#!/usr/bin/env python3 """Run a resumable published-model 200 x 8 diversity experiment.""" from __future__ import annotations import argparse import hashlib import json import re import subprocess import sys from pathlib import Path from common import load_config, unset_proxy_env, write_json from run_checkpoint_curve import check_manifest, training_overlap SAMPLE_NAME = re.compile(r"sample_(\d{4})_seed(\d{4})\.(mp4|wav|json)") def inspect_existing_media(media: Path, prompts: list[str], seed: int, num_seeds: int) -> int: """Validate all existing outputs before adopting a resumable media directory.""" if not media.is_dir(): return 0 existing: dict[tuple[int, int], set[str]] = {} for path in media.iterdir(): match = SAMPLE_NAME.fullmatch(path.name) if match is None: if path.name.startswith("sample_"): raise ValueError(f"Unexpected sample filename: {path}") continue # Existing shard prompt lists and model-init lock are not media. index, seed_idx, suffix = int(match[1]), int(match[2]), match[3] if index >= len(prompts) or seed_idx >= num_seeds or not path.is_file(): raise ValueError(f"Unexpected prompt/seed media: {path}") existing.setdefault((index, seed_idx), set()).add(suffix) if not path.stat().st_size: raise ValueError(f"Empty media file: {path}") for (index, seed_idx), suffixes in existing.items(): # A lone MP4 or WAV may be the result of an interrupted generation. # Both files without a sidecar cannot be provenance-checked. if ("json" in suffixes and suffixes != {"mp4", "wav", "json"}) or ( {"mp4", "wav"} <= suffixes and "json" not in suffixes ): raise ValueError(f"Incomplete provenance for sample_{index:04d}_seed{seed_idx:04d}") if "json" not in suffixes: continue sidecar = media / f"sample_{index:04d}_seed{seed_idx:04d}.json" record = json.loads(sidecar.read_text(encoding="utf-8")) if (record.get("index") != index or record.get("seed_idx") != seed_idx or record.get("seed") != seed + index * num_seeds + seed_idx or record.get("prompt") != prompts[index]): raise ValueError(f"Generated prompt/seed mismatch: {sidecar}") return sum(suffixes == {"mp4", "wav", "json"} for suffixes in existing.values()) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--checkpoint", type=Path, required=True) parser.add_argument("--checkpoint-info", type=Path, required=True) parser.add_argument("--run-config", type=Path, required=True) parser.add_argument("--prompts", type=Path, required=True) parser.add_argument("--train-mapping", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--javis-root", type=Path, required=True) parser.add_argument("--javis-python", type=Path, required=True) parser.add_argument("--ltx-root", type=Path, required=True) parser.add_argument("--ltx-python", type=Path, required=True) parser.add_argument("--gpu", default="0") parser.add_argument("--seed", type=int, default=12345) parser.add_argument("--limit", type=int, default=200) parser.add_argument("--num-seeds", type=int, default=8) parser.add_argument("--batch-size", type=int, default=4) parser.add_argument("--num-workers", type=int, default=4) parser.add_argument("--score-existing-only", action="store_true", help="score a complete media directory without starting generation") parser.add_argument("--cpu", action="store_true", help="CPU scoring only; requires --score-existing-only, never generates") parser.add_argument("--prepare-only", action="store_true", help="validate/adopt existing media and write manifest; never generate or score") parser.add_argument("--dry-run", action="store_true") return parser.parse_args() def run(args: argparse.Namespace) -> int: if args.limit < 1 or args.num_seeds < 2 or args.batch_size < 1 or args.num_workers < 0: raise ValueError("--limit must be positive and --num-seeds at least 2") if args.cpu and not args.score_existing_only and not getattr(args, "prepare_only", False): raise ValueError("--cpu requires --score-existing-only; no CPU generation is supported") checkpoint, info_file = args.checkpoint.resolve(), args.checkpoint_info.resolve() prompts_file, config_file = args.prompts.resolve(), args.run_config.resolve() mapping = args.train_mapping.resolve() output, javis_root, ltx_root = args.output.resolve(), args.javis_root.resolve(), args.ltx_root.resolve() javis_python, ltx_python = args.javis_python.absolute(), args.ltx_python.absolute() infer = ltx_root / "packages/ltx-distillation/src/ltx_distillation/tools/run_av_inference_eval.py" scorer = Path(__file__).absolute().with_name("compute_diversity_imagebind_pairwise.py") for path in (checkpoint, info_file, prompts_file, config_file, mapping, infer, scorer, javis_python, ltx_python): if not path.is_file(): raise FileNotFoundError(path) info = json.loads(info_file.read_text(encoding="utf-8")) if (checkpoint.stat().st_size != info["model_bytes"] or info["format"] != "pytorch_state_dict"): raise ValueError(f"Published checkpoint metadata mismatch: {info_file}") cfg = load_config(config_file) for key in ("checkpoint_path", "gemma_path"): if not Path(cfg[key]).exists(): raise FileNotFoundError(f"{key}: {cfg[key]}") prompts = [p.strip() for p in prompts_file.read_text(encoding="utf-8").splitlines() if p.strip()] if len(prompts) < args.limit: raise ValueError(f"Expected {args.limit} prompts, found {len(prompts)}") prompts = prompts[:args.limit] overlap = training_overlap(mapping, prompts) if not (javis_root / "checkpoints/imagebind_huge.pth").is_file(): raise FileNotFoundError(javis_root / "checkpoints/imagebind_huge.pth") if overlap: print(f"WARNING: {len(overlap)} prompts overlap {mapping}", flush=True) model_name = "published_turbot2av_main" media = output / "media" / model_name metric_root = output / "metrics" # Use this experiment's paths only; never rewrite the prior machine's config. from compute_diversity_imagebind_pairwise import embed_path, sample_paths eval_config = { "paths": {"javis_root": str(javis_root), "diversity_metric_root": str(metric_root)}, "prompt_subset": {"num_prompts": args.limit, "num_seeds": args.num_seeds}, } model_cfg = {"name": model_name, "media_dir": str(media)} manifest = { "model_id": "luyu1021/TurboT2AV", "checkpoint": str(checkpoint), "checkpoint_sha256": info["model_sha256"], "config_sha256": hashlib.sha256(config_file.read_bytes()).hexdigest(), "prompts_sha256": hashlib.sha256("\n".join(prompts).encode("utf-8")).hexdigest(), "num_prompts": args.limit, "num_seeds": args.num_seeds, "seed": args.seed, "seed_rule": "seed + prompt_index * num_seeds + seed_index", "train_overlap_count": len(overlap), "train_mapping": str(mapping), "imagebind": str(javis_root / "checkpoints/imagebind_huge.pth"), "javis_root": str(javis_root), "ltx_infer": str(infer), "ltx_python": str(ltx_python), "javis_python": str(javis_python), } if args.prepare_only: if output.exists(): extras = {p.name for p in output.iterdir()} - {"media", "evaluation_manifest.json"} if extras and not (output / "evaluation_manifest.json").is_file(): raise RuntimeError(f"Cannot adopt output with unknown files: {sorted(extras)}") if (output / "media").exists(): media_extras = {p.name for p in (output / "media").iterdir()} - {model_name} if media_extras and not (output / "evaluation_manifest.json").is_file(): raise RuntimeError(f"Cannot adopt unrelated media: {sorted(media_extras)}") count = inspect_existing_media(media, prompts, args.seed, args.num_seeds) if (output / "evaluation_manifest.json").is_file(): check_manifest(output, manifest, True) elif not args.dry_run: write_json(output / "evaluation_manifest.json", manifest) print(f"validated diversity media: {count}/{args.limit * args.num_seeds} " f"complete prompt/seed pairs; missing={args.limit * args.num_seeds - count}; " f"manifest={'dry-run' if args.dry_run else output / 'evaluation_manifest.json'}") return 0 check_manifest(output, manifest, args.dry_run) generate = [ str(ltx_python), str(infer), "--config_path", str(config_file), "--prompts_file", str(prompts_file), "--output_dir", str(media), "--model_kind", "student", "--student_checkpoint", str(checkpoint), "--student_param", "rcm_trig", "--num_prompts", str(args.limit), "--num_seeds", str(args.num_seeds), "--seed", str(args.seed), ] embedding = embed_path(eval_config, model_cfg) try: sample_paths(eval_config, model_cfg) complete = True except FileNotFoundError: complete = False if args.dry_run: print(f"diversity: {args.limit} prompts x {args.num_seeds} seeds; " f"{args.limit * args.num_seeds * (args.num_seeds - 1) // 2} within-prompt pairs per modality") print(f"generate={not complete}, embed={not embedding.is_file()}") print(f"generate: {generate}") return 0 if args.score_existing_only and not complete: raise RuntimeError(f"--score-existing-only requires all {args.limit * args.num_seeds} pairs in {media}") env = unset_proxy_env() env["CUDA_VISIBLE_DEVICES"] = "" if args.cpu else args.gpu env["PYTHONUNBUFFERED"] = "1" if not complete: media.mkdir(parents=True, exist_ok=True) subprocess.run(generate, cwd=ltx_root, env=env, check=True) sample_paths(eval_config, model_cfg) # The caller may be a pixi Python or another venv; the scoring imports # must always execute through the separate JavisBench environment. scoring = [ str(javis_python), str(scorer), "--config", str(output / "diversity_config.json"), *(["--cpu"] if args.cpu else ["--gpu", str(args.gpu)]), "--only", model_name, "--batch-size", str(args.batch_size), "--num-workers", str(args.num_workers), ] write_json(output / "diversity_config.json", { **eval_config, "models": [model_cfg], }) subprocess.run(scoring, cwd=javis_root, env=env, check=True) summary_path = metric_root / "summary.json" if not summary_path.is_file(): raise RuntimeError(f"Missing diversity scores: {summary_path}") rows = json.loads(summary_path.read_text(encoding="utf-8"))["rows"] if len(rows) != 1 or rows[0]["model_name"] != model_name: raise RuntimeError(f"Unexpected diversity scores: {summary_path}") scores = rows[0] if args.limit == 200 and args.num_seeds == 8: reference = { "video_pairwise_cosine_distance": 0.2322, "audio_pairwise_cosine_distance": 0.4197, "va_average_pairwise_cosine_distance": 0.3259, } write_json(output / "diversity_paper_comparison.json", { "model_identity": model_name, "note": "Numerical comparison only; exact author prompt/enhancement and ImageBind distance protocols unverified", "metrics": { key: {"observed": scores[key], "paper": value, "delta": scores[key] - value} for key, value in reference.items() }, }) print(json.dumps({ k: scores[k] for k in ( "video_pairwise_cosine_distance", "audio_pairwise_cosine_distance", "va_average_pairwise_cosine_distance", ) }, indent=2)) return 0 if __name__ == "__main__": sys.exit(run(parse_args()))