Download code/evaluation/scripts/run_diversity_probe.py from luyu1021/turbot2va-ltx2: direct link, hf CLI and curl.
- Browser
- Download file 12.2 kB
-
https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/evaluation/scripts/run_diversity_probe.py
- Command line
-
hf download hf://luyu1021/turbot2va-ltx2/code/evaluation/scripts/run_diversity_probe.py
-
curl -L -o run_diversity_probe.py https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/evaluation/scripts/run_diversity_probe.py
12.2 kB
| #!/usr/bin/env python3 | |
| """Run a resumable published-model 200 x 8 diversity experiment.""" | |
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import json | |
| import re | |
| import subprocess | |
| import sys | |
| from pathlib import Path | |
| from common import load_config, unset_proxy_env, write_json | |
| from run_checkpoint_curve import check_manifest, training_overlap | |
| SAMPLE_NAME = re.compile(r"sample_(\d{4})_seed(\d{4})\.(mp4|wav|json)") | |
| def inspect_existing_media(media: Path, prompts: list[str], seed: int, num_seeds: int) -> int: | |
| """Validate all existing outputs before adopting a resumable media directory.""" | |
| if not media.is_dir(): | |
| return 0 | |
| existing: dict[tuple[int, int], set[str]] = {} | |
| for path in media.iterdir(): | |
| match = SAMPLE_NAME.fullmatch(path.name) | |
| if match is None: | |
| if path.name.startswith("sample_"): | |
| raise ValueError(f"Unexpected sample filename: {path}") | |
| continue # Existing shard prompt lists and model-init lock are not media. | |
| index, seed_idx, suffix = int(match[1]), int(match[2]), match[3] | |
| if index >= len(prompts) or seed_idx >= num_seeds or not path.is_file(): | |
| raise ValueError(f"Unexpected prompt/seed media: {path}") | |
| existing.setdefault((index, seed_idx), set()).add(suffix) | |
| if not path.stat().st_size: | |
| raise ValueError(f"Empty media file: {path}") | |
| for (index, seed_idx), suffixes in existing.items(): | |
| # A lone MP4 or WAV may be the result of an interrupted generation. | |
| # Both files without a sidecar cannot be provenance-checked. | |
| if ("json" in suffixes and suffixes != {"mp4", "wav", "json"}) or ( | |
| {"mp4", "wav"} <= suffixes and "json" not in suffixes | |
| ): | |
| raise ValueError(f"Incomplete provenance for sample_{index:04d}_seed{seed_idx:04d}") | |
| if "json" not in suffixes: | |
| continue | |
| sidecar = media / f"sample_{index:04d}_seed{seed_idx:04d}.json" | |
| record = json.loads(sidecar.read_text(encoding="utf-8")) | |
| if (record.get("index") != index or record.get("seed_idx") != seed_idx | |
| or record.get("seed") != seed + index * num_seeds + seed_idx | |
| or record.get("prompt") != prompts[index]): | |
| raise ValueError(f"Generated prompt/seed mismatch: {sidecar}") | |
| return sum(suffixes == {"mp4", "wav", "json"} for suffixes in existing.values()) | |
| def parse_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--checkpoint", type=Path, required=True) | |
| parser.add_argument("--checkpoint-info", type=Path, required=True) | |
| parser.add_argument("--run-config", type=Path, required=True) | |
| parser.add_argument("--prompts", type=Path, required=True) | |
| parser.add_argument("--train-mapping", type=Path, required=True) | |
| parser.add_argument("--output", type=Path, required=True) | |
| parser.add_argument("--javis-root", type=Path, required=True) | |
| parser.add_argument("--javis-python", type=Path, required=True) | |
| parser.add_argument("--ltx-root", type=Path, required=True) | |
| parser.add_argument("--ltx-python", type=Path, required=True) | |
| parser.add_argument("--gpu", default="0") | |
| parser.add_argument("--seed", type=int, default=12345) | |
| parser.add_argument("--limit", type=int, default=200) | |
| parser.add_argument("--num-seeds", type=int, default=8) | |
| parser.add_argument("--batch-size", type=int, default=4) | |
| parser.add_argument("--num-workers", type=int, default=4) | |
| parser.add_argument("--score-existing-only", action="store_true", | |
| help="score a complete media directory without starting generation") | |
| parser.add_argument("--cpu", action="store_true", | |
| help="CPU scoring only; requires --score-existing-only, never generates") | |
| parser.add_argument("--prepare-only", action="store_true", | |
| help="validate/adopt existing media and write manifest; never generate or score") | |
| parser.add_argument("--dry-run", action="store_true") | |
| return parser.parse_args() | |
| def run(args: argparse.Namespace) -> int: | |
| if args.limit < 1 or args.num_seeds < 2 or args.batch_size < 1 or args.num_workers < 0: | |
| raise ValueError("--limit must be positive and --num-seeds at least 2") | |
| if args.cpu and not args.score_existing_only and not getattr(args, "prepare_only", False): | |
| raise ValueError("--cpu requires --score-existing-only; no CPU generation is supported") | |
| checkpoint, info_file = args.checkpoint.resolve(), args.checkpoint_info.resolve() | |
| prompts_file, config_file = args.prompts.resolve(), args.run_config.resolve() | |
| mapping = args.train_mapping.resolve() | |
| output, javis_root, ltx_root = args.output.resolve(), args.javis_root.resolve(), args.ltx_root.resolve() | |
| javis_python, ltx_python = args.javis_python.absolute(), args.ltx_python.absolute() | |
| infer = ltx_root / "packages/ltx-distillation/src/ltx_distillation/tools/run_av_inference_eval.py" | |
| scorer = Path(__file__).absolute().with_name("compute_diversity_imagebind_pairwise.py") | |
| for path in (checkpoint, info_file, prompts_file, config_file, mapping, infer, | |
| scorer, javis_python, ltx_python): | |
| if not path.is_file(): | |
| raise FileNotFoundError(path) | |
| info = json.loads(info_file.read_text(encoding="utf-8")) | |
| if (checkpoint.stat().st_size != info["model_bytes"] | |
| or info["format"] != "pytorch_state_dict"): | |
| raise ValueError(f"Published checkpoint metadata mismatch: {info_file}") | |
| cfg = load_config(config_file) | |
| for key in ("checkpoint_path", "gemma_path"): | |
| if not Path(cfg[key]).exists(): | |
| raise FileNotFoundError(f"{key}: {cfg[key]}") | |
| prompts = [p.strip() for p in prompts_file.read_text(encoding="utf-8").splitlines() if p.strip()] | |
| if len(prompts) < args.limit: | |
| raise ValueError(f"Expected {args.limit} prompts, found {len(prompts)}") | |
| prompts = prompts[:args.limit] | |
| overlap = training_overlap(mapping, prompts) | |
| if not (javis_root / "checkpoints/imagebind_huge.pth").is_file(): | |
| raise FileNotFoundError(javis_root / "checkpoints/imagebind_huge.pth") | |
| if overlap: | |
| print(f"WARNING: {len(overlap)} prompts overlap {mapping}", flush=True) | |
| model_name = "published_turbot2av_main" | |
| media = output / "media" / model_name | |
| metric_root = output / "metrics" | |
| # Use this experiment's paths only; never rewrite the prior machine's config. | |
| from compute_diversity_imagebind_pairwise import embed_path, sample_paths | |
| eval_config = { | |
| "paths": {"javis_root": str(javis_root), "diversity_metric_root": str(metric_root)}, | |
| "prompt_subset": {"num_prompts": args.limit, "num_seeds": args.num_seeds}, | |
| } | |
| model_cfg = {"name": model_name, "media_dir": str(media)} | |
| manifest = { | |
| "model_id": "luyu1021/TurboT2AV", "checkpoint": str(checkpoint), | |
| "checkpoint_sha256": info["model_sha256"], | |
| "config_sha256": hashlib.sha256(config_file.read_bytes()).hexdigest(), | |
| "prompts_sha256": hashlib.sha256("\n".join(prompts).encode("utf-8")).hexdigest(), | |
| "num_prompts": args.limit, "num_seeds": args.num_seeds, "seed": args.seed, | |
| "seed_rule": "seed + prompt_index * num_seeds + seed_index", | |
| "train_overlap_count": len(overlap), "train_mapping": str(mapping), | |
| "imagebind": str(javis_root / "checkpoints/imagebind_huge.pth"), | |
| "javis_root": str(javis_root), | |
| "ltx_infer": str(infer), | |
| "ltx_python": str(ltx_python), | |
| "javis_python": str(javis_python), | |
| } | |
| if args.prepare_only: | |
| if output.exists(): | |
| extras = {p.name for p in output.iterdir()} - {"media", "evaluation_manifest.json"} | |
| if extras and not (output / "evaluation_manifest.json").is_file(): | |
| raise RuntimeError(f"Cannot adopt output with unknown files: {sorted(extras)}") | |
| if (output / "media").exists(): | |
| media_extras = {p.name for p in (output / "media").iterdir()} - {model_name} | |
| if media_extras and not (output / "evaluation_manifest.json").is_file(): | |
| raise RuntimeError(f"Cannot adopt unrelated media: {sorted(media_extras)}") | |
| count = inspect_existing_media(media, prompts, args.seed, args.num_seeds) | |
| if (output / "evaluation_manifest.json").is_file(): | |
| check_manifest(output, manifest, True) | |
| elif not args.dry_run: | |
| write_json(output / "evaluation_manifest.json", manifest) | |
| print(f"validated diversity media: {count}/{args.limit * args.num_seeds} " | |
| f"complete prompt/seed pairs; missing={args.limit * args.num_seeds - count}; " | |
| f"manifest={'dry-run' if args.dry_run else output / 'evaluation_manifest.json'}") | |
| return 0 | |
| check_manifest(output, manifest, args.dry_run) | |
| generate = [ | |
| str(ltx_python), str(infer), "--config_path", str(config_file), | |
| "--prompts_file", str(prompts_file), "--output_dir", str(media), | |
| "--model_kind", "student", "--student_checkpoint", str(checkpoint), | |
| "--student_param", "rcm_trig", "--num_prompts", str(args.limit), | |
| "--num_seeds", str(args.num_seeds), "--seed", str(args.seed), | |
| ] | |
| embedding = embed_path(eval_config, model_cfg) | |
| try: | |
| sample_paths(eval_config, model_cfg) | |
| complete = True | |
| except FileNotFoundError: | |
| complete = False | |
| if args.dry_run: | |
| print(f"diversity: {args.limit} prompts x {args.num_seeds} seeds; " | |
| f"{args.limit * args.num_seeds * (args.num_seeds - 1) // 2} within-prompt pairs per modality") | |
| print(f"generate={not complete}, embed={not embedding.is_file()}") | |
| print(f"generate: {generate}") | |
| return 0 | |
| if args.score_existing_only and not complete: | |
| raise RuntimeError(f"--score-existing-only requires all {args.limit * args.num_seeds} pairs in {media}") | |
| env = unset_proxy_env() | |
| env["CUDA_VISIBLE_DEVICES"] = "" if args.cpu else args.gpu | |
| env["PYTHONUNBUFFERED"] = "1" | |
| if not complete: | |
| media.mkdir(parents=True, exist_ok=True) | |
| subprocess.run(generate, cwd=ltx_root, env=env, check=True) | |
| sample_paths(eval_config, model_cfg) | |
| # The caller may be a pixi Python or another venv; the scoring imports | |
| # must always execute through the separate JavisBench environment. | |
| scoring = [ | |
| str(javis_python), str(scorer), "--config", | |
| str(output / "diversity_config.json"), *(["--cpu"] if args.cpu else ["--gpu", str(args.gpu)]), | |
| "--only", model_name, "--batch-size", str(args.batch_size), | |
| "--num-workers", str(args.num_workers), | |
| ] | |
| write_json(output / "diversity_config.json", { | |
| **eval_config, "models": [model_cfg], | |
| }) | |
| subprocess.run(scoring, cwd=javis_root, env=env, check=True) | |
| summary_path = metric_root / "summary.json" | |
| if not summary_path.is_file(): | |
| raise RuntimeError(f"Missing diversity scores: {summary_path}") | |
| rows = json.loads(summary_path.read_text(encoding="utf-8"))["rows"] | |
| if len(rows) != 1 or rows[0]["model_name"] != model_name: | |
| raise RuntimeError(f"Unexpected diversity scores: {summary_path}") | |
| scores = rows[0] | |
| if args.limit == 200 and args.num_seeds == 8: | |
| reference = { | |
| "video_pairwise_cosine_distance": 0.2322, | |
| "audio_pairwise_cosine_distance": 0.4197, | |
| "va_average_pairwise_cosine_distance": 0.3259, | |
| } | |
| write_json(output / "diversity_paper_comparison.json", { | |
| "model_identity": model_name, | |
| "note": "Numerical comparison only; exact author prompt/enhancement and ImageBind distance protocols unverified", | |
| "metrics": { | |
| key: {"observed": scores[key], "paper": value, "delta": scores[key] - value} | |
| for key, value in reference.items() | |
| }, | |
| }) | |
| print(json.dumps({ | |
| k: scores[k] for k in ( | |
| "video_pairwise_cosine_distance", "audio_pairwise_cosine_distance", | |
| "va_average_pairwise_cosine_distance", | |
| ) | |
| }, indent=2)) | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(run(parse_args())) | |