#!/usr/bin/env python3 """Generate and JavisBench-score one published full-state-dict checkpoint.""" from __future__ import annotations import argparse import hashlib import json import subprocess import sys from pathlib import Path from common import load_config, unset_proxy_env from run_checkpoint_curve import METRICS, SCORES, check_manifest, media_complete, score_complete, training_overlap, write_input def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--checkpoint", type=Path, required=True) parser.add_argument("--checkpoint-info", type=Path, required=True) parser.add_argument("--run-config", type=Path, required=True) parser.add_argument("--prompts", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--javis-root", type=Path, required=True) parser.add_argument("--javis-python", type=Path, required=True) parser.add_argument("--ltx-root", type=Path, required=True) parser.add_argument("--ltx-python", type=Path, required=True) parser.add_argument("--gpu", default="0") parser.add_argument("--seed", type=int, default=12345) parser.add_argument("--limit", type=int, default=10) parser.add_argument("--dry-run", action="store_true") return parser.parse_args() def run(args: argparse.Namespace) -> int: checkpoint, info_path = args.checkpoint.resolve(), args.checkpoint_info.resolve() config, prompts_path = args.run_config.resolve(), args.prompts.resolve() output, javis_root, ltx_root = args.output.resolve(), args.javis_root.resolve(), args.ltx_root.resolve() # A venv's python symlink must be executed through its own path. javis_python, ltx_python = args.javis_python.absolute(), args.ltx_python.absolute() infer = ltx_root / "packages/ltx-distillation/src/ltx_distillation/tools/run_av_inference_eval.py" bench = javis_root / "eval/javisbench/main.py" if args.limit < 1: raise ValueError("--limit must be positive") for path in (checkpoint, info_path, config, prompts_path, infer, bench, javis_python, ltx_python): if not path.is_file(): raise FileNotFoundError(path) info = json.loads(info_path.read_text(encoding="utf-8")) if (checkpoint.stat().st_size != info["model_bytes"] or info.get("format") != "pytorch_state_dict" or info.get("checkpoint_name") != "turbot2av_main"): raise ValueError(f"Published checkpoint size/format/name does not match {info_path}") prompts = [line.strip() for line in prompts_path.read_text(encoding="utf-8").splitlines() if line.strip()] if len(prompts) < args.limit: raise ValueError(f"Expected {args.limit} prompts, found {len(prompts)}") prompts = prompts[:args.limit] cfg = load_config(config) for key in ("checkpoint_path", "gemma_path"): if not Path(cfg[key]).exists(): raise FileNotFoundError(f"{key}: {cfg[key]}") mapping = Path(cfg["data_path"]) if not mapping.is_file(): raise FileNotFoundError(f"Training mapping missing: {mapping}") overlap = training_overlap(mapping, prompts) if overlap: print(f"WARNING: {len(overlap)} evaluation prompts overlap training mapping {mapping}", flush=True) manifest = { "model_id": "luyu1021/TurboT2AV", "checkpoint": str(checkpoint), "checkpoint_sha256": info["model_sha256"], "checkpoint_step": info["checkpoint_step"], "config": str(config), "config_sha256": hashlib.sha256(config.read_bytes()).hexdigest(), "prompt_sha256": hashlib.sha256("\n".join(prompts).encode("utf-8")).hexdigest(), "num_prompts": args.limit, "train_overlap_count": len(overlap), "train_mapping": str(mapping), "seed": args.seed, "metrics": METRICS, "javis_root": str(javis_root), "ltx_python": str(ltx_python), "javis_python": str(javis_python), } check_manifest(output, manifest, args.dry_run) media = output / "media" score_file = output / "scores.json" input_csv = output / "input.csv" generate = [ str(ltx_python), str(infer), "--config_path", str(config), "--prompts_file", str(prompts_path), "--output_dir", str(media), "--model_kind", "student", "--student_param", "rcm_trig", "--student_checkpoint", str(checkpoint), "--num_prompts", str(args.limit), "--num_seeds", "1", "--seed", str(args.seed), ] score = [ str(javis_python), "-m", "eval.javisbench.main", "--input_file", str(input_csv), "--output_file", str(score_file), "--metrics", *METRICS, "--max_audio_len_s", "4.0", "--num_workers", "8", ] if args.dry_run: print(f"published probe: generate={not media_complete(media, args.limit)} score={not score_complete(score_file)}") print(f"generate: {generate}") print(f"score: {score}") return 0 env = unset_proxy_env() env["CUDA_VISIBLE_DEVICES"] = args.gpu env["PYTHONUNBUFFERED"] = "1" if not score_complete(score_file): if not media_complete(media, args.limit): media.mkdir(parents=True, exist_ok=True) subprocess.run(generate, cwd=ltx_root, env=env, check=True) if not media_complete(media, args.limit): raise RuntimeError(f"Incomplete media: {media}") write_input(input_csv, prompts, media) subprocess.run(score, cwd=javis_root, env=env, check=True) if not score_complete(score_file): raise RuntimeError(f"Incomplete JavisBench scores: {score_file}") values = json.loads(score_file.read_text(encoding="utf-8")) print(json.dumps({k: values[k] for k in SCORES}, indent=2)) return 0 if __name__ == "__main__": sys.exit(run(parse_args()))