Download code/evaluation/scripts/run_published_probe.py from luyu1021/turbot2va-ltx2: direct link, hf CLI and curl.
- Browser
- Download file 5.86 kB
-
https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/evaluation/scripts/run_published_probe.py
- Command line
-
hf download hf://luyu1021/turbot2va-ltx2/code/evaluation/scripts/run_published_probe.py
-
curl -L -o run_published_probe.py https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/evaluation/scripts/run_published_probe.py
5.86 kB
| #!/usr/bin/env python3 | |
| """Generate and JavisBench-score one published full-state-dict checkpoint.""" | |
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import json | |
| import subprocess | |
| import sys | |
| from pathlib import Path | |
| from common import load_config, unset_proxy_env | |
| from run_checkpoint_curve import METRICS, SCORES, check_manifest, media_complete, score_complete, training_overlap, write_input | |
| def parse_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--checkpoint", type=Path, required=True) | |
| parser.add_argument("--checkpoint-info", type=Path, required=True) | |
| parser.add_argument("--run-config", type=Path, required=True) | |
| parser.add_argument("--prompts", type=Path, required=True) | |
| parser.add_argument("--output", type=Path, required=True) | |
| parser.add_argument("--javis-root", type=Path, required=True) | |
| parser.add_argument("--javis-python", type=Path, required=True) | |
| parser.add_argument("--ltx-root", type=Path, required=True) | |
| parser.add_argument("--ltx-python", type=Path, required=True) | |
| parser.add_argument("--gpu", default="0") | |
| parser.add_argument("--seed", type=int, default=12345) | |
| parser.add_argument("--limit", type=int, default=10) | |
| parser.add_argument("--dry-run", action="store_true") | |
| return parser.parse_args() | |
| def run(args: argparse.Namespace) -> int: | |
| checkpoint, info_path = args.checkpoint.resolve(), args.checkpoint_info.resolve() | |
| config, prompts_path = args.run_config.resolve(), args.prompts.resolve() | |
| output, javis_root, ltx_root = args.output.resolve(), args.javis_root.resolve(), args.ltx_root.resolve() | |
| # A venv's python symlink must be executed through its own path. | |
| javis_python, ltx_python = args.javis_python.absolute(), args.ltx_python.absolute() | |
| infer = ltx_root / "packages/ltx-distillation/src/ltx_distillation/tools/run_av_inference_eval.py" | |
| bench = javis_root / "eval/javisbench/main.py" | |
| if args.limit < 1: | |
| raise ValueError("--limit must be positive") | |
| for path in (checkpoint, info_path, config, prompts_path, infer, bench, javis_python, ltx_python): | |
| if not path.is_file(): | |
| raise FileNotFoundError(path) | |
| info = json.loads(info_path.read_text(encoding="utf-8")) | |
| if (checkpoint.stat().st_size != info["model_bytes"] | |
| or info.get("format") != "pytorch_state_dict" | |
| or info.get("checkpoint_name") != "turbot2av_main"): | |
| raise ValueError(f"Published checkpoint size/format/name does not match {info_path}") | |
| prompts = [line.strip() for line in prompts_path.read_text(encoding="utf-8").splitlines() if line.strip()] | |
| if len(prompts) < args.limit: | |
| raise ValueError(f"Expected {args.limit} prompts, found {len(prompts)}") | |
| prompts = prompts[:args.limit] | |
| cfg = load_config(config) | |
| for key in ("checkpoint_path", "gemma_path"): | |
| if not Path(cfg[key]).exists(): | |
| raise FileNotFoundError(f"{key}: {cfg[key]}") | |
| mapping = Path(cfg["data_path"]) | |
| if not mapping.is_file(): | |
| raise FileNotFoundError(f"Training mapping missing: {mapping}") | |
| overlap = training_overlap(mapping, prompts) | |
| if overlap: | |
| print(f"WARNING: {len(overlap)} evaluation prompts overlap training mapping {mapping}", flush=True) | |
| manifest = { | |
| "model_id": "luyu1021/TurboT2AV", | |
| "checkpoint": str(checkpoint), | |
| "checkpoint_sha256": info["model_sha256"], | |
| "checkpoint_step": info["checkpoint_step"], | |
| "config": str(config), | |
| "config_sha256": hashlib.sha256(config.read_bytes()).hexdigest(), | |
| "prompt_sha256": hashlib.sha256("\n".join(prompts).encode("utf-8")).hexdigest(), | |
| "num_prompts": args.limit, | |
| "train_overlap_count": len(overlap), | |
| "train_mapping": str(mapping), | |
| "seed": args.seed, | |
| "metrics": METRICS, | |
| "javis_root": str(javis_root), | |
| "ltx_python": str(ltx_python), | |
| "javis_python": str(javis_python), | |
| } | |
| check_manifest(output, manifest, args.dry_run) | |
| media = output / "media" | |
| score_file = output / "scores.json" | |
| input_csv = output / "input.csv" | |
| generate = [ | |
| str(ltx_python), str(infer), "--config_path", str(config), | |
| "--prompts_file", str(prompts_path), "--output_dir", str(media), | |
| "--model_kind", "student", "--student_param", "rcm_trig", | |
| "--student_checkpoint", str(checkpoint), "--num_prompts", str(args.limit), | |
| "--num_seeds", "1", "--seed", str(args.seed), | |
| ] | |
| score = [ | |
| str(javis_python), "-m", "eval.javisbench.main", | |
| "--input_file", str(input_csv), "--output_file", str(score_file), | |
| "--metrics", *METRICS, "--max_audio_len_s", "4.0", "--num_workers", "8", | |
| ] | |
| if args.dry_run: | |
| print(f"published probe: generate={not media_complete(media, args.limit)} score={not score_complete(score_file)}") | |
| print(f"generate: {generate}") | |
| print(f"score: {score}") | |
| return 0 | |
| env = unset_proxy_env() | |
| env["CUDA_VISIBLE_DEVICES"] = args.gpu | |
| env["PYTHONUNBUFFERED"] = "1" | |
| if not score_complete(score_file): | |
| if not media_complete(media, args.limit): | |
| media.mkdir(parents=True, exist_ok=True) | |
| subprocess.run(generate, cwd=ltx_root, env=env, check=True) | |
| if not media_complete(media, args.limit): | |
| raise RuntimeError(f"Incomplete media: {media}") | |
| write_input(input_csv, prompts, media) | |
| subprocess.run(score, cwd=javis_root, env=env, check=True) | |
| if not score_complete(score_file): | |
| raise RuntimeError(f"Incomplete JavisBench scores: {score_file}") | |
| values = json.loads(score_file.read_text(encoding="utf-8")) | |
| print(json.dumps({k: values[k] for k in SCORES}, indent=2)) | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(run(parse_args())) | |