Download code/evaluation/scripts/score_existing_cpu_200.py from luyu1021/turbot2va-ltx2: direct link, hf CLI and curl.
- Browser
- Download file 6.32 kB
-
https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/evaluation/scripts/score_existing_cpu_200.py
- Command line
-
hf download hf://luyu1021/turbot2va-ltx2/code/evaluation/scripts/score_existing_cpu_200.py
-
curl -L -o score_existing_cpu_200.py https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/evaluation/scripts/score_existing_cpu_200.py
6.32 kB
| #!/usr/bin/env python3 | |
| """JavisBench-score 200 existing AV pairs on CPU, without model generation.""" | |
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import json | |
| import os | |
| import subprocess | |
| from pathlib import Path | |
| from common import utc_now, write_json | |
| from run_checkpoint_curve import METRICS, SCORES, media_complete, score_complete, write_input | |
| PAPER = { | |
| "student": { | |
| "visual_quality": 2.3892, "motion_quality": 0.8493, "audio_quality": 4.4860, | |
| "ib_tv": 0.2185, "ib_ta": 0.1447, "ib_av": 0.2290, | |
| "clip_score": 0.2921, "clap_score": 0.3472, "cavp_score": 0.8033, | |
| "avh_score": 0.2241, "javis_score": 0.1963, "desync": 0.3880, | |
| }, | |
| "teacher_40_step": { | |
| "visual_quality": 1.9676, "motion_quality": 0.7825, "audio_quality": 4.9023, | |
| "ib_tv": 0.2683, "ib_ta": 0.1393, "ib_av": 0.1991, | |
| "clip_score": 0.2913, "clap_score": 0.3401, "cavp_score": 0.7959, | |
| "avh_score": 0.1951, "javis_score": 0.1648, "desync": 0.6170, | |
| }, | |
| } | |
| def preflight(prompts_file: Path, media: Path, model: str) -> tuple[list[str], str]: | |
| if model not in PAPER: | |
| raise ValueError(f"Unrecognized model identity: {model}") | |
| prompts_file = prompts_file.resolve(strict=True) | |
| prompts = prompts_file.read_text(encoding="utf-8").splitlines() | |
| if len(prompts) != 200 or any(not prompt.strip() for prompt in prompts): | |
| raise ValueError("Expected exactly 200 non-empty prompts") | |
| if not media_complete(media, 200): | |
| missing = [ | |
| i for i in range(200) | |
| if any( | |
| not (media / f"sample_{i:04d}.{suffix}").is_file() | |
| or (media / f"sample_{i:04d}.{suffix}").stat().st_size == 0 | |
| for suffix in ("mp4", "wav") | |
| ) | |
| ] | |
| raise FileNotFoundError(f"Missing AV pairs in {media}; first failing index={missing[0]}") | |
| for index, prompt in enumerate(prompts): | |
| sidecar = media / f"sample_{index:04d}.json" | |
| if not sidecar.is_file(): | |
| continue | |
| payload = json.loads(sidecar.read_text(encoding="utf-8")) | |
| if payload.get("index") != index or payload.get("prompt") != prompt or payload.get("seed_idx") != 0: | |
| raise ValueError(f"Generated prompt/index/seed mismatch: {sidecar}") | |
| digest = hashlib.sha256(prompts_file.read_bytes()).hexdigest() | |
| return prompts, digest | |
| def run(args: argparse.Namespace) -> int: | |
| prompts, digest = preflight(args.prompts, args.media, args.model) | |
| root = args.javis_root.resolve(strict=True) | |
| python = args.javis_python.absolute() | |
| if not python.is_file() or not (root / "eval/javisbench/main.py").is_file(): | |
| raise FileNotFoundError(f"Missing JavisBench or isolated Python: {root}, {python}") | |
| output = args.output.resolve() | |
| manifest_path = output / "cpu_score_manifest.json" | |
| manifest = { | |
| "model_identity": args.model, | |
| "media_dir": str(args.media.resolve()), | |
| "prompts": str(args.prompts.resolve()), | |
| "prompts_sha256": digest, | |
| "sample_count": len(prompts), | |
| "score_device": "cpu", | |
| "javis_root": str(root), | |
| "javis_python": str(python), | |
| "metrics": METRICS, | |
| "paper_reference": "my_repos/turbot2av_paper/main.tex, Table VII", | |
| "sampling_protocol_verified": False, | |
| "provenance_note": "Caller must independently verify model identity and sampling steps; present generation sidecars checked for prompt alignment", | |
| } | |
| if manifest_path.exists(): | |
| if json.loads(manifest_path.read_text(encoding="utf-8")) != manifest: | |
| raise RuntimeError("Settings changed; choose a different output directory") | |
| elif output.exists() and any(output.iterdir()): | |
| raise RuntimeError("Output already contains files without matching manifest") | |
| scores = output / "javis_scores.json" | |
| if not score_complete(scores): | |
| output.mkdir(parents=True, exist_ok=True) | |
| write_json(manifest_path, manifest) | |
| input_csv = output / "javis_input.csv" | |
| write_input(input_csv, prompts, args.media.resolve()) | |
| env = dict(os.environ) | |
| env["CUDA_VISIBLE_DEVICES"] = "" | |
| env["HF_HUB_OFFLINE"] = "1" | |
| env["TRANSFORMERS_OFFLINE"] = "1" | |
| env["TOKENIZERS_PARALLELISM"] = "false" | |
| env["OMP_NUM_THREADS"] = str(args.cpu_threads) | |
| env["MKL_NUM_THREADS"] = str(args.cpu_threads) | |
| env["PATH"] = f"{python.parent}:{env.get('PATH', '')}" | |
| subprocess.run( | |
| [str(python), "-m", "eval.javisbench.main", | |
| "--input_file", str(input_csv), "--output_file", str(scores), | |
| "--metrics", *METRICS, "--max_audio_len_s", "4", | |
| "--num_workers", str(args.workers)], | |
| cwd=root, env=env, check=True, | |
| ) | |
| if not score_complete(scores): | |
| raise RuntimeError(f"Incomplete JavisBench scores: {scores}") | |
| observed = json.loads(scores.read_text(encoding="utf-8")) | |
| reference = PAPER[args.model] | |
| result = { | |
| "compared_at_utc": utc_now(), | |
| "model_identity": args.model, | |
| "note": "Numerical comparison only; prompt enhancement and generation protocol remain unverified", | |
| "scores": {key: { | |
| "observed": observed[key], | |
| "paper": reference[key], | |
| "delta": observed[key] - reference[key], | |
| } for key in SCORES}, | |
| } | |
| write_json(output / "paper_comparison.json", result) | |
| print(json.dumps(result, indent=2)) | |
| return 0 | |
| def main() -> int: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--model", choices=sorted(PAPER), required=True) | |
| parser.add_argument("--media", type=Path, required=True) | |
| parser.add_argument("--prompts", type=Path, required=True) | |
| parser.add_argument("--output", type=Path, required=True) | |
| parser.add_argument("--javis-root", type=Path, required=True) | |
| parser.add_argument("--javis-python", type=Path, required=True) | |
| parser.add_argument("--cpu-threads", type=int, default=4) | |
| parser.add_argument("--workers", type=int, default=0) | |
| args = parser.parse_args() | |
| if args.cpu_threads < 1 or args.workers < 0: | |
| parser.error("--cpu-threads must be positive and --workers nonnegative") | |
| return run(args) | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |