#!/usr/bin/env python3 """JavisBench-score 200 existing AV pairs on CPU, without model generation.""" from __future__ import annotations import argparse import hashlib import json import os import subprocess from pathlib import Path from common import utc_now, write_json from run_checkpoint_curve import METRICS, SCORES, media_complete, score_complete, write_input PAPER = { "student": { "visual_quality": 2.3892, "motion_quality": 0.8493, "audio_quality": 4.4860, "ib_tv": 0.2185, "ib_ta": 0.1447, "ib_av": 0.2290, "clip_score": 0.2921, "clap_score": 0.3472, "cavp_score": 0.8033, "avh_score": 0.2241, "javis_score": 0.1963, "desync": 0.3880, }, "teacher_40_step": { "visual_quality": 1.9676, "motion_quality": 0.7825, "audio_quality": 4.9023, "ib_tv": 0.2683, "ib_ta": 0.1393, "ib_av": 0.1991, "clip_score": 0.2913, "clap_score": 0.3401, "cavp_score": 0.7959, "avh_score": 0.1951, "javis_score": 0.1648, "desync": 0.6170, }, } def preflight(prompts_file: Path, media: Path, model: str) -> tuple[list[str], str]: if model not in PAPER: raise ValueError(f"Unrecognized model identity: {model}") prompts_file = prompts_file.resolve(strict=True) prompts = prompts_file.read_text(encoding="utf-8").splitlines() if len(prompts) != 200 or any(not prompt.strip() for prompt in prompts): raise ValueError("Expected exactly 200 non-empty prompts") if not media_complete(media, 200): missing = [ i for i in range(200) if any( not (media / f"sample_{i:04d}.{suffix}").is_file() or (media / f"sample_{i:04d}.{suffix}").stat().st_size == 0 for suffix in ("mp4", "wav") ) ] raise FileNotFoundError(f"Missing AV pairs in {media}; first failing index={missing[0]}") for index, prompt in enumerate(prompts): sidecar = media / f"sample_{index:04d}.json" if not sidecar.is_file(): continue payload = json.loads(sidecar.read_text(encoding="utf-8")) if payload.get("index") != index or payload.get("prompt") != prompt or payload.get("seed_idx") != 0: raise ValueError(f"Generated prompt/index/seed mismatch: {sidecar}") digest = hashlib.sha256(prompts_file.read_bytes()).hexdigest() return prompts, digest def run(args: argparse.Namespace) -> int: prompts, digest = preflight(args.prompts, args.media, args.model) root = args.javis_root.resolve(strict=True) python = args.javis_python.absolute() if not python.is_file() or not (root / "eval/javisbench/main.py").is_file(): raise FileNotFoundError(f"Missing JavisBench or isolated Python: {root}, {python}") output = args.output.resolve() manifest_path = output / "cpu_score_manifest.json" manifest = { "model_identity": args.model, "media_dir": str(args.media.resolve()), "prompts": str(args.prompts.resolve()), "prompts_sha256": digest, "sample_count": len(prompts), "score_device": "cpu", "javis_root": str(root), "javis_python": str(python), "metrics": METRICS, "paper_reference": "my_repos/turbot2av_paper/main.tex, Table VII", "sampling_protocol_verified": False, "provenance_note": "Caller must independently verify model identity and sampling steps; present generation sidecars checked for prompt alignment", } if manifest_path.exists(): if json.loads(manifest_path.read_text(encoding="utf-8")) != manifest: raise RuntimeError("Settings changed; choose a different output directory") elif output.exists() and any(output.iterdir()): raise RuntimeError("Output already contains files without matching manifest") scores = output / "javis_scores.json" if not score_complete(scores): output.mkdir(parents=True, exist_ok=True) write_json(manifest_path, manifest) input_csv = output / "javis_input.csv" write_input(input_csv, prompts, args.media.resolve()) env = dict(os.environ) env["CUDA_VISIBLE_DEVICES"] = "" env["HF_HUB_OFFLINE"] = "1" env["TRANSFORMERS_OFFLINE"] = "1" env["TOKENIZERS_PARALLELISM"] = "false" env["OMP_NUM_THREADS"] = str(args.cpu_threads) env["MKL_NUM_THREADS"] = str(args.cpu_threads) env["PATH"] = f"{python.parent}:{env.get('PATH', '')}" subprocess.run( [str(python), "-m", "eval.javisbench.main", "--input_file", str(input_csv), "--output_file", str(scores), "--metrics", *METRICS, "--max_audio_len_s", "4", "--num_workers", str(args.workers)], cwd=root, env=env, check=True, ) if not score_complete(scores): raise RuntimeError(f"Incomplete JavisBench scores: {scores}") observed = json.loads(scores.read_text(encoding="utf-8")) reference = PAPER[args.model] result = { "compared_at_utc": utc_now(), "model_identity": args.model, "note": "Numerical comparison only; prompt enhancement and generation protocol remain unverified", "scores": {key: { "observed": observed[key], "paper": reference[key], "delta": observed[key] - reference[key], } for key in SCORES}, } write_json(output / "paper_comparison.json", result) print(json.dumps(result, indent=2)) return 0 def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--model", choices=sorted(PAPER), required=True) parser.add_argument("--media", type=Path, required=True) parser.add_argument("--prompts", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--javis-root", type=Path, required=True) parser.add_argument("--javis-python", type=Path, required=True) parser.add_argument("--cpu-threads", type=int, default=4) parser.add_argument("--workers", type=int, default=0) args = parser.parse_args() if args.cpu_threads < 1 or args.workers < 0: parser.error("--cpu-threads must be positive and --workers nonnegative") return run(args) if __name__ == "__main__": raise SystemExit(main())