File size: 6,324 Bytes
76b7b25 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 | #!/usr/bin/env python3
"""JavisBench-score 200 existing AV pairs on CPU, without model generation."""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import subprocess
from pathlib import Path
from common import utc_now, write_json
from run_checkpoint_curve import METRICS, SCORES, media_complete, score_complete, write_input
PAPER = {
"student": {
"visual_quality": 2.3892, "motion_quality": 0.8493, "audio_quality": 4.4860,
"ib_tv": 0.2185, "ib_ta": 0.1447, "ib_av": 0.2290,
"clip_score": 0.2921, "clap_score": 0.3472, "cavp_score": 0.8033,
"avh_score": 0.2241, "javis_score": 0.1963, "desync": 0.3880,
},
"teacher_40_step": {
"visual_quality": 1.9676, "motion_quality": 0.7825, "audio_quality": 4.9023,
"ib_tv": 0.2683, "ib_ta": 0.1393, "ib_av": 0.1991,
"clip_score": 0.2913, "clap_score": 0.3401, "cavp_score": 0.7959,
"avh_score": 0.1951, "javis_score": 0.1648, "desync": 0.6170,
},
}
def preflight(prompts_file: Path, media: Path, model: str) -> tuple[list[str], str]:
if model not in PAPER:
raise ValueError(f"Unrecognized model identity: {model}")
prompts_file = prompts_file.resolve(strict=True)
prompts = prompts_file.read_text(encoding="utf-8").splitlines()
if len(prompts) != 200 or any(not prompt.strip() for prompt in prompts):
raise ValueError("Expected exactly 200 non-empty prompts")
if not media_complete(media, 200):
missing = [
i for i in range(200)
if any(
not (media / f"sample_{i:04d}.{suffix}").is_file()
or (media / f"sample_{i:04d}.{suffix}").stat().st_size == 0
for suffix in ("mp4", "wav")
)
]
raise FileNotFoundError(f"Missing AV pairs in {media}; first failing index={missing[0]}")
for index, prompt in enumerate(prompts):
sidecar = media / f"sample_{index:04d}.json"
if not sidecar.is_file():
continue
payload = json.loads(sidecar.read_text(encoding="utf-8"))
if payload.get("index") != index or payload.get("prompt") != prompt or payload.get("seed_idx") != 0:
raise ValueError(f"Generated prompt/index/seed mismatch: {sidecar}")
digest = hashlib.sha256(prompts_file.read_bytes()).hexdigest()
return prompts, digest
def run(args: argparse.Namespace) -> int:
prompts, digest = preflight(args.prompts, args.media, args.model)
root = args.javis_root.resolve(strict=True)
python = args.javis_python.absolute()
if not python.is_file() or not (root / "eval/javisbench/main.py").is_file():
raise FileNotFoundError(f"Missing JavisBench or isolated Python: {root}, {python}")
output = args.output.resolve()
manifest_path = output / "cpu_score_manifest.json"
manifest = {
"model_identity": args.model,
"media_dir": str(args.media.resolve()),
"prompts": str(args.prompts.resolve()),
"prompts_sha256": digest,
"sample_count": len(prompts),
"score_device": "cpu",
"javis_root": str(root),
"javis_python": str(python),
"metrics": METRICS,
"paper_reference": "my_repos/turbot2av_paper/main.tex, Table VII",
"sampling_protocol_verified": False,
"provenance_note": "Caller must independently verify model identity and sampling steps; present generation sidecars checked for prompt alignment",
}
if manifest_path.exists():
if json.loads(manifest_path.read_text(encoding="utf-8")) != manifest:
raise RuntimeError("Settings changed; choose a different output directory")
elif output.exists() and any(output.iterdir()):
raise RuntimeError("Output already contains files without matching manifest")
scores = output / "javis_scores.json"
if not score_complete(scores):
output.mkdir(parents=True, exist_ok=True)
write_json(manifest_path, manifest)
input_csv = output / "javis_input.csv"
write_input(input_csv, prompts, args.media.resolve())
env = dict(os.environ)
env["CUDA_VISIBLE_DEVICES"] = ""
env["HF_HUB_OFFLINE"] = "1"
env["TRANSFORMERS_OFFLINE"] = "1"
env["TOKENIZERS_PARALLELISM"] = "false"
env["OMP_NUM_THREADS"] = str(args.cpu_threads)
env["MKL_NUM_THREADS"] = str(args.cpu_threads)
env["PATH"] = f"{python.parent}:{env.get('PATH', '')}"
subprocess.run(
[str(python), "-m", "eval.javisbench.main",
"--input_file", str(input_csv), "--output_file", str(scores),
"--metrics", *METRICS, "--max_audio_len_s", "4",
"--num_workers", str(args.workers)],
cwd=root, env=env, check=True,
)
if not score_complete(scores):
raise RuntimeError(f"Incomplete JavisBench scores: {scores}")
observed = json.loads(scores.read_text(encoding="utf-8"))
reference = PAPER[args.model]
result = {
"compared_at_utc": utc_now(),
"model_identity": args.model,
"note": "Numerical comparison only; prompt enhancement and generation protocol remain unverified",
"scores": {key: {
"observed": observed[key],
"paper": reference[key],
"delta": observed[key] - reference[key],
} for key in SCORES},
}
write_json(output / "paper_comparison.json", result)
print(json.dumps(result, indent=2))
return 0
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--model", choices=sorted(PAPER), required=True)
parser.add_argument("--media", type=Path, required=True)
parser.add_argument("--prompts", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--javis-root", type=Path, required=True)
parser.add_argument("--javis-python", type=Path, required=True)
parser.add_argument("--cpu-threads", type=int, default=4)
parser.add_argument("--workers", type=int, default=0)
args = parser.parse_args()
if args.cpu_threads < 1 or args.workers < 0:
parser.error("--cpu-threads must be positive and --workers nonnegative")
return run(args)
if __name__ == "__main__":
raise SystemExit(main())
|