turbot2va-ltx2 / code /evaluation /scripts /score_existing_cpu_200.py
luyu1021's picture
Add documented LTX-2 training source, recipes and checkpoint release plan
76b7b25 verified
Raw History Blame Contribute Delete
6.32 kB
#!/usr/bin/env python3
"""JavisBench-score 200 existing AV pairs on CPU, without model generation."""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import subprocess
from pathlib import Path
from common import utc_now, write_json
from run_checkpoint_curve import METRICS, SCORES, media_complete, score_complete, write_input
PAPER = {
"student": {
"visual_quality": 2.3892, "motion_quality": 0.8493, "audio_quality": 4.4860,
"ib_tv": 0.2185, "ib_ta": 0.1447, "ib_av": 0.2290,
"clip_score": 0.2921, "clap_score": 0.3472, "cavp_score": 0.8033,
"avh_score": 0.2241, "javis_score": 0.1963, "desync": 0.3880,
},
"teacher_40_step": {
"visual_quality": 1.9676, "motion_quality": 0.7825, "audio_quality": 4.9023,
"ib_tv": 0.2683, "ib_ta": 0.1393, "ib_av": 0.1991,
"clip_score": 0.2913, "clap_score": 0.3401, "cavp_score": 0.7959,
"avh_score": 0.1951, "javis_score": 0.1648, "desync": 0.6170,
},
}
def preflight(prompts_file: Path, media: Path, model: str) -> tuple[list[str], str]:
if model not in PAPER:
raise ValueError(f"Unrecognized model identity: {model}")
prompts_file = prompts_file.resolve(strict=True)
prompts = prompts_file.read_text(encoding="utf-8").splitlines()
if len(prompts) != 200 or any(not prompt.strip() for prompt in prompts):
raise ValueError("Expected exactly 200 non-empty prompts")
if not media_complete(media, 200):
missing = [
i for i in range(200)
if any(
not (media / f"sample_{i:04d}.{suffix}").is_file()
or (media / f"sample_{i:04d}.{suffix}").stat().st_size == 0
for suffix in ("mp4", "wav")
)
]
raise FileNotFoundError(f"Missing AV pairs in {media}; first failing index={missing[0]}")
for index, prompt in enumerate(prompts):
sidecar = media / f"sample_{index:04d}.json"
if not sidecar.is_file():
continue
payload = json.loads(sidecar.read_text(encoding="utf-8"))
if payload.get("index") != index or payload.get("prompt") != prompt or payload.get("seed_idx") != 0:
raise ValueError(f"Generated prompt/index/seed mismatch: {sidecar}")
digest = hashlib.sha256(prompts_file.read_bytes()).hexdigest()
return prompts, digest
def run(args: argparse.Namespace) -> int:
prompts, digest = preflight(args.prompts, args.media, args.model)
root = args.javis_root.resolve(strict=True)
python = args.javis_python.absolute()
if not python.is_file() or not (root / "eval/javisbench/main.py").is_file():
raise FileNotFoundError(f"Missing JavisBench or isolated Python: {root}, {python}")
output = args.output.resolve()
manifest_path = output / "cpu_score_manifest.json"
manifest = {
"model_identity": args.model,
"media_dir": str(args.media.resolve()),
"prompts": str(args.prompts.resolve()),
"prompts_sha256": digest,
"sample_count": len(prompts),
"score_device": "cpu",
"javis_root": str(root),
"javis_python": str(python),
"metrics": METRICS,
"paper_reference": "my_repos/turbot2av_paper/main.tex, Table VII",
"sampling_protocol_verified": False,
"provenance_note": "Caller must independently verify model identity and sampling steps; present generation sidecars checked for prompt alignment",
}
if manifest_path.exists():
if json.loads(manifest_path.read_text(encoding="utf-8")) != manifest:
raise RuntimeError("Settings changed; choose a different output directory")
elif output.exists() and any(output.iterdir()):
raise RuntimeError("Output already contains files without matching manifest")
scores = output / "javis_scores.json"
if not score_complete(scores):
output.mkdir(parents=True, exist_ok=True)
write_json(manifest_path, manifest)
input_csv = output / "javis_input.csv"
write_input(input_csv, prompts, args.media.resolve())
env = dict(os.environ)
env["CUDA_VISIBLE_DEVICES"] = ""
env["HF_HUB_OFFLINE"] = "1"
env["TRANSFORMERS_OFFLINE"] = "1"
env["TOKENIZERS_PARALLELISM"] = "false"
env["OMP_NUM_THREADS"] = str(args.cpu_threads)
env["MKL_NUM_THREADS"] = str(args.cpu_threads)
env["PATH"] = f"{python.parent}:{env.get('PATH', '')}"
subprocess.run(
[str(python), "-m", "eval.javisbench.main",
"--input_file", str(input_csv), "--output_file", str(scores),
"--metrics", *METRICS, "--max_audio_len_s", "4",
"--num_workers", str(args.workers)],
cwd=root, env=env, check=True,
)
if not score_complete(scores):
raise RuntimeError(f"Incomplete JavisBench scores: {scores}")
observed = json.loads(scores.read_text(encoding="utf-8"))
reference = PAPER[args.model]
result = {
"compared_at_utc": utc_now(),
"model_identity": args.model,
"note": "Numerical comparison only; prompt enhancement and generation protocol remain unverified",
"scores": {key: {
"observed": observed[key],
"paper": reference[key],
"delta": observed[key] - reference[key],
} for key in SCORES},
}
write_json(output / "paper_comparison.json", result)
print(json.dumps(result, indent=2))
return 0
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--model", choices=sorted(PAPER), required=True)
parser.add_argument("--media", type=Path, required=True)
parser.add_argument("--prompts", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--javis-root", type=Path, required=True)
parser.add_argument("--javis-python", type=Path, required=True)
parser.add_argument("--cpu-threads", type=int, default=4)
parser.add_argument("--workers", type=int, default=0)
args = parser.parse_args()
if args.cpu_threads < 1 or args.workers < 0:
parser.error("--cpu-threads must be positive and --workers nonnegative")
return run(args)
if __name__ == "__main__":
raise SystemExit(main())