turbot2va-ltx2 / code /evaluation /scripts /score_existing_gpu.py
luyu1021's picture
Add documented LTX-2 training source, recipes and checkpoint release plan
76b7b25 verified
Raw History Blame Contribute Delete
5.01 kB
#!/usr/bin/env python3
"""Score already generated, prompt-aligned media using isolated JavisBench Python."""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import subprocess
from pathlib import Path
from common import write_json
from run_checkpoint_curve import METRICS, SCORES, score_complete, write_input
from score_existing_cpu_200 import PAPER
def run(args: argparse.Namespace) -> int:
if args.limit < 1 or args.limit > 200 or args.workers < 0:
raise ValueError("Expected 1 <= limit <= 200 and nonnegative workers")
prompts_path = args.prompts.resolve(strict=True)
all_prompts = prompts_path.read_text(encoding="utf-8").splitlines()
if len(all_prompts) != 200 or any(not p.strip() for p in all_prompts):
raise ValueError("Expected exactly 200 nonempty prompts")
prompts = all_prompts[:args.limit]
media = args.media.resolve(strict=True)
for idx, prompt in enumerate(prompts):
for suffix in ("mp4", "wav"):
path = media / f"sample_{idx:04d}.{suffix}"
if not path.is_file() or path.stat().st_size == 0:
raise FileNotFoundError(path)
sidecar = media / f"sample_{idx:04d}.json"
if not sidecar.is_file():
raise FileNotFoundError(f"Missing generation provenance: {sidecar}")
metadata = json.loads(sidecar.read_text(encoding="utf-8"))
if metadata.get("prompt") != prompt or metadata.get("index") != idx or metadata.get("seed_idx") != 0:
raise ValueError(f"Generated sample not aligned to prompt: {sidecar}")
output = args.output.resolve()
root = args.javis_root.resolve(strict=True)
python = args.javis_python.absolute()
if not python.is_file():
raise FileNotFoundError(python)
if not (root / "checkpoints/imagebind_huge.pth").is_file():
raise FileNotFoundError(root / "checkpoints/imagebind_huge.pth")
manifest = {
"model": args.model,
"media": str(media),
"prompts_file": str(prompts_path),
"prompt_sha256": hashlib.sha256(prompts_path.read_bytes()).hexdigest(),
"limit": args.limit,
"metrics": METRICS,
"gpu": args.gpu,
"javis_python": str(python),
"javis_root": str(root),
}
manifest_path = output / "manifest.json"
if manifest_path.is_file():
if json.loads(manifest_path.read_text(encoding="utf-8")) != manifest:
raise RuntimeError(f"Scoring settings changed: {manifest_path}")
elif output.exists() and any(output.iterdir()):
raise RuntimeError(f"Nonempty output without matching manifest: {output}")
else:
write_json(manifest_path, manifest)
scores = output / "scores.json"
if not score_complete(scores):
csv_path = output / "input.csv"
write_input(csv_path, prompts, media)
env = dict(os.environ)
env["CUDA_VISIBLE_DEVICES"] = str(args.gpu)
env["HF_HUB_OFFLINE"] = "1"
env["TRANSFORMERS_OFFLINE"] = "1"
env["TOKENIZERS_PARALLELISM"] = "false"
env["OMP_NUM_THREADS"] = str(args.cpu_threads)
env["MKL_NUM_THREADS"] = str(args.cpu_threads)
env["PATH"] = f"{python.parent}:{env.get('PATH', '')}"
subprocess.run([
str(python), "-m", "eval.javisbench.main",
"--input_file", str(csv_path),
"--output_file", str(scores),
"--metrics", *METRICS, "--max_audio_len_s", "4.0",
"--num_workers", str(args.workers),
], cwd=root, env=env, check=True)
if not score_complete(scores):
raise RuntimeError(f"Incomplete scores: {scores}")
values = json.loads(scores.read_text(encoding="utf-8"))
if args.limit == 200:
write_json(output / "paper_comparison.json", {
"model": args.model,
"note": "Numeric comparison; checkpoint identity and prompt enhancement still require verification.",
"scores": {key: {"observed": values[key], "paper": PAPER[args.model][key],
"delta": values[key] - PAPER[args.model][key]} for key in SCORES},
})
print(json.dumps({key: values[key] for key in SCORES}, indent=2))
return 0
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--model", choices=sorted(PAPER), required=True)
parser.add_argument("--media", type=Path, required=True)
parser.add_argument("--prompts", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--javis-root", type=Path, required=True)
parser.add_argument("--javis-python", type=Path, required=True)
parser.add_argument("--gpu", default="7")
parser.add_argument("--limit", type=int, default=200)
parser.add_argument("--workers", type=int, default=4)
parser.add_argument("--cpu-threads", type=int, default=4)
return run(parser.parse_args())
if __name__ == "__main__":
raise SystemExit(main())