Download code/evaluation/scripts/score_existing_gpu.py from luyu1021/turbot2va-ltx2: direct link, hf CLI and curl.
- Browser
- Download file 5.01 kB
-
https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/evaluation/scripts/score_existing_gpu.py
- Command line
-
hf download hf://luyu1021/turbot2va-ltx2/code/evaluation/scripts/score_existing_gpu.py
-
curl -L -o score_existing_gpu.py https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/evaluation/scripts/score_existing_gpu.py
5.01 kB
| #!/usr/bin/env python3 | |
| """Score already generated, prompt-aligned media using isolated JavisBench Python.""" | |
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import json | |
| import os | |
| import subprocess | |
| from pathlib import Path | |
| from common import write_json | |
| from run_checkpoint_curve import METRICS, SCORES, score_complete, write_input | |
| from score_existing_cpu_200 import PAPER | |
| def run(args: argparse.Namespace) -> int: | |
| if args.limit < 1 or args.limit > 200 or args.workers < 0: | |
| raise ValueError("Expected 1 <= limit <= 200 and nonnegative workers") | |
| prompts_path = args.prompts.resolve(strict=True) | |
| all_prompts = prompts_path.read_text(encoding="utf-8").splitlines() | |
| if len(all_prompts) != 200 or any(not p.strip() for p in all_prompts): | |
| raise ValueError("Expected exactly 200 nonempty prompts") | |
| prompts = all_prompts[:args.limit] | |
| media = args.media.resolve(strict=True) | |
| for idx, prompt in enumerate(prompts): | |
| for suffix in ("mp4", "wav"): | |
| path = media / f"sample_{idx:04d}.{suffix}" | |
| if not path.is_file() or path.stat().st_size == 0: | |
| raise FileNotFoundError(path) | |
| sidecar = media / f"sample_{idx:04d}.json" | |
| if not sidecar.is_file(): | |
| raise FileNotFoundError(f"Missing generation provenance: {sidecar}") | |
| metadata = json.loads(sidecar.read_text(encoding="utf-8")) | |
| if metadata.get("prompt") != prompt or metadata.get("index") != idx or metadata.get("seed_idx") != 0: | |
| raise ValueError(f"Generated sample not aligned to prompt: {sidecar}") | |
| output = args.output.resolve() | |
| root = args.javis_root.resolve(strict=True) | |
| python = args.javis_python.absolute() | |
| if not python.is_file(): | |
| raise FileNotFoundError(python) | |
| if not (root / "checkpoints/imagebind_huge.pth").is_file(): | |
| raise FileNotFoundError(root / "checkpoints/imagebind_huge.pth") | |
| manifest = { | |
| "model": args.model, | |
| "media": str(media), | |
| "prompts_file": str(prompts_path), | |
| "prompt_sha256": hashlib.sha256(prompts_path.read_bytes()).hexdigest(), | |
| "limit": args.limit, | |
| "metrics": METRICS, | |
| "gpu": args.gpu, | |
| "javis_python": str(python), | |
| "javis_root": str(root), | |
| } | |
| manifest_path = output / "manifest.json" | |
| if manifest_path.is_file(): | |
| if json.loads(manifest_path.read_text(encoding="utf-8")) != manifest: | |
| raise RuntimeError(f"Scoring settings changed: {manifest_path}") | |
| elif output.exists() and any(output.iterdir()): | |
| raise RuntimeError(f"Nonempty output without matching manifest: {output}") | |
| else: | |
| write_json(manifest_path, manifest) | |
| scores = output / "scores.json" | |
| if not score_complete(scores): | |
| csv_path = output / "input.csv" | |
| write_input(csv_path, prompts, media) | |
| env = dict(os.environ) | |
| env["CUDA_VISIBLE_DEVICES"] = str(args.gpu) | |
| env["HF_HUB_OFFLINE"] = "1" | |
| env["TRANSFORMERS_OFFLINE"] = "1" | |
| env["TOKENIZERS_PARALLELISM"] = "false" | |
| env["OMP_NUM_THREADS"] = str(args.cpu_threads) | |
| env["MKL_NUM_THREADS"] = str(args.cpu_threads) | |
| env["PATH"] = f"{python.parent}:{env.get('PATH', '')}" | |
| subprocess.run([ | |
| str(python), "-m", "eval.javisbench.main", | |
| "--input_file", str(csv_path), | |
| "--output_file", str(scores), | |
| "--metrics", *METRICS, "--max_audio_len_s", "4.0", | |
| "--num_workers", str(args.workers), | |
| ], cwd=root, env=env, check=True) | |
| if not score_complete(scores): | |
| raise RuntimeError(f"Incomplete scores: {scores}") | |
| values = json.loads(scores.read_text(encoding="utf-8")) | |
| if args.limit == 200: | |
| write_json(output / "paper_comparison.json", { | |
| "model": args.model, | |
| "note": "Numeric comparison; checkpoint identity and prompt enhancement still require verification.", | |
| "scores": {key: {"observed": values[key], "paper": PAPER[args.model][key], | |
| "delta": values[key] - PAPER[args.model][key]} for key in SCORES}, | |
| }) | |
| print(json.dumps({key: values[key] for key in SCORES}, indent=2)) | |
| return 0 | |
| def main() -> int: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--model", choices=sorted(PAPER), required=True) | |
| parser.add_argument("--media", type=Path, required=True) | |
| parser.add_argument("--prompts", type=Path, required=True) | |
| parser.add_argument("--output", type=Path, required=True) | |
| parser.add_argument("--javis-root", type=Path, required=True) | |
| parser.add_argument("--javis-python", type=Path, required=True) | |
| parser.add_argument("--gpu", default="7") | |
| parser.add_argument("--limit", type=int, default=200) | |
| parser.add_argument("--workers", type=int, default=4) | |
| parser.add_argument("--cpu-threads", type=int, default=4) | |
| return run(parser.parse_args()) | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |