#!/usr/bin/env python3 """Score already generated, prompt-aligned media using isolated JavisBench Python.""" from __future__ import annotations import argparse import hashlib import json import os import subprocess from pathlib import Path from common import write_json from run_checkpoint_curve import METRICS, SCORES, score_complete, write_input from score_existing_cpu_200 import PAPER def run(args: argparse.Namespace) -> int: if args.limit < 1 or args.limit > 200 or args.workers < 0: raise ValueError("Expected 1 <= limit <= 200 and nonnegative workers") prompts_path = args.prompts.resolve(strict=True) all_prompts = prompts_path.read_text(encoding="utf-8").splitlines() if len(all_prompts) != 200 or any(not p.strip() for p in all_prompts): raise ValueError("Expected exactly 200 nonempty prompts") prompts = all_prompts[:args.limit] media = args.media.resolve(strict=True) for idx, prompt in enumerate(prompts): for suffix in ("mp4", "wav"): path = media / f"sample_{idx:04d}.{suffix}" if not path.is_file() or path.stat().st_size == 0: raise FileNotFoundError(path) sidecar = media / f"sample_{idx:04d}.json" if not sidecar.is_file(): raise FileNotFoundError(f"Missing generation provenance: {sidecar}") metadata = json.loads(sidecar.read_text(encoding="utf-8")) if metadata.get("prompt") != prompt or metadata.get("index") != idx or metadata.get("seed_idx") != 0: raise ValueError(f"Generated sample not aligned to prompt: {sidecar}") output = args.output.resolve() root = args.javis_root.resolve(strict=True) python = args.javis_python.absolute() if not python.is_file(): raise FileNotFoundError(python) if not (root / "checkpoints/imagebind_huge.pth").is_file(): raise FileNotFoundError(root / "checkpoints/imagebind_huge.pth") manifest = { "model": args.model, "media": str(media), "prompts_file": str(prompts_path), "prompt_sha256": hashlib.sha256(prompts_path.read_bytes()).hexdigest(), "limit": args.limit, "metrics": METRICS, "gpu": args.gpu, "javis_python": str(python), "javis_root": str(root), } manifest_path = output / "manifest.json" if manifest_path.is_file(): if json.loads(manifest_path.read_text(encoding="utf-8")) != manifest: raise RuntimeError(f"Scoring settings changed: {manifest_path}") elif output.exists() and any(output.iterdir()): raise RuntimeError(f"Nonempty output without matching manifest: {output}") else: write_json(manifest_path, manifest) scores = output / "scores.json" if not score_complete(scores): csv_path = output / "input.csv" write_input(csv_path, prompts, media) env = dict(os.environ) env["CUDA_VISIBLE_DEVICES"] = str(args.gpu) env["HF_HUB_OFFLINE"] = "1" env["TRANSFORMERS_OFFLINE"] = "1" env["TOKENIZERS_PARALLELISM"] = "false" env["OMP_NUM_THREADS"] = str(args.cpu_threads) env["MKL_NUM_THREADS"] = str(args.cpu_threads) env["PATH"] = f"{python.parent}:{env.get('PATH', '')}" subprocess.run([ str(python), "-m", "eval.javisbench.main", "--input_file", str(csv_path), "--output_file", str(scores), "--metrics", *METRICS, "--max_audio_len_s", "4.0", "--num_workers", str(args.workers), ], cwd=root, env=env, check=True) if not score_complete(scores): raise RuntimeError(f"Incomplete scores: {scores}") values = json.loads(scores.read_text(encoding="utf-8")) if args.limit == 200: write_json(output / "paper_comparison.json", { "model": args.model, "note": "Numeric comparison; checkpoint identity and prompt enhancement still require verification.", "scores": {key: {"observed": values[key], "paper": PAPER[args.model][key], "delta": values[key] - PAPER[args.model][key]} for key in SCORES}, }) print(json.dumps({key: values[key] for key in SCORES}, indent=2)) return 0 def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--model", choices=sorted(PAPER), required=True) parser.add_argument("--media", type=Path, required=True) parser.add_argument("--prompts", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--javis-root", type=Path, required=True) parser.add_argument("--javis-python", type=Path, required=True) parser.add_argument("--gpu", default="7") parser.add_argument("--limit", type=int, default=200) parser.add_argument("--workers", type=int, default=4) parser.add_argument("--cpu-threads", type=int, default=4) return run(parser.parse_args()) if __name__ == "__main__": raise SystemExit(main())