#!/usr/bin/env python3 """Watch in-progress AV media and score each completed experiment on a spare GPU.""" from __future__ import annotations import argparse import json import os import subprocess import sys import time from pathlib import Path from common import utc_now, write_json from run_checkpoint_curve import score_complete def media_count(directory: Path, indices: range, seeds: int | None = None) -> int: def complete(stem: str) -> bool: return all( (directory / f"{stem}.{suffix}").is_file() and (directory / f"{stem}.{suffix}").stat().st_size > 0 for suffix in ("mp4", "wav", "json") ) return sum( complete(f"sample_{i:04d}" + (f"_seed{j:04d}" if seeds is not None else "")) for i in indices for j in (range(seeds) if seeds is not None else (None,)) ) def run(args: argparse.Namespace) -> int: if args.poll_seconds < 2 or args.max_hours <= 0: raise ValueError("--poll-seconds >= 2 and --max-hours > 0 are required") script_root = Path(__file__).resolve().parent javis_python = args.javis_python.absolute() javis_root = args.javis_root.resolve(strict=True) media = {name: path.resolve() for name, path in ( ("student", args.student_media), ("teacher_40_step", args.teacher_media), ("diversity", args.diversity_media))} outputs = {name: args.output.resolve() / name for name in ("student", "teacher_40_step", "diversity")} status_file = args.output.resolve() / "watch_status.json" prompts = args.prompts.resolve(strict=True).read_text(encoding="utf-8").splitlines() if len(prompts) != 200: raise ValueError("Expected 200 prompts") started = time.monotonic() targets = {"student"} if args.student_only else {"student", "teacher_40_step", "diversity"} done: set[str] = { name for name in ("student", "teacher_40_step") if name in targets and score_complete(outputs[name] / "scores.json") } diversity_summary = outputs["diversity"] / "summary.json" if "diversity" in targets and diversity_summary.is_file(): payload = json.loads(diversity_summary.read_text(encoding="utf-8")) if len(payload.get("prompt_indices", [])) == 200 and payload.get("seeds") == 8: done.add("diversity") failures: dict[str, str] = {} while time.monotonic() - started < args.max_hours * 3600: counts = { "student": media_count(media["student"], range(200)), "teacher_40_step": media_count(media["teacher_40_step"], range(200)), "diversity": media_count(media["diversity"], range(200), seeds=8), } if "student" not in done and counts["student"] == 200: target = "student" elif "teacher_40_step" in targets and "teacher_40_step" not in done and counts["teacher_40_step"] == 200: target = "teacher_40_step" elif "diversity" in targets and "diversity" not in done and counts["diversity"] == 1600: target = "diversity" else: target = None write_json(status_file, { "updated_at_utc": utc_now(), "gpu": args.gpu, "counts": counts, "completed_scoring": sorted(done), "failures": failures, "active_scoring": target, }) print(f"{utc_now()} media {counts} scoring {target or 'waiting'}", flush=True) if targets <= done: return 0 if target is None: time.sleep(args.poll_seconds) continue env = dict(os.environ) env.update({ "CUDA_VISIBLE_DEVICES": str(args.gpu), "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", "PYTHONUNBUFFERED": "1", "OMP_NUM_THREADS": "4", "MKL_NUM_THREADS": "4", }) if target == "diversity": command = [ str(javis_python), str(script_root / "score_diversity_completed.py"), "--media", str(media[target]), "--prompts", str(args.prompts), "--javis-root", str(javis_root), "--output", str(outputs[target]), "--gpu", str(args.gpu), "--batch-size", "1", "--workers", "2", ] else: command = [ sys.executable, str(script_root / "score_existing_gpu.py"), "--model", target, "--media", str(media[target]), "--prompts", str(args.prompts), "--output", str(outputs[target]), "--javis-root", str(javis_root), "--javis-python", str(javis_python), "--gpu", str(args.gpu), "--workers", "4", ] print(f"start {target}: {' '.join(command)}", flush=True) result = subprocess.run(command, env=env, check=False) if result.returncode: failures[target] = f"exit code {result.returncode}" write_json(status_file, {"updated_at_utc": utc_now(), "counts": counts, "completed_scoring": sorted(done), "failures": failures}) raise RuntimeError(f"{target} scoring failed with {result.returncode}") if target == "diversity": summary = json.loads((outputs[target] / "summary.json").read_text(encoding="utf-8")) if len(summary["prompt_indices"]) != 200: raise RuntimeError("Diversity result does not cover all 200 prompts") reference = {"video_pairwise_cosine_distance": 0.2322, "audio_pairwise_cosine_distance": 0.4197, "va_average_pairwise_cosine_distance": 0.3259} metrics = summary["metrics"] write_json(outputs[target] / "paper_comparison.json", { "note": "Numeric comparison, pending prompt enhancement/checkpoint identity verification.", "metrics": {key: {"observed": metrics[key], "paper": value, "delta": metrics[key] - value} for key, value in reference.items()}, }) elif not score_complete(outputs[target] / "scores.json"): raise RuntimeError(f"{target} JavisBench output incomplete") done.add(target) raise TimeoutError(f"Evaluation watch timed out after {args.max_hours}h; status: {status_file}") def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--student-media", type=Path, required=True) parser.add_argument("--teacher-media", type=Path, required=True) parser.add_argument("--diversity-media", type=Path, required=True) parser.add_argument("--prompts", type=Path, required=True) parser.add_argument("--javis-root", type=Path, required=True) parser.add_argument("--javis-python", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--gpu", default="7") parser.add_argument("--poll-seconds", type=int, default=30) parser.add_argument("--max-hours", type=float, default=48) parser.add_argument("--student-only", action="store_true", help="Score student and exit without starting other evaluations") return run(parser.parse_args()) if __name__ == "__main__": raise SystemExit(main())