#!/usr/bin/env python3 from __future__ import annotations import argparse import csv import json import subprocess import sys import time from pathlib import Path from common import cfg_path, load_config, media_dir_for_model, select_models, unset_proxy_env, utc_now, write_json def prompt_file(config: dict) -> Path: value = config.get("prompt_subset", {}).get("prompts_file") if value: return Path(value).expanduser() return cfg_path(config, "diversity_root") / "prompts_200.txt" def load_prompts(config: dict) -> list[str]: path = prompt_file(config) if not path.is_file(): raise FileNotFoundError(path) prompts = path.read_text(encoding="utf-8").splitlines() expected = int(config.get("prompt_subset", {}).get("num_prompts", len(prompts))) if len(prompts) != expected: raise RuntimeError(f"Expected {expected} prompts, got {len(prompts)} from {path}") return prompts def expected_keys(config: dict) -> set[str]: return set(config.get("javisbench", {}).get("expected_keys", [])) def output_json(config: dict, model: dict) -> Path: return cfg_path(config, "quality_output_root") / f"{model['name']}.json" def input_csv(config: dict, model: dict) -> Path: return cfg_path(config, "quality_input_root") / str(model["name"]) / "javisbench_input.csv" def load_json(path: Path) -> dict: with path.open(encoding="utf-8") as f: return json.load(f) def is_complete(config: dict, path: Path) -> bool: keys = expected_keys(config) if not path.exists() or path.stat().st_size == 0: return False if not keys: return True try: return keys.issubset(load_json(path).keys()) except Exception: return False def prepare_input_csv(config: dict, model: dict, prompts: list[str], seed_index: int) -> None: media_dir = media_dir_for_model(config, model) if not media_dir.is_dir(): raise FileNotFoundError(f"{model['name']}: missing media dir {media_dir}") csv_path = input_csv(config, model) csv_path.parent.mkdir(parents=True, exist_ok=True) missing = [] rows = [] for index, prompt in enumerate(prompts): stem = f"sample_{index:04d}_seed{seed_index:04d}" mp4 = media_dir / f"{stem}.mp4" wav = media_dir / f"{stem}.wav" if seed_index == 0 and not mp4.is_file() and not wav.is_file(): stem = f"sample_{index:04d}" mp4 = media_dir / f"{stem}.mp4" wav = media_dir / f"{stem}.wav" if not mp4.is_file() or not wav.is_file(): missing.append(stem) continue rows.append( { "video_id": f"sample_{index:04d}", "text": prompt, "path": str(mp4), "audio_path": str(wav), "pred_video_path": str(mp4), "pred_audio_path": str(wav), } ) if missing: raise RuntimeError(f"{model['name']}: missing {len(missing)} media pairs, first={missing[:10]}") with csv_path.open("w", newline="", encoding="utf-8") as f: writer = csv.DictWriter( f, fieldnames=["video_id", "text", "path", "audio_path", "pred_video_path", "pred_audio_path"] ) writer.writeheader() writer.writerows(rows) def launch(config: dict, model: dict, gpu: str, args: argparse.Namespace) -> dict: javis_root = cfg_path(config, "javis_root") python = cfg_path(config, "javis_python") if not python.exists(): raise FileNotFoundError(python) out_json = output_json(config, model) out_json.parent.mkdir(parents=True, exist_ok=True) log_dir = out_json.parent / "logs" log_dir.mkdir(parents=True, exist_ok=True) log_path = log_dir / f"{model['name']}.log" log_f = log_path.open("a", encoding="utf-8") bench = config.get("javisbench", {}) metrics = list(bench.get("metrics", [])) if not metrics: raise RuntimeError("Config javisbench.metrics is empty") cmd = [ str(python), "-m", "eval.javisbench.main", "--input_file", str(input_csv(config, model)), "--output_file", str(out_json), "--metrics", *metrics, "--max_audio_len_s", str(args.max_audio_len_s if args.max_audio_len_s is not None else bench.get("max_audio_len_s", 4.0)), "--num_workers", str(args.num_workers if args.num_workers is not None else bench.get("num_workers", 8)), "--force_eval", ] env = unset_proxy_env() env["CUDA_VISIBLE_DEVICES"] = str(gpu) env.setdefault("PYTHONUNBUFFERED", "1") env.setdefault("TOKENIZERS_PARALLELISM", "false") if bench.get("offline", True): env.setdefault("HF_HUB_OFFLINE", "1") env.setdefault("TRANSFORMERS_OFFLINE", "1") env["PATH"] = f"{python.parent}:{env.get('PATH', '')}" log_f.write(f"\n\n===== launch quality {utc_now()} =====\n") log_f.write("cmd: " + " ".join(cmd) + "\n") log_f.write(f"CUDA_VISIBLE_DEVICES={gpu}\n") log_f.flush() proc = subprocess.Popen(cmd, cwd=str(javis_root), env=env, stdout=log_f, stderr=subprocess.STDOUT, text=True) print(f"launch quality gpu={gpu} model={model['name']} pid={proc.pid} log={log_path}", flush=True) return {"model": model, "gpu": gpu, "proc": proc, "log_f": log_f, "log_path": log_path, "started": time.time()} def write_summary(config: dict, pending: list[dict], active: dict[str, dict], completed: list[str], failed: list[dict]) -> None: out_root = cfg_path(config, "quality_output_root") payload = { "updated_at_utc": utc_now(), "strategy": "javisbench_quality_existing_media_seed", "prompt_file": str(prompt_file(config)), "quality_input_root": str(cfg_path(config, "quality_input_root")), "quality_output_root": str(out_root), "pending": [model["name"] for model in pending], "active": { gpu: { "model": run["model"]["name"], "pid": run["proc"].pid, "log": str(run["log_path"]), "elapsed_seconds": round(time.time() - run["started"], 1), } for gpu, run in active.items() }, "completed": completed, "failed": failed, } write_json(out_root / "batch_summary.json", payload) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--config", required=True) parser.add_argument("--gpus", default="0") parser.add_argument("--only", default=None, help="comma-separated model names") parser.add_argument("--roles", default=None, help="comma-separated role filters") parser.add_argument("--seed-index", type=int, default=0) parser.add_argument("--poll-seconds", type=int, default=60) parser.add_argument("--force", action="store_true") parser.add_argument("--prepare-only", action="store_true") parser.add_argument("--max-audio-len-s", type=float, default=None) parser.add_argument("--num-workers", type=int, default=None) return parser.parse_args() def main() -> int: args = parse_args() config = load_config(args.config) gpus = [gpu.strip() for gpu in args.gpus.split(",") if gpu.strip()] if not gpus: raise RuntimeError("No GPUs specified") prompts = load_prompts(config) models = select_models(config, args.only, args.roles) for model in models: prepare_input_csv(config, model, prompts, args.seed_index) if args.prepare_only: print(f"prepared {len(models)} input CSV files") return 0 pending = [] completed = [] failed = [] for model in models: out_json = output_json(config, model) if args.force or not is_complete(config, out_json): pending.append(model) else: completed.append(model["name"]) active: dict[str, dict] = {} write_summary(config, pending, active, completed, failed) while pending or active: for gpu in gpus: if gpu in active or not pending: continue model = pending.pop(0) if not args.force and is_complete(config, output_json(config, model)): completed.append(model["name"]) continue active[gpu] = launch(config, model, gpu, args) time.sleep(args.poll_seconds) for gpu, run in list(active.items()): ret = run["proc"].poll() model = run["model"] if ret is None: print(f"progress quality gpu={gpu} model={model['name']} elapsed={time.time() - run['started']:.1f}s", flush=True) continue run["log_f"].close() if ret == 0 and is_complete(config, output_json(config, model)): completed.append(model["name"]) print(f"done quality gpu={gpu} model={model['name']} wall={time.time() - run['started']:.1f}s", flush=True) else: failed.append({"model": model["name"], "gpu": gpu, "returncode": ret, "log": str(run["log_path"])}) print(f"failed quality gpu={gpu} model={model['name']} ret={ret}", flush=True) del active[gpu] write_summary(config, pending, active, completed, failed) return 1 if failed else 0 if __name__ == "__main__": sys.exit(main())