Download code/evaluation/scripts/run_quality_javisbench.py from luyu1021/turbot2va-ltx2: direct link, hf CLI and curl.
- Browser
- Download file 9.42 kB
-
https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/evaluation/scripts/run_quality_javisbench.py
- Command line
-
hf download hf://luyu1021/turbot2va-ltx2/code/evaluation/scripts/run_quality_javisbench.py
-
curl -L -o run_quality_javisbench.py https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/evaluation/scripts/run_quality_javisbench.py
9.42 kB
| #!/usr/bin/env python3 | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import json | |
| import subprocess | |
| import sys | |
| import time | |
| from pathlib import Path | |
| from common import cfg_path, load_config, media_dir_for_model, select_models, unset_proxy_env, utc_now, write_json | |
| def prompt_file(config: dict) -> Path: | |
| value = config.get("prompt_subset", {}).get("prompts_file") | |
| if value: | |
| return Path(value).expanduser() | |
| return cfg_path(config, "diversity_root") / "prompts_200.txt" | |
| def load_prompts(config: dict) -> list[str]: | |
| path = prompt_file(config) | |
| if not path.is_file(): | |
| raise FileNotFoundError(path) | |
| prompts = path.read_text(encoding="utf-8").splitlines() | |
| expected = int(config.get("prompt_subset", {}).get("num_prompts", len(prompts))) | |
| if len(prompts) != expected: | |
| raise RuntimeError(f"Expected {expected} prompts, got {len(prompts)} from {path}") | |
| return prompts | |
| def expected_keys(config: dict) -> set[str]: | |
| return set(config.get("javisbench", {}).get("expected_keys", [])) | |
| def output_json(config: dict, model: dict) -> Path: | |
| return cfg_path(config, "quality_output_root") / f"{model['name']}.json" | |
| def input_csv(config: dict, model: dict) -> Path: | |
| return cfg_path(config, "quality_input_root") / str(model["name"]) / "javisbench_input.csv" | |
| def load_json(path: Path) -> dict: | |
| with path.open(encoding="utf-8") as f: | |
| return json.load(f) | |
| def is_complete(config: dict, path: Path) -> bool: | |
| keys = expected_keys(config) | |
| if not path.exists() or path.stat().st_size == 0: | |
| return False | |
| if not keys: | |
| return True | |
| try: | |
| return keys.issubset(load_json(path).keys()) | |
| except Exception: | |
| return False | |
| def prepare_input_csv(config: dict, model: dict, prompts: list[str], seed_index: int) -> None: | |
| media_dir = media_dir_for_model(config, model) | |
| if not media_dir.is_dir(): | |
| raise FileNotFoundError(f"{model['name']}: missing media dir {media_dir}") | |
| csv_path = input_csv(config, model) | |
| csv_path.parent.mkdir(parents=True, exist_ok=True) | |
| missing = [] | |
| rows = [] | |
| for index, prompt in enumerate(prompts): | |
| stem = f"sample_{index:04d}_seed{seed_index:04d}" | |
| mp4 = media_dir / f"{stem}.mp4" | |
| wav = media_dir / f"{stem}.wav" | |
| if seed_index == 0 and not mp4.is_file() and not wav.is_file(): | |
| stem = f"sample_{index:04d}" | |
| mp4 = media_dir / f"{stem}.mp4" | |
| wav = media_dir / f"{stem}.wav" | |
| if not mp4.is_file() or not wav.is_file(): | |
| missing.append(stem) | |
| continue | |
| rows.append( | |
| { | |
| "video_id": f"sample_{index:04d}", | |
| "text": prompt, | |
| "path": str(mp4), | |
| "audio_path": str(wav), | |
| "pred_video_path": str(mp4), | |
| "pred_audio_path": str(wav), | |
| } | |
| ) | |
| if missing: | |
| raise RuntimeError(f"{model['name']}: missing {len(missing)} media pairs, first={missing[:10]}") | |
| with csv_path.open("w", newline="", encoding="utf-8") as f: | |
| writer = csv.DictWriter( | |
| f, fieldnames=["video_id", "text", "path", "audio_path", "pred_video_path", "pred_audio_path"] | |
| ) | |
| writer.writeheader() | |
| writer.writerows(rows) | |
| def launch(config: dict, model: dict, gpu: str, args: argparse.Namespace) -> dict: | |
| javis_root = cfg_path(config, "javis_root") | |
| python = cfg_path(config, "javis_python") | |
| if not python.exists(): | |
| raise FileNotFoundError(python) | |
| out_json = output_json(config, model) | |
| out_json.parent.mkdir(parents=True, exist_ok=True) | |
| log_dir = out_json.parent / "logs" | |
| log_dir.mkdir(parents=True, exist_ok=True) | |
| log_path = log_dir / f"{model['name']}.log" | |
| log_f = log_path.open("a", encoding="utf-8") | |
| bench = config.get("javisbench", {}) | |
| metrics = list(bench.get("metrics", [])) | |
| if not metrics: | |
| raise RuntimeError("Config javisbench.metrics is empty") | |
| cmd = [ | |
| str(python), | |
| "-m", | |
| "eval.javisbench.main", | |
| "--input_file", | |
| str(input_csv(config, model)), | |
| "--output_file", | |
| str(out_json), | |
| "--metrics", | |
| *metrics, | |
| "--max_audio_len_s", | |
| str(args.max_audio_len_s if args.max_audio_len_s is not None else bench.get("max_audio_len_s", 4.0)), | |
| "--num_workers", | |
| str(args.num_workers if args.num_workers is not None else bench.get("num_workers", 8)), | |
| "--force_eval", | |
| ] | |
| env = unset_proxy_env() | |
| env["CUDA_VISIBLE_DEVICES"] = str(gpu) | |
| env.setdefault("PYTHONUNBUFFERED", "1") | |
| env.setdefault("TOKENIZERS_PARALLELISM", "false") | |
| if bench.get("offline", True): | |
| env.setdefault("HF_HUB_OFFLINE", "1") | |
| env.setdefault("TRANSFORMERS_OFFLINE", "1") | |
| env["PATH"] = f"{python.parent}:{env.get('PATH', '')}" | |
| log_f.write(f"\n\n===== launch quality {utc_now()} =====\n") | |
| log_f.write("cmd: " + " ".join(cmd) + "\n") | |
| log_f.write(f"CUDA_VISIBLE_DEVICES={gpu}\n") | |
| log_f.flush() | |
| proc = subprocess.Popen(cmd, cwd=str(javis_root), env=env, stdout=log_f, stderr=subprocess.STDOUT, text=True) | |
| print(f"launch quality gpu={gpu} model={model['name']} pid={proc.pid} log={log_path}", flush=True) | |
| return {"model": model, "gpu": gpu, "proc": proc, "log_f": log_f, "log_path": log_path, "started": time.time()} | |
| def write_summary(config: dict, pending: list[dict], active: dict[str, dict], completed: list[str], failed: list[dict]) -> None: | |
| out_root = cfg_path(config, "quality_output_root") | |
| payload = { | |
| "updated_at_utc": utc_now(), | |
| "strategy": "javisbench_quality_existing_media_seed", | |
| "prompt_file": str(prompt_file(config)), | |
| "quality_input_root": str(cfg_path(config, "quality_input_root")), | |
| "quality_output_root": str(out_root), | |
| "pending": [model["name"] for model in pending], | |
| "active": { | |
| gpu: { | |
| "model": run["model"]["name"], | |
| "pid": run["proc"].pid, | |
| "log": str(run["log_path"]), | |
| "elapsed_seconds": round(time.time() - run["started"], 1), | |
| } | |
| for gpu, run in active.items() | |
| }, | |
| "completed": completed, | |
| "failed": failed, | |
| } | |
| write_json(out_root / "batch_summary.json", payload) | |
| def parse_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--config", required=True) | |
| parser.add_argument("--gpus", default="0") | |
| parser.add_argument("--only", default=None, help="comma-separated model names") | |
| parser.add_argument("--roles", default=None, help="comma-separated role filters") | |
| parser.add_argument("--seed-index", type=int, default=0) | |
| parser.add_argument("--poll-seconds", type=int, default=60) | |
| parser.add_argument("--force", action="store_true") | |
| parser.add_argument("--prepare-only", action="store_true") | |
| parser.add_argument("--max-audio-len-s", type=float, default=None) | |
| parser.add_argument("--num-workers", type=int, default=None) | |
| return parser.parse_args() | |
| def main() -> int: | |
| args = parse_args() | |
| config = load_config(args.config) | |
| gpus = [gpu.strip() for gpu in args.gpus.split(",") if gpu.strip()] | |
| if not gpus: | |
| raise RuntimeError("No GPUs specified") | |
| prompts = load_prompts(config) | |
| models = select_models(config, args.only, args.roles) | |
| for model in models: | |
| prepare_input_csv(config, model, prompts, args.seed_index) | |
| if args.prepare_only: | |
| print(f"prepared {len(models)} input CSV files") | |
| return 0 | |
| pending = [] | |
| completed = [] | |
| failed = [] | |
| for model in models: | |
| out_json = output_json(config, model) | |
| if args.force or not is_complete(config, out_json): | |
| pending.append(model) | |
| else: | |
| completed.append(model["name"]) | |
| active: dict[str, dict] = {} | |
| write_summary(config, pending, active, completed, failed) | |
| while pending or active: | |
| for gpu in gpus: | |
| if gpu in active or not pending: | |
| continue | |
| model = pending.pop(0) | |
| if not args.force and is_complete(config, output_json(config, model)): | |
| completed.append(model["name"]) | |
| continue | |
| active[gpu] = launch(config, model, gpu, args) | |
| time.sleep(args.poll_seconds) | |
| for gpu, run in list(active.items()): | |
| ret = run["proc"].poll() | |
| model = run["model"] | |
| if ret is None: | |
| print(f"progress quality gpu={gpu} model={model['name']} elapsed={time.time() - run['started']:.1f}s", flush=True) | |
| continue | |
| run["log_f"].close() | |
| if ret == 0 and is_complete(config, output_json(config, model)): | |
| completed.append(model["name"]) | |
| print(f"done quality gpu={gpu} model={model['name']} wall={time.time() - run['started']:.1f}s", flush=True) | |
| else: | |
| failed.append({"model": model["name"], "gpu": gpu, "returncode": ret, "log": str(run["log_path"])}) | |
| print(f"failed quality gpu={gpu} model={model['name']} ret={ret}", flush=True) | |
| del active[gpu] | |
| write_summary(config, pending, active, completed, failed) | |
| return 1 if failed else 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |