turbot2va-ltx2 / code /evaluation /scripts /run_quality_javisbench.py
luyu1021's picture
Add documented LTX-2 training source, recipes and checkpoint release plan
76b7b25 verified
Raw History Blame Contribute Delete
9.42 kB
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import csv
import json
import subprocess
import sys
import time
from pathlib import Path
from common import cfg_path, load_config, media_dir_for_model, select_models, unset_proxy_env, utc_now, write_json
def prompt_file(config: dict) -> Path:
value = config.get("prompt_subset", {}).get("prompts_file")
if value:
return Path(value).expanduser()
return cfg_path(config, "diversity_root") / "prompts_200.txt"
def load_prompts(config: dict) -> list[str]:
path = prompt_file(config)
if not path.is_file():
raise FileNotFoundError(path)
prompts = path.read_text(encoding="utf-8").splitlines()
expected = int(config.get("prompt_subset", {}).get("num_prompts", len(prompts)))
if len(prompts) != expected:
raise RuntimeError(f"Expected {expected} prompts, got {len(prompts)} from {path}")
return prompts
def expected_keys(config: dict) -> set[str]:
return set(config.get("javisbench", {}).get("expected_keys", []))
def output_json(config: dict, model: dict) -> Path:
return cfg_path(config, "quality_output_root") / f"{model['name']}.json"
def input_csv(config: dict, model: dict) -> Path:
return cfg_path(config, "quality_input_root") / str(model["name"]) / "javisbench_input.csv"
def load_json(path: Path) -> dict:
with path.open(encoding="utf-8") as f:
return json.load(f)
def is_complete(config: dict, path: Path) -> bool:
keys = expected_keys(config)
if not path.exists() or path.stat().st_size == 0:
return False
if not keys:
return True
try:
return keys.issubset(load_json(path).keys())
except Exception:
return False
def prepare_input_csv(config: dict, model: dict, prompts: list[str], seed_index: int) -> None:
media_dir = media_dir_for_model(config, model)
if not media_dir.is_dir():
raise FileNotFoundError(f"{model['name']}: missing media dir {media_dir}")
csv_path = input_csv(config, model)
csv_path.parent.mkdir(parents=True, exist_ok=True)
missing = []
rows = []
for index, prompt in enumerate(prompts):
stem = f"sample_{index:04d}_seed{seed_index:04d}"
mp4 = media_dir / f"{stem}.mp4"
wav = media_dir / f"{stem}.wav"
if seed_index == 0 and not mp4.is_file() and not wav.is_file():
stem = f"sample_{index:04d}"
mp4 = media_dir / f"{stem}.mp4"
wav = media_dir / f"{stem}.wav"
if not mp4.is_file() or not wav.is_file():
missing.append(stem)
continue
rows.append(
{
"video_id": f"sample_{index:04d}",
"text": prompt,
"path": str(mp4),
"audio_path": str(wav),
"pred_video_path": str(mp4),
"pred_audio_path": str(wav),
}
)
if missing:
raise RuntimeError(f"{model['name']}: missing {len(missing)} media pairs, first={missing[:10]}")
with csv_path.open("w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(
f, fieldnames=["video_id", "text", "path", "audio_path", "pred_video_path", "pred_audio_path"]
)
writer.writeheader()
writer.writerows(rows)
def launch(config: dict, model: dict, gpu: str, args: argparse.Namespace) -> dict:
javis_root = cfg_path(config, "javis_root")
python = cfg_path(config, "javis_python")
if not python.exists():
raise FileNotFoundError(python)
out_json = output_json(config, model)
out_json.parent.mkdir(parents=True, exist_ok=True)
log_dir = out_json.parent / "logs"
log_dir.mkdir(parents=True, exist_ok=True)
log_path = log_dir / f"{model['name']}.log"
log_f = log_path.open("a", encoding="utf-8")
bench = config.get("javisbench", {})
metrics = list(bench.get("metrics", []))
if not metrics:
raise RuntimeError("Config javisbench.metrics is empty")
cmd = [
str(python),
"-m",
"eval.javisbench.main",
"--input_file",
str(input_csv(config, model)),
"--output_file",
str(out_json),
"--metrics",
*metrics,
"--max_audio_len_s",
str(args.max_audio_len_s if args.max_audio_len_s is not None else bench.get("max_audio_len_s", 4.0)),
"--num_workers",
str(args.num_workers if args.num_workers is not None else bench.get("num_workers", 8)),
"--force_eval",
]
env = unset_proxy_env()
env["CUDA_VISIBLE_DEVICES"] = str(gpu)
env.setdefault("PYTHONUNBUFFERED", "1")
env.setdefault("TOKENIZERS_PARALLELISM", "false")
if bench.get("offline", True):
env.setdefault("HF_HUB_OFFLINE", "1")
env.setdefault("TRANSFORMERS_OFFLINE", "1")
env["PATH"] = f"{python.parent}:{env.get('PATH', '')}"
log_f.write(f"\n\n===== launch quality {utc_now()} =====\n")
log_f.write("cmd: " + " ".join(cmd) + "\n")
log_f.write(f"CUDA_VISIBLE_DEVICES={gpu}\n")
log_f.flush()
proc = subprocess.Popen(cmd, cwd=str(javis_root), env=env, stdout=log_f, stderr=subprocess.STDOUT, text=True)
print(f"launch quality gpu={gpu} model={model['name']} pid={proc.pid} log={log_path}", flush=True)
return {"model": model, "gpu": gpu, "proc": proc, "log_f": log_f, "log_path": log_path, "started": time.time()}
def write_summary(config: dict, pending: list[dict], active: dict[str, dict], completed: list[str], failed: list[dict]) -> None:
out_root = cfg_path(config, "quality_output_root")
payload = {
"updated_at_utc": utc_now(),
"strategy": "javisbench_quality_existing_media_seed",
"prompt_file": str(prompt_file(config)),
"quality_input_root": str(cfg_path(config, "quality_input_root")),
"quality_output_root": str(out_root),
"pending": [model["name"] for model in pending],
"active": {
gpu: {
"model": run["model"]["name"],
"pid": run["proc"].pid,
"log": str(run["log_path"]),
"elapsed_seconds": round(time.time() - run["started"], 1),
}
for gpu, run in active.items()
},
"completed": completed,
"failed": failed,
}
write_json(out_root / "batch_summary.json", payload)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--config", required=True)
parser.add_argument("--gpus", default="0")
parser.add_argument("--only", default=None, help="comma-separated model names")
parser.add_argument("--roles", default=None, help="comma-separated role filters")
parser.add_argument("--seed-index", type=int, default=0)
parser.add_argument("--poll-seconds", type=int, default=60)
parser.add_argument("--force", action="store_true")
parser.add_argument("--prepare-only", action="store_true")
parser.add_argument("--max-audio-len-s", type=float, default=None)
parser.add_argument("--num-workers", type=int, default=None)
return parser.parse_args()
def main() -> int:
args = parse_args()
config = load_config(args.config)
gpus = [gpu.strip() for gpu in args.gpus.split(",") if gpu.strip()]
if not gpus:
raise RuntimeError("No GPUs specified")
prompts = load_prompts(config)
models = select_models(config, args.only, args.roles)
for model in models:
prepare_input_csv(config, model, prompts, args.seed_index)
if args.prepare_only:
print(f"prepared {len(models)} input CSV files")
return 0
pending = []
completed = []
failed = []
for model in models:
out_json = output_json(config, model)
if args.force or not is_complete(config, out_json):
pending.append(model)
else:
completed.append(model["name"])
active: dict[str, dict] = {}
write_summary(config, pending, active, completed, failed)
while pending or active:
for gpu in gpus:
if gpu in active or not pending:
continue
model = pending.pop(0)
if not args.force and is_complete(config, output_json(config, model)):
completed.append(model["name"])
continue
active[gpu] = launch(config, model, gpu, args)
time.sleep(args.poll_seconds)
for gpu, run in list(active.items()):
ret = run["proc"].poll()
model = run["model"]
if ret is None:
print(f"progress quality gpu={gpu} model={model['name']} elapsed={time.time() - run['started']:.1f}s", flush=True)
continue
run["log_f"].close()
if ret == 0 and is_complete(config, output_json(config, model)):
completed.append(model["name"])
print(f"done quality gpu={gpu} model={model['name']} wall={time.time() - run['started']:.1f}s", flush=True)
else:
failed.append({"model": model["name"], "gpu": gpu, "returncode": ret, "log": str(run["log_path"])})
print(f"failed quality gpu={gpu} model={model['name']} ret={ret}", flush=True)
del active[gpu]
write_summary(config, pending, active, completed, failed)
return 1 if failed else 0
if __name__ == "__main__":
sys.exit(main())