#!/usr/bin/env python3 """Generate and score a run's FSDP checkpoints, resuming completed points.""" from __future__ import annotations import argparse import csv import hashlib import json import math import re import subprocess import sys from pathlib import Path from common import load_config, unset_proxy_env METRICS = ["video-quality", "audio-quality", "imagebind-score", "cxxp-score", "av-score", "desync"] SCORES = [ "javis_score", "visual_quality", "motion_quality", "audio_quality", "ib_tv", "ib_ta", "clip_score", "clap_score", "ib_av", "cavp_score", "avh_score", "desync", ] COLUMNS = ["run_name", "step", "step_offset", "samples_seen", *SCORES] def checkpoints(run: Path) -> list[tuple[int, Path]]: found = [] for path in (run / "checkpoints").glob("checkpoint_*"): match = re.fullmatch(r"checkpoint_(\d+)", path.name) if match and (path / "sharded").is_dir() and (path / "metadata.pth").is_file(): found.append((int(match.group(1)), path)) return sorted(found) def media_complete(directory: Path, count: int, prompts: list[str] | None = None, seed: int = 12345) -> bool: for index in range(count): for suffix in ("mp4", "wav"): path = directory / f"sample_{index:04d}.{suffix}" if not path.is_file() or path.stat().st_size == 0: return False if prompts is not None: sidecar = directory / f"sample_{index:04d}.json" try: record = json.loads(sidecar.read_text(encoding="utf-8")) except (OSError, ValueError): return False if (record.get("prompt") != prompts[index] or record.get("index") != index or record.get("seed") != seed + index or record.get("seed_idx") != 0): return False return True def score_complete(path: Path, required: list[str] | None = None) -> bool: try: payload = json.loads(path.read_text(encoding="utf-8")) return all( isinstance(payload[key], (int, float)) and not isinstance(payload[key], bool) and math.isfinite(payload[key]) for key in (required or SCORES) ) except (OSError, ValueError, KeyError, TypeError, OverflowError): return False def write_input(path: Path, prompts: list[str], media: Path) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", newline="", encoding="utf-8") as handle: writer = csv.DictWriter(handle, fieldnames=[ "video_id", "text", "path", "audio_path", "pred_video_path", "pred_audio_path" ]) writer.writeheader() for index, prompt in enumerate(prompts): video = media / f"sample_{index:04d}.mp4" audio = media / f"sample_{index:04d}.wav" writer.writerow({ "video_id": f"sample_{index:04d}", "text": prompt, "path": video, "audio_path": audio, "pred_video_path": video, "pred_audio_path": audio, }) def write_summary(path: Path, points: list[tuple[int, Path]], scores: Path, run: Path, offset: int, batch_size: int, required: list[str] | None = None) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", newline="", encoding="utf-8") as handle: writer = csv.DictWriter(handle, fieldnames=COLUMNS) writer.writeheader() for step, _ in points: score_file = scores / f"checkpoint_{step:06d}.json" if not score_complete(score_file, required): continue payload = json.loads(score_file.read_text(encoding="utf-8")) writer.writerow({ "run_name": run.name, "step": step, "step_offset": offset, "samples_seen": (offset + step) * batch_size, **{key: payload.get(key, "") for key in SCORES}, }) def training_overlap(mapping: Path, prompts: list[str]) -> set[str]: wanted = {prompt.strip().casefold() for prompt in prompts} matches = set() with mapping.open(newline="", encoding="utf-8") as handle: reader = csv.DictReader(handle) if not reader.fieldnames or "prompt" not in reader.fieldnames: raise ValueError(f"Training mapping has no prompt column: {mapping}") for row in reader: prompt = row["prompt"].strip().casefold() if prompt in wanted: matches.add(prompt) return matches def check_manifest(output: Path, settings: dict, dry_run: bool) -> None: manifest = output / "evaluation_manifest.json" if manifest.is_file(): if json.loads(manifest.read_text(encoding="utf-8")) != settings: raise RuntimeError(f"Evaluation settings changed; choose a new output directory: {output}") elif output.exists() and any(output.iterdir()): raise RuntimeError(f"Output has files without an evaluation manifest: {output}") elif not dry_run: output.mkdir(parents=True, exist_ok=True) manifest.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8") def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--run", type=Path, required=True, help="run dir with config.yaml and checkpoints/") parser.add_argument("--step-offset", type=int, default=0, help="prior stages' local steps; required for rCM curves") parser.add_argument("--batch-size", type=int, default=16) parser.add_argument("--prompts", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--javis-root", type=Path, required=True) parser.add_argument("--javis-python", type=Path, required=True) parser.add_argument("--ltx-root", type=Path, required=True) parser.add_argument("--ltx-python", type=Path, required=True) parser.add_argument("--gpu", default="0") parser.add_argument("--seed", type=int, default=12345) parser.add_argument("--limit", type=int, default=200) parser.add_argument("--steps", type=int, nargs="+", help="Only run these checkpoint steps; useful for per-GPU scheduling") parser.add_argument("--javis-only", action="store_true", help="Only av-score (Javis and AVH); skip slower auxiliary metrics") parser.add_argument("--num-workers", type=int, default=4) parser.add_argument("--init-lock-path", type=Path, help="Shared lock for sequential model loading across separate GPUs") parser.add_argument("--dry-run", action="store_true") parser.add_argument("--prepare-only", action="store_true", help="Write/validate evaluation manifest, then exit without generation or scoring") return parser.parse_args() def main() -> int: args = parse_args() run, output = args.run.resolve(), args.output.resolve() prompts_file = args.prompts.resolve() ltx_root, javis_root = args.ltx_root.resolve(), args.javis_root.resolve() # Keep the venv/pixi executable path: resolving its symlink escapes the # isolated environment and silently imports packages from the base Python. ltx_python, javis_python = args.ltx_python.absolute(), args.javis_python.absolute() if args.step_offset < 0 or args.batch_size < 1 or args.limit < 1 or args.num_workers < 0: raise ValueError("step-offset must be >= 0; batch-size and limit must be positive") points = checkpoints(run) if args.steps is not None: missing = set(args.steps) - {step for step, _ in points} if missing: raise ValueError(f"Requested checkpoint steps are not complete: {sorted(missing)}") points = [(step, path) for step, path in points if step in set(args.steps)] if not points: raise RuntimeError(f"No complete FSDP checkpoints in {run / 'checkpoints'}") required = ["avh_score", "javis_score"] if args.javis_only else SCORES metrics = ["av-score"] if args.javis_only else METRICS prompts = [p.strip() for p in prompts_file.read_text(encoding="utf-8").splitlines() if p.strip()] if len(prompts) < args.limit: raise ValueError(f"Expected {args.limit} prompts, found {len(prompts)}") prompts = prompts[:args.limit] config = run / "config.yaml" if not config.is_file(): raise FileNotFoundError(config) infer = ltx_root / "packages/ltx-distillation/src/ltx_distillation/tools/run_av_inference_eval.py" bench = javis_root / "eval/javisbench/main.py" for dependency in (infer, bench, ltx_python, javis_python): if not dependency.is_file(): raise FileNotFoundError(dependency) cfg = load_config(config) if str(cfg.get("dmd_style", "")).lower() not in {"rcm", "rcm_trig", "trig"}: raise RuntimeError(f"Expected trig parameterization for this curve: {config}") for key in ("checkpoint_path", "gemma_path"): if not Path(cfg[key]).exists(): raise FileNotFoundError(f"{key}: {cfg[key]}") mapping = Path(cfg["data_path"]) if not mapping.is_file(): raise FileNotFoundError(f"Cannot check training prompt overlap: {mapping}") overlap = training_overlap(mapping, prompts) if overlap: print(f"WARNING: {len(overlap)} evaluation prompts overlap training mapping {mapping}", flush=True) check_manifest(output, { "run": str(run), "config_inference_sha256": hashlib.sha256(json.dumps({ key: cfg.get(key) for key in ( "checkpoint_path", "gemma_path", "dmd_style", "backward_trig_timesteps", "denoising_step_list", "num_frames", "video_height", "video_width", "benchmark_video_fps", "benchmark_audio_sample_rate", "mixed_precision", ) }, sort_keys=True).encode("utf-8")).hexdigest(), "prompt_sha256": hashlib.sha256("\n".join(prompts).encode("utf-8")).hexdigest(), "num_prompts": args.limit, "train_overlap_count": len(overlap), "train_mapping": str(mapping), "seed": args.seed, "step_offset": args.step_offset, "batch_size": args.batch_size, "ltx_infer": str(infer), "javis_root": str(javis_root), "metrics": metrics, "mode": "javis_only" if args.javis_only else "full", "selected_steps": [step for step, _ in points], }, args.dry_run) if args.prepare_only: return 0 media_root, scores = output / "media", output / "scores" env = unset_proxy_env() env["CUDA_VISIBLE_DEVICES"] = args.gpu env["PYTHONUNBUFFERED"] = "1" for step, checkpoint in points: name = f"checkpoint_{step:06d}" media = media_root / name score_file = scores / f"{name}.json" if score_complete(score_file, required) and media_complete(media, args.limit, prompts, args.seed): print(f"skip scored {name}", flush=True) continue generate = [ str(ltx_python), str(infer), "--config_path", str(config), "--prompts_file", str(prompts_file), "--output_dir", str(media), "--model_kind", "student", "--student_checkpoint", str(checkpoint), "--student_param", "rcm_trig", "--num_prompts", str(args.limit), "--num_seeds", "1", "--seed", str(args.seed), ] if args.init_lock_path: generate.extend(["--init_lock_path", str(args.init_lock_path.absolute())]) evaluate = [ str(javis_python), "-m", "eval.javisbench.main", "--input_file", str(output / "inputs" / f"{name}.csv"), "--output_file", str(score_file), "--metrics", *metrics, "--max_audio_len_s", "4.0", "--num_workers", str(args.num_workers), ] if args.dry_run: print(f"{name}: generate={not media_complete(media, args.limit, prompts, args.seed)} score=True", flush=True) continue if not media_complete(media, args.limit, prompts, args.seed): media.mkdir(parents=True, exist_ok=True) subprocess.run(generate, cwd=ltx_root, env=env, check=True) if not media_complete(media, args.limit, prompts, args.seed): raise RuntimeError(f"Incomplete media: {media}") write_input(output / "inputs" / f"{name}.csv", prompts, media) scores.mkdir(parents=True, exist_ok=True) subprocess.run(evaluate, cwd=javis_root, env=env, check=True) if not score_complete(score_file, required): raise RuntimeError(f"JavisBench did not return all curve metrics: {score_file}") write_summary(output / "curve.csv", points, scores, run, args.step_offset, args.batch_size, required) if not args.dry_run: write_summary(output / "curve.csv", points, scores, run, args.step_offset, args.batch_size, required) return 0 if __name__ == "__main__": sys.exit(main())