turbot2va-ltx2 / code /evaluation /scripts /run_checkpoint_curve.py
luyu1021's picture
Add documented LTX-2 training source, recipes and checkpoint release plan
76b7b25 verified
Raw History Blame Contribute Delete
13 kB
#!/usr/bin/env python3
"""Generate and score a run's FSDP checkpoints, resuming completed points."""
from __future__ import annotations
import argparse
import csv
import hashlib
import json
import math
import re
import subprocess
import sys
from pathlib import Path
from common import load_config, unset_proxy_env
METRICS = ["video-quality", "audio-quality", "imagebind-score", "cxxp-score", "av-score", "desync"]
SCORES = [
"javis_score", "visual_quality", "motion_quality", "audio_quality",
"ib_tv", "ib_ta", "clip_score", "clap_score", "ib_av", "cavp_score",
"avh_score", "desync",
]
COLUMNS = ["run_name", "step", "step_offset", "samples_seen", *SCORES]
def checkpoints(run: Path) -> list[tuple[int, Path]]:
found = []
for path in (run / "checkpoints").glob("checkpoint_*"):
match = re.fullmatch(r"checkpoint_(\d+)", path.name)
if match and (path / "sharded").is_dir() and (path / "metadata.pth").is_file():
found.append((int(match.group(1)), path))
return sorted(found)
def media_complete(directory: Path, count: int, prompts: list[str] | None = None,
seed: int = 12345) -> bool:
for index in range(count):
for suffix in ("mp4", "wav"):
path = directory / f"sample_{index:04d}.{suffix}"
if not path.is_file() or path.stat().st_size == 0:
return False
if prompts is not None:
sidecar = directory / f"sample_{index:04d}.json"
try:
record = json.loads(sidecar.read_text(encoding="utf-8"))
except (OSError, ValueError):
return False
if (record.get("prompt") != prompts[index] or record.get("index") != index
or record.get("seed") != seed + index or record.get("seed_idx") != 0):
return False
return True
def score_complete(path: Path, required: list[str] | None = None) -> bool:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
return all(
isinstance(payload[key], (int, float))
and not isinstance(payload[key], bool)
and math.isfinite(payload[key])
for key in (required or SCORES)
)
except (OSError, ValueError, KeyError, TypeError, OverflowError):
return False
def write_input(path: Path, prompts: list[str], media: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", newline="", encoding="utf-8") as handle:
writer = csv.DictWriter(handle, fieldnames=[
"video_id", "text", "path", "audio_path", "pred_video_path", "pred_audio_path"
])
writer.writeheader()
for index, prompt in enumerate(prompts):
video = media / f"sample_{index:04d}.mp4"
audio = media / f"sample_{index:04d}.wav"
writer.writerow({
"video_id": f"sample_{index:04d}", "text": prompt,
"path": video, "audio_path": audio,
"pred_video_path": video, "pred_audio_path": audio,
})
def write_summary(path: Path, points: list[tuple[int, Path]], scores: Path, run: Path,
offset: int, batch_size: int, required: list[str] | None = None) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", newline="", encoding="utf-8") as handle:
writer = csv.DictWriter(handle, fieldnames=COLUMNS)
writer.writeheader()
for step, _ in points:
score_file = scores / f"checkpoint_{step:06d}.json"
if not score_complete(score_file, required):
continue
payload = json.loads(score_file.read_text(encoding="utf-8"))
writer.writerow({
"run_name": run.name, "step": step, "step_offset": offset,
"samples_seen": (offset + step) * batch_size,
**{key: payload.get(key, "") for key in SCORES},
})
def training_overlap(mapping: Path, prompts: list[str]) -> set[str]:
wanted = {prompt.strip().casefold() for prompt in prompts}
matches = set()
with mapping.open(newline="", encoding="utf-8") as handle:
reader = csv.DictReader(handle)
if not reader.fieldnames or "prompt" not in reader.fieldnames:
raise ValueError(f"Training mapping has no prompt column: {mapping}")
for row in reader:
prompt = row["prompt"].strip().casefold()
if prompt in wanted:
matches.add(prompt)
return matches
def check_manifest(output: Path, settings: dict, dry_run: bool) -> None:
manifest = output / "evaluation_manifest.json"
if manifest.is_file():
if json.loads(manifest.read_text(encoding="utf-8")) != settings:
raise RuntimeError(f"Evaluation settings changed; choose a new output directory: {output}")
elif output.exists() and any(output.iterdir()):
raise RuntimeError(f"Output has files without an evaluation manifest: {output}")
elif not dry_run:
output.mkdir(parents=True, exist_ok=True)
manifest.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--run", type=Path, required=True, help="run dir with config.yaml and checkpoints/")
parser.add_argument("--step-offset", type=int, default=0,
help="prior stages' local steps; required for rCM curves")
parser.add_argument("--batch-size", type=int, default=16)
parser.add_argument("--prompts", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--javis-root", type=Path, required=True)
parser.add_argument("--javis-python", type=Path, required=True)
parser.add_argument("--ltx-root", type=Path, required=True)
parser.add_argument("--ltx-python", type=Path, required=True)
parser.add_argument("--gpu", default="0")
parser.add_argument("--seed", type=int, default=12345)
parser.add_argument("--limit", type=int, default=200)
parser.add_argument("--steps", type=int, nargs="+",
help="Only run these checkpoint steps; useful for per-GPU scheduling")
parser.add_argument("--javis-only", action="store_true",
help="Only av-score (Javis and AVH); skip slower auxiliary metrics")
parser.add_argument("--num-workers", type=int, default=4)
parser.add_argument("--init-lock-path", type=Path,
help="Shared lock for sequential model loading across separate GPUs")
parser.add_argument("--dry-run", action="store_true")
parser.add_argument("--prepare-only", action="store_true",
help="Write/validate evaluation manifest, then exit without generation or scoring")
return parser.parse_args()
def main() -> int:
args = parse_args()
run, output = args.run.resolve(), args.output.resolve()
prompts_file = args.prompts.resolve()
ltx_root, javis_root = args.ltx_root.resolve(), args.javis_root.resolve()
# Keep the venv/pixi executable path: resolving its symlink escapes the
# isolated environment and silently imports packages from the base Python.
ltx_python, javis_python = args.ltx_python.absolute(), args.javis_python.absolute()
if args.step_offset < 0 or args.batch_size < 1 or args.limit < 1 or args.num_workers < 0:
raise ValueError("step-offset must be >= 0; batch-size and limit must be positive")
points = checkpoints(run)
if args.steps is not None:
missing = set(args.steps) - {step for step, _ in points}
if missing:
raise ValueError(f"Requested checkpoint steps are not complete: {sorted(missing)}")
points = [(step, path) for step, path in points if step in set(args.steps)]
if not points:
raise RuntimeError(f"No complete FSDP checkpoints in {run / 'checkpoints'}")
required = ["avh_score", "javis_score"] if args.javis_only else SCORES
metrics = ["av-score"] if args.javis_only else METRICS
prompts = [p.strip() for p in prompts_file.read_text(encoding="utf-8").splitlines() if p.strip()]
if len(prompts) < args.limit:
raise ValueError(f"Expected {args.limit} prompts, found {len(prompts)}")
prompts = prompts[:args.limit]
config = run / "config.yaml"
if not config.is_file():
raise FileNotFoundError(config)
infer = ltx_root / "packages/ltx-distillation/src/ltx_distillation/tools/run_av_inference_eval.py"
bench = javis_root / "eval/javisbench/main.py"
for dependency in (infer, bench, ltx_python, javis_python):
if not dependency.is_file():
raise FileNotFoundError(dependency)
cfg = load_config(config)
if str(cfg.get("dmd_style", "")).lower() not in {"rcm", "rcm_trig", "trig"}:
raise RuntimeError(f"Expected trig parameterization for this curve: {config}")
for key in ("checkpoint_path", "gemma_path"):
if not Path(cfg[key]).exists():
raise FileNotFoundError(f"{key}: {cfg[key]}")
mapping = Path(cfg["data_path"])
if not mapping.is_file():
raise FileNotFoundError(f"Cannot check training prompt overlap: {mapping}")
overlap = training_overlap(mapping, prompts)
if overlap:
print(f"WARNING: {len(overlap)} evaluation prompts overlap training mapping {mapping}", flush=True)
check_manifest(output, {
"run": str(run),
"config_inference_sha256": hashlib.sha256(json.dumps({
key: cfg.get(key) for key in (
"checkpoint_path", "gemma_path", "dmd_style", "backward_trig_timesteps",
"denoising_step_list", "num_frames", "video_height", "video_width",
"benchmark_video_fps", "benchmark_audio_sample_rate", "mixed_precision",
)
}, sort_keys=True).encode("utf-8")).hexdigest(),
"prompt_sha256": hashlib.sha256("\n".join(prompts).encode("utf-8")).hexdigest(),
"num_prompts": args.limit,
"train_overlap_count": len(overlap),
"train_mapping": str(mapping),
"seed": args.seed,
"step_offset": args.step_offset,
"batch_size": args.batch_size,
"ltx_infer": str(infer),
"javis_root": str(javis_root),
"metrics": metrics,
"mode": "javis_only" if args.javis_only else "full",
"selected_steps": [step for step, _ in points],
}, args.dry_run)
if args.prepare_only:
return 0
media_root, scores = output / "media", output / "scores"
env = unset_proxy_env()
env["CUDA_VISIBLE_DEVICES"] = args.gpu
env["PYTHONUNBUFFERED"] = "1"
for step, checkpoint in points:
name = f"checkpoint_{step:06d}"
media = media_root / name
score_file = scores / f"{name}.json"
if score_complete(score_file, required) and media_complete(media, args.limit, prompts, args.seed):
print(f"skip scored {name}", flush=True)
continue
generate = [
str(ltx_python), str(infer), "--config_path", str(config),
"--prompts_file", str(prompts_file), "--output_dir", str(media),
"--model_kind", "student", "--student_checkpoint", str(checkpoint),
"--student_param", "rcm_trig",
"--num_prompts", str(args.limit), "--num_seeds", "1", "--seed", str(args.seed),
]
if args.init_lock_path:
generate.extend(["--init_lock_path", str(args.init_lock_path.absolute())])
evaluate = [
str(javis_python), "-m", "eval.javisbench.main",
"--input_file", str(output / "inputs" / f"{name}.csv"),
"--output_file", str(score_file), "--metrics", *metrics,
"--max_audio_len_s", "4.0", "--num_workers", str(args.num_workers),
]
if args.dry_run:
print(f"{name}: generate={not media_complete(media, args.limit, prompts, args.seed)} score=True", flush=True)
continue
if not media_complete(media, args.limit, prompts, args.seed):
media.mkdir(parents=True, exist_ok=True)
subprocess.run(generate, cwd=ltx_root, env=env, check=True)
if not media_complete(media, args.limit, prompts, args.seed):
raise RuntimeError(f"Incomplete media: {media}")
write_input(output / "inputs" / f"{name}.csv", prompts, media)
scores.mkdir(parents=True, exist_ok=True)
subprocess.run(evaluate, cwd=javis_root, env=env, check=True)
if not score_complete(score_file, required):
raise RuntimeError(f"JavisBench did not return all curve metrics: {score_file}")
write_summary(output / "curve.csv", points, scores, run, args.step_offset, args.batch_size, required)
if not args.dry_run:
write_summary(output / "curve.csv", points, scores, run, args.step_offset, args.batch_size, required)
return 0
if __name__ == "__main__":
sys.exit(main())