Download code/evaluation/scripts/run_checkpoint_curve.py from luyu1021/turbot2va-ltx2: direct link, hf CLI and curl.
- Browser
- Download file 13 kB
-
https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/evaluation/scripts/run_checkpoint_curve.py
- Command line
-
hf download hf://luyu1021/turbot2va-ltx2/code/evaluation/scripts/run_checkpoint_curve.py
-
curl -L -o run_checkpoint_curve.py https://huggingface.co/luyu1021/turbot2va-ltx2/resolve/main/code/evaluation/scripts/run_checkpoint_curve.py
13 kB
| #!/usr/bin/env python3 | |
| """Generate and score a run's FSDP checkpoints, resuming completed points.""" | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import hashlib | |
| import json | |
| import math | |
| import re | |
| import subprocess | |
| import sys | |
| from pathlib import Path | |
| from common import load_config, unset_proxy_env | |
| METRICS = ["video-quality", "audio-quality", "imagebind-score", "cxxp-score", "av-score", "desync"] | |
| SCORES = [ | |
| "javis_score", "visual_quality", "motion_quality", "audio_quality", | |
| "ib_tv", "ib_ta", "clip_score", "clap_score", "ib_av", "cavp_score", | |
| "avh_score", "desync", | |
| ] | |
| COLUMNS = ["run_name", "step", "step_offset", "samples_seen", *SCORES] | |
| def checkpoints(run: Path) -> list[tuple[int, Path]]: | |
| found = [] | |
| for path in (run / "checkpoints").glob("checkpoint_*"): | |
| match = re.fullmatch(r"checkpoint_(\d+)", path.name) | |
| if match and (path / "sharded").is_dir() and (path / "metadata.pth").is_file(): | |
| found.append((int(match.group(1)), path)) | |
| return sorted(found) | |
| def media_complete(directory: Path, count: int, prompts: list[str] | None = None, | |
| seed: int = 12345) -> bool: | |
| for index in range(count): | |
| for suffix in ("mp4", "wav"): | |
| path = directory / f"sample_{index:04d}.{suffix}" | |
| if not path.is_file() or path.stat().st_size == 0: | |
| return False | |
| if prompts is not None: | |
| sidecar = directory / f"sample_{index:04d}.json" | |
| try: | |
| record = json.loads(sidecar.read_text(encoding="utf-8")) | |
| except (OSError, ValueError): | |
| return False | |
| if (record.get("prompt") != prompts[index] or record.get("index") != index | |
| or record.get("seed") != seed + index or record.get("seed_idx") != 0): | |
| return False | |
| return True | |
| def score_complete(path: Path, required: list[str] | None = None) -> bool: | |
| try: | |
| payload = json.loads(path.read_text(encoding="utf-8")) | |
| return all( | |
| isinstance(payload[key], (int, float)) | |
| and not isinstance(payload[key], bool) | |
| and math.isfinite(payload[key]) | |
| for key in (required or SCORES) | |
| ) | |
| except (OSError, ValueError, KeyError, TypeError, OverflowError): | |
| return False | |
| def write_input(path: Path, prompts: list[str], media: Path) -> None: | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| with path.open("w", newline="", encoding="utf-8") as handle: | |
| writer = csv.DictWriter(handle, fieldnames=[ | |
| "video_id", "text", "path", "audio_path", "pred_video_path", "pred_audio_path" | |
| ]) | |
| writer.writeheader() | |
| for index, prompt in enumerate(prompts): | |
| video = media / f"sample_{index:04d}.mp4" | |
| audio = media / f"sample_{index:04d}.wav" | |
| writer.writerow({ | |
| "video_id": f"sample_{index:04d}", "text": prompt, | |
| "path": video, "audio_path": audio, | |
| "pred_video_path": video, "pred_audio_path": audio, | |
| }) | |
| def write_summary(path: Path, points: list[tuple[int, Path]], scores: Path, run: Path, | |
| offset: int, batch_size: int, required: list[str] | None = None) -> None: | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| with path.open("w", newline="", encoding="utf-8") as handle: | |
| writer = csv.DictWriter(handle, fieldnames=COLUMNS) | |
| writer.writeheader() | |
| for step, _ in points: | |
| score_file = scores / f"checkpoint_{step:06d}.json" | |
| if not score_complete(score_file, required): | |
| continue | |
| payload = json.loads(score_file.read_text(encoding="utf-8")) | |
| writer.writerow({ | |
| "run_name": run.name, "step": step, "step_offset": offset, | |
| "samples_seen": (offset + step) * batch_size, | |
| **{key: payload.get(key, "") for key in SCORES}, | |
| }) | |
| def training_overlap(mapping: Path, prompts: list[str]) -> set[str]: | |
| wanted = {prompt.strip().casefold() for prompt in prompts} | |
| matches = set() | |
| with mapping.open(newline="", encoding="utf-8") as handle: | |
| reader = csv.DictReader(handle) | |
| if not reader.fieldnames or "prompt" not in reader.fieldnames: | |
| raise ValueError(f"Training mapping has no prompt column: {mapping}") | |
| for row in reader: | |
| prompt = row["prompt"].strip().casefold() | |
| if prompt in wanted: | |
| matches.add(prompt) | |
| return matches | |
| def check_manifest(output: Path, settings: dict, dry_run: bool) -> None: | |
| manifest = output / "evaluation_manifest.json" | |
| if manifest.is_file(): | |
| if json.loads(manifest.read_text(encoding="utf-8")) != settings: | |
| raise RuntimeError(f"Evaluation settings changed; choose a new output directory: {output}") | |
| elif output.exists() and any(output.iterdir()): | |
| raise RuntimeError(f"Output has files without an evaluation manifest: {output}") | |
| elif not dry_run: | |
| output.mkdir(parents=True, exist_ok=True) | |
| manifest.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8") | |
| def parse_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--run", type=Path, required=True, help="run dir with config.yaml and checkpoints/") | |
| parser.add_argument("--step-offset", type=int, default=0, | |
| help="prior stages' local steps; required for rCM curves") | |
| parser.add_argument("--batch-size", type=int, default=16) | |
| parser.add_argument("--prompts", type=Path, required=True) | |
| parser.add_argument("--output", type=Path, required=True) | |
| parser.add_argument("--javis-root", type=Path, required=True) | |
| parser.add_argument("--javis-python", type=Path, required=True) | |
| parser.add_argument("--ltx-root", type=Path, required=True) | |
| parser.add_argument("--ltx-python", type=Path, required=True) | |
| parser.add_argument("--gpu", default="0") | |
| parser.add_argument("--seed", type=int, default=12345) | |
| parser.add_argument("--limit", type=int, default=200) | |
| parser.add_argument("--steps", type=int, nargs="+", | |
| help="Only run these checkpoint steps; useful for per-GPU scheduling") | |
| parser.add_argument("--javis-only", action="store_true", | |
| help="Only av-score (Javis and AVH); skip slower auxiliary metrics") | |
| parser.add_argument("--num-workers", type=int, default=4) | |
| parser.add_argument("--init-lock-path", type=Path, | |
| help="Shared lock for sequential model loading across separate GPUs") | |
| parser.add_argument("--dry-run", action="store_true") | |
| parser.add_argument("--prepare-only", action="store_true", | |
| help="Write/validate evaluation manifest, then exit without generation or scoring") | |
| return parser.parse_args() | |
| def main() -> int: | |
| args = parse_args() | |
| run, output = args.run.resolve(), args.output.resolve() | |
| prompts_file = args.prompts.resolve() | |
| ltx_root, javis_root = args.ltx_root.resolve(), args.javis_root.resolve() | |
| # Keep the venv/pixi executable path: resolving its symlink escapes the | |
| # isolated environment and silently imports packages from the base Python. | |
| ltx_python, javis_python = args.ltx_python.absolute(), args.javis_python.absolute() | |
| if args.step_offset < 0 or args.batch_size < 1 or args.limit < 1 or args.num_workers < 0: | |
| raise ValueError("step-offset must be >= 0; batch-size and limit must be positive") | |
| points = checkpoints(run) | |
| if args.steps is not None: | |
| missing = set(args.steps) - {step for step, _ in points} | |
| if missing: | |
| raise ValueError(f"Requested checkpoint steps are not complete: {sorted(missing)}") | |
| points = [(step, path) for step, path in points if step in set(args.steps)] | |
| if not points: | |
| raise RuntimeError(f"No complete FSDP checkpoints in {run / 'checkpoints'}") | |
| required = ["avh_score", "javis_score"] if args.javis_only else SCORES | |
| metrics = ["av-score"] if args.javis_only else METRICS | |
| prompts = [p.strip() for p in prompts_file.read_text(encoding="utf-8").splitlines() if p.strip()] | |
| if len(prompts) < args.limit: | |
| raise ValueError(f"Expected {args.limit} prompts, found {len(prompts)}") | |
| prompts = prompts[:args.limit] | |
| config = run / "config.yaml" | |
| if not config.is_file(): | |
| raise FileNotFoundError(config) | |
| infer = ltx_root / "packages/ltx-distillation/src/ltx_distillation/tools/run_av_inference_eval.py" | |
| bench = javis_root / "eval/javisbench/main.py" | |
| for dependency in (infer, bench, ltx_python, javis_python): | |
| if not dependency.is_file(): | |
| raise FileNotFoundError(dependency) | |
| cfg = load_config(config) | |
| if str(cfg.get("dmd_style", "")).lower() not in {"rcm", "rcm_trig", "trig"}: | |
| raise RuntimeError(f"Expected trig parameterization for this curve: {config}") | |
| for key in ("checkpoint_path", "gemma_path"): | |
| if not Path(cfg[key]).exists(): | |
| raise FileNotFoundError(f"{key}: {cfg[key]}") | |
| mapping = Path(cfg["data_path"]) | |
| if not mapping.is_file(): | |
| raise FileNotFoundError(f"Cannot check training prompt overlap: {mapping}") | |
| overlap = training_overlap(mapping, prompts) | |
| if overlap: | |
| print(f"WARNING: {len(overlap)} evaluation prompts overlap training mapping {mapping}", flush=True) | |
| check_manifest(output, { | |
| "run": str(run), | |
| "config_inference_sha256": hashlib.sha256(json.dumps({ | |
| key: cfg.get(key) for key in ( | |
| "checkpoint_path", "gemma_path", "dmd_style", "backward_trig_timesteps", | |
| "denoising_step_list", "num_frames", "video_height", "video_width", | |
| "benchmark_video_fps", "benchmark_audio_sample_rate", "mixed_precision", | |
| ) | |
| }, sort_keys=True).encode("utf-8")).hexdigest(), | |
| "prompt_sha256": hashlib.sha256("\n".join(prompts).encode("utf-8")).hexdigest(), | |
| "num_prompts": args.limit, | |
| "train_overlap_count": len(overlap), | |
| "train_mapping": str(mapping), | |
| "seed": args.seed, | |
| "step_offset": args.step_offset, | |
| "batch_size": args.batch_size, | |
| "ltx_infer": str(infer), | |
| "javis_root": str(javis_root), | |
| "metrics": metrics, | |
| "mode": "javis_only" if args.javis_only else "full", | |
| "selected_steps": [step for step, _ in points], | |
| }, args.dry_run) | |
| if args.prepare_only: | |
| return 0 | |
| media_root, scores = output / "media", output / "scores" | |
| env = unset_proxy_env() | |
| env["CUDA_VISIBLE_DEVICES"] = args.gpu | |
| env["PYTHONUNBUFFERED"] = "1" | |
| for step, checkpoint in points: | |
| name = f"checkpoint_{step:06d}" | |
| media = media_root / name | |
| score_file = scores / f"{name}.json" | |
| if score_complete(score_file, required) and media_complete(media, args.limit, prompts, args.seed): | |
| print(f"skip scored {name}", flush=True) | |
| continue | |
| generate = [ | |
| str(ltx_python), str(infer), "--config_path", str(config), | |
| "--prompts_file", str(prompts_file), "--output_dir", str(media), | |
| "--model_kind", "student", "--student_checkpoint", str(checkpoint), | |
| "--student_param", "rcm_trig", | |
| "--num_prompts", str(args.limit), "--num_seeds", "1", "--seed", str(args.seed), | |
| ] | |
| if args.init_lock_path: | |
| generate.extend(["--init_lock_path", str(args.init_lock_path.absolute())]) | |
| evaluate = [ | |
| str(javis_python), "-m", "eval.javisbench.main", | |
| "--input_file", str(output / "inputs" / f"{name}.csv"), | |
| "--output_file", str(score_file), "--metrics", *metrics, | |
| "--max_audio_len_s", "4.0", "--num_workers", str(args.num_workers), | |
| ] | |
| if args.dry_run: | |
| print(f"{name}: generate={not media_complete(media, args.limit, prompts, args.seed)} score=True", flush=True) | |
| continue | |
| if not media_complete(media, args.limit, prompts, args.seed): | |
| media.mkdir(parents=True, exist_ok=True) | |
| subprocess.run(generate, cwd=ltx_root, env=env, check=True) | |
| if not media_complete(media, args.limit, prompts, args.seed): | |
| raise RuntimeError(f"Incomplete media: {media}") | |
| write_input(output / "inputs" / f"{name}.csv", prompts, media) | |
| scores.mkdir(parents=True, exist_ok=True) | |
| subprocess.run(evaluate, cwd=javis_root, env=env, check=True) | |
| if not score_complete(score_file, required): | |
| raise RuntimeError(f"JavisBench did not return all curve metrics: {score_file}") | |
| write_summary(output / "curve.csv", points, scores, run, args.step_offset, args.batch_size, required) | |
| if not args.dry_run: | |
| write_summary(output / "curve.csv", points, scores, run, args.step_offset, args.batch_size, required) | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |