turbot2va-ltx2 / code /evaluation /scripts /run_diversity_probe.py
luyu1021's picture
Add documented LTX-2 training source, recipes and checkpoint release plan
76b7b25 verified
Raw History Blame Contribute Delete
12.2 kB
#!/usr/bin/env python3
"""Run a resumable published-model 200 x 8 diversity experiment."""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import subprocess
import sys
from pathlib import Path
from common import load_config, unset_proxy_env, write_json
from run_checkpoint_curve import check_manifest, training_overlap
SAMPLE_NAME = re.compile(r"sample_(\d{4})_seed(\d{4})\.(mp4|wav|json)")
def inspect_existing_media(media: Path, prompts: list[str], seed: int, num_seeds: int) -> int:
"""Validate all existing outputs before adopting a resumable media directory."""
if not media.is_dir():
return 0
existing: dict[tuple[int, int], set[str]] = {}
for path in media.iterdir():
match = SAMPLE_NAME.fullmatch(path.name)
if match is None:
if path.name.startswith("sample_"):
raise ValueError(f"Unexpected sample filename: {path}")
continue # Existing shard prompt lists and model-init lock are not media.
index, seed_idx, suffix = int(match[1]), int(match[2]), match[3]
if index >= len(prompts) or seed_idx >= num_seeds or not path.is_file():
raise ValueError(f"Unexpected prompt/seed media: {path}")
existing.setdefault((index, seed_idx), set()).add(suffix)
if not path.stat().st_size:
raise ValueError(f"Empty media file: {path}")
for (index, seed_idx), suffixes in existing.items():
# A lone MP4 or WAV may be the result of an interrupted generation.
# Both files without a sidecar cannot be provenance-checked.
if ("json" in suffixes and suffixes != {"mp4", "wav", "json"}) or (
{"mp4", "wav"} <= suffixes and "json" not in suffixes
):
raise ValueError(f"Incomplete provenance for sample_{index:04d}_seed{seed_idx:04d}")
if "json" not in suffixes:
continue
sidecar = media / f"sample_{index:04d}_seed{seed_idx:04d}.json"
record = json.loads(sidecar.read_text(encoding="utf-8"))
if (record.get("index") != index or record.get("seed_idx") != seed_idx
or record.get("seed") != seed + index * num_seeds + seed_idx
or record.get("prompt") != prompts[index]):
raise ValueError(f"Generated prompt/seed mismatch: {sidecar}")
return sum(suffixes == {"mp4", "wav", "json"} for suffixes in existing.values())
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--checkpoint", type=Path, required=True)
parser.add_argument("--checkpoint-info", type=Path, required=True)
parser.add_argument("--run-config", type=Path, required=True)
parser.add_argument("--prompts", type=Path, required=True)
parser.add_argument("--train-mapping", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--javis-root", type=Path, required=True)
parser.add_argument("--javis-python", type=Path, required=True)
parser.add_argument("--ltx-root", type=Path, required=True)
parser.add_argument("--ltx-python", type=Path, required=True)
parser.add_argument("--gpu", default="0")
parser.add_argument("--seed", type=int, default=12345)
parser.add_argument("--limit", type=int, default=200)
parser.add_argument("--num-seeds", type=int, default=8)
parser.add_argument("--batch-size", type=int, default=4)
parser.add_argument("--num-workers", type=int, default=4)
parser.add_argument("--score-existing-only", action="store_true",
help="score a complete media directory without starting generation")
parser.add_argument("--cpu", action="store_true",
help="CPU scoring only; requires --score-existing-only, never generates")
parser.add_argument("--prepare-only", action="store_true",
help="validate/adopt existing media and write manifest; never generate or score")
parser.add_argument("--dry-run", action="store_true")
return parser.parse_args()
def run(args: argparse.Namespace) -> int:
if args.limit < 1 or args.num_seeds < 2 or args.batch_size < 1 or args.num_workers < 0:
raise ValueError("--limit must be positive and --num-seeds at least 2")
if args.cpu and not args.score_existing_only and not getattr(args, "prepare_only", False):
raise ValueError("--cpu requires --score-existing-only; no CPU generation is supported")
checkpoint, info_file = args.checkpoint.resolve(), args.checkpoint_info.resolve()
prompts_file, config_file = args.prompts.resolve(), args.run_config.resolve()
mapping = args.train_mapping.resolve()
output, javis_root, ltx_root = args.output.resolve(), args.javis_root.resolve(), args.ltx_root.resolve()
javis_python, ltx_python = args.javis_python.absolute(), args.ltx_python.absolute()
infer = ltx_root / "packages/ltx-distillation/src/ltx_distillation/tools/run_av_inference_eval.py"
scorer = Path(__file__).absolute().with_name("compute_diversity_imagebind_pairwise.py")
for path in (checkpoint, info_file, prompts_file, config_file, mapping, infer,
scorer, javis_python, ltx_python):
if not path.is_file():
raise FileNotFoundError(path)
info = json.loads(info_file.read_text(encoding="utf-8"))
if (checkpoint.stat().st_size != info["model_bytes"]
or info["format"] != "pytorch_state_dict"):
raise ValueError(f"Published checkpoint metadata mismatch: {info_file}")
cfg = load_config(config_file)
for key in ("checkpoint_path", "gemma_path"):
if not Path(cfg[key]).exists():
raise FileNotFoundError(f"{key}: {cfg[key]}")
prompts = [p.strip() for p in prompts_file.read_text(encoding="utf-8").splitlines() if p.strip()]
if len(prompts) < args.limit:
raise ValueError(f"Expected {args.limit} prompts, found {len(prompts)}")
prompts = prompts[:args.limit]
overlap = training_overlap(mapping, prompts)
if not (javis_root / "checkpoints/imagebind_huge.pth").is_file():
raise FileNotFoundError(javis_root / "checkpoints/imagebind_huge.pth")
if overlap:
print(f"WARNING: {len(overlap)} prompts overlap {mapping}", flush=True)
model_name = "published_turbot2av_main"
media = output / "media" / model_name
metric_root = output / "metrics"
# Use this experiment's paths only; never rewrite the prior machine's config.
from compute_diversity_imagebind_pairwise import embed_path, sample_paths
eval_config = {
"paths": {"javis_root": str(javis_root), "diversity_metric_root": str(metric_root)},
"prompt_subset": {"num_prompts": args.limit, "num_seeds": args.num_seeds},
}
model_cfg = {"name": model_name, "media_dir": str(media)}
manifest = {
"model_id": "luyu1021/TurboT2AV", "checkpoint": str(checkpoint),
"checkpoint_sha256": info["model_sha256"],
"config_sha256": hashlib.sha256(config_file.read_bytes()).hexdigest(),
"prompts_sha256": hashlib.sha256("\n".join(prompts).encode("utf-8")).hexdigest(),
"num_prompts": args.limit, "num_seeds": args.num_seeds, "seed": args.seed,
"seed_rule": "seed + prompt_index * num_seeds + seed_index",
"train_overlap_count": len(overlap), "train_mapping": str(mapping),
"imagebind": str(javis_root / "checkpoints/imagebind_huge.pth"),
"javis_root": str(javis_root),
"ltx_infer": str(infer),
"ltx_python": str(ltx_python),
"javis_python": str(javis_python),
}
if args.prepare_only:
if output.exists():
extras = {p.name for p in output.iterdir()} - {"media", "evaluation_manifest.json"}
if extras and not (output / "evaluation_manifest.json").is_file():
raise RuntimeError(f"Cannot adopt output with unknown files: {sorted(extras)}")
if (output / "media").exists():
media_extras = {p.name for p in (output / "media").iterdir()} - {model_name}
if media_extras and not (output / "evaluation_manifest.json").is_file():
raise RuntimeError(f"Cannot adopt unrelated media: {sorted(media_extras)}")
count = inspect_existing_media(media, prompts, args.seed, args.num_seeds)
if (output / "evaluation_manifest.json").is_file():
check_manifest(output, manifest, True)
elif not args.dry_run:
write_json(output / "evaluation_manifest.json", manifest)
print(f"validated diversity media: {count}/{args.limit * args.num_seeds} "
f"complete prompt/seed pairs; missing={args.limit * args.num_seeds - count}; "
f"manifest={'dry-run' if args.dry_run else output / 'evaluation_manifest.json'}")
return 0
check_manifest(output, manifest, args.dry_run)
generate = [
str(ltx_python), str(infer), "--config_path", str(config_file),
"--prompts_file", str(prompts_file), "--output_dir", str(media),
"--model_kind", "student", "--student_checkpoint", str(checkpoint),
"--student_param", "rcm_trig", "--num_prompts", str(args.limit),
"--num_seeds", str(args.num_seeds), "--seed", str(args.seed),
]
embedding = embed_path(eval_config, model_cfg)
try:
sample_paths(eval_config, model_cfg)
complete = True
except FileNotFoundError:
complete = False
if args.dry_run:
print(f"diversity: {args.limit} prompts x {args.num_seeds} seeds; "
f"{args.limit * args.num_seeds * (args.num_seeds - 1) // 2} within-prompt pairs per modality")
print(f"generate={not complete}, embed={not embedding.is_file()}")
print(f"generate: {generate}")
return 0
if args.score_existing_only and not complete:
raise RuntimeError(f"--score-existing-only requires all {args.limit * args.num_seeds} pairs in {media}")
env = unset_proxy_env()
env["CUDA_VISIBLE_DEVICES"] = "" if args.cpu else args.gpu
env["PYTHONUNBUFFERED"] = "1"
if not complete:
media.mkdir(parents=True, exist_ok=True)
subprocess.run(generate, cwd=ltx_root, env=env, check=True)
sample_paths(eval_config, model_cfg)
# The caller may be a pixi Python or another venv; the scoring imports
# must always execute through the separate JavisBench environment.
scoring = [
str(javis_python), str(scorer), "--config",
str(output / "diversity_config.json"), *(["--cpu"] if args.cpu else ["--gpu", str(args.gpu)]),
"--only", model_name, "--batch-size", str(args.batch_size),
"--num-workers", str(args.num_workers),
]
write_json(output / "diversity_config.json", {
**eval_config, "models": [model_cfg],
})
subprocess.run(scoring, cwd=javis_root, env=env, check=True)
summary_path = metric_root / "summary.json"
if not summary_path.is_file():
raise RuntimeError(f"Missing diversity scores: {summary_path}")
rows = json.loads(summary_path.read_text(encoding="utf-8"))["rows"]
if len(rows) != 1 or rows[0]["model_name"] != model_name:
raise RuntimeError(f"Unexpected diversity scores: {summary_path}")
scores = rows[0]
if args.limit == 200 and args.num_seeds == 8:
reference = {
"video_pairwise_cosine_distance": 0.2322,
"audio_pairwise_cosine_distance": 0.4197,
"va_average_pairwise_cosine_distance": 0.3259,
}
write_json(output / "diversity_paper_comparison.json", {
"model_identity": model_name,
"note": "Numerical comparison only; exact author prompt/enhancement and ImageBind distance protocols unverified",
"metrics": {
key: {"observed": scores[key], "paper": value, "delta": scores[key] - value}
for key, value in reference.items()
},
})
print(json.dumps({
k: scores[k] for k in (
"video_pairwise_cosine_distance", "audio_pairwise_cosine_distance",
"va_average_pairwise_cosine_distance",
)
}, indent=2))
return 0
if __name__ == "__main__":
sys.exit(run(parse_args()))