File size: 12,248 Bytes
76b7b25 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 | #!/usr/bin/env python3
"""Run a resumable published-model 200 x 8 diversity experiment."""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import subprocess
import sys
from pathlib import Path
from common import load_config, unset_proxy_env, write_json
from run_checkpoint_curve import check_manifest, training_overlap
SAMPLE_NAME = re.compile(r"sample_(\d{4})_seed(\d{4})\.(mp4|wav|json)")
def inspect_existing_media(media: Path, prompts: list[str], seed: int, num_seeds: int) -> int:
"""Validate all existing outputs before adopting a resumable media directory."""
if not media.is_dir():
return 0
existing: dict[tuple[int, int], set[str]] = {}
for path in media.iterdir():
match = SAMPLE_NAME.fullmatch(path.name)
if match is None:
if path.name.startswith("sample_"):
raise ValueError(f"Unexpected sample filename: {path}")
continue # Existing shard prompt lists and model-init lock are not media.
index, seed_idx, suffix = int(match[1]), int(match[2]), match[3]
if index >= len(prompts) or seed_idx >= num_seeds or not path.is_file():
raise ValueError(f"Unexpected prompt/seed media: {path}")
existing.setdefault((index, seed_idx), set()).add(suffix)
if not path.stat().st_size:
raise ValueError(f"Empty media file: {path}")
for (index, seed_idx), suffixes in existing.items():
# A lone MP4 or WAV may be the result of an interrupted generation.
# Both files without a sidecar cannot be provenance-checked.
if ("json" in suffixes and suffixes != {"mp4", "wav", "json"}) or (
{"mp4", "wav"} <= suffixes and "json" not in suffixes
):
raise ValueError(f"Incomplete provenance for sample_{index:04d}_seed{seed_idx:04d}")
if "json" not in suffixes:
continue
sidecar = media / f"sample_{index:04d}_seed{seed_idx:04d}.json"
record = json.loads(sidecar.read_text(encoding="utf-8"))
if (record.get("index") != index or record.get("seed_idx") != seed_idx
or record.get("seed") != seed + index * num_seeds + seed_idx
or record.get("prompt") != prompts[index]):
raise ValueError(f"Generated prompt/seed mismatch: {sidecar}")
return sum(suffixes == {"mp4", "wav", "json"} for suffixes in existing.values())
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--checkpoint", type=Path, required=True)
parser.add_argument("--checkpoint-info", type=Path, required=True)
parser.add_argument("--run-config", type=Path, required=True)
parser.add_argument("--prompts", type=Path, required=True)
parser.add_argument("--train-mapping", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--javis-root", type=Path, required=True)
parser.add_argument("--javis-python", type=Path, required=True)
parser.add_argument("--ltx-root", type=Path, required=True)
parser.add_argument("--ltx-python", type=Path, required=True)
parser.add_argument("--gpu", default="0")
parser.add_argument("--seed", type=int, default=12345)
parser.add_argument("--limit", type=int, default=200)
parser.add_argument("--num-seeds", type=int, default=8)
parser.add_argument("--batch-size", type=int, default=4)
parser.add_argument("--num-workers", type=int, default=4)
parser.add_argument("--score-existing-only", action="store_true",
help="score a complete media directory without starting generation")
parser.add_argument("--cpu", action="store_true",
help="CPU scoring only; requires --score-existing-only, never generates")
parser.add_argument("--prepare-only", action="store_true",
help="validate/adopt existing media and write manifest; never generate or score")
parser.add_argument("--dry-run", action="store_true")
return parser.parse_args()
def run(args: argparse.Namespace) -> int:
if args.limit < 1 or args.num_seeds < 2 or args.batch_size < 1 or args.num_workers < 0:
raise ValueError("--limit must be positive and --num-seeds at least 2")
if args.cpu and not args.score_existing_only and not getattr(args, "prepare_only", False):
raise ValueError("--cpu requires --score-existing-only; no CPU generation is supported")
checkpoint, info_file = args.checkpoint.resolve(), args.checkpoint_info.resolve()
prompts_file, config_file = args.prompts.resolve(), args.run_config.resolve()
mapping = args.train_mapping.resolve()
output, javis_root, ltx_root = args.output.resolve(), args.javis_root.resolve(), args.ltx_root.resolve()
javis_python, ltx_python = args.javis_python.absolute(), args.ltx_python.absolute()
infer = ltx_root / "packages/ltx-distillation/src/ltx_distillation/tools/run_av_inference_eval.py"
scorer = Path(__file__).absolute().with_name("compute_diversity_imagebind_pairwise.py")
for path in (checkpoint, info_file, prompts_file, config_file, mapping, infer,
scorer, javis_python, ltx_python):
if not path.is_file():
raise FileNotFoundError(path)
info = json.loads(info_file.read_text(encoding="utf-8"))
if (checkpoint.stat().st_size != info["model_bytes"]
or info["format"] != "pytorch_state_dict"):
raise ValueError(f"Published checkpoint metadata mismatch: {info_file}")
cfg = load_config(config_file)
for key in ("checkpoint_path", "gemma_path"):
if not Path(cfg[key]).exists():
raise FileNotFoundError(f"{key}: {cfg[key]}")
prompts = [p.strip() for p in prompts_file.read_text(encoding="utf-8").splitlines() if p.strip()]
if len(prompts) < args.limit:
raise ValueError(f"Expected {args.limit} prompts, found {len(prompts)}")
prompts = prompts[:args.limit]
overlap = training_overlap(mapping, prompts)
if not (javis_root / "checkpoints/imagebind_huge.pth").is_file():
raise FileNotFoundError(javis_root / "checkpoints/imagebind_huge.pth")
if overlap:
print(f"WARNING: {len(overlap)} prompts overlap {mapping}", flush=True)
model_name = "published_turbot2av_main"
media = output / "media" / model_name
metric_root = output / "metrics"
# Use this experiment's paths only; never rewrite the prior machine's config.
from compute_diversity_imagebind_pairwise import embed_path, sample_paths
eval_config = {
"paths": {"javis_root": str(javis_root), "diversity_metric_root": str(metric_root)},
"prompt_subset": {"num_prompts": args.limit, "num_seeds": args.num_seeds},
}
model_cfg = {"name": model_name, "media_dir": str(media)}
manifest = {
"model_id": "luyu1021/TurboT2AV", "checkpoint": str(checkpoint),
"checkpoint_sha256": info["model_sha256"],
"config_sha256": hashlib.sha256(config_file.read_bytes()).hexdigest(),
"prompts_sha256": hashlib.sha256("\n".join(prompts).encode("utf-8")).hexdigest(),
"num_prompts": args.limit, "num_seeds": args.num_seeds, "seed": args.seed,
"seed_rule": "seed + prompt_index * num_seeds + seed_index",
"train_overlap_count": len(overlap), "train_mapping": str(mapping),
"imagebind": str(javis_root / "checkpoints/imagebind_huge.pth"),
"javis_root": str(javis_root),
"ltx_infer": str(infer),
"ltx_python": str(ltx_python),
"javis_python": str(javis_python),
}
if args.prepare_only:
if output.exists():
extras = {p.name for p in output.iterdir()} - {"media", "evaluation_manifest.json"}
if extras and not (output / "evaluation_manifest.json").is_file():
raise RuntimeError(f"Cannot adopt output with unknown files: {sorted(extras)}")
if (output / "media").exists():
media_extras = {p.name for p in (output / "media").iterdir()} - {model_name}
if media_extras and not (output / "evaluation_manifest.json").is_file():
raise RuntimeError(f"Cannot adopt unrelated media: {sorted(media_extras)}")
count = inspect_existing_media(media, prompts, args.seed, args.num_seeds)
if (output / "evaluation_manifest.json").is_file():
check_manifest(output, manifest, True)
elif not args.dry_run:
write_json(output / "evaluation_manifest.json", manifest)
print(f"validated diversity media: {count}/{args.limit * args.num_seeds} "
f"complete prompt/seed pairs; missing={args.limit * args.num_seeds - count}; "
f"manifest={'dry-run' if args.dry_run else output / 'evaluation_manifest.json'}")
return 0
check_manifest(output, manifest, args.dry_run)
generate = [
str(ltx_python), str(infer), "--config_path", str(config_file),
"--prompts_file", str(prompts_file), "--output_dir", str(media),
"--model_kind", "student", "--student_checkpoint", str(checkpoint),
"--student_param", "rcm_trig", "--num_prompts", str(args.limit),
"--num_seeds", str(args.num_seeds), "--seed", str(args.seed),
]
embedding = embed_path(eval_config, model_cfg)
try:
sample_paths(eval_config, model_cfg)
complete = True
except FileNotFoundError:
complete = False
if args.dry_run:
print(f"diversity: {args.limit} prompts x {args.num_seeds} seeds; "
f"{args.limit * args.num_seeds * (args.num_seeds - 1) // 2} within-prompt pairs per modality")
print(f"generate={not complete}, embed={not embedding.is_file()}")
print(f"generate: {generate}")
return 0
if args.score_existing_only and not complete:
raise RuntimeError(f"--score-existing-only requires all {args.limit * args.num_seeds} pairs in {media}")
env = unset_proxy_env()
env["CUDA_VISIBLE_DEVICES"] = "" if args.cpu else args.gpu
env["PYTHONUNBUFFERED"] = "1"
if not complete:
media.mkdir(parents=True, exist_ok=True)
subprocess.run(generate, cwd=ltx_root, env=env, check=True)
sample_paths(eval_config, model_cfg)
# The caller may be a pixi Python or another venv; the scoring imports
# must always execute through the separate JavisBench environment.
scoring = [
str(javis_python), str(scorer), "--config",
str(output / "diversity_config.json"), *(["--cpu"] if args.cpu else ["--gpu", str(args.gpu)]),
"--only", model_name, "--batch-size", str(args.batch_size),
"--num-workers", str(args.num_workers),
]
write_json(output / "diversity_config.json", {
**eval_config, "models": [model_cfg],
})
subprocess.run(scoring, cwd=javis_root, env=env, check=True)
summary_path = metric_root / "summary.json"
if not summary_path.is_file():
raise RuntimeError(f"Missing diversity scores: {summary_path}")
rows = json.loads(summary_path.read_text(encoding="utf-8"))["rows"]
if len(rows) != 1 or rows[0]["model_name"] != model_name:
raise RuntimeError(f"Unexpected diversity scores: {summary_path}")
scores = rows[0]
if args.limit == 200 and args.num_seeds == 8:
reference = {
"video_pairwise_cosine_distance": 0.2322,
"audio_pairwise_cosine_distance": 0.4197,
"va_average_pairwise_cosine_distance": 0.3259,
}
write_json(output / "diversity_paper_comparison.json", {
"model_identity": model_name,
"note": "Numerical comparison only; exact author prompt/enhancement and ImageBind distance protocols unverified",
"metrics": {
key: {"observed": scores[key], "paper": value, "delta": scores[key] - value}
for key, value in reference.items()
},
})
print(json.dumps({
k: scores[k] for k in (
"video_pairwise_cosine_distance", "audio_pairwise_cosine_distance",
"va_average_pairwise_cosine_distance",
)
}, indent=2))
return 0
if __name__ == "__main__":
sys.exit(run(parse_args()))
|