File size: 12,248 Bytes
76b7b25
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
#!/usr/bin/env python3
"""Run a resumable published-model 200 x 8 diversity experiment."""
from __future__ import annotations

import argparse
import hashlib
import json
import re
import subprocess
import sys
from pathlib import Path

from common import load_config, unset_proxy_env, write_json
from run_checkpoint_curve import check_manifest, training_overlap


SAMPLE_NAME = re.compile(r"sample_(\d{4})_seed(\d{4})\.(mp4|wav|json)")


def inspect_existing_media(media: Path, prompts: list[str], seed: int, num_seeds: int) -> int:
    """Validate all existing outputs before adopting a resumable media directory."""
    if not media.is_dir():
        return 0
    existing: dict[tuple[int, int], set[str]] = {}
    for path in media.iterdir():
        match = SAMPLE_NAME.fullmatch(path.name)
        if match is None:
            if path.name.startswith("sample_"):
                raise ValueError(f"Unexpected sample filename: {path}")
            continue  # Existing shard prompt lists and model-init lock are not media.
        index, seed_idx, suffix = int(match[1]), int(match[2]), match[3]
        if index >= len(prompts) or seed_idx >= num_seeds or not path.is_file():
            raise ValueError(f"Unexpected prompt/seed media: {path}")
        existing.setdefault((index, seed_idx), set()).add(suffix)
        if not path.stat().st_size:
            raise ValueError(f"Empty media file: {path}")
    for (index, seed_idx), suffixes in existing.items():
        # A lone MP4 or WAV may be the result of an interrupted generation.
        # Both files without a sidecar cannot be provenance-checked.
        if ("json" in suffixes and suffixes != {"mp4", "wav", "json"}) or (
            {"mp4", "wav"} <= suffixes and "json" not in suffixes
        ):
            raise ValueError(f"Incomplete provenance for sample_{index:04d}_seed{seed_idx:04d}")
        if "json" not in suffixes:
            continue
        sidecar = media / f"sample_{index:04d}_seed{seed_idx:04d}.json"
        record = json.loads(sidecar.read_text(encoding="utf-8"))
        if (record.get("index") != index or record.get("seed_idx") != seed_idx
                or record.get("seed") != seed + index * num_seeds + seed_idx
                or record.get("prompt") != prompts[index]):
            raise ValueError(f"Generated prompt/seed mismatch: {sidecar}")
    return sum(suffixes == {"mp4", "wav", "json"} for suffixes in existing.values())


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--checkpoint", type=Path, required=True)
    parser.add_argument("--checkpoint-info", type=Path, required=True)
    parser.add_argument("--run-config", type=Path, required=True)
    parser.add_argument("--prompts", type=Path, required=True)
    parser.add_argument("--train-mapping", type=Path, required=True)
    parser.add_argument("--output", type=Path, required=True)
    parser.add_argument("--javis-root", type=Path, required=True)
    parser.add_argument("--javis-python", type=Path, required=True)
    parser.add_argument("--ltx-root", type=Path, required=True)
    parser.add_argument("--ltx-python", type=Path, required=True)
    parser.add_argument("--gpu", default="0")
    parser.add_argument("--seed", type=int, default=12345)
    parser.add_argument("--limit", type=int, default=200)
    parser.add_argument("--num-seeds", type=int, default=8)
    parser.add_argument("--batch-size", type=int, default=4)
    parser.add_argument("--num-workers", type=int, default=4)
    parser.add_argument("--score-existing-only", action="store_true",
                        help="score a complete media directory without starting generation")
    parser.add_argument("--cpu", action="store_true",
                        help="CPU scoring only; requires --score-existing-only, never generates")
    parser.add_argument("--prepare-only", action="store_true",
                        help="validate/adopt existing media and write manifest; never generate or score")
    parser.add_argument("--dry-run", action="store_true")
    return parser.parse_args()


def run(args: argparse.Namespace) -> int:
    if args.limit < 1 or args.num_seeds < 2 or args.batch_size < 1 or args.num_workers < 0:
        raise ValueError("--limit must be positive and --num-seeds at least 2")
    if args.cpu and not args.score_existing_only and not getattr(args, "prepare_only", False):
        raise ValueError("--cpu requires --score-existing-only; no CPU generation is supported")
    checkpoint, info_file = args.checkpoint.resolve(), args.checkpoint_info.resolve()
    prompts_file, config_file = args.prompts.resolve(), args.run_config.resolve()
    mapping = args.train_mapping.resolve()
    output, javis_root, ltx_root = args.output.resolve(), args.javis_root.resolve(), args.ltx_root.resolve()
    javis_python, ltx_python = args.javis_python.absolute(), args.ltx_python.absolute()
    infer = ltx_root / "packages/ltx-distillation/src/ltx_distillation/tools/run_av_inference_eval.py"
    scorer = Path(__file__).absolute().with_name("compute_diversity_imagebind_pairwise.py")
    for path in (checkpoint, info_file, prompts_file, config_file, mapping, infer,
                 scorer, javis_python, ltx_python):
        if not path.is_file():
            raise FileNotFoundError(path)
    info = json.loads(info_file.read_text(encoding="utf-8"))
    if (checkpoint.stat().st_size != info["model_bytes"]
            or info["format"] != "pytorch_state_dict"):
        raise ValueError(f"Published checkpoint metadata mismatch: {info_file}")
    cfg = load_config(config_file)
    for key in ("checkpoint_path", "gemma_path"):
        if not Path(cfg[key]).exists():
            raise FileNotFoundError(f"{key}: {cfg[key]}")
    prompts = [p.strip() for p in prompts_file.read_text(encoding="utf-8").splitlines() if p.strip()]
    if len(prompts) < args.limit:
        raise ValueError(f"Expected {args.limit} prompts, found {len(prompts)}")
    prompts = prompts[:args.limit]
    overlap = training_overlap(mapping, prompts)
    if not (javis_root / "checkpoints/imagebind_huge.pth").is_file():
        raise FileNotFoundError(javis_root / "checkpoints/imagebind_huge.pth")
    if overlap:
        print(f"WARNING: {len(overlap)} prompts overlap {mapping}", flush=True)
    model_name = "published_turbot2av_main"
    media = output / "media" / model_name
    metric_root = output / "metrics"
    # Use this experiment's paths only; never rewrite the prior machine's config.
    from compute_diversity_imagebind_pairwise import embed_path, sample_paths
    eval_config = {
        "paths": {"javis_root": str(javis_root), "diversity_metric_root": str(metric_root)},
        "prompt_subset": {"num_prompts": args.limit, "num_seeds": args.num_seeds},
    }
    model_cfg = {"name": model_name, "media_dir": str(media)}
    manifest = {
        "model_id": "luyu1021/TurboT2AV", "checkpoint": str(checkpoint),
        "checkpoint_sha256": info["model_sha256"],
        "config_sha256": hashlib.sha256(config_file.read_bytes()).hexdigest(),
        "prompts_sha256": hashlib.sha256("\n".join(prompts).encode("utf-8")).hexdigest(),
        "num_prompts": args.limit, "num_seeds": args.num_seeds, "seed": args.seed,
        "seed_rule": "seed + prompt_index * num_seeds + seed_index",
        "train_overlap_count": len(overlap), "train_mapping": str(mapping),
        "imagebind": str(javis_root / "checkpoints/imagebind_huge.pth"),
        "javis_root": str(javis_root),
        "ltx_infer": str(infer),
        "ltx_python": str(ltx_python),
        "javis_python": str(javis_python),
    }
    if args.prepare_only:
        if output.exists():
            extras = {p.name for p in output.iterdir()} - {"media", "evaluation_manifest.json"}
            if extras and not (output / "evaluation_manifest.json").is_file():
                raise RuntimeError(f"Cannot adopt output with unknown files: {sorted(extras)}")
            if (output / "media").exists():
                media_extras = {p.name for p in (output / "media").iterdir()} - {model_name}
                if media_extras and not (output / "evaluation_manifest.json").is_file():
                    raise RuntimeError(f"Cannot adopt unrelated media: {sorted(media_extras)}")
        count = inspect_existing_media(media, prompts, args.seed, args.num_seeds)
        if (output / "evaluation_manifest.json").is_file():
            check_manifest(output, manifest, True)
        elif not args.dry_run:
            write_json(output / "evaluation_manifest.json", manifest)
        print(f"validated diversity media: {count}/{args.limit * args.num_seeds} "
              f"complete prompt/seed pairs; missing={args.limit * args.num_seeds - count}; "
              f"manifest={'dry-run' if args.dry_run else output / 'evaluation_manifest.json'}")
        return 0
    check_manifest(output, manifest, args.dry_run)
    generate = [
        str(ltx_python), str(infer), "--config_path", str(config_file),
        "--prompts_file", str(prompts_file), "--output_dir", str(media),
        "--model_kind", "student", "--student_checkpoint", str(checkpoint),
        "--student_param", "rcm_trig", "--num_prompts", str(args.limit),
        "--num_seeds", str(args.num_seeds), "--seed", str(args.seed),
    ]
    embedding = embed_path(eval_config, model_cfg)
    try:
        sample_paths(eval_config, model_cfg)
        complete = True
    except FileNotFoundError:
        complete = False
    if args.dry_run:
        print(f"diversity: {args.limit} prompts x {args.num_seeds} seeds; "
              f"{args.limit * args.num_seeds * (args.num_seeds - 1) // 2} within-prompt pairs per modality")
        print(f"generate={not complete}, embed={not embedding.is_file()}")
        print(f"generate: {generate}")
        return 0
    if args.score_existing_only and not complete:
        raise RuntimeError(f"--score-existing-only requires all {args.limit * args.num_seeds} pairs in {media}")
    env = unset_proxy_env()
    env["CUDA_VISIBLE_DEVICES"] = "" if args.cpu else args.gpu
    env["PYTHONUNBUFFERED"] = "1"
    if not complete:
        media.mkdir(parents=True, exist_ok=True)
        subprocess.run(generate, cwd=ltx_root, env=env, check=True)
    sample_paths(eval_config, model_cfg)
    # The caller may be a pixi Python or another venv; the scoring imports
    # must always execute through the separate JavisBench environment.
    scoring = [
        str(javis_python), str(scorer), "--config",
        str(output / "diversity_config.json"), *(["--cpu"] if args.cpu else ["--gpu", str(args.gpu)]),
        "--only", model_name, "--batch-size", str(args.batch_size),
        "--num-workers", str(args.num_workers),
    ]
    write_json(output / "diversity_config.json", {
        **eval_config, "models": [model_cfg],
    })
    subprocess.run(scoring, cwd=javis_root, env=env, check=True)
    summary_path = metric_root / "summary.json"
    if not summary_path.is_file():
        raise RuntimeError(f"Missing diversity scores: {summary_path}")
    rows = json.loads(summary_path.read_text(encoding="utf-8"))["rows"]
    if len(rows) != 1 or rows[0]["model_name"] != model_name:
        raise RuntimeError(f"Unexpected diversity scores: {summary_path}")
    scores = rows[0]
    if args.limit == 200 and args.num_seeds == 8:
        reference = {
            "video_pairwise_cosine_distance": 0.2322,
            "audio_pairwise_cosine_distance": 0.4197,
            "va_average_pairwise_cosine_distance": 0.3259,
        }
        write_json(output / "diversity_paper_comparison.json", {
            "model_identity": model_name,
            "note": "Numerical comparison only; exact author prompt/enhancement and ImageBind distance protocols unverified",
            "metrics": {
                key: {"observed": scores[key], "paper": value, "delta": scores[key] - value}
                for key, value in reference.items()
            },
        })
    print(json.dumps({
        k: scores[k] for k in (
            "video_pairwise_cosine_distance", "audio_pairwise_cosine_distance",
            "va_average_pairwise_cosine_distance",
        )
    }, indent=2))
    return 0


if __name__ == "__main__":
    sys.exit(run(parse_args()))