File size: 5,862 Bytes
76b7b25
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
#!/usr/bin/env python3
"""Generate and JavisBench-score one published full-state-dict checkpoint."""
from __future__ import annotations

import argparse
import hashlib
import json
import subprocess
import sys
from pathlib import Path

from common import load_config, unset_proxy_env
from run_checkpoint_curve import METRICS, SCORES, check_manifest, media_complete, score_complete, training_overlap, write_input


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--checkpoint", type=Path, required=True)
    parser.add_argument("--checkpoint-info", type=Path, required=True)
    parser.add_argument("--run-config", type=Path, required=True)
    parser.add_argument("--prompts", type=Path, required=True)
    parser.add_argument("--output", type=Path, required=True)
    parser.add_argument("--javis-root", type=Path, required=True)
    parser.add_argument("--javis-python", type=Path, required=True)
    parser.add_argument("--ltx-root", type=Path, required=True)
    parser.add_argument("--ltx-python", type=Path, required=True)
    parser.add_argument("--gpu", default="0")
    parser.add_argument("--seed", type=int, default=12345)
    parser.add_argument("--limit", type=int, default=10)
    parser.add_argument("--dry-run", action="store_true")
    return parser.parse_args()


def run(args: argparse.Namespace) -> int:
    checkpoint, info_path = args.checkpoint.resolve(), args.checkpoint_info.resolve()
    config, prompts_path = args.run_config.resolve(), args.prompts.resolve()
    output, javis_root, ltx_root = args.output.resolve(), args.javis_root.resolve(), args.ltx_root.resolve()
    # A venv's python symlink must be executed through its own path.
    javis_python, ltx_python = args.javis_python.absolute(), args.ltx_python.absolute()
    infer = ltx_root / "packages/ltx-distillation/src/ltx_distillation/tools/run_av_inference_eval.py"
    bench = javis_root / "eval/javisbench/main.py"
    if args.limit < 1:
        raise ValueError("--limit must be positive")
    for path in (checkpoint, info_path, config, prompts_path, infer, bench, javis_python, ltx_python):
        if not path.is_file():
            raise FileNotFoundError(path)
    info = json.loads(info_path.read_text(encoding="utf-8"))
    if (checkpoint.stat().st_size != info["model_bytes"]
            or info.get("format") != "pytorch_state_dict"
            or info.get("checkpoint_name") != "turbot2av_main"):
        raise ValueError(f"Published checkpoint size/format/name does not match {info_path}")
    prompts = [line.strip() for line in prompts_path.read_text(encoding="utf-8").splitlines() if line.strip()]
    if len(prompts) < args.limit:
        raise ValueError(f"Expected {args.limit} prompts, found {len(prompts)}")
    prompts = prompts[:args.limit]
    cfg = load_config(config)
    for key in ("checkpoint_path", "gemma_path"):
        if not Path(cfg[key]).exists():
            raise FileNotFoundError(f"{key}: {cfg[key]}")
    mapping = Path(cfg["data_path"])
    if not mapping.is_file():
        raise FileNotFoundError(f"Training mapping missing: {mapping}")
    overlap = training_overlap(mapping, prompts)
    if overlap:
        print(f"WARNING: {len(overlap)} evaluation prompts overlap training mapping {mapping}", flush=True)
    manifest = {
        "model_id": "luyu1021/TurboT2AV",
        "checkpoint": str(checkpoint),
        "checkpoint_sha256": info["model_sha256"],
        "checkpoint_step": info["checkpoint_step"],
        "config": str(config),
        "config_sha256": hashlib.sha256(config.read_bytes()).hexdigest(),
        "prompt_sha256": hashlib.sha256("\n".join(prompts).encode("utf-8")).hexdigest(),
        "num_prompts": args.limit,
        "train_overlap_count": len(overlap),
        "train_mapping": str(mapping),
        "seed": args.seed,
        "metrics": METRICS,
        "javis_root": str(javis_root),
        "ltx_python": str(ltx_python),
        "javis_python": str(javis_python),
    }
    check_manifest(output, manifest, args.dry_run)
    media = output / "media"
    score_file = output / "scores.json"
    input_csv = output / "input.csv"
    generate = [
        str(ltx_python), str(infer), "--config_path", str(config),
        "--prompts_file", str(prompts_path), "--output_dir", str(media),
        "--model_kind", "student", "--student_param", "rcm_trig",
        "--student_checkpoint", str(checkpoint), "--num_prompts", str(args.limit),
        "--num_seeds", "1", "--seed", str(args.seed),
    ]
    score = [
        str(javis_python), "-m", "eval.javisbench.main",
        "--input_file", str(input_csv), "--output_file", str(score_file),
        "--metrics", *METRICS, "--max_audio_len_s", "4.0", "--num_workers", "8",
    ]
    if args.dry_run:
        print(f"published probe: generate={not media_complete(media, args.limit)} score={not score_complete(score_file)}")
        print(f"generate: {generate}")
        print(f"score: {score}")
        return 0
    env = unset_proxy_env()
    env["CUDA_VISIBLE_DEVICES"] = args.gpu
    env["PYTHONUNBUFFERED"] = "1"
    if not score_complete(score_file):
        if not media_complete(media, args.limit):
            media.mkdir(parents=True, exist_ok=True)
            subprocess.run(generate, cwd=ltx_root, env=env, check=True)
        if not media_complete(media, args.limit):
            raise RuntimeError(f"Incomplete media: {media}")
        write_input(input_csv, prompts, media)
        subprocess.run(score, cwd=javis_root, env=env, check=True)
    if not score_complete(score_file):
        raise RuntimeError(f"Incomplete JavisBench scores: {score_file}")
    values = json.loads(score_file.read_text(encoding="utf-8"))
    print(json.dumps({k: values[k] for k in SCORES}, indent=2))
    return 0


if __name__ == "__main__":
    sys.exit(run(parse_args()))