File size: 6,324 Bytes
76b7b25
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
#!/usr/bin/env python3
"""JavisBench-score 200 existing AV pairs on CPU, without model generation."""
from __future__ import annotations

import argparse
import hashlib
import json
import os
import subprocess
from pathlib import Path

from common import utc_now, write_json
from run_checkpoint_curve import METRICS, SCORES, media_complete, score_complete, write_input


PAPER = {
    "student": {
        "visual_quality": 2.3892, "motion_quality": 0.8493, "audio_quality": 4.4860,
        "ib_tv": 0.2185, "ib_ta": 0.1447, "ib_av": 0.2290,
        "clip_score": 0.2921, "clap_score": 0.3472, "cavp_score": 0.8033,
        "avh_score": 0.2241, "javis_score": 0.1963, "desync": 0.3880,
    },
    "teacher_40_step": {
        "visual_quality": 1.9676, "motion_quality": 0.7825, "audio_quality": 4.9023,
        "ib_tv": 0.2683, "ib_ta": 0.1393, "ib_av": 0.1991,
        "clip_score": 0.2913, "clap_score": 0.3401, "cavp_score": 0.7959,
        "avh_score": 0.1951, "javis_score": 0.1648, "desync": 0.6170,
    },
}


def preflight(prompts_file: Path, media: Path, model: str) -> tuple[list[str], str]:
    if model not in PAPER:
        raise ValueError(f"Unrecognized model identity: {model}")
    prompts_file = prompts_file.resolve(strict=True)
    prompts = prompts_file.read_text(encoding="utf-8").splitlines()
    if len(prompts) != 200 or any(not prompt.strip() for prompt in prompts):
        raise ValueError("Expected exactly 200 non-empty prompts")
    if not media_complete(media, 200):
        missing = [
            i for i in range(200)
            if any(
                not (media / f"sample_{i:04d}.{suffix}").is_file()
                or (media / f"sample_{i:04d}.{suffix}").stat().st_size == 0
                for suffix in ("mp4", "wav")
            )
        ]
        raise FileNotFoundError(f"Missing AV pairs in {media}; first failing index={missing[0]}")
    for index, prompt in enumerate(prompts):
        sidecar = media / f"sample_{index:04d}.json"
        if not sidecar.is_file():
            continue
        payload = json.loads(sidecar.read_text(encoding="utf-8"))
        if payload.get("index") != index or payload.get("prompt") != prompt or payload.get("seed_idx") != 0:
            raise ValueError(f"Generated prompt/index/seed mismatch: {sidecar}")
    digest = hashlib.sha256(prompts_file.read_bytes()).hexdigest()
    return prompts, digest


def run(args: argparse.Namespace) -> int:
    prompts, digest = preflight(args.prompts, args.media, args.model)
    root = args.javis_root.resolve(strict=True)
    python = args.javis_python.absolute()
    if not python.is_file() or not (root / "eval/javisbench/main.py").is_file():
        raise FileNotFoundError(f"Missing JavisBench or isolated Python: {root}, {python}")
    output = args.output.resolve()
    manifest_path = output / "cpu_score_manifest.json"
    manifest = {
        "model_identity": args.model,
        "media_dir": str(args.media.resolve()),
        "prompts": str(args.prompts.resolve()),
        "prompts_sha256": digest,
        "sample_count": len(prompts),
        "score_device": "cpu",
        "javis_root": str(root),
        "javis_python": str(python),
        "metrics": METRICS,
        "paper_reference": "my_repos/turbot2av_paper/main.tex, Table VII",
        "sampling_protocol_verified": False,
        "provenance_note": "Caller must independently verify model identity and sampling steps; present generation sidecars checked for prompt alignment",
    }
    if manifest_path.exists():
        if json.loads(manifest_path.read_text(encoding="utf-8")) != manifest:
            raise RuntimeError("Settings changed; choose a different output directory")
    elif output.exists() and any(output.iterdir()):
        raise RuntimeError("Output already contains files without matching manifest")

    scores = output / "javis_scores.json"
    if not score_complete(scores):
        output.mkdir(parents=True, exist_ok=True)
        write_json(manifest_path, manifest)
        input_csv = output / "javis_input.csv"
        write_input(input_csv, prompts, args.media.resolve())
        env = dict(os.environ)
        env["CUDA_VISIBLE_DEVICES"] = ""
        env["HF_HUB_OFFLINE"] = "1"
        env["TRANSFORMERS_OFFLINE"] = "1"
        env["TOKENIZERS_PARALLELISM"] = "false"
        env["OMP_NUM_THREADS"] = str(args.cpu_threads)
        env["MKL_NUM_THREADS"] = str(args.cpu_threads)
        env["PATH"] = f"{python.parent}:{env.get('PATH', '')}"
        subprocess.run(
            [str(python), "-m", "eval.javisbench.main",
             "--input_file", str(input_csv), "--output_file", str(scores),
             "--metrics", *METRICS, "--max_audio_len_s", "4",
             "--num_workers", str(args.workers)],
            cwd=root, env=env, check=True,
        )
    if not score_complete(scores):
        raise RuntimeError(f"Incomplete JavisBench scores: {scores}")
    observed = json.loads(scores.read_text(encoding="utf-8"))
    reference = PAPER[args.model]
    result = {
        "compared_at_utc": utc_now(),
        "model_identity": args.model,
        "note": "Numerical comparison only; prompt enhancement and generation protocol remain unverified",
        "scores": {key: {
            "observed": observed[key],
            "paper": reference[key],
            "delta": observed[key] - reference[key],
        } for key in SCORES},
    }
    write_json(output / "paper_comparison.json", result)
    print(json.dumps(result, indent=2))
    return 0


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--model", choices=sorted(PAPER), required=True)
    parser.add_argument("--media", type=Path, required=True)
    parser.add_argument("--prompts", type=Path, required=True)
    parser.add_argument("--output", type=Path, required=True)
    parser.add_argument("--javis-root", type=Path, required=True)
    parser.add_argument("--javis-python", type=Path, required=True)
    parser.add_argument("--cpu-threads", type=int, default=4)
    parser.add_argument("--workers", type=int, default=0)
    args = parser.parse_args()
    if args.cpu_threads < 1 or args.workers < 0:
        parser.error("--cpu-threads must be positive and --workers nonnegative")
    return run(args)


if __name__ == "__main__":
    raise SystemExit(main())