File size: 5,012 Bytes
76b7b25
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
#!/usr/bin/env python3
"""Score already generated, prompt-aligned media using isolated JavisBench Python."""
from __future__ import annotations

import argparse
import hashlib
import json
import os
import subprocess
from pathlib import Path

from common import write_json
from run_checkpoint_curve import METRICS, SCORES, score_complete, write_input
from score_existing_cpu_200 import PAPER


def run(args: argparse.Namespace) -> int:
    if args.limit < 1 or args.limit > 200 or args.workers < 0:
        raise ValueError("Expected 1 <= limit <= 200 and nonnegative workers")
    prompts_path = args.prompts.resolve(strict=True)
    all_prompts = prompts_path.read_text(encoding="utf-8").splitlines()
    if len(all_prompts) != 200 or any(not p.strip() for p in all_prompts):
        raise ValueError("Expected exactly 200 nonempty prompts")
    prompts = all_prompts[:args.limit]
    media = args.media.resolve(strict=True)
    for idx, prompt in enumerate(prompts):
        for suffix in ("mp4", "wav"):
            path = media / f"sample_{idx:04d}.{suffix}"
            if not path.is_file() or path.stat().st_size == 0:
                raise FileNotFoundError(path)
        sidecar = media / f"sample_{idx:04d}.json"
        if not sidecar.is_file():
            raise FileNotFoundError(f"Missing generation provenance: {sidecar}")
        metadata = json.loads(sidecar.read_text(encoding="utf-8"))
        if metadata.get("prompt") != prompt or metadata.get("index") != idx or metadata.get("seed_idx") != 0:
            raise ValueError(f"Generated sample not aligned to prompt: {sidecar}")

    output = args.output.resolve()
    root = args.javis_root.resolve(strict=True)
    python = args.javis_python.absolute()
    if not python.is_file():
        raise FileNotFoundError(python)
    if not (root / "checkpoints/imagebind_huge.pth").is_file():
        raise FileNotFoundError(root / "checkpoints/imagebind_huge.pth")
    manifest = {
        "model": args.model,
        "media": str(media),
        "prompts_file": str(prompts_path),
        "prompt_sha256": hashlib.sha256(prompts_path.read_bytes()).hexdigest(),
        "limit": args.limit,
        "metrics": METRICS,
        "gpu": args.gpu,
        "javis_python": str(python),
        "javis_root": str(root),
    }
    manifest_path = output / "manifest.json"
    if manifest_path.is_file():
        if json.loads(manifest_path.read_text(encoding="utf-8")) != manifest:
            raise RuntimeError(f"Scoring settings changed: {manifest_path}")
    elif output.exists() and any(output.iterdir()):
        raise RuntimeError(f"Nonempty output without matching manifest: {output}")
    else:
        write_json(manifest_path, manifest)
    scores = output / "scores.json"
    if not score_complete(scores):
        csv_path = output / "input.csv"
        write_input(csv_path, prompts, media)
        env = dict(os.environ)
        env["CUDA_VISIBLE_DEVICES"] = str(args.gpu)
        env["HF_HUB_OFFLINE"] = "1"
        env["TRANSFORMERS_OFFLINE"] = "1"
        env["TOKENIZERS_PARALLELISM"] = "false"
        env["OMP_NUM_THREADS"] = str(args.cpu_threads)
        env["MKL_NUM_THREADS"] = str(args.cpu_threads)
        env["PATH"] = f"{python.parent}:{env.get('PATH', '')}"
        subprocess.run([
            str(python), "-m", "eval.javisbench.main",
            "--input_file", str(csv_path),
            "--output_file", str(scores),
            "--metrics", *METRICS, "--max_audio_len_s", "4.0",
            "--num_workers", str(args.workers),
        ], cwd=root, env=env, check=True)
    if not score_complete(scores):
        raise RuntimeError(f"Incomplete scores: {scores}")
    values = json.loads(scores.read_text(encoding="utf-8"))
    if args.limit == 200:
        write_json(output / "paper_comparison.json", {
            "model": args.model,
            "note": "Numeric comparison; checkpoint identity and prompt enhancement still require verification.",
            "scores": {key: {"observed": values[key], "paper": PAPER[args.model][key],
                            "delta": values[key] - PAPER[args.model][key]} for key in SCORES},
        })
    print(json.dumps({key: values[key] for key in SCORES}, indent=2))
    return 0


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--model", choices=sorted(PAPER), required=True)
    parser.add_argument("--media", type=Path, required=True)
    parser.add_argument("--prompts", type=Path, required=True)
    parser.add_argument("--output", type=Path, required=True)
    parser.add_argument("--javis-root", type=Path, required=True)
    parser.add_argument("--javis-python", type=Path, required=True)
    parser.add_argument("--gpu", default="7")
    parser.add_argument("--limit", type=int, default=200)
    parser.add_argument("--workers", type=int, default=4)
    parser.add_argument("--cpu-threads", type=int, default=4)
    return run(parser.parse_args())


if __name__ == "__main__":
    raise SystemExit(main())