File size: 7,209 Bytes
76b7b25
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
#!/usr/bin/env python3
"""Watch in-progress AV media and score each completed experiment on a spare GPU."""
from __future__ import annotations

import argparse
import json
import os
import subprocess
import sys
import time
from pathlib import Path

from common import utc_now, write_json
from run_checkpoint_curve import score_complete


def media_count(directory: Path, indices: range, seeds: int | None = None) -> int:
    def complete(stem: str) -> bool:
        return all(
            (directory / f"{stem}.{suffix}").is_file()
            and (directory / f"{stem}.{suffix}").stat().st_size > 0
            for suffix in ("mp4", "wav", "json")
        )

    return sum(
        complete(f"sample_{i:04d}" + (f"_seed{j:04d}" if seeds is not None else ""))
        for i in indices for j in (range(seeds) if seeds is not None else (None,))
    )


def run(args: argparse.Namespace) -> int:
    if args.poll_seconds < 2 or args.max_hours <= 0:
        raise ValueError("--poll-seconds >= 2 and --max-hours > 0 are required")
    script_root = Path(__file__).resolve().parent
    javis_python = args.javis_python.absolute()
    javis_root = args.javis_root.resolve(strict=True)
    media = {name: path.resolve() for name, path in (
        ("student", args.student_media), ("teacher_40_step", args.teacher_media),
        ("diversity", args.diversity_media))}
    outputs = {name: args.output.resolve() / name for name in ("student", "teacher_40_step", "diversity")}
    status_file = args.output.resolve() / "watch_status.json"
    prompts = args.prompts.resolve(strict=True).read_text(encoding="utf-8").splitlines()
    if len(prompts) != 200:
        raise ValueError("Expected 200 prompts")
    started = time.monotonic()
    targets = {"student"} if args.student_only else {"student", "teacher_40_step", "diversity"}
    done: set[str] = {
        name for name in ("student", "teacher_40_step")
        if name in targets and score_complete(outputs[name] / "scores.json")
    }
    diversity_summary = outputs["diversity"] / "summary.json"
    if "diversity" in targets and diversity_summary.is_file():
        payload = json.loads(diversity_summary.read_text(encoding="utf-8"))
        if len(payload.get("prompt_indices", [])) == 200 and payload.get("seeds") == 8:
            done.add("diversity")
    failures: dict[str, str] = {}
    while time.monotonic() - started < args.max_hours * 3600:
        counts = {
            "student": media_count(media["student"], range(200)),
            "teacher_40_step": media_count(media["teacher_40_step"], range(200)),
            "diversity": media_count(media["diversity"], range(200), seeds=8),
        }
        if "student" not in done and counts["student"] == 200:
            target = "student"
        elif "teacher_40_step" in targets and "teacher_40_step" not in done and counts["teacher_40_step"] == 200:
            target = "teacher_40_step"
        elif "diversity" in targets and "diversity" not in done and counts["diversity"] == 1600:
            target = "diversity"
        else:
            target = None
        write_json(status_file, {
            "updated_at_utc": utc_now(), "gpu": args.gpu, "counts": counts,
            "completed_scoring": sorted(done), "failures": failures,
            "active_scoring": target,
        })
        print(f"{utc_now()} media {counts} scoring {target or 'waiting'}", flush=True)
        if targets <= done:
            return 0
        if target is None:
            time.sleep(args.poll_seconds)
            continue
        env = dict(os.environ)
        env.update({
            "CUDA_VISIBLE_DEVICES": str(args.gpu), "HF_HUB_OFFLINE": "1",
            "TRANSFORMERS_OFFLINE": "1", "PYTHONUNBUFFERED": "1",
            "OMP_NUM_THREADS": "4", "MKL_NUM_THREADS": "4",
        })
        if target == "diversity":
            command = [
                str(javis_python), str(script_root / "score_diversity_completed.py"),
                "--media", str(media[target]), "--prompts", str(args.prompts),
                "--javis-root", str(javis_root), "--output", str(outputs[target]),
                "--gpu", str(args.gpu), "--batch-size", "1", "--workers", "2",
            ]
        else:
            command = [
                sys.executable, str(script_root / "score_existing_gpu.py"),
                "--model", target, "--media", str(media[target]), "--prompts", str(args.prompts),
                "--output", str(outputs[target]), "--javis-root", str(javis_root),
                "--javis-python", str(javis_python), "--gpu", str(args.gpu), "--workers", "4",
            ]
        print(f"start {target}: {' '.join(command)}", flush=True)
        result = subprocess.run(command, env=env, check=False)
        if result.returncode:
            failures[target] = f"exit code {result.returncode}"
            write_json(status_file, {"updated_at_utc": utc_now(), "counts": counts,
                                     "completed_scoring": sorted(done), "failures": failures})
            raise RuntimeError(f"{target} scoring failed with {result.returncode}")
        if target == "diversity":
            summary = json.loads((outputs[target] / "summary.json").read_text(encoding="utf-8"))
            if len(summary["prompt_indices"]) != 200:
                raise RuntimeError("Diversity result does not cover all 200 prompts")
            reference = {"video_pairwise_cosine_distance": 0.2322,
                         "audio_pairwise_cosine_distance": 0.4197,
                         "va_average_pairwise_cosine_distance": 0.3259}
            metrics = summary["metrics"]
            write_json(outputs[target] / "paper_comparison.json", {
                "note": "Numeric comparison, pending prompt enhancement/checkpoint identity verification.",
                "metrics": {key: {"observed": metrics[key], "paper": value,
                                  "delta": metrics[key] - value}
                            for key, value in reference.items()},
            })
        elif not score_complete(outputs[target] / "scores.json"):
            raise RuntimeError(f"{target} JavisBench output incomplete")
        done.add(target)
    raise TimeoutError(f"Evaluation watch timed out after {args.max_hours}h; status: {status_file}")


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--student-media", type=Path, required=True)
    parser.add_argument("--teacher-media", type=Path, required=True)
    parser.add_argument("--diversity-media", type=Path, required=True)
    parser.add_argument("--prompts", type=Path, required=True)
    parser.add_argument("--javis-root", type=Path, required=True)
    parser.add_argument("--javis-python", type=Path, required=True)
    parser.add_argument("--output", type=Path, required=True)
    parser.add_argument("--gpu", default="7")
    parser.add_argument("--poll-seconds", type=int, default=30)
    parser.add_argument("--max-hours", type=float, default=48)
    parser.add_argument("--student-only", action="store_true", help="Score student and exit without starting other evaluations")
    return run(parser.parse_args())


if __name__ == "__main__":
    raise SystemExit(main())