File size: 7,209 Bytes
76b7b25 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 | #!/usr/bin/env python3
"""Watch in-progress AV media and score each completed experiment on a spare GPU."""
from __future__ import annotations
import argparse
import json
import os
import subprocess
import sys
import time
from pathlib import Path
from common import utc_now, write_json
from run_checkpoint_curve import score_complete
def media_count(directory: Path, indices: range, seeds: int | None = None) -> int:
def complete(stem: str) -> bool:
return all(
(directory / f"{stem}.{suffix}").is_file()
and (directory / f"{stem}.{suffix}").stat().st_size > 0
for suffix in ("mp4", "wav", "json")
)
return sum(
complete(f"sample_{i:04d}" + (f"_seed{j:04d}" if seeds is not None else ""))
for i in indices for j in (range(seeds) if seeds is not None else (None,))
)
def run(args: argparse.Namespace) -> int:
if args.poll_seconds < 2 or args.max_hours <= 0:
raise ValueError("--poll-seconds >= 2 and --max-hours > 0 are required")
script_root = Path(__file__).resolve().parent
javis_python = args.javis_python.absolute()
javis_root = args.javis_root.resolve(strict=True)
media = {name: path.resolve() for name, path in (
("student", args.student_media), ("teacher_40_step", args.teacher_media),
("diversity", args.diversity_media))}
outputs = {name: args.output.resolve() / name for name in ("student", "teacher_40_step", "diversity")}
status_file = args.output.resolve() / "watch_status.json"
prompts = args.prompts.resolve(strict=True).read_text(encoding="utf-8").splitlines()
if len(prompts) != 200:
raise ValueError("Expected 200 prompts")
started = time.monotonic()
targets = {"student"} if args.student_only else {"student", "teacher_40_step", "diversity"}
done: set[str] = {
name for name in ("student", "teacher_40_step")
if name in targets and score_complete(outputs[name] / "scores.json")
}
diversity_summary = outputs["diversity"] / "summary.json"
if "diversity" in targets and diversity_summary.is_file():
payload = json.loads(diversity_summary.read_text(encoding="utf-8"))
if len(payload.get("prompt_indices", [])) == 200 and payload.get("seeds") == 8:
done.add("diversity")
failures: dict[str, str] = {}
while time.monotonic() - started < args.max_hours * 3600:
counts = {
"student": media_count(media["student"], range(200)),
"teacher_40_step": media_count(media["teacher_40_step"], range(200)),
"diversity": media_count(media["diversity"], range(200), seeds=8),
}
if "student" not in done and counts["student"] == 200:
target = "student"
elif "teacher_40_step" in targets and "teacher_40_step" not in done and counts["teacher_40_step"] == 200:
target = "teacher_40_step"
elif "diversity" in targets and "diversity" not in done and counts["diversity"] == 1600:
target = "diversity"
else:
target = None
write_json(status_file, {
"updated_at_utc": utc_now(), "gpu": args.gpu, "counts": counts,
"completed_scoring": sorted(done), "failures": failures,
"active_scoring": target,
})
print(f"{utc_now()} media {counts} scoring {target or 'waiting'}", flush=True)
if targets <= done:
return 0
if target is None:
time.sleep(args.poll_seconds)
continue
env = dict(os.environ)
env.update({
"CUDA_VISIBLE_DEVICES": str(args.gpu), "HF_HUB_OFFLINE": "1",
"TRANSFORMERS_OFFLINE": "1", "PYTHONUNBUFFERED": "1",
"OMP_NUM_THREADS": "4", "MKL_NUM_THREADS": "4",
})
if target == "diversity":
command = [
str(javis_python), str(script_root / "score_diversity_completed.py"),
"--media", str(media[target]), "--prompts", str(args.prompts),
"--javis-root", str(javis_root), "--output", str(outputs[target]),
"--gpu", str(args.gpu), "--batch-size", "1", "--workers", "2",
]
else:
command = [
sys.executable, str(script_root / "score_existing_gpu.py"),
"--model", target, "--media", str(media[target]), "--prompts", str(args.prompts),
"--output", str(outputs[target]), "--javis-root", str(javis_root),
"--javis-python", str(javis_python), "--gpu", str(args.gpu), "--workers", "4",
]
print(f"start {target}: {' '.join(command)}", flush=True)
result = subprocess.run(command, env=env, check=False)
if result.returncode:
failures[target] = f"exit code {result.returncode}"
write_json(status_file, {"updated_at_utc": utc_now(), "counts": counts,
"completed_scoring": sorted(done), "failures": failures})
raise RuntimeError(f"{target} scoring failed with {result.returncode}")
if target == "diversity":
summary = json.loads((outputs[target] / "summary.json").read_text(encoding="utf-8"))
if len(summary["prompt_indices"]) != 200:
raise RuntimeError("Diversity result does not cover all 200 prompts")
reference = {"video_pairwise_cosine_distance": 0.2322,
"audio_pairwise_cosine_distance": 0.4197,
"va_average_pairwise_cosine_distance": 0.3259}
metrics = summary["metrics"]
write_json(outputs[target] / "paper_comparison.json", {
"note": "Numeric comparison, pending prompt enhancement/checkpoint identity verification.",
"metrics": {key: {"observed": metrics[key], "paper": value,
"delta": metrics[key] - value}
for key, value in reference.items()},
})
elif not score_complete(outputs[target] / "scores.json"):
raise RuntimeError(f"{target} JavisBench output incomplete")
done.add(target)
raise TimeoutError(f"Evaluation watch timed out after {args.max_hours}h; status: {status_file}")
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--student-media", type=Path, required=True)
parser.add_argument("--teacher-media", type=Path, required=True)
parser.add_argument("--diversity-media", type=Path, required=True)
parser.add_argument("--prompts", type=Path, required=True)
parser.add_argument("--javis-root", type=Path, required=True)
parser.add_argument("--javis-python", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--gpu", default="7")
parser.add_argument("--poll-seconds", type=int, default=30)
parser.add_argument("--max-hours", type=float, default=48)
parser.add_argument("--student-only", action="store_true", help="Score student and exit without starting other evaluations")
return run(parser.parse_args())
if __name__ == "__main__":
raise SystemExit(main())
|