bench / monitor_unityshotsbench.py
ymyy307's picture
Upload folder using huggingface_hub
0d17d13 verified
Raw History Blame Contribute Delete
6.64 kB
#!/usr/bin/env python3
"""Persistent supervisor for the four-GPU UnityShotsBench run."""
import json
import os
import signal
import subprocess
import time
from pathlib import Path
ROOT = Path("/datadrive1/v-zihanyang")
EVAL = ROOT / "evals/unityshotsbench"
STATE = EVAL / "run_state"
LOGS = EVAL / "logs"
H3_OUT = EVAL / "h3_method"
STATUS = STATE / "monitor_status.json"
def atomic(path: Path, value):
tmp = path.with_name(path.name + f".tmp.{os.getpid()}")
tmp.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n")
os.replace(tmp, path)
def alive(pid):
try:
os.kill(pid, 0)
stat = Path(f"/proc/{pid}/stat")
if stat.is_file() and stat.read_text().split()[2] == "Z":
return False
return True
except (ProcessLookupError, PermissionError):
return False
def pid_from(name):
path = STATE / f"{name}.pid"
try:
return int(path.read_text().strip())
except Exception:
return None
def launch(name, command, cwd, log, cuda):
handle = open(log, "a", buffering=1)
env = dict(os.environ, CUDA_VISIBLE_DEVICES=cuda, PYTHONUNBUFFERED="1")
process = subprocess.Popen(command, cwd=cwd, env=env, stdout=handle, stderr=subprocess.STDOUT,
start_new_session=True)
(STATE / f"{name}.pid").write_text(str(process.pid) + "\n")
return process.pid
def h3_command(preflight):
cmd = [str(ROOT / "MiniMax-H3/.conda/envs/minimax-h3-zeroshot/bin/python"), "-u",
"zeroshot/unityshotsbench_runner.py"]
return cmd + (["--preflight"] if preflight else [])
def launch_h3(preflight):
return launch("h3_preflight" if preflight else "h3_full", h3_command(preflight),
ROOT / "MiniMax-H3", LOGS / "h3_method.log", "0,1")
def launch_joy(gpu):
shard = "gpu2" if gpu == 2 else "gpu3"
return launch(f"joyai_{shard}", [str(Path("/home/qid/.conda/envs/echo-long/bin/python")), "-u", "inference.py",
"--config", "configs/inference.yaml", "--prompts-dir", str(EVAL / f"joyai_shards/{shard}"),
"--prompts-glob", "story_*.json", "--output-root", str(EVAL / f"joyai_echo/{shard}")],
ROOT / "JoyAI-Echo", LOGS / f"joyai_{shard}.log", str(gpu))
def valid_h3_shot(story, idx):
base = H3_OUT / story
return all((base / name).is_file() and (base / name).stat().st_size > 0 for name in (
f"shot_{idx}.mp4", f"shot_{idx}.wav", f"shot_{idx}.metadata.json"))
def preflight_valid():
if not all(valid_h3_shot("story_001", i) for i in range(3)):
return False
if not all(valid_h3_shot("story_091", i) for i in range(5)):
return False
for story in ("story_001", "story_091"):
registry_path = H3_OUT / story / "voice_reference_registry.json"
if not registry_path.is_file():
return False
reg = json.loads((H3_OUT / "story_091/voice_reference_registry.json").read_text())
if not all(Path(reg[c]["ref_audio_path"]).is_file() for c in ("char1", "char2")):
return False
if not (reg["char1"]["ref_audio_path"] != reg["char2"]["ref_audio_path"] and (
reg["char1"]["clip_start"], reg["char1"]["clip_end"]) != (
reg["char2"]["clip_start"], reg["char2"]["clip_end"])):
return False
shot4 = json.loads((H3_OUT / "story_091/shot_4.metadata.json").read_text())
return shot4.get("reference_input_order") == [
{"kind": "picture", "label": "<Picture 1>"},
{"kind": "picture", "label": "<Picture 2>"},
{"kind": "audio", "label": "<Audio 1>"},
{"kind": "audio", "label": "<Audio 2>"},
] and [x.get("character_id") for x in shot4.get("audios", [])] == ["char1", "char2"]
def h3_complete_count():
return len(list(H3_OUT.glob("story_*/shot_*.metadata.json")))
def joy_complete(shard):
return len(list((EVAL / f"joyai_echo/{shard}").glob("story_*/.complete.json")))
def failure_key(log):
try:
return "\n".join(Path(log).read_text(errors="replace").splitlines()[-30:])
except Exception:
return "missing-log"
def main():
STATE.mkdir(parents=True, exist_ok=True)
failures = {}
preflight_done = preflight_valid()
while True:
summary = {"time": time.time(), "preflight_complete": preflight_done,
"h3_completed_shots": h3_complete_count(),
"joy_gpu2_completed_stories": joy_complete("gpu2"),
"joy_gpu3_completed_stories": joy_complete("gpu3"), "status": "running"}
if not preflight_done:
pid = pid_from("h3_preflight")
if not pid or not alive(pid):
if preflight_valid():
preflight_done = True
else:
key = failure_key(LOGS / "h3_method.log")
failures[key] = failures.get(key, 0) + 1
if failures[key] >= 3:
summary.update(status="blocked", blocker="H3 preflight repeated the same failure three times")
atomic(STATUS, summary)
return
launch_h3(True)
else:
pid = pid_from("h3_full")
if h3_complete_count() < 992 and (not pid or not alive(pid)):
key = failure_key(LOGS / "h3_method.log")
failures[key] = failures.get(key, 0) + 1
if failures[key] >= 3:
summary.update(status="blocked", blocker="H3 full run repeated the same failure three times")
atomic(STATUS, summary)
return
launch_h3(False)
for gpu, shard in ((2, "gpu2"), (3, "gpu3")):
pid = pid_from(f"joyai_{shard}")
if joy_complete(shard) < 100 and (not pid or not alive(pid)):
key = failure_key(LOGS / f"joyai_{shard}.log")
failures[(shard, key)] = failures.get((shard, key), 0) + 1
if failures[(shard, key)] >= 3:
summary.update(status="blocked", blocker=f"JoyAI {shard} repeated the same failure three times")
atomic(STATUS, summary)
return
launch_joy(gpu)
if preflight_done and h3_complete_count() == 992 and joy_complete("gpu2") == 100 and joy_complete("gpu3") == 100:
summary.update(status="complete", h3_completed_shots=992,
joy_gpu2_completed_stories=100, joy_gpu3_completed_stories=100)
atomic(STATUS, summary)
return
atomic(STATUS, summary)
time.sleep(30)
if __name__ == "__main__":
main()