Download monitor_unityshotsbench.py from ymyy307/bench: direct link, hf CLI and curl.
- Browser
- Download file 6.64 kB
-
https://huggingface.co/ymyy307/bench/resolve/main/monitor_unityshotsbench.py
- Command line
-
hf download hf://ymyy307/bench/monitor_unityshotsbench.py
-
curl -L -o monitor_unityshotsbench.py https://huggingface.co/ymyy307/bench/resolve/main/monitor_unityshotsbench.py
6.64 kB
| #!/usr/bin/env python3 | |
| """Persistent supervisor for the four-GPU UnityShotsBench run.""" | |
| import json | |
| import os | |
| import signal | |
| import subprocess | |
| import time | |
| from pathlib import Path | |
| ROOT = Path("/datadrive1/v-zihanyang") | |
| EVAL = ROOT / "evals/unityshotsbench" | |
| STATE = EVAL / "run_state" | |
| LOGS = EVAL / "logs" | |
| H3_OUT = EVAL / "h3_method" | |
| STATUS = STATE / "monitor_status.json" | |
| def atomic(path: Path, value): | |
| tmp = path.with_name(path.name + f".tmp.{os.getpid()}") | |
| tmp.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n") | |
| os.replace(tmp, path) | |
| def alive(pid): | |
| try: | |
| os.kill(pid, 0) | |
| stat = Path(f"/proc/{pid}/stat") | |
| if stat.is_file() and stat.read_text().split()[2] == "Z": | |
| return False | |
| return True | |
| except (ProcessLookupError, PermissionError): | |
| return False | |
| def pid_from(name): | |
| path = STATE / f"{name}.pid" | |
| try: | |
| return int(path.read_text().strip()) | |
| except Exception: | |
| return None | |
| def launch(name, command, cwd, log, cuda): | |
| handle = open(log, "a", buffering=1) | |
| env = dict(os.environ, CUDA_VISIBLE_DEVICES=cuda, PYTHONUNBUFFERED="1") | |
| process = subprocess.Popen(command, cwd=cwd, env=env, stdout=handle, stderr=subprocess.STDOUT, | |
| start_new_session=True) | |
| (STATE / f"{name}.pid").write_text(str(process.pid) + "\n") | |
| return process.pid | |
| def h3_command(preflight): | |
| cmd = [str(ROOT / "MiniMax-H3/.conda/envs/minimax-h3-zeroshot/bin/python"), "-u", | |
| "zeroshot/unityshotsbench_runner.py"] | |
| return cmd + (["--preflight"] if preflight else []) | |
| def launch_h3(preflight): | |
| return launch("h3_preflight" if preflight else "h3_full", h3_command(preflight), | |
| ROOT / "MiniMax-H3", LOGS / "h3_method.log", "0,1") | |
| def launch_joy(gpu): | |
| shard = "gpu2" if gpu == 2 else "gpu3" | |
| return launch(f"joyai_{shard}", [str(Path("/home/qid/.conda/envs/echo-long/bin/python")), "-u", "inference.py", | |
| "--config", "configs/inference.yaml", "--prompts-dir", str(EVAL / f"joyai_shards/{shard}"), | |
| "--prompts-glob", "story_*.json", "--output-root", str(EVAL / f"joyai_echo/{shard}")], | |
| ROOT / "JoyAI-Echo", LOGS / f"joyai_{shard}.log", str(gpu)) | |
| def valid_h3_shot(story, idx): | |
| base = H3_OUT / story | |
| return all((base / name).is_file() and (base / name).stat().st_size > 0 for name in ( | |
| f"shot_{idx}.mp4", f"shot_{idx}.wav", f"shot_{idx}.metadata.json")) | |
| def preflight_valid(): | |
| if not all(valid_h3_shot("story_001", i) for i in range(3)): | |
| return False | |
| if not all(valid_h3_shot("story_091", i) for i in range(5)): | |
| return False | |
| for story in ("story_001", "story_091"): | |
| registry_path = H3_OUT / story / "voice_reference_registry.json" | |
| if not registry_path.is_file(): | |
| return False | |
| reg = json.loads((H3_OUT / "story_091/voice_reference_registry.json").read_text()) | |
| if not all(Path(reg[c]["ref_audio_path"]).is_file() for c in ("char1", "char2")): | |
| return False | |
| if not (reg["char1"]["ref_audio_path"] != reg["char2"]["ref_audio_path"] and ( | |
| reg["char1"]["clip_start"], reg["char1"]["clip_end"]) != ( | |
| reg["char2"]["clip_start"], reg["char2"]["clip_end"])): | |
| return False | |
| shot4 = json.loads((H3_OUT / "story_091/shot_4.metadata.json").read_text()) | |
| return shot4.get("reference_input_order") == [ | |
| {"kind": "picture", "label": "<Picture 1>"}, | |
| {"kind": "picture", "label": "<Picture 2>"}, | |
| {"kind": "audio", "label": "<Audio 1>"}, | |
| {"kind": "audio", "label": "<Audio 2>"}, | |
| ] and [x.get("character_id") for x in shot4.get("audios", [])] == ["char1", "char2"] | |
| def h3_complete_count(): | |
| return len(list(H3_OUT.glob("story_*/shot_*.metadata.json"))) | |
| def joy_complete(shard): | |
| return len(list((EVAL / f"joyai_echo/{shard}").glob("story_*/.complete.json"))) | |
| def failure_key(log): | |
| try: | |
| return "\n".join(Path(log).read_text(errors="replace").splitlines()[-30:]) | |
| except Exception: | |
| return "missing-log" | |
| def main(): | |
| STATE.mkdir(parents=True, exist_ok=True) | |
| failures = {} | |
| preflight_done = preflight_valid() | |
| while True: | |
| summary = {"time": time.time(), "preflight_complete": preflight_done, | |
| "h3_completed_shots": h3_complete_count(), | |
| "joy_gpu2_completed_stories": joy_complete("gpu2"), | |
| "joy_gpu3_completed_stories": joy_complete("gpu3"), "status": "running"} | |
| if not preflight_done: | |
| pid = pid_from("h3_preflight") | |
| if not pid or not alive(pid): | |
| if preflight_valid(): | |
| preflight_done = True | |
| else: | |
| key = failure_key(LOGS / "h3_method.log") | |
| failures[key] = failures.get(key, 0) + 1 | |
| if failures[key] >= 3: | |
| summary.update(status="blocked", blocker="H3 preflight repeated the same failure three times") | |
| atomic(STATUS, summary) | |
| return | |
| launch_h3(True) | |
| else: | |
| pid = pid_from("h3_full") | |
| if h3_complete_count() < 992 and (not pid or not alive(pid)): | |
| key = failure_key(LOGS / "h3_method.log") | |
| failures[key] = failures.get(key, 0) + 1 | |
| if failures[key] >= 3: | |
| summary.update(status="blocked", blocker="H3 full run repeated the same failure three times") | |
| atomic(STATUS, summary) | |
| return | |
| launch_h3(False) | |
| for gpu, shard in ((2, "gpu2"), (3, "gpu3")): | |
| pid = pid_from(f"joyai_{shard}") | |
| if joy_complete(shard) < 100 and (not pid or not alive(pid)): | |
| key = failure_key(LOGS / f"joyai_{shard}.log") | |
| failures[(shard, key)] = failures.get((shard, key), 0) + 1 | |
| if failures[(shard, key)] >= 3: | |
| summary.update(status="blocked", blocker=f"JoyAI {shard} repeated the same failure three times") | |
| atomic(STATUS, summary) | |
| return | |
| launch_joy(gpu) | |
| if preflight_done and h3_complete_count() == 992 and joy_complete("gpu2") == 100 and joy_complete("gpu3") == 100: | |
| summary.update(status="complete", h3_completed_shots=992, | |
| joy_gpu2_completed_stories=100, joy_gpu3_completed_stories=100) | |
| atomic(STATUS, summary) | |
| return | |
| atomic(STATUS, summary) | |
| time.sleep(30) | |
| if __name__ == "__main__": | |
| main() | |