"""Supervise this experiment and terminate competing CUDA processes on GPU 2. Only compute PIDs reported on the selected GPU are eligible. PID identity is checked again before signaling, and the training process and its descendants are protected. The guard has the same OS permissions as the launching user. """ import csv import fcntl import io import json import os import pwd import signal import subprocess import time from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path ROOT = Path(__file__).resolve().parent OUTPUT = Path("/22tdata/priya/qwen_unsloth/outputs/Qwen38-27B-3shotTrained-GPU2") GPU_INDEX = 2 POLL_SECONDS = 0.5 def now(): return datetime.now(timezone.utc).isoformat() def log(event, **fields): print(json.dumps({"time": now(), "event": event, **fields}), flush=True) def atomic_json(path, value): temporary = path.with_name(path.name + ".supervisor.tmp") temporary.write_text(json.dumps(value, indent=2)) temporary.replace(path) def smi(*arguments): result = subprocess.run(["/usr/bin/nvidia-smi", *arguments], capture_output=True, text=True, timeout=5) if result.returncode: raise RuntimeError(result.stderr.strip() or result.stdout.strip()) return result.stdout def parse_compute_pids(output, gpu_uuid): return { int(row[1].strip()) for row in csv.reader(io.StringIO(output)) if len(row) >= 2 and row[0].strip() == gpu_uuid and row[1].strip().isdigit() } def compute_pids(gpu_uuid): return parse_compute_pids(smi("--query-compute-apps=gpu_uuid,pid", "--format=csv,noheader,nounits"), gpu_uuid) @dataclass(frozen=True) class ProcessIdentity: pid: int parent: int session: int start: int uid: int def identity(pid): try: proc = Path("/proc") / str(pid) fields = (proc / "stat").read_text().rsplit(")", 1)[1].split() return ProcessIdentity(pid, int(fields[1]), int(fields[3]), int(fields[19]), proc.stat().st_uid) except (FileNotFoundError, ProcessLookupError): return None def protected(process, training): if process.pid == os.getpid(): return True if training is None: return False # The job is launched in its own session. Also follow ancestry for children # that deliberately start a separate session (e.g. helper workers). if process.session == training.pid and process.uid == training.uid: return True seen = set() current = process while current and current.pid not in seen: if current.pid == training.pid and current.start == training.start: return True seen.add(current.pid) current = identity(current.parent) if current.parent > 1 else None return False def signal_checked(process, sig): # pidfd prevents PID reuse between verification and delivery on Linux. descriptor = None try: if hasattr(os, "pidfd_open") and hasattr(signal, "pidfd_send_signal"): descriptor = os.pidfd_open(process.pid) if identity(process.pid) != process: return False if descriptor is not None: signal.pidfd_send_signal(descriptor, sig) else: os.kill(process.pid, sig) return True except ProcessLookupError: return False finally: if descriptor is not None: os.close(descriptor) class GPUEnforcer: def __init__(self, gpu_uuid, grace_seconds=2): self.gpu_uuid = gpu_uuid self.grace_seconds = grace_seconds self.pending = {} self.denied = set() def poll(self, training=None): competitors, blocked = [], [] active = compute_pids(self.gpu_uuid) for pid in sorted(active): process = identity(pid) if process is None or protected(process, training): continue competitors.append(pid) try: started = self.pending.get(process) sig = signal.SIGTERM if started is None else signal.SIGKILL if started is not None and time.monotonic() - started < self.grace_seconds: continue # Reconfirm GPU attachment before every destructive action. if pid not in compute_pids(self.gpu_uuid): continue if signal_checked(process, sig): self.pending[process] = time.monotonic() try: owner = pwd.getpwuid(process.uid).pw_name except KeyError: owner = str(process.uid) log("competing_compute_process_signaled", gpu=GPU_INDEX, pid=pid, owner=owner, signal=sig.name) except PermissionError: blocked.append(pid) if process not in self.denied: self.denied.add(process) log("permission_denied", gpu=GPU_INDEX, pid=pid, uid=process.uid) self.pending = {p: started for p, started in self.pending.items() if p.pid in active} return competitors, blocked def completed_checkpoint(directory): candidates = [] for checkpoint in directory.glob("checkpoint-*"): if not checkpoint.is_dir(): continue required = ["trainer_state.json", "optimizer.pt", "scheduler.pt", "rng_state.pth", "adapter_model.safetensors", "adapter_config.json", "recovery_complete.json"] if not all((checkpoint / name).is_file() for name in required): continue try: state = json.loads((checkpoint / "trainer_state.json").read_text()) if state["global_step"] != int(checkpoint.name.split("-")[-1]): continue candidates.append((state["global_step"], checkpoint)) except (ValueError, KeyError, json.JSONDecodeError): continue return max(candidates, default=(None, None))[1] def stop_job(child): if child is None or child.poll() is not None: return os.killpg(child.pid, signal.SIGTERM) try: child.wait(timeout=10) except subprocess.TimeoutExpired: os.killpg(child.pid, signal.SIGKILL) child.wait(timeout=5) def mark_progress(phase, **fields): path = ROOT / "training_progress.json" try: progress = json.loads(path.read_text()) except (FileNotFoundError, json.JSONDecodeError): progress = {} progress.update(phase=phase, updated_at=now(), supervisor_pid=os.getpid(), **fields) if phase != "completed": for key in ("seconds_per_step", "estimated_seconds_remaining", "estimate_includes_evaluation"): progress.pop(key, None) progress["world_size"] = 1 atomic_json(path, progress) def main(): lock = (ROOT / "gpu2_guard.lock").open("a+") try: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError: raise SystemExit("A GPU 2 supervisor is already running.") gpu_uuid = smi("-i", str(GPU_INDEX), "--query-gpu=uuid", "--format=csv,noheader").strip() enforcer = GPUEnforcer(gpu_uuid) run_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") logs = ROOT / "run_logs" / run_id logs.mkdir(parents=True) OUTPUT.mkdir(parents=True, exist_ok=True) stop = False child = training = attempt_log = None batch = int(os.environ.get("TRAIN_BATCH_SIZE", "8")) if batch not in (1, 2, 4, 8, 16, 32): raise SystemExit("Microbatch size must divide the effective batch size of 32.") attempts = failures = query_errors = 0 max_retries = 3 next_launch = last_status = 0 phase = "waiting_for_gpu" def request_stop(signum, frame): nonlocal stop stop = True signal.signal(signal.SIGTERM, request_stop) signal.signal(signal.SIGINT, request_stop) log("guard_started", gpu=GPU_INDEX, gpu_uuid=gpu_uuid, uid=os.getuid(), poll_seconds=POLL_SECONDS, note="Signals GPU 2 compute competitors only; OS permissions apply. Exclusive compute mode is not enabled.") try: while not stop: if child is not None and child.poll() is not None: code = child.returncode tail = attempt_log.read_text(errors="replace")[-32768:] child = training = None if code == 0: phase = "completed" mark_progress(phase, exit_code=0) log("training_completed", output_dir=str(OUTPUT)) break failures += 1 log("training_failed", exit_code=code, attempt=attempts, log=str(attempt_log)) if failures > max_retries: phase = "failed" mark_progress(phase, exit_code=code, error_log=str(attempt_log)) break if "out of memory" in tail.lower() and batch > 1: batch //= 2 phase = "retrying" mark_progress(phase, exit_code=code, error_log=str(attempt_log)) next_launch = time.monotonic() + 10 try: competitors, blocked = enforcer.poll(training) query_errors = 0 except (RuntimeError, subprocess.TimeoutExpired) as exc: query_errors += 1 log("gpu_query_failed", error=str(exc), consecutive_errors=query_errors) if query_errors >= 3: stop_job(child) child = training = None phase = "failed" mark_progress(phase, error="GPU monitoring failed; training stopped to avoid unguarded execution.") break time.sleep(POLL_SECONDS) continue if blocked: if child is not None: stop_job(child) child = training = None phase = "waiting_for_gpu" mark_progress(phase, blocked_pids=blocked) if child is None and not competitors and time.monotonic() >= next_launch: attempts += 1 checkpoint = completed_checkpoint(OUTPUT) env = os.environ.copy() env.update(TRAIN_GPU_UUID=gpu_uuid, TRAIN_BATCH_SIZE=str(batch), TRAIN_GRAD_ACC=str(32 // batch), TRAIN_OUTPUT_DIR=str(OUTPUT), TRAIN_RECOVERY_SAVE_STEPS="10", TRAIN_RESUME_CHECKPOINT=str(checkpoint) if checkpoint else "") attempt_log = logs / f"attempt-{attempts:02d}.log" active_log = ROOT / "training.log" if active_log.exists() and not active_log.is_symlink(): active_log.rename(logs / "previous-training.log") link = ROOT / "training.log.next" link.unlink(missing_ok=True) link.symlink_to(attempt_log.relative_to(ROOT)) link.replace(active_log) mark_progress("starting", step=0 if checkpoint is None else json.loads((checkpoint / "trainer_state.json").read_text())["global_step"], pid=None, total_steps=None, resumed_from=str(checkpoint) if checkpoint else None) with attempt_log.open("w") as stream: child = subprocess.Popen(["bash", str(ROOT / "run_training_gpu2.sh")], cwd=ROOT, env=env, stdin=subprocess.DEVNULL, stdout=stream, stderr=subprocess.STDOUT, start_new_session=True) training = identity(child.pid) if training is None: raise RuntimeError("Training process exited before it could be registered with the guard.") phase = "starting" launch = {"pid": child.pid, "supervisor_pid": os.getpid(), "started_at": now(), "gpu": GPU_INDEX, "gpu_uuid": gpu_uuid, "world_size": 1, "microbatch_size": batch, "gradient_accumulation": 32 // batch, "effective_batch_size": 32, "epochs": 5, "output_dir": str(OUTPUT), "log": str(active_log), "guard_log": str(ROOT / "gpu2_guard.log"), "attempt": attempts, "resumed_from": str(checkpoint) if checkpoint else None} atomic_json(ROOT / "training_launch.json", launch) log("training_started", **launch) if time.monotonic() - last_status >= 5: if child is not None: try: current = json.loads((ROOT / "training_progress.json").read_text()) if current.get("pid") == child.pid: phase = current["phase"] except (FileNotFoundError, json.JSONDecodeError): pass atomic_json(ROOT / "gpu2_supervisor.json", {"updated_at": now(), "supervisor_pid": os.getpid(), "training_pid": child.pid if child else None, "gpu": GPU_INDEX, "gpu_uuid": gpu_uuid, "phase": phase, "attempts": attempts, "failures": failures, "blocked_pids": blocked, "poll_seconds": POLL_SECONDS, "exclusive_compute_mode": False}) last_status = time.monotonic() time.sleep(POLL_SECONDS) except Exception as exc: phase = "failed" mark_progress(phase, error=str(exc)) log("supervisor_failed", error=str(exc)) raise finally: stop_job(child) if stop: phase = "stopped" mark_progress(phase) atomic_json(ROOT / "gpu2_supervisor.json", {"updated_at": now(), "supervisor_pid": os.getpid(), "training_pid": None, "gpu": GPU_INDEX, "gpu_uuid": gpu_uuid, "phase": phase, "attempts": attempts, "failures": failures}) log("guard_stopped", phase=phase) if __name__ == "__main__": main()