temp / project /gpu2_supervisor.py
PriyaMishra's picture
Copy model from AjayRangoji/temp
f5d6a7c verified
Raw History Blame Contribute Delete
14 kB
"""Supervise this experiment and terminate competing CUDA processes on GPU 2.
Only compute PIDs reported on the selected GPU are eligible. PID identity is
checked again before signaling, and the training process and its descendants
are protected. The guard has the same OS permissions as the launching user.
"""
import csv
import fcntl
import io
import json
import os
import pwd
import signal
import subprocess
import time
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parent
OUTPUT = Path("/22tdata/priya/qwen_unsloth/outputs/Qwen38-27B-3shotTrained-GPU2")
GPU_INDEX = 2
POLL_SECONDS = 0.5
def now():
return datetime.now(timezone.utc).isoformat()
def log(event, **fields):
print(json.dumps({"time": now(), "event": event, **fields}), flush=True)
def atomic_json(path, value):
temporary = path.with_name(path.name + ".supervisor.tmp")
temporary.write_text(json.dumps(value, indent=2))
temporary.replace(path)
def smi(*arguments):
result = subprocess.run(["/usr/bin/nvidia-smi", *arguments], capture_output=True, text=True, timeout=5)
if result.returncode:
raise RuntimeError(result.stderr.strip() or result.stdout.strip())
return result.stdout
def parse_compute_pids(output, gpu_uuid):
return {
int(row[1].strip())
for row in csv.reader(io.StringIO(output))
if len(row) >= 2 and row[0].strip() == gpu_uuid and row[1].strip().isdigit()
}
def compute_pids(gpu_uuid):
return parse_compute_pids(smi("--query-compute-apps=gpu_uuid,pid", "--format=csv,noheader,nounits"), gpu_uuid)
@dataclass(frozen=True)
class ProcessIdentity:
pid: int
parent: int
session: int
start: int
uid: int
def identity(pid):
try:
proc = Path("/proc") / str(pid)
fields = (proc / "stat").read_text().rsplit(")", 1)[1].split()
return ProcessIdentity(pid, int(fields[1]), int(fields[3]), int(fields[19]), proc.stat().st_uid)
except (FileNotFoundError, ProcessLookupError):
return None
def protected(process, training):
if process.pid == os.getpid():
return True
if training is None:
return False
# The job is launched in its own session. Also follow ancestry for children
# that deliberately start a separate session (e.g. helper workers).
if process.session == training.pid and process.uid == training.uid:
return True
seen = set()
current = process
while current and current.pid not in seen:
if current.pid == training.pid and current.start == training.start:
return True
seen.add(current.pid)
current = identity(current.parent) if current.parent > 1 else None
return False
def signal_checked(process, sig):
# pidfd prevents PID reuse between verification and delivery on Linux.
descriptor = None
try:
if hasattr(os, "pidfd_open") and hasattr(signal, "pidfd_send_signal"):
descriptor = os.pidfd_open(process.pid)
if identity(process.pid) != process:
return False
if descriptor is not None:
signal.pidfd_send_signal(descriptor, sig)
else:
os.kill(process.pid, sig)
return True
except ProcessLookupError:
return False
finally:
if descriptor is not None:
os.close(descriptor)
class GPUEnforcer:
def __init__(self, gpu_uuid, grace_seconds=2):
self.gpu_uuid = gpu_uuid
self.grace_seconds = grace_seconds
self.pending = {}
self.denied = set()
def poll(self, training=None):
competitors, blocked = [], []
active = compute_pids(self.gpu_uuid)
for pid in sorted(active):
process = identity(pid)
if process is None or protected(process, training):
continue
competitors.append(pid)
try:
started = self.pending.get(process)
sig = signal.SIGTERM if started is None else signal.SIGKILL
if started is not None and time.monotonic() - started < self.grace_seconds:
continue
# Reconfirm GPU attachment before every destructive action.
if pid not in compute_pids(self.gpu_uuid):
continue
if signal_checked(process, sig):
self.pending[process] = time.monotonic()
try:
owner = pwd.getpwuid(process.uid).pw_name
except KeyError:
owner = str(process.uid)
log("competing_compute_process_signaled", gpu=GPU_INDEX, pid=pid, owner=owner, signal=sig.name)
except PermissionError:
blocked.append(pid)
if process not in self.denied:
self.denied.add(process)
log("permission_denied", gpu=GPU_INDEX, pid=pid, uid=process.uid)
self.pending = {p: started for p, started in self.pending.items() if p.pid in active}
return competitors, blocked
def completed_checkpoint(directory):
candidates = []
for checkpoint in directory.glob("checkpoint-*"):
if not checkpoint.is_dir():
continue
required = ["trainer_state.json", "optimizer.pt", "scheduler.pt", "rng_state.pth", "adapter_model.safetensors", "adapter_config.json", "recovery_complete.json"]
if not all((checkpoint / name).is_file() for name in required):
continue
try:
state = json.loads((checkpoint / "trainer_state.json").read_text())
if state["global_step"] != int(checkpoint.name.split("-")[-1]):
continue
candidates.append((state["global_step"], checkpoint))
except (ValueError, KeyError, json.JSONDecodeError):
continue
return max(candidates, default=(None, None))[1]
def stop_job(child):
if child is None or child.poll() is not None:
return
os.killpg(child.pid, signal.SIGTERM)
try:
child.wait(timeout=10)
except subprocess.TimeoutExpired:
os.killpg(child.pid, signal.SIGKILL)
child.wait(timeout=5)
def mark_progress(phase, **fields):
path = ROOT / "training_progress.json"
try:
progress = json.loads(path.read_text())
except (FileNotFoundError, json.JSONDecodeError):
progress = {}
progress.update(phase=phase, updated_at=now(), supervisor_pid=os.getpid(), **fields)
if phase != "completed":
for key in ("seconds_per_step", "estimated_seconds_remaining", "estimate_includes_evaluation"):
progress.pop(key, None)
progress["world_size"] = 1
atomic_json(path, progress)
def main():
lock = (ROOT / "gpu2_guard.lock").open("a+")
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
raise SystemExit("A GPU 2 supervisor is already running.")
gpu_uuid = smi("-i", str(GPU_INDEX), "--query-gpu=uuid", "--format=csv,noheader").strip()
enforcer = GPUEnforcer(gpu_uuid)
run_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
logs = ROOT / "run_logs" / run_id
logs.mkdir(parents=True)
OUTPUT.mkdir(parents=True, exist_ok=True)
stop = False
child = training = attempt_log = None
batch = int(os.environ.get("TRAIN_BATCH_SIZE", "8"))
if batch not in (1, 2, 4, 8, 16, 32):
raise SystemExit("Microbatch size must divide the effective batch size of 32.")
attempts = failures = query_errors = 0
max_retries = 3
next_launch = last_status = 0
phase = "waiting_for_gpu"
def request_stop(signum, frame):
nonlocal stop
stop = True
signal.signal(signal.SIGTERM, request_stop)
signal.signal(signal.SIGINT, request_stop)
log("guard_started", gpu=GPU_INDEX, gpu_uuid=gpu_uuid, uid=os.getuid(), poll_seconds=POLL_SECONDS,
note="Signals GPU 2 compute competitors only; OS permissions apply. Exclusive compute mode is not enabled.")
try:
while not stop:
if child is not None and child.poll() is not None:
code = child.returncode
tail = attempt_log.read_text(errors="replace")[-32768:]
child = training = None
if code == 0:
phase = "completed"
mark_progress(phase, exit_code=0)
log("training_completed", output_dir=str(OUTPUT))
break
failures += 1
log("training_failed", exit_code=code, attempt=attempts, log=str(attempt_log))
if failures > max_retries:
phase = "failed"
mark_progress(phase, exit_code=code, error_log=str(attempt_log))
break
if "out of memory" in tail.lower() and batch > 1:
batch //= 2
phase = "retrying"
mark_progress(phase, exit_code=code, error_log=str(attempt_log))
next_launch = time.monotonic() + 10
try:
competitors, blocked = enforcer.poll(training)
query_errors = 0
except (RuntimeError, subprocess.TimeoutExpired) as exc:
query_errors += 1
log("gpu_query_failed", error=str(exc), consecutive_errors=query_errors)
if query_errors >= 3:
stop_job(child)
child = training = None
phase = "failed"
mark_progress(phase, error="GPU monitoring failed; training stopped to avoid unguarded execution.")
break
time.sleep(POLL_SECONDS)
continue
if blocked:
if child is not None:
stop_job(child)
child = training = None
phase = "waiting_for_gpu"
mark_progress(phase, blocked_pids=blocked)
if child is None and not competitors and time.monotonic() >= next_launch:
attempts += 1
checkpoint = completed_checkpoint(OUTPUT)
env = os.environ.copy()
env.update(TRAIN_GPU_UUID=gpu_uuid, TRAIN_BATCH_SIZE=str(batch), TRAIN_GRAD_ACC=str(32 // batch),
TRAIN_OUTPUT_DIR=str(OUTPUT), TRAIN_RECOVERY_SAVE_STEPS="10",
TRAIN_RESUME_CHECKPOINT=str(checkpoint) if checkpoint else "")
attempt_log = logs / f"attempt-{attempts:02d}.log"
active_log = ROOT / "training.log"
if active_log.exists() and not active_log.is_symlink():
active_log.rename(logs / "previous-training.log")
link = ROOT / "training.log.next"
link.unlink(missing_ok=True)
link.symlink_to(attempt_log.relative_to(ROOT))
link.replace(active_log)
mark_progress("starting", step=0 if checkpoint is None else json.loads((checkpoint / "trainer_state.json").read_text())["global_step"],
pid=None, total_steps=None, resumed_from=str(checkpoint) if checkpoint else None)
with attempt_log.open("w") as stream:
child = subprocess.Popen(["bash", str(ROOT / "run_training_gpu2.sh")], cwd=ROOT, env=env,
stdin=subprocess.DEVNULL, stdout=stream, stderr=subprocess.STDOUT, start_new_session=True)
training = identity(child.pid)
if training is None:
raise RuntimeError("Training process exited before it could be registered with the guard.")
phase = "starting"
launch = {"pid": child.pid, "supervisor_pid": os.getpid(), "started_at": now(), "gpu": GPU_INDEX,
"gpu_uuid": gpu_uuid, "world_size": 1, "microbatch_size": batch, "gradient_accumulation": 32 // batch,
"effective_batch_size": 32, "epochs": 5, "output_dir": str(OUTPUT), "log": str(active_log),
"guard_log": str(ROOT / "gpu2_guard.log"), "attempt": attempts, "resumed_from": str(checkpoint) if checkpoint else None}
atomic_json(ROOT / "training_launch.json", launch)
log("training_started", **launch)
if time.monotonic() - last_status >= 5:
if child is not None:
try:
current = json.loads((ROOT / "training_progress.json").read_text())
if current.get("pid") == child.pid:
phase = current["phase"]
except (FileNotFoundError, json.JSONDecodeError):
pass
atomic_json(ROOT / "gpu2_supervisor.json", {"updated_at": now(), "supervisor_pid": os.getpid(),
"training_pid": child.pid if child else None, "gpu": GPU_INDEX, "gpu_uuid": gpu_uuid,
"phase": phase, "attempts": attempts, "failures": failures, "blocked_pids": blocked,
"poll_seconds": POLL_SECONDS, "exclusive_compute_mode": False})
last_status = time.monotonic()
time.sleep(POLL_SECONDS)
except Exception as exc:
phase = "failed"
mark_progress(phase, error=str(exc))
log("supervisor_failed", error=str(exc))
raise
finally:
stop_job(child)
if stop:
phase = "stopped"
mark_progress(phase)
atomic_json(ROOT / "gpu2_supervisor.json", {"updated_at": now(), "supervisor_pid": os.getpid(), "training_pid": None,
"gpu": GPU_INDEX, "gpu_uuid": gpu_uuid, "phase": phase, "attempts": attempts, "failures": failures})
log("guard_stopped", phase=phase)
if __name__ == "__main__":
main()