Instructions to use PriyaMishra/temp with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use PriyaMishra/temp with PEFT:
Task type is invalid.
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- Unsloth Desktop
File size: 13,957 Bytes
f5d6a7c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 | """Supervise this experiment and terminate competing CUDA processes on GPU 2.
Only compute PIDs reported on the selected GPU are eligible. PID identity is
checked again before signaling, and the training process and its descendants
are protected. The guard has the same OS permissions as the launching user.
"""
import csv
import fcntl
import io
import json
import os
import pwd
import signal
import subprocess
import time
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parent
OUTPUT = Path("/22tdata/priya/qwen_unsloth/outputs/Qwen38-27B-3shotTrained-GPU2")
GPU_INDEX = 2
POLL_SECONDS = 0.5
def now():
return datetime.now(timezone.utc).isoformat()
def log(event, **fields):
print(json.dumps({"time": now(), "event": event, **fields}), flush=True)
def atomic_json(path, value):
temporary = path.with_name(path.name + ".supervisor.tmp")
temporary.write_text(json.dumps(value, indent=2))
temporary.replace(path)
def smi(*arguments):
result = subprocess.run(["/usr/bin/nvidia-smi", *arguments], capture_output=True, text=True, timeout=5)
if result.returncode:
raise RuntimeError(result.stderr.strip() or result.stdout.strip())
return result.stdout
def parse_compute_pids(output, gpu_uuid):
return {
int(row[1].strip())
for row in csv.reader(io.StringIO(output))
if len(row) >= 2 and row[0].strip() == gpu_uuid and row[1].strip().isdigit()
}
def compute_pids(gpu_uuid):
return parse_compute_pids(smi("--query-compute-apps=gpu_uuid,pid", "--format=csv,noheader,nounits"), gpu_uuid)
@dataclass(frozen=True)
class ProcessIdentity:
pid: int
parent: int
session: int
start: int
uid: int
def identity(pid):
try:
proc = Path("/proc") / str(pid)
fields = (proc / "stat").read_text().rsplit(")", 1)[1].split()
return ProcessIdentity(pid, int(fields[1]), int(fields[3]), int(fields[19]), proc.stat().st_uid)
except (FileNotFoundError, ProcessLookupError):
return None
def protected(process, training):
if process.pid == os.getpid():
return True
if training is None:
return False
# The job is launched in its own session. Also follow ancestry for children
# that deliberately start a separate session (e.g. helper workers).
if process.session == training.pid and process.uid == training.uid:
return True
seen = set()
current = process
while current and current.pid not in seen:
if current.pid == training.pid and current.start == training.start:
return True
seen.add(current.pid)
current = identity(current.parent) if current.parent > 1 else None
return False
def signal_checked(process, sig):
# pidfd prevents PID reuse between verification and delivery on Linux.
descriptor = None
try:
if hasattr(os, "pidfd_open") and hasattr(signal, "pidfd_send_signal"):
descriptor = os.pidfd_open(process.pid)
if identity(process.pid) != process:
return False
if descriptor is not None:
signal.pidfd_send_signal(descriptor, sig)
else:
os.kill(process.pid, sig)
return True
except ProcessLookupError:
return False
finally:
if descriptor is not None:
os.close(descriptor)
class GPUEnforcer:
def __init__(self, gpu_uuid, grace_seconds=2):
self.gpu_uuid = gpu_uuid
self.grace_seconds = grace_seconds
self.pending = {}
self.denied = set()
def poll(self, training=None):
competitors, blocked = [], []
active = compute_pids(self.gpu_uuid)
for pid in sorted(active):
process = identity(pid)
if process is None or protected(process, training):
continue
competitors.append(pid)
try:
started = self.pending.get(process)
sig = signal.SIGTERM if started is None else signal.SIGKILL
if started is not None and time.monotonic() - started < self.grace_seconds:
continue
# Reconfirm GPU attachment before every destructive action.
if pid not in compute_pids(self.gpu_uuid):
continue
if signal_checked(process, sig):
self.pending[process] = time.monotonic()
try:
owner = pwd.getpwuid(process.uid).pw_name
except KeyError:
owner = str(process.uid)
log("competing_compute_process_signaled", gpu=GPU_INDEX, pid=pid, owner=owner, signal=sig.name)
except PermissionError:
blocked.append(pid)
if process not in self.denied:
self.denied.add(process)
log("permission_denied", gpu=GPU_INDEX, pid=pid, uid=process.uid)
self.pending = {p: started for p, started in self.pending.items() if p.pid in active}
return competitors, blocked
def completed_checkpoint(directory):
candidates = []
for checkpoint in directory.glob("checkpoint-*"):
if not checkpoint.is_dir():
continue
required = ["trainer_state.json", "optimizer.pt", "scheduler.pt", "rng_state.pth", "adapter_model.safetensors", "adapter_config.json", "recovery_complete.json"]
if not all((checkpoint / name).is_file() for name in required):
continue
try:
state = json.loads((checkpoint / "trainer_state.json").read_text())
if state["global_step"] != int(checkpoint.name.split("-")[-1]):
continue
candidates.append((state["global_step"], checkpoint))
except (ValueError, KeyError, json.JSONDecodeError):
continue
return max(candidates, default=(None, None))[1]
def stop_job(child):
if child is None or child.poll() is not None:
return
os.killpg(child.pid, signal.SIGTERM)
try:
child.wait(timeout=10)
except subprocess.TimeoutExpired:
os.killpg(child.pid, signal.SIGKILL)
child.wait(timeout=5)
def mark_progress(phase, **fields):
path = ROOT / "training_progress.json"
try:
progress = json.loads(path.read_text())
except (FileNotFoundError, json.JSONDecodeError):
progress = {}
progress.update(phase=phase, updated_at=now(), supervisor_pid=os.getpid(), **fields)
if phase != "completed":
for key in ("seconds_per_step", "estimated_seconds_remaining", "estimate_includes_evaluation"):
progress.pop(key, None)
progress["world_size"] = 1
atomic_json(path, progress)
def main():
lock = (ROOT / "gpu2_guard.lock").open("a+")
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
raise SystemExit("A GPU 2 supervisor is already running.")
gpu_uuid = smi("-i", str(GPU_INDEX), "--query-gpu=uuid", "--format=csv,noheader").strip()
enforcer = GPUEnforcer(gpu_uuid)
run_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
logs = ROOT / "run_logs" / run_id
logs.mkdir(parents=True)
OUTPUT.mkdir(parents=True, exist_ok=True)
stop = False
child = training = attempt_log = None
batch = int(os.environ.get("TRAIN_BATCH_SIZE", "8"))
if batch not in (1, 2, 4, 8, 16, 32):
raise SystemExit("Microbatch size must divide the effective batch size of 32.")
attempts = failures = query_errors = 0
max_retries = 3
next_launch = last_status = 0
phase = "waiting_for_gpu"
def request_stop(signum, frame):
nonlocal stop
stop = True
signal.signal(signal.SIGTERM, request_stop)
signal.signal(signal.SIGINT, request_stop)
log("guard_started", gpu=GPU_INDEX, gpu_uuid=gpu_uuid, uid=os.getuid(), poll_seconds=POLL_SECONDS,
note="Signals GPU 2 compute competitors only; OS permissions apply. Exclusive compute mode is not enabled.")
try:
while not stop:
if child is not None and child.poll() is not None:
code = child.returncode
tail = attempt_log.read_text(errors="replace")[-32768:]
child = training = None
if code == 0:
phase = "completed"
mark_progress(phase, exit_code=0)
log("training_completed", output_dir=str(OUTPUT))
break
failures += 1
log("training_failed", exit_code=code, attempt=attempts, log=str(attempt_log))
if failures > max_retries:
phase = "failed"
mark_progress(phase, exit_code=code, error_log=str(attempt_log))
break
if "out of memory" in tail.lower() and batch > 1:
batch //= 2
phase = "retrying"
mark_progress(phase, exit_code=code, error_log=str(attempt_log))
next_launch = time.monotonic() + 10
try:
competitors, blocked = enforcer.poll(training)
query_errors = 0
except (RuntimeError, subprocess.TimeoutExpired) as exc:
query_errors += 1
log("gpu_query_failed", error=str(exc), consecutive_errors=query_errors)
if query_errors >= 3:
stop_job(child)
child = training = None
phase = "failed"
mark_progress(phase, error="GPU monitoring failed; training stopped to avoid unguarded execution.")
break
time.sleep(POLL_SECONDS)
continue
if blocked:
if child is not None:
stop_job(child)
child = training = None
phase = "waiting_for_gpu"
mark_progress(phase, blocked_pids=blocked)
if child is None and not competitors and time.monotonic() >= next_launch:
attempts += 1
checkpoint = completed_checkpoint(OUTPUT)
env = os.environ.copy()
env.update(TRAIN_GPU_UUID=gpu_uuid, TRAIN_BATCH_SIZE=str(batch), TRAIN_GRAD_ACC=str(32 // batch),
TRAIN_OUTPUT_DIR=str(OUTPUT), TRAIN_RECOVERY_SAVE_STEPS="10",
TRAIN_RESUME_CHECKPOINT=str(checkpoint) if checkpoint else "")
attempt_log = logs / f"attempt-{attempts:02d}.log"
active_log = ROOT / "training.log"
if active_log.exists() and not active_log.is_symlink():
active_log.rename(logs / "previous-training.log")
link = ROOT / "training.log.next"
link.unlink(missing_ok=True)
link.symlink_to(attempt_log.relative_to(ROOT))
link.replace(active_log)
mark_progress("starting", step=0 if checkpoint is None else json.loads((checkpoint / "trainer_state.json").read_text())["global_step"],
pid=None, total_steps=None, resumed_from=str(checkpoint) if checkpoint else None)
with attempt_log.open("w") as stream:
child = subprocess.Popen(["bash", str(ROOT / "run_training_gpu2.sh")], cwd=ROOT, env=env,
stdin=subprocess.DEVNULL, stdout=stream, stderr=subprocess.STDOUT, start_new_session=True)
training = identity(child.pid)
if training is None:
raise RuntimeError("Training process exited before it could be registered with the guard.")
phase = "starting"
launch = {"pid": child.pid, "supervisor_pid": os.getpid(), "started_at": now(), "gpu": GPU_INDEX,
"gpu_uuid": gpu_uuid, "world_size": 1, "microbatch_size": batch, "gradient_accumulation": 32 // batch,
"effective_batch_size": 32, "epochs": 5, "output_dir": str(OUTPUT), "log": str(active_log),
"guard_log": str(ROOT / "gpu2_guard.log"), "attempt": attempts, "resumed_from": str(checkpoint) if checkpoint else None}
atomic_json(ROOT / "training_launch.json", launch)
log("training_started", **launch)
if time.monotonic() - last_status >= 5:
if child is not None:
try:
current = json.loads((ROOT / "training_progress.json").read_text())
if current.get("pid") == child.pid:
phase = current["phase"]
except (FileNotFoundError, json.JSONDecodeError):
pass
atomic_json(ROOT / "gpu2_supervisor.json", {"updated_at": now(), "supervisor_pid": os.getpid(),
"training_pid": child.pid if child else None, "gpu": GPU_INDEX, "gpu_uuid": gpu_uuid,
"phase": phase, "attempts": attempts, "failures": failures, "blocked_pids": blocked,
"poll_seconds": POLL_SECONDS, "exclusive_compute_mode": False})
last_status = time.monotonic()
time.sleep(POLL_SECONDS)
except Exception as exc:
phase = "failed"
mark_progress(phase, error=str(exc))
log("supervisor_failed", error=str(exc))
raise
finally:
stop_job(child)
if stop:
phase = "stopped"
mark_progress(phase)
atomic_json(ROOT / "gpu2_supervisor.json", {"updated_at": now(), "supervisor_pid": os.getpid(), "training_pid": None,
"gpu": GPU_INDEX, "gpu_uuid": gpu_uuid, "phase": phase, "attempts": attempts, "failures": failures})
log("guard_stopped", phase=phase)
if __name__ == "__main__":
main()
|