Instructions to use PriyaMishra/temp with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use PriyaMishra/temp with PEFT:
Task type is invalid.
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- Unsloth Desktop
Download project/gpu2_supervisor.py from PriyaMishra/temp: direct link, hf CLI and curl.
- Browser
- Download file 14 kB
-
https://huggingface.co/PriyaMishra/temp/resolve/main/project/gpu2_supervisor.py
- Command line
-
hf download hf://PriyaMishra/temp/project/gpu2_supervisor.py
-
curl -L -o gpu2_supervisor.py https://huggingface.co/PriyaMishra/temp/resolve/main/project/gpu2_supervisor.py
14 kB
| """Supervise this experiment and terminate competing CUDA processes on GPU 2. | |
| Only compute PIDs reported on the selected GPU are eligible. PID identity is | |
| checked again before signaling, and the training process and its descendants | |
| are protected. The guard has the same OS permissions as the launching user. | |
| """ | |
| import csv | |
| import fcntl | |
| import io | |
| import json | |
| import os | |
| import pwd | |
| import signal | |
| import subprocess | |
| import time | |
| from dataclasses import dataclass | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| ROOT = Path(__file__).resolve().parent | |
| OUTPUT = Path("/22tdata/priya/qwen_unsloth/outputs/Qwen38-27B-3shotTrained-GPU2") | |
| GPU_INDEX = 2 | |
| POLL_SECONDS = 0.5 | |
| def now(): | |
| return datetime.now(timezone.utc).isoformat() | |
| def log(event, **fields): | |
| print(json.dumps({"time": now(), "event": event, **fields}), flush=True) | |
| def atomic_json(path, value): | |
| temporary = path.with_name(path.name + ".supervisor.tmp") | |
| temporary.write_text(json.dumps(value, indent=2)) | |
| temporary.replace(path) | |
| def smi(*arguments): | |
| result = subprocess.run(["/usr/bin/nvidia-smi", *arguments], capture_output=True, text=True, timeout=5) | |
| if result.returncode: | |
| raise RuntimeError(result.stderr.strip() or result.stdout.strip()) | |
| return result.stdout | |
| def parse_compute_pids(output, gpu_uuid): | |
| return { | |
| int(row[1].strip()) | |
| for row in csv.reader(io.StringIO(output)) | |
| if len(row) >= 2 and row[0].strip() == gpu_uuid and row[1].strip().isdigit() | |
| } | |
| def compute_pids(gpu_uuid): | |
| return parse_compute_pids(smi("--query-compute-apps=gpu_uuid,pid", "--format=csv,noheader,nounits"), gpu_uuid) | |
| class ProcessIdentity: | |
| pid: int | |
| parent: int | |
| session: int | |
| start: int | |
| uid: int | |
| def identity(pid): | |
| try: | |
| proc = Path("/proc") / str(pid) | |
| fields = (proc / "stat").read_text().rsplit(")", 1)[1].split() | |
| return ProcessIdentity(pid, int(fields[1]), int(fields[3]), int(fields[19]), proc.stat().st_uid) | |
| except (FileNotFoundError, ProcessLookupError): | |
| return None | |
| def protected(process, training): | |
| if process.pid == os.getpid(): | |
| return True | |
| if training is None: | |
| return False | |
| # The job is launched in its own session. Also follow ancestry for children | |
| # that deliberately start a separate session (e.g. helper workers). | |
| if process.session == training.pid and process.uid == training.uid: | |
| return True | |
| seen = set() | |
| current = process | |
| while current and current.pid not in seen: | |
| if current.pid == training.pid and current.start == training.start: | |
| return True | |
| seen.add(current.pid) | |
| current = identity(current.parent) if current.parent > 1 else None | |
| return False | |
| def signal_checked(process, sig): | |
| # pidfd prevents PID reuse between verification and delivery on Linux. | |
| descriptor = None | |
| try: | |
| if hasattr(os, "pidfd_open") and hasattr(signal, "pidfd_send_signal"): | |
| descriptor = os.pidfd_open(process.pid) | |
| if identity(process.pid) != process: | |
| return False | |
| if descriptor is not None: | |
| signal.pidfd_send_signal(descriptor, sig) | |
| else: | |
| os.kill(process.pid, sig) | |
| return True | |
| except ProcessLookupError: | |
| return False | |
| finally: | |
| if descriptor is not None: | |
| os.close(descriptor) | |
| class GPUEnforcer: | |
| def __init__(self, gpu_uuid, grace_seconds=2): | |
| self.gpu_uuid = gpu_uuid | |
| self.grace_seconds = grace_seconds | |
| self.pending = {} | |
| self.denied = set() | |
| def poll(self, training=None): | |
| competitors, blocked = [], [] | |
| active = compute_pids(self.gpu_uuid) | |
| for pid in sorted(active): | |
| process = identity(pid) | |
| if process is None or protected(process, training): | |
| continue | |
| competitors.append(pid) | |
| try: | |
| started = self.pending.get(process) | |
| sig = signal.SIGTERM if started is None else signal.SIGKILL | |
| if started is not None and time.monotonic() - started < self.grace_seconds: | |
| continue | |
| # Reconfirm GPU attachment before every destructive action. | |
| if pid not in compute_pids(self.gpu_uuid): | |
| continue | |
| if signal_checked(process, sig): | |
| self.pending[process] = time.monotonic() | |
| try: | |
| owner = pwd.getpwuid(process.uid).pw_name | |
| except KeyError: | |
| owner = str(process.uid) | |
| log("competing_compute_process_signaled", gpu=GPU_INDEX, pid=pid, owner=owner, signal=sig.name) | |
| except PermissionError: | |
| blocked.append(pid) | |
| if process not in self.denied: | |
| self.denied.add(process) | |
| log("permission_denied", gpu=GPU_INDEX, pid=pid, uid=process.uid) | |
| self.pending = {p: started for p, started in self.pending.items() if p.pid in active} | |
| return competitors, blocked | |
| def completed_checkpoint(directory): | |
| candidates = [] | |
| for checkpoint in directory.glob("checkpoint-*"): | |
| if not checkpoint.is_dir(): | |
| continue | |
| required = ["trainer_state.json", "optimizer.pt", "scheduler.pt", "rng_state.pth", "adapter_model.safetensors", "adapter_config.json", "recovery_complete.json"] | |
| if not all((checkpoint / name).is_file() for name in required): | |
| continue | |
| try: | |
| state = json.loads((checkpoint / "trainer_state.json").read_text()) | |
| if state["global_step"] != int(checkpoint.name.split("-")[-1]): | |
| continue | |
| candidates.append((state["global_step"], checkpoint)) | |
| except (ValueError, KeyError, json.JSONDecodeError): | |
| continue | |
| return max(candidates, default=(None, None))[1] | |
| def stop_job(child): | |
| if child is None or child.poll() is not None: | |
| return | |
| os.killpg(child.pid, signal.SIGTERM) | |
| try: | |
| child.wait(timeout=10) | |
| except subprocess.TimeoutExpired: | |
| os.killpg(child.pid, signal.SIGKILL) | |
| child.wait(timeout=5) | |
| def mark_progress(phase, **fields): | |
| path = ROOT / "training_progress.json" | |
| try: | |
| progress = json.loads(path.read_text()) | |
| except (FileNotFoundError, json.JSONDecodeError): | |
| progress = {} | |
| progress.update(phase=phase, updated_at=now(), supervisor_pid=os.getpid(), **fields) | |
| if phase != "completed": | |
| for key in ("seconds_per_step", "estimated_seconds_remaining", "estimate_includes_evaluation"): | |
| progress.pop(key, None) | |
| progress["world_size"] = 1 | |
| atomic_json(path, progress) | |
| def main(): | |
| lock = (ROOT / "gpu2_guard.lock").open("a+") | |
| try: | |
| fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) | |
| except BlockingIOError: | |
| raise SystemExit("A GPU 2 supervisor is already running.") | |
| gpu_uuid = smi("-i", str(GPU_INDEX), "--query-gpu=uuid", "--format=csv,noheader").strip() | |
| enforcer = GPUEnforcer(gpu_uuid) | |
| run_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") | |
| logs = ROOT / "run_logs" / run_id | |
| logs.mkdir(parents=True) | |
| OUTPUT.mkdir(parents=True, exist_ok=True) | |
| stop = False | |
| child = training = attempt_log = None | |
| batch = int(os.environ.get("TRAIN_BATCH_SIZE", "8")) | |
| if batch not in (1, 2, 4, 8, 16, 32): | |
| raise SystemExit("Microbatch size must divide the effective batch size of 32.") | |
| attempts = failures = query_errors = 0 | |
| max_retries = 3 | |
| next_launch = last_status = 0 | |
| phase = "waiting_for_gpu" | |
| def request_stop(signum, frame): | |
| nonlocal stop | |
| stop = True | |
| signal.signal(signal.SIGTERM, request_stop) | |
| signal.signal(signal.SIGINT, request_stop) | |
| log("guard_started", gpu=GPU_INDEX, gpu_uuid=gpu_uuid, uid=os.getuid(), poll_seconds=POLL_SECONDS, | |
| note="Signals GPU 2 compute competitors only; OS permissions apply. Exclusive compute mode is not enabled.") | |
| try: | |
| while not stop: | |
| if child is not None and child.poll() is not None: | |
| code = child.returncode | |
| tail = attempt_log.read_text(errors="replace")[-32768:] | |
| child = training = None | |
| if code == 0: | |
| phase = "completed" | |
| mark_progress(phase, exit_code=0) | |
| log("training_completed", output_dir=str(OUTPUT)) | |
| break | |
| failures += 1 | |
| log("training_failed", exit_code=code, attempt=attempts, log=str(attempt_log)) | |
| if failures > max_retries: | |
| phase = "failed" | |
| mark_progress(phase, exit_code=code, error_log=str(attempt_log)) | |
| break | |
| if "out of memory" in tail.lower() and batch > 1: | |
| batch //= 2 | |
| phase = "retrying" | |
| mark_progress(phase, exit_code=code, error_log=str(attempt_log)) | |
| next_launch = time.monotonic() + 10 | |
| try: | |
| competitors, blocked = enforcer.poll(training) | |
| query_errors = 0 | |
| except (RuntimeError, subprocess.TimeoutExpired) as exc: | |
| query_errors += 1 | |
| log("gpu_query_failed", error=str(exc), consecutive_errors=query_errors) | |
| if query_errors >= 3: | |
| stop_job(child) | |
| child = training = None | |
| phase = "failed" | |
| mark_progress(phase, error="GPU monitoring failed; training stopped to avoid unguarded execution.") | |
| break | |
| time.sleep(POLL_SECONDS) | |
| continue | |
| if blocked: | |
| if child is not None: | |
| stop_job(child) | |
| child = training = None | |
| phase = "waiting_for_gpu" | |
| mark_progress(phase, blocked_pids=blocked) | |
| if child is None and not competitors and time.monotonic() >= next_launch: | |
| attempts += 1 | |
| checkpoint = completed_checkpoint(OUTPUT) | |
| env = os.environ.copy() | |
| env.update(TRAIN_GPU_UUID=gpu_uuid, TRAIN_BATCH_SIZE=str(batch), TRAIN_GRAD_ACC=str(32 // batch), | |
| TRAIN_OUTPUT_DIR=str(OUTPUT), TRAIN_RECOVERY_SAVE_STEPS="10", | |
| TRAIN_RESUME_CHECKPOINT=str(checkpoint) if checkpoint else "") | |
| attempt_log = logs / f"attempt-{attempts:02d}.log" | |
| active_log = ROOT / "training.log" | |
| if active_log.exists() and not active_log.is_symlink(): | |
| active_log.rename(logs / "previous-training.log") | |
| link = ROOT / "training.log.next" | |
| link.unlink(missing_ok=True) | |
| link.symlink_to(attempt_log.relative_to(ROOT)) | |
| link.replace(active_log) | |
| mark_progress("starting", step=0 if checkpoint is None else json.loads((checkpoint / "trainer_state.json").read_text())["global_step"], | |
| pid=None, total_steps=None, resumed_from=str(checkpoint) if checkpoint else None) | |
| with attempt_log.open("w") as stream: | |
| child = subprocess.Popen(["bash", str(ROOT / "run_training_gpu2.sh")], cwd=ROOT, env=env, | |
| stdin=subprocess.DEVNULL, stdout=stream, stderr=subprocess.STDOUT, start_new_session=True) | |
| training = identity(child.pid) | |
| if training is None: | |
| raise RuntimeError("Training process exited before it could be registered with the guard.") | |
| phase = "starting" | |
| launch = {"pid": child.pid, "supervisor_pid": os.getpid(), "started_at": now(), "gpu": GPU_INDEX, | |
| "gpu_uuid": gpu_uuid, "world_size": 1, "microbatch_size": batch, "gradient_accumulation": 32 // batch, | |
| "effective_batch_size": 32, "epochs": 5, "output_dir": str(OUTPUT), "log": str(active_log), | |
| "guard_log": str(ROOT / "gpu2_guard.log"), "attempt": attempts, "resumed_from": str(checkpoint) if checkpoint else None} | |
| atomic_json(ROOT / "training_launch.json", launch) | |
| log("training_started", **launch) | |
| if time.monotonic() - last_status >= 5: | |
| if child is not None: | |
| try: | |
| current = json.loads((ROOT / "training_progress.json").read_text()) | |
| if current.get("pid") == child.pid: | |
| phase = current["phase"] | |
| except (FileNotFoundError, json.JSONDecodeError): | |
| pass | |
| atomic_json(ROOT / "gpu2_supervisor.json", {"updated_at": now(), "supervisor_pid": os.getpid(), | |
| "training_pid": child.pid if child else None, "gpu": GPU_INDEX, "gpu_uuid": gpu_uuid, | |
| "phase": phase, "attempts": attempts, "failures": failures, "blocked_pids": blocked, | |
| "poll_seconds": POLL_SECONDS, "exclusive_compute_mode": False}) | |
| last_status = time.monotonic() | |
| time.sleep(POLL_SECONDS) | |
| except Exception as exc: | |
| phase = "failed" | |
| mark_progress(phase, error=str(exc)) | |
| log("supervisor_failed", error=str(exc)) | |
| raise | |
| finally: | |
| stop_job(child) | |
| if stop: | |
| phase = "stopped" | |
| mark_progress(phase) | |
| atomic_json(ROOT / "gpu2_supervisor.json", {"updated_at": now(), "supervisor_pid": os.getpid(), "training_pid": None, | |
| "gpu": GPU_INDEX, "gpu_uuid": gpu_uuid, "phase": phase, "attempts": attempts, "failures": failures}) | |
| log("guard_stopped", phase=phase) | |
| if __name__ == "__main__": | |
| main() | |