| """Kill all GPU processes. Run before evals to ensure clean state.""" | |
| import subprocess | |
| import sys | |
| result = subprocess.run( | |
| ["nvidia-smi", "--query-compute-apps=pid", "--format=csv,noheader"], | |
| capture_output=True, text=True, | |
| ) | |
| pids = [p.strip() for p in result.stdout.strip().split("\n") if p.strip()] | |
| if not pids: | |
| print("No GPU processes running.") | |
| sys.exit(0) | |
| print(f"Killing {len(pids)} GPU process(es): {pids}") | |
| for pid in pids: | |
| subprocess.run(["kill", "-9", pid]) | |
| print(f" Killed PID {pid}") | |
| print("Done. GPUs cleared.") | |