| # Launch RL detached with a real pidfile. Three traps this wraps: | |
| # | |
| # * A stale `vllm::router` from a killed run keeps ports 8000/29000, and the next launch dies | |
| # 13s in with `FailedToCreateHTTPListener("Address already in use")`, surfaced by rl.py as | |
| # "Inference failed with exit code -15" — which reads like an OOM/kill but is not. | |
| # * `pkill -f "vllm serve"` does NOT match it: the router renames its own process to | |
| # `vllm::router`, so only an exact-name match (`pkill -x`) finds it. | |
| # * Every process in an RL run **renames itself** — the launcher becomes `PRIME-RL::Launcher`, | |
| # the orchestrator `PRIME-RL::Orchestrator`, the router `vllm::router`. So `pkill -f run_rl` | |
| # matches nothing, `ps | grep orchestrator` shows nothing, and killing the PID in the pidfile | |
| # kills only the transient `setsid` wrapper. The previous run then survives invisibly, and the | |
| # next launch dies on its ports about 80 s in. | |
| # * `pkill -x "PRIME-RL::Launcher"` does not work either: `-x` matches the kernel's `comm`, | |
| # which is capped at 15 characters, so the name on the process is really `PRIME-RL::Launc`. | |
| # The sweep below therefore matches a *prefix* of `comm` via ps, and kills by PID. | |
| # * `-f` (full command line) patterns are never used here. A `-f` pattern matches any shell | |
| # whose command line quotes it — including the shell running this script, or the one that | |
| # wrote it from a heredoc — so it kills the launcher instead of the target, with no output | |
| # at all to explain why. | |
| set -uo pipefail | |
| cd /mnt/pvc/users/simon/agentptb/runs/a-opus-max/workspace | |
| CFG=${1:?usage: launch_rl.sh <config.toml>} | |
| TAG=$(basename "$CFG" .toml) | |
| # Kill every leftover process from a previous run, by comm prefix, excluding this shell. | |
| stale=$(ps -eo pid=,comm= | awk -v me=$$ '$1 != me && ($2 ~ /^(PRIME-RL|vllm|VLLM)::/ || $2 == "torchrun") {print $1}') | |
| if [ -n "$stale" ]; then | |
| echo "clearing stale: $(echo "$stale" | tr '\n' ' ')" | |
| kill -9 $stale 2>/dev/null | |
| fi | |
| sleep 8 | |
| left=$(ps -eo pid=,comm= | awk -v me=$$ '$1 != me && $2 ~ /^(PRIME-RL|vllm|VLLM)::/ {print $1}') | |
| [ -n "$left" ] && { echo "REFUSING TO LAUNCH: processes survived the sweep: $left"; exit 1; } | |
| mkdir -p logs | |
| setsid nohup bash scripts/run_rl.sh "$CFG" > "logs/${TAG}.log" 2>&1 < /dev/null & | |
| echo $! > "logs/${TAG}.pid" | |
| echo "launched $TAG pid=$(cat "logs/${TAG}.pid") log=logs/${TAG}.log" | |
| exit 0 | |