#!/bin/bash
# Launch RL detached with a real pidfile. Three traps this wraps:
#
#  * A stale `vllm::router` from a killed run keeps ports 8000/29000, and the next launch dies
#    13s in with `FailedToCreateHTTPListener("Address already in use")`, surfaced by rl.py as
#    "Inference failed with exit code -15" — which reads like an OOM/kill but is not.
#  * `pkill -f "vllm serve"` does NOT match it: the router renames its own process to
#    `vllm::router`, so only an exact-name match (`pkill -x`) finds it.
#  * Every process in an RL run **renames itself** — the launcher becomes `PRIME-RL::Launcher`,
#    the orchestrator `PRIME-RL::Orchestrator`, the router `vllm::router`. So `pkill -f run_rl`
#    matches nothing, `ps | grep orchestrator` shows nothing, and killing the PID in the pidfile
#    kills only the transient `setsid` wrapper. The previous run then survives invisibly, and the
#    next launch dies on its ports about 80 s in.
#  * `pkill -x "PRIME-RL::Launcher"` does not work either: `-x` matches the kernel's `comm`,
#    which is capped at 15 characters, so the name on the process is really `PRIME-RL::Launc`.
#    The sweep below therefore matches a *prefix* of `comm` via ps, and kills by PID.
#  * `-f` (full command line) patterns are never used here. A `-f` pattern matches any shell
#    whose command line quotes it — including the shell running this script, or the one that
#    wrote it from a heredoc — so it kills the launcher instead of the target, with no output
#    at all to explain why.
set -uo pipefail
cd /mnt/pvc/users/simon/agentptb/runs/a-opus-max/workspace
CFG=${1:?usage: launch_rl.sh <config.toml>}
TAG=$(basename "$CFG" .toml)

# Kill every leftover process from a previous run, by comm prefix, excluding this shell.
stale=$(ps -eo pid=,comm= | awk -v me=$$ '$1 != me && ($2 ~ /^(PRIME-RL|vllm|VLLM)::/ || $2 == "torchrun") {print $1}')
if [ -n "$stale" ]; then
  echo "clearing stale: $(echo "$stale" | tr '\n' ' ')"
  kill -9 $stale 2>/dev/null
fi
sleep 8
left=$(ps -eo pid=,comm= | awk -v me=$$ '$1 != me && $2 ~ /^(PRIME-RL|vllm|VLLM)::/ {print $1}')
[ -n "$left" ] && { echo "REFUSING TO LAUNCH: processes survived the sweep: $left"; exit 1; }
mkdir -p logs
setsid nohup bash scripts/run_rl.sh "$CFG" > "logs/${TAG}.log" 2>&1 < /dev/null &
echo $! > "logs/${TAG}.pid"
echo "launched $TAG pid=$(cat "logs/${TAG}.pid") log=logs/${TAG}.log"
exit 0