#!/bin/bash
# topup_loop.sh <target_graded> <run_dir> [run_dir...]
# Repeatedly resume each run until it has <target_graded> graded episodes.
#
# Why a loop rather than one long run: the shared sandbox pool alternates between bursts and long
# stalls (finding 22). A single pass leaves episodes waiting out their readiness budget, they
# expire as SandboxError, and the run "finishes" incomplete. Re-resuming on a cadence turns those
# into fresh sandbox requests, which is what actually gets served when a burst arrives.
#
# TWO HARD CONSTRAINTS, both learned the expensive way (finding 24):
#
#  1. **Never SIGKILL an eval run.** Each in-flight episode holds a host tunnel, and the account is
#     capped at 32 concurrent tunnels. `PrimeTunnel.expose` releases its tunnel in a shielded
#     `finally`, so SIGTERM cleans up — SIGKILL does not. Kill -9 a run with 24 episodes in flight
#     and you leak 24 tunnels; a few passes and all 32 slots are gone, after which every episode
#     dies instantly with `TunnelError: Maximum number of tunnels (32) reached`. That looks exactly
#     like a sandbox-pool problem and is not one — and it is invisible in `/resources`.
#  2. **Total concurrency across ALL running eval processes must stay under 32.** Not per run —
#     per API token. Two arms at 24 each is 48 and cannot work however healthy the pool looks.
#
# After stopping the runs each pass, the loop reclaims any stray tunnels. That is safe only
# because it has just stopped every run it started; do not run it alongside an unrelated eval.
WS=/mnt/pvc/users/simon/agentptb/runs/a-opus-max/workspace
cd "$WS" || exit 1
export PRIME_API_KEY="$(cat "$AGENTPTB_PRIME_KEY_FILE")"
PY=/root/work/a/prime-rl/.venv/bin/python
TARGET=${1:?target graded}; shift

stop_runs() {
  local pids
  pids=$(ps -eo pid=,args= | awk '/eval --resume/ && !/awk/ {print $1}')
  if [ -n "$pids" ]; then
    kill -TERM $pids 2>/dev/null          # SIGTERM so each episode releases its tunnel
    for _ in $(seq 1 20); do
      sleep 3
      ps -eo pid=,args= | grep -q "[e]val --resume" || break
    done
    pids=$(ps -eo pid=,args= | awk '/eval --resume/ && !/awk/ {print $1}')
    [ -n "$pids" ] && kill -9 $pids 2>/dev/null
    sleep 5
  fi
  "$PY" scripts/tunnels.py --delete 2>&1 | tail -1
}

for pass in $(seq 1 200); do
  alldone=1
  for d in "$@"; do
    g=$("$PY" scripts/graded.py -n "$d")
    echo "$(date -Is) pass=$pass $(basename "$d") graded=$g/$TARGET"
    if [ "${g:-0}" -lt "$TARGET" ]; then
      alldone=0
      setsid nohup bash scripts/resume_eval.sh "$WS/$d" > "logs/$(basename "$d")_resume.log" 2>&1 < /dev/null &
    fi
  done
  [ "$alldone" = "1" ] && { echo "$(date -Is) ALL RUNS AT TARGET"; exit 0; }
  sleep 900
  stop_runs
done