harness/scripts/finalize_eval.sh

#!/bin/bash
# Final confirmation evals for the submission checkpoint: tb2 (89) then swe (100), one server.
set -u
WS=/mnt/pvc/users/simon/agentptb/runs/d/workspace
CKPT=$1
TAG=$2
PORT=8500
cd /root/work/a/prime-rl
export PRIME_API_KEY="$(cat $WS/.prime-api-key)"

nohup uv run inference --model.name "$CKPT" \
  --model.tool-call-parser qwen3_coder --model.reasoning-parser qwen3 \
  --server.port $PORT --parallel.tp 1 --parallel.dp 1 \
  --vllm-extra '{"limit_mm_per_prompt": {"image": 0, "video": 0}}' \
  > $WS/logs/inference-final-$TAG.log 2>&1 &
SPID=$!

ok=0
for i in $(seq 1 60); do
  curl -sf http://localhost:$PORT/health >/dev/null 2>&1 && { ok=1; break; }
  sleep 10
done
[ $ok -eq 1 ] || { echo "SERVER FAILED"; tail -5 $WS/logs/inference-final-$TAG.log; exit 1; }

OUT=$WS/evals/final-$TAG
mkdir -p "$OUT"

echo "=== tb2 89 ==="
uv run --no-sync eval terminal-bench-2-v1 -m "$CKPT" \
  --client.base-url http://127.0.0.1:$PORT/v1 \
  --env.agent.harness.id pi --env.agent.runtime.type broker \
  --env.agent.runtime.ready_timeout_seconds 1800 \
  --sampling.temperature 0.3 \
  -n 89 -r 1 -s -c 24 --no-push --rich false \
  -o "$OUT/tb2" > "$OUT/tb2.log" 2>&1
uv run --no-sync python $WS/scripts/score.py "$OUT/tb2/traces.jsonl" || true

echo "=== swe 100 ==="
uv run --no-sync eval swebench-verified-v1 -m "$CKPT" \
  --client.base-url http://127.0.0.1:$PORT/v1 \
  --env.agent.harness.id pi --env.agent.runtime.type broker \
  --env.agent.runtime.ready_timeout_seconds 1800 \
  --sampling.temperature 0.3 \
  -n 100 -r 1 -s -c 64 --no-push --rich false \
  -o "$OUT/swe" > "$OUT/swe.log" 2>&1
uv run --no-sync python $WS/scripts/score.py "$OUT/swe/traces.jsonl" || true

kill $SPID 2>/dev/null
echo "=== FINAL DONE $TAG ==="