File size: 1,742 Bytes
7fde66e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 | #!/bin/bash
# Final confirmation evals for the submission checkpoint: tb2 (89) then swe (100), one server.
set -u
WS=/mnt/pvc/users/simon/agentptb/runs/d/workspace
CKPT=$1
TAG=$2
PORT=8500
cd /root/work/a/prime-rl
export PRIME_API_KEY="$(cat $WS/.prime-api-key)"
nohup uv run inference --model.name "$CKPT" \
--model.tool-call-parser qwen3_coder --model.reasoning-parser qwen3 \
--server.port $PORT --parallel.tp 1 --parallel.dp 1 \
--vllm-extra '{"limit_mm_per_prompt": {"image": 0, "video": 0}}' \
> $WS/logs/inference-final-$TAG.log 2>&1 &
SPID=$!
ok=0
for i in $(seq 1 60); do
curl -sf http://localhost:$PORT/health >/dev/null 2>&1 && { ok=1; break; }
sleep 10
done
[ $ok -eq 1 ] || { echo "SERVER FAILED"; tail -5 $WS/logs/inference-final-$TAG.log; exit 1; }
OUT=$WS/evals/final-$TAG
mkdir -p "$OUT"
echo "=== tb2 89 ==="
uv run --no-sync eval terminal-bench-2-v1 -m "$CKPT" \
--client.base-url http://127.0.0.1:$PORT/v1 \
--env.agent.harness.id pi --env.agent.runtime.type broker \
--env.agent.runtime.ready_timeout_seconds 1800 \
--sampling.temperature 0.3 \
-n 89 -r 1 -s -c 24 --no-push --rich false \
-o "$OUT/tb2" > "$OUT/tb2.log" 2>&1
uv run --no-sync python $WS/scripts/score.py "$OUT/tb2/traces.jsonl" || true
echo "=== swe 100 ==="
uv run --no-sync eval swebench-verified-v1 -m "$CKPT" \
--client.base-url http://127.0.0.1:$PORT/v1 \
--env.agent.harness.id pi --env.agent.runtime.type broker \
--env.agent.runtime.ready_timeout_seconds 1800 \
--sampling.temperature 0.3 \
-n 100 -r 1 -s -c 64 --no-push --rich false \
-o "$OUT/swe" > "$OUT/swe.log" 2>&1
uv run --no-sync python $WS/scripts/score.py "$OUT/swe/traces.jsonl" || true
kill $SPID 2>/dev/null
echo "=== FINAL DONE $TAG ==="
|