File size: 1,131 Bytes
7fde66e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | #!/bin/bash
# Usage: run_eval.sh <model-name-or-path> <port> <tag> [n_tb2] [n_swe]
# Serves nothing — expects an inference server already running on <port>.
# Runs both suites with fixed seeds so checkpoints compare apples-to-apples.
set -u
MODEL=$1
PORT=$2
TAG=$3
N_TB2=${4:-89}
N_SWE=${5:-100}
cd /root/work/a/prime-rl
export PRIME_API_KEY="$(cat "$AGENTPTB_PRIME_KEY_FILE")"
OUT=$AGENTPTB_WORKSPACE/evals/$TAG
mkdir -p "$OUT"
echo "=== terminal-bench-2: $N_TB2 tasks ==="
uv run eval terminal-bench-2-v1 -m "$MODEL" \
--client.base-url "http://127.0.0.1:$PORT/v1" \
--env.agent.harness.id pi --env.agent.runtime.type broker \
--env.agent.runtime.ready_timeout_seconds 1800 \
-n "$N_TB2" -r 1 -s --no-push --rich false \
-o "$OUT/tb2" 2>&1 | tail -25
echo "=== swe-bench-verified: $N_SWE tasks ==="
uv run eval swebench-verified-v1 -m "$MODEL" \
--client.base-url "http://127.0.0.1:$PORT/v1" \
--env.agent.harness.id pi --env.agent.runtime.type broker \
--env.agent.runtime.ready_timeout_seconds 1800 \
-n "$N_SWE" -r 1 -s --no-push --rich false \
-o "$OUT/swe" 2>&1 | tail -25
echo "=== DONE $TAG ==="
|