harness/scripts/run_eval.sh

#!/bin/bash
# Usage: run_eval.sh <model-name-or-path> <port> <tag> [n_tb2] [n_swe]
# Serves nothing — expects an inference server already running on <port>.
# Runs both suites with fixed seeds so checkpoints compare apples-to-apples.
set -u
MODEL=$1
PORT=$2
TAG=$3
N_TB2=${4:-89}
N_SWE=${5:-100}

cd /root/work/a/prime-rl
export PRIME_API_KEY="$(cat "$AGENTPTB_PRIME_KEY_FILE")"
OUT=$AGENTPTB_WORKSPACE/evals/$TAG
mkdir -p "$OUT"

echo "=== terminal-bench-2: $N_TB2 tasks ==="
uv run eval terminal-bench-2-v1 -m "$MODEL" \
  --client.base-url "http://127.0.0.1:$PORT/v1" \
  --env.agent.harness.id pi --env.agent.runtime.type broker \
  --env.agent.runtime.ready_timeout_seconds 1800 \
  -n "$N_TB2" -r 1 -s --no-push --rich false \
  -o "$OUT/tb2" 2>&1 | tail -25

echo "=== swe-bench-verified: $N_SWE tasks ==="
uv run eval swebench-verified-v1 -m "$MODEL" \
  --client.base-url "http://127.0.0.1:$PORT/v1" \
  --env.agent.harness.id pi --env.agent.runtime.type broker \
  --env.agent.runtime.ready_timeout_seconds 1800 \
  -n "$N_SWE" -r 1 -s --no-push --rich false \
  -o "$OUT/swe" 2>&1 | tail -25

echo "=== DONE $TAG ==="