File size: 6,437 Bytes
9589849 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 | #!/usr/bin/env bash
set -euo pipefail
workspace="/mnt/pvc/users/simon/agentptb/runs/sol-max-s1/workspace"
weights="$workspace/outputs/sft-agent-mix-clean-full-v1/weights/step_150"
train_session="agentptb-sft-clean-resume50b"
inference_pid=""
cleanup() {
if [[ -n "$inference_pid" ]]; then
kill -TERM -- "-$inference_pid" 2>/dev/null || true
for _ in {1..12}; do
kill -0 -- "-$inference_pid" 2>/dev/null || break
sleep 5
done
kill -KILL -- "-$inference_pid" 2>/dev/null || true
wait "$inference_pid" 2>/dev/null || true
fi
}
trap cleanup EXIT INT TERM
while [[ ! -f "$weights/STABLE" ]]; do
sleep 30
done
while tmux has-session -t "$train_session" 2>/dev/null; do
sleep 10
done
python "$workspace/scripts/ensure_chat_stop_metadata.py" "$weights" \
>"$workspace/logs/chat-stop-metadata-step150.json"
mkdir -p "$workspace/candidates"
if [[ ! -e "$workspace/candidates/step_150" ]]; then
# Archive mode attempts a chown/chmod after linking and fails under the
# shared filesystem's root-squash policy. Recursive hard links preserve
# the exact file inodes without metadata mutations.
cp -Rl "$weights" "$workspace/candidates/step_150"
fi
python "$workspace/scripts/ensure_chat_stop_metadata.py" \
"$workspace/candidates/step_150" --check >/dev/null
python "$workspace/scripts/check_checkpoint.py" \
"$weights" "$workspace/outputs/sft-agent-mix-clean-full-v1/metrics.jsonl" \
--expected-step 150 >"$workspace/logs/checkpoint-health-step150.json"
cd /root/work/a/prime-rl
setsid bash -c 'exec "$@"' _ \
env CUDA_VISIBLE_DEVICES=4,5,6,7 \
TMPDIR=/tmp/agentptb-s1 \
PYTHONUNBUFFERED=1 \
uv run --no-sync inference @ "$workspace/configs/inference-candidate.toml" \
>"$workspace/logs/inference-step150-launch.log" 2>&1 &
inference_pid=$!
ready=false
for _ in {1..120}; do
# The DP router's generic /v1/models forwarding can rewrite the backend
# address to 0.0.0.0 even while inference is fully healthy. Readiness only
# needs the router health endpoint; the explicit-model smoke below verifies
# an actual structured completion.
if curl --fail --silent http://127.0.0.1:8200/health >/dev/null; then
ready=true
break
fi
kill -0 "$inference_pid"
sleep 10
done
[[ "$ready" == true ]]
cd "$workspace"
python scripts/smoke_tool_call.py --model "$weights" \
>logs/smoke-tool-call-step150.log 2>&1
# Keep each fixed-sample wave at no more than 24 live sandboxes. The broker is
# shared, and a prior 120-wide wave spent most of its wall time pending despite
# ample model capacity. Grouping by suite and sampling temperature preserves
# every paired comparison while avoiding shared-pool head-of-line blocking.
scripts/run_paired_evals.sh \
configs/eval-stock-tb2-32.toml \
configs/eval-custom-noreview-tb2-32.toml \
configs/eval-custom-tb2-32.toml
scripts/run_paired_evals.sh \
configs/eval-stock-swe-32.toml \
configs/eval-custom-noreview-swe-32.toml \
configs/eval-custom-swe-32.toml
scripts/run_paired_evals.sh \
configs/eval-custom-noreview-t06-tb2-32.toml \
configs/eval-custom-t06-tb2-32.toml \
configs/eval-stock-t06-tb2-32.toml
scripts/run_paired_evals.sh \
configs/eval-custom-noreview-t06-swe-32.toml \
configs/eval-custom-t06-swe-32.toml \
configs/eval-stock-t06-swe-32.toml
# Repair only explicit sandbox/network failures after both fixed waves have
# exited. This remains part of the precommitted protocol, but running it in the
# same process makes GPU and server ownership unambiguous.
scripts/repair_step150_infra.sh
for label in \
stock-tb2 custom-noreview-tb2 custom-tb2 \
stock-swe custom-noreview-swe custom-swe \
custom-noreview-t06-tb2 custom-t06-tb2 \
custom-noreview-t06-swe custom-t06-swe \
stock-t06-tb2 stock-t06-swe; do
python scripts/summarize_eval.py \
"evals/step150-${label}-32-run2" \
>"logs/summary-step150-${label}-32-run2.json"
done
python scripts/compare_evals.py \
evals/step150-stock-tb2-32-run2 evals/step150-custom-noreview-tb2-32-run2 \
--label-a stock --label-b aligned \
>logs/compare-step150-stock-vs-aligned-tb2.json
python scripts/compare_evals.py \
evals/step150-custom-noreview-tb2-32-run2 evals/step150-custom-tb2-32-run2 \
--label-a aligned --label-b review \
>logs/compare-step150-aligned-vs-review-tb2.json
python scripts/compare_evals.py \
evals/step150-stock-swe-32-run2 evals/step150-custom-noreview-swe-32-run2 \
--label-a stock --label-b aligned \
>logs/compare-step150-stock-vs-aligned-swe.json
python scripts/compare_evals.py \
evals/step150-custom-noreview-swe-32-run2 evals/step150-custom-swe-32-run2 \
--label-a aligned --label-b review \
>logs/compare-step150-aligned-vs-review-swe.json
python scripts/compare_evals.py \
evals/step150-custom-noreview-tb2-32-run2 \
evals/step150-custom-noreview-t06-tb2-32-run2 \
--label-a t02 --label-b t06 \
>logs/compare-step150-t02-vs-t06-tb2.json
python scripts/compare_evals.py \
evals/step150-custom-noreview-swe-32-run2 \
evals/step150-custom-noreview-t06-swe-32-run2 \
--label-a t02 --label-b t06 \
>logs/compare-step150-t02-vs-t06-swe.json
python scripts/compare_evals.py \
evals/step150-custom-noreview-t06-tb2-32-run2 \
evals/step150-custom-t06-tb2-32-run2 \
--label-a aligned --label-b review \
>logs/compare-step150-t06-aligned-vs-review-tb2.json
python scripts/compare_evals.py \
evals/step150-custom-noreview-t06-swe-32-run2 \
evals/step150-custom-t06-swe-32-run2 \
--label-a aligned --label-b review \
>logs/compare-step150-t06-aligned-vs-review-swe.json
python scripts/select_harness.py \
evals/step150-custom-noreview-tb2-32-run2 \
evals/step150-custom-tb2-32-run2 \
evals/step150-custom-noreview-t06-tb2-32-run2 \
evals/step150-custom-t06-tb2-32-run2 \
evals/step150-custom-noreview-swe-32-run2 \
evals/step150-custom-swe-32-run2 \
evals/step150-custom-noreview-t06-swe-32-run2 \
evals/step150-custom-t06-swe-32-run2 \
evals/step150-stock-tb2-32-run2 \
evals/step150-stock-t06-tb2-32-run2 \
evals/step150-stock-swe-32-run2 \
evals/step150-stock-t06-swe-32-run2 \
>logs/select-harness-step150.json
python -c \
'import json; print(json.load(open("logs/select-harness-step150.json"))["choice"])' \
>state/harness-choice.txt
python -c \
'import json; print(json.load(open("logs/select-harness-step150.json"))["temperature"])' \
>state/sampling-temperature.txt
touch state/step150-eval-complete
|