| # Run expert-iteration collection only while the sandbox pool has real headroom. | |
| # | |
| # The pool oscillates between free (100-360 CPU) and jammed (60-100 pods queued behind other | |
| # tenants). Sampling through a jam turns every episode into a SandboxError and deepens the | |
| # queue for everyone, so this starts a round when there is room and kills it when there isn't. | |
| # | |
| # Tracks its child by pidfile: `pgrep -f <config path>` also matches any shell that merely | |
| # mentions the path, which made an earlier version believe a round was permanently running. | |
| WS=/mnt/pvc/users/simon/agentptb/runs/a-opus-max/workspace | |
| cd "$WS" || exit 1 | |
| PIDF=/tmp/ei_round.pid | |
| i=${EI_START_ROUND:-0} | |
| while true; do | |
| free=$(curl -s -m 30 "$SANDBOX_BASE_URL/resources" -H "X-API-Key: $SANDBOX_API_KEY" | python3 -c " | |
| import json,sys,collections | |
| try: | |
| d=json.load(sys.stdin); c=collections.Counter(p['phase'] for p in d['sandbox_pods']) | |
| print(1 if (c.get('Pending',0) < 8 and d['available']['cpu'] > 60) else 0) | |
| except Exception: print(0)" 2>/dev/null) | |
| alive=0 | |
| if [ -f "$PIDF" ] && kill -0 "$(cat "$PIDF")" 2>/dev/null; then alive=1; fi | |
| if [ "$free" = "1" ] && [ "$alive" -eq 0 ]; then | |
| i=$((i+1)) | |
| echo "$(date -Is) pool free -> ei round $i" | |
| setsid nohup bash scripts/run_eval.sh "$WS/cfg/ei_r2e.toml" "$WS/runs/ei_round$i" \ | |
| > "logs/ei_round$i.log" 2>&1 < /dev/null & | |
| echo $! > "$PIDF" | |
| elif [ "$free" = "0" ] && [ "$alive" -eq 1 ]; then | |
| echo "$(date -Is) pool jammed -> pausing round $i" | |
| kill -TERM "$(cat "$PIDF")" 2>/dev/null | |
| sleep 20 | |
| rm -f "$PIDF" | |
| fi | |
| sleep 600 | |
| done | |