opus-max-record / harness /scripts /pool_probe.sh
simonycl's picture
Upload folder using huggingface_hub
b2ebc95 verified
Raw
History Blame Contribute Delete
1.42 kB
#!/bin/bash
# Sandbox pool health + capacity, every 15 minutes.
# NOTE: the readiness JSON puts "ready":true ~190 chars in — do NOT truncate the response
# below that, or every healthy probe reads as a failure (this misdiagnosis cost hours once).
while true; do
CAP=$(curl -s -m 30 "$SANDBOX_BASE_URL/resources" -H "X-API-Key: $SANDBOX_API_KEY" | python3 -c "
import json,sys,collections
try:
d=json.load(sys.stdin); c=collections.Counter(p['phase'] for p in d['sandbox_pods'])
print(f\"pods={dict(c)} avail_cpu={round(d['available']['cpu'])}\")
except Exception: print('capacity-unknown')" 2>/dev/null)
R=$(curl -s -m 60 -X POST "$SANDBOX_BASE_URL/sandboxes" -H "X-API-Key: $SANDBOX_API_KEY" \
-H 'Content-Type: application/json' -d '{"image":"ubuntu:22.04","block_network":false,"cpu":"1","memory":"4Gi"}')
ID=$(echo "$R" | python3 -c "import json,sys;print(json.load(sys.stdin).get('sandbox_id',''))" 2>/dev/null)
if [ -z "$ID" ]; then
echo "$(date -Is) CREATE_FAIL $CAP $(echo "$R" | head -c 120)"
else
S=$(date +%s)
RES=$(timeout 260 curl -s "$SANDBOX_BASE_URL/sandboxes/$ID/wait?timeout=240" -H "X-API-Key: $SANDBOX_API_KEY")
E=$(date +%s)
case "$RES" in *'"ready":true'*) OK=READY;; *) OK=FAIL;; esac
echo "$(date -Is) $OK $((E-S))s $CAP"
curl -s -X DELETE "$SANDBOX_BASE_URL/sandboxes/$ID" -H "X-API-Key: $SANDBOX_API_KEY" >/dev/null
fi
sleep 900
done