sweep-all.sh

#!/usr/bin/env bash
# harvest — full model sweep v2 (PIPELINED: GPU never idles).
#
# molab reaps pods whose GPU utilization sits ~0% for even a few minutes (UI warns,
# then kills ~1s later — observed 2026-07-16). v1 idled the GPU ~4-7 min during
# setup + the first 38GB download and died exactly when benching began. v2 fixes it
# structurally: a small fast model (16GB, ~60s download) is served and benchmarked
# FIRST, and every big-model download runs in the BACKGROUND while the GPU is busy
# benchmarking the previous model. All GPU load is genuine benchmark work we want.
#
# Logs: /tmp/sweep.log · results: /tmp/bench_*.json /tmp/pack_*.json
# Ends with: SWEEP COMPLETE
set -uo pipefail
cd /tmp
RT="https://huggingface.co/PS4Research/harvest-runtime/resolve/main"
HFM="https://huggingface.co/unsloth"
MODELS_DIR=/tmp/harvest/models
log(){ echo "[$(date +%H:%M:%S)] $*"; }
gpu(){ nvidia-smi --query-gpu=utilization.gpu,memory.used --format=csv,noheader 2>/dev/null; }

python3 -c "import requests" 2>/dev/null || pip install -q requests
[ -f bench-pod.py ] || curl -sL -o bench-pod.py "$RT/bench-pod.py"
mkdir -p "$MODELS_DIR"

# registry: file + repo per model
declare -A GGUF=(
  [qwen3.6-35b]="Qwen3.6-35B-A3B-GGUF/Qwen3.6-35B-A3B-UD-Q8_K_XL.gguf"
  [glm-4.7-flash]="GLM-4.7-Flash-GGUF/GLM-4.7-Flash-UD-Q8_K_XL.gguf"
  [gemma-4-26b]="gemma-4-26B-A4B-it-GGUF/gemma-4-26B-A4B-it-UD-Q8_K_XL.gguf"
  [nemotron-3-nano-30b]="Nemotron-3-Nano-30B-A3B-GGUF/Nemotron-3-Nano-30B-A3B-UD-Q8_K_XL.gguf"
)

bg_download(){ # $1=model-name  (idempotent; .part + rename)
  local repo="${GGUF[$1]%/*}" file="${GGUF[$1]##*/}"
  [ -f "$MODELS_DIR/$file" ] && { log "dl $1: already cached"; return 0; }
  # HF needs /resolve/main/ between repo and file
  ( curl -fL --retry 3 -s -o "$MODELS_DIR/$file.part" "$HFM/$repo/resolve/main/$file" \
      && mv "$MODELS_DIR/$file.part" "$MODELS_DIR/$file" \
      && log "dl $1: done" || log "dl $1: FAILED" ) &
  DL_PID=$!
  log "dl $1: started in background (pid $DL_PID)"
}
wait_model(){ # $1=model-name — block until file present
  local file="${GGUF[$1]##*/}"
  while [ ! -f "$MODELS_DIR/$file" ]; do sleep 5; done
}

serve_only(){ # $1=models-csv $2=slots $3=ctx  (models must be cached already)
  pkill -x llama-server 2>/dev/null; sleep 2
  curl -sL "$RT/pod-bootstrap.sh" | bash -s -- --model "$1" --slots "$2" --ctx "$3" --no-smoke \
    || { log "BOOTSTRAP FAILED for $1"; return 1; }
}

sweep(){ # $1=model $2=port $3=mode $4=max_tokens
  log "sweep $1 thinking=$3 (gpu: $(gpu))"
  python3 bench-pod.py --base-url "http://127.0.0.1:$2/v1" --model "$1" \
    --concurrencies 1,8,32,64 --rounds 2 --max-tokens "$4" --thinking "$3" \
    --out "/tmp/bench_$1_$3.json" || log "sweep $1/$3 FAILED"
}

# ---------- 0. FAST START: small model busies the GPU within ~90s ----------
log "PHASE fast-start: gemma-4-26b-q4 (16GB) + qwen download in background"
curl -sL "$RT/pod-bootstrap.sh" | bash -s -- --model gemma-4-26b-q4 --slots 32 --ctx 32768 --no-smoke \
  || { log "FAST-START BOOTSTRAP FAILED"; exit 1; }
bg_download qwen3.6-35b
sweep gemma-4-26b-q4 18080 off 256      # real benchmark data; GPU busy while qwen lands
sweep gemma-4-26b-q4 18080 on  384

# ---------- 1. Qwen3.6-35B ----------
log "PHASE qwen (waiting for download if needed)"
wait_model qwen3.6-35b
serve_only qwen3.6-35b 64 131072 || exit 1
bg_download glm-4.7-flash
sweep qwen3.6-35b 18080 off 256
sweep qwen3.6-35b 18080 on  384

# ---------- 2. long-context (Qwen, 16k/slot) ----------
log "PHASE longctx"
pkill -x llama-server 2>/dev/null; sleep 2
setsid nohup /tmp/harvest/portable-fat/run-server.sh \
  -m "$MODELS_DIR/${GGUF[qwen3.6-35b]##*/}" -a qwen3.6-35b \
  --host 127.0.0.1 --port 18090 --api-key sk-harvest-local \
  -ngl 999 -fa on -ctk q8_0 -ctv q8_0 -c 131072 -np 8 > /tmp/s_longctx.log 2>&1 < /dev/null &
for i in $(seq 1 120); do
  curl -sf -m 3 -H "Authorization: Bearer sk-harvest-local" http://127.0.0.1:18090/v1/models 2>/dev/null | grep -q qwen && { log "longctx up (${i}s)"; break; }
  sleep 1
done
python3 - <<'PY' 2>&1 || log "longctx FAILED"
import requests, time, json, statistics
base='http://127.0.0.1:18090/v1'; H={'Authorization':'Bearer sk-harvest-local'}
hist='Week %d: rainfall %dmm, soil moisture %d%%, pest scouting score %d, action taken: %s. '
long_ctx=''.join(hist%(i,(i*37)%80,30+(i*13)%50,(i*7)%10,'irrigated' if i%3 else 'sprayed') for i in range(1,400))
msgs=[{'role':'system','content':'You are a season-long advisory agent. Consider the full season history.'},
      {'role':'user','content':long_ctx+' Given this full season history, advise on harvest timing and marketing in 3 sentences.'}]
rows=[]
for r_ in range(3):
    t0=time.time()
    d=requests.post(base+'/chat/completions',headers=H,json={'model':'qwen3.6-35b','messages':msgs,
        'max_tokens':160,'temperature':0.3,'chat_template_kwargs':{'enable_thinking':False}},timeout=900).json()
    u=d.get('usage',{}); w=time.time()-t0
    rows.append({'prompt_tokens':u.get('prompt_tokens'),'completion_tokens':u.get('completion_tokens'),'wall_s':round(w,1)})
    print('longctx run',r_,rows[-1])
json.dump(rows,open('/tmp/bench_longctx.json','w'),indent=2)
print('LONGCTX p50 wall: %.1fs (prompt ~%s toks)'%(statistics.median(r['wall_s'] for r in rows),rows[0]['prompt_tokens']))
PY

# ---------- 3. GLM-4.7-Flash ----------
log "PHASE glm"
wait_model glm-4.7-flash
serve_only glm-4.7-flash 64 131072 && {
  bg_download nemotron-3-nano-30b
  sweep glm-4.7-flash 18080 off 256
  sweep glm-4.7-flash 18080 on  384
}
rm -f "$MODELS_DIR/${GGUF[glm-4.7-flash]##*/}"

# ---------- 4. Nemotron-3-Nano ----------
log "PHASE nemotron"
wait_model nemotron-3-nano-30b
serve_only nemotron-3-nano-30b 64 131072 && {
  bg_download gemma-4-26b
  sweep nemotron-3-nano-30b 18080 off 256
  sweep nemotron-3-nano-30b 18080 on  384
}
rm -f "$MODELS_DIR/${GGUF[nemotron-3-nano-30b]##*/}"

# ---------- 5. gemma-4-26B (Q8) ----------
log "PHASE gemma"
wait_model gemma-4-26b
serve_only gemma-4-26b 64 131072 && {
  sweep gemma-4-26b 18080 off 256
  sweep gemma-4-26b 18080 on  384
}

# ---------- 6. packing: qwen + gemma co-resident ----------
log "PHASE packing (gpu: $(gpu))"
serve_only "qwen3.6-35b,gemma-4-26b" 24 49152 && {
  log "endpoints:"; cat /tmp/harvest/endpoints.txt
  ( python3 bench-pod.py --base-url http://127.0.0.1:18080/v1 --model qwen3.6-35b \
      --concurrencies 16 --rounds 3 --max-tokens 256 --thinking off --out /tmp/pack_qwen.json ) &
  P1=$!
  ( python3 bench-pod.py --base-url http://127.0.0.1:18081/v1 --model gemma-4-26b \
      --concurrencies 16 --rounds 3 --max-tokens 256 --thinking off --out /tmp/pack_gemma.json ) &
  P2=$!
  wait $P1 $P2
}

log "disk: $(du -sh $MODELS_DIR 2>/dev/null | cut -f1); gpu now: $(gpu)"
pkill -x llama-server 2>/dev/null
echo "SWEEP COMPLETE"