#!/usr/bin/env bash # harvest — plug-and-play pod bootstrap # # Takes a bare GPU pod (no persistent storage, nothing installed) to a VERIFIED # OpenAI-compatible endpoint. Built for molab Blackwell pods; also runs on A100s. # Everything is fetched from PUBLIC sources — no credentials required, ever. # # curl -sL https://huggingface.co/PS4Research/harvest-runtime/resolve/main/pod-bootstrap.sh \ # | bash -s -- --model gemma-4-26b # # Options: # --model NAME[,NAME...] model(s) from the registry (multiple = packed on one GPU) # --port N base port (default 18080 — NOT 8080, molab uses that) # --ctx N context per server (default 65536) # --slots N parallel slots (default 32) # --no-smoke skip the post-launch smoke test # --list print the model registry and exit # # Hard-won pod lessons baked in (do not "simplify" these away): # * molab occupies :8080 and its proxy answers /health 200 -> never trust a bare # health 200; we verify OUR model alias via /v1/models. # * servers must be setsid-detached or they die with the bootstrap shell. # * pod overlay df reports a nonsense 8.0E -> tolerate absurd disk values. set -uo pipefail RUNTIME_URL="https://huggingface.co/PS4Research/harvest-runtime/resolve/main/portable-fat.tar.gz" WORK="${HARVEST_WORK:-/tmp/harvest}" PORT=18080; CTX=65536; SLOTS=32; SMOKE=1; MODELS="" API_KEY="${HARVEST_API_KEY:-sk-harvest-local}" # name|hf_repo|gguf_file|approx_gb|reasoning_mechanism (all locally validated) REGISTRY=' qwen3.6-35b|unsloth/Qwen3.6-35B-A3B-GGUF|Qwen3.6-35B-A3B-UD-Q8_K_XL.gguf|38|enable_thinking glm-4.7-flash|unsloth/GLM-4.7-Flash-GGUF|GLM-4.7-Flash-UD-Q8_K_XL.gguf|36|enable_thinking gemma-4-26b|unsloth/gemma-4-26B-A4B-it-GGUF|gemma-4-26B-A4B-it-UD-Q8_K_XL.gguf|28|enable_thinking nemotron-3-nano-30b|unsloth/Nemotron-3-Nano-30B-A3B-GGUF|Nemotron-3-Nano-30B-A3B-UD-Q8_K_XL.gguf|40|enable_thinking gemma-4-26b-q4|unsloth/gemma-4-26B-A4B-it-GGUF|gemma-4-26B-A4B-it-UD-Q4_K_XL.gguf|16|enable_thinking ' die() { echo "❌ ERROR: $*" >&2; exit 1; } say() { echo "[$(date +%H:%M:%S)] $*"; } reg_line() { echo "$REGISTRY" | grep "^$1|" | head -1; } port_free() { # exit 0 == port is FREE python3 -c " import socket, sys s = socket.socket() r = s.connect_ex(('127.0.0.1', $1)) s.close() sys.exit(0 if r != 0 else 1) # connect refused => nobody there => free " 2>/dev/null } next_free_port() { local p=$1 for _ in $(seq 1 50); do port_free "$p" && { echo "$p"; return; }; p=$((p+1)); done die "no free port found from $1" } while [ $# -gt 0 ]; do case "$1" in --model) MODELS="$2"; shift 2 ;; --port) PORT="$2"; shift 2 ;; --ctx) CTX="$2"; shift 2 ;; --slots) SLOTS="$2"; shift 2 ;; --no-smoke) SMOKE=0; shift ;; --list) echo "$REGISTRY" | grep '|' | awk -F'|' '{printf " %-22s %s (%sGB)\n", $1, $2, $4}'; exit 0 ;; *) die "unknown option: $1" ;; esac done [ -z "$MODELS" ] && die "need --model NAME (see --list)" echo "==============================================" echo " harvest pod bootstrap" echo "==============================================" # ---- 1. preflight ------------------------------------------------------------ say "preflight..." command -v nvidia-smi >/dev/null 2>&1 || die "no nvidia-smi — no GPU driver on this pod" GPU_NAME=$(nvidia-smi --query-gpu=name --format=csv,noheader | head -1) CC=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '. ') VRAM=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits | head -1) say " GPU: $GPU_NAME (sm_${CC}, ${VRAM} MiB)" case "$CC" in 80|120) : ;; # the two arches our fat bundle is compiled for *) echo "⚠️ WARNING: bundle targets sm_80/sm_120; this GPU is sm_${CC}. Expect" >&2 echo " 'no kernel image for device'. Report this arch so we can add it." >&2 ;; esac mkdir -p "$WORK/models" || die "cannot write to $WORK" NEED=2 for m in ${MODELS//,/ }; do L=$(reg_line "$m"); [ -z "$L" ] && die "unknown model '$m' (see --list)" NEED=$((NEED + $(echo "$L" | cut -d'|' -f4))) done AVAIL=$(df -BG --output=avail "$WORK" 2>/dev/null | tail -1 | tr -dc '0-9') if [ -z "${AVAIL:-}" ] || [ "${AVAIL:-0}" -gt 1000000 ]; then say " disk: need ~${NEED}GB, available: unreported (overlay fs) — continuing" else say " disk: need ~${NEED}GB, have ${AVAIL}GB" [ "$AVAIL" -lt "$NEED" ] && die "insufficient disk in $WORK (~${NEED}GB needed, ${AVAIL}GB free)" fi # ---- 2. runtime bundle (cached) --------------------------------------------- if [ -x "$WORK/portable-fat/run-server.sh" ]; then say "runtime: cached ✓" else say "runtime: downloading (~888MB)..." curl -fsSL --retry 3 -o "$WORK/portable-fat.tar.gz" "$RUNTIME_URL" || die "bundle download failed" tar xzf "$WORK/portable-fat.tar.gz" -C "$WORK" || die "bundle extract failed" rm -f "$WORK/portable-fat.tar.gz" [ -x "$WORK/portable-fat/run-server.sh" ] || die "bundle is missing run-server.sh" say "runtime: ready ✓ (self-contained — no system CUDA needed)" fi # ---- 3. models (cached, resumable) ------------------------------------------ for m in ${MODELS//,/ }; do L=$(reg_line "$m"); REPO=$(echo "$L" | cut -d'|' -f2); FILE=$(echo "$L" | cut -d'|' -f3); GB=$(echo "$L" | cut -d'|' -f4) DEST="$WORK/models/$FILE" if [ -f "$DEST" ] && [ "$(stat -Lc %s "$DEST" 2>/dev/null || echo 0)" -gt $(( (GB - 3) * 1000000000 )) ]; then say "model $m: cached ✓" else say "model $m: downloading ~${GB}GB ..." t0=$(date +%s) curl -fL -C - --retry 3 -s -o "$DEST" "https://huggingface.co/$REPO/resolve/main/$FILE" \ || die "model download failed: $m" say "model $m: ready ✓ ($(( $(date +%s) - t0 ))s)" fi done # ---- 4. serve (detached, identity-verified) --------------------------------- ENDPOINTS="" for m in ${MODELS//,/ }; do L=$(reg_line "$m"); FILE=$(echo "$L" | cut -d'|' -f3) P=$(next_free_port "$PORT"); PORT=$((P + 1)) say "serving $m on :$P ..." # setsid + "$WORK/server-$m.log" 2>&1 < /dev/null & echo $! > "$WORK/server-$m.pid" ok=0 for i in $(seq 1 400); do # IDENTITY check — a bare 200 may be molab's proxy, not us if curl -sf -m 3 -H "Authorization: Bearer $API_KEY" "http://127.0.0.1:$P/v1/models" 2>/dev/null | grep -q "\"$m\""; then ok=1; say " up in ${i}s ✓ (verified: serving '$m')"; break fi kill -0 "$(cat "$WORK/server-$m.pid")" 2>/dev/null || { echo "--- server log ---"; tail -20 "$WORK/server-$m.log"; die "$m server process died during startup"; } grep -qi "error while handling\|failed to load\|no kernel image\|address already in use\|bind: " "$WORK/server-$m.log" 2>/dev/null && { echo "--- server log ---"; tail -20 "$WORK/server-$m.log"; die "$m failed to start"; } sleep 1 done [ $ok -eq 1 ] || { echo "--- server log ---"; tail -20 "$WORK/server-$m.log"; die "$m not healthy after 400s"; } ENDPOINTS="${ENDPOINTS}${m}|http://127.0.0.1:$P/v1"$'\n' done # ---- 5. smoke test: a real advisory call with reasoning OFF ------------------ if [ $SMOKE -eq 1 ]; then m1="${MODELS%%,*}" U1=$(printf '%s' "$ENDPOINTS" | grep "^$m1|" | cut -d'|' -f2-) say "smoke test on $m1 (reasoning off → direct answer)..." RESP=$(curl -s -m 180 -H "Authorization: Bearer $API_KEY" -H "Content-Type: application/json" \ "$U1/chat/completions" -d "{\"model\":\"$m1\", \"messages\":[{\"role\":\"user\",\"content\":\"In one sentence: should a smallholder cotton farmer sow now if the monsoon is two weeks late?\"}], \"max_tokens\":80,\"temperature\":0.3,\"chat_template_kwargs\":{\"enable_thinking\":false}}") TXT=$(echo "$RESP" | python3 -c "import sys,json;d=json.load(sys.stdin);m=d['choices'][0]['message'];print((m.get('content') or '').strip()[:160])" 2>/dev/null) [ -z "$TXT" ] && { echo "--- response ---"; echo "$RESP" | head -c 400; die "smoke test failed — no content"; } say " model says: \"$TXT\"" say " smoke test ✓" fi # ---- 6. report --------------------------------------------------------------- printf '%s' "$ENDPOINTS" > "$WORK/endpoints.txt" echo echo "==============================================" echo " ✅ POD READY" echo "==============================================" printf '%s' "$ENDPOINTS" | while IFS='|' read -r n u; do [ -n "$n" ] && echo " $n -> $u"; done echo " api key : $API_KEY" echo " logs : $WORK/server-.log" echo " stop : pkill -x llama-server" echo echo " reasoning OFF (baseline) : \"chat_template_kwargs\": {\"enable_thinking\": false}" echo " reasoning ON (condition): \"chat_template_kwargs\": {\"enable_thinking\": true}" echo "=============================================="