pod-bootstrap.sh

#!/usr/bin/env bash
# harvest — plug-and-play pod bootstrap
#
# Takes a bare GPU pod (no persistent storage, nothing installed) to a VERIFIED
# OpenAI-compatible endpoint. Built for molab Blackwell pods; also runs on A100s.
# Everything is fetched from PUBLIC sources — no credentials required, ever.
#
#   curl -sL https://huggingface.co/PS4Research/harvest-runtime/resolve/main/pod-bootstrap.sh \
#     | bash -s -- --model gemma-4-26b
#
# Options:
#   --model NAME[,NAME...]  model(s) from the registry (multiple = packed on one GPU)
#   --port N                base port (default 18080 — NOT 8080, molab uses that)
#   --ctx N                 context per server (default 65536)
#   --slots N               parallel slots (default 32)
#   --no-smoke              skip the post-launch smoke test
#   --list                  print the model registry and exit
#
# Hard-won pod lessons baked in (do not "simplify" these away):
#   * molab occupies :8080 and its proxy answers /health 200 -> never trust a bare
#     health 200; we verify OUR model alias via /v1/models.
#   * servers must be setsid-detached or they die with the bootstrap shell.
#   * pod overlay df reports a nonsense 8.0E -> tolerate absurd disk values.
set -uo pipefail

RUNTIME_URL="https://huggingface.co/PS4Research/harvest-runtime/resolve/main/portable-fat.tar.gz"
WORK="${HARVEST_WORK:-/tmp/harvest}"
PORT=18080; CTX=65536; SLOTS=32; SMOKE=1; MODELS=""
API_KEY="${HARVEST_API_KEY:-sk-harvest-local}"

# name|hf_repo|gguf_file|approx_gb|reasoning_mechanism   (all locally validated)
REGISTRY='
qwen3.6-35b|unsloth/Qwen3.6-35B-A3B-GGUF|Qwen3.6-35B-A3B-UD-Q8_K_XL.gguf|38|enable_thinking
glm-4.7-flash|unsloth/GLM-4.7-Flash-GGUF|GLM-4.7-Flash-UD-Q8_K_XL.gguf|36|enable_thinking
gemma-4-26b|unsloth/gemma-4-26B-A4B-it-GGUF|gemma-4-26B-A4B-it-UD-Q8_K_XL.gguf|28|enable_thinking
nemotron-3-nano-30b|unsloth/Nemotron-3-Nano-30B-A3B-GGUF|Nemotron-3-Nano-30B-A3B-UD-Q8_K_XL.gguf|40|enable_thinking
gemma-4-26b-q4|unsloth/gemma-4-26B-A4B-it-GGUF|gemma-4-26B-A4B-it-UD-Q4_K_XL.gguf|16|enable_thinking
'

die() { echo "❌ ERROR: $*" >&2; exit 1; }
say() { echo "[$(date +%H:%M:%S)] $*"; }
reg_line() { echo "$REGISTRY" | grep "^$1|" | head -1; }

port_free() {  # exit 0 == port is FREE
  python3 -c "
import socket, sys
s = socket.socket()
r = s.connect_ex(('127.0.0.1', $1))
s.close()
sys.exit(0 if r != 0 else 1)   # connect refused => nobody there => free
" 2>/dev/null
}
next_free_port() {
  local p=$1
  for _ in $(seq 1 50); do port_free "$p" && { echo "$p"; return; }; p=$((p+1)); done
  die "no free port found from $1"
}

while [ $# -gt 0 ]; do
  case "$1" in
    --model) MODELS="$2"; shift 2 ;;
    --port) PORT="$2"; shift 2 ;;
    --ctx) CTX="$2"; shift 2 ;;
    --slots) SLOTS="$2"; shift 2 ;;
    --no-smoke) SMOKE=0; shift ;;
    --list) echo "$REGISTRY" | grep '|' | awk -F'|' '{printf "  %-22s %s (%sGB)\n", $1, $2, $4}'; exit 0 ;;
    *) die "unknown option: $1" ;;
  esac
done
[ -z "$MODELS" ] && die "need --model NAME (see --list)"

echo "=============================================="
echo " harvest pod bootstrap"
echo "=============================================="

# ---- 1. preflight ------------------------------------------------------------
say "preflight..."
command -v nvidia-smi >/dev/null 2>&1 || die "no nvidia-smi — no GPU driver on this pod"
GPU_NAME=$(nvidia-smi --query-gpu=name --format=csv,noheader | head -1)
CC=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '. ')
VRAM=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits | head -1)
say "  GPU: $GPU_NAME (sm_${CC}, ${VRAM} MiB)"
case "$CC" in
  80|120) : ;;   # the two arches our fat bundle is compiled for
  *) echo "⚠️  WARNING: bundle targets sm_80/sm_120; this GPU is sm_${CC}. Expect" >&2
     echo "   'no kernel image for device'. Report this arch so we can add it." >&2 ;;
esac
mkdir -p "$WORK/models" || die "cannot write to $WORK"
NEED=2
for m in ${MODELS//,/ }; do
  L=$(reg_line "$m"); [ -z "$L" ] && die "unknown model '$m' (see --list)"
  NEED=$((NEED + $(echo "$L" | cut -d'|' -f4)))
done
AVAIL=$(df -BG --output=avail "$WORK" 2>/dev/null | tail -1 | tr -dc '0-9')
if [ -z "${AVAIL:-}" ] || [ "${AVAIL:-0}" -gt 1000000 ]; then
  say "  disk: need ~${NEED}GB, available: unreported (overlay fs) — continuing"
else
  say "  disk: need ~${NEED}GB, have ${AVAIL}GB"
  [ "$AVAIL" -lt "$NEED" ] && die "insufficient disk in $WORK (~${NEED}GB needed, ${AVAIL}GB free)"
fi

# ---- 2. runtime bundle (cached) ---------------------------------------------
if [ -x "$WORK/portable-fat/run-server.sh" ]; then
  say "runtime: cached ✓"
else
  say "runtime: downloading (~888MB)..."
  curl -fsSL --retry 3 -o "$WORK/portable-fat.tar.gz" "$RUNTIME_URL" || die "bundle download failed"
  tar xzf "$WORK/portable-fat.tar.gz" -C "$WORK" || die "bundle extract failed"
  rm -f "$WORK/portable-fat.tar.gz"
  [ -x "$WORK/portable-fat/run-server.sh" ] || die "bundle is missing run-server.sh"
  say "runtime: ready ✓ (self-contained — no system CUDA needed)"
fi

# ---- 3. models (cached, resumable) ------------------------------------------
for m in ${MODELS//,/ }; do
  L=$(reg_line "$m"); REPO=$(echo "$L" | cut -d'|' -f2); FILE=$(echo "$L" | cut -d'|' -f3); GB=$(echo "$L" | cut -d'|' -f4)
  DEST="$WORK/models/$FILE"
  if [ -f "$DEST" ] && [ "$(stat -Lc %s "$DEST" 2>/dev/null || echo 0)" -gt $(( (GB - 3) * 1000000000 )) ]; then
    say "model $m: cached ✓"
  else
    say "model $m: downloading ~${GB}GB ..."
    t0=$(date +%s)
    curl -fL -C - --retry 3 -s -o "$DEST" "https://huggingface.co/$REPO/resolve/main/$FILE" \
      || die "model download failed: $m"
    say "model $m: ready ✓ ($(( $(date +%s) - t0 ))s)"
  fi
done

# ---- 4. serve (detached, identity-verified) ---------------------------------
ENDPOINTS=""
for m in ${MODELS//,/ }; do
  L=$(reg_line "$m"); FILE=$(echo "$L" | cut -d'|' -f3)
  P=$(next_free_port "$PORT"); PORT=$((P + 1))
  say "serving $m on :$P ..."
  # setsid + </dev/null: MUST survive this script exiting
  setsid nohup "$WORK/portable-fat/run-server.sh" \
      -m "$WORK/models/$FILE" -a "$m" \
      --host 127.0.0.1 --port "$P" --api-key "$API_KEY" \
      -ngl 999 -fa on -ctk q8_0 -ctv q8_0 -c "$CTX" -np "$SLOTS" \
      > "$WORK/server-$m.log" 2>&1 < /dev/null &
  echo $! > "$WORK/server-$m.pid"

  ok=0
  for i in $(seq 1 400); do
    # IDENTITY check — a bare 200 may be molab's proxy, not us
    if curl -sf -m 3 -H "Authorization: Bearer $API_KEY" "http://127.0.0.1:$P/v1/models" 2>/dev/null | grep -q "\"$m\""; then
      ok=1; say "  up in ${i}s ✓ (verified: serving '$m')"; break
    fi
    kill -0 "$(cat "$WORK/server-$m.pid")" 2>/dev/null || {
      echo "--- server log ---"; tail -20 "$WORK/server-$m.log"; die "$m server process died during startup"; }
    grep -qi "error while handling\|failed to load\|no kernel image\|address already in use\|bind: " "$WORK/server-$m.log" 2>/dev/null && {
      echo "--- server log ---"; tail -20 "$WORK/server-$m.log"; die "$m failed to start"; }
    sleep 1
  done
  [ $ok -eq 1 ] || { echo "--- server log ---"; tail -20 "$WORK/server-$m.log"; die "$m not healthy after 400s"; }
  ENDPOINTS="${ENDPOINTS}${m}|http://127.0.0.1:$P/v1"$'\n'
done

# ---- 5. smoke test: a real advisory call with reasoning OFF ------------------
if [ $SMOKE -eq 1 ]; then
  m1="${MODELS%%,*}"
  U1=$(printf '%s' "$ENDPOINTS" | grep "^$m1|" | cut -d'|' -f2-)
  say "smoke test on $m1 (reasoning off → direct answer)..."
  RESP=$(curl -s -m 180 -H "Authorization: Bearer $API_KEY" -H "Content-Type: application/json" \
    "$U1/chat/completions" -d "{\"model\":\"$m1\",
      \"messages\":[{\"role\":\"user\",\"content\":\"In one sentence: should a smallholder cotton farmer sow now if the monsoon is two weeks late?\"}],
      \"max_tokens\":80,\"temperature\":0.3,\"chat_template_kwargs\":{\"enable_thinking\":false}}")
  TXT=$(echo "$RESP" | python3 -c "import sys,json;d=json.load(sys.stdin);m=d['choices'][0]['message'];print((m.get('content') or '').strip()[:160])" 2>/dev/null)
  [ -z "$TXT" ] && { echo "--- response ---"; echo "$RESP" | head -c 400; die "smoke test failed — no content"; }
  say "  model says: \"$TXT\""
  say "  smoke test ✓"
fi

# ---- 6. report ---------------------------------------------------------------
printf '%s' "$ENDPOINTS" > "$WORK/endpoints.txt"
echo
echo "=============================================="
echo " ✅ POD READY"
echo "=============================================="
printf '%s' "$ENDPOINTS" | while IFS='|' read -r n u; do [ -n "$n" ] && echo "  $n  ->  $u"; done
echo "  api key : $API_KEY"
echo "  logs    : $WORK/server-<model>.log"
echo "  stop    : pkill -x llama-server"
echo
echo "  reasoning OFF (baseline) : \"chat_template_kwargs\": {\"enable_thinking\": false}"
echo "  reasoning ON  (condition): \"chat_template_kwargs\": {\"enable_thinking\": true}"
echo "=============================================="