| #!/usr/bin/env bash |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| set -uo pipefail |
|
|
| RUNTIME_URL="https://huggingface.co/PS4Research/harvest-runtime/resolve/main/portable-fat.tar.gz" |
| WORK="${HARVEST_WORK:-/tmp/harvest}" |
| PORT=18080; CTX=65536; SLOTS=32; SMOKE=1; MODELS="" |
| API_KEY="${HARVEST_API_KEY:-sk-harvest-local}" |
|
|
| |
| REGISTRY=' |
| qwen3.6-35b|unsloth/Qwen3.6-35B-A3B-GGUF|Qwen3.6-35B-A3B-UD-Q8_K_XL.gguf|38|enable_thinking |
| glm-4.7-flash|unsloth/GLM-4.7-Flash-GGUF|GLM-4.7-Flash-UD-Q8_K_XL.gguf|36|enable_thinking |
| gemma-4-26b|unsloth/gemma-4-26B-A4B-it-GGUF|gemma-4-26B-A4B-it-UD-Q8_K_XL.gguf|28|enable_thinking |
| nemotron-3-nano-30b|unsloth/Nemotron-3-Nano-30B-A3B-GGUF|Nemotron-3-Nano-30B-A3B-UD-Q8_K_XL.gguf|40|enable_thinking |
| gemma-4-26b-q4|unsloth/gemma-4-26B-A4B-it-GGUF|gemma-4-26B-A4B-it-UD-Q4_K_XL.gguf|16|enable_thinking |
| ' |
|
|
| die() { echo "❌ ERROR: $*" >&2; exit 1; } |
| say() { echo "[$(date +%H:%M:%S)] $*"; } |
| reg_line() { echo "$REGISTRY" | grep "^$1|" | head -1; } |
|
|
| port_free() { |
| python3 -c " |
| import socket, sys |
| s = socket.socket() |
| r = s.connect_ex(('127.0.0.1', $1)) |
| s.close() |
| sys.exit(0 if r != 0 else 1) # connect refused => nobody there => free |
| " 2>/dev/null |
| } |
| next_free_port() { |
| local p=$1 |
| for _ in $(seq 1 50); do port_free "$p" && { echo "$p"; return; }; p=$((p+1)); done |
| die "no free port found from $1" |
| } |
|
|
| while [ $# -gt 0 ]; do |
| case "$1" in |
| --model) MODELS="$2"; shift 2 ;; |
| --port) PORT="$2"; shift 2 ;; |
| --ctx) CTX="$2"; shift 2 ;; |
| --slots) SLOTS="$2"; shift 2 ;; |
| --no-smoke) SMOKE=0; shift ;; |
| --list) echo "$REGISTRY" | grep '|' | awk -F'|' '{printf " %-22s %s (%sGB)\n", $1, $2, $4}'; exit 0 ;; |
| *) die "unknown option: $1" ;; |
| esac |
| done |
| [ -z "$MODELS" ] && die "need --model NAME (see --list)" |
|
|
| echo "==============================================" |
| echo " harvest pod bootstrap" |
| echo "==============================================" |
|
|
| |
| say "preflight..." |
| command -v nvidia-smi >/dev/null 2>&1 || die "no nvidia-smi — no GPU driver on this pod" |
| GPU_NAME=$(nvidia-smi --query-gpu=name --format=csv,noheader | head -1) |
| CC=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '. ') |
| VRAM=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits | head -1) |
| say " GPU: $GPU_NAME (sm_${CC}, ${VRAM} MiB)" |
| case "$CC" in |
| 80|120) : ;; |
| *) echo "⚠️ WARNING: bundle targets sm_80/sm_120; this GPU is sm_${CC}. Expect" >&2 |
| echo " 'no kernel image for device'. Report this arch so we can add it." >&2 ;; |
| esac |
| mkdir -p "$WORK/models" || die "cannot write to $WORK" |
| NEED=2 |
| for m in ${MODELS//,/ }; do |
| L=$(reg_line "$m"); [ -z "$L" ] && die "unknown model '$m' (see --list)" |
| NEED=$((NEED + $(echo "$L" | cut -d'|' -f4))) |
| done |
| AVAIL=$(df -BG --output=avail "$WORK" 2>/dev/null | tail -1 | tr -dc '0-9') |
| if [ -z "${AVAIL:-}" ] || [ "${AVAIL:-0}" -gt 1000000 ]; then |
| say " disk: need ~${NEED}GB, available: unreported (overlay fs) — continuing" |
| else |
| say " disk: need ~${NEED}GB, have ${AVAIL}GB" |
| [ "$AVAIL" -lt "$NEED" ] && die "insufficient disk in $WORK (~${NEED}GB needed, ${AVAIL}GB free)" |
| fi |
|
|
| |
| if [ -x "$WORK/portable-fat/run-server.sh" ]; then |
| say "runtime: cached ✓" |
| else |
| say "runtime: downloading (~888MB)..." |
| curl -fsSL --retry 3 -o "$WORK/portable-fat.tar.gz" "$RUNTIME_URL" || die "bundle download failed" |
| tar xzf "$WORK/portable-fat.tar.gz" -C "$WORK" || die "bundle extract failed" |
| rm -f "$WORK/portable-fat.tar.gz" |
| [ -x "$WORK/portable-fat/run-server.sh" ] || die "bundle is missing run-server.sh" |
| say "runtime: ready ✓ (self-contained — no system CUDA needed)" |
| fi |
|
|
| |
| for m in ${MODELS//,/ }; do |
| L=$(reg_line "$m"); REPO=$(echo "$L" | cut -d'|' -f2); FILE=$(echo "$L" | cut -d'|' -f3); GB=$(echo "$L" | cut -d'|' -f4) |
| DEST="$WORK/models/$FILE" |
| if [ -f "$DEST" ] && [ "$(stat -Lc %s "$DEST" 2>/dev/null || echo 0)" -gt $(( (GB - 3) * 1000000000 )) ]; then |
| say "model $m: cached ✓" |
| else |
| say "model $m: downloading ~${GB}GB ..." |
| t0=$(date +%s) |
| curl -fL -C - --retry 3 -s -o "$DEST" "https://huggingface.co/$REPO/resolve/main/$FILE" \ |
| || die "model download failed: $m" |
| say "model $m: ready ✓ ($(( $(date +%s) - t0 ))s)" |
| fi |
| done |
|
|
| |
| ENDPOINTS="" |
| for m in ${MODELS//,/ }; do |
| L=$(reg_line "$m"); FILE=$(echo "$L" | cut -d'|' -f3) |
| P=$(next_free_port "$PORT"); PORT=$((P + 1)) |
| say "serving $m on :$P ..." |
| |
| setsid nohup "$WORK/portable-fat/run-server.sh" \ |
| -m "$WORK/models/$FILE" -a "$m" \ |
| --host 127.0.0.1 --port "$P" --api-key "$API_KEY" \ |
| -ngl 999 -fa on -ctk q8_0 -ctv q8_0 -c "$CTX" -np "$SLOTS" \ |
| > "$WORK/server-$m.log" 2>&1 < /dev/null & |
| echo $! > "$WORK/server-$m.pid" |
|
|
| ok=0 |
| for i in $(seq 1 400); do |
| |
| if curl -sf -m 3 -H "Authorization: Bearer $API_KEY" "http://127.0.0.1:$P/v1/models" 2>/dev/null | grep -q "\"$m\""; then |
| ok=1; say " up in ${i}s ✓ (verified: serving '$m')"; break |
| fi |
| kill -0 "$(cat "$WORK/server-$m.pid")" 2>/dev/null || { |
| echo "--- server log ---"; tail -20 "$WORK/server-$m.log"; die "$m server process died during startup"; } |
| grep -qi "error while handling\|failed to load\|no kernel image\|address already in use\|bind: " "$WORK/server-$m.log" 2>/dev/null && { |
| echo "--- server log ---"; tail -20 "$WORK/server-$m.log"; die "$m failed to start"; } |
| sleep 1 |
| done |
| [ $ok -eq 1 ] || { echo "--- server log ---"; tail -20 "$WORK/server-$m.log"; die "$m not healthy after 400s"; } |
| ENDPOINTS="${ENDPOINTS}${m}|http://127.0.0.1:$P/v1"$'\n' |
| done |
|
|
| |
| if [ $SMOKE -eq 1 ]; then |
| m1="${MODELS%%,*}" |
| U1=$(printf '%s' "$ENDPOINTS" | grep "^$m1|" | cut -d'|' -f2-) |
| say "smoke test on $m1 (reasoning off → direct answer)..." |
| RESP=$(curl -s -m 180 -H "Authorization: Bearer $API_KEY" -H "Content-Type: application/json" \ |
| "$U1/chat/completions" -d "{\"model\":\"$m1\", |
| \"messages\":[{\"role\":\"user\",\"content\":\"In one sentence: should a smallholder cotton farmer sow now if the monsoon is two weeks late?\"}], |
| \"max_tokens\":80,\"temperature\":0.3,\"chat_template_kwargs\":{\"enable_thinking\":false}}") |
| TXT=$(echo "$RESP" | python3 -c "import sys,json;d=json.load(sys.stdin);m=d['choices'][0]['message'];print((m.get('content') or '').strip()[:160])" 2>/dev/null) |
| [ -z "$TXT" ] && { echo "--- response ---"; echo "$RESP" | head -c 400; die "smoke test failed — no content"; } |
| say " model says: \"$TXT\"" |
| say " smoke test ✓" |
| fi |
|
|
| |
| printf '%s' "$ENDPOINTS" > "$WORK/endpoints.txt" |
| echo |
| echo "==============================================" |
| echo " ✅ POD READY" |
| echo "==============================================" |
| printf '%s' "$ENDPOINTS" | while IFS='|' read -r n u; do [ -n "$n" ] && echo " $n -> $u"; done |
| echo " api key : $API_KEY" |
| echo " logs : $WORK/server-<model>.log" |
| echo " stop : pkill -x llama-server" |
| echo |
| echo " reasoning OFF (baseline) : \"chat_template_kwargs\": {\"enable_thinking\": false}" |
| echo " reasoning ON (condition): \"chat_template_kwargs\": {\"enable_thinking\": true}" |
| echo "==============================================" |
|
|
