Update start_server.sh

start_server.sh

@@ -1,23 +1,37 @@
 #!/bin/bash
 set -e
+export PYTHONUNBUFFERED=1
 
-# Defaults if not passed in
-MODEL_NAME="${MODEL_NAME:-unsloth/llama-2-7b-bnb-4bit}"
-HOST="${HOST:-0.0.0.0}"
-VLLM_PORT="${VLLM_PORT:-8000}"
-TP_SIZE="${TP_SIZE:-1}"
-GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.90}"
+# ================================
+# Fixed configuration for your setup
+# ================================
+MODEL_NAME="unsloth/Llama-3.2-3B-bnb-4bit"
+HOST="0.0.0.0"
+VLLM_PORT="7860"
+CPU_KVCACHE_SPACE="8"     # in GiB
+DTYPE="auto"              # auto, float16, float32, etc.
+VLLM_EXTRA_ARGS=""        # add extra vLLM flags here if needed
 
 echo "[vLLM] Starting server with:"
 echo "  MODEL_NAME=$MODEL_NAME"
 echo "  HOST=$HOST"
 echo "  VLLM_PORT=$VLLM_PORT"
-echo "  TP_SIZE=$TP_SIZE"
-echo "  GPU_MEMORY_UTILIZATION=$GPU_MEMORY_UTILIZATION"
+echo "  CPU_KVCACHE_SPACE=${CPU_KVCACHE_SPACE}GiB"
+echo "  DTYPE=$DTYPE"
+echo "  EXTRA_ARGS=$VLLM_EXTRA_ARGS"
 
-exec python3 -m vllm.entrypoints.openai.api_server \
+# Warn if /sys not mounted (lscpu detection will fail)
+if [ ! -e /sys/devices/system/cpu/possible ]; then
+  echo "[WARN] /sys not mounted — CPU topology detection may fail."
+  echo "       Run with: docker run -v /sys:/sys:ro ..."
+fi
+
+# Start the vLLM CPU server (logs will stream to console)
+exec python3 -u -m vllm.entrypoints.openai.api_server \
     --model "$MODEL_NAME" \
     --host "$HOST" \
     --port "$VLLM_PORT" \
-    --tensor-parallel-size "$TP_SIZE" \
-    --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+    --cpu-offload \
+    --cpu-kv-cache-space "$CPU_KVCACHE_SPACE" \
+    --dtype "$DTYPE" \
+    $VLLM_EXTRA_ARGS