fix: deploy native llama-server sidecar from pre-built tarball (supports n_parallel, fixes fallback mode)

Dockerfile

@@ -1,7 +1,13 @@
 FROM python:3.10-slim
 
-# No system-level binary installs needed — llama-cpp-python server is installed via pip
-
+# Install curl and tar to download and extract pre-built native llama-server
+RUN apt-get update && apt-get install -y curl tar && rm -rf /var/lib/apt/lists/*
+
+# Download native llama-server binary from official llama.cpp releases
+RUN curl -L -s https://github.com/ggml-org/llama.cpp/releases/download/b9964/llama-b9964-bin-ubuntu-x64.tar.gz -o /tmp/llama.tar.gz \
+    && tar -xzf /tmp/llama.tar.gz -C /tmp \
+    && find /tmp -name 'llama-server' -type f | head -1 | xargs -I{} install -m 0755 {} /usr/local/bin/llama-server \
+    && rm -rf /tmp/llama.tar.gz /tmp/bin
 
 WORKDIR /code
 

start.sh

@@ -17,13 +17,34 @@ else
     echo "Model already exists at $MODEL_PATH"
 fi
 
-# 2. Set environment for the FastAPI app (uses llama-cpp-python directly, no sidecar needed)
-export MODEL_PATH="$MODEL_PATH"
+# 2. Start native llama-server in the background (runs on CPU, port 8001)
+echo "Starting native llama-server on port 8001..."
+llama-server \
+    --model "$MODEL_PATH" \
+    --port 8001 \
+    --host 127.0.0.1 \
+    --ctx-size 6144 \
+    --parallel 3 \
+    --cont-batching \
+    --threads 2 \
+    --threads-batch 2 &
+
+# 3. Wait for the local llama.cpp server to be ready
+echo "Waiting for llama.cpp server to initialize..."
+until curl -s http://127.0.0.1:8001/v1/models > /dev/null; do
+    sleep 2
+    echo "Still waiting..."
+done
+echo "llama.cpp server is ready!"
+
+# 4. Set environment variables to point FastAPI to the local llama.cpp server
+export OPENAI_BASE_URL="http://127.0.0.1:8001/v1"
+export LLM_BASE_URL="http://127.0.0.1:8001/v1"
+export MODEL_NAME="qwen2.5-3b-instruct-q4_k_m.gguf"
+export LLM_MODEL_NAME="qwen2.5-3b-instruct-q4_k_m.gguf"
 export LLM_MAX_TOKENS=2048
-export LLM_N_CTX=4096
-export LLM_N_THREADS=2
 
-# 3. Run ingest only if the collection is empty or new files exist
+# 5. Run ingest only if the collection is empty or new files exist
 echo "Checking knowledge base..."
 CHROMA_COUNT=$(python -c "
 import sys
@@ -43,6 +64,6 @@ else
     echo "Knowledge base already indexed ($CHROMA_COUNT chunks). Skipping ingest."
 fi
 
-# 4. Start FastAPI server on port 7860 (Hugging Face default)
+# 6. Start FastAPI server on port 7860 (Hugging Face default)
 echo "Starting FastAPI backend server on port 7860..."
 exec uvicorn app.main:app --host 0.0.0.0 --port 7860