Add optimized llama.cpp CPU inference backend

.dockerignore

@@ -0,0 +1,9 @@
+.git
+.venv
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+.env
+models
+hf-cache

.gitignore

@@ -0,0 +1 @@
+**/*.env

Dockerfile

@@ -0,0 +1,62 @@
+FROM debian:bookworm-slim AS builder
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    build-essential \
+    cmake \
+    libopenblas-dev \
+    ca-certificates \
+  && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /opt
+RUN git clone --depth 1 https://github.com/ggml-org/llama.cpp.git
+
+WORKDIR /opt/llama.cpp
+RUN cmake -B build -S . \
+    -DGGML_BLAS=ON \
+    -DGGML_BLAS_VENDOR=OpenBLAS \
+    -DGGML_NATIVE=ON \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON \
+    -DCMAKE_C_FLAGS_RELEASE="-Ofast -march=native -flto" \
+    -DCMAKE_CXX_FLAGS_RELEASE="-Ofast -march=native -flto" \
+  && cmake --build build -j"$(nproc)" --target llama-server
+
+FROM debian:bookworm-slim
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    PYTHONUNBUFFERED=1 \
+    HF_HUB_DISABLE_TELEMETRY=1 \
+    HF_HOME=/data/hf-cache \
+    MODEL_DIR=/data/models \
+    MODEL_PATH=/data/models/model.gguf
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    python3 \
+    python3-pip \
+    ca-certificates \
+    libopenblas0-pthread \
+    libgomp1 \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /opt/llama.cpp/build/bin/llama-server /usr/local/bin/llama-server
+
+WORKDIR /app
+COPY requirements.txt /app/requirements.txt
+RUN python3 -m pip install --no-cache-dir --upgrade pip \
+  && python3 -m pip install --no-cache-dir -r /app/requirements.txt \
+  && rm -rf /root/.cache/pip
+
+COPY download_model.py /app/download_model.py
+COPY start.sh /app/start.sh
+RUN chmod +x /app/start.sh \
+  && mkdir -p /data/models /data/hf-cache \
+  && chown -R 65532:65532 /app /data
+
+EXPOSE 7860
+
+USER 65532:65532
+
+ENTRYPOINT ["/app/start.sh"]

README.md

@@ -7,4 +7,100 @@ sdk: docker
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# CPU Inference Backend
+
+This Space is a backend-only OpenAI-compatible inference server built on `llama.cpp` and optimized for CPU-only Hugging Face Docker Spaces.
+
+It automatically downloads the target Hugging Face model from `MODEL_NAME`, stores it under `/data/models/model.gguf`, and starts `llama-server` with OpenAI-compatible endpoints.
+
+## Environment variables
+
+Set these in the Space settings:
+
+- `MODEL_NAME`: Hugging Face repo id that contains one or more GGUF files
+- `HF_TOKEN`: Hugging Face token used for the download
+- `API_PASSWORD`: bearer token required by the API
+
+Optional tuning variables:
+
+- `CTX_SIZE`: context size, default `4096`
+- `THREADS`: CPU thread count, default `2`
+- `THREADS_BATCH`: batch thread count, default matches `THREADS`
+- `BATCH_SIZE`: prompt batch size, default `512`
+- `UBATCH_SIZE`: micro-batch size, default `512`
+- `CACHE_TYPE_K`: KV cache type for keys, default `q8_0`
+- `CACHE_TYPE_V`: KV cache type for values, default `q8_0`
+- `PARALLEL`: optional override for concurrent prompt decode slots
+- `PORT`: listen port, default `7860`
+
+## Endpoints
+
+The server exposes:
+
+- `/v1/chat/completions`
+- `/v1/completions`
+- `/v1/models`
+
+Streaming is supported by `llama-server`. Bearer token authentication uses `--api-key` directly.
+
+## Build and runtime
+
+The Docker image uses a multi-stage build on Ubuntu 24.04. The builder stage installs `git`, `build-essential`, `cmake`, and `libopenblas-dev`, clones `llama.cpp`, and compiles `llama-server` with `-Ofast`, `-march=native`, `-flto`, OpenBLAS, and native CPU optimizations. The runtime stage keeps only Python, OpenBLAS runtime libraries, the compiled server, and the downloader.
+
+Model downloads and HF cache live on the `/data` bucket so restarts do not redownload the model.
+
+## curl examples
+
+```bash
+curl http://localhost:7860/v1/models \
+  -H "Authorization: Bearer $API_PASSWORD"
+```
+
+```bash
+curl http://localhost:7860/v1/chat/completions \
+  -H "Authorization: Bearer $API_PASSWORD" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "model",
+    "messages": [
+      {"role": "user", "content": "Write a one-sentence summary of llama.cpp."}
+    ],
+    "stream": false
+  }'
+```
+
+```bash
+curl http://localhost:7860/v1/completions \
+  -H "Authorization: Bearer $API_PASSWORD" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "model",
+    "prompt": "Explain KV cache in one paragraph.",
+    "max_tokens": 128,
+    "stream": false
+  }'
+```
+
+## OpenAI SDK example
+
+```python
+import os
+
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="http://localhost:7860/v1",
+    api_key=os.environ["API_PASSWORD"],
+)
+
+response = client.chat.completions.create(
+    model="model",
+    messages=[{"role": "user", "content": "Hello"}],
+)
+
+print(response.choices[0].message.content)
+```
+
+## Hugging Face Spaces notes
+
+This repository is ready for a Docker Space with no frontend. The only required changes at deployment time are the three environment variables above.

download_model.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+
+import os
+import shutil
+from pathlib import Path
+
+from huggingface_hub import snapshot_download
+
+
+def require_env(name: str) -> str:
+    value = os.environ.get(name, "").strip()
+    if not value:
+        raise SystemExit(f"missing required environment variable: {name}")
+    return value
+
+
+def quant_rank(path: Path) -> tuple[int, int, str]:
+    name = path.name.lower()
+    size = path.stat().st_size
+
+    q8_tokens = ("q8_k_l", "q8_k_m", "q8_k_s", "q8_k", "q8_0", "q8_1")
+    if any(token in name for token in q8_tokens):
+        return (0, -size, path.name)
+
+    if "f16" in name or "bf16" in name:
+        return (1, -size, path.name)
+
+    if any(token in name for token in ("q6_k", "q6_0", "q6_1", "q6")):
+        return (2, -size, path.name)
+
+    if any(token in name for token in ("q5_k_m", "q5_k_s", "q5_k", "q5_0", "q5_1", "q5")):
+        return (3, -size, path.name)
+
+    if any(token in name for token in ("q4_k_m", "q4_k_s", "q4_k", "q4_0", "q4_1", "q4")):
+        return (4, -size, path.name)
+
+    if any(token in name for token in ("q3_k_m", "q3_k_s", "q3_k", "q3_0", "q3_1", "q3")):
+        return (5, -size, path.name)
+
+    if any(token in name for token in ("q2_k", "q2_0", "q2_1", "q2")):
+        return (6, -size, path.name)
+
+    if "f32" in name:
+        return (7, -size, path.name)
+
+    return (8, -size, path.name)
+
+
+def main() -> None:
+    model_name = require_env("MODEL_NAME")
+    hf_token = require_env("HF_TOKEN")
+    require_env("API_PASSWORD")
+
+    models_dir = Path(os.environ.get("MODEL_DIR", "/data/models"))
+    models_dir.mkdir(parents=True, exist_ok=True)
+
+    target_model = models_dir / "model.gguf"
+    repo_marker = models_dir / "model.repo"
+    download_dir = models_dir / ".download"
+
+    if target_model.exists() and repo_marker.exists() and repo_marker.read_text(encoding="utf-8").strip() == model_name:
+        print(f"using cached model: {target_model}")
+        return
+
+    if download_dir.exists():
+        shutil.rmtree(download_dir)
+    download_dir.mkdir(parents=True, exist_ok=True)
+
+    print(f"downloading model: {model_name}")
+    snapshot_download(
+        repo_id=model_name,
+        token=hf_token,
+        allow_patterns=["*.gguf"],
+        local_dir=str(download_dir),
+        local_dir_use_symlinks=False,
+        cache_dir=os.environ.get("HF_HOME", "/data/hf-cache"),
+    )
+
+    candidates = [path for path in download_dir.rglob("*.gguf") if path.is_file()]
+    if not candidates:
+        raise SystemExit(f"no gguf files found in repository: {model_name}")
+
+    selected = min(candidates, key=quant_rank)
+    print(f"selected model: {selected.name}")
+
+    if target_model.exists():
+        target_model.unlink()
+    shutil.move(str(selected), str(target_model))
+
+    if repo_marker.exists():
+        repo_marker.unlink()
+    repo_marker.write_text(model_name, encoding="utf-8")
+
+    shutil.rmtree(download_dir, ignore_errors=True)
+    print(f"ready: {target_model}")
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1 @@
+huggingface_hub>=0.24,<1

start.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+THREADS="${THREADS:-$(nproc)}"
+export OMP_NUM_THREADS="$THREADS"
+export OPENBLAS_NUM_THREADS="$THREADS"
+export MKL_NUM_THREADS="$THREADS"
+export VECLIB_MAXIMUM_THREADS="$THREADS"
+export NUMEXPR_NUM_THREADS="$THREADS"
+export HF_HOME="${HF_HOME:-/data/hf-cache}"
+export MODEL_DIR="${MODEL_DIR:-/data/models}"
+export MODEL_PATH="${MODEL_PATH:-/data/models/model.gguf}"
+
+python3 /app/download_model.py
+
+LLAMA_SERVER_BIN="${LLAMA_SERVER_BIN:-/usr/local/bin/llama-server}"
+
+server_args=(
+  --model "$MODEL_PATH"
+  --host 0.0.0.0
+  --port "${PORT:-7860}"
+  --api-key "$API_PASSWORD"
+  --mmap
+  -ngl 0
+  -t "$THREADS"
+  --threads-batch "${THREADS_BATCH:-$THREADS}"
+  --batch-size "${BATCH_SIZE:-512}"
+  --ubatch-size "${UBATCH_SIZE:-512}"
+  --ctx-size "${CTX_SIZE:-4096}"
+  --cache-type-k "${CACHE_TYPE_K:-f16}"
+  --cache-type-v "${CACHE_TYPE_V:-f16}"
+  --metrics
+)
+
+if [[ -n "${PARALLEL:-}" ]]; then
+  server_args+=(--parallel "$PARALLEL")
+fi
+
+exec "$LLAMA_SERVER_BIN" "${server_args[@]}"