v2: Alpine base + runtime /admin/load (deepseek defaults + template kept)

Dockerfile

@@ -1,52 +1,32 @@
-# ── GGUF LLM API — Hugging Face Spaces (Docker SDK), CPU tier by default ───────
-# Builds llama-cpp-python from source, so the C/C++ toolchain is installed first.
-FROM python:3.11-slim
+# ── GGUF LLM API — HF Spaces (Docker SDK), Alpine base ────────────────────────
+# Alpine (musl libc) is REQUIRED here: the prebuilt llama-cpp-python CPU wheels
+# on abetlen's index are musl builds, and compiling from source exceeds HF's
+# build-job timeout on the free builder. With the wheel, the build is minutes.
+FROM python:3.11-alpine
 
-# 1) Build toolchain required to compile llama-cpp-python.
-#    build-essential = gcc + g++ + make; cmake is what its build backend invokes.
-RUN apt-get update && apt-get install -y --no-install-recommends \
-        build-essential \
-        gcc \
-        g++ \
-        make \
-        cmake \
-        git \
-    && rm -rf /var/lib/apt/lists/*
+# Runtime libraries the llama.cpp shared objects need (C++/OpenMP), plus a
+# compiler-free environment — no gcc/cmake on purpose: wheels only.
+RUN apk add --no-cache libstdc++ libgomp curl
 
-# 2) Hugging Face Spaces runs the container as a non-root user (uid 1000).
-#    Create it and put ALL writable paths under its home so model downloads work.
-RUN useradd -m -u 1000 user
+# HF Spaces runs containers as uid 1000 with writable paths under its home.
+RUN adduser -D -u 1000 user
 USER user
 ENV HOME=/home/user \
     PATH=/home/user/.local/bin:$PATH \
-    # HF_HOME is where huggingface_hub caches the downloaded .gguf at startup.
-    # It MUST be writable by uid 1000 — /home/user is, /root and /app often are not.
     HF_HOME=/home/user/.cache/huggingface \
-    # HF CPU-basic = 2 vCPUs. Pin llama.cpp + BLAS to 2 threads so they don't
-    # over-subscribe the host's core count and thrash. Raise these (or set the
-    # N_THREADS Space variable) if you move to bigger CPU/GPU hardware.
     N_THREADS=2 \
     OMP_NUM_THREADS=2
 
 WORKDIR /home/user/app
 
-# 3) Install Python deps first (better layer caching than copying everything).
-#    The --extra-index-url hosts PREBUILT CPU wheels for llama-cpp-python, so pip
-#    installs a wheel instead of compiling from source. This avoids the #1 cause
-#    of failed HF CPU builds: the source compile running out of memory. The gcc/
-#    cmake toolchain above stays only as a fallback if no wheel matches.
 COPY --chown=user requirements.txt ./
+# --only-binary: fail loudly if any dependency would need a compiler instead of
+# silently hitting the build timeout again.
 RUN pip install --no-cache-dir --upgrade pip \
-    && pip install --no-cache-dir -r requirements.txt \
+    && pip install --no-cache-dir --only-binary=:all: -r requirements.txt \
         --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
 
-# 4) Copy the (lightweight) application code. No model file is baked in — it is
-#    pulled from the HF model repo on startup.
 COPY --chown=user . ./
 
-# 5) Hugging Face Spaces routes public traffic to port 7860.
 EXPOSE 7860
-
-# 6) Single worker: llama.cpp holds one non-thread-safe context; the app serialises
-#    calls internally. Do NOT add --workers > 1 (each would reload the whole model).
 CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

main.py

@@ -1,26 +1,13 @@
-"""
-GGUF LLM API for Hugging Face Spaces (Docker SDK) — "Download on Startup".
-
-On boot this app downloads a .gguf model from a Hugging Face **Model** repo with
-huggingface_hub, loads it once with llama-cpp-python, and serves it behind an
-OpenAI-compatible chat endpoint plus a simple /generate endpoint.
-
-Nothing here is model-specific: which repo/file to pull is chosen entirely by
-environment variables, so you redeploy a new model by changing a variable in the
-Space settings — no code edits.
-
-Environment variables
-----------------------
-REPO_ID            (required)  HF model repo, e.g. "your-username/my-gguf-models"
-FILENAME           (required)  the .gguf file inside that repo,
-                               e.g. "mistral-7b-instruct-v0.2.Q4_K_M.gguf"
-HF_TOKEN           (optional)  a READ token — ONLY needed if the model repo is PRIVATE
-MODEL_ID           (optional)  name reported to clients (default: FILENAME)
-N_CTX              (optional)  context window in tokens        (default 4096)
-N_THREADS          (optional)  CPU threads for inference       (default: all cores)
-N_GPU_LAYERS       (optional)  layers to offload to GPU        (default 0 = CPU only)
-CHAT_FORMAT        (optional)  e.g. "llama-3", "chatml", "mistral-instruct"
-DEFAULT_MAX_TOKENS (optional)  fallback max_tokens per request (default 512)
+"""GGUF LLM API for Hugging Face Spaces (Docker SDK) — runtime-swappable models.
+
+Downloads a .gguf from a HF model repo, loads it with llama-cpp-python and
+serves an OpenAI-compatible API. The initial model comes from environment
+variables; afterwards a NEW model can be loaded at runtime via POST /admin/load
+— no Space variable change, therefore NO image rebuild.
+
+Environment variables (initial model only):
+    REPO_ID, FILENAME, HF_TOKEN, MODEL_ID, N_CTX, N_THREADS, N_GPU_LAYERS,
+    CHAT_FORMAT ("" = auto from GGUF metadata), DEFAULT_MAX_TOKENS
 """
 import os
 import time
@@ -39,9 +26,10 @@ from llama_cpp import llama_chat_format
 logging.basicConfig(level=logging.INFO, format="%(asctime)s  %(levelname)s  %(message)s")
 log = logging.getLogger("gguf-api")
 
-# DeepSeek-LLM chat template — NOT built into llama-cpp-python (and TheBloke's
-# 2023 GGUFs predate embedded chat_template metadata), so register it ourselves.
-# Official format:  [system\n\n]User: {msg}\n\nAssistant: {reply}<eos>...Assistant:
+
+# DeepSeek-LLM chat template — not built into llama-cpp-python, and TheBloke's
+# 2023 GGUFs predate embedded chat_template metadata. Registered here so this
+# Space can keep serving deepseek-7b-chat (its default model).
 @llama_chat_format.register_chat_format("deepseek")
 def _format_deepseek(messages, **kwargs) -> llama_chat_format.ChatFormatterResponse:
     prompt = ""
@@ -56,100 +44,65 @@ def _format_deepseek(messages, **kwargs) -> llama_chat_format.ChatFormatterRespo
     prompt += "Assistant:"
     return llama_chat_format.ChatFormatterResponse(prompt=prompt, stop=["User:"])
 
-
-# ── Configuration (all via environment) ───────────────────────────────────────
-# Defaults point at DeepSeek-LLM-7B-CHAT (Q4_K_M), a PUBLIC GGUF — so the Space
-# runs with zero setup. Override REPO_ID/FILENAME in the Space settings to swap
-# models without touching the code.
-REPO_ID            = os.getenv("REPO_ID", "TheBloke/deepseek-llm-7B-chat-GGUF").strip()
-FILENAME           = os.getenv("FILENAME", "deepseek-llm-7b-chat.Q4_K_M.gguf").strip()
-HF_TOKEN           = os.getenv("HF_TOKEN") or None          # None => anonymous (public repo)
-MODEL_ID           = os.getenv("MODEL_ID", "") or "deepseek-7b-chat"  # name reported to clients
+HF_TOKEN           = os.getenv("HF_TOKEN") or None
 N_CTX              = int(os.getenv("N_CTX", "4096"))
-# Default to the number of *usable* CPUs (respects cgroup cpuset), NOT os.cpu_count()
-# which over-reports the host's cores inside a container and causes thread thrash.
-_USABLE_CPUS = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else (os.cpu_count() or 4)
-N_THREADS          = int(os.getenv("N_THREADS", str(_USABLE_CPUS)))
-N_GPU_LAYERS       = int(os.getenv("N_GPU_LAYERS", "0"))    # 0 = CPU only; -1 = all layers on GPU
-# "deepseek" = the custom template registered above, matching the default chat
-# model. Set to "" (empty) when running a BASE model — the endpoints then bypass
-# chat templating and use raw text completion instead.
-CHAT_FORMAT        = os.getenv("CHAT_FORMAT", "deepseek").strip()
+_USABLE = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else (os.cpu_count() or 4)
+N_THREADS          = int(os.getenv("N_THREADS", str(_USABLE)))
+N_GPU_LAYERS       = int(os.getenv("N_GPU_LAYERS", "0"))
 DEFAULT_MAX_TOKENS = int(os.getenv("DEFAULT_MAX_TOKENS", "512"))
 
-# The loaded model lives here. It is None until startup finishes (or if it failed).
-# llama-cpp is NOT thread-safe for concurrent generation on one context, so every
-# call into the model is serialised behind this lock.
+# Current model state (mutable at runtime via /admin/load).
 _llm: Optional[Llama] = None
 _llm_lock = threading.Lock()
 _load_error: Optional[str] = None
-
-
-def _download_and_load() -> None:
-    """Download the .gguf from the HF model repo, then load it with llama-cpp."""
-    global _llm, _load_error
-
-    if not REPO_ID or not FILENAME:
-        _load_error = (
-            "REPO_ID and FILENAME must both be set. In your Space go to "
-            "Settings -> Variables and secrets and add REPO_ID (e.g. "
-            "'your-username/my-gguf-models') and FILENAME (e.g. 'model.Q4_K_M.gguf')."
-        )
-        log.error(_load_error)
-        return
-
+_loading: bool = False
+_cfg: Dict[str, str] = {
+    "repo_id":     os.getenv("REPO_ID", "TheBloke/deepseek-llm-7B-chat-GGUF").strip(),
+    "filename":    os.getenv("FILENAME", "deepseek-llm-7b-chat.Q4_K_M.gguf").strip(),
+    "model_id":    (os.getenv("MODEL_ID", "") or "deepseek-7b-chat").strip(),
+    "chat_format": os.getenv("CHAT_FORMAT", "deepseek").strip(),
+}
+
+
+def _download_and_load(cfg: Dict[str, str]) -> None:
+    """Download cfg's .gguf and load it, replacing the current model."""
+    global _llm, _load_error, _loading, _cfg
+    _loading = True
+    _load_error = None
     try:
-        log.info("Downloading model '%s' from repo '%s' ...", FILENAME, REPO_ID)
+        log.info("Downloading %s from %s ...", cfg["filename"], cfg["repo_id"])
         t0 = time.time()
-        # hf_hub_download caches under HF_HOME and returns the local file path.
-        # Re-runs are instant once the file is cached in the container layer.
-        model_path = hf_hub_download(
-            repo_id=REPO_ID,
-            filename=FILENAME,
-            token=HF_TOKEN,             # ignored for public repos, required for private
-        )
-        log.info("Model downloaded to %s (%.1fs)", model_path, time.time() - t0)
-
-        kwargs: Dict[str, Any] = dict(
-            model_path=model_path,
-            n_ctx=N_CTX,
-            n_threads=N_THREADS,
-            n_gpu_layers=N_GPU_LAYERS,
-            verbose=False,
-        )
-        if CHAT_FORMAT:
-            kwargs["chat_format"] = CHAT_FORMAT
-
-        log.info("Loading model into memory (n_ctx=%d, n_gpu_layers=%d) ...", N_CTX, N_GPU_LAYERS)
-        t0 = time.time()
-        _llm = Llama(**kwargs)
-        log.info("Model ready (%.1fs).", time.time() - t0)
-    except Exception as exc:  # keep the server up so /health can report the reason
-        _load_error = f"Failed to download/load model: {exc}"
+        path = hf_hub_download(repo_id=cfg["repo_id"], filename=cfg["filename"],
+                               token=HF_TOKEN)
+        log.info("Downloaded in %.1fs; loading ...", time.time() - t0)
+        kwargs: Dict[str, Any] = dict(model_path=path, n_ctx=N_CTX,
+                                      n_threads=N_THREADS, n_gpu_layers=N_GPU_LAYERS,
+                                      verbose=False)
+        if cfg["chat_format"]:
+            kwargs["chat_format"] = cfg["chat_format"]
+        new_llm = Llama(**kwargs)
+        with _llm_lock:                      # swap atomically; old model is freed
+            _llm = new_llm
+            _cfg = dict(cfg)
+        log.info("Model '%s' ready.", cfg["model_id"])
+    except Exception as exc:
+        _load_error = f"Failed to load {cfg['model_id']}: {exc}"
         log.exception(_load_error)
+    finally:
+        _loading = False
 
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    """Start loading the model in a BACKGROUND thread so the HTTP port opens
-    immediately.
-
-    Hugging Face marks a Space "in error" if port 7860 doesn't open within its
-    startup window. Downloading a ~2 GB model and loading it *before* the server
-    binds would blow past that window (this is exactly what crashes naive Spaces).
-    Loading off-thread lets uvicorn bind 7860 in seconds; /health reports
-    "loading_or_error" until the model is ready, and inference endpoints return a
-    clear 503 until then instead of failing the whole container.
-    """
-    threading.Thread(target=_download_and_load, name="model-loader", daemon=True).start()
+    # Load the initial model in the background so port 7860 opens immediately.
+    threading.Thread(target=_download_and_load, args=(dict(_cfg),),
+                     name="model-loader", daemon=True).start()
     yield
-    # (nothing to clean up: process exit releases the model)
 
 
-app = FastAPI(title="GGUF LLM API (HF Spaces)", version="1.0.0", lifespan=lifespan)
+app = FastAPI(title="GGUF LLM API (runtime-swappable)", version="2.0.0", lifespan=lifespan)
 
 
-# ── Request/response schemas ──────────────────────────────────────────────────
 class ChatMessage(BaseModel):
     role: str
     content: str = ""
@@ -162,164 +115,96 @@ class ChatCompletionRequest(BaseModel):
     top_p: float = 0.95
     max_tokens: Optional[int] = None
     stop: Optional[List[str]] = None
-    # OpenAI JSON mode: {"type":"json_object"} -> llama.cpp grammar-constrains the
-    # reply to VALID JSON. Essential for small models, which otherwise emit
-    # unbalanced brackets (e.g. a stray "]" after each array element).
     response_format: Optional[Dict[str, Any]] = None
 
 
 class GenerateRequest(BaseModel):
-    prompt: str = Field(..., description="Plain text prompt for the model.",
-                        examples=["State Newton's second law of motion in one sentence."])
-    max_tokens: int = Field(DEFAULT_MAX_TOKENS, description="Max tokens to generate.")
-    temperature: float = Field(0.7, description="Sampling temperature (0 = deterministic).")
-
-
-def _require_model() -> Llama:
-    """Return the loaded model or raise 503 with a helpful message."""
-    if _llm is None:
-        raise HTTPException(
-            status_code=503,
-            detail=_load_error or "Model is still loading, try again in a few seconds.",
-        )
-    return _llm
-
+    prompt: str
+    max_tokens: int = Field(default=512)
+    temperature: float = 0.7
 
-def _messages_to_raw_prompt(messages: List["ChatMessage"]) -> str:
-    """Flatten chat messages into a plain-text prompt for BASE (non-chat) models.
 
-    A base model was never trained on a chat template — wrapping its input in
-    one (e.g. llama-2 [INST] tags) makes it emit only <s>/</s> tokens. Instead
-    we join the message contents into plain text and let the model continue it.
-    """
-    parts = [m.content.strip() for m in messages if m.content and m.content.strip()]
-    return "\n\n".join(parts)
+class LoadRequest(BaseModel):
+    repo_id: str
+    filename: str
+    model_id: str
+    chat_format: str = ""     # "" = auto from GGUF metadata
 
 
-def _raw_chat_completion(llm: Llama, req: "ChatCompletionRequest") -> Dict[str, Any]:
-    """Text-completion path used when CHAT_FORMAT is empty (base model)."""
-    prompt = _messages_to_raw_prompt(req.messages)
-    result = llm.create_completion(
-        prompt=prompt,
-        temperature=req.temperature,
-        top_p=req.top_p,
-        max_tokens=req.max_tokens if req.max_tokens is not None else DEFAULT_MAX_TOKENS,
-        stop=req.stop,
-    )
-    text = result["choices"][0]["text"]
-    # Re-shape the completion result into an OpenAI chat-completion response so
-    # clients (which always call /v1/chat/completions) need no changes.
-    return {
-        "id": f"chatcmpl-{uuid.uuid4().hex}",
-        "object": "chat.completion",
-        "created": int(time.time()),
-        "model": MODEL_ID,
-        "choices": [{
-            "index": 0,
-            "message": {"role": "assistant", "content": text},
-            "finish_reason": result["choices"][0].get("finish_reason", "stop"),
-        }],
-        "usage": result.get("usage", {}),
-    }
+def _require_model() -> Llama:
+    if _llm is None or _loading:
+        raise HTTPException(503, _load_error or "Model is loading, try again shortly.")
+    return _llm
 
 
-# ── Endpoints ─────────────────────────────────────────────────────────────────
 @app.get("/health")
 def health() -> Dict[str, Any]:
-    """Liveness probe. status='ok' only once the model is loaded and ready."""
-    # Report the CPU picture so thread over-subscription (the #1 cause of slow
-    # CPU generation) is visible: if cpu_count >> usable cores, llama.cpp threads
-    # thrash. N_THREADS is what we actually hand to llama.cpp.
-    usable = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else None
     return {
-        "status": "ok" if _llm is not None else "loading_or_error",
-        "model": MODEL_ID,
-        "repo_id": REPO_ID,
-        "filename": FILENAME,
-        "n_ctx": N_CTX,
-        "n_gpu_layers": N_GPU_LAYERS,
-        "n_threads": N_THREADS,
-        "cpu_count": os.cpu_count(),
-        "usable_cpus": usable,
-        "error": _load_error,
+        "status": "ok" if (_llm is not None and not _loading) else "loading_or_error",
+        "model": _cfg["model_id"], "repo_id": _cfg["repo_id"],
+        "filename": _cfg["filename"], "chat_format": _cfg["chat_format"],
+        "loading": _loading, "n_ctx": N_CTX, "n_threads": N_THREADS,
+        "n_gpu_layers": N_GPU_LAYERS, "error": _load_error,
     }
 
 
+@app.post("/admin/load")
+def admin_load(req: LoadRequest) -> Dict[str, Any]:
+    """Load a different GGUF at runtime (async; poll /health until status ok)."""
+    global _loading
+    if _loading:
+        raise HTTPException(409, "another model is already loading")
+    cfg = {"repo_id": req.repo_id.strip(), "filename": req.filename.strip(),
+           "model_id": req.model_id.strip(), "chat_format": req.chat_format.strip()}
+    threading.Thread(target=_download_and_load, args=(cfg,),
+                     name="model-loader", daemon=True).start()
+    return {"status": "loading", "model": cfg["model_id"]}
+
+
 @app.get("/v1/models")
 def list_models() -> Dict[str, Any]:
-    """OpenAI-shaped model list (a single loaded model)."""
-    return {"object": "list", "data": [{"id": MODEL_ID, "object": "model", "owned_by": "local"}]}
+    return {"object": "list",
+            "data": [{"id": _cfg["model_id"], "object": "model", "owned_by": "local"}]}
 
 
 @app.post("/v1/chat/completions")
 def chat_completions(req: ChatCompletionRequest) -> Dict[str, Any]:
-    """OpenAI-compatible chat completion (non-streaming)."""
     llm = _require_model()
     if not req.messages:
-        raise HTTPException(status_code=400, detail="`messages` must not be empty.")
-
-    # Base model (no chat template): use raw text completion instead of a chat
-    # template the model was never trained on.
-    if not CHAT_FORMAT:
-        try:
-            with _llm_lock:
-                return _raw_chat_completion(llm, req)
-        except Exception as exc:
-            raise HTTPException(status_code=500, detail=f"generation failed: {exc}") from exc
-
+        raise HTTPException(400, "`messages` must not be empty.")
     kwargs: Dict[str, Any] = dict(
         messages=[m.model_dump() for m in req.messages],
-        temperature=req.temperature,
-        top_p=req.top_p,
+        temperature=req.temperature, top_p=req.top_p,
         max_tokens=req.max_tokens if req.max_tokens is not None else DEFAULT_MAX_TOKENS,
         stop=req.stop,
     )
-    # Honour OpenAI JSON mode -> llama.cpp constrains generation to valid JSON.
     if req.response_format and req.response_format.get("type") == "json_object":
         kwargs["response_format"] = {"type": "json_object"}
-
     try:
-        with _llm_lock:  # serialise access to the single llama.cpp context
+        with _llm_lock:
             result = llm.create_chat_completion(**kwargs)
     except Exception as exc:
-        raise HTTPException(status_code=500, detail=f"generation failed: {exc}") from exc
-
-    # llama-cpp already returns an OpenAI-shaped dict; normalise id/model/created.
+        raise HTTPException(500, f"generation failed: {exc}") from exc
     result["id"] = result.get("id") or f"chatcmpl-{uuid.uuid4().hex}"
-    result["model"] = MODEL_ID
+    result["model"] = _cfg["model_id"]
     result["created"] = result.get("created") or int(time.time())
     return result
 
 
 @app.post("/generate")
 def generate(req: GenerateRequest) -> Dict[str, Any]:
-    """Simplified endpoint: one prompt in, generated text out (plus timing)."""
     llm = _require_model()
     t0 = time.time()
     try:
         with _llm_lock:
-            if not CHAT_FORMAT:  # base model -> raw continuation of the prompt
-                raw = llm.create_completion(prompt=req.prompt,
-                                            temperature=req.temperature,
-                                            max_tokens=req.max_tokens)
-                result = {
-                    "choices": [{"message": {"content": raw["choices"][0]["text"]}}],
-                    "usage": raw.get("usage", {}),
-                }
-            else:
-                result = llm.create_chat_completion(
-                    messages=[{"role": "user", "content": req.prompt}],
-                    temperature=req.temperature,
-                    max_tokens=req.max_tokens,
-                )
+            result = llm.create_chat_completion(
+                messages=[{"role": "user", "content": req.prompt}],
+                temperature=req.temperature, max_tokens=req.max_tokens)
     except Exception as exc:
-        raise HTTPException(status_code=500, detail=f"generation failed: {exc}") from exc
-
+        raise HTTPException(500, f"generation failed: {exc}") from exc
     usage = result.get("usage", {}) or {}
-    return {
-        "prompt": req.prompt,
-        "response": result["choices"][0]["message"]["content"],
-        "model": MODEL_ID,
-        "time_seconds": round(time.time() - t0, 2),
-        "output_tokens": usage.get("completion_tokens"),
-    }
+    return {"prompt": req.prompt,
+            "response": result["choices"][0]["message"]["content"],
+            "model": _cfg["model_id"],
+            "time_seconds": round(time.time() - t0, 2),
+            "output_tokens": usage.get("completion_tokens")}

requirements.txt

@@ -1,12 +1,13 @@
-# Pinned for reproducible Hugging Face Space builds.
-# Bump these only deliberately — a floating llama-cpp-python version is the most
-# common cause of a Space that built yesterday and fails to build today.
+# Pinned for reproducible builds. ALL packages must install as prebuilt wheels
+# (the Alpine image ships no compiler - see Dockerfile --only-binary).
 
 fastapi==0.115.6
-uvicorn[standard]==0.34.0
+# plain uvicorn (not [standard]): avoids uvloop/httptools, which may lack
+# musllinux wheels for this pin - h11 worker is fine for a single-model server.
+uvicorn==0.34.0
 pydantic==2.10.4
 huggingface_hub==0.27.1
 
-# llama-cpp-python is compiled from source in the Dockerfile (needs gcc/g++/make/cmake).
-# 0.3.x ships the current llama.cpp; CPU-only build is the default.
-llama-cpp-python==0.3.7
+# musl (Alpine) prebuilt CPU wheel from abetlen's index - 2025 llama.cpp with
+# qwen3 / gemma3 / granite architecture support.
+llama-cpp-python==0.3.19