Replace Gradio SmolVLM app with DeepSeek-7B-base GGUF OpenAI-compatible API (Docker)

.dockerignore

@@ -0,0 +1,12 @@
+# Keep the build context (and the image) small and model-free.
+*.gguf
+*.bin
+*.safetensors
+venv/
+.venv/
+__pycache__/
+*.pyc
+.git/
+.github/
+.env
+*.md

.gitignore

@@ -0,0 +1,16 @@
+# Never commit models or caches — the model is served from a HF model repo.
+*.gguf
+*.bin
+*.safetensors
+.cache/
+huggingface/
+hf_home/
+
+# Python
+venv/
+.venv/
+__pycache__/
+*.pyc
+
+# Local env
+.env

Dockerfile

@@ -0,0 +1,52 @@
+# ── GGUF LLM API — Hugging Face Spaces (Docker SDK), CPU tier by default ───────
+# Builds llama-cpp-python from source, so the C/C++ toolchain is installed first.
+FROM python:3.11-slim
+
+# 1) Build toolchain required to compile llama-cpp-python.
+#    build-essential = gcc + g++ + make; cmake is what its build backend invokes.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential \
+        gcc \
+        g++ \
+        make \
+        cmake \
+        git \
+    && rm -rf /var/lib/apt/lists/*
+
+# 2) Hugging Face Spaces runs the container as a non-root user (uid 1000).
+#    Create it and put ALL writable paths under its home so model downloads work.
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH \
+    # HF_HOME is where huggingface_hub caches the downloaded .gguf at startup.
+    # It MUST be writable by uid 1000 — /home/user is, /root and /app often are not.
+    HF_HOME=/home/user/.cache/huggingface \
+    # HF CPU-basic = 2 vCPUs. Pin llama.cpp + BLAS to 2 threads so they don't
+    # over-subscribe the host's core count and thrash. Raise these (or set the
+    # N_THREADS Space variable) if you move to bigger CPU/GPU hardware.
+    N_THREADS=2 \
+    OMP_NUM_THREADS=2
+
+WORKDIR /home/user/app
+
+# 3) Install Python deps first (better layer caching than copying everything).
+#    The --extra-index-url hosts PREBUILT CPU wheels for llama-cpp-python, so pip
+#    installs a wheel instead of compiling from source. This avoids the #1 cause
+#    of failed HF CPU builds: the source compile running out of memory. The gcc/
+#    cmake toolchain above stays only as a fallback if no wheel matches.
+COPY --chown=user requirements.txt ./
+RUN pip install --no-cache-dir --upgrade pip \
+    && pip install --no-cache-dir -r requirements.txt \
+        --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
+
+# 4) Copy the (lightweight) application code. No model file is baked in — it is
+#    pulled from the HF model repo on startup.
+COPY --chown=user . ./
+
+# 5) Hugging Face Spaces routes public traffic to port 7860.
+EXPOSE 7860
+
+# 6) Single worker: llama.cpp holds one non-thread-safe context; the app serialises
+#    calls internally. Do NOT add --workers > 1 (each would reload the whole model).
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,14 +1,66 @@
 ---
-title: Model
-emoji: 👁
+title: DeepSeek 7B GGUF API
+emoji: 🧠
 colorFrom: blue
-colorTo: yellow
-sdk: gradio
-sdk_version: 6.19.0
-python_version: '3.13'
-app_file: app.py
+colorTo: indigo
+sdk: docker
+app_port: 7860
 pinned: false
 license: apache-2.0
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# DeepSeek-LLM-7B-base GGUF API (Docker Space)
+
+An OpenAI-compatible FastAPI server that runs
+**`TheBloke/deepseek-llm-7B-base-GGUF` → `deepseek-llm-7b-base.Q4_K_M.gguf`**
+with `llama-cpp-python`. The model file is **not** stored in this repo — it is
+downloaded from the Hugging Face model hub on container startup, so this code
+repo stays tiny.
+
+> ⚠️ **The YAML block at the very top of this file is required.** Hugging Face
+> reads `sdk: docker` and `app_port: 7860` from it to know how to build and
+> route the Space. Do not delete it.
+
+## Endpoints
+
+Base URL: `https://electro0023-model.hf.space`
+
+| Method | Path                   | Purpose                                       |
+|--------|------------------------|-----------------------------------------------|
+| GET    | `/health`              | Liveness + whether the model finished loading |
+| GET    | `/v1/models`           | OpenAI-style model list                       |
+| POST   | `/v1/chat/completions` | OpenAI-compatible chat completion             |
+| POST   | `/generate`            | Simple `{prompt, max_tokens}` → text          |
+
+## Quick test
+
+```bash
+curl https://electro0023-model.hf.space/health
+
+curl -X POST https://electro0023-model.hf.space/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{"model":"deepseek-7b-base","messages":[{"role":"user","content":"Newton'\''s second law is"}],"max_tokens":64}'
+```
+
+## Swapping models (no code changes)
+
+Set Space **Variables** (Settings → Variables and secrets) and restart:
+
+- `REPO_ID` — HF model repo holding the GGUF
+- `FILENAME` — the exact `.gguf` file inside that repo
+- `MODEL_ID` — name reported to API clients
+- `CHAT_FORMAT` — chat template for instruct models (e.g. `llama-3`, `chatml`);
+  leave empty for base models
+- `N_CTX`, `N_THREADS`, `N_GPU_LAYERS`, `DEFAULT_MAX_TOKENS` — tuning knobs
+
+## Notes
+
+- **This default model is a BASE model** (raw text completion, not
+  instruction-tuned). For chat/instruction behaviour, switch to
+  `TheBloke/deepseek-llm-7B-chat-GGUF` / `deepseek-llm-7b-chat.Q4_K_M.gguf`
+  with `CHAT_FORMAT=deepseek` — or any other instruct GGUF.
+- On the free `cpu-basic` tier (2 vCPU, 16 GB RAM) a 7B Q4_K_M generates
+  roughly 1–3 tokens/sec. Expect long response times; set generous client
+  timeouts.
+- First boot downloads ~4.1 GB, so allow several minutes before `/health`
+  reports `status: ok`.

app.py

@@ -1,84 +0,0 @@
-import subprocess
-import sys
-
-print("Installing dependencies...")
-subprocess.run([
-    sys.executable, "-m", "pip", "install",
-    "--force-reinstall", "--no-deps",
-    "transformers==4.53.0",
-    "tokenizers==0.23.1",
-], check=True)
-print("Done installing")
-
-import torch
-import gradio as gr
-from transformers import AutoProcessor, AutoModelForVision2Seq
-from PIL import Image
-
-print(f"Transformers version: {__import__('transformers').__version__}")
-
-model_id = "HuggingFaceTB/SmolVLM-256M-Instruct"
-
-print("Loading processor...")
-processor = AutoProcessor.from_pretrained(model_id)
-
-print("Loading model...")
-model = AutoModelForVision2Seq.from_pretrained(
-    model_id,
-    torch_dtype=torch.float32,
-    device_map="auto"
-)
-model.eval()
-print("Model ready!")
-
-
-def extract_text(image: Image.Image) -> str:
-    if image is None:
-        return "Please upload an image."
-
-    messages = [
-        {
-            "role": "user",
-            "content": [
-                {"type": "image"},
-                {"type": "text", "text": (
-                    "Extract all text from this image exactly as it appears. "
-                    "Preserve question numbers, options A B C D, "
-                    "tables, and any mathematical or chemical expressions. "
-                    "Format clearly."
-                )}
-            ]
-        }
-    ]
-
-    prompt = processor.apply_chat_template(
-        messages,
-        add_generation_prompt=True
-    )
-
-    inputs = processor(
-        text=prompt,
-        images=[image],
-        return_tensors="pt"
-    ).to(model.device)
-
-    with torch.no_grad():
-        outputs = model.generate(
-            **inputs,
-            max_new_tokens=1024,
-            do_sample=False
-        )
-
-    generated = outputs[0][inputs["input_ids"].shape[1]:]
-    return processor.decode(generated, skip_special_tokens=True)
-
-
-demo = gr.Interface(
-    fn=extract_text,
-    inputs=gr.Image(type="pil", label="Upload NEET Question Image"),
-    outputs=gr.Textbox(label="Extracted Text", lines=20),
-    title="NEET Question Extractor",
-    description="Upload scanned NEET question paper image to extract text"
-)
-
-demo.launch()

main.py

@@ -0,0 +1,250 @@
+"""
+GGUF LLM API for Hugging Face Spaces (Docker SDK) — "Download on Startup".
+
+On boot this app downloads a .gguf model from a Hugging Face **Model** repo with
+huggingface_hub, loads it once with llama-cpp-python, and serves it behind an
+OpenAI-compatible chat endpoint plus a simple /generate endpoint.
+
+Nothing here is model-specific: which repo/file to pull is chosen entirely by
+environment variables, so you redeploy a new model by changing a variable in the
+Space settings — no code edits.
+
+Environment variables
+----------------------
+REPO_ID            (required)  HF model repo, e.g. "your-username/my-gguf-models"
+FILENAME           (required)  the .gguf file inside that repo,
+                               e.g. "mistral-7b-instruct-v0.2.Q4_K_M.gguf"
+HF_TOKEN           (optional)  a READ token — ONLY needed if the model repo is PRIVATE
+MODEL_ID           (optional)  name reported to clients (default: FILENAME)
+N_CTX              (optional)  context window in tokens        (default 4096)
+N_THREADS          (optional)  CPU threads for inference       (default: all cores)
+N_GPU_LAYERS       (optional)  layers to offload to GPU        (default 0 = CPU only)
+CHAT_FORMAT        (optional)  e.g. "llama-3", "chatml", "mistral-instruct"
+DEFAULT_MAX_TOKENS (optional)  fallback max_tokens per request (default 512)
+"""
+import os
+import time
+import uuid
+import logging
+import threading
+from contextlib import asynccontextmanager
+from typing import List, Optional, Dict, Any
+
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel, Field
+from huggingface_hub import hf_hub_download
+from llama_cpp import Llama
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s  %(levelname)s  %(message)s")
+log = logging.getLogger("gguf-api")
+
+# ── Configuration (all via environment) ───────────────────────────────────────
+# Defaults point at DeepSeek-LLM-7B-base (Q4_K_M), a PUBLIC GGUF — so the Space
+# runs with zero setup. Override REPO_ID/FILENAME in the Space settings to swap
+# models without touching the code.
+REPO_ID            = os.getenv("REPO_ID", "TheBloke/deepseek-llm-7B-base-GGUF").strip()
+FILENAME           = os.getenv("FILENAME", "deepseek-llm-7b-base.Q4_K_M.gguf").strip()
+HF_TOKEN           = os.getenv("HF_TOKEN") or None          # None => anonymous (public repo)
+MODEL_ID           = os.getenv("MODEL_ID", "") or "deepseek-7b-base"  # name reported to clients
+N_CTX              = int(os.getenv("N_CTX", "4096"))
+# Default to the number of *usable* CPUs (respects cgroup cpuset), NOT os.cpu_count()
+# which over-reports the host's cores inside a container and causes thread thrash.
+_USABLE_CPUS = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else (os.cpu_count() or 4)
+N_THREADS          = int(os.getenv("N_THREADS", str(_USABLE_CPUS)))
+N_GPU_LAYERS       = int(os.getenv("N_GPU_LAYERS", "0"))    # 0 = CPU only; -1 = all layers on GPU
+# deepseek-llm-7b-BASE has no chat template (it is a raw completion model), so no
+# chat format is forced by default; llama-cpp falls back to its generic template.
+# If you swap to an instruct/chat GGUF, set CHAT_FORMAT accordingly (e.g. "llama-3").
+CHAT_FORMAT        = os.getenv("CHAT_FORMAT", "").strip()
+DEFAULT_MAX_TOKENS = int(os.getenv("DEFAULT_MAX_TOKENS", "512"))
+
+# The loaded model lives here. It is None until startup finishes (or if it failed).
+# llama-cpp is NOT thread-safe for concurrent generation on one context, so every
+# call into the model is serialised behind this lock.
+_llm: Optional[Llama] = None
+_llm_lock = threading.Lock()
+_load_error: Optional[str] = None
+
+
+def _download_and_load() -> None:
+    """Download the .gguf from the HF model repo, then load it with llama-cpp."""
+    global _llm, _load_error
+
+    if not REPO_ID or not FILENAME:
+        _load_error = (
+            "REPO_ID and FILENAME must both be set. In your Space go to "
+            "Settings -> Variables and secrets and add REPO_ID (e.g. "
+            "'your-username/my-gguf-models') and FILENAME (e.g. 'model.Q4_K_M.gguf')."
+        )
+        log.error(_load_error)
+        return
+
+    try:
+        log.info("Downloading model '%s' from repo '%s' ...", FILENAME, REPO_ID)
+        t0 = time.time()
+        # hf_hub_download caches under HF_HOME and returns the local file path.
+        # Re-runs are instant once the file is cached in the container layer.
+        model_path = hf_hub_download(
+            repo_id=REPO_ID,
+            filename=FILENAME,
+            token=HF_TOKEN,             # ignored for public repos, required for private
+        )
+        log.info("Model downloaded to %s (%.1fs)", model_path, time.time() - t0)
+
+        kwargs: Dict[str, Any] = dict(
+            model_path=model_path,
+            n_ctx=N_CTX,
+            n_threads=N_THREADS,
+            n_gpu_layers=N_GPU_LAYERS,
+            verbose=False,
+        )
+        if CHAT_FORMAT:
+            kwargs["chat_format"] = CHAT_FORMAT
+
+        log.info("Loading model into memory (n_ctx=%d, n_gpu_layers=%d) ...", N_CTX, N_GPU_LAYERS)
+        t0 = time.time()
+        _llm = Llama(**kwargs)
+        log.info("Model ready (%.1fs).", time.time() - t0)
+    except Exception as exc:  # keep the server up so /health can report the reason
+        _load_error = f"Failed to download/load model: {exc}"
+        log.exception(_load_error)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Start loading the model in a BACKGROUND thread so the HTTP port opens
+    immediately.
+
+    Hugging Face marks a Space "in error" if port 7860 doesn't open within its
+    startup window. Downloading a ~2 GB model and loading it *before* the server
+    binds would blow past that window (this is exactly what crashes naive Spaces).
+    Loading off-thread lets uvicorn bind 7860 in seconds; /health reports
+    "loading_or_error" until the model is ready, and inference endpoints return a
+    clear 503 until then instead of failing the whole container.
+    """
+    threading.Thread(target=_download_and_load, name="model-loader", daemon=True).start()
+    yield
+    # (nothing to clean up: process exit releases the model)
+
+
+app = FastAPI(title="GGUF LLM API (HF Spaces)", version="1.0.0", lifespan=lifespan)
+
+
+# ── Request/response schemas ──────────────────────────────────────────────────
+class ChatMessage(BaseModel):
+    role: str
+    content: str = ""
+
+
+class ChatCompletionRequest(BaseModel):
+    model: Optional[str] = None
+    messages: List[ChatMessage]
+    temperature: float = 0.7
+    top_p: float = 0.95
+    max_tokens: Optional[int] = None
+    stop: Optional[List[str]] = None
+    # OpenAI JSON mode: {"type":"json_object"} -> llama.cpp grammar-constrains the
+    # reply to VALID JSON. Essential for small models, which otherwise emit
+    # unbalanced brackets (e.g. a stray "]" after each array element).
+    response_format: Optional[Dict[str, Any]] = None
+
+
+class GenerateRequest(BaseModel):
+    prompt: str = Field(..., description="Plain text prompt for the model.",
+                        examples=["State Newton's second law of motion in one sentence."])
+    max_tokens: int = Field(DEFAULT_MAX_TOKENS, description="Max tokens to generate.")
+    temperature: float = Field(0.7, description="Sampling temperature (0 = deterministic).")
+
+
+def _require_model() -> Llama:
+    """Return the loaded model or raise 503 with a helpful message."""
+    if _llm is None:
+        raise HTTPException(
+            status_code=503,
+            detail=_load_error or "Model is still loading, try again in a few seconds.",
+        )
+    return _llm
+
+
+# ── Endpoints ─────────────────────────────────────────────────────────────────
+@app.get("/health")
+def health() -> Dict[str, Any]:
+    """Liveness probe. status='ok' only once the model is loaded and ready."""
+    # Report the CPU picture so thread over-subscription (the #1 cause of slow
+    # CPU generation) is visible: if cpu_count >> usable cores, llama.cpp threads
+    # thrash. N_THREADS is what we actually hand to llama.cpp.
+    usable = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else None
+    return {
+        "status": "ok" if _llm is not None else "loading_or_error",
+        "model": MODEL_ID,
+        "repo_id": REPO_ID,
+        "filename": FILENAME,
+        "n_ctx": N_CTX,
+        "n_gpu_layers": N_GPU_LAYERS,
+        "n_threads": N_THREADS,
+        "cpu_count": os.cpu_count(),
+        "usable_cpus": usable,
+        "error": _load_error,
+    }
+
+
+@app.get("/v1/models")
+def list_models() -> Dict[str, Any]:
+    """OpenAI-shaped model list (a single loaded model)."""
+    return {"object": "list", "data": [{"id": MODEL_ID, "object": "model", "owned_by": "local"}]}
+
+
+@app.post("/v1/chat/completions")
+def chat_completions(req: ChatCompletionRequest) -> Dict[str, Any]:
+    """OpenAI-compatible chat completion (non-streaming)."""
+    llm = _require_model()
+    if not req.messages:
+        raise HTTPException(status_code=400, detail="`messages` must not be empty.")
+
+    kwargs: Dict[str, Any] = dict(
+        messages=[m.model_dump() for m in req.messages],
+        temperature=req.temperature,
+        top_p=req.top_p,
+        max_tokens=req.max_tokens if req.max_tokens is not None else DEFAULT_MAX_TOKENS,
+        stop=req.stop,
+    )
+    # Honour OpenAI JSON mode -> llama.cpp constrains generation to valid JSON.
+    if req.response_format and req.response_format.get("type") == "json_object":
+        kwargs["response_format"] = {"type": "json_object"}
+
+    try:
+        with _llm_lock:  # serialise access to the single llama.cpp context
+            result = llm.create_chat_completion(**kwargs)
+    except Exception as exc:
+        raise HTTPException(status_code=500, detail=f"generation failed: {exc}") from exc
+
+    # llama-cpp already returns an OpenAI-shaped dict; normalise id/model/created.
+    result["id"] = result.get("id") or f"chatcmpl-{uuid.uuid4().hex}"
+    result["model"] = MODEL_ID
+    result["created"] = result.get("created") or int(time.time())
+    return result
+
+
+@app.post("/generate")
+def generate(req: GenerateRequest) -> Dict[str, Any]:
+    """Simplified endpoint: one prompt in, generated text out (plus timing)."""
+    llm = _require_model()
+    t0 = time.time()
+    try:
+        with _llm_lock:
+            result = llm.create_chat_completion(
+                messages=[{"role": "user", "content": req.prompt}],
+                temperature=req.temperature,
+                max_tokens=req.max_tokens,
+            )
+    except Exception as exc:
+        raise HTTPException(status_code=500, detail=f"generation failed: {exc}") from exc
+
+    usage = result.get("usage", {}) or {}
+    return {
+        "prompt": req.prompt,
+        "response": result["choices"][0]["message"]["content"],
+        "model": MODEL_ID,
+        "time_seconds": round(time.time() - t0, 2),
+        "output_tokens": usage.get("completion_tokens"),
+    }

packages.txt

@@ -1 +0,0 @@
-python3-pip

requirements.txt

@@ -1,8 +1,12 @@
-torch
-Pillow
-accelerate
-python-multipart
-einops
-regex
-safetensors
-tokenizers
+# Pinned for reproducible Hugging Face Space builds.
+# Bump these only deliberately — a floating llama-cpp-python version is the most
+# common cause of a Space that built yesterday and fails to build today.
+
+fastapi==0.115.6
+uvicorn[standard]==0.34.0
+pydantic==2.10.4
+huggingface_hub==0.27.1
+
+# llama-cpp-python is compiled from source in the Dockerfile (needs gcc/g++/make/cmake).
+# 0.3.x ships the current llama.cpp; CPU-only build is the default.
+llama-cpp-python==0.3.7