Upload 5 files

Dockerfile

@@ -0,0 +1,53 @@
+# ============================================================
+# Qwen3-14B  –  OpenAI-compatible API  –  CPU-only Docker image
+# ============================================================
+FROM python:3.11-slim
+
+# Build-time deps for llama-cpp-python (needs a C++ compiler)
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential \
+        cmake \
+        git \
+        wget \
+        ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# ── Python deps ──────────────────────────────────────────────
+# Install everything EXCEPT llama-cpp-python first (lighter layer)
+COPY requirements.txt .
+RUN pip install --no-cache-dir fastapi==0.111.0 uvicorn[standard]==0.29.0 pydantic==2.7.1
+
+# Install llama-cpp-python with CPU-only build (no CUDA/Metal/OpenCL)
+# CMAKE_ARGS forces a plain CPU build; FORCE_CMAKE=1 ensures the wheel
+# is compiled from source so the flags are respected.
+RUN CMAKE_ARGS="-DLLAMA_CUBLAS=OFF -DLLAMA_METAL=OFF -DLLAMA_OPENCL=OFF" \
+    FORCE_CMAKE=1 \
+    pip install --no-cache-dir llama-cpp-python==0.2.77
+
+# ── App code ─────────────────────────────────────────────────
+COPY app.py .
+
+# ── Model volume ─────────────────────────────────────────────
+# Mount your GGUF file here at runtime:
+#   docker run -v /path/to/models:/models ...
+# OR bake it into the image by uncommenting the COPY line below
+# (image will be ~9 GB for Q4_K_M):
+# COPY models/qwen3-14b-q4_k_m.gguf /models/qwen3-14b-q4_k_m.gguf
+RUN mkdir -p /models
+
+# ── Runtime env defaults (override with -e flags) ────────────
+ENV MODEL_PATH=/models/qwen3-14b-q4_k_m.gguf \
+    MODEL_ID=qwen3-14b \
+    N_CTX=4096 \
+    N_THREADS=8 \
+    N_BATCH=512 \
+    VERBOSE=false
+
+EXPOSE 8000
+
+HEALTHCHECK --interval=30s --timeout=10s --start-period=120s --retries=3 \
+    CMD wget -qO- http://localhost:8000/health || exit 1
+
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"]

README.md

@@ -1,10 +1,123 @@
+# Qwen3-14B — OpenAI-compatible API (CPU · Docker)
+
+A lightweight FastAPI server that wraps **Qwen3-14B** (GGUF quantised) and
+exposes an **OpenAI-compatible REST API** so tools like **Paperclip** can talk
+to it as a drop-in OpenAI provider.
+
+## Endpoints
+
+| Method | Path | Description |
+|--------|------|-------------|
+| `GET`  | `/v1/models` | List available models |
+| `POST` | `/v1/chat/completions` | Chat completions (streaming + non-streaming) |
+| `GET`  | `/health` | Docker health check |
+
+---
+
+## 1 — Download the model
+
+You need a **GGUF quantised** version of Qwen3-14B.
+The recommended variant for CPU is **Q4_K_M** (~9 GB RAM at runtime).
+
+```bash
+mkdir -p models
+
+# Option A — Hugging Face CLI
+pip install huggingface_hub
+huggingface-cli download \
+  bartowski/Qwen3-14B-GGUF \
+  Qwen3-14B-Q4_K_M.gguf \
+  --local-dir ./models
+
+# Option B — wget (find the direct URL on the HF repo)
+# wget -O models/qwen3-14b-q4_k_m.gguf <url>
+```
+
+Rename the file so it matches `MODEL_PATH` in `docker-compose.yml`
+(or change the env var to match your filename).
+
+---
+
+## 2 — Build & run
+
+```bash
+# Build the image (one-time, ~5–10 min — compiles llama-cpp from source)
+docker compose build
+
+# Start the server
+docker compose up -d
+
+# Tail logs
+docker compose logs -f
+```
+
+The API will be available at **http://localhost:8000** once the model has
+loaded (allow ~1–3 min for the GGUF to map into memory on first start).
+
+---
+
+## 3 — Connect Paperclip
+
+In Paperclip's settings, add a new **OpenAI-compatible** provider:
+
+| Setting | Value |
+|---------|-------|
+| **Base URL** | `http://localhost:8000/v1` |
+| **API Key** | *(any non-empty string, e.g. `local`)* |
+| **Model** | `qwen3-14b` |
+
+Paperclip will call `/v1/models` to verify the connection and
+`/v1/chat/completions` for inference — both are implemented.
+
+---
+
+## 4 — Environment variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `MODEL_PATH` | `/models/qwen3-14b-q4_k_m.gguf` | Path to GGUF file inside container |
+| `MODEL_ID` | `qwen3-14b` | Model name returned by the API |
+| `N_CTX` | `4096` | Context window size (tokens) |
+| `N_THREADS` | `8` | CPU threads — set to your physical core count |
+| `N_BATCH` | `512` | Prompt processing batch size |
+| `VERBOSE` | `false` | Enable llama.cpp verbose logging |
+
+Override in `docker-compose.yml` or pass with `-e` flags to `docker run`.
+
 ---
-title: Fallback Module Trial
-emoji: 🌖
-colorFrom: indigo
-colorTo: gray
-sdk: docker
-pinned: false
+
+## 5 — Performance tips (CPU)
+
+- Set `N_THREADS` to your **physical** core count (not hyper-threaded).
+  On a modern 8-core machine `N_THREADS=8` is a good start.
+- Expect ~3–8 tokens/sec on a modern laptop; a server with many cores does better.
+- If you have more RAM, try **Q5_K_M** or **Q6_K** for better quality.
+- Reduce `N_CTX` to `2048` if you hit memory pressure.
+
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+## Quick test (curl)
+
+```bash
+# List models
+curl http://localhost:8000/v1/models
+
+# Non-streaming chat
+curl http://localhost:8000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "qwen3-14b",
+    "messages": [{"role": "user", "content": "Hello!"}],
+    "max_tokens": 256
+  }'
+
+# Streaming chat
+curl http://localhost:8000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "qwen3-14b",
+    "messages": [{"role": "user", "content": "Tell me a joke."}],
+    "max_tokens": 256,
+    "stream": true
+  }'
+```

app.py

@@ -0,0 +1,224 @@
+"""
+OpenAI-compatible FastAPI wrapper for Qwen3-14B (GGUF / llama-cpp-python)
+Endpoints: GET /v1/models, POST /v1/chat/completions
+Supports streaming (SSE) and non-streaming responses.
+"""
+
+import os
+import time
+import uuid
+import json
+import asyncio
+import logging
+from typing import AsyncIterator, List, Optional
+
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.responses import StreamingResponse, JSONResponse
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel, Field
+from llama_cpp import Llama
+
+# ---------------------------------------------------------------------------
+# Logging
+# ---------------------------------------------------------------------------
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Config (override via environment variables)
+# ---------------------------------------------------------------------------
+MODEL_PATH   = os.environ.get("MODEL_PATH", "/models/qwen3-14b-q4_k_m.gguf")
+MODEL_ID     = os.environ.get("MODEL_ID",   "qwen3-14b")
+N_CTX        = int(os.environ.get("N_CTX",  "4096"))
+N_THREADS    = int(os.environ.get("N_THREADS", str(os.cpu_count() or 4)))
+N_BATCH      = int(os.environ.get("N_BATCH", "512"))
+VERBOSE      = os.environ.get("VERBOSE", "false").lower() == "true"
+
+# ---------------------------------------------------------------------------
+# Load model at startup
+# ---------------------------------------------------------------------------
+logger.info(f"Loading model from {MODEL_PATH} — this may take a few minutes on CPU …")
+llm = Llama(
+    model_path=MODEL_PATH,
+    n_ctx=N_CTX,
+    n_threads=N_THREADS,
+    n_batch=N_BATCH,
+    n_gpu_layers=0,        # CPU only
+    verbose=VERBOSE,
+    chat_format="chatml",  # Qwen3 uses ChatML
+)
+logger.info("Model loaded ✓")
+
+# ---------------------------------------------------------------------------
+# FastAPI app
+# ---------------------------------------------------------------------------
+app = FastAPI(title="Qwen3-14B OpenAI-compatible API", version="1.0.0")
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# ---------------------------------------------------------------------------
+# Pydantic schemas (OpenAI-compatible subset)
+# ---------------------------------------------------------------------------
+
+class Message(BaseModel):
+    role: str
+    content: str
+
+class ChatCompletionRequest(BaseModel):
+    model: str = MODEL_ID
+    messages: List[Message]
+    max_tokens: Optional[int] = Field(default=1024, ge=1, le=8192)
+    temperature: Optional[float] = Field(default=0.7, ge=0.0, le=2.0)
+    top_p: Optional[float] = Field(default=0.9, ge=0.0, le=1.0)
+    stream: Optional[bool] = False
+    stop: Optional[List[str]] = None
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _make_chunk(delta_content: str, finish_reason: Optional[str], request_id: str) -> str:
+    chunk = {
+        "id": request_id,
+        "object": "chat.completion.chunk",
+        "created": int(time.time()),
+        "model": MODEL_ID,
+        "choices": [
+            {
+                "index": 0,
+                "delta": {"content": delta_content} if delta_content else {},
+                "finish_reason": finish_reason,
+            }
+        ],
+    }
+    return f"data: {json.dumps(chunk)}\n\n"
+
+
+async def _stream_response(request: ChatCompletionRequest, request_id: str) -> AsyncIterator[str]:
+    """Run llama-cpp in a thread pool and yield SSE chunks."""
+    messages = [{"role": m.role, "content": m.content} for m in request.messages]
+
+    loop = asyncio.get_event_loop()
+
+    def _run():
+        return llm.create_chat_completion(
+            messages=messages,
+            max_tokens=request.max_tokens,
+            temperature=request.temperature,
+            top_p=request.top_p,
+            stop=request.stop or [],
+            stream=True,
+        )
+
+    # llama-cpp streaming returns a generator; run initial call in thread pool
+    gen = await loop.run_in_executor(None, _run)
+
+    # Yield first role delta
+    yield _make_chunk("", None, request_id)
+
+    for chunk in gen:
+        choice = chunk["choices"][0]
+        delta  = choice.get("delta", {})
+        content = delta.get("content", "")
+        finish  = choice.get("finish_reason")
+        if content:
+            yield _make_chunk(content, None, request_id)
+        if finish:
+            yield _make_chunk("", finish, request_id)
+            break
+
+    yield "data: [DONE]\n\n"
+
+
+# ---------------------------------------------------------------------------
+# Routes
+# ---------------------------------------------------------------------------
+
+@app.get("/")
+async def root():
+    return {"status": "ok", "model": MODEL_ID}
+
+
+@app.get("/v1/models")
+async def list_models():
+    return {
+        "object": "list",
+        "data": [
+            {
+                "id": MODEL_ID,
+                "object": "model",
+                "created": 1700000000,
+                "owned_by": "local",
+            }
+        ],
+    }
+
+
+@app.post("/v1/chat/completions")
+async def chat_completions(request: ChatCompletionRequest):
+    messages = [{"role": m.role, "content": m.content} for m in request.messages]
+
+    if request.stream:
+        request_id = f"chatcmpl-{uuid.uuid4().hex}"
+        return StreamingResponse(
+            _stream_response(request, request_id),
+            media_type="text/event-stream",
+            headers={
+                "Cache-Control": "no-cache",
+                "X-Accel-Buffering": "no",
+            },
+        )
+
+    # Non-streaming
+    loop = asyncio.get_event_loop()
+
+    def _run():
+        return llm.create_chat_completion(
+            messages=messages,
+            max_tokens=request.max_tokens,
+            temperature=request.temperature,
+            top_p=request.top_p,
+            stop=request.stop or [],
+            stream=False,
+        )
+
+    result = await loop.run_in_executor(None, _run)
+
+    choice   = result["choices"][0]
+    usage    = result.get("usage", {})
+
+    return {
+        "id": f"chatcmpl-{uuid.uuid4().hex}",
+        "object": "chat.completion",
+        "created": int(time.time()),
+        "model": MODEL_ID,
+        "choices": [
+            {
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": choice["message"]["content"],
+                },
+                "finish_reason": choice.get("finish_reason", "stop"),
+            }
+        ],
+        "usage": {
+            "prompt_tokens":     usage.get("prompt_tokens", 0),
+            "completion_tokens": usage.get("completion_tokens", 0),
+            "total_tokens":      usage.get("total_tokens", 0),
+        },
+    }
+
+
+# ---------------------------------------------------------------------------
+# Health check (useful for Docker HEALTHCHECK)
+# ---------------------------------------------------------------------------
+
+@app.get("/health")
+async def health():
+    return {"status": "healthy", "model": MODEL_ID}

docker-compose.yml

@@ -0,0 +1,22 @@
+version: "3.9"
+
+services:
+  qwen3-api:
+    build: .
+    image: qwen3-api:latest
+    container_name: qwen3-api
+    ports:
+      - "8000:8000"
+    volumes:
+      # Put your GGUF file in ./models/ on the host
+      - ./models:/models:ro
+    environment:
+      MODEL_PATH: /models/qwen3-14b-q4_k_m.gguf
+      MODEL_ID: qwen3-14b
+      N_CTX: "4096"
+      # Set to number of physical CPU cores for best performance
+      N_THREADS: "8"
+      N_BATCH: "512"
+      VERBOSE: "false"
+    restart: unless-stopped
+    # CPU-only — no deploy.resources.reservations needed

requirements.txt

@@ -0,0 +1,6 @@
+fastapi==0.111.0
+uvicorn[standard]==0.29.0
+pydantic==2.7.1
+# llama-cpp-python is installed via pip in the Dockerfile
+# with CPU-only build flags (no CUDA / Metal)
+llama-cpp-python==0.2.77