Upload 3 files

Dockerfile

@@ -15,13 +15,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 WORKDIR /app
 
 # ── Python deps ──────────────────────────────────────────────
-# Install everything EXCEPT llama-cpp-python first (lighter layer)
-COPY requirements.txt .
-RUN pip install --no-cache-dir fastapi==0.111.0 uvicorn[standard]==0.29.0 pydantic==2.7.1
+RUN pip install --no-cache-dir \
+        fastapi==0.111.0 \
+        "uvicorn[standard]==0.29.0" \
+        pydantic==2.7.1
 
-# Install llama-cpp-python with CPU-only build (no CUDA/Metal/OpenCL)
-# CMAKE_ARGS forces a plain CPU build; FORCE_CMAKE=1 ensures the wheel
-# is compiled from source so the flags are respected.
+# Install llama-cpp-python CPU-only (compiled from source)
 RUN CMAKE_ARGS="-DLLAMA_CUBLAS=OFF -DLLAMA_METAL=OFF -DLLAMA_OPENCL=OFF" \
     FORCE_CMAKE=1 \
     pip install --no-cache-dir llama-cpp-python==0.2.77
@@ -30,15 +29,16 @@ RUN CMAKE_ARGS="-DLLAMA_CUBLAS=OFF -DLLAMA_METAL=OFF -DLLAMA_OPENCL=OFF" \
 COPY app.py .
 
 # ── Model volume ─────────────────────────────────────────────
-# Mount your GGUF file here at runtime:
+# The app auto-downloads the model on first boot into /models.
+# Mount a named volume here so the download persists across restarts:
+#   docker run -v qwen3_models:/models ...
+# Or pre-populate with your own GGUF:
 #   docker run -v /path/to/models:/models ...
-# OR bake it into the image by uncommenting the COPY line below
-# (image will be ~9 GB for Q4_K_M):
-# COPY models/qwen3-14b-q4_k_m.gguf /models/qwen3-14b-q4_k_m.gguf
 RUN mkdir -p /models
 
-# ── Runtime env defaults (override with -e flags) ────────────
+# ── Runtime env defaults (override with -e or docker-compose) ─
 ENV MODEL_PATH=/models/qwen3-14b-q4_k_m.gguf \
+    MODEL_URL=https://huggingface.co/bartowski/Qwen3-14B-GGUF/resolve/main/Qwen3-14B-Q4_K_M.gguf \
     MODEL_ID=qwen3-14b \
     N_CTX=4096 \
     N_THREADS=8 \
@@ -47,7 +47,8 @@ ENV MODEL_PATH=/models/qwen3-14b-q4_k_m.gguf \
 
 EXPOSE 8000
 
-HEALTHCHECK --interval=30s --timeout=10s --start-period=120s --retries=3 \
-    CMD wget -qO- http://localhost:8000/health || exit 1
+# Health check — /health returns {"ready": true} once the model is loaded
+HEALTHCHECK --interval=30s --timeout=10s --start-period=600s --retries=20 \
+    CMD wget -qO- http://localhost:8000/health | grep -q '"ready": true' || exit 1
 
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"]

app.py

@@ -2,6 +2,8 @@
 OpenAI-compatible FastAPI wrapper for Qwen3-14B (GGUF / llama-cpp-python)
 Endpoints: GET /v1/models, POST /v1/chat/completions
 Supports streaming (SSE) and non-streaming responses.
+
+Model is downloaded automatically on first boot if not already present.
 """
 
 import os
@@ -10,10 +12,12 @@ import uuid
 import json
 import asyncio
 import logging
+import threading
+from pathlib import Path
 from typing import AsyncIterator, List, Optional
 
-from fastapi import FastAPI, HTTPException, Request
-from fastapi.responses import StreamingResponse, JSONResponse
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import StreamingResponse
 from fastapi.middleware.cors import CORSMiddleware
 from pydantic import BaseModel, Field
 from llama_cpp import Llama
@@ -27,27 +31,95 @@ logger = logging.getLogger(__name__)
 # ---------------------------------------------------------------------------
 # Config (override via environment variables)
 # ---------------------------------------------------------------------------
-MODEL_PATH   = os.environ.get("MODEL_PATH", "/models/qwen3-14b-q4_k_m.gguf")
-MODEL_ID     = os.environ.get("MODEL_ID",   "qwen3-14b")
-N_CTX        = int(os.environ.get("N_CTX",  "4096"))
-N_THREADS    = int(os.environ.get("N_THREADS", str(os.cpu_count() or 4)))
-N_BATCH      = int(os.environ.get("N_BATCH", "512"))
-VERBOSE      = os.environ.get("VERBOSE", "false").lower() == "true"
+MODEL_PATH    = os.environ.get("MODEL_PATH",    "/models/qwen3-14b-q4_k_m.gguf")
+MODEL_URL     = os.environ.get("MODEL_URL",
+    "https://huggingface.co/bartowski/Qwen3-14B-GGUF/resolve/main/Qwen3-14B-Q4_K_M.gguf")
+MODEL_ID      = os.environ.get("MODEL_ID",      "qwen3-14b")
+N_CTX         = int(os.environ.get("N_CTX",     "4096"))
+N_THREADS     = int(os.environ.get("N_THREADS", str(os.cpu_count() or 4)))
+N_BATCH       = int(os.environ.get("N_BATCH",   "512"))
+VERBOSE       = os.environ.get("VERBOSE",       "false").lower() == "true"
 
 # ---------------------------------------------------------------------------
-# Load model at startup
+# Lazy model holder
 # ---------------------------------------------------------------------------
-logger.info(f"Loading model from {MODEL_PATH} — this may take a few minutes on CPU …")
-llm = Llama(
-    model_path=MODEL_PATH,
-    n_ctx=N_CTX,
-    n_threads=N_THREADS,
-    n_batch=N_BATCH,
-    n_gpu_layers=0,        # CPU only
-    verbose=VERBOSE,
-    chat_format="chatml",  # Qwen3 uses ChatML
-)
-logger.info("Model loaded ✓")
+_llm: Optional[Llama] = None
+_llm_lock   = threading.Lock()
+_llm_ready  = threading.Event()   # set once the model is loaded
+_llm_error: Optional[str] = None  # set if loading failed
+
+
+def _download_model() -> None:
+    """Download the GGUF file from MODEL_URL if MODEL_PATH doesn't exist."""
+    path = Path(MODEL_PATH)
+    if path.exists():
+        logger.info(f"Model already present at {MODEL_PATH}")
+        return
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    logger.info(f"Model not found — downloading from {MODEL_URL} …")
+    logger.info("This will take a while on first boot (file is ~9 GB).")
+
+    import urllib.request
+
+    tmp = Path(str(MODEL_PATH) + ".part")
+    last_pct = -1
+
+    def _progress(block_num, block_size, total_size):
+        nonlocal last_pct
+        if total_size <= 0:
+            return
+        pct = int(block_num * block_size * 100 / total_size)
+        pct = min(pct, 100)
+        if pct != last_pct and pct % 5 == 0:
+            logger.info(f"Download progress: {pct}%")
+            last_pct = pct
+
+    urllib.request.urlretrieve(MODEL_URL, tmp, reporthook=_progress)
+    tmp.rename(path)
+    logger.info(f"Download complete → {MODEL_PATH}")
+
+
+def _load_model_background() -> None:
+    """Download (if needed) then load the model. Runs in a daemon thread."""
+    global _llm, _llm_error
+    try:
+        _download_model()
+        logger.info(f"Loading model into memory from {MODEL_PATH} …")
+        llm = Llama(
+            model_path=MODEL_PATH,
+            n_ctx=N_CTX,
+            n_threads=N_THREADS,
+            n_batch=N_BATCH,
+            n_gpu_layers=0,        # CPU only
+            verbose=VERBOSE,
+            chat_format="chatml",  # Qwen3 uses ChatML
+        )
+        with _llm_lock:
+            _llm = llm
+        logger.info("Model loaded and ready ✓")
+    except Exception as exc:
+        _llm_error = str(exc)
+        logger.error(f"Failed to load model: {exc}")
+    finally:
+        _llm_ready.set()
+
+
+def _get_llm() -> Llama:
+    """Return the loaded model or raise a 503 if it's not ready yet."""
+    if not _llm_ready.is_set():
+        raise HTTPException(
+            status_code=503,
+            detail="Model is still loading (or downloading). "
+                   "Check /health for status and retry in a moment.",
+        )
+    if _llm_error:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Model failed to load: {_llm_error}",
+        )
+    return _llm
+
 
 # ---------------------------------------------------------------------------
 # FastAPI app
@@ -61,6 +133,16 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
+
+@app.on_event("startup")
+async def startup_event():
+    """Kick off model download + load in a background thread so the server
+    starts immediately and stays responsive during the (long) load phase."""
+    t = threading.Thread(target=_load_model_background, daemon=True)
+    t.start()
+    logger.info("Server is up. Model loading in background — see /health for status.")
+
+
 # ---------------------------------------------------------------------------
 # Pydantic schemas (OpenAI-compatible subset)
 # ---------------------------------------------------------------------------
@@ -69,6 +151,7 @@ class Message(BaseModel):
     role: str
     content: str
 
+
 class ChatCompletionRequest(BaseModel):
     model: str = MODEL_ID
     messages: List[Message]
@@ -78,6 +161,7 @@ class ChatCompletionRequest(BaseModel):
     stream: Optional[bool] = False
     stop: Optional[List[str]] = None
 
+
 # ---------------------------------------------------------------------------
 # Helpers
 # ---------------------------------------------------------------------------
@@ -100,9 +184,8 @@ def _make_chunk(delta_content: str, finish_reason: Optional[str], request_id: st
 
 
 async def _stream_response(request: ChatCompletionRequest, request_id: str) -> AsyncIterator[str]:
-    """Run llama-cpp in a thread pool and yield SSE chunks."""
+    llm = _get_llm()
     messages = [{"role": m.role, "content": m.content} for m in request.messages]
-
     loop = asyncio.get_event_loop()
 
     def _run():
@@ -115,15 +198,13 @@ async def _stream_response(request: ChatCompletionRequest, request_id: str) -> A
             stream=True,
         )
 
-    # llama-cpp streaming returns a generator; run initial call in thread pool
     gen = await loop.run_in_executor(None, _run)
 
-    # Yield first role delta
-    yield _make_chunk("", None, request_id)
+    yield _make_chunk("", None, request_id)  # opening delta
 
     for chunk in gen:
-        choice = chunk["choices"][0]
-        delta  = choice.get("delta", {})
+        choice  = chunk["choices"][0]
+        delta   = choice.get("delta", {})
         content = delta.get("content", "")
         finish  = choice.get("finish_reason")
         if content:
@@ -141,7 +222,17 @@ async def _stream_response(request: ChatCompletionRequest, request_id: str) -> A
 
 @app.get("/")
 async def root():
-    return {"status": "ok", "model": MODEL_ID}
+    ready = _llm_ready.is_set() and _llm is not None
+    return {"status": "ready" if ready else "loading", "model": MODEL_ID}
+
+
+@app.get("/health")
+async def health():
+    if not _llm_ready.is_set():
+        return {"status": "loading", "model": MODEL_ID, "ready": False}
+    if _llm_error:
+        return {"status": "error", "error": _llm_error, "ready": False}
+    return {"status": "healthy", "model": MODEL_ID, "ready": True}
 
 
 @app.get("/v1/models")
@@ -161,6 +252,8 @@ async def list_models():
 
 @app.post("/v1/chat/completions")
 async def chat_completions(request: ChatCompletionRequest):
+    llm = _get_llm()  # raises 503 if not ready
+
     messages = [{"role": m.role, "content": m.content} for m in request.messages]
 
     if request.stream:
@@ -168,10 +261,7 @@ async def chat_completions(request: ChatCompletionRequest):
         return StreamingResponse(
             _stream_response(request, request_id),
             media_type="text/event-stream",
-            headers={
-                "Cache-Control": "no-cache",
-                "X-Accel-Buffering": "no",
-            },
+            headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
         )
 
     # Non-streaming
@@ -188,9 +278,8 @@ async def chat_completions(request: ChatCompletionRequest):
         )
 
     result = await loop.run_in_executor(None, _run)
-
-    choice   = result["choices"][0]
-    usage    = result.get("usage", {})
+    choice = result["choices"][0]
+    usage  = result.get("usage", {})
 
     return {
         "id": f"chatcmpl-{uuid.uuid4().hex}",
@@ -213,12 +302,3 @@ async def chat_completions(request: ChatCompletionRequest):
             "total_tokens":      usage.get("total_tokens", 0),
         },
     }
-
-
-# ---------------------------------------------------------------------------
-# Health check (useful for Docker HEALTHCHECK)
-# ---------------------------------------------------------------------------
-
-@app.get("/health")
-async def health():
-    return {"status": "healthy", "model": MODEL_ID}

docker-compose.yml

@@ -8,15 +8,20 @@ services:
     ports:
       - "8000:8000"
     volumes:
-      # Put your GGUF file in ./models/ on the host
-      - ./models:/models:ro
+      # Named volume — model is downloaded here on first boot and reused after
+      - qwen3_models:/models
+      # Alternative: mount your own pre-downloaded folder instead:
+      # - ./models:/models
     environment:
       MODEL_PATH: /models/qwen3-14b-q4_k_m.gguf
+      # Override MODEL_URL to use a different GGUF variant (e.g. Q5_K_M)
+      MODEL_URL: https://huggingface.co/bartowski/Qwen3-14B-GGUF/resolve/main/Qwen3-14B-Q4_K_M.gguf
       MODEL_ID: qwen3-14b
       N_CTX: "4096"
-      # Set to number of physical CPU cores for best performance
-      N_THREADS: "8"
+      N_THREADS: "8"   # set to your physical CPU core count
       N_BATCH: "512"
       VERBOSE: "false"
     restart: unless-stopped
-    # CPU-only — no deploy.resources.reservations needed
+
+volumes:
+  qwen3_models:   # persists the downloaded GGUF across container restarts