Upload 4 files

Dockerfile

@@ -0,0 +1,133 @@
+# syntax=docker/dockerfile:1
+# ---------------------------------------------------------------------------
+# Dockerfile for a Hugging Face "Docker Space" that serves a GGUF model
+# (empero-ai/Qwythos-9B-Claude-Mythos-5-1M-GGUF) through llama-cpp-python
+# with a streaming Gradio chat UI.
+#
+# Design goals:
+#   - Small final image  -> multi-stage build. All compilers / build tools
+#     (cmake, ninja, gcc) live ONLY in the builder stage. The runtime stage
+#     just installs the pre-built wheel.
+#   - Reliable build      -> pin apt/pip behaviour (no interactive prompts,
+#     no cache dirs left behind, explicit CMake flags for llama.cpp so the
+#     build never silently falls back to something incompatible with the
+#     CPU the Space actually runs on).
+#   - CPU-only            -> no CUDA/ROCm toolkits anywhere in the image.
+# ---------------------------------------------------------------------------
+
+# =========================== 1. Builder stage ===============================
+FROM python:3.11-slim AS builder
+
+# Build-time system dependencies for compiling llama-cpp-python's C++/CMake
+# backend (llama.cpp). None of this ends up in the final image.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential \
+        cmake \
+        ninja-build \
+        git \
+        ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+# CMake flags for llama.cpp's CPU backend:
+#   - GGML_NATIVE=OFF   : do NOT auto-detect the build machine's CPU flags.
+#                         The Docker image is built on different hardware
+#                         than it may eventually run on, so "native" builds
+#                         can crash with "illegal instruction" on the actual
+#                         Space runner. We instead opt in to a conservative,
+#                         broadly-supported instruction set explicitly.
+#   - GGML_AVX2/FMA/F16C: virtually all HF CPU Space runners support these
+#                         (modern x86_64 cloud CPUs). This gives good
+#                         performance without the risk of AVX-512-only code
+#                         paths.
+#   - CMAKE_BUILD_TYPE=Release + Ninja generator: faster, smaller, more
+#                         reliable builds than the default Makefiles.
+ENV CMAKE_ARGS="-DGGML_NATIVE=OFF -DGGML_AVX2=ON -DGGML_FMA=ON -DGGML_F16C=ON -DCMAKE_BUILD_TYPE=Release -GNinja"
+ENV FORCE_CMAKE=1
+
+WORKDIR /build
+
+# Build a wheel for the latest stable llama-cpp-python (and its light
+# transitive deps) instead of `pip install`-ing it directly. This lets the
+# final runtime stage install from a local wheel with zero compilers
+# present, which is both faster and smaller.
+RUN pip install --no-cache-dir --upgrade pip wheel setuptools && \
+    pip wheel --no-cache-dir --wheel-dir /wheels "llama-cpp-python"
+
+# ============================ 2. Runtime stage ================================
+FROM python:3.11-slim AS runtime
+
+LABEL maintainer="hf-space" \
+      description="Gradio chat UI serving empero-ai/Qwythos-9B-Claude-Mythos-5-1M-GGUF via llama-cpp-python"
+
+# Runtime-only system dependencies:
+#   - libgomp1: OpenMP runtime required by llama.cpp's multithreaded kernels.
+#   - ca-certificates: needed for HTTPS downloads from the Hugging Face Hub.
+# NOTE: no compilers, no cmake, no git here -> keeps the final image lean.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        libgomp1 \
+        ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+# Run as a non-root user (Hugging Face Spaces requirement/best practice).
+RUN useradd --create-home --uid 1000 appuser
+WORKDIR /app
+
+# Install the pre-built llama-cpp-python wheel from the builder stage.
+COPY --from=builder /wheels /wheels
+COPY requirements.txt .
+
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir /wheels/*.whl && \
+    pip install --no-cache-dir -r requirements.txt && \
+    rm -rf /wheels /root/.cache/pip ~/.cache/pip
+
+# Application code
+COPY app.py .
+
+# ---------------------------------------------------------------------------
+# Runtime configuration (all overridable as Space "Variables and secrets"
+# without touching the Dockerfile). See README.md for the full list.
+# ---------------------------------------------------------------------------
+ENV \
+    # Where to look for the GGUF model on the Hub.
+    GGUF_REPO_ID="empero-ai/Qwythos-9B-Claude-Mythos-5-1M-GGUF" \
+    # Leave empty to auto-select; set to force an exact filename.
+    GGUF_FILENAME="" \
+    # Quantization to prefer when auto-selecting (falls back automatically
+    # if this exact quant isn't present in the repo).
+    PREFERRED_QUANT="Q4_K_M" \
+    # Local, persistent-within-container cache for downloaded model files
+    # and Hub metadata so restarts of a *running* Space don't re-download.
+    MODEL_CACHE_DIR="/data/models" \
+    HF_HOME="/data/hf_home" \
+    HF_HUB_ENABLE_HF_TRANSFER="0" \
+    # Context window (tokens). The model supports up to 1,048,576 via
+    # baked-in YaRN scaling, but free CPU Spaces cannot allocate that much
+    # KV-cache. Default is a safe value for a 16GB-RAM CPU Space; raise it
+    # via the Space's Variables UI if you have more headroom.
+    N_CTX="8192" \
+    # 0 = auto-detect available CPU cores.
+    N_THREADS="0" \
+    N_BATCH="256" \
+    MAX_NEW_TOKENS="1024" \
+    TEMPERATURE="0.6" \
+    TOP_P="0.95" \
+    TOP_K="20" \
+    REPEAT_PENALTY="1.05" \
+    SYSTEM_PROMPT="" \
+    # Number of retries when downloading the model file from the Hub.
+    DOWNLOAD_MAX_RETRIES="5" \
+    # Gradio server bind settings (must be 0.0.0.0 + 7860 for HF Spaces).
+    GRADIO_SERVER_NAME="0.0.0.0" \
+    GRADIO_SERVER_PORT="7860" \
+    PYTHONUNBUFFERED="1"
+
+# Cache/data directories must be writable by the non-root user.
+RUN mkdir -p /data/models /data/hf_home && \
+    chown -R appuser:appuser /data /app
+
+USER appuser
+
+EXPOSE 7860
+
+CMD ["python", "app.py"]

README.md

@@ -1,10 +1,184 @@
 ---
-title: DOC
-emoji: 👁
-colorFrom: green
-colorTo: yellow
+title: Qwythos 9B GGUF Chat
+emoji: 🧠
+colorFrom: indigo
+colorTo: purple
 sdk: docker
+app_port: 7860
 pinned: false
+license: apache-2.0
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Qwythos-9B GGUF Chat (Docker Space)
+
+A self-contained Hugging Face **Docker Space** that downloads a GGUF quant of
+[`empero-ai/Qwythos-9B-Claude-Mythos-5-1M-GGUF`](https://huggingface.co/empero-ai/Qwythos-9B-Claude-Mythos-5-1M-GGUF)
+and serves it through a streaming Gradio chat UI, powered by
+[`llama-cpp-python`](https://github.com/abetlen/llama-cpp-python).
+
+> ⚠️ **About this model.** This is a third-party community fine-tune (based
+> on Qwen3.5-9B), not an Anthropic model. Its model card describes it as
+> "uncensored" with no built-in safety layer. If you deploy this Space
+> publicly, add your own moderation/review layer appropriate to your
+> audience — the model card recommends the same.
+
+---
+
+## What's in this repo
+
+| File              | Purpose                                                              |
+|-------------------|-----------------------------------------------------------------------|
+| `Dockerfile`      | Multi-stage build: compiles `llama-cpp-python` in a builder stage, then ships a slim runtime image with no compilers. |
+| `requirements.txt`| Pure-Python runtime deps (`gradio`, `huggingface_hub`).              |
+| `app.py`          | Downloads/caches the GGUF, loads it with `llama-cpp-python`, and serves a streaming Gradio chat UI. |
+| `README.md`       | This file (also the Space's metadata card, via the YAML frontmatter above). |
+
+This repo is ready to push directly to a new **Docker** Space with no
+further edits.
+
+---
+
+## Deploying to Hugging Face Spaces
+
+1. Create a new Space at <https://huggingface.co/new-space>.
+2. Choose **Docker** as the Space SDK (not "Gradio" or "Streamlit" — this
+   project builds and runs its own Dockerfile).
+3. Pick the **CPU basic (free)** hardware tier — this project is built to
+   run entirely on CPU.
+4. Push these four files to the Space repo:
+
+   ```bash
+   git clone https://huggingface.co/spaces/<your-username>/<your-space-name>
+   cd <your-space-name>
+   cp /path/to/Dockerfile /path/to/requirements.txt /path/to/app.py /path/to/README.md .
+   git add .
+   git commit -m "Deploy Qwythos-9B GGUF chat Space"
+   git push
+   ```
+
+5. The Space will build the Docker image (this takes several minutes the
+   first time — `llama-cpp-python` is compiled from source) and then start
+   the container. On first launch, `app.py` downloads the selected GGUF
+   file from the Hub in the background; watch the Space's **Logs** tab for
+   download/load progress.
+6. Once the model finishes loading, the chat UI becomes responsive.
+
+No secrets or tokens are required for the default (public) repo. If you
+point this at a **gated/private** GGUF repo, add an `HF_TOKEN` secret in the
+Space's **Settings → Variables and secrets**; `huggingface_hub` picks it up
+automatically.
+
+---
+
+## Configuration (environment variables)
+
+All of these are set with sensible defaults in the `Dockerfile` and can be
+overridden per-Space under **Settings → Variables and secrets** without
+touching any code:
+
+| Variable              | Default                                              | Description |
+|-----------------------|-------------------------------------------------------|--------------|
+| `GGUF_REPO_ID`         | `empero-ai/Qwythos-9B-Claude-Mythos-5-1M-GGUF`        | Hub repo to pull the GGUF from. |
+| `GGUF_FILENAME`        | *(empty = auto-select)*                                | Force an exact filename instead of auto-selecting by quant. |
+| `PREFERRED_QUANT`      | `Q4_K_M`                                              | Preferred quantization. Falls back automatically (`Q4_K_S` → `Q5_K_M` → ... → smallest available `.gguf`) if not present. `mmproj` (vision) and `-MTP-` (speculative-decoding draft head) files are skipped by the auto-selector in favor of a plain text-chat quant. |
+| `MODEL_CACHE_DIR`      | `/data/models`                                        | Local cache directory for downloaded model weights. |
+| `HF_HOME`              | `/data/hf_home`                                       | Cache directory for Hub metadata. |
+| `N_CTX`                | `8192`                                                 | Context window (tokens) allocated at load time. The model supports up to **1,048,576** tokens via baked-in YaRN scaling, but a free 16GB-RAM CPU Space cannot allocate a multi-hundred-thousand-token KV-cache — raise this only if you've upgraded hardware (see "Long context" below). |
+| `N_THREADS`            | `0` (= auto, all cores)                                | CPU threads for inference. |
+| `N_BATCH`              | `256`                                                  | Prompt processing batch size. |
+| `MAX_NEW_TOKENS`       | `1024`                                                 | Max tokens generated per reply. |
+| `TEMPERATURE`          | `0.6`                                                  | Sampling temperature (the model card recommends 0.6 for its thinking mode; avoid ≤0.3, which the card notes can cause repetition loops). |
+| `TOP_P`                | `0.95`                                                 | Nucleus sampling. |
+| `TOP_K`                | `20`                                                   | Top-k sampling. |
+| `REPEAT_PENALTY`       | `1.05`                                                 | Repetition penalty. |
+| `SYSTEM_PROMPT`        | *(empty)*                                              | Optional system prompt prepended to every conversation. |
+| `DOWNLOAD_MAX_RETRIES` | `5`                                                     | Retry attempts (exponential backoff) for the model download. |
+
+---
+
+## Model caching & the free tier's storage caveat
+
+`app.py` downloads the model once into `MODEL_CACHE_DIR` and reuses the
+cached file for every subsequent chat request — it will **not** re-download
+on every message, and it survives the container going to sleep/waking back
+up from inactivity.
+
+However, **the free Spaces tier has no *persistent* storage**: the
+container's disk (including `/data`) is rebuilt from scratch whenever the
+Space is fully **restarted or rebuilt** (e.g. after a `git push`, a factory
+reboot, or an infrastructure migration). In that case, the model will be
+re-downloaded once on the next startup — this is a platform limitation, not
+a bug in this app. If you need the cache to survive restarts, enable
+**Persistent Storage** for the Space (a paid add-on) and point
+`MODEL_CACHE_DIR`/`HF_HOME` at the mounted persistent volume (typically
+`/data`, which is already the default here).
+
+---
+
+## Long context ("1M context") notes
+
+The GGUF files in this repo ship with YaRN rope-scaling baked in for up to a
+1,048,576-token context window. That is a *ceiling*, not something you get
+for free on CPU:
+
+- Free **CPU basic** Spaces (16GB RAM) can realistically handle a `Q4_K_M`
+  9B model with a context window in the **low thousands to ~16-32k tokens**,
+  depending on available RAM after the model weights are loaded.
+- Attempting to set `N_CTX` far beyond what your Space's RAM allows will
+  cause the model load to fail or the container to be OOM-killed.
+- If you need genuinely long context (hundreds of thousands of tokens),
+  you'll need a GPU Space or a machine with substantially more RAM — the
+  model card itself notes the full 1M window typically needs multi-GPU or
+  aggressive KV-cache offload even outside of CPU constraints.
+
+`N_CTX` is fully configurable via the environment variable above so you can
+tune it to whatever hardware tier you're running on.
+
+---
+
+## Local development (outside Docker)
+
+```bash
+# Build llama-cpp-python with a build appropriate for your machine:
+CMAKE_ARGS="-DGGML_NATIVE=ON" pip install llama-cpp-python
+pip install -r requirements.txt
+
+python app.py
+# then open http://localhost:7860
+```
+
+## Building/running the Docker image locally
+
+```bash
+docker build -t qwythos-space .
+docker run -it -p 7860:7860 qwythos-space
+# then open http://localhost:7860
+```
+
+---
+
+## Technical notes
+
+- **`llama-cpp-python` build**: built from source in a dedicated builder
+  stage with `CMAKE_ARGS="-DGGML_NATIVE=OFF -DGGML_AVX2=ON -DGGML_FMA=ON
+  -DGGML_F16C=ON"`. `GGML_NATIVE` is deliberately disabled because the
+  machine that *builds* the Docker image is not guaranteed to be the same
+  CPU that *runs* it; auto-detected "native" builds can otherwise crash
+  with `SIGILL` on the Space's actual runner. AVX2/FMA/F16C are supported
+  by essentially all modern cloud x86_64 CPUs and give good performance
+  without that risk.
+- **Chat template**: the `Llama` object is created without a hardcoded
+  `chat_format`, so `llama-cpp-python` auto-detects and applies the Jinja
+  chat template embedded in the GGUF's own metadata — the current,
+  non-deprecated approach (no reliance on a manually-specified/legacy
+  template name).
+- **Streaming**: implemented via `llm.create_chat_completion(..., stream=True)`,
+  yielding incrementally-growing text to Gradio's `ChatInterface` for
+  token-by-token display.
+- **GPU layers**: `n_gpu_layers=0` — this Space is CPU-only by design, matching
+  the free Spaces hardware tier.
+- **File selection**: uses `huggingface_hub.HfApi().model_info(..., files_metadata=True)`
+  to inspect all files with sizes, filters out `mmproj` (vision projector)
+  and `-MTP-` (speculative decoding draft-head) variants by default, then
+  picks the smallest file matching `PREFERRED_QUANT`, falling back through
+  a quant-quality-ordered list if needed.

app.py

@@ -0,0 +1,329 @@
+"""
+Hugging Face Docker Space: streaming chat UI for a GGUF model served with
+llama-cpp-python.
+
+Model : empero-ai/Qwythos-9B-Claude-Mythos-5-1M-GGUF (configurable via env)
+Quant : Q4_K_M by default, with automatic fallback to another available
+        quantization if Q4_K_M isn't present in the repo.
+UI    : Gradio ChatInterface, token-by-token streaming.
+
+Everything here is driven by environment variables (see Dockerfile / README)
+so the Space can be reconfigured entirely from the "Variables and secrets"
+tab without editing code.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import threading
+import time
+from typing import Iterator
+
+import gradio as gr
+from huggingface_hub import HfApi, hf_hub_download
+from huggingface_hub.utils import HfHubHTTPError
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+)
+log = logging.getLogger("qwythos-space")
+
+# ---------------------------------------------------------------------------
+# Configuration (all overridable via environment variables)
+# ---------------------------------------------------------------------------
+
+GGUF_REPO_ID = os.environ.get("GGUF_REPO_ID", "empero-ai/Qwythos-9B-Claude-Mythos-5-1M-GGUF")
+GGUF_FILENAME = os.environ.get("GGUF_FILENAME", "").strip()  # force exact file if set
+PREFERRED_QUANT = os.environ.get("PREFERRED_QUANT", "Q4_K_M").strip()
+
+MODEL_CACHE_DIR = os.environ.get("MODEL_CACHE_DIR", "/data/models")
+os.environ.setdefault("HF_HOME", os.environ.get("HF_HOME", "/data/hf_home"))
+
+N_CTX = int(os.environ.get("N_CTX", "8192"))
+N_THREADS_ENV = int(os.environ.get("N_THREADS", "0"))
+N_BATCH = int(os.environ.get("N_BATCH", "256"))
+MAX_NEW_TOKENS = int(os.environ.get("MAX_NEW_TOKENS", "1024"))
+TEMPERATURE = float(os.environ.get("TEMPERATURE", "0.6"))
+TOP_P = float(os.environ.get("TOP_P", "0.95"))
+TOP_K = int(os.environ.get("TOP_K", "20"))
+REPEAT_PENALTY = float(os.environ.get("REPEAT_PENALTY", "1.05"))
+SYSTEM_PROMPT = os.environ.get("SYSTEM_PROMPT", "").strip()
+
+DOWNLOAD_MAX_RETRIES = int(os.environ.get("DOWNLOAD_MAX_RETRIES", "5"))
+
+# Fallback quantization preference order if PREFERRED_QUANT isn't available.
+QUANT_FALLBACK_ORDER = [
+    "Q4_K_M", "Q4_K_S", "Q4_0",
+    "Q5_K_M", "Q5_K_S",
+    "Q6_K", "Q8_0",
+    "Q3_K_M", "Q3_K_S",
+    "IQ4_XS",
+]
+
+os.makedirs(MODEL_CACHE_DIR, exist_ok=True)
+
+# ---------------------------------------------------------------------------
+# Global (lazily-initialised, thread-guarded) model state
+# ---------------------------------------------------------------------------
+
+_llm = None
+_llm_lock = threading.Lock()
+_init_status = {"ready": False, "error": None, "model_path": None, "filename": None}
+
+
+def _is_excluded(filename: str) -> bool:
+    """Filter out files we never want to auto-select as the chat model."""
+    lower = filename.lower()
+    if not lower.endswith(".gguf"):
+        return True
+    if "mmproj" in lower:  # vision projector, not a text model
+        return True
+    if lower.endswith(".sha256") or lower.endswith(".json"):
+        return True
+    return False
+
+
+def _select_gguf_file(repo_id: str, preferred_quant: str) -> str:
+    """
+    Inspect the repo's file list (with sizes) and pick the best GGUF file:
+
+      1. If an exact match for the preferred quant exists among "plain"
+         (non multi-token-prediction / non mmproj) files, use it.
+      2. Otherwise, fall back through QUANT_FALLBACK_ORDER.
+      3. Otherwise, fall back to the smallest remaining .gguf file (best
+         chance of fitting/running on modest CPU hardware).
+
+    "MTP" (multi-token-prediction draft-head) variants are deprioritized:
+    they require extra `--spec-type draft-mtp` speculative-decoding flags
+    that plain llama-cpp-python chat completion does not use, so a plain
+    quant is the safer default for a generic chat UI.
+    """
+    api = HfApi()
+    info = api.model_info(repo_id, files_metadata=True)
+    siblings = [s for s in info.siblings if not _is_excluded(s.rfilename)]
+    if not siblings:
+        raise RuntimeError(f"No usable .gguf files found in repo '{repo_id}'.")
+
+    def is_mtp(name: str) -> bool:
+        low = name.lower()
+        return "-mtp-" in low or low.startswith("mtp-") or "_mtp_" in low
+
+    plain = [s for s in siblings if not is_mtp(s.rfilename)]
+    pool = plain if plain else siblings  # only use MTP files if nothing else exists
+
+    # 1. Exact preferred-quant match among the pool, smallest file wins ties.
+    exact = [s for s in pool if preferred_quant.lower() in s.rfilename.lower()]
+    if exact:
+        best = min(exact, key=lambda s: (s.size or float("inf")))
+        return best.rfilename
+
+    # 2. Fallback quant order.
+    for quant in QUANT_FALLBACK_ORDER:
+        matches = [s for s in pool if quant.lower() in s.rfilename.lower()]
+        if matches:
+            best = min(matches, key=lambda s: (s.size or float("inf")))
+            log.warning(
+                "Preferred quant '%s' not found in %s; falling back to '%s' (%s).",
+                preferred_quant, repo_id, quant, best.rfilename,
+            )
+            return best.rfilename
+
+    # 3. Last resort: smallest .gguf available at all.
+    best = min(pool, key=lambda s: (s.size or float("inf")))
+    log.warning(
+        "No known quant matched preferences in %s; falling back to smallest "
+        "available file: %s", repo_id, best.rfilename,
+    )
+    return best.rfilename
+
+
+def _download_with_retries(repo_id: str, filename: str, cache_dir: str, max_retries: int) -> str:
+    """Download (or reuse the local cache for) a file from the Hub, with
+    exponential-backoff retries so transient network issues don't crash the
+    Space on startup."""
+    last_err: Exception | None = None
+    for attempt in range(1, max_retries + 1):
+        try:
+            log.info("Downloading %s (attempt %d/%d)...", filename, attempt, max_retries)
+            path = hf_hub_download(
+                repo_id=repo_id,
+                filename=filename,
+                cache_dir=cache_dir,
+                # local_files_only=False -> if already cached, this returns
+                # instantly without re-downloading (cache is content-hashed).
+            )
+            log.info("Model file ready at: %s", path)
+            return path
+        except (HfHubHTTPError, OSError, ValueError) as exc:
+            last_err = exc
+            wait = min(2 ** attempt, 30)
+            log.warning("Download attempt %d failed (%s). Retrying in %ds...", attempt, exc, wait)
+            time.sleep(wait)
+    raise RuntimeError(
+        f"Failed to download '{filename}' from '{repo_id}' after {max_retries} attempts: {last_err}"
+    ) from last_err
+
+
+def _load_llm():
+    """Resolve, download and load the GGUF model. Populates module-level
+    state; raises on unrecoverable failure (caught by the caller)."""
+    from llama_cpp import Llama  # imported here so a load failure doesn't
+
+    filename = GGUF_FILENAME or _select_gguf_file(GGUF_REPO_ID, PREFERRED_QUANT)
+    log.info("Selected GGUF file: %s", filename)
+
+    model_path = _download_with_retries(
+        repo_id=GGUF_REPO_ID,
+        filename=filename,
+        cache_dir=MODEL_CACHE_DIR,
+        max_retries=DOWNLOAD_MAX_RETRIES,
+    )
+
+    n_threads = N_THREADS_ENV if N_THREADS_ENV > 0 else max(1, os.cpu_count() or 4)
+    log.info(
+        "Loading model into llama.cpp (n_ctx=%d, n_threads=%d, n_batch=%d)...",
+        N_CTX, n_threads, N_BATCH,
+    )
+
+    llm = Llama(
+        model_path=model_path,
+        n_ctx=N_CTX,
+        n_threads=n_threads,
+        n_batch=N_BATCH,
+        n_gpu_layers=0,  # CPU-only Space: keep everything on CPU.
+        # chat_format left as None (default): llama-cpp-python auto-detects
+        # and applies the chat template embedded in the GGUF's metadata
+        # (the model ships its own Jinja chat template), which is the
+        # current, non-deprecated way to get correct prompting without
+        # hardcoding a template name.
+        verbose=False,
+    )
+    _init_status.update(ready=True, error=None, model_path=model_path, filename=filename)
+    return llm
+
+
+def get_llm():
+    """Thread-safe lazy singleton accessor for the loaded model."""
+    global _llm
+    if _llm is not None:
+        return _llm
+    with _llm_lock:
+        if _llm is None:
+            try:
+                _llm = _load_llm()
+            except Exception as exc:  # noqa: BLE001 - surface any failure to the UI
+                log.exception("Model initialization failed.")
+                _init_status.update(ready=False, error=str(exc))
+                raise
+    return _llm
+
+
+# ---------------------------------------------------------------------------
+# Chat inference
+# ---------------------------------------------------------------------------
+
+def _build_messages(message: str, history: list[dict]) -> list[dict]:
+    messages: list[dict] = []
+    if SYSTEM_PROMPT:
+        messages.append({"role": "system", "content": SYSTEM_PROMPT})
+    # `history` from gr.ChatInterface(type="messages") is already a list of
+    # {"role": ..., "content": ...} dicts.
+    messages.extend(history)
+    messages.append({"role": "user", "content": message})
+    return messages
+
+
+def respond(message: str, history: list[dict]) -> Iterator[str]:
+    """Gradio streaming callback: yields the growing response string as new
+    tokens arrive from llama.cpp's chat-completion stream."""
+    try:
+        llm = get_llm()
+    except Exception as exc:  # noqa: BLE001
+        yield (
+            "⚠️ The model failed to load, so I can't respond right now.\n\n"
+            f"**Error:** {exc}\n\n"
+            "Check the Space's logs for details. If this is a download error, "
+            "it will often resolve itself on a retry/restart; if it persists, "
+            "verify `GGUF_REPO_ID` / `GGUF_FILENAME` are correct and that the "
+            "repo/file are publicly accessible."
+        )
+        return
+
+    messages = _build_messages(message, history)
+
+    try:
+        stream = llm.create_chat_completion(
+            messages=messages,
+            max_tokens=MAX_NEW_TOKENS,
+            temperature=TEMPERATURE,
+            top_p=TOP_P,
+            top_k=TOP_K,
+            repeat_penalty=REPEAT_PENALTY,
+            stream=True,
+        )
+    except Exception as exc:  # noqa: BLE001
+        yield f"⚠️ Generation failed: {exc}"
+        return
+
+    partial = ""
+    for chunk in stream:
+        choice = chunk.get("choices", [{}])[0]
+        delta = choice.get("delta", {})
+        token = delta.get("content")
+        if token:
+            partial += token
+            yield partial
+
+
+# ---------------------------------------------------------------------------
+# Gradio UI
+# ---------------------------------------------------------------------------
+
+def _status_markdown() -> str:
+    return (
+        f"**Model repo:** `{GGUF_REPO_ID}`  \n"
+        f"**Requested quant:** `{PREFERRED_QUANT}`" +
+        (f" (forced file: `{GGUF_FILENAME}`)" if GGUF_FILENAME else "") + "  \n"
+        f"**Context window:** {N_CTX:,} tokens  \n"
+        "*The model downloads on first request and is cached for the life "
+        "of this running container.*"
+    )
+
+
+with gr.Blocks(title="Qwythos-9B Chat") as demo:
+    gr.Markdown("# 🧠 Qwythos-9B Chat (GGUF / llama.cpp)")
+    gr.Markdown(_status_markdown())
+    gr.Markdown(
+        "> ⚠️ This model's card describes it as an uncensored fine-tune with "
+        "no built-in safety layer. If you deploy this publicly, consider "
+        "adding your own moderation/review layer."
+    )
+
+    gr.ChatInterface(
+        fn=respond,
+        type="messages",
+        chatbot=gr.Chatbot(type="messages", height=550, show_copy_button=True),
+        textbox=gr.Textbox(
+            placeholder="Ask something...",
+            scale=7,
+        ),
+        title=None,
+        examples=[
+            "Give me a short summary of what you can do.",
+            "Explain the difference between a stack and a queue.",
+        ],
+        cache_examples=False,
+    )
+
+if __name__ == "__main__":
+    # Kick off model loading in the background as soon as the app starts,
+    # rather than waiting for the first chat message, so the download/load
+    # progress is visible in the Space's build/runtime logs immediately.
+    threading.Thread(target=lambda: (get_llm() if not _init_status["ready"] else None), daemon=True).start()
+
+    demo.queue(max_size=32).launch(
+        server_name=os.environ.get("GRADIO_SERVER_NAME", "0.0.0.0"),
+        server_port=int(os.environ.get("GRADIO_SERVER_PORT", "7860")),
+    )

requirements.txt

@@ -0,0 +1,18 @@
+# ---------------------------------------------------------------------------
+# Python runtime dependencies.
+#
+# NOTE: llama-cpp-python is intentionally NOT listed here. It is built from
+# source in the Dockerfile's builder stage with explicit CMake flags tuned
+# for CPU-only HF Spaces (see Dockerfile comments), then installed from the
+# resulting wheel. Listing an unpinned "llama-cpp-python" here as well would
+# risk pip re-resolving/reinstalling a different (source) build without
+# those flags. If you need to run app.py OUTSIDE Docker (e.g. local dev),
+# install it manually first:
+#
+#   CMAKE_ARGS="-DGGML_NATIVE=ON" pip install llama-cpp-python
+#
+# Then `pip install -r requirements.txt` for the rest.
+# ---------------------------------------------------------------------------
+
+gradio>=5.0,<6.0
+huggingface_hub>=0.24.0