Add backend router and GPU GGUF llama.cpp support

README.md

@@ -6,38 +6,55 @@ colorTo: indigo
 sdk: gradio
 sdk_version: 5.50.0
 app_file: app.py
-short_description: Load and test HF causal LLMs on ZeroGPU
+short_description: Test Transformers and GGUF LLM quants on ZeroGPU
 python_version: "3.10"
 startup_duration_timeout: 1h
 ---
 
 # Dynamic LLM ZeroGPU Playground
 
-A small, general-purpose playground for testing standard Hugging Face causal
-language models by model ID.
+A small, general-purpose playground for testing Hugging Face LLMs by model ID.
+It supports standard Transformers checkpoints and direct, non-dequantized GGUF
+inference through a CUDA-enabled llama.cpp backend.
 
 ## How it works
 
-1. Enter a model ID such as `Qwen/Qwen2.5-0.5B-Instruct`.
-2. Click **Download** to fetch the snapshot into the Space's CPU-side cache.
-3. Click **Load** to put that model on the ZeroGPU worker.
-4. Chat with the model, or use **Unload** before switching models.
-5. Use **Delete from disk** to unload first, then remove all cached revisions for the selected ID.
+1. Enter a model ID.
+2. Click **Inspect / list GGUF**. If the repository contains GGUF, choose one
+   quant file such as Q4_K_M, Q5_K_M, Q8_0, IQ4, or a newer type.
+3. Choose **Backend: Auto** (recommended), or force Transformers/llama.cpp.
+4. Click **Download**. Standard repositories use a CPU-side snapshot download;
+   GGUF repositories download only the selected `.gguf` file.
+5. Click **Load**, then chat. Use **Unload** before switching models and
+   **Delete from disk** to remove the cached revisions/files.
 
 Only one model is kept active by the runtime. Switching models releases the
 previous model with `del`, `gc.collect()`, and `torch.cuda.empty_cache()` before
 the new model is loaded. Chat templates are used whenever the tokenizer
 provides `apply_chat_template()`.
 
-This MVP intentionally targets standard `transformers` +
-`AutoModelForCausalLM` checkpoints. The loader is isolated in
-`TransformersCausalLMRuntime`, so AWQ, GPTQ, or FP8 backends can be added later.
+## Backend routing
+
+- `Auto` routes GGUF to llama.cpp and Transformers weight repositories to
+  `AutoModelForCausalLM`.
+- AWQ, GPTQ, bitsandbytes 4/8-bit, compressed-tensors, and FP8 metadata are
+  detected from `config.json` and filenames. Transformers receives the
+  repository quantization config and uses the installed optional loaders.
+- GGUF is never passed to Transformers or dequantized. llama.cpp is loaded with
+  `n_gpu_layers=-1`, and the GGUF's embedded chat template/metadata is used by
+  the Python binding for current Qwen and other supported architectures.
+- Only one model/backend is active at a time. Cleanup calls `del`,
+  `gc.collect()`, `torch.cuda.empty_cache()`, and llama.cpp's close method when
+  available.
 
 ## Notes
 
 - Model downloads happen on CPU and are never triggered by the chat handler.
 - A model must be downloaded before it can be loaded or used.
 - Large models may exceed ZeroGPU memory or take a long time to load.
-- Remote model code is disabled in this first version for safety and stability.
-- GGUF-only, vision, and other non-causal checkpoints are detected and reported
-  as unsupported instead of producing an opaque Transformers traceback.
+- Remote model code is disabled for safety and stability.
+- A GGUF repository normally embeds its tokenizer/chat metadata, so the large
+  companion files are not downloaded. Multimodal `mmproj` files are listed but
+  are not selected as the default quant.
+- Very large models can still exceed the temporary ZeroGPU memory budget; Q4/Q5
+  GGUF files are generally the best starting point on an A10G.

app.py

@@ -1,4 +1,4 @@
-"""Dynamic Hugging Face causal-LLM playground for ZeroGPU Spaces."""
+"""Dynamic quantized LLM playground for Hugging Face ZeroGPU Spaces."""
 
 from __future__ import annotations
 
@@ -8,17 +8,18 @@ from typing import Any
 
 os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
 
-# ZeroGPU must be imported before torch or any library that may touch CUDA.
+# ZeroGPU must be imported before torch or a library that may initialize CUDA.
 import spaces
-import torch
+import torch  # noqa: F401  # imported after spaces by design
 import gradio as gr
 
-from model_manager import (
-    ModelCache,
-    TransformersCausalLMRuntime,
-    UnsupportedModelError,
-    validate_model_id,
+from backend_router import (
+    BACKEND_AUTO,
+    BACKEND_CHOICES,
+    BackendRouterError,
+    ModelInspection,
 )
+from model_manager import ModelCache, ModelRuntime, validate_model_id
 
 
 logging.basicConfig(level=logging.INFO)
@@ -26,13 +27,13 @@ LOGGER = logging.getLogger(__name__)
 
 DEFAULT_MODEL_ID = "Qwen/Qwen2.5-0.5B-Instruct"
 cache = ModelCache()
-runtime = TransformersCausalLMRuntime(cache)
+runtime = ModelRuntime(cache)
 
 
 def _short_error(prefix: str, exc: Exception) -> str:
     LOGGER.exception("%s", prefix)
     detail = str(exc).strip().splitlines()[0] if str(exc).strip() else exc.__class__.__name__
-    return f"Error: {prefix} {detail[:300]}"
+    return f"Error: {prefix} {detail[:320]}"
 
 
 def _safe_generation_settings(
@@ -46,40 +47,114 @@ def _safe_generation_settings(
     return tokens, temp, nucleus
 
 
-def download_model(model_id: str) -> tuple[str, str]:
-    """Download a model snapshot on CPU into the playground cache."""
+def _dropdown_update(inspection: ModelInspection | None) -> Any:
+    choices = inspection.gguf_files if inspection else []
+    value = inspection.default_gguf if inspection else None
+    return gr.update(choices=choices, value=value)
+
+
+def _inspection_markdown(
+    inspection: ModelInspection | None,
+    backend: str | None = None,
+    selected_file: str | None = None,
+) -> str:
+    if inspection is None:
+        return "**Detected format:** not inspected yet  \n**Backend:** Auto"
+    return inspection.markdown(backend, selected_file)
+
+
+def inspect_model(model_id: str) -> tuple[str, Any, str, str]:
+    """Inspect repository metadata and list GGUF choices without downloading weights."""
 
     try:
         model_id = validate_model_id(model_id)
-        snapshot_path = cache.download(model_id)
-        try:
-            cache.ensure_transformers_checkpoint(model_id)
-            status = f"Downloaded on CPU: `{model_id}`"
-        except UnsupportedModelError as exc:
-            status = f"Downloaded on CPU, but this MVP cannot Load it: {exc}"
+        inspection = cache.inspect_remote(model_id)
+        selected = inspection.default_gguf
+        status = f"Inspected `{model_id}` on CPU."
+        if inspection.gguf_files:
+            status += " Select a GGUF file, then click Download."
+        else:
+            status += " No GGUF file was found; Auto will use Transformers."
         return (
             status,
-            f"Disk cache: ready ({snapshot_path.name}).",
+            _dropdown_update(inspection),
+            _inspection_markdown(inspection, selected_file=selected),
+            cache.describe(model_id, selected),
         )
     except Exception as exc:
-        message = _short_error("Could not download the model:", exc)
-        return message, cache.describe(model_id)
+        return (
+            _short_error("Could not inspect the repository:", exc),
+            _dropdown_update(None),
+            _inspection_markdown(None),
+            cache.describe(model_id),
+        )
+
+
+def download_model(
+    model_id: str,
+    backend_choice: str,
+    gguf_file: str | None,
+) -> tuple[str, Any, str, str]:
+    """Download a standard snapshot or exactly one selected GGUF on CPU."""
+
+    try:
+        model_id = validate_model_id(model_id)
+        inspection = cache.inspect_remote(model_id)
+        selected = (gguf_file or inspection.default_gguf or "").strip()
+        backend = cache.router.resolve_backend(inspection, backend_choice, selected or None)
+
+        if backend == "llama.cpp":
+            path = cache.download_gguf(model_id, selected)
+            status = f"Downloaded one GGUF file on CPU: `{selected}`"
+            cache_status = cache.describe(model_id, selected)
+            return (
+                status,
+                _dropdown_update(inspection),
+                _inspection_markdown(inspection, backend, selected),
+                cache_status,
+            )
+
+        path = cache.download(model_id)
+        return (
+            f"Downloaded Transformers files on CPU: `{model_id}`",
+            _dropdown_update(inspection),
+            _inspection_markdown(inspection, backend),
+            f"Disk cache: snapshot ready (`{path.name}`).",
+        )
+    except Exception as exc:
+        return (
+            _short_error("Could not download the model:", exc),
+            _dropdown_update(None),
+            _inspection_markdown(None),
+            cache.describe(model_id, gguf_file),
+        )
 
 
 @spaces.GPU(duration=420)
-def load_model_on_gpu(model_id: str) -> tuple[str, str, str]:
-    """Load one downloaded Transformers causal LM on the ZeroGPU worker."""
+def load_model_on_gpu(
+    model_id: str,
+    backend_choice: str,
+    gguf_file: str | None,
+) -> tuple[str, str, str, str]:
+    """Load one selected model on ZeroGPU using the routed backend."""
 
     try:
         model_id = validate_model_id(model_id)
-        active = runtime.ensure_loaded(model_id)
+        target = runtime.ensure_loaded(model_id, backend_choice, gguf_file)
+        selected = target.selected_file
         return (
-            f"Loaded on ZeroGPU: `{active}`",
-            active,
-            cache.describe(active),
+            f"Loaded on ZeroGPU: `{model_id}` via `{target.backend}`",
+            runtime.active_label(),
+            _inspection_markdown(target.inspection, target.backend, selected),
+            cache.describe(model_id, selected),
         )
     except Exception as exc:
-        return _short_error("Could not load the model:", exc), "No model loaded", cache.describe(model_id)
+        return (
+            _short_error("Could not load the model:", exc),
+            "No model loaded",
+            _inspection_markdown(None),
+            cache.describe(model_id, gguf_file),
+        )
 
 
 @spaces.GPU(duration=180)
@@ -87,12 +162,14 @@ def chat_with_model(
     message: str,
     history: list[Any] | None,
     model_id: str,
+    backend_choice: str,
+    gguf_file: str | None,
     system_prompt: str,
     max_new_tokens: Any,
     temperature: Any,
     top_p: Any,
 ) -> str:
-    """Generate a reply using the selected cached Hugging Face model."""
+    """Generate a reply through the active Transformers or llama.cpp runtime."""
 
     try:
         model_id = validate_model_id(model_id)
@@ -101,6 +178,8 @@ def chat_with_model(
         )
         return runtime.generate(
             model_id=model_id,
+            requested_backend=backend_choice,
+            selected_file=gguf_file,
             message=message,
             history=history,
             system_prompt=system_prompt,
@@ -113,29 +192,55 @@ def chat_with_model(
 
 
 @spaces.GPU(duration=30)
-def unload_model_on_gpu(model_id: str) -> tuple[str, str, str]:
-    """Unload the active model and release RAM/VRAM on the ZeroGPU worker."""
+def unload_model_on_gpu(model_id: str) -> tuple[str, str, str, str]:
+    """Unload the active runtime and release RAM/VRAM."""
 
     try:
         runtime.unload()
-        return "Unloaded; RAM/VRAM cleanup requested.", "No model loaded", cache.describe(model_id)
+        return (
+            "Unloaded; RAM/VRAM cleanup requested.",
+            "No model loaded",
+            "**Detected format:** none active  \n**Backend:** none",
+            cache.describe(model_id),
+        )
     except Exception as exc:
-        return _short_error("Could not unload the model:", exc), "Unknown", cache.describe(model_id)
+        return (
+            _short_error("Could not unload the model:", exc),
+            "Unknown",
+            _inspection_markdown(None),
+            cache.describe(model_id),
+        )
 
 
-def delete_model_from_disk(model_id: str) -> tuple[str, str, str]:
-    """Delete every cached revision of the selected model from disk."""
+def delete_model_from_disk(
+    model_id: str,
+    gguf_file: str | None,
+) -> tuple[str, str, str, str, Any]:
+    """Remove all cached revisions/files for the selected model on CPU."""
 
     try:
         model_id = validate_model_id(model_id)
         deleted = cache.delete(model_id)
-        if deleted:
-            status = f"Deleted from disk: `{model_id}`"
-        else:
-            status = f"No cached files found for `{model_id}`"
-        return status, "No model loaded", cache.describe(model_id)
+        status = (
+            f"Deleted from disk: `{model_id}`"
+            if deleted
+            else f"No cached files found for `{model_id}`"
+        )
+        return (
+            status,
+            "No model loaded",
+            "**Detected format:** none active  \n**Backend:** none",
+            cache.describe(model_id),
+            _dropdown_update(None),
+        )
     except Exception as exc:
-        return _short_error("Could not delete the model cache:", exc), "Unknown", cache.describe(model_id)
+        return (
+            _short_error("Could not delete the model cache:", exc),
+            "Unknown",
+            _inspection_markdown(None),
+            cache.describe(model_id, gguf_file),
+            _dropdown_update(None),
+        )
 
 
 CSS = """
@@ -144,15 +249,15 @@ CSS = """
 """
 
 
-with gr.Blocks(title="Dynamic LLM ZeroGPU Playground", css=CSS) as demo:
+with gr.Blocks(title="Quantized LLM ZeroGPU Playground", css=CSS) as demo:
     gr.Markdown(
         """
-        # Dynamic LLM ZeroGPU Playground
+        # Quantized LLM ZeroGPU Playground
 
-        Download standard `transformers` causal language models, load one model
-        at a time on ZeroGPU, and test it through a Gradio chat interface.
-        Downloading and cache management stay on CPU; model loading and
-        inference use the GPU only when requested.
+        Download and test one Hugging Face LLM at a time. **Auto** routes
+        standard Transformers checkpoints to Transformers and GGUF files to
+        the CUDA-enabled llama.cpp backend. GGUF repositories are inspected
+        first so you can select only the Q4/Q5/Q8 (or newer) quant you want.
         """
     )
 
@@ -163,6 +268,25 @@ with gr.Blocks(title="Dynamic LLM ZeroGPU Playground", css=CSS) as demo:
             placeholder="namespace/model-name",
             scale=4,
         )
+        backend_choice = gr.Radio(
+            label="Backend",
+            choices=BACKEND_CHOICES,
+            value=BACKEND_AUTO,
+            scale=2,
+        )
+
+    with gr.Row():
+        inspect_button = gr.Button("Inspect / list GGUF", variant="secondary")
+        gguf_file = gr.Dropdown(
+            label="GGUF quant file (choose one)",
+            choices=[],
+            value=None,
+            allow_custom_value=False,
+            interactive=True,
+            scale=4,
+        )
+
+    with gr.Row():
         download_button = gr.Button("Download", variant="secondary", scale=1)
         load_button = gr.Button("Load", variant="primary", scale=1)
         unload_button = gr.Button("Unload", variant="secondary", scale=1)
@@ -170,14 +294,17 @@ with gr.Blocks(title="Dynamic LLM ZeroGPU Playground", css=CSS) as demo:
 
     with gr.Row():
         current_model = gr.Textbox(
-            label="Active model",
+            label="Active model / backend",
             value="No model loaded",
             interactive=False,
             scale=1,
         )
         cache_status = gr.Markdown("Disk cache: no model selected.")
 
-    status = gr.Markdown("Status: enter a model ID, then click Download.")
+    status = gr.Markdown("Status: inspect a repository, then Download and Load it.")
+    format_backend = gr.Markdown(
+        "**Detected format:** not inspected yet  \n**Backend:** Auto"
+    )
 
     with gr.Accordion("Generation settings", open=True):
         system_prompt = gr.Textbox(
@@ -199,40 +326,53 @@ with gr.Blocks(title="Dynamic LLM ZeroGPU Playground", css=CSS) as demo:
         fn=chat_with_model,
         chatbot=chatbot,
         type="messages",
-        additional_inputs=[model_id, system_prompt, max_new_tokens, temperature, top_p],
+        additional_inputs=[
+            model_id,
+            backend_choice,
+            gguf_file,
+            system_prompt,
+            max_new_tokens,
+            temperature,
+            top_p,
+        ],
         textbox=gr.Textbox(placeholder="Write a message…", container=False),
         api_name="chat",
     )
 
+    inspect_button.click(
+        fn=inspect_model,
+        inputs=[model_id],
+        outputs=[status, gguf_file, format_backend, cache_status],
+        api_name="inspect",
+    )
     download_button.click(
         fn=download_model,
-        inputs=[model_id],
-        outputs=[status, cache_status],
+        inputs=[model_id, backend_choice, gguf_file],
+        outputs=[status, gguf_file, format_backend, cache_status],
         api_name="download",
     )
     load_button.click(
         fn=load_model_on_gpu,
-        inputs=[model_id],
-        outputs=[status, current_model, cache_status],
+        inputs=[model_id, backend_choice, gguf_file],
+        outputs=[status, current_model, format_backend, cache_status],
         api_name="load",
     )
     unload_button.click(
         fn=unload_model_on_gpu,
         inputs=[model_id],
-        outputs=[status, current_model, cache_status],
+        outputs=[status, current_model, format_backend, cache_status],
         api_name="unload",
     )
     delete_event = delete_button.click(
-        # Release a possibly active GPU copy before removing its CPU cache.
-        # The actual deletion remains a CPU-only operation in the next step.
+        # Release a possibly active GPU copy before deleting its CPU cache.
         fn=unload_model_on_gpu,
         inputs=[model_id],
-        outputs=[status, current_model, cache_status],
+        outputs=[status, current_model, format_backend, cache_status],
     )
     delete_event.then(
         fn=delete_model_from_disk,
-        inputs=[model_id],
-        outputs=[status, current_model, cache_status],
+        inputs=[model_id, gguf_file],
+        outputs=[status, current_model, format_backend, cache_status, gguf_file],
         api_name="delete_from_disk",
     )
 
@@ -240,3 +380,4 @@ with gr.Blocks(title="Dynamic LLM ZeroGPU Playground", css=CSS) as demo:
 if __name__ == "__main__":
     demo.queue(default_concurrency_limit=1)
     demo.launch(mcp_server=True)
+

backend_router.py

@@ -0,0 +1,374 @@
+"""CPU-side model inspection and backend routing.
+
+The router never imports torch or initializes a model.  It only looks at the
+Hub file list and (when useful) the small ``config.json`` file.  This keeps
+download/inspection work outside ZeroGPU allocations and makes it possible to
+add another runtime without changing the UI contract.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Iterable
+
+from huggingface_hub import HfApi, hf_hub_download
+
+
+BACKEND_AUTO = "Auto"
+BACKEND_TRANSFORMERS = "Transformers"
+BACKEND_LLAMACPP = "llama.cpp"
+BACKEND_CHOICES = [BACKEND_AUTO, BACKEND_TRANSFORMERS, BACKEND_LLAMACPP]
+
+QUANTIZED_TRANSFORMERS_KINDS = {
+    "awq",
+    "gptq",
+    "bitsandbytes",
+    "compressed-tensors",
+    "fp8",
+}
+
+_GGUF_QUANT_RE = re.compile(
+    r"(?i)(?:^|[_\-.])((?:iq|q|tq)\d+(?:[_\-][a-z0-9]+)*|mxfp4|nvfp4|fp8|bf16|f16|f32)(?:[_\-.]|$)"
+)
+_STANDARD_WEIGHT_NAMES = {
+    "model.safetensors.index.json",
+    "pytorch_model.bin.index.json",
+}
+_STANDARD_WEIGHT_SUFFIXES = (".safetensors", ".bin", ".pt", ".pth")
+
+
+class BackendRouterError(ValueError):
+    """Raised when a model cannot be mapped to a supported backend."""
+
+
+@dataclass
+class ModelInspection:
+    """A small, serializable description of one Hub repository."""
+
+    model_id: str
+    files: list[str] = field(default_factory=list)
+    gguf_files: list[str] = field(default_factory=list)
+    has_standard_weights: bool = False
+    quantization_kind: str = "none"
+    format_label: str = "Unknown"
+    preferred_backend: str = BACKEND_TRANSFORMERS
+    config: dict[str, Any] = field(default_factory=dict)
+    source: str = "remote"
+
+    @property
+    def is_gguf(self) -> bool:
+        return bool(self.gguf_files)
+
+    @property
+    def is_transformers_quantized(self) -> bool:
+        return self.quantization_kind in QUANTIZED_TRANSFORMERS_KINDS
+
+    @property
+    def default_gguf(self) -> str | None:
+        """Prefer a normal LLM quant over a multimodal projector file."""
+
+        candidates = [
+            name for name in self.gguf_files if "mmproj" not in name.lower()
+        ] or list(self.gguf_files)
+        if not candidates:
+            return None
+
+        def rank(name: str) -> tuple[int, str]:
+            lowered = name.lower()
+            preferred = (
+                "q4_k_m",
+                "q5_k_m",
+                "q4_k_s",
+                "q5_k_s",
+                "q6_k",
+                "q8_0",
+                "iq4",
+            )
+            for index, token in enumerate(preferred):
+                if token in lowered:
+                    return index, lowered
+            return len(preferred), lowered
+
+        return min(candidates, key=rank)
+
+    @property
+    def gguf_quantizations(self) -> list[str]:
+        values: set[str] = set()
+        for filename in self.gguf_files:
+            for match in _GGUF_QUANT_RE.finditer(filename):
+                values.add(match.group(1).replace("-", "_"))
+        return sorted(values, key=str.lower)
+
+    def markdown(self, resolved_backend: str | None = None, selected_file: str | None = None) -> str:
+        backend = resolved_backend or self.preferred_backend
+        lines = [
+            f"**Detected format:** `{self.format_label}`  ",
+            f"**Backend:** `{backend}`",
+        ]
+        if selected_file:
+            lines.append(f"  \n**Selected GGUF:** `{selected_file}`")
+        if self.gguf_files:
+            lines.append(
+                f"  \n**GGUF files:** {len(self.gguf_files)} found; only the selected file is downloaded."
+            )
+        if self.is_transformers_quantized:
+            lines.append(
+                "  \nThe Transformers quantization config will be passed to the corresponding loader."
+            )
+        return "\n".join(lines)
+
+
+@dataclass
+class ResolvedBackend:
+    """A cached model path plus the backend selected for it."""
+
+    backend: str
+    path: Path
+    inspection: ModelInspection
+    selected_file: str | None = None
+
+
+def _is_gguf(filename: str) -> bool:
+    return filename.lower().endswith(".gguf")
+
+
+def _has_standard_weights(files: Iterable[str]) -> bool:
+    return any(
+        name.lower().endswith(_STANDARD_WEIGHT_SUFFIXES)
+        or Path(name).name.lower() in _STANDARD_WEIGHT_NAMES
+        for name in files
+    )
+
+
+def _read_json(path: Path) -> dict[str, Any]:
+    try:
+        value = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, UnicodeDecodeError, json.JSONDecodeError):
+        return {}
+    return value if isinstance(value, dict) else {}
+
+
+def _quantization_from_config(config: dict[str, Any]) -> tuple[str, str | None]:
+    raw_config = config.get("quantization_config")
+    quant_config = raw_config if isinstance(raw_config, dict) else {}
+    raw = json.dumps(quant_config, sort_keys=True).lower()
+    model_text = json.dumps(config, sort_keys=True).lower()
+
+    quant_method = str(
+        quant_config.get("quant_method")
+        or quant_config.get("quantization_method")
+        or quant_config.get("method")
+        or ""
+    ).lower()
+
+    if "bitsandbytes" in quant_method or any(
+        key in quant_config
+        for key in ("load_in_4bit", "load_in_8bit", "_load_in_4bit", "_load_in_8bit")
+    ):
+        bits = "4-bit" if quant_config.get("load_in_4bit", quant_config.get("_load_in_4bit")) else "8-bit"
+        return "bitsandbytes", f"bitsandbytes {bits}"
+    if "awq" in quant_method or "awq" in raw:
+        return "awq", "AWQ"
+    if "gptq" in quant_method or "gptq" in raw:
+        return "gptq", "GPTQ"
+    if "compressed" in quant_method or "compressed-tensors" in raw:
+        if "fp8" in raw or "float8" in raw or "nvfp4" in raw:
+            return "compressed-tensors", "compressed-tensors / FP8 or FP4"
+        return "compressed-tensors", "compressed-tensors"
+    if "fp8" in quant_method or "float8" in raw or "float8" in model_text:
+        return "fp8", "FP8"
+    return "none", None
+
+
+def _quantization_from_filenames(files: Iterable[str]) -> tuple[str, str | None]:
+    text = " ".join(files).lower()
+    if "bitsandbytes" in text or "bnb" in text:
+        if "4bit" in text or "4-bit" in text:
+            return "bitsandbytes", "bitsandbytes 4-bit (filename heuristic)"
+        if "8bit" in text or "8-bit" in text:
+            return "bitsandbytes", "bitsandbytes 8-bit (filename heuristic)"
+    if "compressed-tensors" in text or "compressed_tensors" in text:
+        return "compressed-tensors", "compressed-tensors (filename heuristic)"
+    if "gptq" in text:
+        return "gptq", "GPTQ (filename heuristic)"
+    if "awq" in text:
+        return "awq", "AWQ (filename heuristic)"
+    if "fp8" in text or "float8" in text or "nvfp4" in text:
+        return "fp8", "FP8 / FP4 (filename heuristic)"
+    return "none", None
+
+
+def _dtype_label(config: dict[str, Any]) -> str:
+    value = str(config.get("torch_dtype") or config.get("dtype") or "").lower()
+    if "bfloat16" in value or value == "bf16":
+        return "BF16"
+    if "float16" in value or value in {"fp16", "half"}:
+        return "FP16"
+    if "float8" in value or "fp8" in value:
+        return "FP8"
+    if "float32" in value or value == "fp32":
+        return "FP32"
+    return "dtype auto"
+
+
+def inspection_from_files(
+    model_id: str,
+    files: Iterable[str],
+    config: dict[str, Any] | None = None,
+    source: str = "remote",
+) -> ModelInspection:
+    """Build an inspection from a file list and an optional config."""
+
+    file_list = sorted(set(str(name) for name in files))
+    gguf_files = sorted(name for name in file_list if _is_gguf(name))
+    config = config or {}
+    has_weights = _has_standard_weights(file_list)
+
+    if gguf_files:
+        quantizations = ModelInspection(
+            model_id=model_id,
+            files=file_list,
+            gguf_files=gguf_files,
+        ).gguf_quantizations
+        quant_label = ", ".join(quantizations) if quantizations else "quantized"
+        format_label = f"GGUF / {quant_label}"
+        return ModelInspection(
+            model_id=model_id,
+            files=file_list,
+            gguf_files=gguf_files,
+            has_standard_weights=has_weights,
+            quantization_kind="gguf",
+            format_label=format_label,
+            preferred_backend=BACKEND_LLAMACPP,
+            config=config,
+            source=source,
+        )
+
+    kind, label = _quantization_from_config(config)
+    if kind == "none":
+        kind, label = _quantization_from_filenames(file_list)
+    if label is None:
+        label = f"safetensors / {_dtype_label(config)}" if has_weights else "Unknown"
+
+    return ModelInspection(
+        model_id=model_id,
+        files=file_list,
+        gguf_files=[],
+        has_standard_weights=has_weights,
+        quantization_kind=kind,
+        format_label=label,
+        preferred_backend=BACKEND_TRANSFORMERS,
+        config=config,
+        source=source,
+    )
+
+
+class BackendRouter:
+    """Inspect Hub repositories and resolve an explicit runtime backend."""
+
+    def __init__(self, api: HfApi | None = None) -> None:
+        self.api = api or HfApi()
+
+    def inspect_remote(self, model_id: str, cache_dir: str | Path | None = None) -> ModelInspection:
+        files = list(self.api.list_repo_files(repo_id=model_id, repo_type="model"))
+        gguf_files = [name for name in files if _is_gguf(name)]
+        config: dict[str, Any] = {}
+
+        # GGUF contains its own architecture/template metadata.  Avoid even
+        # fetching config.json for a GGUF-only repository; the selected GGUF
+        # is the only large artifact downloaded later.
+        if not gguf_files or _has_standard_weights(files):
+            try:
+                config_path = hf_hub_download(
+                    repo_id=model_id,
+                    filename="config.json",
+                    repo_type="model",
+                    cache_dir=str(cache_dir) if cache_dir else None,
+                )
+                config = _read_json(Path(config_path))
+            except Exception:
+                config = {}
+
+        return inspection_from_files(
+            model_id=model_id,
+            files=files,
+            config=config,
+            source="remote",
+        )
+
+    def inspect_snapshot(self, model_id: str, snapshot_path: str | Path) -> ModelInspection:
+        root = Path(snapshot_path)
+        files = [str(path.relative_to(root)) for path in root.rglob("*") if path.is_file()]
+        return inspection_from_files(
+            model_id=model_id,
+            files=files,
+            config=_read_json(root / "config.json"),
+            source="cache",
+        )
+
+    @staticmethod
+    def synthetic_gguf(model_id: str, filename: str) -> ModelInspection:
+        return inspection_from_files(
+            model_id=model_id,
+            files=[filename],
+            config={},
+            source="cache",
+        )
+
+    @staticmethod
+    def _normalize_backend(value: str | None) -> str:
+        normalized = (value or BACKEND_AUTO).strip().lower()
+        if normalized in {"auto", "automatic"}:
+            return BACKEND_AUTO
+        if normalized in {"transformers", "transformer"}:
+            return BACKEND_TRANSFORMERS
+        if normalized in {"llama.cpp", "llama-cpp", "llamacpp", "llama"}:
+            return BACKEND_LLAMACPP
+        raise BackendRouterError(
+            f"Unknown backend `{value}`. Choose Auto, Transformers, or llama.cpp."
+        )
+
+    def resolve_backend(
+        self,
+        inspection: ModelInspection,
+        requested_backend: str | None = BACKEND_AUTO,
+        selected_file: str | None = None,
+    ) -> str:
+        requested = self._normalize_backend(requested_backend)
+        selected = (selected_file or "").strip()
+        selected_is_gguf = bool(selected) and _is_gguf(selected)
+
+        if selected and selected not in inspection.gguf_files:
+            raise BackendRouterError(
+                f"`{selected}` is not one of the GGUF files detected in `{inspection.model_id}`."
+            )
+
+        if requested == BACKEND_AUTO:
+            if selected_is_gguf:
+                return BACKEND_LLAMACPP
+            if inspection.is_gguf and not inspection.has_standard_weights:
+                raise BackendRouterError(
+                    "This is a GGUF repository. Select a `.gguf` quant file before downloading or loading it."
+                )
+            return BACKEND_TRANSFORMERS
+
+        if requested == BACKEND_LLAMACPP:
+            if not selected_is_gguf:
+                raise BackendRouterError(
+                    "llama.cpp requires a selected `.gguf` file. Inspect the repository and choose a quant."
+                )
+            return BACKEND_LLAMACPP
+
+        if selected_is_gguf:
+            raise BackendRouterError(
+                "Transformers cannot load a GGUF file here; choose llama.cpp or select Transformers weights."
+            )
+        if not inspection.has_standard_weights:
+            raise BackendRouterError(
+                "No standard Transformers weight file was found. This repository needs a GGUF file and llama.cpp."
+            )
+        return BACKEND_TRANSFORMERS
+

model_manager.py

@@ -1,14 +1,8 @@
-"""CPU cache helpers and the single-model Transformers runtime.
-
-The runtime deliberately keeps model loading behind the GPU handlers in
-``app.py``.  That makes model IDs dynamic while still ensuring that no model
-weights are loaded during Space startup.
-"""
+"""Cache management and single-model runtimes for the ZeroGPU playground."""
 
 from __future__ import annotations
 
 import gc
-import json
 import logging
 import os
 import re
@@ -16,19 +10,29 @@ import threading
 from pathlib import Path
 from typing import Any
 
+# ZeroGPU must be imported before torch.  The lazy llama.cpp import below is
+# also intentionally kept inside the GPU-side loader.
 import spaces
 import torch
-from huggingface_hub import scan_cache_dir, snapshot_download
+from huggingface_hub import hf_hub_download, scan_cache_dir, snapshot_download
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
+from backend_router import (
+    BACKEND_AUTO,
+    BACKEND_LLAMACPP,
+    BACKEND_TRANSFORMERS,
+    BackendRouter,
+    BackendRouterError,
+    ModelInspection,
+    ResolvedBackend,
+)
+
 
 LOGGER = logging.getLogger(__name__)
 MODEL_ID_PATTERN = re.compile(r"^[^/\s]+/[^/\s]+$")
 
 
 def validate_model_id(model_id: str) -> str:
-    """Validate and normalize a Hugging Face model repository ID."""
-
     normalized = (model_id or "").strip()
     if not MODEL_ID_PATTERN.fullmatch(normalized):
         raise ValueError("Model ID must look like namespace/model-name.")
@@ -36,92 +40,137 @@ def validate_model_id(model_id: str) -> str:
 
 
 class UnsupportedModelError(RuntimeError):
-    """Raised when a repository is outside the MVP's Transformers LM scope."""
+    """Compatibility alias for callers that want a user-facing load error."""
 
 
 class ModelCache:
-    """A dedicated Hugging Face cache for downloaded model snapshots."""
+    """Keep standard snapshots and individually selected GGUF files in one cache."""
 
     def __init__(self, cache_dir: str | None = None) -> None:
         default_dir = Path.home() / ".cache" / "huggingface" / "llm-playground"
         self.root = Path(cache_dir or os.getenv("PLAYGROUND_CACHE_DIR", default_dir))
         self.root.mkdir(parents=True, exist_ok=True)
+        self.router = BackendRouter()
+
+    def inspect_remote(self, model_id: str) -> ModelInspection:
+        return self.router.inspect_remote(validate_model_id(model_id), cache_dir=self.root)
 
     def download(self, model_id: str) -> Path:
-        """Download a complete model snapshot without initializing a model."""
+        """Download a standard Transformers snapshot on CPU."""
 
-        model_id = validate_model_id(model_id)
-        snapshot_path = snapshot_download(
-            repo_id=model_id,
-            repo_type="model",
-            cache_dir=str(self.root),
+        return Path(
+            snapshot_download(
+                repo_id=validate_model_id(model_id),
+                repo_type="model",
+                cache_dir=str(self.root),
+            )
         )
-        return Path(snapshot_path)
 
-    def cached_snapshot(self, model_id: str) -> Path:
-        """Return a locally cached snapshot, raising if it is not complete."""
+    def download_gguf(self, model_id: str, filename: str) -> Path:
+        """Download exactly one GGUF file, never the whole repository."""
 
         model_id = validate_model_id(model_id)
-        try:
-            snapshot_path = snapshot_download(
+        filename = (filename or "").strip()
+        if not filename or not filename.lower().endswith(".gguf"):
+            raise ValueError("Choose one `.gguf` file before downloading.")
+
+        inspection = self.inspect_remote(model_id)
+        if filename not in inspection.gguf_files:
+            raise ValueError(f"`{filename}` is not a GGUF file in `{model_id}`.")
+
+        return Path(
+            hf_hub_download(
                 repo_id=model_id,
+                filename=filename,
                 repo_type="model",
                 cache_dir=str(self.root),
-                local_files_only=True,
             )
-        except Exception as exc:  # hub versions expose different local-cache errors
+        )
+
+    def cached_snapshot(self, model_id: str) -> Path:
+        model_id = validate_model_id(model_id)
+        try:
+            return Path(
+                snapshot_download(
+                    repo_id=model_id,
+                    repo_type="model",
+                    cache_dir=str(self.root),
+                    local_files_only=True,
+                )
+            )
+        except Exception as exc:
             raise FileNotFoundError(
                 f"{model_id} is not downloaded yet. Click Download first."
             ) from exc
-        return Path(snapshot_path)
-
-    def ensure_transformers_checkpoint(self, model_id: str) -> Path:
-        """Validate that a cached repo looks like a standard Transformers LM."""
-
-        snapshot_path = self.cached_snapshot(model_id)
-        files = [path for path in snapshot_path.rglob("*") if path.is_file()]
-        has_gguf = any(path.name.lower().endswith(".gguf") for path in files)
-        has_standard_weights = any(
-            path.name.lower().endswith((".safetensors", ".bin", ".pt", ".pth"))
-            or path.name.lower() in {"model.safetensors.index.json", "pytorch_model.bin.index.json"}
-            for path in files
-        )
 
-        if has_gguf and not has_standard_weights:
-            raise UnsupportedModelError(
-                "This Space currently supports standard Transformers checkpoints only. "
-                "The selected repository contains GGUF/quantized files; use a repo with "
-                "safetensors or PyTorch weights, or wait for a GGUF backend."
+    def cached_gguf(self, model_id: str, filename: str) -> Path:
+        model_id = validate_model_id(model_id)
+        filename = (filename or "").strip()
+        if not filename.lower().endswith(".gguf"):
+            raise ValueError("Choose one `.gguf` file before loading.")
+        try:
+            return Path(
+                hf_hub_download(
+                    repo_id=model_id,
+                    filename=filename,
+                    repo_type="model",
+                    cache_dir=str(self.root),
+                    local_files_only=True,
+                )
             )
+        except Exception as exc:
+            raise FileNotFoundError(
+                f"`{filename}` is not downloaded yet. Click Download for this GGUF file first."
+            ) from exc
 
-        config_path = snapshot_path / "config.json"
-        if config_path.is_file():
-            try:
-                config = json.loads(config_path.read_text(encoding="utf-8"))
-            except (OSError, json.JSONDecodeError):
-                config = {}
-            if config.get("model_type") == "bit":
-                raise UnsupportedModelError(
-                    "The selected repository declares model_type=bit, which is a vision "
-                    "backbone and not a causal language model."
-                )
+    def resolve_cached(
+        self,
+        model_id: str,
+        requested_backend: str | None,
+        selected_file: str | None,
+    ) -> ResolvedBackend:
+        """Resolve a local artifact without doing network I/O on a GPU call."""
 
-        return snapshot_path
+        model_id = validate_model_id(model_id)
+        selected = (selected_file or "").strip()
+        if selected:
+            path = self.cached_gguf(model_id, selected)
+            inspection = self.router.synthetic_gguf(model_id, selected)
+        else:
+            path = self.cached_snapshot(model_id)
+            inspection = self.router.inspect_snapshot(model_id, path)
 
-    def describe(self, model_id: str) -> str:
-        """Return a short cache status for the UI."""
+        backend = self.router.resolve_backend(
+            inspection,
+            requested_backend=requested_backend,
+            selected_file=selected or None,
+        )
+        if backend == BACKEND_LLAMACPP and not selected:
+            raise BackendRouterError(
+                "Choose the GGUF file you downloaded before loading it with llama.cpp."
+            )
+        return ResolvedBackend(
+            backend=backend,
+            path=path,
+            inspection=inspection,
+            selected_file=selected or None,
+        )
 
+    def describe(self, model_id: str, selected_file: str | None = None) -> str:
         model_id = (model_id or "").strip()
         if not model_id:
             return "Disk cache: no model selected."
         try:
+            if selected_file:
+                path = self.cached_gguf(model_id, selected_file)
+                return f"Disk cache: GGUF ready (`{path.name}`)."
             path = self.cached_snapshot(model_id)
+            return f"Disk cache: snapshot ready (`{path.name}`)."
         except (ValueError, FileNotFoundError):
-            return f"Disk cache: {model_id} is not downloaded."
-        return f"Disk cache: ready ({path.name})."
+            return f"Disk cache: `{model_id}` is not downloaded."
 
     def delete(self, model_id: str) -> bool:
-        """Delete every cached revision of one model from this cache."""
+        """Remove all cached revisions for one model, including selected GGUFs."""
 
         model_id = validate_model_id(model_id)
         cache_info = scan_cache_dir(cache_dir=str(self.root))
@@ -129,81 +178,225 @@ class ModelCache:
         for repo in cache_info.repos:
             if repo.repo_id == model_id:
                 revisions.extend(revision.commit_hash for revision in repo.revisions)
-
         if not revisions:
             return False
-
-        # The cache manager removes snapshots, refs, and blobs that are no
-        # longer shared by another cached revision.
         cache_info.delete_revisions(*revisions).execute()
         return True
 
 
-class TransformersCausalLMRuntime:
-    """Single active standard Transformers causal language model."""
+class ModelRuntime:
+    """Route one active model to Transformers or llama.cpp."""
 
     def __init__(self, cache: ModelCache) -> None:
         self.cache = cache
         self._model: Any | None = None
         self._tokenizer: Any | None = None
+        self._llama: Any | None = None
         self._model_id: str | None = None
+        self._backend: str | None = None
+        self._selected_file: str | None = None
+        self._inspection: ModelInspection | None = None
         self._lock = threading.RLock()
 
     @property
     def active_model_id(self) -> str | None:
         return self._model_id
 
+    @property
+    def active_backend(self) -> str | None:
+        return self._backend
+
+    @property
+    def active_selected_file(self) -> str | None:
+        return self._selected_file
+
+    @property
+    def active_inspection(self) -> ModelInspection | None:
+        return self._inspection
+
+    def active_label(self) -> str:
+        if not self._model_id:
+            return "No model loaded"
+        suffix = f" · {self._backend}"
+        if self._selected_file:
+            suffix += f" · {self._selected_file}"
+        return f"{self._model_id}{suffix}"
+
     def unload(self) -> None:
-        """Release the active model and clear CUDA's allocator cache."""
+        """Release both possible runtimes and clear CUDA allocator state."""
 
         with self._lock:
             old_model = self._model
+            old_llama = self._llama
             self._model = None
             self._tokenizer = None
+            self._llama = None
             self._model_id = None
-
+            self._backend = None
+            self._selected_file = None
+            self._inspection = None
+
+            if old_llama is not None:
+                close = getattr(old_llama, "close", None)
+                if callable(close):
+                    try:
+                        close()
+                    except Exception:
+                        LOGGER.debug("llama.cpp close failed during cleanup", exc_info=True)
+                del old_llama
             if old_model is not None:
                 del old_model
+
             gc.collect()
             if torch.cuda.is_available():
                 torch.cuda.empty_cache()
 
-    def ensure_loaded(self, model_id: str) -> str:
-        """Load one cached model on CUDA, replacing any previously active model."""
+    def ensure_loaded(
+        self,
+        model_id: str,
+        requested_backend: str | None = BACKEND_AUTO,
+        selected_file: str | None = None,
+    ) -> ResolvedBackend:
+        """Load one local artifact on GPU, replacing any active model."""
 
         model_id = validate_model_id(model_id)
+        target = self.cache.resolve_cached(model_id, requested_backend, selected_file)
+        identity = (model_id, target.backend, target.selected_file)
+
         with self._lock:
-            snapshot_path = self.cache.ensure_transformers_checkpoint(model_id)
-            if self._model is not None and self._model_id == model_id:
-                return model_id
+            current = (self._model_id, self._backend, self._selected_file)
+            if current == identity and (self._model is not None or self._llama is not None):
+                return target
 
-            # Switching models always releases the old object before reading
-            # the new checkpoint, keeping the one-model invariant explicit.
+            # The one-model invariant is enforced before constructing either
+            # a new Transformers model or a new llama.cpp context.
             self.unload()
-            tokenizer = AutoTokenizer.from_pretrained(
-                str(snapshot_path),
-                local_files_only=True,
-                use_fast=True,
-                trust_remote_code=False,
-            )
-            model = AutoModelForCausalLM.from_pretrained(
-                str(snapshot_path),
-                local_files_only=True,
-                torch_dtype=torch.bfloat16,
-                low_cpu_mem_usage=True,
-                trust_remote_code=False,
+            if target.backend == BACKEND_LLAMACPP:
+                self._load_llama(target)
+            else:
+                self._load_transformers(target)
+
+            self._model_id = model_id
+            self._backend = target.backend
+            self._selected_file = target.selected_file
+            self._inspection = target.inspection
+            return target
+
+    @staticmethod
+    def _preferred_dtype(inspection: ModelInspection) -> Any:
+        value = str(
+            inspection.config.get("torch_dtype") or inspection.config.get("dtype") or ""
+        ).lower()
+        if "float16" in value or value in {"fp16", "half"}:
+            return torch.float16
+        if "float32" in value or value == "fp32":
+            return torch.float32
+        if "float8" in value or "fp8" in value:
+            return "auto"
+        return torch.bfloat16
+
+    @staticmethod
+    def _bnb_config(inspection: ModelInspection) -> Any | None:
+        """Create a BitsAndBytesConfig only for filename-only quant repos."""
+
+        quant_config = inspection.config.get("quantization_config")
+        if isinstance(quant_config, dict) and any(
+            key in quant_config
+            for key in ("load_in_4bit", "load_in_8bit", "_load_in_4bit", "_load_in_8bit")
+        ):
+            return None  # Transformers will consume the repository config itself.
+
+        if inspection.quantization_kind != "bitsandbytes":
+            return None
+        try:
+            from transformers import BitsAndBytesConfig
+
+            text = inspection.format_label.lower()
+            load_in_8bit = "8-bit" in text
+            return BitsAndBytesConfig(
+                load_in_4bit=not load_in_8bit,
+                load_in_8bit=load_in_8bit,
+                bnb_4bit_compute_dtype=torch.bfloat16,
+                bnb_4bit_quant_type="nf4",
+                bnb_4bit_use_double_quant=True,
             )
-            model = model.to("cuda").eval()
+        except Exception:
+            LOGGER.debug("Could not construct a BitsAndBytesConfig", exc_info=True)
+            return None
+
+    def _load_transformers(self, target: ResolvedBackend) -> None:
+        inspection = target.inspection
+        tokenizer = AutoTokenizer.from_pretrained(
+            str(target.path),
+            local_files_only=True,
+            use_fast=True,
+            trust_remote_code=False,
+        )
 
-            if tokenizer.pad_token_id is None and tokenizer.eos_token_id is not None:
-                tokenizer.pad_token = tokenizer.eos_token
-            if getattr(model.config, "pad_token_id", None) is None:
-                model.config.pad_token_id = tokenizer.pad_token_id
+        quantized = inspection.is_transformers_quantized
+        load_kwargs: dict[str, Any] = {
+            "local_files_only": True,
+            "low_cpu_mem_usage": True,
+            "trust_remote_code": False,
+            "dtype": "auto" if quantized else self._preferred_dtype(inspection),
+        }
+        if quantized:
+            # Quantized modules must be placed by Accelerate/Transformers and
+            # must not receive a later blanket `.to("cuda")` call.
+            load_kwargs["device_map"] = "cuda"
+            bnb_config = self._bnb_config(inspection)
+            if bnb_config is not None:
+                load_kwargs["quantization_config"] = bnb_config
 
-            self._tokenizer = tokenizer
-            self._model = model
-            self._model_id = model_id
-            return model_id
+        try:
+            model = AutoModelForCausalLM.from_pretrained(str(target.path), **load_kwargs)
+            if not quantized:
+                model = model.to("cuda")
+            model = model.eval()
+        except Exception as exc:
+            label = inspection.format_label
+            raise RuntimeError(
+                f"Could not load `{label}` with Transformers. "
+                "The repository's quantization runtime may need a compatible loader package. "
+                f"Details: {str(exc).splitlines()[0][:260]}"
+            ) from exc
+
+        if tokenizer.pad_token_id is None and tokenizer.eos_token_id is not None:
+            tokenizer.pad_token = tokenizer.eos_token
+        if getattr(model.config, "pad_token_id", None) is None:
+            model.config.pad_token_id = tokenizer.pad_token_id
+
+        self._tokenizer = tokenizer
+        self._model = model
+
+    def _load_llama(self, target: ResolvedBackend) -> None:
+        try:
+            from llama_cpp import Llama
+        except Exception as exc:
+            raise RuntimeError(
+                "llama.cpp is not available. The Space needs the CUDA-enabled llama-cpp-python wheel."
+            ) from exc
+
+        kwargs: dict[str, Any] = {
+            "model_path": str(target.path),
+            "n_gpu_layers": -1,
+            "n_ctx": 8192,
+            "n_batch": 512,
+            "n_threads": max(2, min(8, os.cpu_count() or 4)),
+            "verbose": False,
+        }
+        try:
+            llama = Llama(**kwargs, flash_attn=True)
+        except TypeError:
+            # Keep compatibility with older wheels that predate flash_attn in
+            # the Python constructor; the GPU offload remains explicit.
+            llama = Llama(**kwargs)
+        except Exception as exc:
+            raise RuntimeError(
+                f"Could not load `{target.selected_file}` with llama.cpp GPU offload. "
+                f"Details: {str(exc).splitlines()[0][:260]}"
+            ) from exc
+        self._llama = llama
 
     @staticmethod
     def _history_to_messages(history: list[Any] | None) -> list[dict[str, str]]:
@@ -224,13 +417,12 @@ class TransformersCausalLMRuntime:
 
     @staticmethod
     def _plain_prompt(messages: list[dict[str, str]]) -> str:
-        lines = [f"{message['role'].capitalize()}: {message['content']}" for message in messages]
+        lines = [f"{m['role'].capitalize()}: {m['content']}" for m in messages]
         return "\n".join(lines) + "\nAssistant:"
 
     def _tokenize(self, messages: list[dict[str, str]]) -> Any:
         assert self._tokenizer is not None
         tokenizer = self._tokenizer
-
         if hasattr(tokenizer, "apply_chat_template"):
             try:
                 return tokenizer.apply_chat_template(
@@ -252,12 +444,13 @@ class TransformersCausalLMRuntime:
                     LOGGER.debug("Chat template without return_dict failed", exc_info=True)
             except Exception:
                 LOGGER.debug("Chat template failed; using plain prompt", exc_info=True)
-
         return tokenizer(self._plain_prompt(messages), return_tensors="pt")
 
     def generate(
         self,
         model_id: str,
+        requested_backend: str | None,
+        selected_file: str | None,
         message: str,
         history: list[Any] | None,
         system_prompt: str,
@@ -265,19 +458,32 @@ class TransformersCausalLMRuntime:
         temperature: float,
         top_p: float,
     ) -> str:
-        """Generate one answer from the active cached Transformers model."""
+        target = self.ensure_loaded(model_id, requested_backend, selected_file)
+        messages: list[dict[str, str]] = []
+        if (system_prompt or "").strip():
+            messages.append({"role": "system", "content": system_prompt.strip()})
+        messages.extend(self._history_to_messages(history))
+        messages.append({"role": "user", "content": (message or "").strip()})
 
         with self._lock:
-            self.ensure_loaded(model_id)
-            assert self._model is not None
-            assert self._tokenizer is not None
-
-            messages: list[dict[str, str]] = []
-            if (system_prompt or "").strip():
-                messages.append({"role": "system", "content": system_prompt.strip()})
-            messages.extend(self._history_to_messages(history))
-            messages.append({"role": "user", "content": (message or "").strip()})
+            if target.backend == BACKEND_LLAMACPP:
+                if self._llama is None:
+                    raise RuntimeError("llama.cpp runtime is not loaded.")
+                result = self._llama.create_chat_completion(
+                    messages=messages,
+                    max_tokens=max_new_tokens,
+                    temperature=temperature,
+                    top_p=top_p,
+                )
+                answer = ""
+                if isinstance(result, dict) and result.get("choices"):
+                    answer = str(
+                        result["choices"][0].get("message", {}).get("content", "")
+                    )
+                return answer.strip() or "The model returned an empty response."
 
+            if self._model is None or self._tokenizer is None:
+                raise RuntimeError("Transformers runtime is not loaded.")
             encoded = self._tokenize(messages)
             encoded = {
                 key: value.to("cuda")
@@ -285,17 +491,15 @@ class TransformersCausalLMRuntime:
                 if torch.is_tensor(value)
             }
             input_length = int(encoded["input_ids"].shape[-1])
-
             generation_kwargs: dict[str, Any] = {
                 "max_new_tokens": max_new_tokens,
                 "do_sample": temperature > 0,
             }
             if temperature > 0:
                 generation_kwargs.update({"temperature": temperature, "top_p": top_p})
-
             with torch.inference_mode():
                 generated = self._model.generate(**encoded, **generation_kwargs)
-
             new_tokens = generated[0, input_length:]
             answer = self._tokenizer.decode(new_tokens, skip_special_tokens=True).strip()
             return answer or "The model returned an empty response."
+

requirements.txt

@@ -1,4 +1,9 @@
 transformers==4.57.6
-accelerate>=0.34.0
+accelerate>=1.1.0
 safetensors>=0.4.3
 sentencepiece>=0.2.0
+bitsandbytes>=0.50.0
+compressed-tensors>=0.18.0
+autoawq==0.2.9
+gptqmodel==2.2.0
+llama-cpp-python @ https://github.com/abetlen/llama-cpp-python/releases/download/v0.3.35-cu130/llama_cpp_python-0.3.35-py3-none-manylinux_2_35_x86_64.whl