Switch to Gradio SDK -- ZeroGPU only works with Gradio SDK, not Docker/FastAPI

Dockerfile

@@ -1,18 +0,0 @@
-FROM nvidia/cuda:12.1.0-cudnn8-devel-ubuntu22.04
-
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get install -y python3 python3-pip git && rm -rf /var/lib/apt/lists/*
-
-WORKDIR /app
-COPY requirements.txt .
-
-# Build llama-cpp-python with CUDA support
-ENV CMAKE_ARGS="-DGGML_CUDA=on"
-RUN pip3 install --no-cache-dir llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121
-
-RUN pip3 install --no-cache-dir fastapi uvicorn huggingface_hub spaces
-
-COPY app.py .
-
-EXPOSE 7860
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -3,13 +3,18 @@ title: chatPDB API
 emoji: 🧬
 colorFrom: green
 colorTo: blue
-sdk: docker
-app_port: 7860
+sdk: gradio
+app_file: app.py
 pinned: false
 ---
 
 # chatPDB Inference API
 
-ZeroGPU-backed streaming inference endpoint for chatPDB 32B v1 (Q4_K_M GGUF).
+ZeroGPU-backed inference endpoint for chatPDB 32B v1 (Q4_K_M GGUF).
 
-Endpoint: `POST /generate` — returns `text/event-stream` of token chunks.
+Consumed by the Flask PTY app at [chatpdb.mdeller.com](https://chatpdb.mdeller.com).
+
+**Endpoint:** `POST /generate` — returns `text/event-stream` of token chunks.
+
+**Cold start:** first request after idle downloads the ~18.4 GB GGUF and allocates the ZeroGPU
+A10G (~60-120 s). This is a portfolio demo; availability is best-effort.

app.py

@@ -1,97 +1,111 @@
 """
-chatPDB inference API — HuggingFace Space (ZeroGPU)
+chatPDB inference API — HuggingFace Space (Gradio SDK, ZeroGPU)
 
-Serves a streaming /generate endpoint backed by llama-cpp-python.
-The GGUF is pulled from the Hub on first request and cached for the session.
+Exposes POST /generate (SSE) consumed by the Flask PTY app on the droplet.
+Gradio SDK is required for ZeroGPU (confirmed live 2026-07-23: requesting ZeroGPU hardware for a
+Docker-SDK Space returns "ZeroGPU Spaces only work with Gradio SDK") -- the Gradio UI itself is
+just a minimal landing page; a custom FastAPI route is mounted on Gradio's own underlying app to
+keep the exact same /generate contract chat_remote.py already expects.
+
+Cold-start note: first request after idle downloads the GGUF and allocates the GPU
+(~60-120 s). Subsequent requests within the same GPU lease are fast.
 """
 from __future__ import annotations
 
 import json
-import os
-from pathlib import Path
-from typing import Generator
 
+import gradio as gr
 import spaces
-from fastapi import FastAPI
+from fastapi import Request
 from fastapi.responses import StreamingResponse
 from huggingface_hub import hf_hub_download
 
-# ---------------------------------------------------------------------------
-# Model config
-# ---------------------------------------------------------------------------
-
 REPO_ID = "Dellboy/chatpdb_32b_v1-GGUF"
 FILENAME = "chatpdb_32b_v1_q4km.gguf"
-MODEL_PATH: Path | None = None   # set after first download
-
 N_CTX = 1536   # matches chatPDB's real training max_seq_length (config/train_config.yaml)
 N_GPU_LAYERS = -1  # offload all layers to GPU
 
-# ---------------------------------------------------------------------------
-# App
-# ---------------------------------------------------------------------------
-
-app = FastAPI(title="chatPDB API")
-
-
-def _get_model():
-    """Download GGUF on first call, return cached Llama instance."""
-    global MODEL_PATH
-    from llama_cpp import Llama
+_model_path: str | None = None
 
-    if MODEL_PATH is None:
-        MODEL_PATH = Path(hf_hub_download(repo_id=REPO_ID, filename=FILENAME))
 
-    return Llama(
-        model_path=str(MODEL_PATH),
-        n_ctx=N_CTX,
-        n_gpu_layers=N_GPU_LAYERS,
-        verbose=False,
-    )
+def _get_model_path() -> str:
+    global _model_path
+    if _model_path is None:
+        _model_path = hf_hub_download(repo_id=REPO_ID, filename=FILENAME)
+    return _model_path
 
 
-@spaces.GPU
+@spaces.GPU(duration=180)
 def _generate_tokens(
     prompt: str,
     max_tokens: int,
     temperature: float,
     repeat_penalty: float,
-) -> Generator[str, None, None]:
-    """Run inference inside the ZeroGPU lease."""
-    llm = _get_model()
-    stream = llm(
+) -> list[str]:
+    """Run inside the ZeroGPU lease; collect all tokens and return.
+
+    ZeroGPU-decorated functions run in a bounded lease and can't hold a live generator open
+    across the function boundary, so tokens are collected here and streamed out afterward by the
+    /generate route below -- the client sees compute-then-deliver rather than true live
+    token-by-token latency, a real trade-off of the ZeroGPU model.
+    """
+    from llama_cpp import Llama
+
+    llm = Llama(
+        model_path=_get_model_path(),
+        n_ctx=N_CTX,
+        n_gpu_layers=N_GPU_LAYERS,
+        verbose=False,
+    )
+    tokens: list[str] = []
+    for chunk in llm(
         prompt,
         max_tokens=max_tokens,
         temperature=temperature,
         repeat_penalty=repeat_penalty,
         stream=True,
+    ):
+        tok = chunk["choices"][0]["text"]
+        if tok:
+            tokens.append(tok)
+    return tokens
+
+
+# -- Gradio UI (minimal -- required for Gradio SDK / ZeroGPU) --
+
+with gr.Blocks(title="chatPDB API") as demo:
+    gr.Markdown(
+        "## 🧬 chatPDB Inference API\n\n"
+        "Internal endpoint for [chatpdb.mdeller.com](https://chatpdb.mdeller.com). "
+        "Use `POST /generate` — returns `text/event-stream` of token chunks.\n\n"
+        "**Cold start:** first request after idle takes ~60-120 s (GGUF download + GPU alloc)."
     )
-    for chunk in stream:
-        token = chunk["choices"][0]["text"]
-        if token:
-            yield token
 
 
-@app.post("/generate")
-async def generate(request: dict):
-    """
-    POST /generate
-    Body: { "prompt": "...", "max_tokens": 512, "temperature": 0.15, "repeat_penalty": 1.15 }
-    Returns: text/event-stream of token strings
-    """
-    prompt = request.get("prompt", "")
-    max_tokens = int(request.get("max_tokens", 512))
-    temperature = float(request.get("temperature", 0.15))
-    repeat_penalty = float(request.get("repeat_penalty", 1.15))
+# -- Custom FastAPI route mounted on Gradio's app --
+
+@demo.app.post("/generate")
+async def generate(request: Request):
+    body = await request.json()
+    prompt = body.get("prompt", "")
+    max_tokens = int(body.get("max_tokens", 512))
+    temperature = float(body.get("temperature", 0.15))
+    repeat_penalty = float(body.get("repeat_penalty", 1.15))
+
+    tokens = _generate_tokens(prompt, max_tokens, temperature, repeat_penalty)
 
     def event_stream():
-        for token in _generate_tokens(prompt, max_tokens, temperature, repeat_penalty):
-            yield f"data: {json.dumps({'token': token})}\n\n"
+        for tok in tokens:
+            yield f"data: {json.dumps({'token': tok})}\n\n"
         yield "data: [DONE]\n\n"
 
     return StreamingResponse(event_stream(), media_type="text/event-stream")
 
 
-@app.get("/health")
-def health():
+@demo.app.get("/health")
+async def health():
     return {"status": "ok"}
+
+
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -1,5 +1,5 @@
-fastapi
-uvicorn
-llama-cpp-python[cuda]
-huggingface_hub
+--extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121
+llama-cpp-python
+gradio>=4.0
 spaces
+huggingface_hub