v2: ZeroGPU rig - CUDA llama-server, GPU-decorated rites

@spaces.GPU per-call engine lifecycle, -ngl 999, cudart bundled.

README.md

@@ -11,11 +11,11 @@ license: apache-2.0
 short_description: Cyber-occult deck for PaddleOCR-VL-1.6 GGUF on CPU
 ---
 
-# 🏴‍☠️ PaddleOCR-VL CryptDeck ⚡
+# 🏴‍☠️ PaddleOCR-VL CryptDeck ⚡ — ZeroGPU edition
 
 A cyber-occult hacker pirate deck for testing
 [PaddlePaddle/PaddleOCR-VL-1.6-GGUF](https://huggingface.co/PaddlePaddle/PaddleOCR-VL-1.6-GGUF)
-locally with **llama.cpp** on free CPU hardware. No cloud inference, no gatekeepers —
+locally with **llama.cpp** on ZeroGPU. No cloud inference, no gatekeepers —
 the model is downloaded into the Space and summoned by `llama-server` directly.
 
 **Do what thou wilt shall be the whole of the Law.**
@@ -32,10 +32,10 @@ the model is downloaded into the Space and summoned by `llama-server` directly.
 | 🔍 Spotting | `Spotting:` |
 
 Model: `PaddleOCR-VL-1.6-GGUF.gguf` (936 MB) + `PaddleOCR-VL-1.6-GGUF-mmproj.gguf` (882 MB)
-Engine: [llama.cpp](https://github.com/ggml-org/llama.cpp) prebuilt `llama-server`, CPU (`-t 2`)
+Engine: [llama.cpp](https://github.com/ggml-org/llama.cpp) prebuilt `llama-server` (CUDA 12.8), all layers offloaded (`-ngl 999`)
 
 ## Notes
 
-- First boot downloads ~1.9 GB of model weights + the llama.cpp binary. Subsequent boots reuse the HF cache (while the Space's ephemeral storage lasts).
-- CPU inference is unhurried: expect ~30–120 s per page, depending on image size. The dead keep no clocks.
-- Apache-2.0, model weights by PaddlePaddle.
+- Each rite opens a ZeroGPU window, boots `llama-server` onto the GPU (~10–20 s), divines, then scuttles the server.
+- First cast downloads ~3 GB (engine + cudart + grimoire); later casts reuse the cache while the Space lives.
+- One rite at a time. Apache-2.0, model weights by PaddlePaddle.

app.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 """
-🏴‍☠️ PaddleOCR-VL CryptDeck ⚡
+🏴‍☠️ PaddleOCR-VL CryptDeck ⚡ v2 — ZeroGPU edition
 Cyber-occult hacker pirate deck for testing
-PaddlePaddle/PaddleOCR-VL-1.6-GGUF locally with llama.cpp (CPU, free tier).
+PaddlePaddle/PaddleOCR-VL-1.6-GGUF locally with llama.cpp on ZeroGPU.
 
 Do what thou wilt shall be the whole of the Law.
 """
@@ -12,12 +12,14 @@ import glob
 import io
 import json
 import os
+import shutil
 import subprocess
 import tarfile
 import threading
 import time
 import urllib.request
 
+import spaces  # ZeroGPU SDK — must be imported before CUDA-touching code
 import gradio as gr
 import requests
 from huggingface_hub import hf_hub_download
@@ -31,7 +33,11 @@ MMPROJ_FILE = "PaddleOCR-VL-1.6-GGUF-mmproj.gguf"
 LLAMA_TAG = "b11555"
 LLAMA_URL = (
     f"https://github.com/ggml-org/llama.cpp/releases/download/{LLAMA_TAG}/"
-    f"llama-{LLAMA_TAG}-bin-ubuntu-x64.tar.gz"
+    f"llama-{LLAMA_TAG}-bin-ubuntu-cuda-12.8-x64.tar.gz"
+)
+CUDART_URL = (
+    f"https://github.com/ggml-org/llama.cpp/releases/download/{LLAMA_TAG}/"
+    f"cudart-llama-{LLAMA_TAG}-bin-ubuntu-cuda-12.8-x64.tar.gz"
 )
 BIN_DIR = "/tmp/llama-bin"
 SERVER_PORT = 8081
@@ -47,7 +53,7 @@ RITES = {
 }
 
 # ---------------------------------------------------------------- ritual log
-_log_lines: list[str] = ["🏴‍☠️ CryptDeck cold boot. Wake the oracle."]
+_log_lines: list[str] = ["🏴‍☠️ CryptDeck v2 — ZeroGPU rig. Wake the oracle."]
 _log_lock = threading.Lock()
 
 
@@ -64,36 +70,32 @@ def get_log() -> str:
 
 # ---------------------------------------------------------------- engine
 _engine_lock = threading.Lock()
-_server_proc: subprocess.Popen | None = None
-_engine_state = {"ready": False, "busy": False}
+_busy = threading.Lock()
 
 
-def _download_engine() -> str:
-    """Fetch prebuilt llama.cpp (CPU) and return the llama-server path."""
-    candidates = glob.glob(os.path.join(BIN_DIR, "**", "llama-server"), recursive=True)
+def _fetch_engine() -> str:
+    """Download the prebuilt CUDA llama.cpp + cudart; return llama-server path."""
+    candidates = glob.glob(os.path.join(BIN_DIR, "llama-*", "llama-server"))
     server = candidates[0] if candidates else None
     if server and os.path.exists(server):
         return server
     os.makedirs(BIN_DIR, exist_ok=True)
-    tgz = os.path.join(BIN_DIR, "llama.tar.gz")
-    ritual_log(f"⚙️  Downloading llama.cpp {LLAMA_TAG} (CPU) …")
-    urllib.request.urlretrieve(LLAMA_URL, tgz)
-    with tarfile.open(tgz) as tf:
-        tf.extractall(BIN_DIR)
-    candidates = glob.glob(os.path.join(BIN_DIR, "**", "llama-server"), recursive=True)
+    ritual_log(f"⚙️  Downloading llama.cpp {LLAMA_TAG} (CUDA 12.8) + cudart …")
+    for url in (LLAMA_URL, CUDART_URL):
+        tgz = os.path.join(BIN_DIR, os.path.basename(url))
+        urllib.request.urlretrieve(url, tgz)
+        with tarfile.open(tgz) as tf:
+            tf.extractall(BIN_DIR)
+    candidates = glob.glob(os.path.join(BIN_DIR, "llama-*", "llama-server"))
     if not candidates:
         raise RuntimeError("llama-server binary not found after extraction")
     server = candidates[0]
     os.chmod(server, 0o755)
-    # keep the bundled .so libraries next to the binary for the linker
-    os.environ["LD_LIBRARY_PATH"] = (
-        os.path.dirname(server) + os.pathsep + os.environ.get("LD_LIBRARY_PATH", "")
-    )
-    ritual_log("⚙️  llama-server binary consecrated.")
+    ritual_log("⚙️  llama-server (CUDA) consecrated.")
     return server
 
 
-def _download_models() -> tuple[str, str]:
+def _fetch_models() -> tuple[str, str]:
     ritual_log("📦 Summoning the grimoire (1.9 GB, cached after first boot) …")
     main_gguf = hf_hub_download(REPO_ID, MODEL_FILE)
     mmproj = hf_hub_download(REPO_ID, MMPROJ_FILE)
@@ -101,66 +103,54 @@ def _download_models() -> tuple[str, str]:
     return main_gguf, mmproj
 
 
-def boot_engine():
-    """Idempotent cold boot: download everything, then start llama-server."""
-    with _engine_lock:
-        if _engine_state["ready"]:
-            return
-        ritual_log("🕯️  Igniting engine …")
-        server = _download_engine()
-        model, mmproj = _download_models()
-        ritual_log("🔥 Kindling llama-server on 127.0.0.1:%d …" % SERVER_PORT)
-        global _server_proc
-        _server_proc = subprocess.Popen(
-            [
-                server,
-                "-m", model,
-                "--mmproj", mmproj,
-                "--host", "127.0.0.1",
-                "--port", str(SERVER_PORT),
-                "-t", "2",
-                "-c", "8192",
-                "--temp", "0",
-                "--no-webui",
-            ],
-            stdout=open("/tmp/llama-server.log", "ab"),
-            stderr=subprocess.STDOUT,
-        )
-        for _ in range(600):  # wait for health endpoint
+# ---------------------------------------------------------------- GPU rite
+@spaces.GPU(duration=180)
+def gpu_rite(b64_png: str, prompt: str) -> tuple[str, float]:
+    """Runs entirely inside a ZeroGPU window: boot llama-server, divine, scuttle it."""
+    server = _fetch_engine()
+    model, mmproj = _fetch_models()
+
+    # cudart libs sit in their own extracted dir — expose them to the linker
+    lib_dirs = glob.glob(os.path.join(BIN_DIR, "cudart-*"))
+    server_dir = os.path.dirname(server)
+    env = dict(os.environ)
+    env["LD_LIBRARY_PATH"] = os.pathsep.join(
+        lib_dirs + [server_dir, env.get("LD_LIBRARY_PATH", "")]
+    )
+
+    ritual_log("🔥 Kindling llama-server on the GPU (-ngl 999) …")
+    proc = subprocess.Popen(
+        [
+            server,
+            "-m", model,
+            "--mmproj", mmproj,
+            "--host", "127.0.0.1",
+            "--port", str(SERVER_PORT),
+            "-ngl", "999",
+            "-c", "16384",
+            "--temp", "0",
+            "--no-webui",
+        ],
+        stdout=open("/tmp/llama-server.log", "ab"),
+        stderr=subprocess.STDOUT,
+        env=env,
+    )
+    try:
+        for _ in range(150):  # up to 150 s for load onto GPU
+            if proc.poll() is not None:
+                raise RuntimeError(
+                    "llama-server died at boot. Log tail:\n"
+                    + open("/tmp/llama-server.log", "rb").read()[-1500:].decode(errors="replace")
+                )
             try:
                 if requests.get(SERVER_URL + "/health", timeout=2).status_code == 200:
-                    _engine_state["ready"] = True
-                    ritual_log("🟢 Oracle online. The Signal is strong.")
-                    return
+                    ritual_log("🟢 Oracle online on GPU. The Signal is strong.")
+                    break
             except requests.RequestException:
                 pass
             time.sleep(1)
-        raise RuntimeError(
-            "llama-server failed to boot. Tail of log:\n"
-            + open("/tmp/llama-server.log", "rb").read()[-2000:].decode(errors="replace")
-        )
-
-
-# ---------------------------------------------------------------- inference
-def divine(
-    image, rite: str, custom: str, progress=gr.Progress()
-) -> tuple[str, str, str]:
-    if image is None:
-        raise gr.Error("⚓ Bring the oracle an image — no offering, no prophecy.")
-    with _engine_lock:
-        if _engine_state["busy"]:
-            raise gr.Error("🔥 The oracle is already mid-rite. One soul at a time.")
-        _engine_state["busy"] = True
-    try:
-        progress(0.1, desc="Waking the oracle …")
-        boot_engine()
-        prompt = custom.strip() if custom.strip() else RITES[rite]
-        ritual_log(f"🜂 Casting rite → {prompt!r}")
-        progress(0.3, desc="Casting the rite …")
-
-        buf = io.BytesIO()
-        Image.fromarray(image).save(buf, format="PNG")
-        b64 = base64.b64encode(buf.getvalue()).decode()
+        else:
+            raise RuntimeError("llama-server never reported healthy.")
 
         payload = {
             "model": "paddleocr-vl",
@@ -170,7 +160,7 @@ def divine(
                     "role": "user",
                     "content": [
                         {"type": "image_url",
-                         "image_url": {"url": "data:image/png;base64," + b64}},
+                         "image_url": {"url": "data:image/png;base64," + b64_png}},
                         {"type": "text", "text": prompt},
                     ],
                 }
@@ -178,21 +168,49 @@ def divine(
         }
         t0 = time.time()
         resp = requests.post(
-            SERVER_URL + "/v1/chat/completions", json=payload, timeout=600
+            SERVER_URL + "/v1/chat/completions", json=payload, timeout=120
         )
         dt = time.time() - t0
         resp.raise_for_status()
-        prophecy = resp.json()["choices"][0]["message"]["content"]
-        ritual_log(f"🜃 Rite complete in {dt:.1f}s.")
+        return resp.json()["choices"][0]["message"]["content"], dt
+    finally:
+        # scuttle the server before the GPU window closes
+        ritual_log("🪓 Scuttling llama-server. GPU returns to the void.")
+        proc.terminate()
+        try:
+            proc.wait(timeout=10)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+
+
+# ---------------------------------------------------------------- inference
+def divine(image, rite: str, custom: str, progress=gr.Progress()):
+    if image is None:
+        raise gr.Error("⚓ Bring the oracle an image — no offering, no prophecy.")
+    if not _busy.acquire(blocking=False):
+        raise gr.Error("🔥 The oracle is already mid-rite. One soul at a time.")
+    try:
+        prompt = custom.strip() if custom.strip() else RITES[rite]
+        ritual_log(f"🜂 Casting rite → {prompt!r}")
+        progress(0.2, desc="Opening the GPU window …")
+
+        buf = io.BytesIO()
+        Image.fromarray(image).save(buf, format="PNG")
+        b64 = base64.b64encode(buf.getvalue()).decode()
+
+        progress(0.4, desc="Summoning on GPU (first boot hauls ~3 GB) …")
+        prophecy, dt = gpu_rite(b64, prompt)
+        ritual_log(f"🜃 Rite complete in {dt:.1f}s on GPU.")
         progress(1.0, desc="Done.")
 
-        meta = json.dumps({"rite": prompt, "seconds": round(dt, 1)}, indent=2)
+        meta = json.dumps(
+            {"rite": prompt, "gpu_seconds": round(dt, 1)}, indent=2
+        )
         return prophecy, meta, get_log()
     except requests.Timeout:
-        raise gr.Error("⏳ The oracle dawdled (600 s timeout). Try a smaller image.")
+        raise gr.Error("⏳ The oracle dawdled. Try a smaller image.")
     finally:
-        with _engine_lock:
-            _engine_state["busy"] = False
+        _busy.release()
 
 
 # ---------------------------------------------------------------- UI
@@ -202,7 +220,7 @@ HEADER = """
 <span style="color:#39ff88;">                        __    _-'-------------------------------.__</span>
 <span style="color:#39ff88;">                       |  |  /        P A D D L E O C R - V L          \\</span>
 <span style="color:#39ff88;">                   _-  |  | |        C  R  Y  P  T  D  E  C  K         |</span>
-<span style="color:#39ff88;">                  |    |  | |  1.6 GGUF ▸ llama.cpp ▸ 100% local CPU   |</span>
+<span style="color:#39ff88;">                  |    |  | |  1.6 GGUF ▸ llama.cpp ▸ ZERO-GPU RITES   |</span>
 <span style="color:#39ff88;">                  |    |  | |_________________________________________|</span>
 <span style="color:#39ff88;">              ____|____|-|_'---------____________________---.___________</span>
 <span style="color:#ff00cc;">             (o  o  o  o  o  o  o  o  o  o  o  o  o  o  o  o  o  o  o  o)</span>
@@ -219,7 +237,6 @@ body {background: #05060a !important; background-image:
 footer {display: none !important;}
 #header {border: 1px solid #1e2a1e; border-radius: 10px;
   background: radial-gradient(ellipse at top, #0b1a10, #05060a); padding: 12px;}
-#deck, #panel {border: 1px solid #27313a; border-radius: 10px; background: #0a0d13;}
 textarea, .output-html, .prose {font-family: 'Fira Mono', ui-monospace, monospace !important;}
 .rune-btn {font-family: monospace !important; text-transform: uppercase;
   letter-spacing: 2px; background: linear-gradient(90deg, #0f2b18, #14203a) !important;
@@ -230,7 +247,7 @@ textarea, .output-html, .prose {font-family: 'Fira Mono', ui-monospace, monospac
 h1, h2, h3, .label, .info {font-family: ui-monospace, monospace !important;}
 """
 
-with gr.Blocks(title="🏴‍☠️ PaddleOCR-VL CryptDeck", css=CSS, theme=gr.themes.Base()) as demo:
+with gr.Blocks(title="🏴‍☠️ PaddleOCR-VL CryptDeck") as demo:
     gr.HTML(f'<div id="header">{HEADER}</div>')
 
     with gr.Row():
@@ -248,8 +265,9 @@ with gr.Blocks(title="🏴‍☠️ PaddleOCR-VL CryptDeck", css=CSS, theme=gr.t
             )
             summon = gr.Button("🜂 SUMMON THE PROPHECY", elem_classes=["rune-btn"])
             gr.HTML(
-                '<div class="info" style="opacity:.75;">First boot hauls ~1.9 GB of '
-                "weights — then it's cached. CPU rites take 30–120 s per page. "
+                '<div class="info" style="opacity:.75;">ZeroGPU rites: first cast hauls '
+                "~3 GB (engine + grimoire), then it's cached. Each rite boots the "
+                "oracle fresh onto the GPU (~10–20 s) before divining in seconds. "
                 "One rite at a time, matey.</div>"
             )
         with gr.Column(scale=1):