Fix ZeroGPU: preload CUDA runtime libs for the llama.cpp wheel

The cu124 llama-cpp-python wheel is dynamically linked against
libcudart/libcublas, which are not on the loader path in the HF Spaces
image (OSError: libcudart.so.12). Ship them as nvidia-* pip wheels and
dlopen them with RTLD_GLOBAL (cudart first) before importing llama_cpp
so libllama.so resolves its symbols.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

requirements.txt

@@ -1,6 +1,9 @@
 # llama.cpp with CUDA wheels for Hugging Face ZeroGPU (CUDA 12.4).
 --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu124
 llama-cpp-python
+# CUDA runtime libs the wheel is linked against (preloaded in id/llm/local.py).
+nvidia-cuda-runtime-cu12>=12.4,<12.5
+nvidia-cublas-cu12>=12.4,<12.5
 spaces
 huggingface_hub>=0.24
 

src/id/llm/local.py

@@ -38,12 +38,50 @@ except Exception:  # pragma: no cover - exercised only off ZeroGPU
         return fn
 
 
+def _preload_cuda() -> None:
+    """Make the CUDA runtime resolvable for llama.cpp's CUDA-linked .so.
+
+    The prebuilt ``cu124`` ``llama-cpp-python`` wheel is dynamically linked
+    against ``libcudart``/``libcublas``, which are not on the loader path in the
+    HF Spaces image. We ship them as ``nvidia-*`` pip wheels and ``dlopen`` them
+    with ``RTLD_GLOBAL`` (cudart first) so ``libllama.so`` finds the symbols.
+    """
+    if N_GPU_LAYERS == 0:
+        return
+    import ctypes
+    import glob
+    import site
+
+    roots: set[str] = set()
+    getsp = getattr(site, "getsitepackages", None)
+    if getsp:
+        roots.update(getsp())
+    import sys
+
+    roots.update(p for p in sys.path if p.endswith("site-packages"))
+    # Load order matters: cudart -> cublasLt -> cublas.
+    for pattern in (
+        "nvidia/cuda_runtime/lib/libcudart.so*",
+        "nvidia/cublas/lib/libcublasLt.so*",
+        "nvidia/cublas/lib/libcublas.so*",
+    ):
+        for root in roots:
+            hits = glob.glob(os.path.join(root, pattern))
+            if hits:
+                try:
+                    ctypes.CDLL(hits[0], mode=ctypes.RTLD_GLOBAL)
+                except OSError:
+                    pass
+                break
+
+
 def _load_model() -> Any:
     """Build (once) the llama.cpp model from the cached GGUF."""
     global _LLM
     if _LLM is None:
         with _LOCK:
             if _LLM is None:
+                _preload_cuda()
                 from llama_cpp import Llama
 
                 _LLM = Llama.from_pretrained(