Real fix: preload nvidia-*-cu12 pip packages' shared libraries via ctypes before llama_cpp import (libcudart.so.12 not found on ZeroGPU worker, confirmed via real runtime logs)

app.py

@@ -61,7 +61,39 @@ def _generate_tokens(
     across the function boundary, so tokens are collected here rather than streamed token-by-token
     -- the client sees compute-then-deliver rather than true live latency, a real trade-off of the
     ZeroGPU model.
+
+    Real fix (found + fixed 2026-07-23): the CUDA-built llama-cpp-python wheel failed to load on
+    the real ZeroGPU worker with `OSError: libcudart.so.12: cannot open shared object file` --
+    confirmed via the Space's actual runtime logs (fetch_space_logs), not guessed. The
+    `nvidia-cuda-runtime-cu12`/`nvidia-cublas-cu12` pip packages bundle the needed .so files (real
+    fix pattern confirmed via github.com/abetlen/llama-cpp-python#1460), but llama-cpp-python loads
+    its library via a raw `ctypes.CDLL()` call that only searches the system's standard library
+    path / LD_LIBRARY_PATH -- it doesn't know to look inside a pip package's install directory
+    (unlike PyTorch's own wheels, which have special-cased loader logic for exactly this). The real
+    subpath is `<package>.__path__[0]/lib/*.so*` (a `lib/` subdirectory, not a Python submodule).
+    Pre-loading every nvidia-*-cu12 package's .so files explicitly via ctypes here, before
+    llama_cpp's own import triggers its CDLL() call, makes it resolve to the already-loaded
+    libraries instead of searching (and failing) on its own -- done generically across every
+    installed `nvidia.*` package rather than hardcoding exact library names, since llama.cpp's
+    CUDA backend may need more than just cudart+cublas depending on the exact build.
     """
+    import ctypes
+    import glob
+    import importlib
+    import pkgutil
+
+    try:
+        import nvidia
+        for _finder, _pkg_name, _ in pkgutil.iter_modules(nvidia.__path__):
+            try:
+                _pkg = importlib.import_module(f"nvidia.{_pkg_name}")
+                for _so in glob.glob(f"{_pkg.__path__[0]}/lib/*.so*"):
+                    ctypes.CDLL(_so, mode=ctypes.RTLD_GLOBAL)
+            except Exception:
+                continue
+    except Exception:
+        pass  # fall through -- if this fails, llama_cpp's own import error will surface as before
+
     from llama_cpp import Llama
 
     llm = Llama(

requirements.txt

@@ -1,5 +1,7 @@
 --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121
 llama-cpp-python
+nvidia-cuda-runtime-cu12
+nvidia-cublas-cu12
 gradio>=4.0
 spaces
 huggingface_hub