First pass

llm.py

@@ -47,19 +47,31 @@ _llm = None
 
 
 def _preload_cuda_libs():
-    """Preload the nvidia-*-cu12 CUDA runtime libs RTLD_GLOBAL so the prebuilt llama.cpp
-    CUDA wheel can resolve libcudart.so.12 / libcublas.so.12 / etc. No-op if unavailable."""
+    """Load the CUDA runtime libs (libcudart.so.12, libcublas*, ...) RTLD_GLOBAL by full path
+    so the prebuilt llama.cpp CUDA wheel can resolve them. They ship in the nvidia-*-cu12 pip
+    packages and inside torch/lib, but neither is on the dynamic loader's search path. No-op
+    for anything not found. Order matters: cudart before cublasLt before cublas."""
     import ctypes
     import glob
+    import os
+    import site
 
+    dirs = []
     try:
-        import nvidia  # namespace package provided by the nvidia-*-cu12 wheels
+        import torch
+        dirs.append(os.path.join(os.path.dirname(torch.__file__), "lib"))
     except Exception:
-        return
-    # runtime + math libs before everything else
-    for sub in ("cuda_runtime", "cublas", "cuda_nvrtc", "cuda_cupti", "cudnn"):
-        for root in list(nvidia.__path__):
-            for lib in sorted(glob.glob(f"{root}/{sub}/lib/*.so*")):
+        pass
+    site_dirs = []
+    if hasattr(site, "getsitepackages"):
+        site_dirs += site.getsitepackages()
+    site_dirs.append(os.path.dirname(os.path.dirname(os.__file__)))  # fallback
+    for sp in dict.fromkeys(site_dirs):
+        dirs += glob.glob(os.path.join(sp, "nvidia", "*", "lib"))
+
+    for prefix in ("libcudart", "libnvrtc", "libcublasLt", "libcublas", "libcudnn"):
+        for d in dict.fromkeys(dirs):
+            for lib in sorted(glob.glob(os.path.join(d, prefix + "*.so*"))):
                 try:
                     ctypes.CDLL(lib, mode=ctypes.RTLD_GLOBAL)
                 except OSError:

requirements.txt

@@ -7,6 +7,12 @@ spaces
 transformers>=4.44.2
 accelerate
 torch
+# CUDA runtime libs the prebuilt cu124 llama-cpp-python wheel links against (libcudart.so.12,
+# libcublas.so.12, ...). Installed explicitly so the .so files definitely exist on disk; llm.py
+# then preloads them by full path. (libcudart.so.12 is ABI-compatible across all CUDA 12.x.)
+nvidia-cuda-runtime-cu12
+nvidia-cublas-cu12
+nvidia-cuda-nvrtc-cu12
 pillow
 hf-transfer
 huggingface-hub