Preload nvidia CUDA libs (RTLD_GLOBAL) so libllama.so finds libcudart on ZeroGPU; eager model download; duration=120

app.py

@@ -76,26 +76,37 @@ impl Point {
 ]
 
 _llm = None
-_model_path = None
 
+_model_path = hf_hub_download(
+    repo_id=MODEL_REPO,
+    filename=MODEL_FILE,
+    token=os.environ.get("HF_TOKEN"),
+)
 
-def model_path():
-    global _model_path
-    if _model_path is None:
-        _model_path = hf_hub_download(
-            repo_id=MODEL_REPO,
-            filename=MODEL_FILE,
-            token=os.environ.get("HF_TOKEN"),
-        )
-    return _model_path
+
+def _preload_cuda():
+    import ctypes
+    import glob
+    import site
+
+    dirs = []
+    for sp in site.getsitepackages():
+        dirs += glob.glob(sp + "/nvidia/*/lib")
+    for d in sorted(dirs):
+        for so in sorted(glob.glob(d + "/*.so*")):
+            try:
+                ctypes.CDLL(so, mode=ctypes.RTLD_GLOBAL)
+            except OSError:
+                pass
 
 
 def get_llm():
     global _llm
     if _llm is None:
+        _preload_cuda()
         from llama_cpp import Llama
         _llm = Llama(
-            model_path=model_path(),
+            model_path=_model_path,
             n_gpu_layers=N_GPU_LAYERS,
             n_ctx=CTX,
             n_batch=512,
@@ -115,7 +126,7 @@ def clean(text: str) -> str:
     return text.rstrip()
 
 
-@gpu(duration=60)
+@gpu(duration=120)
 def complete(code, temperature, max_tokens, top_p):
     if code.strip() == "":
         return "", code, "*Empty input.*"