Fix llama.cpp CUDA runtime loading

app.py

@@ -12,6 +12,38 @@ MMPROJ_PATH = os.path.join("/tmp", MMPROJ_FILE)
 
 server_process = None
 
+LLAMA_DIR = "/tmp/llama"
+LLAMA_SERVER = os.path.join(LLAMA_DIR, "llama-server")
+LLAMA_RELEASE = "b10182"
+LLAMA_ARCHIVE = f"llama.cpp-b{LLAMA_RELEASE}-cuda-12.8-amd64.tar.gz"
+LLAMA_URL = f"https://github.com/ai-dock/llama.cpp-cuda/releases/download/b{LLAMA_RELEASE}/{LLAMA_ARCHIVE}"
+
+
+def download_llama():
+    if os.path.exists(LLAMA_SERVER):
+        return
+
+    os.makedirs(LLAMA_DIR, exist_ok=True)
+
+    archive = os.path.join("/tmp", LLAMA_ARCHIVE)
+
+    subprocess.run(
+        ["wget", "-q", "--show-progress", "-O", archive, LLAMA_URL],
+        check=True,
+    )
+
+    subprocess.run(
+        ["tar", "-xzf", archive, "-C", LLAMA_DIR, "--strip-components=1"],
+        check=True,
+    )
+
+    if not os.path.exists(LLAMA_SERVER):
+        raise RuntimeError(
+            f"llama-server was not found after extraction: {LLAMA_SERVER}"
+        )
+
+    os.chmod(LLAMA_SERVER, 0o755)
+
 
 def download_models():
     from huggingface_hub import hf_hub_download
@@ -35,11 +67,12 @@ def download_models():
 def generate(prompt, max_tokens, temperature):
     global server_process
 
+    download_llama()
     download_models()
 
     if server_process is None or server_process.poll() is not None:
         cmd = [
-            "llama-server",
+            LLAMA_SERVER,
             "-m", MODEL_PATH,
             "--mmproj", MMPROJ_PATH,
             "--host", "127.0.0.1",
@@ -50,11 +83,19 @@ def generate(prompt, max_tokens, temperature):
             "-t", "2",
         ]
 
+        server_env = os.environ.copy()
+        server_env["LD_LIBRARY_PATH"] = (
+            LLAMA_DIR
+            + ":"
+            + server_env.get("LD_LIBRARY_PATH", "")
+        )
+
         server_process = subprocess.Popen(
             cmd,
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
             text=True,
+            env=server_env,
         )
 
     import time