Update app.py

app.py

@@ -3,7 +3,7 @@ import subprocess
 import time
 import requests
 import gradio as gr
-from huggingface_hub import snapshot_download
+from huggingface_hub import hf_hub_download
 
 MODEL_REPO = "bartowski/Qwen_Qwen3.5-9B-GGUF"
 MODEL_FILE = "Qwen3.5-9B-Q4_K_M.gguf"
@@ -12,54 +12,54 @@ LLAMA_DIR = "llama.cpp"
 SERVER_PORT = "8000"
 
 
-def install_build_tools():
-    subprocess.run(
-        ["apt-get", "update"],
-        stdout=subprocess.DEVNULL,
-        stderr=subprocess.DEVNULL
-    )
-
-    subprocess.run(
-        ["apt-get", "install", "-y", "build-essential", "cmake", "git"],
-        stdout=subprocess.DEVNULL,
-        stderr=subprocess.DEVNULL
-    )
+def install_dependencies():
+    subprocess.run(["apt-get", "update"])
+    subprocess.run([
+        "apt-get", "install", "-y",
+        "build-essential",
+        "cmake",
+        "git"
+    ])
 
 
 def download_model():
-    path = snapshot_download(
+    return hf_hub_download(
         repo_id=MODEL_REPO,
-        allow_patterns=[MODEL_FILE]
+        filename=MODEL_FILE,
+        repo_type="model"
     )
 
-    return os.path.join(path, MODEL_FILE)
-
 
 def setup_llama():
     if not os.path.exists(LLAMA_DIR):
-        subprocess.run(
-            ["git", "clone", "https://github.com/ggerganov/llama.cpp"],
-            check=True
-        )
-
-    subprocess.run(
-        ["make", "-j"],
-        cwd=LLAMA_DIR,
-        check=True
-    )
+        subprocess.run([
+            "git", "clone",
+            "https://github.com/ggml-org/llama.cpp"
+        ], check=True)
 
+    os.makedirs(f"{LLAMA_DIR}/build", exist_ok=True)
 
-def start_llama_server(model_path):
-    subprocess.Popen(
-        [
-            "./server",
-            "-m", model_path,
-            "--port", SERVER_PORT,
-            "--ctx-size", "4096",
-            "--api"
-        ],
-        cwd=LLAMA_DIR
-    )
+    subprocess.run([
+        "cmake",
+        "-B", "build"
+    ], cwd=LLAMA_DIR, check=True)
+
+    subprocess.run([
+        "cmake",
+        "--build",
+        "build",
+        "-j"
+    ], cwd=LLAMA_DIR, check=True)
+
+
+def start_server(model_path):
+    subprocess.Popen([
+        "./build/bin/llama-server",
+        "-m", model_path,
+        "--port", SERVER_PORT,
+        "--ctx-size", "4096",
+        "--api"
+    ], cwd=LLAMA_DIR)
 
 
 def wait_for_server():
@@ -89,22 +89,22 @@ def chat(prompt):
     return data["choices"][0]["message"]["content"]
 
 
-install_build_tools()
+install_dependencies()
 
 model_path = download_model()
 
 setup_llama()
 
-start_llama_server(model_path)
+start_server(model_path)
 
 wait_for_server()
 
 
 ui = gr.Interface(
     fn=chat,
-    inputs=gr.Textbox(lines=5, placeholder="Ask Operon something..."),
+    inputs=gr.Textbox(lines=5),
     outputs="text",
-    title="Operon Dev LLM (Qwen3.5-9B)",
+    title="Operon Dev LLM (Qwen3.5-9B)"
 )
 
 ui.launch(server_name="0.0.0.0", server_port=7860)