migrate to llamacpp server

app.py

@@ -1,18 +1,15 @@
-import re
+import socket
+import subprocess
 import gradio as gr
-from llama_cpp import Llama
+from openai import OpenAI
 
-# model = "NousResearch/Hermes-3-Llama-3.2-3B-GGUF"
-model = "lmstudio-community/Phi-4-mini-reasoning-GGUF"
-llm = Llama.from_pretrained(
-    repo_id=model,
-    filename="*Q6_K.gguf",
-    verbose=True,
-    use_mmap=True,
-    use_mlock=True,
-    n_threads=4,
-    n_threads_batch=4,
-    n_ctx=8000,
+
+subprocess.Popen("bash /home/user/app/start.sh", shell=True)
+
+client = OpenAI(
+    base_url="http://0.0.0.0:8000/v1",
+    api_key="sk-local",
+    timeout=600
 )
 
 
@@ -24,57 +21,45 @@ def respond(
     temperature,
     top_p,
 ):
-    if len(system_message) > 0:
-        messages = [{"role": "system", "content": system_message}]
-    else:
-        messages = []
-        
-
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
+    messages = [{"role": "system", "content": system_message}]
+    
+    for user, assistant in history:
+        if user:
+            messages.append({"role": "user", "content": user})
+        if assistant:
+            messages.append({"role": "assistant", "content": assistant})
 
     messages.append({"role": "user", "content": message})
 
-    response = ""
-    completion = llm.create_chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p
-    )
+    try:
+        stream = client.chat.completions.create(
+            model="qwen3",  # ⚠️ Replace it with the name of the model loaded by your llama.cpp
+            messages=messages,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            top_p=top_p,
+            stream=True,
+        )
 
-    for message in completion:
-        delta = message['choices'][0]['delta']
-        
-        if 'content' in delta:
-            response += delta['content']
-            yield response.replace("<think>", "*").replace("</think>", "*")
+        output = ""
+        for chunk in stream:
+            delta = chunk.choices[0].delta.content or ""
+            output += delta
+            yield output
 
+    except Exception as e:
+        print(f"[Error] {e}")
+        yield "⚠️ Llama.cpp server error"
 
 demo = gr.ChatInterface(
     respond,
     additional_inputs=[
-        gr.Textbox(
-            value="",
-            label="System message",
-        ),
-        gr.Slider(minimum=200, maximum=100000, value=4000, step=100, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.6, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
+        gr.Textbox(value="You are a friendly assistant.", label="System message"),
+        gr.Slider(minimum=1, maximum=2048, value=4096, step=1, label="Max new tokens"),
+        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
+        gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)"),
     ],
-    description=model,
 )
 
-
 if __name__ == "__main__":
     demo.launch()