Update app.py

app.py

@@ -1,10 +1,6 @@
 import socket
 import gradio as gr
-import requests
-import json
-import time
-import threading
-import queue
+from openai import OpenAI
 
 def get_local_ip():
     s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
@@ -19,146 +15,90 @@ def get_local_ip():
 
 print("本機 IP:", get_local_ip())
 
-def llama_http_stream_worker(message, history, system_message, max_tokens, temperature, top_p, output_queue):
-    """直接使用HTTP請求到llama.cpp，完全繞過OpenAI library"""
+# ✅ 設定 base URL 連接本地 llama.cpp API
+client = OpenAI(
+    base_url="http://0.0.0.0:8000/v1",
+    api_key="sk-local",  # llama.cpp 不檢查內容，只要有就行
+    timeout=1200  # 增加 OpenAI 客戶端超時時間
+)
+
+# ✅ 回應函式 (流式 generator)
+def respond(message, history, system_message, max_tokens, temperature, top_p):
+    # history 是 list of dict: [{"role": "user"/"assistant", "content": "..."}]
+    messages = [{"role": "system", "content": system_message}]
+    messages.extend(history)  # 直接加入舊對話
+    messages.append({"role": "user", "content": message})
+
     try:
-        url = "http://0.0.0.0:8000/v1/chat/completions"
-        headers = {
-            "Content-Type": "application/json",
-            "Authorization": "Bearer sk-local"
-        }
-        
-        messages = [{"role": "system", "content": system_message}]
-        messages.extend(history)
-        messages.append({"role": "user", "content": message})
+        # 先立即返回一個等待消息，保持連接活躍
+        yield {"role": "assistant", "content": "⏳ 正在處理您的請求，這可能需要較長時間..."}
         
-        payload = {
-            "model": "qwen3",
-            "messages": messages,
-            "max_tokens": max_tokens,
-            "temperature": temperature,
-            "top_p": top_p,
-            "stream": True
-        }
-        
-        print(f"[Request] Sending request to llama.cpp...")
-        
-        # 使用非常長的超時時間，並禁用連接超時
-        response = requests.post(
-            url, 
-            json=payload, 
-            headers=headers, 
+        stream = client.chat.completions.create(
+            model="qwen3",   # ⚠️ 替換成你 llama.cpp 載入的模型 general.name
+            messages=messages,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            top_p=top_p,
             stream=True,
-            timeout=(60, 3600)  # 連接超時60秒，讀取超時3600秒（1小時）
         )
-        
-        if response.status_code == 200:
-            output = ""
-            for line in response.iter_lines(decode_unicode=True, chunk_size=1):
-                if line and line.startswith('data: '):
-                    data = line[6:].strip()
-                    if data == '[DONE]':
-                        break
-                    
-                    try:
-                        chunk = json.loads(data)
-                        if 'choices' in chunk and chunk['choices']:
-                            delta = chunk['choices'][0].get('delta', {})
-                            if delta and delta.get('content'):
-                                content = delta['content']
-                                output += content
-                                output_queue.put(("chunk", output))
-                                print(f"[Chunk]: {content}", end="", flush=True)
-                    except json.JSONDecodeError as e:
-                        print(f"[JSON Error] {e}, line: {line}")
-                        continue
-            
-            output_queue.put(("complete", output))
-            print(f"[Request] Completed successfully")
-            
-        else:
-            error_msg = f"⚠️ HTTP錯誤: {response.status_code} - {response.text}"
-            print(f"[Error] {error_msg}")
-            output_queue.put(("error", error_msg))
-            
-    except requests.exceptions.Timeout:
-        error_msg = "⚠️ 請求超時（第一個token生成時間太長）"
-        print(f"[Error] {error_msg}")
-        output_queue.put(("error", error_msg))
-    except requests.exceptions.ConnectionError:
-        error_msg = "⚠️ 連接錯誤（請檢查llama.cpp伺服器是否運行）"
-        print(f"[Error] {error_msg}")
-        output_queue.put(("error", error_msg))
-    except Exception as e:
-        error_msg = f"⚠️ 未知錯誤: {str(e)}"
-        print(f"[Error] {error_msg}")
-        output_queue.put(("error", error_msg))
 
-def respond(message, history, system_message, max_tokens, temperature, top_p):
-    """處理Gradio請求"""
-    output_queue = queue.Queue()
-    
-    # 啟動工作線程
-    worker_thread = threading.Thread(
-        target=llama_http_stream_worker,
-        args=(message, history, system_message, max_tokens, temperature, top_p, output_queue),
-        daemon=True
-    )
-    worker_thread.start()
-    
-    output = ""
-    last_output_time = time.time()
-    heartbeat_interval = 2.0  # 每2秒發送一次心跳
-    
-    while True:
-        try:
-            # 等待輸出，設置較短超時以保持響應性
-            item_type, content = output_queue.get(timeout=0.5)
-            
-            if item_type == "chunk":
-                output = content
-                yield {"role": "assistant", "content": output}
-                last_output_time = time.time()
-                
-            elif item_type == "complete":
-                yield {"role": "assistant", "content": content}
-                break
-                
-            elif item_type == "error":
-                yield {"role": "assistant", "content": content}
-                break
-                
-        except queue.Empty:
-            # 檢查工作線程是否還在運行
-            if not worker_thread.is_alive():
-                # 線程已結束但沒有發送完成信號，可能出錯了
-                yield {"role": "assistant", "content": "⚠️ 伺服器處理異常中斷"}
-                break
-            
-            # 發送心跳保持連接
-            current_time = time.time()
-            if current_time - last_output_time > heartbeat_interval:
-                if output:  # 如果有內容，發送當前內容作為心跳
+        output = ""
+        for chunk in stream:
+            # 🔍 Debug log
+            # print("[DEBUG] chunk:", chunk)
+
+            if chunk.choices:
+                delta = chunk.choices[0].delta
+                if delta and delta.content:
+                    output += delta.content
                     yield {"role": "assistant", "content": output}
-                last_output_time = current_time
 
-# Gradio 介面
+    except Exception as e:
+        print(f"[Error] {e}")
+        yield {"role": "assistant", "content": "⚠️ Llama.cpp server 沒有回應，請稍後再試。"}
+
+# ✅ Gradio 介面 (新版必須用 type="messages")
 demo = gr.ChatInterface(
     respond,
-    type="messages",
+    type="messages",  # 🔑 使用 OpenAI 風格訊息格式
     additional_inputs=[
         gr.Textbox(value="You are a friendly assistant.", label="System message"),
         gr.Slider(minimum=1, maximum=4096, value=1024, step=1, label="Max new tokens"),
         gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
         gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)"),
     ],
-    title="Llama.cpp Chat Interface",
-    description="直接連接llama.cpp伺服器，避免OpenAI library超時問題"
 )
 
 if __name__ == "__main__":
+    # 修改 Gradio 的 App 類以支持長時間超時
+    from gradio.routes import App
+    
+    class CustomApp(App):
+        def __init__(self, *args, **kwargs):
+            super().__init__(*args, **kwargs)
+            # 修改關鍵的超時參數
+            self.keepalive_timeout = 1800  # 30分鐘
+            if hasattr(self, 'timeout_keep_alive'):
+                self.timeout_keep_alive = 1800  # 30分鐘
+    
+    # 替換默認的 App 類
+    gr.routes.App = CustomApp
+    
+    # 啟動應用程序並設置超時參數
     demo.launch(
         server_name="0.0.0.0",
         server_port=7860,
+        # 關鍵：禁用 Gradio 的心跳檢測和設置長時間超時
+        app_kwargs={
+            "keepalive_timeout": 1800,    # 30分鐘
+            "timeout_keep_alive": 1800,   # 30分鐘
+        },
+        # 禁用心跳檢測
+        heartbeat=False,
+        # 顯示詳細錯誤信息
+        show_error=True,
+        # 增加隊列大小
+        max_threads=20,
+        # 允許共享
         share=False
     )