Update app.py

app.py

@@ -7,38 +7,33 @@ import sys
 import gradio as gr
 from huggingface_hub import hf_hub_download
 
-# --- 配置 ---
+# Llama-cpp-python, f- you for not updating for months! :(
+
 LLAMA_CPP_RELEASE_URL = "https://github.com/ggml-org/llama.cpp/releases/download/b8093/llama-b8093-bin-ubuntu-x64.tar.gz"
 BINARY_NAME = "llama-server"
 SERVER_PORT = "8080"
 REPO_ID = "huzpsb/heru"
 FILENAME = "qwq_q4k.gguf"
 
+
 def setup_server():
-    """下载并启动 llama-server，处理动态链接库及符号链接"""
     print(f"[*] Downloading model: {FILENAME}...")
     model_path = hf_hub_download(repo_id=REPO_ID, filename=FILENAME)
-
     if not os.path.exists(BINARY_NAME):
         print("[*] Downloading llama.cpp binary package...")
         response = requests.get(LLAMA_CPP_RELEASE_URL, stream=True)
         with open("llama.tar.gz", "wb") as f:
             f.write(response.content)
-
         print("[*] Extracting files and handling symlinks...")
         with tarfile.open("llama.tar.gz", "r:gz") as tar:
             for member in tar.getmembers():
                 base_name = os.path.basename(member.name)
-                if not base_name: continue # 跳过目录本身
-
+                if not base_name: continue
                 if member.isfile():
-                    # 提取普通文件并去除路径前缀
                     member.name = base_name
                     tar.extract(member, path=".")
                 elif member.issym():
-                    # 处理符号链接
                     link_target = os.path.basename(member.linkname)
-                    # 如果链接已存在，先删除
                     if os.path.lexists(base_name):
                         os.remove(base_name)
                     try:
@@ -55,25 +50,21 @@ def setup_server():
     print("[*] Starting llama-server with LD_LIBRARY_PATH...")
     new_env = os.environ.copy()
     current_dir = os.getcwd()
-    # 确保 LD_LIBRARY_PATH 包含当前目录，以便加载 .so 文件
     new_env["LD_LIBRARY_PATH"] = f"{current_dir}:{new_env.get('LD_LIBRARY_PATH', '')}"
-    
     cmd = [
         f"./{BINARY_NAME}",
         "-m", model_path,
         "--port", SERVER_PORT,
-        "--ctx-size", "8192",
+        "--ctx-size", "81920",
         "--n-gpu-layers", "0",
         "--host", "127.0.0.1"
     ]
-    
     proc = subprocess.Popen(
         cmd,
         stdout=sys.stdout,
         stderr=sys.stderr,
         env=new_env
     )
-
     print("[*] Waiting for server to respond...")
     retries = 0
     while retries < 60:
@@ -85,34 +76,27 @@ def setup_server():
         except:
             time.sleep(2)
             retries += 1
-    
+
     raise Exception("Server failed to start. Check logs for missing .so files.")
 
-# 初始化
+
 server_process = setup_server()
 
-# --- 修改后的预测函数 ---
+
 def predict(message, history, system_prompt, temperature):
-    """
-    Gradio 回调：现在支持自定义 system prompt 和 temperature
-    """
     messages = [{"role": "system", "content": system_prompt}]
     for user_msg, bot_msg in history:
         messages.append({"role": "user", "content": user_msg})
         messages.append({"role": "assistant", "content": bot_msg})
     messages.append({"role": "user", "content": message})
-
     payload = {
         "messages": messages,
         "temperature": temperature,
         "max_tokens": 2048,
         "stream": False
     }
-
-    # --- 调试日志 ---
     print("\n--- [Request Payload] ---")
     print(payload)
-
     try:
         response = requests.post(
             f"http://127.0.0.1:{SERVER_PORT}/v1/chat/completions",
@@ -121,19 +105,17 @@ def predict(message, history, system_prompt, temperature):
         )
         response.raise_for_status()
         result = response.json()
-        
         print("--- [Response] ---")
-        print(result) # 打印完整响应
-        
+        print(result)
         return result["choices"][0]["message"]["content"]
     except Exception as e:
         print(f"--- [Error] --- \n{str(e)}")
         return f"Error: {str(e)}"
 
-# --- Gradio UI (支持 API 参数) ---
+
 with gr.Blocks(theme="soft") as demo:
     gr.Markdown("## Qwen3 Inference via llama-server")
-    
+
     chat_interface = gr.ChatInterface(
         fn=predict,
         additional_inputs=[
@@ -144,4 +126,4 @@ with gr.Blocks(theme="soft") as demo:
     )
 
 if __name__ == "__main__":
-    demo.launch(server_name="0.0.0.0", server_port=7860)
+    demo.launch(server_name="0.0.0.0", server_port=7860)