Update app.py

app.py

@@ -1,63 +1,110 @@
+import json
+import spaces
+import subprocess
 import gradio as gr
 from llama_cpp import Llama
 from huggingface_hub import hf_hub_download
 
 CSS = """
 #qwen-md .katex-display { display: inline; }
+#qwen-md .katex-display>.katex { display: inline; }
+#qwen-md .katex-display>.katex>.katex-html { display: inline; }
 """
 
-# 下载GGUF模型文件（如果本地已下载，此代码会跳过）
 hf_hub_download(
     repo_id="bartowski/Qwen2.5-Math-7B-Instruct-GGUF",
     filename="Qwen2.5-Math-7B-Instruct-Q6_K_L.gguf",
     local_dir="./models",
 )
 
-# 仅CPU模式加载模型，去除flash attention和显存优化参数
 llm = Llama(
     model_path="models/Qwen2.5-Math-7B-Instruct-Q6_K_L.gguf",
-    n_ctx=2048,   # 更短的上下文提高速度
-    n_batch=256,  # 小批量减少CPU负载
-    n_threads=8,  # 明确线程数 (可根据你的CPU核心数调整)
+    flash_attn=True,
+    n_ctx=8192,
+    n_batch=1024,
     chat_format="chatml",
-    verbose=False
 )
 
-# Gradio 组件定义（简化配置）
-input_text = gr.Textbox(label="Ask math questions here")
-output_md = gr.Markdown(label="Answer", elem_id="qwen-md", show_copy_button=True)
-target_lang = gr.Dropdown(choices=["Chinese", "English"], value="Chinese", label="Output Language")
-new_tokens = gr.Slider(minimum=1, maximum=512, value=256, step=1, label="Max new tokens")
-temperature = gr.Slider(minimum=0, maximum=1.0, value=0.2, step=0.05, label="Temperature")
-top_p = gr.Slider(minimum=0.0, maximum=1.0, value=0.9, step=0.05, label="Top P")
-submit_btn = gr.Button("Ask")
-banner = gr.Markdown(value="### 📖 **Qwen2.5-Math 7B GGUF** - Optimized for CPU")
+# Gradio 组件
+output_md = gr.Markdown(
+    label="Answer",
+    value="Answer will be presented here",
+    latex_delimiters=[
+        {"left": "\\(", "right": "\\)", "display": True},
+        {"left": "\\begin\{equation\}", "right": "\\end\{equation\}", "display": True},
+        {"left": "\\begin\{align\}", "right": "\\end\{align\}", "display": True},
+        {"left": "\\begin\{alignat\}", "right": "\\end\{alignat\}", "display": True},
+        {"left": "\\begin\{gather\}", "right": "\\end\{gather\}", "display": True},
+        {"left": "\\begin\{CD\}", "right": "\\end\{CD\}", "display": True},
+        {"left": "\\[", "right": "\\]", "display": True},
+    ],
+    elem_id="qwen-md",
+    show_copy_button=True,
+    container=True,
+    render=False,
+)
+target_lang = gr.Dropdown(
+    choices=["Chinese", "English"],
+    value="Chinese",
+    label="Output Language",
+    interactive=True,
+    render=False,
+)
+new_tokens = gr.Slider(
+    minimum=1, maximum=8192, value=2048, step=1, label="Max new tokens", render=False
+)
+temperature = gr.Slider(
+    minimum=0, maximum=2.0, value=0.5, step=0.1, label="Temperature", render=False
+)
+top_p = gr.Slider(minimum=0.0, maximum=1.0, value=0.95, step=0.05, label="Top P", render=False)
+input_text = gr.Textbox(label="Ask math questions here", render=False)
+submit_btn = gr.Button(value="Ask", render=False)
+banner = gr.Markdown(value="""
+# 📖 Qwen2.5-Math GGUF
+This WebUI is based on Qwen2.5-Math-7B-Instruct-GGUF for mathematical reasoning. You can input texts of mathematical or arithmetic problems.
+"""
+)
+
 
-# 响应函数（精简版，更快速）
-def respond(input_text, lang, max_tokens, temperature, top_p):
-    sys_msg = "你是一个乐于助人的数学助手，请使用中文回答。" if lang == "Chinese" else "You are a helpful math assistant. Please answer in English."
+# Gradio 函数
+def respond(
+    input_text,
+    lang="Chinese",
+    max_tokens=2048,
+    temperature=0.5,
+    top_p=0.95,
+):
+    if lang == "Chinese":
+        sys_msg = "你是一个乐于助人的数学助手. 你使用中文回答问题"
+    else:
+        sys_msg = "You are a helpful math assistant. You should always provide your answer in English."
     messages = [
-        {"role": "system", "content": sys_msg},
+        {
+            "role": "system",
+            "content": sys_msg,
+        },
         {"role": "user", "content": input_text},
     ]
 
-    stream_response = llm.create_chat_completion(
+    response = ""
+    response = llm.create_chat_completion(
         messages=messages,
         stream=True,
         max_tokens=max_tokens,
         temperature=temperature,
         top_p=top_p,
     )
+    message_repl = ""
+    for chunk in response:
+        if len(chunk['choices'][0]["delta"]) != 0 and "content" in chunk['choices'][0]["delta"]:
+            message_repl = message_repl + \
+                chunk['choices'][0]["delta"]["content"]
+        yield message_repl
 
-    result = ""
-    for chunk in stream_response:
-        content = chunk['choices'][0]["delta"].get("content", "")
-        result += content
-        yield result.strip()
 
 with gr.Blocks(css=CSS, theme="NoCrypt/miku") as demo:
     submit_btn.click(
-        respond,
+        fn=respond,
         inputs=[input_text, target_lang, new_tokens, temperature, top_p],
         outputs=output_md,
     )
@@ -75,4 +122,4 @@ with gr.Blocks(css=CSS, theme="NoCrypt/miku") as demo:
                 output_md.render()
 
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch()