fix: remove concurrency_count; add CPU fallback + cache

app.py

@@ -1,104 +1,77 @@
 
 import os, torch, gradio as gr
-from transformers import AutoTokenizer, AutoModelForCausalLM, TextIteratorStreamer, BitsAndBytesConfig
-
-TITLE    = "LanguageBridge — Multimodal Chatbot (Mistral-7B)"
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, TextIteratorStreamer
+os.environ.setdefault("HF_HOME", "/data/.cache")
+os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1")
 MODEL_ID = os.getenv("MODEL_ID", "aciang/mistral7b-tk-sft-20251019-merged")
-
-SYSTEM_PROMPT = (
-    "你是『語言橋』學習助教。規則："
-    "1) 嚴謹、分段、先重點後細節；"
-    "2) 若為數學/規則題：先列步驟，再給最終答案；"
-    "3) 若資訊不足，請明確指出缺口，勿捏造；"
-    "4) 優先以繁體中文回答。"
-)
+TITLE = "LanguageBridge — Multimodal Chatbot (Mistral-7B)"
+SYSTEM_PROMPT = "你是教學助教。先讀【任務】，按【格式】作答；資料不足先列缺口，勿猜測。"
 
 def load_llm():
-    # 4-bit 後援；失敗則 fp16/CPU fallback
-    kwargs = dict(trust_remote_code=False)
-    bnb = BitsAndBytesConfig(
-        load_in_4bit=True, bnb_4bit_quant_type="nf4",
-        bnb_4bit_use_double_quant=True,
-        bnb_4bit_compute_dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float16
-    )
+    has_cuda = torch.cuda.is_available()
+    kwargs = dict(trust_remote_code=False, low_cpu_mem_usage=True)
     try:
-        model = AutoModelForCausalLM.from_pretrained(MODEL_ID, quantization_config=bnb, device_map="auto", **kwargs)
+        if has_cuda:
+            bnb = BitsAndBytesConfig(
+                load_in_4bit=True, bnb_4bit_use_double_quant=True,
+                bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.bfloat16
+            )
+            model = AutoModelForCausalLM.from_pretrained(MODEL_ID, device_map="auto", quantization_config=bnb, **kwargs)
+        else:
+            print("[no CUDA] using CPU fp32")
+            model = AutoModelForCausalLM.from_pretrained(MODEL_ID, device_map="cpu", torch_dtype=torch.float32, **kwargs)
     except Exception as e:
-        print("[4-bit failed] -> fp16/CPU fallback:", e)
+        print("[loader fallback fp16/cpu]:", e)
         model = AutoModelForCausalLM.from_pretrained(
             MODEL_ID,
-            torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
-            device_map="auto" if torch.cuda.is_available() else None,
+            device_map="auto" if has_cuda else "cpu",
+            torch_dtype=torch.float16 if has_cuda else torch.float32,
             **kwargs
         )
     tok = AutoTokenizer.from_pretrained(MODEL_ID, use_fast=True)
     if tok.pad_token is None: tok.pad_token = tok.eos_token
     tok.padding_side = "left"
-    if torch.cuda.is_available():
-        torch.backends.cuda.matmul.allow_tf32 = True
     model.config.use_cache = True
     return tok, model
 
 tokenizer, llm = load_llm(); llm.eval()
 
-def build_prompt(context, question, longform):
-    head = SYSTEM_PROMPT
-    if context.strip():
-        head += f"\n\n[上下文]\n{context.strip()}"
-    ask  = f"\n\n[問題]\n{question.strip()}\n"
-    tail = "\n請以條列步驟與小結回覆；若可計算，先算再答。"
-    if longform:
-        tail += "\n（長文模式）請分段、標題化、最後給出『摘要重點』。"
-    return head + ask + tail
+def build_prompt(task, ctx=None):
+    head = "你是教學助教。先讀任務，依：1) 摘要要點；2) 逐步推理；3) 結論條列。\n\n"
+    if ctx:
+        ctx = ctx[-6000:]
+        return f"{head}【參考上下文】\n{ctx}\n\n【使用者問題】\n{task}\n\n【回答】"
+    return f"{head}【使用者問題】\n{task}\n\n【回答】"
 
 @torch.inference_mode()
-def stream_answer(context, question, longform, mx, temp, top_p, rep):
-    prompt = build_prompt(context, question, longform)
+def stream_answer(task, context, mx=256, temp=0.15, top_p=0.9):
+    prompt = build_prompt(task, context.strip() or None)
     inputs = tokenizer(prompt, return_tensors="pt").to(llm.device)
     streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
-    gen = dict(
-        **inputs, streamer=streamer, max_new_tokens=int(mx),
-        temperature=float(temp), top_p=float(top_p),
-        repetition_penalty=float(rep),
-        do_sample=True if float(temp)>0 else False,
-        eos_token_id=tokenizer.eos_token_id, pad_token_id=tokenizer.pad_token_id
-    )
-    import threading
-    t = threading.Thread(target=llm.generate, kwargs=gen); t.start()
+    kwargs = dict(**inputs, streamer=streamer, max_new_tokens=int(mx),
+                  temperature=float(temp), top_p=float(top_p),
+                  do_sample=True if float(temp)>0 else False,
+                  eos_token_id=tokenizer.eos_token_id, pad_token_id=tokenizer.pad_token_id)
+    import threading; threading.Thread(target=llm.generate, kwargs=kwargs).start()
     buf=""
     for tok in streamer:
         buf += tok
         yield buf
 
-def warmup():
-    try:
-        _ = list(stream_answer("", "簡述本系統的用途。", False, 96, 0.2, 0.9, 1.05))[-1]
-        print("[warmup] done")
-    except Exception as e:
-        print("[warmup] skip:", e)
-
 with gr.Blocks(title=TITLE, theme="soft") as demo:
-    gr.Markdown(f"## {TITLE}\n模型：`{MODEL_ID}`｜已啟用：上下文欄位、長文模式、流式輸出、暖機")
-    with gr.Row():
-        ctx = gr.Textbox(label="上下文（長文，可空白）", placeholder="選填的背景內容/段落/資料摘錄", lines=6)
-    with gr.Row():
-        q = gr.Textbox(label="問題 / 指令", placeholder="請清楚描述你的問題", lines=3)
-    with gr.Row():
-        longf = gr.Checkbox(label="長文模式（章節化 + 摘要）", value=True)
-    with gr.Row():
-        mx   = gr.Slider(128, 1024, value=512, step=32, label="max_new_tokens")
-        temp = gr.Slider(0.0, 0.8,  value=0.2, step=0.05, label="temperature")
-        top  = gr.Slider(0.6, 1.0,  value=0.9, step=0.01, label="top_p")
-        rep  = gr.Slider(1.0, 1.3,  value=1.05, step=0.01, label="repetition_penalty")
-    go = gr.Button("送出 🚀", variant="primary")
-    out = gr.Textbox(label="輸出（流式）", lines=14)
-    clr = gr.Button("清除")
-
-    go.click(stream_answer, inputs=[ctx,q,longf,mx,temp,top,rep], outputs=out)
+    gr.Markdown(f"## {TITLE}｜模型：`{MODEL_ID}`（流式）")
+    q   = gr.Textbox(label="你的問題 / 指令", lines=5, placeholder="可貼長文；我會先摘要→推理→結論")
+    ctx = gr.Textbox(label="（可選）上下文", lines=6)
+    mx   = gr.Slider(64, 512, value=256, step=32, label="max_new_tokens")
+    temp = gr.Slider(0.0, 0.8, value=0.15, step=0.05, label="temperature")
+    top  = gr.Slider(0.6, 1.0, value=0.9, step=0.01, label="top_p")
+    go   = gr.Button("送出 🚀", variant="primary")
+    out  = gr.Textbox(label="輸出（流式）", lines=14)
+    clr  = gr.Button("清除")
+    go.click(stream_answer, inputs=[q, ctx, mx, temp, top], outputs=out)
     clr.click(lambda:"", outputs=out)
-
-    demo.queue(concurrency_count=4, max_size=32, api_open=False)
-    warmup()
+    # ← 修正：不要用舊參數 concurrency_count
+    demo.queue(max_size=32, status_update_rate=1, api_open=False)
 
 if __name__ == "__main__":
     demo.launch(share=False, server_name="0.0.0.0", server_port=7860, show_error=True)