Init/Update LanguageBridge Multimodal Chatbot Space (final)

README.md

@@ -1,10 +1,20 @@
+---
+title: LanguageBridge — Multimodal Chatbot (Mistral-7B)
+emoji: 🌉
+colorFrom: indigo
+colorTo: blue
+sdk: gradio
+app_file: app.py
+pinned: false
+license: other
+---
+
 # LanguageBridge — Multimodal Chatbot (Mistral-7B)
 
-文字先穩定，影像/語音可用 USE_IMAGE/USE_AUDIO 變數開關。
+以文字為主穩定推理；影像/語音可透過 Variables 開關（USE_IMAGE/USE_AUDIO）。
 
 - Core model: `aciang/mistral7b-tk-sft-20251019-merged`
-- 若要啟用影像/語音：
-  - 進入 **Settings → Variables**，新增：
-    - `USE_IMAGE=1`
-    - `USE_AUDIO=1`
-  - 如遇套件相依衝突，可在 **Settings → Runtime** 切換到 **T4** 或 **A10G**。
+- **啟用影像/語音**：到 **Settings → Variables** 新增  
+  - `USE_IMAGE=1`
+  - `USE_AUDIO=1`  
+- 若遇 CUDA/依賴衝突，請到 **Settings → Hardware** 改用 T4 或 A10G。

app.py

@@ -1,96 +1,104 @@
 
-import os, torch, time
+import os, torch
 import gradio as gr
 from transformers import AutoTokenizer, AutoModelForCausalLM
 
-TITLE     = os.getenv("SPACE_TITLE", "LanguageBridge — Multimodal Chatbot (Mistral-7B)")
-MODEL_ID  = os.getenv("MODEL_ID", "aciang/mistral7b-tk-sft-20251019-merged")
+TITLE    = os.getenv("SPACE_TITLE", "LanguageBridge — Multimodal Chatbot (Mistral-7B)")
+MODEL_ID = os.getenv("MODEL_ID", "aciang/mistral7b-tk-sft-20251019-merged")
+
 SYSTEM_PROMPT = os.getenv("SYSTEM_PROMPT",
-    "你是語言橋助教，回答務必：條列、準確、可引用部落知識與科學。必要時給出步驟清單。")
+    "你是語言橋助教，回答務必：條列、準確，能連結部落知識與科學方法；必要時提供清楚步驟。")
 
-# 可用 Space Variables 覆蓋：USE_IMAGE=1 / USE_AUDIO=1
-USE_IMAGE = os.getenv("USE_IMAGE", "0") in ("1", "true", "True")
-USE_AUDIO = os.getenv("USE_AUDIO", "0") in ("1", "true", "True")
+USE_IMAGE = os.getenv("USE_IMAGE", "0") in ("1","true","True")
+USE_AUDIO = os.getenv("USE_AUDIO", "0") in ("1","true","True")
 
 def load_llm():
     dtype = torch.float16 if torch.cuda.is_available() else torch.float32
     try:
-        model = AutoModelForCausalLM.from_pretrained(
-            MODEL_ID, torch_dtype=dtype, device_map="auto"
-        )
+        model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype=dtype, device_map="auto")
     except Exception as e:
-        print(f"[Fallback CPU] load error: {e}")
-        model = AutoModelForCausalLM.from_pretrained(
-            MODEL_ID, torch_dtype=torch.float32, device_map=None
-        )
+        print(f"[LLM fallback CPU] {e}")
+        model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype=torch.float32, device_map=None)
     tok = AutoTokenizer.from_pretrained(MODEL_ID)
     return tok, model
 
 tokenizer, llm = load_llm()
 llm.eval()
 
-def lazy_load_captioner():
+# (可選) 影像 caption 懶載入
+CAP_PROC, CAP_MOD = None, None
+def ensure_captioner():
+    global CAP_PROC, CAP_MOD
+    if CAP_PROC is not None:
+        return CAP_PROC, CAP_MOD
     try:
         from transformers import BlipProcessor, BlipForConditionalGeneration
         cap_id = os.getenv("CAPTION_MODEL_ID", "Salesforce/blip-image-captioning-base")
-        proc = BlipProcessor.from_pretrained(cap_id)
-        vmod = BlipForConditionalGeneration.from_pretrained(
+        CAP_PROC = BlipProcessor.from_pretrained(cap_id)
+        CAP_MOD  = BlipForConditionalGeneration.from_pretrained(
             cap_id, torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32
         )
-        if torch.cuda.is_available():
-            vmod = vmod.to("cuda")
-        return proc, vmod
+        if torch.cuda.is_available(): CAP_MOD.to("cuda")
     except Exception as e:
         print(f"[Image OFF] {e}")
-        return None, None
-
-CAP_PROC, CAP_MOD = (None, None)
-if USE_IMAGE:
-    CAP_PROC, CAP_MOD = lazy_load_captioner()
-
-def lazy_load_asr():
+        CAP_PROC, CAP_MOD = None, None
+    return CAP_PROC, CAP_MOD
+
+# (可選) Whisper 懶載入
+ASR = None
+def ensure_asr():
+    global ASR
+    if ASR is not None:
+        return ASR
     try:
         import whisper
         asr_id = os.getenv("ASR_MODEL_ID", "tiny")
-        return whisper.load_model(asr_id)
+        ASR = whisper.load_model(asr_id)
     except Exception as e:
         print(f"[Audio OFF] {e}")
-        return None
-
-ASR = lazy_load_asr() if USE_AUDIO else None
+        ASR = None
+    return ASR
 
 @torch.inference_mode()
 def generate_reply(history, image, audio, max_new_tokens, temperature, top_p):
-    sys_prompt = SYSTEM_PROMPT
-    user_parts = []
-
-    if image is not None and USE_IMAGE and CAP_PROC and CAP_MOD:
-        try:
-            from PIL import Image
-            im = Image.open(image).convert("RGB")
-            inputs = CAP_PROC(im, return_tensors="pt").to(CAP_MOD.device)
-            out = CAP_MOD.generate(**inputs, max_new_tokens=64)
-            cap = CAP_PROC.decode(out[0], skip_special_tokens=True)
-            user_parts.append(f"[影像描述] {cap}")
-        except Exception as e:
-            user_parts.append(f"[影像處理失敗: {e}]")
-
-    if audio is not None and USE_AUDIO and ASR is not None:
-        try:
-            result = ASR.transcribe(audio, fp16=torch.cuda.is_available())
-            user_parts.append(f"[語音辨識] {result.get('text','')}")
-        except Exception as e:
-            user_parts.append(f"[語音處理失敗: {e}]")
-
-    text = ""
-    for role, msg in reversed(history or []):
-        if role == "user":
-            text = msg.get("content", "") if isinstance(msg, dict) else str(msg)
-            break
-    if user_parts:
-        text = (text + "\n" if text else "") + "\n".join(user_parts)
-
-    prompt = f"{sys_prompt}\n\n使用者：{text}\n助教："
+    # 取最後一輪 user 文字
+    user_text = ""
+    if history:
+        for role, msg in history[::-1]:
+            if role == "user":
+                user_text = msg.get("content","") if isinstance(msg, dict) else str(msg)
+                break
+
+    extra_parts = []
+
+    # 影像
+    if image and USE_IMAGE:
+        proc, vmod = ensure_captioner()
+        if proc and vmod:
+            try:
+                from PIL import Image
+                im = Image.open(image).convert("RGB")
+                inputs = proc(im, return_tensors="pt").to(vmod.device)
+                out = vmod.generate(**inputs, max_new_tokens=64)
+                cap = proc.decode(out[0], skip_special_tokens=True)
+                extra_parts.append(f"[影像描述] {cap}")
+            except Exception as e:
+                extra_parts.append(f"[影像處理失敗] {e}")
+
+    # 語音
+    if audio and USE_AUDIO:
+        asr = ensure_asr()
+        if asr:
+            try:
+                res = asr.transcribe(audio, fp16=torch.cuda.is_available())
+                extra_parts.append(f"[語音辨識] {res.get('text','')}")
+            except Exception as e:
+                extra_parts.append(f"[語音處理失敗] {e}")
+
+    if extra_parts:
+        user_text = (user_text + "\n" if user_text else "") + "\n".join(extra_parts)
+
+    prompt = f"{SYSTEM_PROMPT}\n\n使用者：{user_text}\n助教："
     inputs = tokenizer(prompt, return_tensors="pt").to(llm.device)
     out = llm.generate(
         **inputs,
@@ -102,23 +110,23 @@ def generate_reply(history, image, audio, max_new_tokens, temperature, top_p):
         pad_token_id=tokenizer.eos_token_id
     )
     ans = tokenizer.decode(out[0], skip_special_tokens=True)
-    if "助教：" in ans:
-        ans = ans.split("助教：", 1)[-1].strip()
+    if "助教：" in ans: ans = ans.split("助教：",1)[-1].strip()
     return ans
 
 with gr.Blocks(title=TITLE, fill_height=True) as demo:
-    gr.Markdown(f"## {TITLE}\n- Core model: `{MODEL_ID}`\n- 影像/語音預設關閉（可在 Space Variables: `USE_IMAGE=1` / `USE_AUDIO=1` 開啟）")
+    gr.Markdown(f"## {TITLE}\n- 模型：`{MODEL_ID}`\n- 影像/語音預設關閉（可在 Variables 開 `USE_IMAGE=1` / `USE_AUDIO=1`）")
+
     with gr.Row():
-        chat = gr.Chatbot(height=450, type="messages", show_copy_button=True)
+        chat = gr.Chatbot(height=460, type="messages", show_copy_button=True)
         with gr.Column(scale=0):
             user_txt = gr.Textbox(label="你的問題 / 指令", placeholder="請輸入文字…", interactive=True)
-            img = gr.Image(label="(可選) 上傳圖片", type="filepath", visible=bool(USE_IMAGE), interactive=True)
-            aud = gr.Audio(label="(可選) 上傳語音", type="filepath", sources=["upload","microphone"], visible=bool(USE_AUDIO), interactive=True)
+            img = gr.Image(label="(可選) 圖片", type="filepath", visible=bool(USE_IMAGE), interactive=True)
+            aud = gr.Audio(label="(可選) 語音", type="filepath", sources=["upload","microphone"], visible=bool(USE_AUDIO), interactive=True)
             mx  = gr.Slider(64, 1024, value=512, step=32, label="max_new_tokens")
-            tp  = gr.Slider(0.1, 1.2,  value=0.6, step=0.05, label="temperature")
-            top = gr.Slider(0.5, 1.0,  value=0.95, step=0.01,  label="top_p")
+            tp  = gr.Slider(0.1, 1.2, value=0.6, step=0.05, label="temperature")
+            top = gr.Slider(0.5, 1.0, value=0.95, step=0.01, label="top_p")
             btn = gr.Button("送出 🚀", variant="primary")
-            clr = gr.Button("清除對話")
+            clr = gr.Button("清除")
 
     def respond(history, text, image, audio, mx, tp, top):
         history = history or []
@@ -133,8 +141,10 @@ with gr.Blocks(title=TITLE, fill_height=True) as demo:
     btn.click(respond, inputs=[chat, user_txt, img, aud, mx, tp, top], outputs=[chat, user_txt])
     clr.click(lambda: ([], ""), outputs=[chat, user_txt])
 
-try:
-    demo.queue().launch(share=False, show_error=True, show_api=False)
-except Exception as e:
-    print(f"[local 失敗，改用 share=True] {e}")
-    demo.queue().launch(share=True, show_error=True, show_api=False)
+# 僅在 Space 環境啟動；此檔由 HF 自動執行，Colab 不需啟動。
+if __name__ == "__main__":
+    try:
+        demo.queue().launch(share=False, show_error=True, show_api=False)
+    except Exception as e:
+        print(f"[local 失敗，改用 share=True] {e}")
+        demo.queue().launch(share=True, show_error=True, show_api=False)

requirements.txt

@@ -2,10 +2,8 @@ gradio>=4.44.0
 transformers>=4.44.0
 accelerate>=0.31.0
 bitsandbytes
-torch
 huggingface_hub
-# 若要啟用影像/語音，請於 Space Variables 設 USE_IMAGE/USE_AUDIO=1，
-# 並可視情況取消註解下列依賴（或在 Runtime 選擇 T4/A10G）
+# 需要影像/語音時再取消以下註解（或在 Space 的 Docker 環境中安裝）
 # pillow
 # torchaudio
 # soundfile