Init/Update LanguageBridge Multimodal Chatbot Space

README.md

@@ -1,12 +1,10 @@
----
-title: LanguageBridge Mistral7B Multimodal Chat
-emoji: 📈
-colorFrom: yellow
-colorTo: red
-sdk: gradio
-sdk_version: 5.49.1
-app_file: app.py
-pinned: false
----
+# LanguageBridge — Multimodal Chatbot (Mistral-7B)
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+文字先穩定，影像/語音可用 USE_IMAGE/USE_AUDIO 變數開關。
+
+- Core model: `aciang/mistral7b-tk-sft-20251019-merged`
+- 若要啟用影像/語音：
+  - 進入 **Settings → Variables**，新增：
+    - `USE_IMAGE=1`
+    - `USE_AUDIO=1`
+  - 如遇套件相依衝突，可在 **Settings → Runtime** 切換到 **T4** 或 **A10G**。

app.py

@@ -0,0 +1,140 @@
+
+import os, torch, time
+import gradio as gr
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+TITLE     = os.getenv("SPACE_TITLE", "LanguageBridge — Multimodal Chatbot (Mistral-7B)")
+MODEL_ID  = os.getenv("MODEL_ID", "aciang/mistral7b-tk-sft-20251019-merged")
+SYSTEM_PROMPT = os.getenv("SYSTEM_PROMPT",
+    "你是語言橋助教，回答務必：條列、準確、可引用部落知識與科學。必要時給出步驟清單。")
+
+# 可用 Space Variables 覆蓋：USE_IMAGE=1 / USE_AUDIO=1
+USE_IMAGE = os.getenv("USE_IMAGE", "0") in ("1", "true", "True")
+USE_AUDIO = os.getenv("USE_AUDIO", "0") in ("1", "true", "True")
+
+def load_llm():
+    dtype = torch.float16 if torch.cuda.is_available() else torch.float32
+    try:
+        model = AutoModelForCausalLM.from_pretrained(
+            MODEL_ID, torch_dtype=dtype, device_map="auto"
+        )
+    except Exception as e:
+        print(f"[Fallback CPU] load error: {e}")
+        model = AutoModelForCausalLM.from_pretrained(
+            MODEL_ID, torch_dtype=torch.float32, device_map=None
+        )
+    tok = AutoTokenizer.from_pretrained(MODEL_ID)
+    return tok, model
+
+tokenizer, llm = load_llm()
+llm.eval()
+
+def lazy_load_captioner():
+    try:
+        from transformers import BlipProcessor, BlipForConditionalGeneration
+        cap_id = os.getenv("CAPTION_MODEL_ID", "Salesforce/blip-image-captioning-base")
+        proc = BlipProcessor.from_pretrained(cap_id)
+        vmod = BlipForConditionalGeneration.from_pretrained(
+            cap_id, torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32
+        )
+        if torch.cuda.is_available():
+            vmod = vmod.to("cuda")
+        return proc, vmod
+    except Exception as e:
+        print(f"[Image OFF] {e}")
+        return None, None
+
+CAP_PROC, CAP_MOD = (None, None)
+if USE_IMAGE:
+    CAP_PROC, CAP_MOD = lazy_load_captioner()
+
+def lazy_load_asr():
+    try:
+        import whisper
+        asr_id = os.getenv("ASR_MODEL_ID", "tiny")
+        return whisper.load_model(asr_id)
+    except Exception as e:
+        print(f"[Audio OFF] {e}")
+        return None
+
+ASR = lazy_load_asr() if USE_AUDIO else None
+
+@torch.inference_mode()
+def generate_reply(history, image, audio, max_new_tokens, temperature, top_p):
+    sys_prompt = SYSTEM_PROMPT
+    user_parts = []
+
+    if image is not None and USE_IMAGE and CAP_PROC and CAP_MOD:
+        try:
+            from PIL import Image
+            im = Image.open(image).convert("RGB")
+            inputs = CAP_PROC(im, return_tensors="pt").to(CAP_MOD.device)
+            out = CAP_MOD.generate(**inputs, max_new_tokens=64)
+            cap = CAP_PROC.decode(out[0], skip_special_tokens=True)
+            user_parts.append(f"[影像描述] {cap}")
+        except Exception as e:
+            user_parts.append(f"[影像處理失敗: {e}]")
+
+    if audio is not None and USE_AUDIO and ASR is not None:
+        try:
+            result = ASR.transcribe(audio, fp16=torch.cuda.is_available())
+            user_parts.append(f"[語音辨識] {result.get('text','')}")
+        except Exception as e:
+            user_parts.append(f"[語音處理失敗: {e}]")
+
+    text = ""
+    for role, msg in reversed(history or []):
+        if role == "user":
+            text = msg.get("content", "") if isinstance(msg, dict) else str(msg)
+            break
+    if user_parts:
+        text = (text + "\n" if text else "") + "\n".join(user_parts)
+
+    prompt = f"{sys_prompt}\n\n使用者：{text}\n助教："
+    inputs = tokenizer(prompt, return_tensors="pt").to(llm.device)
+    out = llm.generate(
+        **inputs,
+        max_new_tokens=int(max_new_tokens),
+        temperature=float(temperature),
+        top_p=float(top_p),
+        do_sample=True,
+        eos_token_id=tokenizer.eos_token_id,
+        pad_token_id=tokenizer.eos_token_id
+    )
+    ans = tokenizer.decode(out[0], skip_special_tokens=True)
+    if "助教：" in ans:
+        ans = ans.split("助教：", 1)[-1].strip()
+    return ans
+
+with gr.Blocks(title=TITLE, fill_height=True) as demo:
+    gr.Markdown(f"## {TITLE}\n- Core model: `{MODEL_ID}`\n- 影像/語音預設關閉（可在 Space Variables: `USE_IMAGE=1` / `USE_AUDIO=1` 開啟）")
+    with gr.Row():
+        chat = gr.Chatbot(height=450, type="messages", show_copy_button=True)
+        with gr.Column(scale=0):
+            user_txt = gr.Textbox(label="你的問題 / 指令", placeholder="請輸入文字…", interactive=True)
+            img = gr.Image(label="(可選) 上傳圖片", type="filepath", visible=bool(USE_IMAGE), interactive=True)
+            aud = gr.Audio(label="(可選) 上傳語音", type="filepath", sources=["upload","microphone"], visible=bool(USE_AUDIO), interactive=True)
+            mx  = gr.Slider(64, 1024, value=512, step=32, label="max_new_tokens")
+            tp  = gr.Slider(0.1, 1.2,  value=0.6, step=0.05, label="temperature")
+            top = gr.Slider(0.5, 1.0,  value=0.95, step=0.01,  label="top_p")
+            btn = gr.Button("送出 🚀", variant="primary")
+            clr = gr.Button("清除對話")
+
+    def respond(history, text, image, audio, mx, tp, top):
+        history = history or []
+        history.append(("user", {"content": text}))
+        try:
+            ans = generate_reply(history, image, audio, mx, tp, top)
+        except Exception as e:
+            ans = f"(推理失敗：{e})"
+        history.append(("assistant", {"content": ans}))
+        return history, ""
+
+    btn.click(respond, inputs=[chat, user_txt, img, aud, mx, tp, top], outputs=[chat, user_txt])
+    clr.click(lambda: ([], ""), outputs=[chat, user_txt])
+
+try:
+    demo.queue().launch(share=False, show_error=True, show_api=False)
+except Exception as e:
+    print(f"[local 失敗，改用 share=True] {e}")
+    demo.queue().launch(share=True, show_error=True, show_api=False)

requirements.txt

@@ -0,0 +1,12 @@
+gradio>=4.44.0
+transformers>=4.44.0
+accelerate>=0.31.0
+bitsandbytes
+torch
+huggingface_hub
+# 若要啟用影像/語音，請於 Space Variables 設 USE_IMAGE/USE_AUDIO=1，
+# 並可視情況取消註解下列依賴（或在 Runtime 選擇 T4/A10G）
+# pillow
+# torchaudio
+# soundfile
+# ffmpeg-python