Update emotion.py

emotion.py

@@ -6,7 +6,8 @@ import gradio as gr
 import numpy as np
 from PIL import Image
 
-import cv2  # ✅ 新增：用來畫框
+import cv2
+import mediapipe as mp
 
 from deepface import DeepFace
 
@@ -14,19 +15,24 @@ from deepface import DeepFace
 # =========================
 # 參數（可微調）
 # =========================
-ANALYZE_EVERY_SEC = 0.8
-DOWNSAMPLE_W = 720
+ANALYZE_EVERY_SEC = 0.8         # 情緒分析頻率（秒）
+DOWNSAMPLE_W = 720              # 目前保留（可不使用）
 EMA_ALPHA = 0.45
 CONF_TH = 40.0
 SWITCH_CONFIRM_SEC = 2.5
-IOU_TH = 0.25
-ZOOM_FACTOR = 1.5
+ZOOM_FACTOR = 1.5               # 你原本就有的 zoom
 
-DETECTOR_BACKEND = "mtcnn"     # 想快一點可改 "opencv"
-ALIGN_FACE = True
+# DeepFace 相關（我們這版用 MediaPipe 偵測臉，再把 ROI 丟進 DeepFace.analyze）
+ALIGN_FACE = True               # 保留參數（這版 ROI 已是人臉）
+# DETECTOR_BACKEND = "mtcnn"    # 這版不再用 extract_faces，就不需要
 
-# ✅ 新增：bbox 過期時間（秒），避免框卡死
-BBOX_TTL_SEC = 2.0
+
+# =========================
+# MediaPipe Face Detection（白框 + 紅點 keypoints）
+# =========================
+_mp_face = mp.solutions.face_detection
+_FACE_DET = _mp_face.FaceDetection(model_selection=0, min_detection_confidence=0.5)
+# model_selection=0 偏短距離偵測（接近 blaze_face_short_range）
 
 
 # =========================
@@ -43,7 +49,7 @@ EMO_ZH = {
     "unknown": "未知",
 }
 
-# 你要的：把 7 種情緒塞進 5 分心情分數
+# 把 7 種情緒塞進 5 分心情分數
 EMO_TO_SCORE = {
     "angry": 1,
     "disgust": 1,
@@ -73,17 +79,11 @@ SCORE_CHOICES = [SCORE_LABELS[i] for i in [1, 2, 3, 4, 5]]
 # =========================
 @dataclass
 class AppState:
-    mode: str = "攝影機辨識" 
-
+    mode: str = "攝影機辨識"
     running: bool = True
     finished: bool = False
     last_analyze_t: float = 0.0
 
-    # tracking（bbox 在 small 座標系）
-    track_bbox: Optional[Tuple[int, int, int, int]] = None
-    track_small_shape: Optional[Tuple[int, int]] = None  # ✅ 新增：(h, w) of small
-    track_bbox_time: float = 0.0                          # ✅ 新增：bbox 更新時間
-
     # smoothing
     ema_scores: Optional[np.ndarray] = None
     emo_keys: List[str] = field(default_factory=list)
@@ -96,8 +96,7 @@ class AppState:
     # final
     final_emo: Optional[str] = None
     final_conf: Optional[float] = None
-
-    final_score: int = 3  
+    final_score: int = 3
 
 
 # =========================
@@ -110,21 +109,20 @@ def _hint_html(msg: str) -> str:
     </div>
     """
 
-
 def _camera_hint_before_start() -> str:
     return _hint_html(
         "請先按「Click to access webcam / 允許」開啟攝影機；"
         "接著，請按「錄製（Record）」開始。"
     )
 
-
 def _camera_hint_done() -> str:
     return _hint_html("已完成辨識，可按「重新攝影機辨識」再次進行辨識。")
 
-
 def _camera_hint_stopped() -> str:
     return _hint_html("已停止辨識。可按「重新攝影機辨識」再試一次。")
 
+def _camera_hint_need_face() -> str:
+    return _hint_html("⚠️ 目前未偵測到人臉，請讓臉部出現在畫面中央、靠近一點並面向鏡頭，才可進行情緒辨識。")
 
 def _result_card_html(emo_key: str, conf: Optional[float]) -> str:
     zh = EMO_ZH.get(emo_key, emo_key)
@@ -138,7 +136,7 @@ def _result_card_html(emo_key: str, conf: Optional[float]) -> str:
         padding:20px;
         border:1px solid #E6E6E6;
         background:#FFFFFF;
-        color:#000000; 
+        color:#000000;
     ">
       <div style="font-size:14px;color:#000000;margin-bottom:8px;">
         辨識結果
@@ -164,13 +162,15 @@ def _result_card_html(emo_key: str, conf: Optional[float]) -> str:
     </div>
     """
 
-
 def _score_radio_value(score: int) -> str:
     score = int(score)
     score = min(5, max(1, score))
     return SCORE_LABELS[score]
 
 
+# =========================
+# 影像處理
+# =========================
 def _downsample_rgb(frame: np.ndarray, target_w: int) -> np.ndarray:
     if frame is None:
         return None
@@ -186,7 +186,7 @@ def _downsample_rgb(frame: np.ndarray, target_w: int) -> np.ndarray:
 def _zoom_center(frame: np.ndarray, zoom: float) -> np.ndarray:
     if frame is None or zoom <= 1.0:
         return frame
-        
+
     h, w = frame.shape[:2]
     new_w = int(w / zoom)
     new_h = int(h / zoom)
@@ -196,105 +196,84 @@ def _zoom_center(frame: np.ndarray, zoom: float) -> np.ndarray:
     return cropped
 
 
-def _iou(a: Tuple[int, int, int, int], b: Tuple[int, int, int, int]) -> float:
-    ax, ay, aw, ah = a
-    bx, by, bw, bh = b
-    ax2, ay2 = ax + aw, ay + ah
-    bx2, by2 = bx + bw, by + bh
-
-    ix1, iy1 = max(ax, bx), max(ay, by)
-    ix2, iy2 = min(ax2, bx2), min(ay2, by2)
-    iw, ih = max(0, ix2 - ix1), max(0, iy2 - iy1)
-    inter = iw * ih
-    if inter <= 0:
-        return 0.0
-    union = aw * ah + bw * bh - inter
-    return inter / max(1e-6, union)
-
-
-def _pick_face_by_tracking(
-    faces: List[Dict],
-    prev_bbox: Optional[Tuple[int, int, int, int]]
-) -> Tuple[Optional[Dict], Optional[Tuple[int, int, int, int]]]:
-    if not faces:
-        return None, None
-
-    cand = []
-    for f in faces:
-        fa = f.get("facial_area", {}) or {}
-        x = int(fa.get("x", 0) or 0)
-        y = int(fa.get("y", 0) or 0)
-        w = int(fa.get("w", 0) or 0)
-        h = int(fa.get("h", 0) or 0)
-        if w > 0 and h > 0 and f.get("face") is not None:
-            cand.append((f, (x, y, w, h), w * h))
-
-    if not cand:
-        return None, None
-
-    if prev_bbox is not None:
-        best = None
-        best_i = 0.0
-        for f, bb, area in cand:
-            i = _iou(prev_bbox, bb)
-            if i > best_i:
-                best_i = i
-                best = (f, bb)
-        if best is not None and best_i >= IOU_TH:
-            return best[0], best[1]
-
-    cand.sort(key=lambda x: x[2], reverse=True)
-    return cand[0][0], cand[0][1]
-
-
-# ✅ 新增：每一幀都畫上 bbox（如果 bbox 沒過期）
-def _draw_bbox_always(frame_rgb: np.ndarray, st: AppState, ttl_sec: float = BBOX_TTL_SEC) -> np.ndarray:
+# =========================
+# MediaPipe：偵測臉 + 畫白框/紅點
+# =========================
+def _mp_detect_draw_bbox_kp_largest(frame_rgb: np.ndarray):
+    """
+    最大臉策略：
+    - 多人入鏡：挑 bbox 面積最大者
+    回傳：
+      drawn_rgb：有臉就畫白框+紅點；沒臉回原圖
+      bbox_px： (x, y, w, h) pixel；沒臉則 None
+    """
     if frame_rgb is None:
-        return frame_rgb
+        return frame_rgb, None
 
-    if st.track_bbox is None or st.track_small_shape is None:
-        return frame_rgb
+    h, w = frame_rgb.shape[:2]
+    res = _FACE_DET.process(frame_rgb)  # MediaPipe 吃 RGB
 
-    now = time.time()
-    if st.track_bbox_time and (now - st.track_bbox_time) > ttl_sec:
-        return frame_rgb
+    if not res.detections:
+        return frame_rgb, None
+
+    best_det = None
+    best_area = -1
+    best_bbox = None
 
-    fh, fw = frame_rgb.shape[:2]
-    sh, sw = st.track_small_shape
-    if sw <= 0 or sh <= 0:
-        return frame_rgb
+    # 逐個算 bbox 面積，挑最大
+    for det in res.detections:
+        loc = det.location_data
+        rbb = loc.relative_bounding_box
+        x = int(rbb.xmin * w)
+        y = int(rbb.ymin * h)
+        bw = int(rbb.width * w)
+        bh = int(rbb.height * h)
 
-    x, y, w, h = st.track_bbox  # small 座標系
-    sx = fw / sw
-    sy = fh / sh
+        # clamp
+        x = max(0, min(w - 1, x))
+        y = max(0, min(h - 1, y))
+        bw = max(1, min(w - x, bw))
+        bh = max(1, min(h - y, bh))
 
-    x1 = int(x * sx)
-    y1 = int(y * sy)
-    x2 = int((x + w) * sx)
-    y2 = int((y + h) * sy)
+        area = bw * bh
+        if area > best_area:
+            best_area = area
+            best_det = det
+            best_bbox = (x, y, bw, bh)
 
-    x1 = max(0, min(fw - 1, x1))
-    y1 = max(0, min(fh - 1, y1))
-    x2 = max(0, min(fw - 1, x2))
-    y2 = max(0, min(fh - 1, y2))
+    if best_det is None or best_bbox is None:
+        return frame_rgb, None
+
+    x, y, bw, bh = best_bbox
+    loc = best_det.location_data
 
     bgr = cv2.cvtColor(frame_rgb, cv2.COLOR_RGB2BGR)
-    cv2.rectangle(bgr, (x1, y1), (x2, y2), (255, 255, 255), 3)  # 白框
-    return cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
+
+    # 白框
+    cv2.rectangle(bgr, (x, y), (x + bw, y + bh), (255, 255, 255), 3)
+
+    # 紅點（五官特徵點）
+    for kp in loc.relative_keypoints:
+        cx = int(kp.x * w)
+        cy = int(kp.y * h)
+        cv2.circle(bgr, (cx, cy), 6, (0, 0, 255), -1)
+
+    drawn = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
+    return drawn, (x, y, bw, bh)
 
 
 def _reset_state_for_camera(st: AppState) -> AppState:
     st.running = True
     st.finished = False
     st.last_analyze_t = 0.0
-    st.track_bbox = None
-    st.track_small_shape = None
-    st.track_bbox_time = 0.0
+
     st.ema_scores = None
     st.emo_keys = []
+
     st.stable_emo = "unknown"
     st.candidate_emo = None
     st.candidate_since = None
+
     st.final_emo = None
     st.final_conf = None
     st.final_score = 3
@@ -319,7 +298,7 @@ def on_stream(frame_rgb: np.ndarray, st: AppState):
             hint_html = _camera_hint_stopped()
 
         return (
-            gr.update(value=None, visible=cam_visible),  # ✅ cam
+            gr.update(value=None, visible=cam_visible),
             gr.update(value=result_html),
             gr.update(value=mood_val),
             gr.update(value=hint_html),
@@ -330,7 +309,7 @@ def on_stream(frame_rgb: np.ndarray, st: AppState):
     # --- no frame yet ---
     if frame_rgb is None:
         return (
-            gr.update(value=None, visible=True),  # ✅ cam
+            gr.update(value=None, visible=True),
             gr.update(value=_hint_html("等待影像…（尚未開始串流）")),
             gr.update(value=_score_radio_value(3)),
             gr.update(value=_camera_hint_before_start()),
@@ -341,42 +320,27 @@ def on_stream(frame_rgb: np.ndarray, st: AppState):
     # zoom
     frame_rgb = _zoom_center(frame_rgb, ZOOM_FACTOR)
 
-    # ✅ 每一幀先畫「上一個 bbox」，讓框一直存在
-    frame_rgb = _draw_bbox_always(frame_rgb, st, ttl_sec=BBOX_TTL_SEC)
-
-    now = time.time()
+    # 每一幀先偵測臉：有臉才畫框/紅點
+    frame_drawn, bbox_px = _mp_detect_draw_bbox_kp_largest(frame_rgb)
 
-    # --- not time to analyze: return frame with bbox ---
-    if now - st.last_analyze_t < ANALYZE_EVERY_SEC:
-        show = st.stable_emo or "unknown"
+    # 沒有人臉：不畫框/點，並提醒
+    if bbox_px is None:
         return (
-            gr.update(value=frame_rgb, visible=True),  # ✅ cam：回傳影像
-            gr.update(value=_result_card_html(show, st.final_conf)),
-            gr.update(value=_score_radio_value(EMO_TO_SCORE.get(show, 3))),
-            gr.update(value=_camera_hint_before_start()),
+            gr.update(value=frame_rgb, visible=True),  # 回原圖（無框）
+            gr.update(value=_hint_html("尚未偵測到人臉")),
+            gr.update(value=_score_radio_value(3)),
+            gr.update(value=_camera_hint_need_face()),
             st,
             gr.update(value="重新攝影機辨識"),
         )
 
-    # --- analyze tick ---
-    st.last_analyze_t = now
-
-    small = _downsample_rgb(frame_rgb, DOWNSAMPLE_W)
-
-    faces = DeepFace.extract_faces(
-        img_path=small,
-        detector_backend=DETECTOR_BACKEND,
-        enforce_detection=False,
-        align=ALIGN_FACE,
-    )
-
-    pick, bbox = _pick_face_by_tracking(faces, st.track_bbox)
+    now = time.time()
 
-    # 若偵測不到：仍回傳「上一個框」(已經畫過)，不要閃爍
-    if pick is None or bbox is None or pick.get("face") is None:
+    # 還沒到分析時間：只更新畫面（框/點一直在），不做情緒分析
+    if now - st.last_analyze_t < ANALYZE_EVERY_SEC:
         show = st.stable_emo or "unknown"
         return (
-            gr.update(value=frame_rgb, visible=True),  # ✅ cam
+            gr.update(value=frame_drawn, visible=True),
             gr.update(value=_result_card_html(show, st.final_conf)),
             gr.update(value=_score_radio_value(EMO_TO_SCORE.get(show, 3))),
             gr.update(value=_camera_hint_before_start()),
@@ -384,21 +348,23 @@ def on_stream(frame_rgb: np.ndarray, st: AppState):
             gr.update(value="重新攝影機辨識"),
         )
 
-    # ✅ 更新追蹤 bbox（small 座標系）
-    st.track_bbox = bbox
-    st.track_small_shape = small.shape[:2]
-    st.track_bbox_time = now
+    # --- analyze tick ---
+    st.last_analyze_t = now
 
-    # ✅ 立刻再畫一次最新 bbox（讓框馬上跟上）
-    frame_rgb = _draw_bbox_always(frame_rgb, st, ttl_sec=BBOX_TTL_SEC)
+    # 用 bbox 裁 ROI 做情緒辨識（可加 padding）
+    x, y, bw, bh = bbox_px
+    pad_x = int(bw * 0.10)
+    pad_y = int(bh * 0.10)
+    H, W = frame_rgb.shape[:2]
+    x1 = max(0, x - pad_x)
+    y1 = max(0, y - pad_y)
+    x2 = min(W, x + bw + pad_x)
+    y2 = min(H, y + bh + pad_y)
 
-    # ROI face for emotion
-    face = pick["face"]
-    if face.dtype != np.uint8:
-        face = np.clip(face * 255.0, 0, 255).astype(np.uint8)
+    face_roi = frame_rgb[y1:y2, x1:x2].astype(np.uint8)
 
     result = DeepFace.analyze(
-        img_path=face,
+        img_path=face_roi,
         actions=["emotion"],
         enforce_detection=False,
     )
@@ -409,7 +375,7 @@ def on_stream(frame_rgb: np.ndarray, st: AppState):
     if not isinstance(emo_dict, dict) or len(emo_dict) == 0:
         show = st.stable_emo or "unknown"
         return (
-            gr.update(value=frame_rgb, visible=True),
+            gr.update(value=frame_drawn, visible=True),
             gr.update(value=_result_card_html(show, st.final_conf)),
             gr.update(value=_score_radio_value(EMO_TO_SCORE.get(show, 3))),
             gr.update(value=_camera_hint_before_start()),
@@ -420,10 +386,11 @@ def on_stream(frame_rgb: np.ndarray, st: AppState):
     top_key = max(emo_dict, key=lambda k: float(emo_dict[k]))
     top_conf = float(emo_dict[top_key])
 
+    # 低信心就不切換，顯示目前穩定值
     if top_conf < CONF_TH:
         show = st.stable_emo or "unknown"
         return (
-            gr.update(value=frame_rgb, visible=True),
+            gr.update(value=frame_drawn, visible=True),
             gr.update(value=_result_card_html(show, st.final_conf)),
             gr.update(value=_score_radio_value(EMO_TO_SCORE.get(show, 3))),
             gr.update(value=_camera_hint_before_start()),
@@ -444,7 +411,7 @@ def on_stream(frame_rgb: np.ndarray, st: AppState):
     stable_key = st.emo_keys[stable_idx]
     st.stable_emo = stable_key
 
-    # 延遲確認
+    # 延遲確認（避免一直跳）
     if st.candidate_emo != stable_key:
         st.candidate_emo = stable_key
         st.candidate_since = now
@@ -457,6 +424,7 @@ def on_stream(frame_rgb: np.ndarray, st: AppState):
         and (now - st.candidate_since) >= SWITCH_CONFIRM_SEC
     )
 
+    # 一旦 confirmed：完成一次辨識��你原本就是這個邏輯）
     if confirmed and st.final_emo is None:
         st.final_emo = stable_key
         st.final_conf = top_conf
@@ -466,7 +434,7 @@ def on_stream(frame_rgb: np.ndarray, st: AppState):
         st.finished = True
 
         return (
-            gr.update(value=None, visible=False),  # ✅ cam hide
+            gr.update(value=None, visible=False),
             gr.update(value=_result_card_html(stable_key, top_conf)),
             gr.update(value=_score_radio_value(EMO_TO_SCORE.get(stable_key, 3))),
             gr.update(value=_camera_hint_done()),
@@ -475,7 +443,7 @@ def on_stream(frame_rgb: np.ndarray, st: AppState):
         )
 
     return (
-        gr.update(value=frame_rgb, visible=True),  # ✅ cam
+        gr.update(value=frame_drawn, visible=True),
         gr.update(value=_result_card_html(stable_key, top_conf)),
         gr.update(value=_score_radio_value(EMO_TO_SCORE.get(stable_key, 3))),
         gr.update(value=_camera_hint_before_start()),
@@ -485,13 +453,12 @@ def on_stream(frame_rgb: np.ndarray, st: AppState):
 
 
 # =========================
-# Mode / Buttons
+# Buttons
 # =========================
-
 def on_restart(st: AppState):
     st = _reset_state_for_camera(st)
     return (
-        gr.update(value=None, visible=True),                               # ✅ cam
+        gr.update(value=None, visible=True),
         gr.update(value=_hint_html("等待影像…（尚未開始串流）")),
         gr.update(value=_score_radio_value(3)),
         gr.update(value=_camera_hint_before_start()),
@@ -499,13 +466,13 @@ def on_restart(st: AppState):
         gr.update(value="重新攝影機辨識"),
     )
 
-
 def on_stop(st: AppState):
     st.running = False
     st.finished = True
+
     if st.final_emo is None:
         return (
-            gr.update(value=None, visible=False),  # ✅ cam
+            gr.update(value=None, visible=False),
             gr.update(value=_hint_html("已停止辨識。")),
             gr.update(value=_score_radio_value(3)),
             gr.update(value=_camera_hint_stopped()),
@@ -514,7 +481,7 @@ def on_stop(st: AppState):
         )
 
     return (
-        gr.update(value=None, visible=False),  # ✅ cam
+        gr.update(value=None, visible=False),
         gr.update(value=_result_card_html(st.final_emo, st.final_conf)),
         gr.update(value=_score_radio_value(EMO_TO_SCORE.get(st.final_emo, 3))),
         gr.update(value=_camera_hint_done()),
@@ -526,9 +493,6 @@ def on_stop(st: AppState):
 # =========================
 # Gradio UI
 # =========================
-
-# 你原本這裡的 css 變數在 pasted 檔裡沒顯示出來
-# 如果你原檔有 css = "...", 請保持原樣；沒有的話請自行加上或移除 css 參數
 css = ""
 
 if __name__ == "__main__":
@@ -536,7 +500,7 @@ if __name__ == "__main__":
         st = gr.State(AppState())
 
         gr.Markdown("## 情緒辨識")
-        
+
         hint = gr.HTML(_camera_hint_before_start())
 
         cam = gr.Image(