Update emotion.py

emotion.py

@@ -11,42 +11,23 @@ import mediapipe as mp
 from mediapipe.tasks import python
 from mediapipe.tasks.python import vision
 
-
 from deepface import DeepFace
 
 
 # =========================
 # 參數（可微調）
 # =========================
-ANALYZE_EVERY_SEC = 0.8         # 情緒分析頻率（秒）
-DOWNSAMPLE_W = 720              # 目前保留（可不使用）
+ANALYZE_EVERY_SEC = 0.8          # 情緒分析頻率（秒）
 EMA_ALPHA = 0.45
-CONF_TH = 40.0
-SWITCH_CONFIRM_SEC = 2.5
-ZOOM_FACTOR = 1.5               # 你原本就有的 zoom
+CONF_TH = 40.0                  # DeepFace emotion confidence 門檻
+SWITCH_CONFIRM_SEC = 2.5        # 連續穩定多久才算完成
+ZOOM_FACTOR = 1.5               # 只用在「顯示」(你要保留 zoom 畫面)
 
-# DeepFace 相關（我們這版用 MediaPipe 偵測臉，再把 ROI 丟進 DeepFace.analyze）
-ALIGN_FACE = True               # 保留參數（這版 ROI 已是人臉）
-# DETECTOR_BACKEND = "mtcnn"    # 這版不再用 extract_faces，就不需要
-
-
-# =========================
-# MediaPipe Tasks Face Detector
-# =========================
-BaseOptions = python.BaseOptions
-FaceDetector = vision.FaceDetector
-FaceDetectorOptions = vision.FaceDetectorOptions
-VisionRunningMode = vision.RunningMode
-
-_face_detector = FaceDetector.create_from_options(
-    FaceDetectorOptions(
-        base_options=BaseOptions(
-            model_asset_path="blaze_face_short_range.tflite"
-        ),
-        running_mode=VisionRunningMode.IMAGE,
-    )
-)
+# MediaPipe face detector 門檻（太高很容易漏）
+MP_MIN_DET_CONF = 0.3
 
+# 模型路徑（請確認你的 repo 真的有這個檔案）
+MP_MODEL_PATH = "blaze_face_short_range.tflite"
 
 
 # =========================
@@ -63,7 +44,6 @@ EMO_ZH = {
     "unknown": "未知",
 }
 
-# 把 7 種情緒塞進 5 分心情分數
 EMO_TO_SCORE = {
     "angry": 1,
     "disgust": 1,
@@ -75,9 +55,6 @@ EMO_TO_SCORE = {
     "unknown": 3,
 }
 
-def get_emotion_score(emo: str) -> int:
-    return EMO_TO_SCORE.get(emo, 3)
-
 SCORE_LABELS = {
     1: "1（心情差）",
     2: "2（不太好）",
@@ -94,6 +71,7 @@ SCORE_CHOICES = [SCORE_LABELS[i] for i in [1, 2, 3, 4, 5]]
 @dataclass
 class AppState:
     mode: str = "攝影機辨識"
+
     running: bool = True
     finished: bool = False
     last_analyze_t: float = 0.0
@@ -114,7 +92,7 @@ class AppState:
 
 
 # =========================
-# UI HTML (強制黑色文字)
+# UI HTML
 # =========================
 def _hint_html(msg: str) -> str:
     return f"""
@@ -129,14 +107,21 @@ def _camera_hint_before_start() -> str:
         "接著，請按「錄製（Record）」開始。"
     )
 
+def _camera_hint_need_face() -> str:
+    return _hint_html(
+        "⚠️ 目前未偵測到人臉，請將臉部移至畫面中央、靠近鏡頭，並避免瀏海遮住眉眼，才可進行情緒辨識。"
+    )
+
 def _camera_hint_done() -> str:
     return _hint_html("已完成辨識，可按「重新攝影機辨識」再次進行辨識。")
 
 def _camera_hint_stopped() -> str:
     return _hint_html("已停止辨識。可按「重新攝影機辨識」再試一次。")
 
-def _camera_hint_need_face() -> str:
-    return _hint_html("⚠️ 目前未偵測到人臉，請讓臉部出現在畫面中央、靠近一點並面向鏡頭，才可進行情緒辨識。")
+def _score_radio_value(score: int) -> str:
+    score = int(score)
+    score = min(5, max(1, score))
+    return SCORE_LABELS[score]
 
 def _result_card_html(emo_key: str, conf: Optional[float]) -> str:
     zh = EMO_ZH.get(emo_key, emo_key)
@@ -176,26 +161,38 @@ def _result_card_html(emo_key: str, conf: Optional[float]) -> str:
     </div>
     """
 
-def _score_radio_value(score: int) -> str:
-    score = int(score)
-    score = min(5, max(1, score))
-    return SCORE_LABELS[score]
-
 
 # =========================
-# 影像處理
+# 影像處理（重點：保命轉換）
 # =========================
-def _downsample_rgb(frame: np.ndarray, target_w: int) -> np.ndarray:
+def _to_rgb_uint8(frame: np.ndarray) -> np.ndarray:
+    """
+    MediaPipe Tasks 最穩格式：
+    - RGB
+    - uint8
+    - 0~255
+    - contiguous
+    """
     if frame is None:
-        return None
-    h, w = frame.shape[:2]
-    if w <= target_w:
         return frame
-    scale = target_w / w
-    new_h = max(1, int(h * scale))
-    img = Image.fromarray(frame)
-    img = img.resize((target_w, new_h))
-    return np.array(img)
+
+    # RGBA -> RGB
+    if frame.ndim == 3 and frame.shape[2] == 4:
+        frame = frame[:, :, :3]
+
+    # 灰階 -> RGB
+    if frame.ndim == 2:
+        frame = np.stack([frame, frame, frame], axis=-1)
+
+    # dtype / range 修正
+    if frame.dtype != np.uint8:
+        mx = float(np.max(frame)) if frame.size else 0.0
+        if mx <= 1.5:  # 多半是 0~1
+            frame = (frame * 255.0).clip(0, 255).astype(np.uint8)
+        else:          # 多半是 0~255 但 float
+            frame = np.clip(frame, 0, 255).astype(np.uint8)
+
+    return np.ascontiguousarray(frame)
 
 def _zoom_center(frame: np.ndarray, zoom: float) -> np.ndarray:
     if frame is None or zoom <= 1.0:
@@ -207,40 +204,56 @@ def _zoom_center(frame: np.ndarray, zoom: float) -> np.ndarray:
     start_x = (w - new_w) // 2
     start_y = (h - new_h) // 2
     cropped = frame[start_y : start_y + new_h, start_x : start_x + new_w]
-    return cropped
+    return np.ascontiguousarray(cropped)
 
 
+# =========================
+# MediaPipe Tasks Face Detector
+# =========================
+BaseOptions = python.BaseOptions
+FaceDetector = vision.FaceDetector
+FaceDetectorOptions = vision.FaceDetectorOptions
+VisionRunningMode = vision.RunningMode
+
+_face_detector = FaceDetector.create_from_options(
+    FaceDetectorOptions(
+        base_options=BaseOptions(model_asset_path=MP_MODEL_PATH),
+        running_mode=VisionRunningMode.IMAGE,
+        min_detection_confidence=MP_MIN_DET_CONF,
+    )
+)
+
 def mp_tasks_detect_and_draw(frame_rgb: np.ndarray):
     """
-    使用 MediaPipe Tasks
-    - 有臉：畫白框 + 紅色 keypoints
-    - 回傳 bbox (x,y,w,h)
+    - 最大臉策略（bbox 面積最大）
+    - 有臉：畫白框 + 紅點 keypoints
+    - 回 bbox_px (x, y, w, h) 供 ROI 裁切
     """
+    frame_rgb = _to_rgb_uint8(frame_rgb)
     h, w = frame_rgb.shape[:2]
 
-    mp_image = mp.Image(
-        image_format=mp.ImageFormat.SRGB,
-        data=frame_rgb
-    )
-
+    mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=frame_rgb)
     result = _face_detector.detect(mp_image)
 
     if not result.detections:
         return frame_rgb, None
 
-    # 取「最大臉」
     best_det = None
     best_area = -1
     best_bbox = None
 
+    # 最大臉：bbox 面積最大
     for det in result.detections:
         bbox = det.bounding_box
-        area = bbox.width * bbox.height
+        area = float(bbox.width) * float(bbox.height)
         if area > best_area:
             best_area = area
             best_det = det
             best_bbox = bbox
 
+    if best_det is None or best_bbox is None:
+        return frame_rgb, None
+
     lx = int(best_bbox.origin_x)
     ly = int(best_bbox.origin_y)
     bw = int(best_bbox.width)
@@ -254,16 +267,10 @@ def mp_tasks_detect_and_draw(frame_rgb: np.ndarray):
 
     bgr = cv2.cvtColor(frame_rgb, cv2.COLOR_RGB2BGR)
 
-    # 白色框
-    cv2.rectangle(
-        bgr,
-        (lx, ly),
-        (lx + bw, ly + bh),
-        (255, 255, 255),
-        3
-    )
+    # 白框
+    cv2.rectangle(bgr, (lx, ly), (lx + bw, ly + bh), (255, 255, 255), 3)
 
-    # 紅色五官點
+    # 紅點：五官特徵點（相對座標）
     for kp in best_det.keypoints:
         cx = int(kp.x * w)
         cy = int(kp.y * h)
@@ -273,7 +280,9 @@ def mp_tasks_detect_and_draw(frame_rgb: np.ndarray):
     return drawn, (lx, ly, bw, bh)
 
 
-
+# =========================
+# state helper
+# =========================
 def _reset_state_for_camera(st: AppState) -> AppState:
     st.running = True
     st.finished = False
@@ -293,10 +302,10 @@ def _reset_state_for_camera(st: AppState) -> AppState:
 
 
 # =========================
-# Camera streaming
+# streaming callback
 # =========================
 def on_stream(frame_rgb: np.ndarray, st: AppState):
-    # --- stopped/finished ---
+    # stopped / finished
     if st.finished or (not st.running):
         cam_visible = False
 
@@ -318,7 +327,7 @@ def on_stream(frame_rgb: np.ndarray, st: AppState):
             gr.update(value="重新攝影機辨識"),
         )
 
-    # --- no frame yet ---
+    # no frame yet
     if frame_rgb is None:
         return (
             gr.update(value=None, visible=True),
@@ -329,16 +338,19 @@ def on_stream(frame_rgb: np.ndarray, st: AppState):
             gr.update(value="重新攝影機辨識"),
         )
 
-    # zoom
-    frame_rgb = _zoom_center(frame_rgb, ZOOM_FACTOR)
+    # 先整理格式（超重要：不然可能永遠偵測不到）
+    frame_rgb = _to_rgb_uint8(frame_rgb)
 
-    # 每一幀先偵測臉：有臉才畫框/紅點
+    # ✅ 偵測用原圖（穩）
     frame_drawn, bbox_px = mp_tasks_detect_and_draw(frame_rgb)
-    
-    # 沒有人臉：不畫框/點，並提醒
+
+    # ✅ 顯示用 zoom（你要保留 zoom 畫面）
+    frame_show = _zoom_center(frame_drawn if bbox_px is not None else frame_rgb, ZOOM_FACTOR)
+
+    # 沒有人臉：不畫框/點 + 提示
     if bbox_px is None:
         return (
-            gr.update(value=frame_rgb, visible=True),  # 回原圖（無框）
+            gr.update(value=frame_show, visible=True),
             gr.update(value=_hint_html("尚未偵測到人臉")),
             gr.update(value=_score_radio_value(3)),
             gr.update(value=_camera_hint_need_face()),
@@ -348,11 +360,11 @@ def on_stream(frame_rgb: np.ndarray, st: AppState):
 
     now = time.time()
 
-    # 還沒到分析時間：只更新畫面（框/點一直在），不做情緒分析
+    # 還沒到分析時間：只更新畫面（框/點會一直在），不做情緒分析
     if now - st.last_analyze_t < ANALYZE_EVERY_SEC:
         show = st.stable_emo or "unknown"
         return (
-            gr.update(value=frame_drawn, visible=True),
+            gr.update(value=frame_show, visible=True),
             gr.update(value=_result_card_html(show, st.final_conf)),
             gr.update(value=_score_radio_value(EMO_TO_SCORE.get(show, 3))),
             gr.update(value=_camera_hint_before_start()),
@@ -360,13 +372,13 @@ def on_stream(frame_rgb: np.ndarray, st: AppState):
             gr.update(value="重新攝影機辨識"),
         )
 
-    # --- analyze tick ---
+    # 到分析時間：用「原圖 bbox」裁 ROI 做情緒辨識（不要用 zoom 後的圖裁）
     st.last_analyze_t = now
 
-    # 用 bbox 裁 ROI 做情緒辨識（可加 padding）
     x, y, bw, bh = bbox_px
     pad_x = int(bw * 0.10)
     pad_y = int(bh * 0.10)
+
     H, W = frame_rgb.shape[:2]
     x1 = max(0, x - pad_x)
     y1 = max(0, y - pad_y)
@@ -375,6 +387,7 @@ def on_stream(frame_rgb: np.ndarray, st: AppState):
 
     face_roi = frame_rgb[y1:y2, x1:x2].astype(np.uint8)
 
+    # DeepFace.analyze
     result = DeepFace.analyze(
         img_path=face_roi,
         actions=["emotion"],
@@ -382,12 +395,12 @@ def on_stream(frame_rgb: np.ndarray, st: AppState):
     )
     if isinstance(result, list):
         result = result[0]
-    emo_dict = result.get("emotion", None)
 
+    emo_dict = result.get("emotion", None)
     if not isinstance(emo_dict, dict) or len(emo_dict) == 0:
         show = st.stable_emo or "unknown"
         return (
-            gr.update(value=frame_drawn, visible=True),
+            gr.update(value=frame_show, visible=True),
             gr.update(value=_result_card_html(show, st.final_conf)),
             gr.update(value=_score_radio_value(EMO_TO_SCORE.get(show, 3))),
             gr.update(value=_camera_hint_before_start()),
@@ -398,11 +411,10 @@ def on_stream(frame_rgb: np.ndarray, st: AppState):
     top_key = max(emo_dict, key=lambda k: float(emo_dict[k]))
     top_conf = float(emo_dict[top_key])
 
-    # 低信心就不切換，顯示目前穩定值
     if top_conf < CONF_TH:
         show = st.stable_emo or "unknown"
         return (
-            gr.update(value=frame_drawn, visible=True),
+            gr.update(value=frame_show, visible=True),
             gr.update(value=_result_card_html(show, st.final_conf)),
             gr.update(value=_score_radio_value(EMO_TO_SCORE.get(show, 3))),
             gr.update(value=_camera_hint_before_start()),
@@ -410,6 +422,7 @@ def on_stream(frame_rgb: np.ndarray, st: AppState):
             gr.update(value="重新攝影機辨識"),
         )
 
+    # EMA smoothing
     keys = list(emo_dict.keys())
     vec = np.array([float(emo_dict[k]) for k in keys], dtype=np.float32)
 
@@ -423,7 +436,7 @@ def on_stream(frame_rgb: np.ndarray, st: AppState):
     stable_key = st.emo_keys[stable_idx]
     st.stable_emo = stable_key
 
-    # 延遲確認（避免一直跳）
+    # 延遲確認，避免跳動
     if st.candidate_emo != stable_key:
         st.candidate_emo = stable_key
         st.candidate_since = now
@@ -436,7 +449,7 @@ def on_stream(frame_rgb: np.ndarray, st: AppState):
         and (now - st.candidate_since) >= SWITCH_CONFIRM_SEC
     )
 
-    # 一旦 confirmed：完成一次辨識（你原本就是這個邏輯）
+    # 完成一次辨識後停止（沿用你原本需求）
     if confirmed and st.final_emo is None:
         st.final_emo = stable_key
         st.final_conf = top_conf
@@ -455,7 +468,7 @@ def on_stream(frame_rgb: np.ndarray, st: AppState):
         )
 
     return (
-        gr.update(value=frame_drawn, visible=True),
+        gr.update(value=frame_show, visible=True),
         gr.update(value=_result_card_html(stable_key, top_conf)),
         gr.update(value=_score_radio_value(EMO_TO_SCORE.get(stable_key, 3))),
         gr.update(value=_camera_hint_before_start()),
@@ -512,14 +525,13 @@ if __name__ == "__main__":
         st = gr.State(AppState())
 
         gr.Markdown("## 情緒辨識")
-
         hint = gr.HTML(_camera_hint_before_start())
 
         cam = gr.Image(
             sources=["webcam"],
             streaming=True,
             type="numpy",
-            label="攝影機",
+            label="攝影機畫面",
             visible=True,
         )