Update app.py

app.py

@@ -57,6 +57,9 @@ MIN_HAND_PRESENCE_RATIO = 1 / 3
 # Fraction of processed frames with 2 hands to trigger automatic LSTM
 TWO_HAND_RATIO = 1 / 3
 
+# Movement threshold for wrist position change (normalized coordinates)
+MOVEMENT_THRESHOLD = 0.03  # ~3% of frame width
+
 # ============================================================
 # Model Architectures
 # ============================================================
@@ -196,10 +199,10 @@ download_if_missing(
     'pose_landmarker.task'
 )
 
-# IMAGE mode detectors
+# VIDEO mode detectors
 hand_options_alphabet = mp_vision.HandLandmarkerOptions(
     base_options=mp_python.BaseOptions(model_asset_path='hand_landmarker.task'),
-    running_mode=mp_vision.RunningMode.IMAGE,
+    running_mode=mp_vision.RunningMode.VIDEO,
     num_hands=1,
     min_hand_detection_confidence=0.7,
     min_hand_presence_confidence=0.6,
@@ -207,7 +210,7 @@ hand_options_alphabet = mp_vision.HandLandmarkerOptions(
 )
 hand_options_phrase = mp_vision.HandLandmarkerOptions(
     base_options=mp_python.BaseOptions(model_asset_path='hand_landmarker.task'),
-    running_mode=mp_vision.RunningMode.IMAGE,
+    running_mode=mp_vision.RunningMode.VIDEO,
     num_hands=2,
     min_hand_detection_confidence=0.5,
     min_hand_presence_confidence=0.5,
@@ -215,7 +218,7 @@ hand_options_phrase = mp_vision.HandLandmarkerOptions(
 )
 pose_options = mp_vision.PoseLandmarkerOptions(
     base_options=mp_python.BaseOptions(model_asset_path='pose_landmarker.task'),
-    running_mode=mp_vision.RunningMode.IMAGE,
+    running_mode=mp_vision.RunningMode.VIDEO,
     num_poses=1,
     min_pose_detection_confidence=0.3,
     min_pose_presence_confidence=0.3,
@@ -244,7 +247,7 @@ def bytes_to_mp_image(data: bytes):
     arr   = np.frombuffer(data, np.uint8)
     frame = cv2.imdecode(arr, cv2.IMREAD_COLOR)
     frame = cv2.flip(frame, 1)
-    #frame = cv2.convertScaleAbs(frame, alpha=1.3, beta=20)
+    frame = cv2.convertScaleAbs(frame, alpha=1.3, beta=20)
     rgb   = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
     return mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb)
 
@@ -382,7 +385,7 @@ def health():
 async def predict_alphabet(file: UploadFile = File(...)):
     data     = await file.read()
     mp_img   = bytes_to_mp_image(data)
-    result   = hand_detector_alpha.detect(mp_img)
+    result = hand_detector_alpha.detect_for_video(mp_img, 0)
     hand_lms = result.hand_landmarks[0] if result.hand_landmarks else None
 
     if hand_lms is None:
@@ -408,10 +411,10 @@ async def predict_alphabet_batch(files: List[UploadFile] = File(...)):
         raise HTTPException(status_code=400, detail="No frames provided")
 
     votes = {}
-    for file in files:
+    for i, file in enumerate(files):
         data     = await file.read()
         mp_img   = bytes_to_mp_image(data)
-        result   = hand_detector_alpha.detect(mp_img)
+        result = hand_detector_alpha.detect_for_video(mp_img, i * 33)
         hand_lms = result.hand_landmarks[0] if result.hand_landmarks else None
         if hand_lms is None:
             continue
@@ -435,13 +438,14 @@ async def predict_alphabet_batch(files: List[UploadFile] = File(...)):
 async def predict_auto(files: List[UploadFile] = File(...)):
     """
     Accepts 60 JPEG frames.
-    Runs MediaPipe on every other frame (30 calls).
+    Processes all frames (no skipping).
     
     Decision logic:
     1. If hand presence < 1/3 of processed frames → not detected
     2. If 2 hands detected in >= 1/3 of processed frames → skip CNN, go straight to LSTM
-    3. If CNN vote_ratio >= 0.50 → return letter
-    4. Else → run LSTM → return phrase
+    3. If wrist movement >= threshold → skip CNN, go straight to LSTM
+    4. If CNN vote_ratio >= 0.65 → return letter
+    5. Else → run LSTM → return phrase
     """
     if not files:
         raise HTTPException(status_code=400, detail="No frames provided")
@@ -452,7 +456,7 @@ async def predict_auto(files: List[UploadFile] = File(...)):
         frame_data.append(data)
 
     votes                  = {}
-    total_processed_frames = 0
+    total_processed_frames = len(frame_data)
     total_hand_frames      = 0
     two_hand_frames        = 0
 
@@ -465,43 +469,42 @@ async def predict_auto(files: List[UploadFile] = File(...)):
     last_right_hand_vec = np.zeros(HAND_SIZE, dtype=np.float32)
 
     for i, data in enumerate(frame_data):
-        if i % 2 == 0:
-            total_processed_frames += 1
-            mp_img = bytes_to_mp_image(data)
-
-            # CNN — single hand for alphabet
-            cnn_result = hand_detector_alpha.detect(mp_img)
-            hand_lms   = cnn_result.hand_landmarks[0] \
-                         if cnn_result.hand_landmarks else None
-
-            if hand_lms is not None:
-                total_hand_frames += 1
-                # Track wrist position for movement detection
-                wrist_positions.append((hand_lms[WRIST].x, hand_lms[WRIST].y))
-
-                skel   = render_hand_skeleton(hand_lms)
-                tensor = infer_transform(skel).unsqueeze(0).to(device)
-                with torch.no_grad():
-                    probs = torch.softmax(alphabet_model(tensor), dim=1)[0]
-                conf, idx = probs.max(dim=0)
-                if conf.item() >= 0.6:  # only count confident predictions
-                    letter = ALPHA_CLASSES[idx.item()]
-                    votes[letter] = votes.get(letter, 0) + 1
-
-            # LSTM — pose + both hands
-            pose_result = pose_detector.detect(mp_img)
-            pose_lms    = pose_result.pose_landmarks[0] \
-                          if pose_result.pose_landmarks else None
-            if pose_lms:
-                last_pose_vec = normalize_pose(pose_lms)
-
-            lstm_result          = hand_detector_phrase.detect(mp_img)
-            left_lms, right_lms  = get_hands_by_side(lstm_result)
-            if left_lms  is not None: last_left_hand_vec  = normalize_hand(left_lms)
-            if right_lms is not None: last_right_hand_vec = normalize_hand(right_lms)
-
-            if left_lms is not None and right_lms is not None:
-                two_hand_frames += 1
+        timestamp_ms = int(i * (1000 / 30))
+        mp_img = bytes_to_mp_image(data)
+
+        # CNN — single hand for alphabet
+        cnn_result = hand_detector_alpha.detect_for_video(mp_img, timestamp_ms)
+        hand_lms   = cnn_result.hand_landmarks[0] \
+                     if cnn_result.hand_landmarks else None
+
+        if hand_lms is not None:
+            total_hand_frames += 1
+            # Track wrist position for movement detection
+            wrist_positions.append((hand_lms[WRIST].x, hand_lms[WRIST].y))
+
+            skel   = render_hand_skeleton(hand_lms)
+            tensor = infer_transform(skel).unsqueeze(0).to(device)
+            with torch.no_grad():
+                probs = torch.softmax(alphabet_model(tensor), dim=1)[0]
+            conf, idx = probs.max(dim=0)
+            if conf.item() >= 0.6:  # only count confident predictions
+                letter = ALPHA_CLASSES[idx.item()]
+                votes[letter] = votes.get(letter, 0) + 1
+
+        # LSTM — pose + both hands
+        pose_result = pose_detector.detect_for_video(mp_img, timestamp_ms)
+        pose_lms    = pose_result.pose_landmarks[0] \
+                      if pose_result.pose_landmarks else None
+        if pose_lms:
+            last_pose_vec = normalize_pose(pose_lms)
+
+        lstm_result = hand_detector_phrase.detect_for_video(mp_img, timestamp_ms)
+        left_lms, right_lms  = get_hands_by_side(lstm_result)
+        if left_lms  is not None: last_left_hand_vec  = normalize_hand(left_lms)
+        if right_lms is not None: last_right_hand_vec = normalize_hand(right_lms)
+
+        if left_lms is not None and right_lms is not None:
+            two_hand_frames += 1
 
         vec = np.concatenate([last_pose_vec, last_left_hand_vec, last_right_hand_vec])
         coord_buffer.append(vec)
@@ -535,7 +538,6 @@ async def predict_auto(files: List[UploadFile] = File(...)):
 
         print(f"[auto] Wrist movement (mid→end): {movement:.4f}")
 
-        MOVEMENT_THRESHOLD = 0.70  # ~20% of frame width, tunable
         if movement >= MOVEMENT_THRESHOLD:
             print(f"[auto] Significant movement ({movement:.4f}) → LSTM")
             return run_lstm(coord_buffer, vote_ratio=0.0)
@@ -562,7 +564,7 @@ async def predict_auto(files: List[UploadFile] = File(...)):
             detected=True
         )
 
-    # ── Step 4: LSTM ─────────────────────────────────────────
+    # ── Step 5: LSTM ─────────────────────────────────────────
     return run_lstm(coord_buffer, vote_ratio)
 
 
@@ -576,17 +578,19 @@ async def predict_phrase(files: List[UploadFile] = File(...)):
     last_left_hand_vec  = np.zeros(HAND_SIZE,  dtype=np.float32)
     last_right_hand_vec = np.zeros(HAND_SIZE,  dtype=np.float32)
 
-    for file in files:
+    for i, file in enumerate(files):
         data   = await file.read()
         mp_img = bytes_to_mp_image(data)
-
-        pose_result = pose_detector.detect(mp_img)
+        
+        timestamp_ms = int(i * (1000 / 30))
+        
+        pose_result = pose_detector.detect_for_video(mp_img, timestamp_ms)
         pose_lms    = pose_result.pose_landmarks[0] \
                       if pose_result.pose_landmarks else None
         if pose_lms:
             last_pose_vec = normalize_pose(pose_lms)
 
-        hand_result          = hand_detector_phrase.detect(mp_img)
+        hand_result = hand_detector_phrase.detect_for_video(mp_img, timestamp_ms)
         left_lms, right_lms  = get_hands_by_side(hand_result)
         if left_lms  is not None: last_left_hand_vec  = normalize_hand(left_lms)
         if right_lms is not None: last_right_hand_vec = normalize_hand(right_lms)