Update app.py

app.py

@@ -385,7 +385,7 @@ def health():
 async def predict_alphabet(file: UploadFile = File(...)):
     data     = await file.read()
     mp_img   = bytes_to_mp_image(data)
-    result = hand_detector_alpha.detect_for_video(mp_img, 0)
+    result = hand_detector_alpha.detect_for_video(mp_img, int(0))
     hand_lms = result.hand_landmarks[0] if result.hand_landmarks else None
 
     if hand_lms is None:
@@ -414,7 +414,8 @@ async def predict_alphabet_batch(files: List[UploadFile] = File(...)):
     for i, file in enumerate(files):
         data     = await file.read()
         mp_img   = bytes_to_mp_image(data)
-        result = hand_detector_alpha.detect_for_video(mp_img, i * 33)
+        timestamp_ms = int(i * (1000 / 30))
+        result = hand_detector_alpha.detect_for_video(mp_img, timestamp_ms)
         hand_lms = result.hand_landmarks[0] if result.hand_landmarks else None
         if hand_lms is None:
             continue
@@ -436,31 +437,16 @@ async def predict_alphabet_batch(files: List[UploadFile] = File(...)):
 
 @app.post("/predict/auto", response_model=AutoResponse)
 async def predict_auto(files: List[UploadFile] = File(...)):
-    """
-    Accepts 60 JPEG frames.
-    Processes all frames (no skipping).
-    
-    Decision logic:
-    1. If hand presence < 1/3 of processed frames → not detected
-    2. If 2 hands detected in >= 1/3 of processed frames → skip CNN, go straight to LSTM
-    3. If wrist movement >= threshold → skip CNN, go straight to LSTM
-    4. If CNN vote_ratio >= 0.65 → return letter
-    5. Else → run LSTM → return phrase
-    """
+
     if not files:
         raise HTTPException(status_code=400, detail="No frames provided")
 
-    frame_data = []
-    for file in files:
-        data = await file.read()
-        frame_data.append(data)
+    frame_data = [await f.read() for f in files]
 
     votes                  = {}
     total_processed_frames = len(frame_data)
     total_hand_frames      = 0
     two_hand_frames        = 0
-
-    # Movement tracking — store wrist position of each processed frame
     wrist_positions        = []
 
     coord_buffer        = []
@@ -468,40 +454,54 @@ async def predict_auto(files: List[UploadFile] = File(...)):
     last_left_hand_vec  = np.zeros(HAND_SIZE, dtype=np.float32)
     last_right_hand_vec = np.zeros(HAND_SIZE, dtype=np.float32)
 
+    # ===============================
+    # Frame Processing Loop
+    # ===============================
     for i, data in enumerate(frame_data):
+
+        # ✅ FIXED timestamp
         timestamp_ms = int(i * (1000 / 30))
+
         mp_img = bytes_to_mp_image(data)
 
-        # CNN — single hand for alphabet
+        # ── CNN (alphabet) ──
         cnn_result = hand_detector_alpha.detect_for_video(mp_img, timestamp_ms)
-        hand_lms   = cnn_result.hand_landmarks[0] \
-                     if cnn_result.hand_landmarks else None
+        hand_lms   = cnn_result.hand_landmarks[0] if cnn_result.hand_landmarks else None
 
         if hand_lms is not None:
             total_hand_frames += 1
-            # Track wrist position for movement detection
+
+            # Track wrist position
             wrist_positions.append((hand_lms[WRIST].x, hand_lms[WRIST].y))
 
             skel   = render_hand_skeleton(hand_lms)
             tensor = infer_transform(skel).unsqueeze(0).to(device)
+
             with torch.no_grad():
                 probs = torch.softmax(alphabet_model(tensor), dim=1)[0]
+
             conf, idx = probs.max(dim=0)
-            if conf.item() >= 0.6:  # only count confident predictions
+
+            if conf.item() >= 0.6:
                 letter = ALPHA_CLASSES[idx.item()]
                 votes[letter] = votes.get(letter, 0) + 1
 
-        # LSTM — pose + both hands
+        # ── Pose ──
         pose_result = pose_detector.detect_for_video(mp_img, timestamp_ms)
-        pose_lms    = pose_result.pose_landmarks[0] \
-                      if pose_result.pose_landmarks else None
+        pose_lms    = pose_result.pose_landmarks[0] if pose_result.pose_landmarks else None
+
         if pose_lms:
             last_pose_vec = normalize_pose(pose_lms)
 
+        # ── Hands (LSTM) ──
         lstm_result = hand_detector_phrase.detect_for_video(mp_img, timestamp_ms)
-        left_lms, right_lms  = get_hands_by_side(lstm_result)
-        if left_lms  is not None: last_left_hand_vec  = normalize_hand(left_lms)
-        if right_lms is not None: last_right_hand_vec = normalize_hand(right_lms)
+        left_lms, right_lms = get_hands_by_side(lstm_result)
+
+        if left_lms is not None:
+            last_left_hand_vec = normalize_hand(left_lms)
+
+        if right_lms is not None:
+            last_right_hand_vec = normalize_hand(right_lms)
 
         if left_lms is not None and right_lms is not None:
             two_hand_frames += 1
@@ -509,43 +509,40 @@ async def predict_auto(files: List[UploadFile] = File(...)):
         vec = np.concatenate([last_pose_vec, last_left_hand_vec, last_right_hand_vec])
         coord_buffer.append(vec)
 
-    # ── Step 1: Minimum hand presence check ──────────────────
+    # ===============================
+    # Step 1: Hand Presence Check
+    # ===============================
     min_required = total_processed_frames * MIN_HAND_PRESENCE_RATIO
+
     if total_hand_frames < min_required:
-        print(f"[auto] Hand presence too low: {total_hand_frames}/{total_processed_frames} "
-              f"(need >= {min_required:.1f}) — skipping")
         return AutoResponse(
             result="", result_type="none",
             confidence=0.0, vote_ratio=0.0, detected=False
         )
 
-    # ── Step 2: Two hands detected → LSTM directly ───────────
-    two_hand_threshold = total_processed_frames * TWO_HAND_RATIO
-    if two_hand_frames >= two_hand_threshold:
-        print(f"[auto] Two hands in {two_hand_frames}/{total_processed_frames} frames → LSTM")
+    # ===============================
+    # Step 2: Two Hands → LSTM
+    # ===============================
+    if two_hand_frames >= total_processed_frames * TWO_HAND_RATIO:
         return run_lstm(coord_buffer, vote_ratio=0.0)
 
-    # ── Step 3: Obvious movement detected → LSTM directly ────
-    # Compare wrist position from midpoint to end of sequence.
-    # Ignores initial arm raising to position the hand.
-    if len(wrist_positions) >= 4:
-        mid_idx   = len(wrist_positions) // 2
-        mid_pos   = wrist_positions[mid_idx]
-        end_pos   = wrist_positions[-1]
-        dx        = end_pos[0] - mid_pos[0]
-        dy        = end_pos[1] - mid_pos[1]
-        movement  = (dx**2 + dy**2) ** 0.5
+    # ===============================
+    # Step 3: Movement Detection (FIXED)
+    # ===============================
+    movement = 0.0
 
-        print(f"[auto] Wrist movement (mid→end): {movement:.4f}")
+    if len(wrist_positions) >= 2:
+        for j in range(1, len(wrist_positions)):
+            dx = wrist_positions[j][0] - wrist_positions[j-1][0]
+            dy = wrist_positions[j][1] - wrist_positions[j-1][1]
+            movement += (dx**2 + dy**2) ** 0.5
 
-        if movement >= MOVEMENT_THRESHOLD:
-            print(f"[auto] Significant movement ({movement:.4f}) → LSTM")
-            return run_lstm(coord_buffer, vote_ratio=0.0)
-
-    # ── Step 4: CNN vote decision ─────────────────────────────
-    vote_ratio  = 0.0
-    best_letter = ""
+        # ✅ Normalize (important)
+        movement /= len(wrist_positions)
 
+    # ===============================
+    # Step 4: CNN Voting
+    # ===============================
     if not votes:
         return AutoResponse(
             result="", result_type="none",
@@ -555,7 +552,13 @@ async def predict_auto(files: List[UploadFile] = File(...)):
     best_letter = max(votes, key=votes.get)
     vote_ratio  = votes[best_letter] / sum(votes.values())
 
-    if vote_ratio >= 0.65 and best_letter:
+    # ===============================
+    # Step 5: Smart Decision Logic
+    # ===============================
+    if movement >= MOVEMENT_THRESHOLD and vote_ratio < 0.75:
+        return run_lstm(coord_buffer, vote_ratio)
+
+    if vote_ratio >= 0.65:
         return AutoResponse(
             result=best_letter,
             result_type="letter",
@@ -564,7 +567,9 @@ async def predict_auto(files: List[UploadFile] = File(...)):
             detected=True
         )
 
-    # ── Step 5: LSTM ─────────────────────────────────────────
+    # ===============================
+    # Step 6: Default → LSTM
+    # ===============================
     return run_lstm(coord_buffer, vote_ratio)