Update app.py

app.py

@@ -414,9 +414,12 @@ class AutoResponse(BaseModel):
 async def predict_auto(files: List[UploadFile] = File(...)):
     """
     Accepts 60 JPEG frames.
-    1. Runs CNN on every frame → majority vote
-    2. If vote_ratio >= 0.75 → return letter
-    3. Else → run LSTM on same frames → return phrase
+    Optimization: runs MediaPipe on every other frame (30 calls instead of 60).
+    - CNN: votes on the 30 processed frames
+    - LSTM: 60 coord vectors — processed frames get real coords,
+            in-between frames reuse the previous frame's coords
+    1. If CNN vote_ratio >= 0.75 → return letter
+    2. Else → run LSTM on 60 coords → return phrase
     """
     if not files:
         raise HTTPException(status_code=400, detail="No frames provided")
@@ -427,27 +430,55 @@ async def predict_auto(files: List[UploadFile] = File(...)):
         data = await file.read()
         frame_data.append(data)
 
-    # ── Step 2: CNN majority vote on all frames ───────────────
-    votes = {}
-
-    for data in frame_data:
-        mp_img   = bytes_to_mp_image(data)
-        result   = hand_detector_alpha.detect(mp_img)
-        hand_lms = result.hand_landmarks[0] if result.hand_landmarks else None
-
-        if hand_lms is None:
-            continue
-
-        skel   = render_hand_skeleton(hand_lms)
-        tensor = infer_transform(skel).unsqueeze(0).to(device)
+    # ── Step 2: Run MediaPipe on every other frame ────────────
+    # For each frame, store: hand_lms, pose_vec, left_vec, right_vec
+    # Odd-indexed frames reuse the previous even frame's results
 
-        with torch.no_grad():
-            probs = torch.softmax(alphabet_model(tensor), dim=1)[0]
+    votes = {}
 
-        conf, idx = probs.max(dim=0)
-        letter    = ALPHA_CLASSES[idx.item()]
-        votes[letter] = votes.get(letter, 0) + 1
+    # LSTM coord buffer — 60 entries total
+    coord_buffer        = []
+    last_pose_vec       = np.zeros(POSE_SIZE, dtype=np.float32)
+    last_left_hand_vec  = np.zeros(HAND_SIZE, dtype=np.float32)
+    last_right_hand_vec = np.zeros(HAND_SIZE, dtype=np.float32)
+
+    for i, data in enumerate(frame_data):
+        if i % 2 == 0:
+            # ── Process this frame through MediaPipe ──────────
+            mp_img = bytes_to_mp_image(data)
+
+            # CNN — hand landmarks
+            cnn_result = hand_detector_alpha.detect(mp_img)
+            hand_lms   = cnn_result.hand_landmarks[0] \
+                         if cnn_result.hand_landmarks else None
+
+            if hand_lms is not None:
+                skel   = render_hand_skeleton(hand_lms)
+                tensor = infer_transform(skel).unsqueeze(0).to(device)
+                with torch.no_grad():
+                    probs = torch.softmax(alphabet_model(tensor), dim=1)[0]
+                conf, idx = probs.max(dim=0)
+                letter = ALPHA_CLASSES[idx.item()]
+                votes[letter] = votes.get(letter, 0) + 1
+
+            # LSTM — pose + hand coords
+            pose_result = pose_detector.detect(mp_img)
+            pose_lms    = pose_result.pose_landmarks[0] \
+                          if pose_result.pose_landmarks else None
+            if pose_lms:
+                last_pose_vec = normalize_pose(pose_lms)
+
+            lstm_result          = hand_detector_phrase.detect(mp_img)
+            left_lms, right_lms  = get_hands_by_side(lstm_result)
+            if left_lms  is not None: last_left_hand_vec  = normalize_hand(left_lms)
+            if right_lms is not None: last_right_hand_vec = normalize_hand(right_lms)
+
+        # Both even and odd frames get a coord entry
+        # Odd frames reuse the last updated coords (from the previous even frame)
+        vec = np.concatenate([last_pose_vec, last_left_hand_vec, last_right_hand_vec])
+        coord_buffer.append(vec)
 
+    # ── Step 3: CNN vote decision ─────────────────────────────
     vote_ratio  = 0.0
     best_letter = ""
 
@@ -455,7 +486,6 @@ async def predict_auto(files: List[UploadFile] = File(...)):
         best_letter = max(votes, key=votes.get)
         vote_ratio  = votes[best_letter] / sum(votes.values())
 
-    # ── Step 3: Letter decision ───────────────────────────────
     if vote_ratio >= 0.75 and best_letter:
         return AutoResponse(
             result=best_letter,
@@ -465,29 +495,7 @@ async def predict_auto(files: List[UploadFile] = File(...)):
             detected=True
         )
 
-    # ── Step 4: LSTM on same frames ───────────────────────────
-    coord_buffer        = []
-    last_pose_vec       = np.zeros(POSE_SIZE,  dtype=np.float32)
-    last_left_hand_vec  = np.zeros(HAND_SIZE,  dtype=np.float32)
-    last_right_hand_vec = np.zeros(HAND_SIZE,  dtype=np.float32)
-
-    for data in frame_data:
-        mp_img = bytes_to_mp_image(data)
-
-        pose_result = pose_detector.detect(mp_img)
-        pose_lms    = pose_result.pose_landmarks[0] \
-                      if pose_result.pose_landmarks else None
-        if pose_lms:
-            last_pose_vec = normalize_pose(pose_lms)
-
-        hand_result          = hand_detector_phrase.detect(mp_img)
-        left_lms, right_lms  = get_hands_by_side(hand_result)
-        if left_lms  is not None: last_left_hand_vec  = normalize_hand(left_lms)
-        if right_lms is not None: last_right_hand_vec = normalize_hand(right_lms)
-
-        vec = np.concatenate([last_pose_vec, last_left_hand_vec, last_right_hand_vec])
-        coord_buffer.append(vec)
-
+    # ── Step 4: LSTM on 60 coords ─────────────────────────────
     if len(coord_buffer) < MIN_COORD_FRAMES:
         return AutoResponse(
             result="", result_type="phrase",
@@ -512,6 +520,10 @@ async def predict_auto(files: List[UploadFile] = File(...)):
         detected=True
     )
 
+        vote_ratio=vote_ratio,
+        detected=True
+    )
+
 
 async def predict_phrase(files: List[UploadFile] = File(...)):
     """