Update app.py

app.py

@@ -2,26 +2,15 @@ import gradio as gr
 import cv2
 import mediapipe as mp
 import tempfile
+from micro_gestures import *
+from composite_gestures import detect_composite_gesture
 
-# initialize mediapipe modules
 mp_hands = mp.solutions.hands
 mp_pose = mp.solutions.pose
 mp_drawing = mp.solutions.drawing_utils
 
-hands = mp_hands.Hands(
-    static_image_mode=False,
-    max_num_hands=2,
-    min_detection_confidence=0.5,
-    min_tracking_confidence=0.5
-)
-
-pose = mp_pose.Pose(
-    static_image_mode=False,
-    model_complexity=1,
-    enable_segmentation=False,
-    min_detection_confidence=0.5,
-    min_tracking_confidence=0.5
-)
+hands = mp_hands.Hands(static_image_mode=False,max_num_hands=2,min_detection_confidence=0.5,min_tracking_confidence=0.5)
+pose = mp_pose.Pose(static_image_mode=False,model_complexity=1,enable_segmentation=False,min_detection_confidence=0.5,min_tracking_confidence=0.5)
 
 def process_video(video_path, target_width=640):
     cap = cv2.VideoCapture(video_path)
@@ -29,7 +18,7 @@ def process_video(video_path, target_width=640):
     temp_output = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4")
     out = None
 
-    last_label = "None"  # store last detected gesture label
+    sequence_buffer = []
 
     while cap.isOpened():
         ret, frame = cap.read()
@@ -37,77 +26,64 @@ def process_video(video_path, target_width=640):
             break
 
         h0, w0 = frame.shape[:2]
-        # resize frame keeping aspect ratio
         scale = target_width / float(w0)
         target_height = int(round(h0 * scale))
         frame_small = cv2.resize(frame, (target_width, target_height), interpolation=cv2.INTER_AREA)
-
-        # convert to rgb
         rgb_small = cv2.cvtColor(frame_small, cv2.COLOR_BGR2RGB)
 
-        # hand detection
         hand_results = hands.process(rgb_small)
+        micro_label = None
         if hand_results.multi_hand_landmarks:
             for hand_landmarks in hand_results.multi_hand_landmarks:
-                mp_drawing.draw_landmarks(
-                    frame_small,
-                    hand_landmarks,
-                    mp_hands.HAND_CONNECTIONS,
-                    mp_drawing.DrawingSpec(color=(0,0,255), thickness=1, circle_radius=1),
-                    mp_drawing.DrawingSpec(color=(0,255,0), thickness=1, circle_radius=1)
-                )
-
-        # pose detection
-        pose_results = pose.process(rgb_small)
-        if pose_results.pose_landmarks:
-            mp_drawing.draw_landmarks(
-                frame_small,
-                pose_results.pose_landmarks,
-                mp_pose.POSE_CONNECTIONS,
-                mp_drawing.DrawingSpec(color=(255,0,0), thickness=1, circle_radius=1),
-                mp_drawing.DrawingSpec(color=(0,255,255), thickness=1, circle_radius=1)
-            )
-
-        # here you would detect gesture label (example placeholder)
-        # last_label = detect_composite_gesture([...])
-        last_label = "example_label"
-
-        # initialize video writer
+                mp_drawing.draw_landmarks(frame_small, hand_landmarks, mp_hands.HAND_CONNECTIONS)
+                # detect micro-gestures
+                if fist_closed(hand_landmarks.landmark):
+                    micro_label = "fist_closed"
+                elif palm_open(hand_landmarks.landmark):
+                    micro_label = "palm_open"
+                elif index_pointing_up(hand_landmarks.landmark):
+                    micro_label = "index_up"
+                elif thumb_up(hand_landmarks.landmark):
+                    micro_label = "thumb_up"
+                # add label to buffer
+                if micro_label:
+                    sequence_buffer.append(micro_label)
+                    if len(sequence_buffer) > 5:
+                        sequence_buffer.pop(0)
+
+        # detect composite gesture
+        composite_label = detect_composite_gesture(sequence_buffer)
+        
         if out is None:
             fps = cap.get(cv2.CAP_PROP_FPS)
             if fps <= 0 or fps > 120:
                 fps = 30
-            
-            out = cv2.VideoWriter(
-                temp_output.name,
-                fourcc,
-                fps,
-                (frame_small.shape[1], frame_small.shape[0])
-            )
+            out = cv2.VideoWriter(temp_output.name, fourcc, fps, (frame_small.shape[1], frame_small.shape[0]))
 
-        # write processed frame
-        out.write(frame_small)
+        # write label under frame
+        if composite_label:
+            # create separate label image
+            label_img = cv2.putText(frame_small.copy(), composite_label, (10, target_height-10),
+                        cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), 2, cv2.LINE_AA)
+            out.write(label_img)
+        else:
+            out.write(frame_small)
 
     cap.release()
     if out:
         out.release()
 
-    # return both video path and last label for gradio
-    return temp_output.name, last_label
+    return temp_output.name
 
-# gradio interface
 iface = gr.Interface(
     fn=process_video,
     inputs=[
         gr.Video(label="Upload or Record Video"),
         gr.Slider(minimum=160, maximum=1280, value=640, step=20, label="Processing Width")
     ],
-    outputs=[
-        gr.Video(label="Processed Video with Landmarks"),
-        gr.Textbox(label="Detected Gesture", interactive=False)
-    ],
+    outputs=gr.Video(label="Processed Video with Gestures"),
     title="Hand & Body Pose Detection",
-    description="Upload or record a video, and see MediaPipe detect hand and body landmarks with connections."
+    description="Upload or record a video, see MediaPipe detect hand landmarks and composite gestures."
 )
 
 if __name__ == "__main__":