Update app.py

app.py

@@ -5,7 +5,6 @@ import time
 from streamlit_webrtc import webrtc_streamer, VideoTransformerBase
 import av
 from queue import Queue
-import os
 
 # MediaPipe components for gesture recognition and hand landmarks
 mp_hands = mp.solutions.hands
@@ -22,38 +21,52 @@ model_path = 'model/gesture_recognizer.task'
 if not os.path.exists(model_path):
     raise FileNotFoundError(f"Model file not found at {model_path}")
 
-# Gesture recognizer configuration
-options = GestureRecognizerOptions(
-    base_options=BaseOptions(model_asset_path=model_path),
-    running_mode=VisionRunningMode.LIVE_STREAM
-)
-
 # Custom video processor class for handling webcam input
 class VideoProcessor(VideoTransformerBase):
     def __init__(self):
-        self.recognizer = GestureRecognizer.create_from_options(options)
+        self.gesture_queue = Queue()  # Queue to store gesture results
+
+        # Configure the Gesture Recognizer with a callback
+        self.recognizer = GestureRecognizer.create_from_options(
+            GestureRecognizerOptions(
+                base_options=BaseOptions(model_asset_path=model_path),
+                running_mode=VisionRunningMode.LIVE_STREAM,
+                result_callback=self.handle_result,  # Set the result callback
+            )
+        )
         self.hands = mp_hands.Hands(
             max_num_hands=2,
             model_complexity=0,
             min_detection_confidence=0.3,
             min_tracking_confidence=0.3
         )
-        self.gesture_queue = Queue()
+
+    def handle_result(self, result: GestureRecognizerResult, output_image: mp.Image, timestamp_ms: int):
+        # Process the result and add it to the queue
+        gestures = []
+        if result.gestures:
+            for hand_gestures in result.gestures:
+                for gesture in hand_gestures:
+                    gestures.append(f"{gesture.category_name} (Confidence: {gesture.score:.2f})")
+        else:
+            gestures.append("No gestures detected.")
+        
+        self.gesture_queue.put(gestures)
 
     def recv(self, frame):
         img = frame.to_ndarray(format="bgr24")
-        img = cv2.flip(img, 1)  # Flip the image horizontally for a mirror effect
+        img = cv2.flip(img, 1)  # Flip horizontally
         img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 
         # Perform hand landmark detection
         hand_results = self.hands.process(img_rgb)
 
-        # Perform gesture recognition
+        # Perform gesture recognition asynchronously
         mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=img_rgb)
         current_time_ms = int(time.time() * 1000)
-        result = self.recognizer.recognize(mp_image, current_time_ms)
+        self.recognizer.recognize_async(mp_image, current_time_ms)
 
-        # Draw landmarks and recognized gestures on the frame
+        # Draw hand landmarks on the frame
         if hand_results.multi_hand_landmarks:
             for hand_landmarks in hand_results.multi_hand_landmarks:
                 mp_drawing.draw_landmarks(
@@ -64,12 +77,19 @@ class VideoProcessor(VideoTransformerBase):
                     mp_drawing_styles.get_default_hand_connections_style()
                 )
 
-        # Display recognized gestures
-        if result.gestures:
-            for hand_gestures in result.gestures:
-                for gesture in hand_gestures:
-                    gesture_text = f"{gesture.category_name} ({gesture.score:.2f})"
-                    cv2.putText(img, gesture_text, (10, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2)
+        # Optionally display detected gestures from the queue
+        if not self.gesture_queue.empty():
+            gestures = self.gesture_queue.get()
+            for i, gesture in enumerate(gestures):
+                cv2.putText(
+                    img,
+                    gesture,
+                    (10, 50 + i * 30),
+                    cv2.FONT_HERSHEY_SIMPLEX,
+                    1,
+                    (0, 255, 0),
+                    2,
+                )
 
         return av.VideoFrame.from_ndarray(img, format="bgr24")