Update app.py

app.py

@@ -209,12 +209,13 @@
 
 
 
+
+
 import streamlit as st
 import mediapipe as mp
 import cv2
+import numpy as np
 import os
-import time
-from queue import Queue
 from utils import display_gesture_chart
 
 # Initialize MediaPipe Hands for hand landmark detection
@@ -236,27 +237,6 @@ model_path = 'gesture_recognizer.task'
 if not os.path.exists(model_path):
     raise FileNotFoundError(f"Model file not found at {model_path}")
 
-# Queue to share gesture results between the callback and the main thread
-gesture_queue = Queue()
-
-# Callback function to process gesture results and add them to the queue
-def print_result(result: GestureRecognizerResult, output_image: mp.Image, timestamp_ms: int):
-    results = []
-    if result.gestures:
-        for hand_gestures in result.gestures:
-            for gesture in hand_gestures:
-                results.append(f"{gesture.category_name} (Confidence: {gesture.score:.2f})")
-    else:
-        results.append("No gestures detected.")
-    gesture_queue.put(results)
-
-# Configure the Gesture Recognizer options
-options = GestureRecognizerOptions(
-    base_options=BaseOptions(model_asset_path=model_path),
-    running_mode=VisionRunningMode.LIVE_STREAM,
-    result_callback=print_result
-)
-
 # Initialize session state for saving recognized gestures
 if "recognized_gestures" not in st.session_state:
     st.session_state.recognized_gestures = []
@@ -282,104 +262,40 @@ st.sidebar.markdown("<hr>", unsafe_allow_html=True)
 gesture_chart_path = "./gestureReference.png"  # Update this with the actual path to the image
 display_gesture_chart(gesture_chart_path)
 
-# User-configurable options
-max_num_hands = st.sidebar.slider("Max Number of Hands", 1, 2, 1)
-skip_frames = st.sidebar.slider("Process Every Nth Frame", 1, 10, 5)
-resolution = st.sidebar.selectbox("Frame Resolution", ["320x240", "640x480"], index=0)
-
-st.sidebar.markdown("<hr>", unsafe_allow_html=True)
-
-# Start and Stop buttons
-if "run_app" not in st.session_state:
-    st.session_state.run_app = False
-
-col1, col2 = st.sidebar.columns(2)
-if col1.button("▶ Start"):
-    st.session_state.run_app = True
-
-if col2.button("⏹ Stop"):
-    st.session_state.run_app = False
-
 # Clear gesture history button
 if st.sidebar.button("🚲 Clear History"):
     st.session_state.recognized_gestures = []
 
-# Layout with columns: Live camera feed on the left, gesture log box on the right
+# Layout with columns: Webcam input on the left, gesture log box on the right
 col_feed, col_log = st.columns([5, 2])
 
 with col_feed:
-    st.markdown("### Live Camera Feed")
-    video_placeholder = st.empty()  # Placeholder for displaying the video feed
-
-with col_log:
-    st.markdown("### Gesture Log")
-    current_gesture_box = st.empty()  # Box for the most recent gesture
-    st.markdown("### Gesture History")
-    gesture_history_box = st.empty()  # Box for gesture history
-
-# Footer with branding
-st.sidebar.markdown(
-    """
-    <style>
-    .footer {text-align: center; font-size: 12px; color: grey; margin-top: 20px;}
-    </style>
-    <p class="footer">Made by Marco Chen, William Taka, Rigoberto Ponce using Streamlit, MediaPipe & OpenCV</p>
-    """,
-    unsafe_allow_html=True,
-)
-
-if st.session_state.run_app:
-    # Open a video capture device (webcam)
-    cap = cv2.VideoCapture(0)
-
-    # Parse resolution to width and height
-    res_width, res_height = map(int, resolution.split("x"))
-
-    # Start a timestamp for gesture recognition
-    start_time = time.time()
-
-    # Initialize MediaPipe components
-    with GestureRecognizer.create_from_options(options) as recognizer, mp_hands.Hands(
-        max_num_hands=max_num_hands,
-        model_complexity=1,  # Use simplified model for better performance
-        min_detection_confidence=0.5, 
-        min_tracking_confidence=0.5
-    ) as hands:
-        frame_count = 0  # Counter to track frames processed
-
-        while st.session_state.run_app and cap.isOpened():
-            success, frame = cap.read()
-            if not success:
-                st.error("Unable to access the camera. Please check your camera connection.")
-                st.session_state.run_app = False
-                break
-
-            frame_count += 1
-
-            # Skip frames based on the user-configured interval
-            if frame_count % skip_frames != 0:
-                continue
-
-            # Flip the frame horizontally for a mirror-like view
-            frame = cv2.flip(frame, 1)
-
-            # Resize the frame to the selected resolution
-            frame = cv2.resize(frame, (res_width, res_height))
-
-            # Convert the frame to RGB for processing by MediaPipe
-            frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
-
+    st.markdown("### Webcam Input")
+    # Use st.camera_input() to capture an image from the browser-based webcam
+    image_data = st.camera_input("Take a picture using your webcam")
+
+    if image_data:
+        # Read the image as a numpy array
+        file_bytes = np.asarray(bytearray(image_data.read()), dtype=np.uint8)
+        frame = cv2.imdecode(file_bytes, 1)
+        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+
+        # Configure Gesture Recognizer
+        options = GestureRecognizerOptions(
+            base_options=BaseOptions(model_asset_path=model_path),
+            running_mode=VisionRunningMode.IMAGE  # Use IMAGE mode for single-frame processing
+        )
+
+        # Initialize MediaPipe Gesture Recognizer
+        with GestureRecognizer.create_from_options(options) as recognizer, mp_hands.Hands(
+            model_complexity=1,
+            min_detection_confidence=0.5,
+            min_tracking_confidence=0.5,
+        ) as hands:
             # Perform hand landmark detection
             hand_results = hands.process(frame_rgb)
 
-            # Prepare frame for gesture recognition
-            mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=frame)
-            current_time_ms = int((time.time() - start_time) * 1000)
-
-            # Perform asynchronous gesture recognition
-            recognizer.recognize_async(mp_image, current_time_ms)
-
-            # Draw hand landmarks on the frame
+            # Recognize gestures if landmarks are detected
             if hand_results.multi_hand_landmarks:
                 for hand_landmarks in hand_results.multi_hand_landmarks:
                     mp_drawing.draw_landmarks(
@@ -389,45 +305,41 @@ if st.session_state.run_app:
                         mp_drawing_styles.get_default_hand_landmarks_style(),
                         mp_drawing_styles.get_default_hand_connections_style(),
                     )
+                st.image(frame, channels="BGR", caption="Processed Image with Landmarks")
+
+                # Prepare frame for gesture recognition
+                mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=frame_rgb)
+                recognizer_result = recognizer.recognize(mp_image)
+
+                if recognizer_result.gestures:
+                    for hand_gestures in recognizer_result.gestures:
+                        for gesture in hand_gestures:
+                            label = gesture.category_name
+                            confidence = f"{gesture.score:.2f}"
 
-            # Process gesture results from the queue
-            while not gesture_queue.empty():
-                results = gesture_queue.get()
-                if results:
-                    new_gesture = results[-1]
-
-                    # Extract gesture label and confidence
-                    if " (Confidence: " in new_gesture:
-                        label, confidence = new_gesture.split(" (Confidence: ")
-                        confidence = confidence.rstrip(")")
-                    else:
-                        label = new_gesture
-                        confidence = "N/A"
-
-                    # Add gesture to history if not already logged
-                    if label.isalpha() and new_gesture not in st.session_state.recognized_gestures:
-                        st.session_state.recognized_gestures.append(new_gesture)
-
-                        # Update current gesture display
-                        current_gesture_box.markdown(
-                            f"<h4 style='text-align: center; color: #4CAF50;'>Gesture: {label}<br>Confidence: {confidence}</h4>",
-                            unsafe_allow_html=True,
-                        )
-
-                        # Update gesture history display
-                        gesture_history_box.text_area(
-                            "Gesture History:",
-                            value="\n".join(reversed(st.session_state.recognized_gestures)),
-                            height=300,
-                            disabled=True,
-                        )
-
-            # Display the processed frame with landmarks and gesture details
-            video_placeholder.image(frame, channels="BGR", caption="Gesture & Hand Landmark Detection", use_column_width=True)
-
-    # Release the video capture resource
-    cap.release()
+                            # Add gesture to history
+                            if label not in st.session_state.recognized_gestures:
+                                st.session_state.recognized_gestures.append(f"{label} (Confidence: {confidence})")
 
+with col_log:
+    st.markdown("### Gesture History")
+    gesture_history_box = st.text_area(
+        "Recognized Gestures:",
+        value="\n".join(reversed(st.session_state.recognized_gestures)),
+        height=300,
+        disabled=True,
+    )
+
+# Footer with branding
+st.sidebar.markdown(
+    """
+    <style>
+    .footer {text-align: center; font-size: 12px; color: grey; margin-top: 20px;}
+    </style>
+    <p class="footer">Made by Marco Chen, William Taka, Rigoberto Ponce using Streamlit, MediaPipe & OpenCV</p>
+    """,
+    unsafe_allow_html=True,
+)