RP-Azul commited on
Commit
c75c910
·
verified ·
1 Parent(s): c607c40

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +60 -164
app.py CHANGED
@@ -1,199 +1,95 @@
1
  import streamlit as st
2
  import mediapipe as mp
3
  import cv2
4
- import os
5
  import time
 
 
6
  from queue import Queue
7
 
8
- # Initialize MediaPipe Hands for hand landmark detection
9
  mp_hands = mp.solutions.hands
10
  mp_drawing = mp.solutions.drawing_utils
11
  mp_drawing_styles = mp.solutions.drawing_styles
12
-
13
- # Import necessary components from MediaPipe Gesture Recognizer
14
  BaseOptions = mp.tasks.BaseOptions
15
  GestureRecognizer = mp.tasks.vision.GestureRecognizer
16
  GestureRecognizerOptions = mp.tasks.vision.GestureRecognizerOptions
17
  GestureRecognizerResult = mp.tasks.vision.GestureRecognizerResult
18
  VisionRunningMode = mp.tasks.vision.RunningMode
19
 
20
- # Correct path to the Gesture Recognizer model file
21
- #model_path = './model/model/gesture_recognizer.task'
22
  model_path = 'model/gesture_recognizer.task'
23
- # Check if file exists
24
  if not os.path.exists(model_path):
25
  raise FileNotFoundError(f"Model file not found at {model_path}")
26
 
27
- # Queue to share results between the callback and main thread
28
- gesture_queue = Queue()
29
-
30
- # Callback function to process gesture results and add them to the queue
31
- def print_result(result: GestureRecognizerResult, output_image: mp.Image, timestamp_ms: int):
32
- results = [] # Collect gesture results
33
- if result.gestures:
34
- for hand_gestures in result.gestures:
35
- for gesture in hand_gestures:
36
- results.append(f"{gesture.category_name} (Confidence: {gesture.score:.2f})") # Include confidence
37
- else:
38
- results.append("No gestures detected.")
39
- gesture_queue.put(results)
40
-
41
- # Configure the Gesture Recognizer
42
  options = GestureRecognizerOptions(
43
  base_options=BaseOptions(model_asset_path=model_path),
44
- running_mode=VisionRunningMode.LIVE_STREAM,
45
- result_callback=print_result
46
- )
47
-
48
- # Initialize session state for saving gestures
49
- if "recognized_gestures" not in st.session_state:
50
- st.session_state.recognized_gestures = []
51
-
52
- # Custom App Header
53
- st.markdown(
54
- """
55
- <style>
56
- .header {text-align: center; color: #4CAF50; margin-top: -50px;}
57
- .description {text-align: center; color: grey; font-size: 16px;}
58
- </style>
59
- <h1 class="header">Gesture & Hand Landmark Detection 🚀</h1>
60
- <p class="description">Recognize and save hand gestures in real time with MediaPipe.</p>
61
- """,
62
- unsafe_allow_html=True,
63
  )
64
 
65
- # Sidebar for User Controls
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
66
  st.sidebar.title("Control Panel")
67
  st.sidebar.markdown("<hr>", unsafe_allow_html=True)
68
 
69
- max_num_hands = st.sidebar.slider("Max Number of Hands", 1, 2, 1)
70
- skip_frames = st.sidebar.slider("Process Every Nth Frame", 1, 10, 5)
71
- resolution = st.sidebar.selectbox("Frame Resolution", ["320x240", "640x480"], index=0)
72
-
73
- st.sidebar.markdown("<hr>", unsafe_allow_html=True)
74
-
75
- # Start and Stop buttons
76
- if "run_app" not in st.session_state:
77
- st.session_state.run_app = False
78
-
79
- col1, col2 = st.sidebar.columns(2)
80
- if col1.button("▶ Start"):
81
- st.session_state.run_app = True
82
-
83
- if col2.button("⏹ Stop"):
84
- st.session_state.run_app = False
85
-
86
- # Clear history button
87
- if st.sidebar.button("🗑️ Clear History"):
88
- st.session_state.recognized_gestures = []
89
-
90
- # Layout with columns: Live camera feed on the left, gesture log box on the right
91
- col_feed, col_log = st.columns([3, 2])
92
 
93
- with col_feed:
94
- st.markdown("### Live Camera Feed")
95
- video_placeholder = st.empty() # Placeholder for the video feed
96
-
97
- with col_log:
98
- st.markdown("### Gesture Log")
99
- current_gesture_box = st.empty() # Box to display the most recent gesture dynamically
100
- st.markdown("### Gesture History")
101
- gesture_history_box = st.empty() # Box to display all recognized gestures dynamically
102
-
103
- # Footer with branding
104
  st.sidebar.markdown(
105
  """
106
  <style>
107
  .footer {text-align: center; font-size: 12px; color: grey; margin-top: 20px;}
108
  </style>
109
- <p class="footer">Made by Marco Chen, William Taka, Rigoberto Ponce using Streamlit, MediaPipe & OpenCV</p>
110
  """,
111
  unsafe_allow_html=True,
112
  )
113
-
114
- if st.session_state.run_app:
115
- cap = cv2.VideoCapture(0)
116
-
117
- # Parse resolution
118
- res_width, res_height = map(int, resolution.split("x"))
119
-
120
- # Initialize a monotonically increasing timestamp
121
- start_time = time.time()
122
-
123
- with GestureRecognizer.create_from_options(options) as recognizer, mp_hands.Hands(
124
- max_num_hands=max_num_hands,
125
- model_complexity=0, # Simplified model for performance
126
- min_detection_confidence=0.3,
127
- min_tracking_confidence=0.3
128
- ) as hands:
129
- frame_count = 0
130
- while st.session_state.run_app and cap.isOpened():
131
- success, frame = cap.read()
132
- if not success:
133
- st.warning("No frames available from the video feed.")
134
- break
135
-
136
- frame_count += 1
137
- if frame_count % skip_frames != 0:
138
- continue
139
-
140
- # Flip and resize frame
141
- frame = cv2.flip(frame, 1)
142
- frame = cv2.resize(frame, (res_width, res_height))
143
- frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
144
-
145
- # Perform hand landmark detection
146
- hand_results = hands.process(frame_rgb)
147
-
148
- # Perform gesture recognition
149
- mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=frame)
150
- current_time_ms = int((time.time() - start_time) * 1000)
151
- recognizer.recognize_async(mp_image, current_time_ms)
152
-
153
- # Draw hand landmarks on the frame
154
- if hand_results.multi_hand_landmarks:
155
- for hand_landmarks in hand_results.multi_hand_landmarks:
156
- mp_drawing.draw_landmarks(
157
- frame,
158
- hand_landmarks,
159
- mp_hands.HAND_CONNECTIONS,
160
- mp_drawing_styles.get_default_hand_landmarks_style(),
161
- mp_drawing_styles.get_default_hand_connections_style(),
162
- )
163
-
164
- # Retrieve and display gesture results from the queue
165
- while not gesture_queue.empty():
166
- results = gesture_queue.get()
167
- if results:
168
- new_gesture = results[-1]
169
-
170
- # Extract label and confidence safely
171
- if " (Confidence: " in new_gesture:
172
- label, confidence = new_gesture.split(" (Confidence: ")
173
- confidence = confidence.rstrip(")") # Remove the trailing parenthesis
174
- else:
175
- label = new_gesture
176
- confidence = "N/A"
177
-
178
- # Add new gesture to history only if it's not already logged
179
- if label.isalpha() and new_gesture not in st.session_state.recognized_gestures:
180
- st.session_state.recognized_gestures.append(new_gesture)
181
-
182
- # Update current gesture display
183
- current_gesture_box.markdown(
184
- f"<h4 style='text-align: center; color: #4CAF50;'>Gesture: {label}<br>Confidence: {confidence}</h4>",
185
- unsafe_allow_html=True,
186
- )
187
-
188
- # Update gesture history display
189
- gesture_history_box.text_area(
190
- "Gesture History:",
191
- value="\n".join(reversed(st.session_state.recognized_gestures)),
192
- height=300,
193
- disabled=True,
194
- )
195
-
196
- # Display the frame with hand landmarks and gesture results
197
- video_placeholder.image(frame, channels="BGR", caption="Gesture & Hand Landmark Detection", use_column_width=True)
198
-
199
- cap.release()
 
1
  import streamlit as st
2
  import mediapipe as mp
3
  import cv2
 
4
  import time
5
+ from streamlit_webrtc import webrtc_streamer, VideoTransformerBase
6
+ import av
7
  from queue import Queue
8
 
9
+ # MediaPipe components for gesture recognition and hand landmarks
10
  mp_hands = mp.solutions.hands
11
  mp_drawing = mp.solutions.drawing_utils
12
  mp_drawing_styles = mp.solutions.drawing_styles
 
 
13
  BaseOptions = mp.tasks.BaseOptions
14
  GestureRecognizer = mp.tasks.vision.GestureRecognizer
15
  GestureRecognizerOptions = mp.tasks.vision.GestureRecognizerOptions
16
  GestureRecognizerResult = mp.tasks.vision.GestureRecognizerResult
17
  VisionRunningMode = mp.tasks.vision.RunningMode
18
 
19
+ # Path to the gesture recognizer model
 
20
  model_path = 'model/gesture_recognizer.task'
 
21
  if not os.path.exists(model_path):
22
  raise FileNotFoundError(f"Model file not found at {model_path}")
23
 
24
+ # Gesture recognizer configuration
 
 
 
 
 
 
 
 
 
 
 
 
 
 
25
  options = GestureRecognizerOptions(
26
  base_options=BaseOptions(model_asset_path=model_path),
27
+ running_mode=VisionRunningMode.LIVE_STREAM
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  )
29
 
30
+ # Custom video processor class for handling webcam input
31
+ class VideoProcessor(VideoTransformerBase):
32
+ def __init__(self):
33
+ self.recognizer = GestureRecognizer.create_from_options(options)
34
+ self.hands = mp_hands.Hands(
35
+ max_num_hands=2,
36
+ model_complexity=0,
37
+ min_detection_confidence=0.3,
38
+ min_tracking_confidence=0.3
39
+ )
40
+ self.gesture_queue = Queue()
41
+
42
+ def recv(self, frame):
43
+ img = frame.to_ndarray(format="bgr24")
44
+ img = cv2.flip(img, 1) # Flip the image horizontally for a mirror effect
45
+ img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
46
+
47
+ # Perform hand landmark detection
48
+ hand_results = self.hands.process(img_rgb)
49
+
50
+ # Perform gesture recognition
51
+ mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=img_rgb)
52
+ current_time_ms = int(time.time() * 1000)
53
+ result = self.recognizer.recognize(mp_image, current_time_ms)
54
+
55
+ # Draw landmarks and recognized gestures on the frame
56
+ if hand_results.multi_hand_landmarks:
57
+ for hand_landmarks in hand_results.multi_hand_landmarks:
58
+ mp_drawing.draw_landmarks(
59
+ img,
60
+ hand_landmarks,
61
+ mp_hands.HAND_CONNECTIONS,
62
+ mp_drawing_styles.get_default_hand_landmarks_style(),
63
+ mp_drawing_styles.get_default_hand_connections_style()
64
+ )
65
+
66
+ # Display recognized gestures
67
+ if result.gestures:
68
+ for hand_gestures in result.gestures:
69
+ for gesture in hand_gestures:
70
+ gesture_text = f"{gesture.category_name} ({gesture.score:.2f})"
71
+ cv2.putText(img, gesture_text, (10, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2)
72
+
73
+ return av.VideoFrame.from_ndarray(img, format="bgr24")
74
+
75
+ # Streamlit UI
76
+ st.title("Gesture & Hand Landmark Detection 🚀")
77
+ st.markdown("### Recognize and save hand gestures in real time using MediaPipe and Streamlit WebRTC.")
78
+
79
+ # Sidebar controls
80
  st.sidebar.title("Control Panel")
81
  st.sidebar.markdown("<hr>", unsafe_allow_html=True)
82
 
83
+ # Run the Streamlit WebRTC app
84
+ webrtc_streamer(key="gesture-detection", video_processor_factory=VideoProcessor)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
85
 
86
+ # Footer
 
 
 
 
 
 
 
 
 
 
87
  st.sidebar.markdown(
88
  """
89
  <style>
90
  .footer {text-align: center; font-size: 12px; color: grey; margin-top: 20px;}
91
  </style>
92
+ <p class="footer">Made using Streamlit, MediaPipe & OpenCV</p>
93
  """,
94
  unsafe_allow_html=True,
95
  )