Update app.py
Browse files
app.py
CHANGED
|
@@ -1,199 +1,95 @@
|
|
| 1 |
import streamlit as st
|
| 2 |
import mediapipe as mp
|
| 3 |
import cv2
|
| 4 |
-
import os
|
| 5 |
import time
|
|
|
|
|
|
|
| 6 |
from queue import Queue
|
| 7 |
|
| 8 |
-
#
|
| 9 |
mp_hands = mp.solutions.hands
|
| 10 |
mp_drawing = mp.solutions.drawing_utils
|
| 11 |
mp_drawing_styles = mp.solutions.drawing_styles
|
| 12 |
-
|
| 13 |
-
# Import necessary components from MediaPipe Gesture Recognizer
|
| 14 |
BaseOptions = mp.tasks.BaseOptions
|
| 15 |
GestureRecognizer = mp.tasks.vision.GestureRecognizer
|
| 16 |
GestureRecognizerOptions = mp.tasks.vision.GestureRecognizerOptions
|
| 17 |
GestureRecognizerResult = mp.tasks.vision.GestureRecognizerResult
|
| 18 |
VisionRunningMode = mp.tasks.vision.RunningMode
|
| 19 |
|
| 20 |
-
#
|
| 21 |
-
#model_path = './model/model/gesture_recognizer.task'
|
| 22 |
model_path = 'model/gesture_recognizer.task'
|
| 23 |
-
# Check if file exists
|
| 24 |
if not os.path.exists(model_path):
|
| 25 |
raise FileNotFoundError(f"Model file not found at {model_path}")
|
| 26 |
|
| 27 |
-
#
|
| 28 |
-
gesture_queue = Queue()
|
| 29 |
-
|
| 30 |
-
# Callback function to process gesture results and add them to the queue
|
| 31 |
-
def print_result(result: GestureRecognizerResult, output_image: mp.Image, timestamp_ms: int):
|
| 32 |
-
results = [] # Collect gesture results
|
| 33 |
-
if result.gestures:
|
| 34 |
-
for hand_gestures in result.gestures:
|
| 35 |
-
for gesture in hand_gestures:
|
| 36 |
-
results.append(f"{gesture.category_name} (Confidence: {gesture.score:.2f})") # Include confidence
|
| 37 |
-
else:
|
| 38 |
-
results.append("No gestures detected.")
|
| 39 |
-
gesture_queue.put(results)
|
| 40 |
-
|
| 41 |
-
# Configure the Gesture Recognizer
|
| 42 |
options = GestureRecognizerOptions(
|
| 43 |
base_options=BaseOptions(model_asset_path=model_path),
|
| 44 |
-
running_mode=VisionRunningMode.LIVE_STREAM
|
| 45 |
-
result_callback=print_result
|
| 46 |
-
)
|
| 47 |
-
|
| 48 |
-
# Initialize session state for saving gestures
|
| 49 |
-
if "recognized_gestures" not in st.session_state:
|
| 50 |
-
st.session_state.recognized_gestures = []
|
| 51 |
-
|
| 52 |
-
# Custom App Header
|
| 53 |
-
st.markdown(
|
| 54 |
-
"""
|
| 55 |
-
<style>
|
| 56 |
-
.header {text-align: center; color: #4CAF50; margin-top: -50px;}
|
| 57 |
-
.description {text-align: center; color: grey; font-size: 16px;}
|
| 58 |
-
</style>
|
| 59 |
-
<h1 class="header">Gesture & Hand Landmark Detection 🚀</h1>
|
| 60 |
-
<p class="description">Recognize and save hand gestures in real time with MediaPipe.</p>
|
| 61 |
-
""",
|
| 62 |
-
unsafe_allow_html=True,
|
| 63 |
)
|
| 64 |
|
| 65 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
st.sidebar.title("Control Panel")
|
| 67 |
st.sidebar.markdown("<hr>", unsafe_allow_html=True)
|
| 68 |
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
resolution = st.sidebar.selectbox("Frame Resolution", ["320x240", "640x480"], index=0)
|
| 72 |
-
|
| 73 |
-
st.sidebar.markdown("<hr>", unsafe_allow_html=True)
|
| 74 |
-
|
| 75 |
-
# Start and Stop buttons
|
| 76 |
-
if "run_app" not in st.session_state:
|
| 77 |
-
st.session_state.run_app = False
|
| 78 |
-
|
| 79 |
-
col1, col2 = st.sidebar.columns(2)
|
| 80 |
-
if col1.button("▶ Start"):
|
| 81 |
-
st.session_state.run_app = True
|
| 82 |
-
|
| 83 |
-
if col2.button("⏹ Stop"):
|
| 84 |
-
st.session_state.run_app = False
|
| 85 |
-
|
| 86 |
-
# Clear history button
|
| 87 |
-
if st.sidebar.button("🗑️ Clear History"):
|
| 88 |
-
st.session_state.recognized_gestures = []
|
| 89 |
-
|
| 90 |
-
# Layout with columns: Live camera feed on the left, gesture log box on the right
|
| 91 |
-
col_feed, col_log = st.columns([3, 2])
|
| 92 |
|
| 93 |
-
|
| 94 |
-
st.markdown("### Live Camera Feed")
|
| 95 |
-
video_placeholder = st.empty() # Placeholder for the video feed
|
| 96 |
-
|
| 97 |
-
with col_log:
|
| 98 |
-
st.markdown("### Gesture Log")
|
| 99 |
-
current_gesture_box = st.empty() # Box to display the most recent gesture dynamically
|
| 100 |
-
st.markdown("### Gesture History")
|
| 101 |
-
gesture_history_box = st.empty() # Box to display all recognized gestures dynamically
|
| 102 |
-
|
| 103 |
-
# Footer with branding
|
| 104 |
st.sidebar.markdown(
|
| 105 |
"""
|
| 106 |
<style>
|
| 107 |
.footer {text-align: center; font-size: 12px; color: grey; margin-top: 20px;}
|
| 108 |
</style>
|
| 109 |
-
<p class="footer">Made
|
| 110 |
""",
|
| 111 |
unsafe_allow_html=True,
|
| 112 |
)
|
| 113 |
-
|
| 114 |
-
if st.session_state.run_app:
|
| 115 |
-
cap = cv2.VideoCapture(0)
|
| 116 |
-
|
| 117 |
-
# Parse resolution
|
| 118 |
-
res_width, res_height = map(int, resolution.split("x"))
|
| 119 |
-
|
| 120 |
-
# Initialize a monotonically increasing timestamp
|
| 121 |
-
start_time = time.time()
|
| 122 |
-
|
| 123 |
-
with GestureRecognizer.create_from_options(options) as recognizer, mp_hands.Hands(
|
| 124 |
-
max_num_hands=max_num_hands,
|
| 125 |
-
model_complexity=0, # Simplified model for performance
|
| 126 |
-
min_detection_confidence=0.3,
|
| 127 |
-
min_tracking_confidence=0.3
|
| 128 |
-
) as hands:
|
| 129 |
-
frame_count = 0
|
| 130 |
-
while st.session_state.run_app and cap.isOpened():
|
| 131 |
-
success, frame = cap.read()
|
| 132 |
-
if not success:
|
| 133 |
-
st.warning("No frames available from the video feed.")
|
| 134 |
-
break
|
| 135 |
-
|
| 136 |
-
frame_count += 1
|
| 137 |
-
if frame_count % skip_frames != 0:
|
| 138 |
-
continue
|
| 139 |
-
|
| 140 |
-
# Flip and resize frame
|
| 141 |
-
frame = cv2.flip(frame, 1)
|
| 142 |
-
frame = cv2.resize(frame, (res_width, res_height))
|
| 143 |
-
frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
| 144 |
-
|
| 145 |
-
# Perform hand landmark detection
|
| 146 |
-
hand_results = hands.process(frame_rgb)
|
| 147 |
-
|
| 148 |
-
# Perform gesture recognition
|
| 149 |
-
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=frame)
|
| 150 |
-
current_time_ms = int((time.time() - start_time) * 1000)
|
| 151 |
-
recognizer.recognize_async(mp_image, current_time_ms)
|
| 152 |
-
|
| 153 |
-
# Draw hand landmarks on the frame
|
| 154 |
-
if hand_results.multi_hand_landmarks:
|
| 155 |
-
for hand_landmarks in hand_results.multi_hand_landmarks:
|
| 156 |
-
mp_drawing.draw_landmarks(
|
| 157 |
-
frame,
|
| 158 |
-
hand_landmarks,
|
| 159 |
-
mp_hands.HAND_CONNECTIONS,
|
| 160 |
-
mp_drawing_styles.get_default_hand_landmarks_style(),
|
| 161 |
-
mp_drawing_styles.get_default_hand_connections_style(),
|
| 162 |
-
)
|
| 163 |
-
|
| 164 |
-
# Retrieve and display gesture results from the queue
|
| 165 |
-
while not gesture_queue.empty():
|
| 166 |
-
results = gesture_queue.get()
|
| 167 |
-
if results:
|
| 168 |
-
new_gesture = results[-1]
|
| 169 |
-
|
| 170 |
-
# Extract label and confidence safely
|
| 171 |
-
if " (Confidence: " in new_gesture:
|
| 172 |
-
label, confidence = new_gesture.split(" (Confidence: ")
|
| 173 |
-
confidence = confidence.rstrip(")") # Remove the trailing parenthesis
|
| 174 |
-
else:
|
| 175 |
-
label = new_gesture
|
| 176 |
-
confidence = "N/A"
|
| 177 |
-
|
| 178 |
-
# Add new gesture to history only if it's not already logged
|
| 179 |
-
if label.isalpha() and new_gesture not in st.session_state.recognized_gestures:
|
| 180 |
-
st.session_state.recognized_gestures.append(new_gesture)
|
| 181 |
-
|
| 182 |
-
# Update current gesture display
|
| 183 |
-
current_gesture_box.markdown(
|
| 184 |
-
f"<h4 style='text-align: center; color: #4CAF50;'>Gesture: {label}<br>Confidence: {confidence}</h4>",
|
| 185 |
-
unsafe_allow_html=True,
|
| 186 |
-
)
|
| 187 |
-
|
| 188 |
-
# Update gesture history display
|
| 189 |
-
gesture_history_box.text_area(
|
| 190 |
-
"Gesture History:",
|
| 191 |
-
value="\n".join(reversed(st.session_state.recognized_gestures)),
|
| 192 |
-
height=300,
|
| 193 |
-
disabled=True,
|
| 194 |
-
)
|
| 195 |
-
|
| 196 |
-
# Display the frame with hand landmarks and gesture results
|
| 197 |
-
video_placeholder.image(frame, channels="BGR", caption="Gesture & Hand Landmark Detection", use_column_width=True)
|
| 198 |
-
|
| 199 |
-
cap.release()
|
|
|
|
| 1 |
import streamlit as st
|
| 2 |
import mediapipe as mp
|
| 3 |
import cv2
|
|
|
|
| 4 |
import time
|
| 5 |
+
from streamlit_webrtc import webrtc_streamer, VideoTransformerBase
|
| 6 |
+
import av
|
| 7 |
from queue import Queue
|
| 8 |
|
| 9 |
+
# MediaPipe components for gesture recognition and hand landmarks
|
| 10 |
mp_hands = mp.solutions.hands
|
| 11 |
mp_drawing = mp.solutions.drawing_utils
|
| 12 |
mp_drawing_styles = mp.solutions.drawing_styles
|
|
|
|
|
|
|
| 13 |
BaseOptions = mp.tasks.BaseOptions
|
| 14 |
GestureRecognizer = mp.tasks.vision.GestureRecognizer
|
| 15 |
GestureRecognizerOptions = mp.tasks.vision.GestureRecognizerOptions
|
| 16 |
GestureRecognizerResult = mp.tasks.vision.GestureRecognizerResult
|
| 17 |
VisionRunningMode = mp.tasks.vision.RunningMode
|
| 18 |
|
| 19 |
+
# Path to the gesture recognizer model
|
|
|
|
| 20 |
model_path = 'model/gesture_recognizer.task'
|
|
|
|
| 21 |
if not os.path.exists(model_path):
|
| 22 |
raise FileNotFoundError(f"Model file not found at {model_path}")
|
| 23 |
|
| 24 |
+
# Gesture recognizer configuration
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
options = GestureRecognizerOptions(
|
| 26 |
base_options=BaseOptions(model_asset_path=model_path),
|
| 27 |
+
running_mode=VisionRunningMode.LIVE_STREAM
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
)
|
| 29 |
|
| 30 |
+
# Custom video processor class for handling webcam input
|
| 31 |
+
class VideoProcessor(VideoTransformerBase):
|
| 32 |
+
def __init__(self):
|
| 33 |
+
self.recognizer = GestureRecognizer.create_from_options(options)
|
| 34 |
+
self.hands = mp_hands.Hands(
|
| 35 |
+
max_num_hands=2,
|
| 36 |
+
model_complexity=0,
|
| 37 |
+
min_detection_confidence=0.3,
|
| 38 |
+
min_tracking_confidence=0.3
|
| 39 |
+
)
|
| 40 |
+
self.gesture_queue = Queue()
|
| 41 |
+
|
| 42 |
+
def recv(self, frame):
|
| 43 |
+
img = frame.to_ndarray(format="bgr24")
|
| 44 |
+
img = cv2.flip(img, 1) # Flip the image horizontally for a mirror effect
|
| 45 |
+
img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
|
| 46 |
+
|
| 47 |
+
# Perform hand landmark detection
|
| 48 |
+
hand_results = self.hands.process(img_rgb)
|
| 49 |
+
|
| 50 |
+
# Perform gesture recognition
|
| 51 |
+
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=img_rgb)
|
| 52 |
+
current_time_ms = int(time.time() * 1000)
|
| 53 |
+
result = self.recognizer.recognize(mp_image, current_time_ms)
|
| 54 |
+
|
| 55 |
+
# Draw landmarks and recognized gestures on the frame
|
| 56 |
+
if hand_results.multi_hand_landmarks:
|
| 57 |
+
for hand_landmarks in hand_results.multi_hand_landmarks:
|
| 58 |
+
mp_drawing.draw_landmarks(
|
| 59 |
+
img,
|
| 60 |
+
hand_landmarks,
|
| 61 |
+
mp_hands.HAND_CONNECTIONS,
|
| 62 |
+
mp_drawing_styles.get_default_hand_landmarks_style(),
|
| 63 |
+
mp_drawing_styles.get_default_hand_connections_style()
|
| 64 |
+
)
|
| 65 |
+
|
| 66 |
+
# Display recognized gestures
|
| 67 |
+
if result.gestures:
|
| 68 |
+
for hand_gestures in result.gestures:
|
| 69 |
+
for gesture in hand_gestures:
|
| 70 |
+
gesture_text = f"{gesture.category_name} ({gesture.score:.2f})"
|
| 71 |
+
cv2.putText(img, gesture_text, (10, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2)
|
| 72 |
+
|
| 73 |
+
return av.VideoFrame.from_ndarray(img, format="bgr24")
|
| 74 |
+
|
| 75 |
+
# Streamlit UI
|
| 76 |
+
st.title("Gesture & Hand Landmark Detection 🚀")
|
| 77 |
+
st.markdown("### Recognize and save hand gestures in real time using MediaPipe and Streamlit WebRTC.")
|
| 78 |
+
|
| 79 |
+
# Sidebar controls
|
| 80 |
st.sidebar.title("Control Panel")
|
| 81 |
st.sidebar.markdown("<hr>", unsafe_allow_html=True)
|
| 82 |
|
| 83 |
+
# Run the Streamlit WebRTC app
|
| 84 |
+
webrtc_streamer(key="gesture-detection", video_processor_factory=VideoProcessor)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
|
| 86 |
+
# Footer
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
st.sidebar.markdown(
|
| 88 |
"""
|
| 89 |
<style>
|
| 90 |
.footer {text-align: center; font-size: 12px; color: grey; margin-top: 20px;}
|
| 91 |
</style>
|
| 92 |
+
<p class="footer">Made using Streamlit, MediaPipe & OpenCV</p>
|
| 93 |
""",
|
| 94 |
unsafe_allow_html=True,
|
| 95 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|