Update gesture_classifier.py

gesture_classifier.py

@@ -2,88 +2,101 @@ import cv2
 import numpy as np
 import json
 import mediapipe as mp
+import math
 
-# Initialize MediaPipe Hands
+# Initialize MediaPipe
 mp_hands = mp.solutions.hands
+mp_drawing = mp.solutions.drawing_utils
+mp_drawing_styles = mp.solutions.drawing_styles
+
 hands = mp_hands.Hands(
     static_image_mode=True,
     max_num_hands=1,
-    min_detection_confidence=0.5
+    min_detection_confidence=0.7 # Increased confidence threshold
 )
 
-# ---------------- LOAD GESTURE DEFINITIONS ----------------
+# Load Rules
 with open("gesture_rules.json", "r") as f:
     GESTURE_DATA = json.load(f)["gestures"]
 
-def get_finger_states_mediapipe(image_rgb):
+def calculate_distance(p1, p2):
+    """Calculates Euclidean distance between two 3D landmarks."""
+    return math.sqrt((p1.x - p2.x)**2 + (p1.y - p2.y)**2 + (p1.z - p2.z)**2)
+
+def get_finger_states_robust(landmarks):
     """
     Returns [thumb, index, middle, ring, pinky] states (1=Open, 0=Closed)
-    using MediaPipe Landmarks.
+    using Euclidean distance relative to the Wrist (Landmark 0).
     """
-    results = hands.process(image_rgb)
-
-    if not results.multi_hand_landmarks:
-        return [0, 0, 0, 0, 0] # No hand detected
+    states = []
+    wrist = landmarks[0]
 
-    landmarks = results.multi_hand_landmarks[0].landmark
+    # --- FINGERS (Index, Middle, Ring, Pinky) ---
+    # Logic: If Distance(Tip, Wrist) < Distance(PIP, Wrist), the finger is curled.
     
-    # MediaPipe Landmark Indices:
-    # Thumb: 1-4 (Compare Tip 4 vs IP 3 or MCP 2 depending on axis)
-    # Index: 5-8 (Tip 8 vs PIP 6)
-    # Middle: 9-12 (Tip 12 vs PIP 10)
-    # Ring: 13-16 (Tip 16 vs PIP 14)
-    # Pinky: 17-20 (Tip 20 vs PIP 18)
-
-    states = []
+    # Index (Tip 8, PIP 6)
+    states.append(1 if calculate_distance(landmarks[8], wrist) > calculate_distance(landmarks[6], wrist) else 0)
+    
+    # Middle (Tip 12, PIP 10)
+    states.append(1 if calculate_distance(landmarks[12], wrist) > calculate_distance(landmarks[10], wrist) else 0)
 
-    # --- THUMB LOGIC ---
-    # Thumb is open if tip (4) is to the outside of IP (3) relative to the palm
-    # Simple check: Is tip x-coordinate further from pinky base x-coordinate?
-    # A generic check for "Thumb Up/Open":
-    # We compare the X coordinates for the thumb (horizontal movement)
-    # Assuming right hand for simplicity, or checking relative distance to MCP
-    if landmarks[4].x < landmarks[3].x: # Adjust logic if needed for specific hand orientation
-         states.append(1)
-    else:
-         states.append(0)
+    # Ring (Tip 16, PIP 14)
+    states.append(1 if calculate_distance(landmarks[16], wrist) > calculate_distance(landmarks[14], wrist) else 0)
 
-    # --- FINGERS (Index, Middle, Ring, Pinky) ---
-    # Check if Tip Y < PIP Y (Note: Y increases downwards in images)
-    # Index (8 vs 6)
-    states.append(1 if landmarks[8].y < landmarks[6].y else 0)
-    # Middle (12 vs 10)
-    states.append(1 if landmarks[12].y < landmarks[10].y else 0)
-    # Ring (16 vs 14)
-    states.append(1 if landmarks[16].y < landmarks[14].y else 0)
-    # Pinky (20 vs 18)
-    states.append(1 if landmarks[20].y < landmarks[18].y else 0)
+    # Pinky (Tip 20, PIP 18)
+    states.append(1 if calculate_distance(landmarks[20], wrist) > calculate_distance(landmarks[18], wrist) else 0)
 
-    # Note: The Thumb logic above is simplistic. 
-    # For a purely static robust check, we often check if Tip is "higher" than MCP,
-    # but thumbs move laterally. 
-    # For this specific JSON, let's treat Thumb as "Open" if the Tip is far from the Index base.
+    # --- THUMB LOGIC (Complex) ---
+    # The thumb doesn't curl to the wrist like other fingers.
+    # We check if the Tip (4) is further away from the Pinky MCP (17) than the IP (3) is.
+    # Basically, is the thumb sticking out?
+    
+    dist_tip_pinky = calculate_distance(landmarks[4], landmarks[17])
+    dist_ip_pinky = calculate_distance(landmarks[3], landmarks[17])
     
-    # Overriding thumb for better stability in static images:
-    # Calculate distance between Thumb Tip (4) and Pinky MCP (17)
-    # If distance is large -> Open. Small -> Closed.
+    # We insert Thumb at the start of the list to match JSON format
+    if dist_tip_pinky > dist_ip_pinky:
+        states.insert(0, 1) # Open
+    else:
+        states.insert(0, 0) # Closed
+
     return states
 
-def classify_gesture(image: np.ndarray) -> str:
+def classify_gesture(image: np.ndarray):
     """
-    Match extracted finger states against JSON-defined patterns
+    Returns: (gesture_name, annotated_image)
     """
-    # Ensure Image is RGB (MediaPipe expects RGB)
-    # If the input is BGR from OpenCV standard, convert it. 
-    # But Streamlit/PIL usually gives RGB. We assume RGB here.
+    # Convert image to writable for MediaPipe
+    image.flags.writeable = False
+    results = hands.process(image)
+    image.flags.writeable = True
+
+    # If no hand found
+    if not results.multi_hand_landmarks:
+        return None, image
+
+    # Get landmarks for the first hand
+    hand_landmarks = results.multi_hand_landmarks[0]
     
-    # If input is a numpy array from PIL, it is RGB.
-    states = get_finger_states_mediapipe(image)
+    # Draw the Skeleton on the image
+    annotated_image = image.copy()
+    mp_drawing.draw_landmarks(
+        annotated_image,
+        hand_landmarks,
+        mp_hands.HAND_CONNECTIONS,
+        mp_drawing_styles.get_default_hand_landmarks_style(),
+        mp_drawing_styles.get_default_hand_connections_style()
+    )
 
-    # Debug print to help you tune thresholds if needed
-    print(f"Detected States: {states}") 
+    # Get binary states
+    states = get_finger_states_robust(hand_landmarks.landmark)
+    print(f"Detected States (T,I,M,R,P): {states}")
 
+    # Match against DB
+    detected_gesture = "UNKNOWN"
     for name, info in GESTURE_DATA.items():
         if states == info["pattern"]:
-            return name
-
-    return "UNKNOWN"
+            detected_gesture = name
+            break
+            
+    return detected_gesture, annotated_image