Update app.py

app.py

@@ -18,6 +18,69 @@ face_mesh = mp_face_mesh.FaceMesh(
     min_detection_confidence=0.5
 )
 
+# Define key facial landmarks for expressions
+FACIAL_LANDMARKS = {
+    'left_brow': [52, 65, 46],  # inner, middle, outer
+    'right_brow': [285, 295, 276],  # inner, middle, outer
+    'left_eye': [159, 145, 133],  # top, bottom, outer
+    'right_eye': [386, 374, 362],  # top, bottom, outer
+    'nose': [6, 197],  # bridge, tip
+    'mouth': [61, 291, 0, 17, 13, 14],  # left corner, right corner, top lip, bottom lip, upper inner, lower inner
+    'jaw': [17, 84, 314]  # center, left, right
+}
+
+def calculate_distances(points, landmarks):
+    """Calculate normalized distances between facial landmarks."""
+    def distance(p1_idx, p2_idx):
+        try:
+            p1 = points[p1_idx]
+            p2 = points[p2_idx]
+            return np.linalg.norm(p1 - p2)
+        except:
+            return 0.0
+    
+    # Get face height for normalization
+    face_height = distance(FACIAL_LANDMARKS['nose'][0], FACIAL_LANDMARKS['jaw'][0])
+    if face_height == 0:
+        return {}
+    
+    measurements = {
+        # Inner brow raising (AU1)
+        'inner_brow_raise': (
+            distance(FACIAL_LANDMARKS['left_brow'][0], FACIAL_LANDMARKS['nose'][0]) +
+            distance(FACIAL_LANDMARKS['right_brow'][0], FACIAL_LANDMARKS['nose'][0])
+        ) / (2 * face_height),
+        
+        # Outer brow raising (AU2)
+        'outer_brow_raise': (
+            distance(FACIAL_LANDMARKS['left_brow'][2], FACIAL_LANDMARKS['nose'][0]) +
+            distance(FACIAL_LANDMARKS['right_brow'][2], FACIAL_LANDMARKS['nose'][0])
+        ) / (2 * face_height),
+        
+        # Brow lowering (AU4)
+        'brow_furrow': distance(FACIAL_LANDMARKS['left_brow'][0], FACIAL_LANDMARKS['right_brow'][0]) / face_height,
+        
+        # Eye opening (AU5)
+        'eye_opening': (
+            distance(FACIAL_LANDMARKS['left_eye'][0], FACIAL_LANDMARKS['left_eye'][1]) +
+            distance(FACIAL_LANDMARKS['right_eye'][0], FACIAL_LANDMARKS['right_eye'][1])
+        ) / (2 * face_height),
+        
+        # Smile width (AU12)
+        'smile_width': distance(FACIAL_LANDMARKS['mouth'][0], FACIAL_LANDMARKS['mouth'][1]) / face_height,
+        
+        # Mouth height (AU25/26)
+        'mouth_opening': distance(FACIAL_LANDMARKS['mouth'][4], FACIAL_LANDMARKS['mouth'][5]) / face_height,
+        
+        # Lip corner height (for smile/frown detection)
+        'lip_corner_height': (
+            (points[FACIAL_LANDMARKS['mouth'][0]][1] + points[FACIAL_LANDMARKS['mouth'][1]][1])/2 -
+            points[FACIAL_LANDMARKS['mouth'][2]][1]
+        ) / face_height
+    }
+    
+    return measurements
+
 def analyze_expression(image):
     if image is None:
         return None, "No image provided"
@@ -38,53 +101,86 @@ def analyze_expression(image):
     landmarks = results.multi_face_landmarks[0]
     points = np.array([[lm.x, lm.y, lm.z] for lm in landmarks.landmark])
     
-    # Calculate basic measurements
-    # Smile detection (AU12)
-    smile_width = abs(points[61][0] - points[291][0])
-    smile_height = (points[61][1] + points[291][1])/2 - points[0][1]
-    
-    # Brow movement (AU1, AU4)
-    brow_height = abs(points[52][1] - points[65][1])
-    brow_furrow = abs(points[9][0] - points[336][0])
-    
-    # Mouth opening (AU25, AU26)
-    mouth_open = abs(points[13][1] - points[14][1])
+    # Calculate facial measurements
+    measurements = calculate_distances(points, landmarks)
+    
+    # Analyze Action Units
+    aus = {
+        'AU01': measurements['inner_brow_raise'] > 0.15,  # Inner Brow Raiser
+        'AU02': measurements['outer_brow_raise'] > 0.15,  # Outer Brow Raiser
+        'AU04': measurements['brow_furrow'] < 0.2,        # Brow Lowerer
+        'AU05': measurements['eye_opening'] > 0.12,       # Upper Lid Raiser
+        'AU12': measurements['smile_width'] > 0.45,       # Lip Corner Puller
+        'AU25': measurements['mouth_opening'] > 0.1,      # Lips Part
+        'AU26': measurements['mouth_opening'] > 0.2       # Jaw Drop
+    }
     
-    # Simple emotion classification
+    # Emotion classification based on AUs
     emotions = {
-        "Happy": (smile_width > 0.3 and smile_height < 0),
-        "Sad": (smile_height > 0 and brow_height > 0.1),
-        "Surprised": (mouth_open > 0.1 and brow_height > 0.15),
-        "Neutral": (smile_width < 0.3 and mouth_open < 0.1 and brow_height < 0.1)
+        "Happy": (aus['AU12'] and measurements['lip_corner_height'] < 0),
+        "Sad": (not aus['AU12'] and measurements['lip_corner_height'] > 0 and (aus['AU01'] or aus['AU04'])),
+        "Surprised": (aus['AU01'] and aus['AU02'] and (aus['AU25'] or aus['AU26'])),
+        "Neutral": not any([aus['AU01'], aus['AU02'], aus['AU04'], aus['AU12'], aus['AU26']])
     }
     
     # Create visualization
     viz_image = image.copy()
     h, w = viz_image.shape[:2]
     
-    # Draw landmarks
-    for landmark in landmarks.landmark:
-        pos = (int(landmark.x * w), int(landmark.y * h))
-        cv2.circle(viz_image, pos, 1, (0, 255, 0), -1)
+    # Draw facial landmarks with different colors for key points
+    colors = {
+        'brow': (0, 255, 0),    # Green
+        'eye': (255, 255, 0),   # Yellow
+        'nose': (0, 255, 255),  # Cyan
+        'mouth': (255, 0, 255), # Magenta
+        'jaw': (255, 128, 0)    # Orange
+    }
+    
+    # Draw landmarks with feature-specific colors
+    for feature, points_list in FACIAL_LANDMARKS.items():
+        color = colors.get(feature.split('_')[0], (0, 255, 0))
+        for point_idx in points_list:
+            pos = (int(landmarks.landmark[point_idx].x * w), 
+                  int(landmarks.landmark[point_idx].y * h))
+            cv2.circle(viz_image, pos, 2, color, -1)
     
     # Add emotion text
     detected_emotions = [emotion for emotion, is_present in emotions.items() if is_present]
     emotion_text = " + ".join(detected_emotions) if detected_emotions else "Neutral"
     
+    # Create detailed analysis text
+    analysis = f"Expression: {emotion_text}\n\nActive Action Units:\n"
+    au_descriptions = {
+        'AU01': 'Inner Brow Raiser',
+        'AU02': 'Outer Brow Raiser',
+        'AU04': 'Brow Lowerer',
+        'AU05': 'Upper Lid Raiser',
+        'AU12': 'Lip Corner Puller (Smile)',
+        'AU25': 'Lips Part',
+        'AU26': 'Jaw Drop'
+    }
+    
+    active_aus = [f"{au} ({desc})" for au, active in aus.items() if active 
+                 for au_name, desc in au_descriptions.items() if au == au_name]
+    
+    if active_aus:
+        analysis += "\n".join(f"• {au}" for au in active_aus)
+    else:
+        analysis += "No significant Action Units detected"
+    
     # Add text with black background
-    text = f"Expression: {emotion_text}"
     font = cv2.FONT_HERSHEY_SIMPLEX
-    font_scale = 0.8
+    font_scale = 0.7
     thickness = 2
-    (text_w, text_h), _ = cv2.getTextSize(text, font, font_scale, thickness)
-    
-    # Draw black background
-    cv2.rectangle(viz_image, (10, 10), (text_w + 20, text_h + 20), (0, 0, 0), -1)
+    y_pos = 30
     
-    # Draw text
-    cv2.putText(viz_image, text, (15, text_h + 15), font, font_scale, (255, 255, 255), thickness)
+    for line in emotion_text.split('\n'):
+        (text_w, text_h), _ = cv2.getTextSize(line, font, font_scale, thickness)
+        cv2.rectangle(viz_image, (10, y_pos - text_h - 5), (text_w + 20, y_pos + 5), (0, 0, 0), -1)
+        cv2.putText(viz_image, line, (15, y_pos), font, font_scale, (255, 255, 255), thickness)
+        y_pos += text_h + 20
     
-    return viz_image, emotion_text
+    return viz_image, analysis
 
 # Create Gradio interface
 with gr.Blocks(css="app.css") as demo:
@@ -102,13 +198,18 @@ with gr.Blocks(css="app.css") as demo:
             gr.Markdown("""
             ### Instructions:
             1. Upload a clear facial image
-            2. View the detected expression
-            3. Green dots show facial landmarks
+            2. View the detected expression and Action Units (AUs)
+            3. Colored dots show key facial features:
+               - Green: Eyebrows
+               - Yellow: Eyes
+               - Cyan: Nose
+               - Magenta: Mouth
+               - Orange: Jaw
             """)
         
         with gr.Column():
             output_image = gr.Image(label="Analysis")
-            emotion_text = gr.Textbox(label="Detected Expression")
+            analysis_text = gr.Textbox(label="Expression Analysis", lines=8)
     
     # Footer
     with gr.Row(elem_classes="center-content"):
@@ -120,7 +221,7 @@ with gr.Blocks(css="app.css") as demo:
     input_image.change(
         fn=analyze_expression,
         inputs=input_image,
-        outputs=[output_image, emotion_text]
+        outputs=[output_image, analysis_text]
     )
 
 if __name__ == "__main__":