""" Facial Expression Recognition App LittleMonkeyLab | Goldsmiths Observatory """ import gradio as gr import cv2 import mediapipe as mp import numpy as np import os from datetime import datetime # Initialize MediaPipe Face Mesh mp_face_mesh = mp.solutions.face_mesh face_mesh = mp_face_mesh.FaceMesh( static_image_mode=True, max_num_faces=1, refine_landmarks=True, min_detection_confidence=0.5 ) # Define key facial landmarks for expressions FACIAL_LANDMARKS = { 'left_brow': [52, 65, 46], # inner, middle, outer 'right_brow': [285, 295, 276], # inner, middle, outer 'left_eye': [159, 145, 133], # top, bottom, outer 'right_eye': [386, 374, 362], # top, bottom, outer 'nose': [6, 197], # bridge, tip 'mouth': [61, 291, 0, 17, 13, 14], # left corner, right corner, top lip, bottom lip, upper inner, lower inner 'jaw': [17, 84, 314] # center, left, right } def calculate_distances(points, landmarks): """Calculate normalized distances between facial landmarks.""" def distance(p1_idx, p2_idx): try: p1 = points[p1_idx] p2 = points[p2_idx] return np.linalg.norm(p1 - p2) except: return 0.0 # Get face height for normalization face_height = distance(FACIAL_LANDMARKS['nose'][0], FACIAL_LANDMARKS['jaw'][0]) if face_height == 0: return {} measurements = { # Inner brow raising (AU1) 'inner_brow_raise': ( distance(FACIAL_LANDMARKS['left_brow'][0], FACIAL_LANDMARKS['nose'][0]) + distance(FACIAL_LANDMARKS['right_brow'][0], FACIAL_LANDMARKS['nose'][0]) ) / (2 * face_height), # Outer brow raising (AU2) 'outer_brow_raise': ( distance(FACIAL_LANDMARKS['left_brow'][2], FACIAL_LANDMARKS['nose'][0]) + distance(FACIAL_LANDMARKS['right_brow'][2], FACIAL_LANDMARKS['nose'][0]) ) / (2 * face_height), # Brow lowering (AU4) 'brow_furrow': distance(FACIAL_LANDMARKS['left_brow'][0], FACIAL_LANDMARKS['right_brow'][0]) / face_height, # Eye opening (AU5) 'eye_opening': ( distance(FACIAL_LANDMARKS['left_eye'][0], FACIAL_LANDMARKS['left_eye'][1]) + distance(FACIAL_LANDMARKS['right_eye'][0], FACIAL_LANDMARKS['right_eye'][1]) ) / (2 * face_height), # Smile width (AU12) 'smile_width': distance(FACIAL_LANDMARKS['mouth'][0], FACIAL_LANDMARKS['mouth'][1]) / face_height, # Mouth height (AU25/26) 'mouth_opening': distance(FACIAL_LANDMARKS['mouth'][4], FACIAL_LANDMARKS['mouth'][5]) / face_height, # Lip corner height (for smile/frown detection) 'lip_corner_height': ( (points[FACIAL_LANDMARKS['mouth'][0]][1] + points[FACIAL_LANDMARKS['mouth'][1]][1])/2 - points[FACIAL_LANDMARKS['mouth'][2]][1] ) / face_height } return measurements def analyze_expression(image): if image is None: return None, "No image provided", None # Convert to RGB if needed if len(image.shape) == 2: image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB) elif image.shape[2] == 4: image = cv2.cvtColor(image, cv2.COLOR_RGBA2RGB) # Process the image results = face_mesh.process(image) if not results.multi_face_landmarks: return None, "No face detected", None # Get landmarks landmarks = results.multi_face_landmarks[0] points = np.array([[lm.x, lm.y, lm.z] for lm in landmarks.landmark]) # Calculate facial measurements measurements = calculate_distances(points, landmarks) # Analyze Action Units with refined thresholds aus = { 'AU01': measurements['inner_brow_raise'] > 0.12, # Inner Brow Raiser 'AU02': measurements['outer_brow_raise'] > 0.12, # Outer Brow Raiser 'AU04': measurements['brow_furrow'] < 0.2, # Brow Lowerer (tighter threshold for anger) 'AU05': measurements['eye_opening'] > 0.1, # Upper Lid Raiser 'AU12': measurements['smile_width'] > 0.45, # Lip Corner Puller 'AU25': measurements['mouth_opening'] > 0.08, # Lips Part 'AU26': measurements['mouth_opening'] > 0.15 # Jaw Drop } # Refined emotion classification with mutual exclusion emotions = {} # Check Anger first (takes precedence due to distinctive features) if aus['AU04'] and not aus['AU12']: # Lowered brows without smile emotions["Angry"] = True # Happy - clear smile without anger indicators elif aus['AU12'] and measurements['lip_corner_height'] < -0.02 and not aus['AU04']: emotions["Happy"] = True # Sad - raised inner brow with neutral/down mouth elif aus['AU01'] and measurements['lip_corner_height'] > 0.01 and not aus['AU12']: emotions["Sad"] = True # Surprised - raised brows with open mouth elif (aus['AU01'] or aus['AU02']) and (aus['AU25'] or aus['AU26']) and not aus['AU04']: emotions["Surprised"] = True # Neutral - no strong indicators of other emotions elif not any([aus['AU01'], aus['AU02'], aus['AU04'], aus['AU12'], aus['AU26']]) and abs(measurements['lip_corner_height']) < 0.02: emotions["Neutral"] = True else: emotions["Neutral"] = True # Default to neutral if no clear emotion is detected # Create visualization viz_image = image.copy() h, w = viz_image.shape[:2] # Draw facial landmarks with different colors for key points colors = { 'brow': (0, 255, 0), # Green 'eye': (255, 255, 0), # Yellow 'nose': (0, 255, 255), # Cyan 'mouth': (255, 0, 255), # Magenta 'jaw': (255, 128, 0) # Orange } # Draw landmarks with feature-specific colors - made more visible for feature, points_list in FACIAL_LANDMARKS.items(): color = colors.get(feature.split('_')[0], (0, 255, 0)) for point_idx in points_list: pos = (int(landmarks.landmark[point_idx].x * w), int(landmarks.landmark[point_idx].y * h)) # Larger circles with white outline for visibility cv2.circle(viz_image, pos, 4, (255, 255, 255), -1) # White background cv2.circle(viz_image, pos, 3, color, -1) # Colored center # Add emotion text detected_emotions = [emotion for emotion, is_present in emotions.items() if is_present] emotion_text = " + ".join(detected_emotions) if detected_emotions else "Neutral" # Create detailed analysis text analysis = f"Expression: {emotion_text}\n\nActive Action Units:\n" au_descriptions = { 'AU01': 'Inner Brow Raiser', 'AU02': 'Outer Brow Raiser', 'AU04': 'Brow Lowerer', 'AU05': 'Upper Lid Raiser', 'AU12': 'Lip Corner Puller (Smile)', 'AU25': 'Lips Part', 'AU26': 'Jaw Drop' } active_aus = [f"{au}" for au, active in aus.items() if active] aus_text = "_".join(active_aus) if active_aus else "NoAUs" # Create filename with timestamp, emotion, and AUs timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") download_filename = f"FER_{timestamp}_{emotion_text.replace(' + ', '_')}_{aus_text}.jpg" # Add text with black background font = cv2.FONT_HERSHEY_SIMPLEX font_scale = 0.7 thickness = 2 y_pos = 30 for line in emotion_text.split('\n'): (text_w, text_h), _ = cv2.getTextSize(line, font, font_scale, thickness) cv2.rectangle(viz_image, (10, y_pos - text_h - 5), (text_w + 20, y_pos + 5), (0, 0, 0), -1) cv2.putText(viz_image, line, (15, y_pos), font, font_scale, (255, 255, 255), thickness) y_pos += text_h + 20 return viz_image, analysis, download_filename def save_original_image(image, filename): if image is None or filename is None: return None return image # Create Gradio interface with gr.Blocks(css="app.css") as demo: # Header with Observatory logo with gr.Row(elem_classes="header-container"): with gr.Column(): gr.Image("images/LMLOBS.png", show_label=False, container=False, elem_classes="header-logo") gr.Markdown("# Facial Expression Recognition") gr.Markdown("### LittleMonkeyLab | Goldsmiths Observatory") with gr.Row(): with gr.Column(): input_image = gr.Image(label="Upload Image", type="numpy") download_button = gr.Button("Download Original Image with Expression", visible=False) gr.Markdown(""" ### Instructions: 1. Upload a clear facial image 2. View the detected expression and Action Units (AUs) 3. Colored dots show key facial features: - Green: Eyebrows - Yellow: Eyes - Cyan: Nose - Magenta: Mouth - Orange: Jaw 4. Click 'Download' to save the original image """) with gr.Column(): output_image = gr.Image(label="Analysis") analysis_text = gr.Textbox(label="Expression Analysis", lines=8) download_output = gr.File(label="Download", visible=False) # Footer with gr.Row(elem_classes="center-content"): with gr.Column(): gr.Image("images/LMLLOGO.png", show_label=False, container=False, elem_classes="footer-logo") gr.Markdown("© LittleMonkeyLab | Goldsmiths Observatory", elem_classes="footer-text") # Set up the event handlers filename = gr.State() def update_interface(image): viz_image, analysis, download_name = analyze_expression(image) download_button.visible = True if image is not None else False return viz_image, analysis, download_name input_image.change( fn=update_interface, inputs=[input_image], outputs=[output_image, analysis_text, filename] ) download_button.click( fn=save_original_image, inputs=[input_image, filename], outputs=download_output ) if __name__ == "__main__": demo.launch()