Upload StressDetection_V1 (1).ipynb
Browse files- StressDetection_V1 (1).ipynb +1017 -0
StressDetection_V1 (1).ipynb
ADDED
|
@@ -0,0 +1,1017 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"cells": [
|
| 3 |
+
{
|
| 4 |
+
"cell_type": "code",
|
| 5 |
+
"execution_count": null,
|
| 6 |
+
"id": "a69b5d99",
|
| 7 |
+
"metadata": {},
|
| 8 |
+
"outputs": [],
|
| 9 |
+
"source": [
|
| 10 |
+
"# Complete Stress Detection System - 10 Action Units\n",
|
| 11 |
+
"# Real-time Multi-AU Detection with Comprehensive Analysis\n",
|
| 12 |
+
"# Research Assistant: [Your Name]\n",
|
| 13 |
+
"# Guide: Prof. Anup Nandy\n",
|
| 14 |
+
"# Based on Facial Action Coding System (FACS) - Ekman & Friesen\n",
|
| 15 |
+
"\n",
|
| 16 |
+
"import cv2\n",
|
| 17 |
+
"import mediapipe as mp\n",
|
| 18 |
+
"import numpy as np\n",
|
| 19 |
+
"import pandas as pd\n",
|
| 20 |
+
"import matplotlib.pyplot as plt\n",
|
| 21 |
+
"from matplotlib.gridspec import GridSpec\n",
|
| 22 |
+
"from collections import deque\n",
|
| 23 |
+
"import time\n",
|
| 24 |
+
"from datetime import datetime\n",
|
| 25 |
+
"import warnings\n",
|
| 26 |
+
"warnings.filterwarnings('ignore')\n",
|
| 27 |
+
"\n",
|
| 28 |
+
"# ==================== CONFIGURATION ====================\n",
|
| 29 |
+
"WINDOW_SIZE = 30\n",
|
| 30 |
+
"RECORDING_DURATION = 15\n",
|
| 31 |
+
"FPS = 30\n",
|
| 32 |
+
"\n",
|
| 33 |
+
"# ==================== MediaPipe Setup ====================\n",
|
| 34 |
+
"mp_face_mesh = mp.solutions.face_mesh\n",
|
| 35 |
+
"face_mesh = mp_face_mesh.FaceMesh(\n",
|
| 36 |
+
" min_detection_confidence=0.5,\n",
|
| 37 |
+
" min_tracking_confidence=0.5,\n",
|
| 38 |
+
" refine_landmarks=True\n",
|
| 39 |
+
")\n",
|
| 40 |
+
"\n",
|
| 41 |
+
"# ==================== LANDMARK INDICES (468 landmarks) ====================\n",
|
| 42 |
+
"\n",
|
| 43 |
+
"# AU01 - Inner Brow Raiser (Surprise, Fear, Sadness)\n",
|
| 44 |
+
"AU01_LANDMARKS = {\n",
|
| 45 |
+
" 'left_inner_brow': 336,\n",
|
| 46 |
+
" 'right_inner_brow': 107,\n",
|
| 47 |
+
" 'nose_bridge': 6,\n",
|
| 48 |
+
" 'left_outer_brow': 285,\n",
|
| 49 |
+
" 'right_outer_brow': 55\n",
|
| 50 |
+
"}\n",
|
| 51 |
+
"\n",
|
| 52 |
+
"# AU04 - Brow Lowerer (Anger, Sadness, Concentration)\n",
|
| 53 |
+
"AU04_LANDMARKS = {\n",
|
| 54 |
+
" 'left_inner_brow': 336,\n",
|
| 55 |
+
" 'right_inner_brow': 107,\n",
|
| 56 |
+
" 'left_mid_brow': 285,\n",
|
| 57 |
+
" 'right_mid_brow': 55,\n",
|
| 58 |
+
" 'left_eyelid': 159,\n",
|
| 59 |
+
" 'right_eyelid': 386,\n",
|
| 60 |
+
" 'nose_bridge': 6\n",
|
| 61 |
+
"}\n",
|
| 62 |
+
"\n",
|
| 63 |
+
"# AU06 - Cheek Raiser (Genuine Smile - Duchenne)\n",
|
| 64 |
+
"AU06_LANDMARKS = {\n",
|
| 65 |
+
" 'left_cheek': 205,\n",
|
| 66 |
+
" 'right_cheek': 425,\n",
|
| 67 |
+
" 'left_lower_eyelid': 145,\n",
|
| 68 |
+
" 'right_lower_eyelid': 374,\n",
|
| 69 |
+
" 'left_eye_outer': 33,\n",
|
| 70 |
+
" 'right_eye_outer': 263\n",
|
| 71 |
+
"}\n",
|
| 72 |
+
"\n",
|
| 73 |
+
"# AU07 - Lid Tightener (Concentration, Anger, Disgust)\n",
|
| 74 |
+
"AU07_LANDMARKS = {\n",
|
| 75 |
+
" 'left_upper_lid': 159,\n",
|
| 76 |
+
" 'right_upper_lid': 386,\n",
|
| 77 |
+
" 'left_lower_lid': 145,\n",
|
| 78 |
+
" 'right_lower_lid': 374,\n",
|
| 79 |
+
" 'left_eye_top': 159,\n",
|
| 80 |
+
" 'right_eye_top': 386\n",
|
| 81 |
+
"}\n",
|
| 82 |
+
"\n",
|
| 83 |
+
"# AU12 - Lip Corner Puller (Happiness)\n",
|
| 84 |
+
"AU12_LANDMARKS = {\n",
|
| 85 |
+
" 'left_corner': 61,\n",
|
| 86 |
+
" 'right_corner': 291,\n",
|
| 87 |
+
" 'upper_center': 13,\n",
|
| 88 |
+
" 'lower_center': 14\n",
|
| 89 |
+
"}\n",
|
| 90 |
+
"\n",
|
| 91 |
+
"# AU14 - Dimpler (Smile Intensity)\n",
|
| 92 |
+
"AU14_LANDMARKS = {\n",
|
| 93 |
+
" 'left_dimple': 206,\n",
|
| 94 |
+
" 'right_dimple': 426,\n",
|
| 95 |
+
" 'left_corner': 61,\n",
|
| 96 |
+
" 'right_corner': 291\n",
|
| 97 |
+
"}\n",
|
| 98 |
+
"\n",
|
| 99 |
+
"# AU17 - Chin Raiser (Doubt, Sadness, Pouting)\n",
|
| 100 |
+
"AU17_LANDMARKS = {\n",
|
| 101 |
+
" 'chin_center': 152,\n",
|
| 102 |
+
" 'lower_lip': 17,\n",
|
| 103 |
+
" 'chin_left': 176,\n",
|
| 104 |
+
" 'chin_right': 400\n",
|
| 105 |
+
"}\n",
|
| 106 |
+
"\n",
|
| 107 |
+
"# AU23 - Lip Tightener (Anger, Tension)\n",
|
| 108 |
+
"AU23_LANDMARKS = {\n",
|
| 109 |
+
" 'left_corner': 61,\n",
|
| 110 |
+
" 'right_corner': 291,\n",
|
| 111 |
+
" 'left_outer': 57,\n",
|
| 112 |
+
" 'right_outer': 287\n",
|
| 113 |
+
"}\n",
|
| 114 |
+
"\n",
|
| 115 |
+
"# AU24 - Lip Pressor (Stress, Tension, Anger)\n",
|
| 116 |
+
"AU24_LANDMARKS = {\n",
|
| 117 |
+
" 'upper_lip_top': 0,\n",
|
| 118 |
+
" 'upper_lip_bottom': 13,\n",
|
| 119 |
+
" 'lower_lip_top': 14,\n",
|
| 120 |
+
" 'lower_lip_bottom': 17\n",
|
| 121 |
+
"}\n",
|
| 122 |
+
"\n",
|
| 123 |
+
"# AU26 - Jaw Drop (Surprise, Shock, Mouth Opening)\n",
|
| 124 |
+
"AU26_LANDMARKS = {\n",
|
| 125 |
+
" 'upper_lip': 13,\n",
|
| 126 |
+
" 'lower_lip': 14,\n",
|
| 127 |
+
" 'chin': 152,\n",
|
| 128 |
+
" 'nose': 1\n",
|
| 129 |
+
"}\n",
|
| 130 |
+
"\n",
|
| 131 |
+
"# ==================== UTILITY FUNCTIONS ====================\n",
|
| 132 |
+
"def calculate_distance(point1, point2):\n",
|
| 133 |
+
" return np.sqrt((point1[0] - point2[0])**2 + (point1[1] - point2[1])**2)\n",
|
| 134 |
+
"\n",
|
| 135 |
+
"def get_landmark_coords(landmarks, idx, frame_width, frame_height):\n",
|
| 136 |
+
" lm = landmarks[idx]\n",
|
| 137 |
+
" return np.array([lm.x * frame_width, lm.y * frame_height])\n",
|
| 138 |
+
"\n",
|
| 139 |
+
"# ==================== AU DETECTOR CLASSES ====================\n",
|
| 140 |
+
"\n",
|
| 141 |
+
"class AU01Detector:\n",
|
| 142 |
+
" \"\"\"AU01 - Inner Brow Raiser (Surprise, Fear, Worry)\"\"\"\n",
|
| 143 |
+
" def __init__(self, window_size=30):\n",
|
| 144 |
+
" self.name = \"AU01_InnerBrowRaise\"\n",
|
| 145 |
+
" self.activation_history = deque(maxlen=window_size)\n",
|
| 146 |
+
" self.intensity_history = deque(maxlen=window_size)\n",
|
| 147 |
+
" \n",
|
| 148 |
+
" def detect(self, landmarks, frame_width, frame_height):\n",
|
| 149 |
+
" left_inner = get_landmark_coords(landmarks, AU01_LANDMARKS['left_inner_brow'], frame_width, frame_height)\n",
|
| 150 |
+
" right_inner = get_landmark_coords(landmarks, AU01_LANDMARKS['right_inner_brow'], frame_width, frame_height)\n",
|
| 151 |
+
" left_outer = get_landmark_coords(landmarks, AU01_LANDMARKS['left_outer_brow'], frame_width, frame_height)\n",
|
| 152 |
+
" right_outer = get_landmark_coords(landmarks, AU01_LANDMARKS['right_outer_brow'], frame_width, frame_height)\n",
|
| 153 |
+
" nose = get_landmark_coords(landmarks, AU01_LANDMARKS['nose_bridge'], frame_width, frame_height)\n",
|
| 154 |
+
" \n",
|
| 155 |
+
" # Calculate inner vs outer brow height\n",
|
| 156 |
+
" inner_height = ((nose[1] - left_inner[1]) + (nose[1] - right_inner[1])) / 2\n",
|
| 157 |
+
" outer_height = ((nose[1] - left_outer[1]) + (nose[1] - right_outer[1])) / 2\n",
|
| 158 |
+
" \n",
|
| 159 |
+
" # AU01 active when inner brows raised MORE than outer (creates worried look)\n",
|
| 160 |
+
" raise_ratio = inner_height / (outer_height + 1e-6)\n",
|
| 161 |
+
" \n",
|
| 162 |
+
" is_active = raise_ratio > 1.15 # Inner brows 15% higher than outer\n",
|
| 163 |
+
" intensity = min(100, max(0, (raise_ratio - 1.0) * 500))\n",
|
| 164 |
+
" \n",
|
| 165 |
+
" self.activation_history.append(int(is_active))\n",
|
| 166 |
+
" self.intensity_history.append(intensity)\n",
|
| 167 |
+
" \n",
|
| 168 |
+
" return is_active, intensity\n",
|
| 169 |
+
"\n",
|
| 170 |
+
"\n",
|
| 171 |
+
"class AU04Detector:\n",
|
| 172 |
+
" \"\"\"AU04 - Brow Lowerer (Anger, Concentration, Stress)\"\"\"\n",
|
| 173 |
+
" def __init__(self, window_size=30):\n",
|
| 174 |
+
" self.name = \"AU04_BrowLower\"\n",
|
| 175 |
+
" self.activation_history = deque(maxlen=window_size)\n",
|
| 176 |
+
" self.intensity_history = deque(maxlen=window_size)\n",
|
| 177 |
+
" \n",
|
| 178 |
+
" def detect(self, landmarks, frame_width, frame_height):\n",
|
| 179 |
+
" left_inner_brow = get_landmark_coords(landmarks, AU04_LANDMARKS['left_inner_brow'], frame_width, frame_height)\n",
|
| 180 |
+
" right_inner_brow = get_landmark_coords(landmarks, AU04_LANDMARKS['right_inner_brow'], frame_width, frame_height)\n",
|
| 181 |
+
" left_eyelid = get_landmark_coords(landmarks, AU04_LANDMARKS['left_eyelid'], frame_width, frame_height)\n",
|
| 182 |
+
" right_eyelid = get_landmark_coords(landmarks, AU04_LANDMARKS['right_eyelid'], frame_width, frame_height)\n",
|
| 183 |
+
" nose_bridge = get_landmark_coords(landmarks, AU04_LANDMARKS['nose_bridge'], frame_width, frame_height)\n",
|
| 184 |
+
" \n",
|
| 185 |
+
" left_brow_eyelid_dist = left_inner_brow[1] - left_eyelid[1]\n",
|
| 186 |
+
" right_brow_eyelid_dist = right_inner_brow[1] - right_eyelid[1]\n",
|
| 187 |
+
" avg_brow_eyelid_dist = (left_brow_eyelid_dist + right_brow_eyelid_dist) / 2\n",
|
| 188 |
+
" \n",
|
| 189 |
+
" face_height = calculate_distance(left_inner_brow, nose_bridge)\n",
|
| 190 |
+
" normalized_distance = avg_brow_eyelid_dist / (face_height + 1e-6)\n",
|
| 191 |
+
" \n",
|
| 192 |
+
" inner_brow_distance = calculate_distance(left_inner_brow, right_inner_brow)\n",
|
| 193 |
+
" outer_eye_distance = calculate_distance(left_eyelid, right_eyelid)\n",
|
| 194 |
+
" brow_compression_ratio = inner_brow_distance / (outer_eye_distance + 1e-6)\n",
|
| 195 |
+
" \n",
|
| 196 |
+
" is_active = (normalized_distance > -0.30 or brow_compression_ratio < 0.95)\n",
|
| 197 |
+
" intensity = min(100, max(0, (normalized_distance + 0.40) / 0.40 * 100))\n",
|
| 198 |
+
" \n",
|
| 199 |
+
" self.activation_history.append(int(is_active))\n",
|
| 200 |
+
" self.intensity_history.append(intensity)\n",
|
| 201 |
+
" \n",
|
| 202 |
+
" return is_active, intensity\n",
|
| 203 |
+
"\n",
|
| 204 |
+
"\n",
|
| 205 |
+
"class AU06Detector:\n",
|
| 206 |
+
" \"\"\"AU06 - Cheek Raiser (Genuine Smile)\"\"\"\n",
|
| 207 |
+
" def __init__(self, window_size=30):\n",
|
| 208 |
+
" self.name = \"AU06_CheekRaise\"\n",
|
| 209 |
+
" self.activation_history = deque(maxlen=window_size)\n",
|
| 210 |
+
" self.intensity_history = deque(maxlen=window_size)\n",
|
| 211 |
+
" \n",
|
| 212 |
+
" def detect(self, landmarks, frame_width, frame_height):\n",
|
| 213 |
+
" left_cheek = get_landmark_coords(landmarks, AU06_LANDMARKS['left_cheek'], frame_width, frame_height)\n",
|
| 214 |
+
" right_cheek = get_landmark_coords(landmarks, AU06_LANDMARKS['right_cheek'], frame_width, frame_height)\n",
|
| 215 |
+
" left_lower_lid = get_landmark_coords(landmarks, AU06_LANDMARKS['left_lower_eyelid'], frame_width, frame_height)\n",
|
| 216 |
+
" right_lower_lid = get_landmark_coords(landmarks, AU06_LANDMARKS['right_lower_eyelid'], frame_width, frame_height)\n",
|
| 217 |
+
" \n",
|
| 218 |
+
" # When cheeks raise, distance between cheek and lower eyelid decreases\n",
|
| 219 |
+
" left_distance = calculate_distance(left_cheek, left_lower_lid)\n",
|
| 220 |
+
" right_distance = calculate_distance(right_cheek, right_lower_lid)\n",
|
| 221 |
+
" avg_distance = (left_distance + right_distance) / 2\n",
|
| 222 |
+
" \n",
|
| 223 |
+
" # Also check if lower eyelid moves up\n",
|
| 224 |
+
" left_eye_outer = get_landmark_coords(landmarks, AU06_LANDMARKS['left_eye_outer'], frame_width, frame_height)\n",
|
| 225 |
+
" eye_height = abs(left_eye_outer[1] - left_lower_lid[1])\n",
|
| 226 |
+
" \n",
|
| 227 |
+
" cheek_raise_score = eye_height / (avg_distance + 1e-6)\n",
|
| 228 |
+
" \n",
|
| 229 |
+
" is_active = cheek_raise_score > 0.8\n",
|
| 230 |
+
" intensity = min(100, max(0, (cheek_raise_score - 0.5) * 200))\n",
|
| 231 |
+
" \n",
|
| 232 |
+
" self.activation_history.append(int(is_active))\n",
|
| 233 |
+
" self.intensity_history.append(intensity)\n",
|
| 234 |
+
" \n",
|
| 235 |
+
" return is_active, intensity\n",
|
| 236 |
+
"\n",
|
| 237 |
+
"\n",
|
| 238 |
+
"class AU07Detector:\n",
|
| 239 |
+
" \"\"\"AU07 - Lid Tightener (Tension, Squinting)\"\"\"\n",
|
| 240 |
+
" def __init__(self, window_size=30):\n",
|
| 241 |
+
" self.name = \"AU07_LidTighten\"\n",
|
| 242 |
+
" self.activation_history = deque(maxlen=window_size)\n",
|
| 243 |
+
" self.intensity_history = deque(maxlen=window_size)\n",
|
| 244 |
+
" \n",
|
| 245 |
+
" def detect(self, landmarks, frame_width, frame_height):\n",
|
| 246 |
+
" left_upper = get_landmark_coords(landmarks, AU07_LANDMARKS['left_upper_lid'], frame_width, frame_height)\n",
|
| 247 |
+
" right_upper = get_landmark_coords(landmarks, AU07_LANDMARKS['right_upper_lid'], frame_width, frame_height)\n",
|
| 248 |
+
" left_lower = get_landmark_coords(landmarks, AU07_LANDMARKS['left_lower_lid'], frame_width, frame_height)\n",
|
| 249 |
+
" right_lower = get_landmark_coords(landmarks, AU07_LANDMARKS['right_lower_lid'], frame_width, frame_height)\n",
|
| 250 |
+
" \n",
|
| 251 |
+
" # Eye opening (smaller = more tightened)\n",
|
| 252 |
+
" left_eye_opening = abs(left_upper[1] - left_lower[1])\n",
|
| 253 |
+
" right_eye_opening = abs(right_upper[1] - right_lower[1])\n",
|
| 254 |
+
" avg_eye_opening = (left_eye_opening + right_eye_opening) / 2\n",
|
| 255 |
+
" \n",
|
| 256 |
+
" # Normalize by face height\n",
|
| 257 |
+
" face_ref = calculate_distance(left_upper, \n",
|
| 258 |
+
" get_landmark_coords(landmarks, 152, frame_width, frame_height))\n",
|
| 259 |
+
" normalized_opening = avg_eye_opening / (face_ref + 1e-6)\n",
|
| 260 |
+
" \n",
|
| 261 |
+
" is_active = normalized_opening < 0.025 # Eyes tightened/squinted\n",
|
| 262 |
+
" intensity = min(100, max(0, (0.035 - normalized_opening) / 0.035 * 100))\n",
|
| 263 |
+
" \n",
|
| 264 |
+
" self.activation_history.append(int(is_active))\n",
|
| 265 |
+
" self.intensity_history.append(intensity)\n",
|
| 266 |
+
" \n",
|
| 267 |
+
" return is_active, intensity\n",
|
| 268 |
+
"\n",
|
| 269 |
+
"\n",
|
| 270 |
+
"class AU12Detector:\n",
|
| 271 |
+
" \"\"\"AU12 - Lip Corner Puller (Smile)\"\"\"\n",
|
| 272 |
+
" def __init__(self, window_size=30):\n",
|
| 273 |
+
" self.name = \"AU12_LipCornerPull\"\n",
|
| 274 |
+
" self.activation_history = deque(maxlen=window_size)\n",
|
| 275 |
+
" self.intensity_history = deque(maxlen=window_size)\n",
|
| 276 |
+
" \n",
|
| 277 |
+
" def detect(self, landmarks, frame_width, frame_height):\n",
|
| 278 |
+
" left_corner = get_landmark_coords(landmarks, AU12_LANDMARKS['left_corner'], frame_width, frame_height)\n",
|
| 279 |
+
" right_corner = get_landmark_coords(landmarks, AU12_LANDMARKS['right_corner'], frame_width, frame_height)\n",
|
| 280 |
+
" upper_center = get_landmark_coords(landmarks, AU12_LANDMARKS['upper_center'], frame_width, frame_height)\n",
|
| 281 |
+
" lower_center = get_landmark_coords(landmarks, AU12_LANDMARKS['lower_center'], frame_width, frame_height)\n",
|
| 282 |
+
" \n",
|
| 283 |
+
" mouth_width = calculate_distance(left_corner, right_corner)\n",
|
| 284 |
+
" mouth_height = calculate_distance(upper_center, lower_center)\n",
|
| 285 |
+
" mouth_center_y = (upper_center[1] + lower_center[1]) / 2\n",
|
| 286 |
+
" \n",
|
| 287 |
+
" left_corner_lift = mouth_center_y - left_corner[1]\n",
|
| 288 |
+
" right_corner_lift = mouth_center_y - right_corner[1]\n",
|
| 289 |
+
" avg_corner_lift = (left_corner_lift + right_corner_lift) / 2\n",
|
| 290 |
+
" \n",
|
| 291 |
+
" mouth_ratio = mouth_width / (mouth_height + 1e-6)\n",
|
| 292 |
+
" normalized_lift = avg_corner_lift / mouth_height if mouth_height > 0 else 0\n",
|
| 293 |
+
" \n",
|
| 294 |
+
" lift_difference = abs(left_corner_lift - right_corner_lift)\n",
|
| 295 |
+
" symmetry_score = 1.0 - min(1.0, lift_difference / (mouth_height + 1e-6))\n",
|
| 296 |
+
" \n",
|
| 297 |
+
" is_active = (normalized_lift > 0.25 and mouth_ratio > 2.8 and symmetry_score > 0.6)\n",
|
| 298 |
+
" intensity = min(100, max(0, normalized_lift * 250))\n",
|
| 299 |
+
" \n",
|
| 300 |
+
" self.activation_history.append(int(is_active))\n",
|
| 301 |
+
" self.intensity_history.append(intensity)\n",
|
| 302 |
+
" \n",
|
| 303 |
+
" return is_active, intensity\n",
|
| 304 |
+
"\n",
|
| 305 |
+
"\n",
|
| 306 |
+
"class AU14Detector:\n",
|
| 307 |
+
" \"\"\"AU14 - Dimpler (Smile Depth Indicator)\"\"\"\n",
|
| 308 |
+
" def __init__(self, window_size=30):\n",
|
| 309 |
+
" self.name = \"AU14_Dimpler\"\n",
|
| 310 |
+
" self.activation_history = deque(maxlen=window_size)\n",
|
| 311 |
+
" self.intensity_history = deque(maxlen=window_size)\n",
|
| 312 |
+
" \n",
|
| 313 |
+
" def detect(self, landmarks, frame_width, frame_height):\n",
|
| 314 |
+
" left_dimple = get_landmark_coords(landmarks, AU14_LANDMARKS['left_dimple'], frame_width, frame_height)\n",
|
| 315 |
+
" right_dimple = get_landmark_coords(landmarks, AU14_LANDMARKS['right_dimple'], frame_width, frame_height)\n",
|
| 316 |
+
" left_corner = get_landmark_coords(landmarks, AU14_LANDMARKS['left_corner'], frame_width, frame_height)\n",
|
| 317 |
+
" right_corner = get_landmark_coords(landmarks, AU14_LANDMARKS['right_corner'], frame_width, frame_height)\n",
|
| 318 |
+
" \n",
|
| 319 |
+
" # Dimples appear when corners pull back and create indentation\n",
|
| 320 |
+
" left_depth = calculate_distance(left_dimple, left_corner)\n",
|
| 321 |
+
" right_depth = calculate_distance(right_dimple, right_corner)\n",
|
| 322 |
+
" avg_depth = (left_depth + right_depth) / 2\n",
|
| 323 |
+
" \n",
|
| 324 |
+
" # Check corner retraction\n",
|
| 325 |
+
" corner_distance = calculate_distance(left_corner, right_corner)\n",
|
| 326 |
+
" dimple_score = avg_depth / (corner_distance + 1e-6)\n",
|
| 327 |
+
" \n",
|
| 328 |
+
" is_active = dimple_score > 0.15\n",
|
| 329 |
+
" intensity = min(100, max(0, (dimple_score - 0.10) * 500))\n",
|
| 330 |
+
" \n",
|
| 331 |
+
" self.activation_history.append(int(is_active))\n",
|
| 332 |
+
" self.intensity_history.append(intensity)\n",
|
| 333 |
+
" \n",
|
| 334 |
+
" return is_active, intensity\n",
|
| 335 |
+
"\n",
|
| 336 |
+
"\n",
|
| 337 |
+
"class AU17Detector:\n",
|
| 338 |
+
" \"\"\"AU17 - Chin Raiser (Doubt, Pouting, Sadness)\"\"\"\n",
|
| 339 |
+
" def __init__(self, window_size=30):\n",
|
| 340 |
+
" self.name = \"AU17_ChinRaise\"\n",
|
| 341 |
+
" self.activation_history = deque(maxlen=window_size)\n",
|
| 342 |
+
" self.intensity_history = deque(maxlen=window_size)\n",
|
| 343 |
+
" \n",
|
| 344 |
+
" def detect(self, landmarks, frame_width, frame_height):\n",
|
| 345 |
+
" chin = get_landmark_coords(landmarks, AU17_LANDMARKS['chin_center'], frame_width, frame_height)\n",
|
| 346 |
+
" lower_lip = get_landmark_coords(landmarks, AU17_LANDMARKS['lower_lip'], frame_width, frame_height)\n",
|
| 347 |
+
" \n",
|
| 348 |
+
" # When chin raises, distance between chin and lower lip decreases\n",
|
| 349 |
+
" chin_lip_distance = calculate_distance(chin, lower_lip)\n",
|
| 350 |
+
" \n",
|
| 351 |
+
" # Normalize by face height\n",
|
| 352 |
+
" nose = get_landmark_coords(landmarks, 1, frame_width, frame_height)\n",
|
| 353 |
+
" face_height = calculate_distance(nose, chin)\n",
|
| 354 |
+
" normalized_distance = chin_lip_distance / (face_height + 1e-6)\n",
|
| 355 |
+
" \n",
|
| 356 |
+
" is_active = normalized_distance < 0.08 # Chin pushed up\n",
|
| 357 |
+
" intensity = min(100, max(0, (0.12 - normalized_distance) / 0.12 * 100))\n",
|
| 358 |
+
" \n",
|
| 359 |
+
" self.activation_history.append(int(is_active))\n",
|
| 360 |
+
" self.intensity_history.append(intensity)\n",
|
| 361 |
+
" \n",
|
| 362 |
+
" return is_active, intensity\n",
|
| 363 |
+
"\n",
|
| 364 |
+
"\n",
|
| 365 |
+
"class AU23Detector:\n",
|
| 366 |
+
" \"\"\"AU23 - Lip Tightener (Anger, Tension)\"\"\"\n",
|
| 367 |
+
" def __init__(self, window_size=30):\n",
|
| 368 |
+
" self.name = \"AU23_LipTighten\"\n",
|
| 369 |
+
" self.activation_history = deque(maxlen=window_size)\n",
|
| 370 |
+
" self.intensity_history = deque(maxlen=window_size)\n",
|
| 371 |
+
" \n",
|
| 372 |
+
" def detect(self, landmarks, frame_width, frame_height):\n",
|
| 373 |
+
" left_corner = get_landmark_coords(landmarks, AU23_LANDMARKS['left_corner'], frame_width, frame_height)\n",
|
| 374 |
+
" right_corner = get_landmark_coords(landmarks, AU23_LANDMARKS['right_corner'], frame_width, frame_height)\n",
|
| 375 |
+
" left_outer = get_landmark_coords(landmarks, AU23_LANDMARKS['left_outer'], frame_width, frame_height)\n",
|
| 376 |
+
" right_outer = get_landmark_coords(landmarks, AU23_LANDMARKS['right_outer'], frame_width, frame_height)\n",
|
| 377 |
+
" \n",
|
| 378 |
+
" corner_width = calculate_distance(left_corner, right_corner)\n",
|
| 379 |
+
" outer_width = calculate_distance(left_outer, right_outer)\n",
|
| 380 |
+
" tightness_ratio = corner_width / (outer_width + 1e-6)\n",
|
| 381 |
+
" \n",
|
| 382 |
+
" is_active = (tightness_ratio < 0.85)\n",
|
| 383 |
+
" intensity = min(100, max(0, (0.95 - tightness_ratio) / 0.20 * 100))\n",
|
| 384 |
+
" \n",
|
| 385 |
+
" self.activation_history.append(int(is_active))\n",
|
| 386 |
+
" self.intensity_history.append(intensity)\n",
|
| 387 |
+
" \n",
|
| 388 |
+
" return is_active, intensity\n",
|
| 389 |
+
"\n",
|
| 390 |
+
"\n",
|
| 391 |
+
"class AU24Detector:\n",
|
| 392 |
+
" \"\"\"AU24 - Lip Pressor (Stress, Tension)\"\"\"\n",
|
| 393 |
+
" def __init__(self, window_size=30):\n",
|
| 394 |
+
" self.name = \"AU24_LipPress\"\n",
|
| 395 |
+
" self.activation_history = deque(maxlen=window_size)\n",
|
| 396 |
+
" self.intensity_history = deque(maxlen=window_size)\n",
|
| 397 |
+
" \n",
|
| 398 |
+
" def detect(self, landmarks, frame_width, frame_height):\n",
|
| 399 |
+
" upper_lip_top = get_landmark_coords(landmarks, AU24_LANDMARKS['upper_lip_top'], frame_width, frame_height)\n",
|
| 400 |
+
" upper_lip_bottom = get_landmark_coords(landmarks, AU24_LANDMARKS['upper_lip_bottom'], frame_width, frame_height)\n",
|
| 401 |
+
" lower_lip_top = get_landmark_coords(landmarks, AU24_LANDMARKS['lower_lip_top'], frame_width, frame_height)\n",
|
| 402 |
+
" lower_lip_bottom = get_landmark_coords(landmarks, AU24_LANDMARKS['lower_lip_bottom'], frame_width, frame_height)\n",
|
| 403 |
+
" \n",
|
| 404 |
+
" upper_lip_thickness = calculate_distance(upper_lip_top, upper_lip_bottom)\n",
|
| 405 |
+
" lower_lip_thickness = calculate_distance(lower_lip_top, lower_lip_bottom)\n",
|
| 406 |
+
" total_lip_thickness = upper_lip_thickness + lower_lip_thickness\n",
|
| 407 |
+
" \n",
|
| 408 |
+
" mouth_opening = calculate_distance(upper_lip_bottom, lower_lip_top)\n",
|
| 409 |
+
" lip_press_score = mouth_opening / (total_lip_thickness + 1e-6)\n",
|
| 410 |
+
" \n",
|
| 411 |
+
" is_active = (lip_press_score < 0.4 and total_lip_thickness < 15)\n",
|
| 412 |
+
" intensity = min(100, max(0, (0.6 - lip_press_score) / 0.6 * 100))\n",
|
| 413 |
+
" \n",
|
| 414 |
+
" self.activation_history.append(int(is_active))\n",
|
| 415 |
+
" self.intensity_history.append(intensity)\n",
|
| 416 |
+
" \n",
|
| 417 |
+
" return is_active, intensity\n",
|
| 418 |
+
"\n",
|
| 419 |
+
"\n",
|
| 420 |
+
"class AU26Detector:\n",
|
| 421 |
+
" \"\"\"AU26 - Jaw Drop (Surprise, Shock)\"\"\"\n",
|
| 422 |
+
" def __init__(self, window_size=30):\n",
|
| 423 |
+
" self.name = \"AU26_JawDrop\"\n",
|
| 424 |
+
" self.activation_history = deque(maxlen=window_size)\n",
|
| 425 |
+
" self.intensity_history = deque(maxlen=window_size)\n",
|
| 426 |
+
" \n",
|
| 427 |
+
" def detect(self, landmarks, frame_width, frame_height):\n",
|
| 428 |
+
" upper_lip = get_landmark_coords(landmarks, AU26_LANDMARKS['upper_lip'], frame_width, frame_height)\n",
|
| 429 |
+
" lower_lip = get_landmark_coords(landmarks, AU26_LANDMARKS['lower_lip'], frame_width, frame_height)\n",
|
| 430 |
+
" chin = get_landmark_coords(landmarks, AU26_LANDMARKS['chin'], frame_width, frame_height)\n",
|
| 431 |
+
" nose = get_landmark_coords(landmarks, AU26_LANDMARKS['nose'], frame_width, frame_height)\n",
|
| 432 |
+
" \n",
|
| 433 |
+
" # Mouth opening\n",
|
| 434 |
+
" mouth_opening = calculate_distance(upper_lip, lower_lip)\n",
|
| 435 |
+
" \n",
|
| 436 |
+
" # Jaw drop (distance from nose to chin increases)\n",
|
| 437 |
+
" jaw_length = calculate_distance(nose, chin)\n",
|
| 438 |
+
" \n",
|
| 439 |
+
" # Normalize\n",
|
| 440 |
+
" mouth_opening_ratio = mouth_opening / (jaw_length + 1e-6)\n",
|
| 441 |
+
" \n",
|
| 442 |
+
" is_active = mouth_opening_ratio > 0.15 # Mouth significantly open\n",
|
| 443 |
+
" intensity = min(100, max(0, (mouth_opening_ratio - 0.10) / 0.20 * 100))\n",
|
| 444 |
+
" \n",
|
| 445 |
+
" self.activation_history.append(int(is_active))\n",
|
| 446 |
+
" self.intensity_history.append(intensity)\n",
|
| 447 |
+
" \n",
|
| 448 |
+
" return is_active, intensity\n",
|
| 449 |
+
"\n",
|
| 450 |
+
"\n",
|
| 451 |
+
"# ==================== FEATURE EXTRACTOR ====================\n",
|
| 452 |
+
"\n",
|
| 453 |
+
"class MultiAUFeatureExtractor:\n",
|
| 454 |
+
" def __init__(self, detectors):\n",
|
| 455 |
+
" self.detectors = detectors\n",
|
| 456 |
+
" self.feature_log = []\n",
|
| 457 |
+
" \n",
|
| 458 |
+
" def extract_features(self, timestamp):\n",
|
| 459 |
+
" features = {'timestamp': timestamp}\n",
|
| 460 |
+
" \n",
|
| 461 |
+
" for detector in self.detectors:\n",
|
| 462 |
+
" is_active = detector.activation_history[-1] if detector.activation_history else 0\n",
|
| 463 |
+
" intensity = detector.intensity_history[-1] if detector.intensity_history else 0\n",
|
| 464 |
+
" \n",
|
| 465 |
+
" activation_rate = sum(detector.activation_history) / len(detector.activation_history) if detector.activation_history else 0\n",
|
| 466 |
+
" avg_intensity = np.mean(detector.intensity_history) if detector.intensity_history else 0\n",
|
| 467 |
+
" max_intensity = np.max(detector.intensity_history) if detector.intensity_history else 0\n",
|
| 468 |
+
" intensity_std = np.std(detector.intensity_history) if detector.intensity_history else 0\n",
|
| 469 |
+
" \n",
|
| 470 |
+
" features[f'{detector.name}_active'] = is_active\n",
|
| 471 |
+
" features[f'{detector.name}_intensity'] = intensity\n",
|
| 472 |
+
" features[f'{detector.name}_activation_rate'] = activation_rate\n",
|
| 473 |
+
" features[f'{detector.name}_avg_intensity'] = avg_intensity\n",
|
| 474 |
+
" features[f'{detector.name}_max_intensity'] = max_intensity\n",
|
| 475 |
+
" features[f'{detector.name}_intensity_std'] = intensity_std\n",
|
| 476 |
+
" \n",
|
| 477 |
+
" self.feature_log.append(features)\n",
|
| 478 |
+
" return features\n",
|
| 479 |
+
" \n",
|
| 480 |
+
" def get_dataframe(self):\n",
|
| 481 |
+
" return pd.DataFrame(self.feature_log)\n",
|
| 482 |
+
" \n",
|
| 483 |
+
" def save_features(self, filename):\n",
|
| 484 |
+
" df = self.get_dataframe()\n",
|
| 485 |
+
" df.to_csv(filename, index=False)\n",
|
| 486 |
+
" print(f\"β Features saved to {filename}\")\n",
|
| 487 |
+
"\n",
|
| 488 |
+
"\n",
|
| 489 |
+
"# ==================== DETECTION SESSION ====================\n",
|
| 490 |
+
"\n",
|
| 491 |
+
"def run_detection_session(duration_seconds=15, save_data=True):\n",
|
| 492 |
+
" # Initialize all 10 AU detectors\n",
|
| 493 |
+
" au01 = AU01Detector()\n",
|
| 494 |
+
" au04 = AU04Detector()\n",
|
| 495 |
+
" au06 = AU06Detector()\n",
|
| 496 |
+
" au07 = AU07Detector()\n",
|
| 497 |
+
" au12 = AU12Detector()\n",
|
| 498 |
+
" au14 = AU14Detector()\n",
|
| 499 |
+
" au17 = AU17Detector()\n",
|
| 500 |
+
" au23 = AU23Detector()\n",
|
| 501 |
+
" au24 = AU24Detector()\n",
|
| 502 |
+
" au26 = AU26Detector()\n",
|
| 503 |
+
" \n",
|
| 504 |
+
" detectors = [au01, au04, au06, au07, au12, au14, au17, au23, au24, au26]\n",
|
| 505 |
+
" feature_extractor = MultiAUFeatureExtractor(detectors)\n",
|
| 506 |
+
" \n",
|
| 507 |
+
" cap = cv2.VideoCapture(0)\n",
|
| 508 |
+
" \n",
|
| 509 |
+
" print(f\"\\n{'='*70}\")\n",
|
| 510 |
+
" print(f\" COMPLETE 10-AU STRESS DETECTION SYSTEM\")\n",
|
| 511 |
+
" print(f\" Recording for {duration_seconds} seconds...\")\n",
|
| 512 |
+
" print(f\"{'='*70}\\n\")\n",
|
| 513 |
+
" \n",
|
| 514 |
+
" start_time = time.time()\n",
|
| 515 |
+
" frame_count = 0\n",
|
| 516 |
+
" \n",
|
| 517 |
+
" while True:\n",
|
| 518 |
+
" ret, frame = cap.read()\n",
|
| 519 |
+
" if not ret:\n",
|
| 520 |
+
" break\n",
|
| 521 |
+
" \n",
|
| 522 |
+
" current_time = time.time()\n",
|
| 523 |
+
" elapsed = current_time - start_time\n",
|
| 524 |
+
" \n",
|
| 525 |
+
" if elapsed >= duration_seconds:\n",
|
| 526 |
+
" break\n",
|
| 527 |
+
" \n",
|
| 528 |
+
" frame = cv2.flip(frame, 1)\n",
|
| 529 |
+
" frame_height, frame_width = frame.shape[:2]\n",
|
| 530 |
+
" rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n",
|
| 531 |
+
" \n",
|
| 532 |
+
" results = face_mesh.process(rgb_frame)\n",
|
| 533 |
+
" \n",
|
| 534 |
+
" if results.multi_face_landmarks:\n",
|
| 535 |
+
" landmarks = results.multi_face_landmarks[0].landmark\n",
|
| 536 |
+
" \n",
|
| 537 |
+
" # Detect all 10 AUs\n",
|
| 538 |
+
" au01_active, au01_intensity = au01.detect(landmarks, frame_width, frame_height)\n",
|
| 539 |
+
" au04_active, au04_intensity = au04.detect(landmarks, frame_width, frame_height)\n",
|
| 540 |
+
" au06_active, au06_intensity = au06.detect(landmarks, frame_width, frame_height)\n",
|
| 541 |
+
" au07_active, au07_intensity = au07.detect(landmarks, frame_width, frame_height)\n",
|
| 542 |
+
" au12_active, au12_intensity = au12.detect(landmarks, frame_width, frame_height)\n",
|
| 543 |
+
" au14_active, au14_intensity = au14.detect(landmarks, frame_width, frame_height)\n",
|
| 544 |
+
" au17_active, au17_intensity = au17.detect(landmarks, frame_width, frame_height)\n",
|
| 545 |
+
" au23_active, au23_intensity = au23.detect(landmarks, frame_width, frame_height)\n",
|
| 546 |
+
" au24_active, au24_intensity = au24.detect(landmarks, frame_width, frame_height)\n",
|
| 547 |
+
" au26_active, au26_intensity = au26.detect(landmarks, frame_width, frame_height)\n",
|
| 548 |
+
" \n",
|
| 549 |
+
" features = feature_extractor.extract_features(elapsed)\n",
|
| 550 |
+
" \n",
|
| 551 |
+
" # Display (2 columns)\n",
|
| 552 |
+
" y_offset = 25\n",
|
| 553 |
+
" col1_x = 10\n",
|
| 554 |
+
" col2_x = frame_width // 2 + 10\n",
|
| 555 |
+
" \n",
|
| 556 |
+
" # Header\n",
|
| 557 |
+
" cv2.putText(frame, f\"Time: {elapsed:.1f}s / {duration_seconds}s\", \n",
|
| 558 |
+
" (10, y_offset), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2)\n",
|
| 559 |
+
" \n",
|
| 560 |
+
" y_offset += 35\n",
|
| 561 |
+
" \n",
|
| 562 |
+
" # Column 1: Stress Indicators\n",
|
| 563 |
+
" cv2.putText(frame, \"STRESS INDICATORS:\", (col1_x, y_offset), \n",
|
| 564 |
+
" cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 2)\n",
|
| 565 |
+
" y_offset += 25\n",
|
| 566 |
+
" \n",
|
| 567 |
+
" stress_aus = [\n",
|
| 568 |
+
" (au01_active, au01_intensity, \"AU01-BrowRaise\"),\n",
|
| 569 |
+
" (au04_active, au04_intensity, \"AU04-BrowLower\"),\n",
|
| 570 |
+
" (au07_active, au07_intensity, \"AU07-LidTight\"),\n",
|
| 571 |
+
" (au17_active, au17_intensity, \"AU17-ChinRaise\"),\n",
|
| 572 |
+
" (au23_active, au23_intensity, \"AU23-LipTight\"),\n",
|
| 573 |
+
" (au24_active, au24_intensity, \"AU24-LipPress\")\n",
|
| 574 |
+
" ]\n",
|
| 575 |
+
" \n",
|
| 576 |
+
" for active, intensity, name in stress_aus:\n",
|
| 577 |
+
" color = (0, 0, 255) if active else (100, 100, 100)\n",
|
| 578 |
+
" cv2.putText(frame, f\"{name}: {intensity:.0f}%\", \n",
|
| 579 |
+
" (col1_x, y_offset), cv2.FONT_HERSHEY_SIMPLEX, 0.4, color, 1)\n",
|
| 580 |
+
" y_offset += 20\n",
|
| 581 |
+
" \n",
|
| 582 |
+
" # Column 2: Positive Indicators\n",
|
| 583 |
+
" y_offset = 60\n",
|
| 584 |
+
" cv2.putText(frame, \"POSITIVE INDICATORS:\", (col2_x, y_offset), \n",
|
| 585 |
+
" cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)\n",
|
| 586 |
+
" y_offset += 25\n",
|
| 587 |
+
" \n",
|
| 588 |
+
" positive_aus = [\n",
|
| 589 |
+
" (au06_active, au06_intensity, \"AU06-CheekRaise\"),\n",
|
| 590 |
+
" (au12_active, au12_intensity, \"AU12-SmilePull\"),\n",
|
| 591 |
+
" (au14_active, au14_intensity, \"AU14-Dimpler\"),\n",
|
| 592 |
+
" (au26_active, au26_intensity, \"AU26-JawDrop\")\n",
|
| 593 |
+
" ]\n",
|
| 594 |
+
" \n",
|
| 595 |
+
" for active, intensity, name in positive_aus:\n",
|
| 596 |
+
" color = (0, 255, 0) if active else (100, 100, 100)\n",
|
| 597 |
+
" cv2.putText(frame, f\"{name}: {intensity:.0f}%\", \n",
|
| 598 |
+
" (col2_x, y_offset), cv2.FONT_HERSHEY_SIMPLEX, 0.4, color, 1)\n",
|
| 599 |
+
" y_offset += 20\n",
|
| 600 |
+
" \n",
|
| 601 |
+
" # Bottom summary\n",
|
| 602 |
+
" stress_count = sum([au01_active, au04_active, au07_active, au17_active, au23_active, au24_active])\n",
|
| 603 |
+
" positive_count = sum([au06_active, au12_active, au14_active])\n",
|
| 604 |
+
" \n",
|
| 605 |
+
" cv2.rectangle(frame, (10, frame_height - 60), (frame_width - 10, frame_height - 10), (50, 50, 50), -1)\n",
|
| 606 |
+
" cv2.putText(frame, f\"Stress AUs: {stress_count}/6 | Positive AUs: {positive_count}/4\", \n",
|
| 607 |
+
" (20, frame_height - 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2)\n",
|
| 608 |
+
" \n",
|
| 609 |
+
" cv2.imshow('Complete 10-AU Stress Detection', frame)\n",
|
| 610 |
+
" frame_count += 1\n",
|
| 611 |
+
" \n",
|
| 612 |
+
" if cv2.waitKey(1) & 0xFF == ord('q'):\n",
|
| 613 |
+
" break\n",
|
| 614 |
+
" \n",
|
| 615 |
+
" cap.release()\n",
|
| 616 |
+
" cv2.destroyAllWindows()\n",
|
| 617 |
+
" \n",
|
| 618 |
+
" if save_data:\n",
|
| 619 |
+
" timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
|
| 620 |
+
" filename = f\"complete_au_features_{timestamp}.csv\"\n",
|
| 621 |
+
" feature_extractor.save_features(filename)\n",
|
| 622 |
+
" \n",
|
| 623 |
+
" print(f\"\\nβ Session complete! Processed {frame_count} frames\")\n",
|
| 624 |
+
" print(f\"β Average FPS: {frame_count/duration_seconds:.1f}\")\n",
|
| 625 |
+
" \n",
|
| 626 |
+
" return feature_extractor.get_dataframe()\n",
|
| 627 |
+
"\n",
|
| 628 |
+
"\n",
|
| 629 |
+
"# ==================== ADVANCED ANALYSIS ====================\n",
|
| 630 |
+
"\n",
|
| 631 |
+
"def calculate_comprehensive_stress_score(df):\n",
|
| 632 |
+
" \"\"\"\n",
|
| 633 |
+
" Research-based stress scoring using all 10 AUs\n",
|
| 634 |
+
" Weights based on affective computing literature\n",
|
| 635 |
+
" \"\"\"\n",
|
| 636 |
+
" \n",
|
| 637 |
+
" # STRESS INDICATORS (weighted by research evidence)\n",
|
| 638 |
+
" # AU04 (Brow Lower) - Primary anger/stress indicator\n",
|
| 639 |
+
" au04_score = (df['AU04_BrowLower_intensity'].mean() / 100) * (df['AU04_BrowLower_activation_rate'].mean()) * 25\n",
|
| 640 |
+
" \n",
|
| 641 |
+
" # AU01 (Inner Brow Raise) - Worry/sadness indicator\n",
|
| 642 |
+
" au01_score = (df['AU01_InnerBrowRaise_intensity'].mean() / 100) * (df['AU01_InnerBrowRaise_activation_rate'].mean()) * 15\n",
|
| 643 |
+
" \n",
|
| 644 |
+
" # AU07 (Lid Tighten) - Tension indicator\n",
|
| 645 |
+
" au07_score = (df['AU07_LidTighten_intensity'].mean() / 100) * (df['AU07_LidTighten_activation_rate'].mean()) * 12\n",
|
| 646 |
+
" \n",
|
| 647 |
+
" # AU24 (Lip Press) - Stress/tension\n",
|
| 648 |
+
" au24_score = (df['AU24_LipPress_intensity'].mean() / 100) * (df['AU24_LipPress_activation_rate'].mean()) * 15\n",
|
| 649 |
+
" \n",
|
| 650 |
+
" # AU23 (Lip Tighten) - Anger/tension\n",
|
| 651 |
+
" au23_score = (df['AU23_LipTighten_intensity'].mean() / 100) * (df['AU23_LipTighten_activation_rate'].mean()) * 12\n",
|
| 652 |
+
" \n",
|
| 653 |
+
" # AU17 (Chin Raise) - Doubt/sadness\n",
|
| 654 |
+
" au17_score = (df['AU17_ChinRaise_intensity'].mean() / 100) * (df['AU17_ChinRaise_activation_rate'].mean()) * 8\n",
|
| 655 |
+
" \n",
|
| 656 |
+
" # POSITIVE INDICATORS (reduce stress score)\n",
|
| 657 |
+
" # AU06 + AU12 (Duchenne Smile - genuine happiness)\n",
|
| 658 |
+
" duchenne_smile = ((df['AU06_CheekRaise_active'] == 1) & (df['AU12_LipCornerPull_active'] == 1)).sum() / len(df)\n",
|
| 659 |
+
" positive_reduction = duchenne_smile * 15\n",
|
| 660 |
+
" \n",
|
| 661 |
+
" # AU12 alone (social smile - may mask stress)\n",
|
| 662 |
+
" social_smile = ((df['AU12_LipCornerPull_active'] == 1) & (df['AU06_CheekRaise_active'] == 0)).sum() / len(df)\n",
|
| 663 |
+
" masking_indicator = social_smile * 5 # Adds to stress if smiling without cheek raise\n",
|
| 664 |
+
" \n",
|
| 665 |
+
" # TEMPORAL PATTERNS\n",
|
| 666 |
+
" # Sustained stress (continuous activation 3+ seconds)\n",
|
| 667 |
+
" sustained_stress = 0\n",
|
| 668 |
+
" for au_name in ['AU04_BrowLower', 'AU24_LipPress', 'AU23_LipTighten']:\n",
|
| 669 |
+
" streak = 0\n",
|
| 670 |
+
" for val in df[f'{au_name}_active']:\n",
|
| 671 |
+
" if val == 1:\n",
|
| 672 |
+
" streak += 1\n",
|
| 673 |
+
" if streak >= 90: # 3 seconds at 30fps\n",
|
| 674 |
+
" sustained_stress += 1\n",
|
| 675 |
+
" break\n",
|
| 676 |
+
" else:\n",
|
| 677 |
+
" streak = 0\n",
|
| 678 |
+
" sustained_score = min(10, sustained_stress * 3)\n",
|
| 679 |
+
" \n",
|
| 680 |
+
" # Co-occurrence of multiple stress AUs\n",
|
| 681 |
+
" stress_cols = ['AU01_InnerBrowRaise_active', 'AU04_BrowLower_active', \n",
|
| 682 |
+
" 'AU07_LidTighten_active', 'AU23_LipTighten_active', \n",
|
| 683 |
+
" 'AU24_LipPress_active', 'AU17_ChinRaise_active']\n",
|
| 684 |
+
" co_occurrence = (df[stress_cols].sum(axis=1) >= 3).sum() / len(df)\n",
|
| 685 |
+
" co_occurrence_score = co_occurrence * 8\n",
|
| 686 |
+
" \n",
|
| 687 |
+
" # COMBINED STRESS SCORE\n",
|
| 688 |
+
" raw_stress = (au04_score + au01_score + au07_score + au24_score + \n",
|
| 689 |
+
" au23_score + au17_score + sustained_score + \n",
|
| 690 |
+
" co_occurrence_score + masking_indicator - positive_reduction)\n",
|
| 691 |
+
" \n",
|
| 692 |
+
" stress_score = min(100, max(0, raw_stress))\n",
|
| 693 |
+
" \n",
|
| 694 |
+
" # Classification\n",
|
| 695 |
+
" if stress_score < 25:\n",
|
| 696 |
+
" classification = \"NOT STRESSED\"\n",
|
| 697 |
+
" color = \"π’\"\n",
|
| 698 |
+
" elif stress_score < 55:\n",
|
| 699 |
+
" classification = \"POSSIBLY STRESSED\"\n",
|
| 700 |
+
" color = \"π‘\"\n",
|
| 701 |
+
" else:\n",
|
| 702 |
+
" classification = \"STRESSED\"\n",
|
| 703 |
+
" color = \"π΄\"\n",
|
| 704 |
+
" \n",
|
| 705 |
+
" return {\n",
|
| 706 |
+
" 'classification': classification,\n",
|
| 707 |
+
" 'color': color,\n",
|
| 708 |
+
" 'stress_score': stress_score,\n",
|
| 709 |
+
" 'components': {\n",
|
| 710 |
+
" 'au04': au04_score,\n",
|
| 711 |
+
" 'au01': au01_score,\n",
|
| 712 |
+
" 'au07': au07_score,\n",
|
| 713 |
+
" 'au24': au24_score,\n",
|
| 714 |
+
" 'au23': au23_score,\n",
|
| 715 |
+
" 'au17': au17_score,\n",
|
| 716 |
+
" 'sustained': sustained_score,\n",
|
| 717 |
+
" 'co_occurrence': co_occurrence_score,\n",
|
| 718 |
+
" 'duchenne_smile': duchenne_smile,\n",
|
| 719 |
+
" 'social_smile_masking': masking_indicator\n",
|
| 720 |
+
" },\n",
|
| 721 |
+
" 'activation_percentages': {\n",
|
| 722 |
+
" 'AU01': (df['AU01_InnerBrowRaise_active'].sum() / len(df)) * 100,\n",
|
| 723 |
+
" 'AU04': (df['AU04_BrowLower_active'].sum() / len(df)) * 100,\n",
|
| 724 |
+
" 'AU06': (df['AU06_CheekRaise_active'].sum() / len(df)) * 100,\n",
|
| 725 |
+
" 'AU07': (df['AU07_LidTighten_active'].sum() / len(df)) * 100,\n",
|
| 726 |
+
" 'AU12': (df['AU12_LipCornerPull_active'].sum() / len(df)) * 100,\n",
|
| 727 |
+
" 'AU14': (df['AU14_Dimpler_active'].sum() / len(df)) * 100,\n",
|
| 728 |
+
" 'AU17': (df['AU17_ChinRaise_active'].sum() / len(df)) * 100,\n",
|
| 729 |
+
" 'AU23': (df['AU23_LipTighten_active'].sum() / len(df)) * 100,\n",
|
| 730 |
+
" 'AU24': (df['AU24_LipPress_active'].sum() / len(df)) * 100,\n",
|
| 731 |
+
" 'AU26': (df['AU26_JawDrop_active'].sum() / len(df)) * 100\n",
|
| 732 |
+
" }\n",
|
| 733 |
+
" }\n",
|
| 734 |
+
"\n",
|
| 735 |
+
"\n",
|
| 736 |
+
"# ==================== COMPREHENSIVE VISUALIZATION ====================\n",
|
| 737 |
+
"\n",
|
| 738 |
+
"def plot_comprehensive_analysis(df):\n",
|
| 739 |
+
" \"\"\"Create 10 publication-quality plots\"\"\"\n",
|
| 740 |
+
" \n",
|
| 741 |
+
" fig = plt.figure(figsize=(20, 16))\n",
|
| 742 |
+
" gs = GridSpec(5, 3, figure=fig, hspace=0.35, wspace=0.3)\n",
|
| 743 |
+
" \n",
|
| 744 |
+
" fig.suptitle('Comprehensive 10-AU Facial Expression Analysis\\nStress Detection System', \n",
|
| 745 |
+
" fontsize=18, fontweight='bold', y=0.995)\n",
|
| 746 |
+
" \n",
|
| 747 |
+
" # Plot 1: All AU Activations Over Time\n",
|
| 748 |
+
" ax1 = fig.add_subplot(gs[0, :2])\n",
|
| 749 |
+
" stress_aus = ['AU01_InnerBrowRaise', 'AU04_BrowLower', 'AU07_LidTighten', \n",
|
| 750 |
+
" 'AU17_ChinRaise', 'AU23_LipTighten', 'AU24_LipPress']\n",
|
| 751 |
+
" colors_stress = ['orange', 'red', 'darkred', 'brown', 'purple', 'magenta']\n",
|
| 752 |
+
" \n",
|
| 753 |
+
" for au, color in zip(stress_aus, colors_stress):\n",
|
| 754 |
+
" ax1.plot(df['timestamp'], df[f'{au}_active'], label=au.split('_')[1], \n",
|
| 755 |
+
" color=color, linewidth=1.5, alpha=0.7)\n",
|
| 756 |
+
" \n",
|
| 757 |
+
" ax1.set_xlabel('Time (seconds)', fontweight='bold')\n",
|
| 758 |
+
" ax1.set_ylabel('Activation (Binary)', fontweight='bold')\n",
|
| 759 |
+
" ax1.set_title('Stress-Related AU Temporal Patterns')\n",
|
| 760 |
+
" ax1.legend(loc='upper right', ncol=3, fontsize=8)\n",
|
| 761 |
+
" ax1.grid(True, alpha=0.3)\n",
|
| 762 |
+
" ax1.set_ylim(-0.1, 1.1)\n",
|
| 763 |
+
" \n",
|
| 764 |
+
" # Plot 2: Positive AUs Over Time\n",
|
| 765 |
+
" ax2 = fig.add_subplot(gs[0, 2])\n",
|
| 766 |
+
" positive_aus = ['AU06_CheekRaise', 'AU12_LipCornerPull', 'AU14_Dimpler', 'AU26_JawDrop']\n",
|
| 767 |
+
" colors_pos = ['lightgreen', 'green', 'darkgreen', 'blue']\n",
|
| 768 |
+
" \n",
|
| 769 |
+
" for au, color in zip(positive_aus, colors_pos):\n",
|
| 770 |
+
" ax2.plot(df['timestamp'], df[f'{au}_active'], label=au.split('_')[1], \n",
|
| 771 |
+
" color=color, linewidth=1.5, alpha=0.7)\n",
|
| 772 |
+
" \n",
|
| 773 |
+
" ax2.set_xlabel('Time (s)', fontweight='bold')\n",
|
| 774 |
+
" ax2.set_ylabel('Activation', fontweight='bold')\n",
|
| 775 |
+
" ax2.set_title('Positive AU Patterns')\n",
|
| 776 |
+
" ax2.legend(fontsize=7)\n",
|
| 777 |
+
" ax2.grid(True, alpha=0.3)\n",
|
| 778 |
+
" ax2.set_ylim(-0.1, 1.1)\n",
|
| 779 |
+
" \n",
|
| 780 |
+
" # Plot 3: Intensity Heatmap (All 10 AUs)\n",
|
| 781 |
+
" ax3 = fig.add_subplot(gs[1, :])\n",
|
| 782 |
+
" all_aus = ['AU01_InnerBrowRaise', 'AU04_BrowLower', 'AU06_CheekRaise', \n",
|
| 783 |
+
" 'AU07_LidTighten', 'AU12_LipCornerPull', 'AU14_Dimpler',\n",
|
| 784 |
+
" 'AU17_ChinRaise', 'AU23_LipTighten', 'AU24_LipPress', 'AU26_JawDrop']\n",
|
| 785 |
+
" \n",
|
| 786 |
+
" intensity_data = df[[f'{au}_intensity' for au in all_aus]].T\n",
|
| 787 |
+
" im = ax3.imshow(intensity_data, aspect='auto', cmap='RdYlGn_r', interpolation='nearest')\n",
|
| 788 |
+
" ax3.set_yticks(range(10))\n",
|
| 789 |
+
" ax3.set_yticklabels([au.split('_')[0] for au in all_aus])\n",
|
| 790 |
+
" ax3.set_xlabel('Frame Number', fontweight='bold')\n",
|
| 791 |
+
" ax3.set_title('Complete AU Intensity Heatmap (All 10 Action Units)')\n",
|
| 792 |
+
" plt.colorbar(im, ax=ax3, label='Intensity (%)')\n",
|
| 793 |
+
" \n",
|
| 794 |
+
" # Plot 4: AU Activation Frequency Bar Chart\n",
|
| 795 |
+
" ax4 = fig.add_subplot(gs[2, 0])\n",
|
| 796 |
+
" result = calculate_comprehensive_stress_score(df)\n",
|
| 797 |
+
" au_names = list(result['activation_percentages'].keys())\n",
|
| 798 |
+
" au_values = list(result['activation_percentages'].values())\n",
|
| 799 |
+
" colors = ['red' if 'AU04' in au or 'AU24' in au or 'AU23' in au \n",
|
| 800 |
+
" else 'orange' if 'AU01' in au or 'AU07' in au or 'AU17' in au\n",
|
| 801 |
+
" else 'green' for au in au_names]\n",
|
| 802 |
+
" \n",
|
| 803 |
+
" bars = ax4.barh(au_names, au_values, color=colors, alpha=0.7)\n",
|
| 804 |
+
" ax4.set_xlabel('Activation Percentage (%)', fontweight='bold')\n",
|
| 805 |
+
" ax4.set_title('AU Activation Frequencies')\n",
|
| 806 |
+
" ax4.grid(True, alpha=0.3, axis='x')\n",
|
| 807 |
+
" \n",
|
| 808 |
+
" # Plot 5: Duchenne vs Non-Duchenne Smile Detection\n",
|
| 809 |
+
" ax5 = fig.add_subplot(gs[2, 1])\n",
|
| 810 |
+
" duchenne = ((df['AU06_CheekRaise_active'] == 1) & (df['AU12_LipCornerPull_active'] == 1)).sum()\n",
|
| 811 |
+
" non_duchenne = ((df['AU12_LipCornerPull_active'] == 1) & (df['AU06_CheekRaise_active'] == 0)).sum()\n",
|
| 812 |
+
" no_smile = len(df) - duchenne - non_duchenne\n",
|
| 813 |
+
" \n",
|
| 814 |
+
" labels = ['Genuine\\n(Duchenne)', 'Social\\n(Masking)', 'No Smile']\n",
|
| 815 |
+
" sizes = [duchenne, non_duchenne, no_smile]\n",
|
| 816 |
+
" colors_pie = ['green', 'yellow', 'lightgray']\n",
|
| 817 |
+
" \n",
|
| 818 |
+
" ax5.pie(sizes, labels=labels, colors=colors_pie, autopct='%1.1f%%', startangle=90)\n",
|
| 819 |
+
" ax5.set_title('Smile Type Distribution')\n",
|
| 820 |
+
" \n",
|
| 821 |
+
" # Plot 6: Stress vs Positive Balance\n",
|
| 822 |
+
" ax6 = fig.add_subplot(gs[2, 2])\n",
|
| 823 |
+
" stress_intensity_avg = df[['AU01_InnerBrowRaise_intensity', 'AU04_BrowLower_intensity',\n",
|
| 824 |
+
" 'AU07_LidTighten_intensity', 'AU23_LipTighten_intensity',\n",
|
| 825 |
+
" 'AU24_LipPress_intensity', 'AU17_ChinRaise_intensity']].mean(axis=1)\n",
|
| 826 |
+
" positive_intensity_avg = df[['AU06_CheekRaise_intensity', 'AU12_LipCornerPull_intensity',\n",
|
| 827 |
+
" 'AU14_Dimpler_intensity']].mean(axis=1)\n",
|
| 828 |
+
" \n",
|
| 829 |
+
" ax6.plot(df['timestamp'], stress_intensity_avg, color='red', linewidth=2, label='Stress AUs')\n",
|
| 830 |
+
" ax6.plot(df['timestamp'], positive_intensity_avg, color='green', linewidth=2, label='Positive AUs')\n",
|
| 831 |
+
" ax6.fill_between(df['timestamp'], stress_intensity_avg, alpha=0.3, color='red')\n",
|
| 832 |
+
" ax6.fill_between(df['timestamp'], positive_intensity_avg, alpha=0.3, color='green')\n",
|
| 833 |
+
" ax6.set_xlabel('Time (seconds)', fontweight='bold')\n",
|
| 834 |
+
" ax6.set_ylabel('Average Intensity (%)', fontweight='bold')\n",
|
| 835 |
+
" ax6.set_title('Stress vs Positive Affect Balance')\n",
|
| 836 |
+
" ax6.legend()\n",
|
| 837 |
+
" ax6.grid(True, alpha=0.3)\n",
|
| 838 |
+
" \n",
|
| 839 |
+
" # Plot 7: Correlation Matrix (All AUs)\n",
|
| 840 |
+
" ax7 = fig.add_subplot(gs[3, :2])\n",
|
| 841 |
+
" correlation_cols = [f'{au}_intensity' for au in all_aus]\n",
|
| 842 |
+
" corr_matrix = df[correlation_cols].corr()\n",
|
| 843 |
+
" \n",
|
| 844 |
+
" im = ax7.imshow(corr_matrix, cmap='coolwarm', vmin=-1, vmax=1, aspect='auto')\n",
|
| 845 |
+
" ax7.set_xticks(range(10))\n",
|
| 846 |
+
" ax7.set_yticks(range(10))\n",
|
| 847 |
+
" ax7.set_xticklabels([au.split('_')[0] for au in all_aus], rotation=45, ha='right')\n",
|
| 848 |
+
" ax7.set_yticklabels([au.split('_')[0] for au in all_aus])\n",
|
| 849 |
+
" ax7.set_title('Complete AU Correlation Matrix')\n",
|
| 850 |
+
" \n",
|
| 851 |
+
" # Add correlation values\n",
|
| 852 |
+
" for i in range(10):\n",
|
| 853 |
+
" for j in range(10):\n",
|
| 854 |
+
" if abs(corr_matrix.iloc[i, j]) > 0.3: # Only show strong correlations\n",
|
| 855 |
+
" ax7.text(j, i, f'{corr_matrix.iloc[i, j]:.2f}',\n",
|
| 856 |
+
" ha=\"center\", va=\"center\", color=\"black\", fontsize=7)\n",
|
| 857 |
+
" \n",
|
| 858 |
+
" plt.colorbar(im, ax=ax7)\n",
|
| 859 |
+
" \n",
|
| 860 |
+
" # Plot 8: Time-Windowed Stress Evolution\n",
|
| 861 |
+
" ax8 = fig.add_subplot(gs[3, 2])\n",
|
| 862 |
+
" window_size = 90 # 3 seconds\n",
|
| 863 |
+
" windowed_stress = []\n",
|
| 864 |
+
" window_times = []\n",
|
| 865 |
+
" \n",
|
| 866 |
+
" for i in range(0, len(df) - window_size, window_size // 2):\n",
|
| 867 |
+
" window_df = df.iloc[i:i+window_size]\n",
|
| 868 |
+
" if len(window_df) > 0:\n",
|
| 869 |
+
" window_result = calculate_comprehensive_stress_score(window_df)\n",
|
| 870 |
+
" windowed_stress.append(window_result['stress_score'])\n",
|
| 871 |
+
" window_times.append(window_df['timestamp'].mean())\n",
|
| 872 |
+
" \n",
|
| 873 |
+
" ax8.plot(window_times, windowed_stress, color='red', linewidth=2, marker='o')\n",
|
| 874 |
+
" ax8.fill_between(window_times, windowed_stress, alpha=0.3, color='red')\n",
|
| 875 |
+
" ax8.axhline(y=25, color='green', linestyle='--', label='Low threshold', alpha=0.5)\n",
|
| 876 |
+
" ax8.axhline(y=55, color='orange', linestyle='--', label='High threshold', alpha=0.5)\n",
|
| 877 |
+
" ax8.set_xlabel('Time (seconds)', fontweight='bold')\n",
|
| 878 |
+
" ax8.set_ylabel('Stress Score', fontweight='bold')\n",
|
| 879 |
+
" ax8.set_title('Stress Score Evolution (3s windows)')\n",
|
| 880 |
+
" ax8.legend()\n",
|
| 881 |
+
" ax8.grid(True, alpha=0.3)\n",
|
| 882 |
+
" \n",
|
| 883 |
+
" # Plot 9: AU Co-occurrence Matrix\n",
|
| 884 |
+
" ax9 = fig.add_subplot(gs[4, 0])\n",
|
| 885 |
+
" stress_au_cols = [f'{au}_active' for au in stress_aus]\n",
|
| 886 |
+
" co_occur_matrix = np.zeros((6, 6))\n",
|
| 887 |
+
" \n",
|
| 888 |
+
" for i in range(6):\n",
|
| 889 |
+
" for j in range(6):\n",
|
| 890 |
+
" co_occur = ((df[stress_au_cols[i]] == 1) & (df[stress_au_cols[j]] == 1)).sum()\n",
|
| 891 |
+
" co_occur_matrix[i, j] = co_occur / len(df) * 100\n",
|
| 892 |
+
" \n",
|
| 893 |
+
" im = ax9.imshow(co_occur_matrix, cmap='Reds', aspect='auto')\n",
|
| 894 |
+
" ax9.set_xticks(range(6))\n",
|
| 895 |
+
" ax9.set_yticks(range(6))\n",
|
| 896 |
+
" ax9.set_xticklabels([au.split('_')[0] for au in stress_aus], rotation=45, ha='right')\n",
|
| 897 |
+
" ax9.set_yticklabels([au.split('_')[0] for au in stress_aus])\n",
|
| 898 |
+
" ax9.set_title('Stress AU Co-occurrence (%)')\n",
|
| 899 |
+
" plt.colorbar(im, ax=ax9)\n",
|
| 900 |
+
" \n",
|
| 901 |
+
" # Plot 10: Comprehensive Summary Report\n",
|
| 902 |
+
" ax10 = fig.add_subplot(gs[4, 1:])\n",
|
| 903 |
+
" ax10.axis('off')\n",
|
| 904 |
+
" \n",
|
| 905 |
+
" result = calculate_comprehensive_stress_score(df)\n",
|
| 906 |
+
" \n",
|
| 907 |
+
" summary_text = f\"\"\"\n",
|
| 908 |
+
"βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ\n",
|
| 909 |
+
"β COMPREHENSIVE STRESS ASSESSMENT REPORT β\n",
|
| 910 |
+
"β ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ£\n",
|
| 911 |
+
"β β\n",
|
| 912 |
+
"β CLASSIFICATION: {result['color']} {result['classification']:<25} | STRESS SCORE: {result['stress_score']:.1f}/100 β\n",
|
| 913 |
+
"β β\n",
|
| 914 |
+
"β ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ£\n",
|
| 915 |
+
"β COMPONENT CONTRIBUTIONS: β\n",
|
| 916 |
+
"β β’ AU04 (Brow Lower): {result['components']['au04']:.2f} / 25.0 [{result['activation_percentages']['AU04']:5.1f}% active] β\n",
|
| 917 |
+
"β β’ AU01 (Inner Brow): {result['components']['au01']:.2f} / 15.0 [{result['activation_percentages']['AU01']:5.1f}% active] β\n",
|
| 918 |
+
"β β’ AU07 (Lid Tighten): {result['components']['au07']:.2f} / 12.0 [{result['activation_percentages']['AU07']:5.1f}% active] β\n",
|
| 919 |
+
"β β’ AU24 (Lip Press): {result['components']['au24']:.2f} / 15.0 [{result['activation_percentages']['AU24']:5.1f}% active] β\n",
|
| 920 |
+
"β β’ AU23 (Lip Tighten): {result['components']['au23']:.2f} / 12.0 [{result['activation_percentages']['AU23']:5.1f}% active] β\n",
|
| 921 |
+
"β β’ AU17 (Chin Raise): {result['components']['au17']:.2f} / 8.0 [{result['activation_percentages']['AU17']:5.1f}% active] β\n",
|
| 922 |
+
"β β’ Sustained Activation: {result['components']['sustained']:.2f} / 10.0 β\n",
|
| 923 |
+
"β β’ Co-occurrence Pattern: {result['components']['co_occurrence']:.2f} / 8.0 β\n",
|
| 924 |
+
"β β’ Smile Masking Effect: {result['components']['social_smile_masking']:.2f} (adds stress if present) β\n",
|
| 925 |
+
"β β’ Duchenne Smile Bonus: -{result['components']['duchenne_smile']*15:.2f} (reduces stress) β\n",
|
| 926 |
+
"β β\n",
|
| 927 |
+
"β ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ£\n",
|
| 928 |
+
"β POSITIVE AFFECT INDICATORS: β\n",
|
| 929 |
+
"β β’ AU06 (Cheek Raise): {result['activation_percentages']['AU06']:5.1f}% active β\n",
|
| 930 |
+
"β β’ AU12 (Lip Pull): {result['activation_percentages']['AU12']:5.1f}% active β\n",
|
| 931 |
+
"β β’ AU14 (Dimpler): {result['activation_percentages']['AU14']:5.1f}% active β\n",
|
| 932 |
+
"β β’ Duchenne Smile Rate: {result['components']['duchenne_smile']*100:.1f}% β\n",
|
| 933 |
+
"β β\n",
|
| 934 |
+
"β ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ£\n",
|
| 935 |
+
"β RESEARCH BASIS: β\n",
|
| 936 |
+
"β Weights based on Facial Action Coding System (FACS) research: β\n",
|
| 937 |
+
"β β’ AU04 highest weight (Ekman & Friesen, 1978) - primary anger/stress indicator β\n",
|
| 938 |
+
"β β’ AU06+AU12 combination identifies genuine happiness (Duchenne marker) β\n",
|
| 939 |
+
"β β’ Sustained activation and co-occurrence patterns enhance stress detection accuracy β\n",
|
| 940 |
+
"β β’ Temporal windowing allows detection of acute stress episodes vs chronic patterns β\n",
|
| 941 |
+
"βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ\n",
|
| 942 |
+
" \"\"\"\n",
|
| 943 |
+
" \n",
|
| 944 |
+
" ax10.text(0.05, 0.5, summary_text, fontsize=8, family='monospace',\n",
|
| 945 |
+
" verticalalignment='center',\n",
|
| 946 |
+
" bbox=dict(boxstyle='round', facecolor='lightgray', alpha=0.2))\n",
|
| 947 |
+
" \n",
|
| 948 |
+
" plt.tight_layout()\n",
|
| 949 |
+
" return fig, result\n",
|
| 950 |
+
"\n",
|
| 951 |
+
"\n",
|
| 952 |
+
"# ==================== MAIN EXECUTION ====================\n",
|
| 953 |
+
"\n",
|
| 954 |
+
"if __name__ == \"__main__\":\n",
|
| 955 |
+
" print(\"\\n\" + \"=\"*70)\n",
|
| 956 |
+
" print(\" COMPLETE 10-AU STRESS DETECTION SYSTEM\")\n",
|
| 957 |
+
" print(\" Based on Facial Action Coding System (FACS)\")\n",
|
| 958 |
+
" print(\" Research Guide: Prof. Anup Nandy\")\n",
|
| 959 |
+
" print(\"=\"*70)\n",
|
| 960 |
+
" \n",
|
| 961 |
+
" print(\"\\n Action Units Detected:\")\n",
|
| 962 |
+
" print(\" STRESS: AU01, AU04, AU07, AU17, AU23, AU24\")\n",
|
| 963 |
+
" print(\" POSITIVE: AU06, AU12, AU14, AU26\")\n",
|
| 964 |
+
" print(\"\\n Press Enter to start 15-second recording...\")\n",
|
| 965 |
+
" \n",
|
| 966 |
+
" input()\n",
|
| 967 |
+
" \n",
|
| 968 |
+
" df = run_detection_session(duration_seconds=15, save_data=True)\n",
|
| 969 |
+
" \n",
|
| 970 |
+
" print(\"\\n\" + \"=\"*70)\n",
|
| 971 |
+
" print(\" Generating comprehensive analysis...\")\n",
|
| 972 |
+
" print(\"=\"*70 + \"\\n\")\n",
|
| 973 |
+
" \n",
|
| 974 |
+
" fig, result = plot_comprehensive_analysis(df)\n",
|
| 975 |
+
" \n",
|
| 976 |
+
" print(f\"\\n{result['color']} FINAL ASSESSMENT: {result['classification']}\")\n",
|
| 977 |
+
" print(f\" Stress Score: {result['stress_score']:.1f}/100\")\n",
|
| 978 |
+
" print(f\"\\n Data saved with {len(df)} frames\")\n",
|
| 979 |
+
" print(f\" Total features per frame: {len(df.columns) - 1}\")\n",
|
| 980 |
+
" print(\"\\n\" + \"=\"*70)\n",
|
| 981 |
+
" \n",
|
| 982 |
+
" plt.show()\n",
|
| 983 |
+
" \n",
|
| 984 |
+
" print(\"\\nβ Analysis complete!\")"
|
| 985 |
+
]
|
| 986 |
+
},
|
| 987 |
+
{
|
| 988 |
+
"cell_type": "code",
|
| 989 |
+
"execution_count": null,
|
| 990 |
+
"id": "a7551dd6",
|
| 991 |
+
"metadata": {},
|
| 992 |
+
"outputs": [],
|
| 993 |
+
"source": []
|
| 994 |
+
}
|
| 995 |
+
],
|
| 996 |
+
"metadata": {
|
| 997 |
+
"kernelspec": {
|
| 998 |
+
"display_name": "Python 3.8 (Global)",
|
| 999 |
+
"language": "python",
|
| 1000 |
+
"name": "globalpython38"
|
| 1001 |
+
},
|
| 1002 |
+
"language_info": {
|
| 1003 |
+
"codemirror_mode": {
|
| 1004 |
+
"name": "ipython",
|
| 1005 |
+
"version": 3
|
| 1006 |
+
},
|
| 1007 |
+
"file_extension": ".py",
|
| 1008 |
+
"mimetype": "text/x-python",
|
| 1009 |
+
"name": "python",
|
| 1010 |
+
"nbconvert_exporter": "python",
|
| 1011 |
+
"pygments_lexer": "ipython3",
|
| 1012 |
+
"version": "3.8.10"
|
| 1013 |
+
}
|
| 1014 |
+
},
|
| 1015 |
+
"nbformat": 4,
|
| 1016 |
+
"nbformat_minor": 5
|
| 1017 |
+
}
|