Sohan2004 commited on
Commit
68f97f8
Β·
verified Β·
1 Parent(s): 51a1e75

Upload StressDetection_V1 (1).ipynb

Browse files
Files changed (1) hide show
  1. StressDetection_V1 (1).ipynb +1017 -0
StressDetection_V1 (1).ipynb ADDED
@@ -0,0 +1,1017 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": null,
6
+ "id": "a69b5d99",
7
+ "metadata": {},
8
+ "outputs": [],
9
+ "source": [
10
+ "# Complete Stress Detection System - 10 Action Units\n",
11
+ "# Real-time Multi-AU Detection with Comprehensive Analysis\n",
12
+ "# Research Assistant: [Your Name]\n",
13
+ "# Guide: Prof. Anup Nandy\n",
14
+ "# Based on Facial Action Coding System (FACS) - Ekman & Friesen\n",
15
+ "\n",
16
+ "import cv2\n",
17
+ "import mediapipe as mp\n",
18
+ "import numpy as np\n",
19
+ "import pandas as pd\n",
20
+ "import matplotlib.pyplot as plt\n",
21
+ "from matplotlib.gridspec import GridSpec\n",
22
+ "from collections import deque\n",
23
+ "import time\n",
24
+ "from datetime import datetime\n",
25
+ "import warnings\n",
26
+ "warnings.filterwarnings('ignore')\n",
27
+ "\n",
28
+ "# ==================== CONFIGURATION ====================\n",
29
+ "WINDOW_SIZE = 30\n",
30
+ "RECORDING_DURATION = 15\n",
31
+ "FPS = 30\n",
32
+ "\n",
33
+ "# ==================== MediaPipe Setup ====================\n",
34
+ "mp_face_mesh = mp.solutions.face_mesh\n",
35
+ "face_mesh = mp_face_mesh.FaceMesh(\n",
36
+ " min_detection_confidence=0.5,\n",
37
+ " min_tracking_confidence=0.5,\n",
38
+ " refine_landmarks=True\n",
39
+ ")\n",
40
+ "\n",
41
+ "# ==================== LANDMARK INDICES (468 landmarks) ====================\n",
42
+ "\n",
43
+ "# AU01 - Inner Brow Raiser (Surprise, Fear, Sadness)\n",
44
+ "AU01_LANDMARKS = {\n",
45
+ " 'left_inner_brow': 336,\n",
46
+ " 'right_inner_brow': 107,\n",
47
+ " 'nose_bridge': 6,\n",
48
+ " 'left_outer_brow': 285,\n",
49
+ " 'right_outer_brow': 55\n",
50
+ "}\n",
51
+ "\n",
52
+ "# AU04 - Brow Lowerer (Anger, Sadness, Concentration)\n",
53
+ "AU04_LANDMARKS = {\n",
54
+ " 'left_inner_brow': 336,\n",
55
+ " 'right_inner_brow': 107,\n",
56
+ " 'left_mid_brow': 285,\n",
57
+ " 'right_mid_brow': 55,\n",
58
+ " 'left_eyelid': 159,\n",
59
+ " 'right_eyelid': 386,\n",
60
+ " 'nose_bridge': 6\n",
61
+ "}\n",
62
+ "\n",
63
+ "# AU06 - Cheek Raiser (Genuine Smile - Duchenne)\n",
64
+ "AU06_LANDMARKS = {\n",
65
+ " 'left_cheek': 205,\n",
66
+ " 'right_cheek': 425,\n",
67
+ " 'left_lower_eyelid': 145,\n",
68
+ " 'right_lower_eyelid': 374,\n",
69
+ " 'left_eye_outer': 33,\n",
70
+ " 'right_eye_outer': 263\n",
71
+ "}\n",
72
+ "\n",
73
+ "# AU07 - Lid Tightener (Concentration, Anger, Disgust)\n",
74
+ "AU07_LANDMARKS = {\n",
75
+ " 'left_upper_lid': 159,\n",
76
+ " 'right_upper_lid': 386,\n",
77
+ " 'left_lower_lid': 145,\n",
78
+ " 'right_lower_lid': 374,\n",
79
+ " 'left_eye_top': 159,\n",
80
+ " 'right_eye_top': 386\n",
81
+ "}\n",
82
+ "\n",
83
+ "# AU12 - Lip Corner Puller (Happiness)\n",
84
+ "AU12_LANDMARKS = {\n",
85
+ " 'left_corner': 61,\n",
86
+ " 'right_corner': 291,\n",
87
+ " 'upper_center': 13,\n",
88
+ " 'lower_center': 14\n",
89
+ "}\n",
90
+ "\n",
91
+ "# AU14 - Dimpler (Smile Intensity)\n",
92
+ "AU14_LANDMARKS = {\n",
93
+ " 'left_dimple': 206,\n",
94
+ " 'right_dimple': 426,\n",
95
+ " 'left_corner': 61,\n",
96
+ " 'right_corner': 291\n",
97
+ "}\n",
98
+ "\n",
99
+ "# AU17 - Chin Raiser (Doubt, Sadness, Pouting)\n",
100
+ "AU17_LANDMARKS = {\n",
101
+ " 'chin_center': 152,\n",
102
+ " 'lower_lip': 17,\n",
103
+ " 'chin_left': 176,\n",
104
+ " 'chin_right': 400\n",
105
+ "}\n",
106
+ "\n",
107
+ "# AU23 - Lip Tightener (Anger, Tension)\n",
108
+ "AU23_LANDMARKS = {\n",
109
+ " 'left_corner': 61,\n",
110
+ " 'right_corner': 291,\n",
111
+ " 'left_outer': 57,\n",
112
+ " 'right_outer': 287\n",
113
+ "}\n",
114
+ "\n",
115
+ "# AU24 - Lip Pressor (Stress, Tension, Anger)\n",
116
+ "AU24_LANDMARKS = {\n",
117
+ " 'upper_lip_top': 0,\n",
118
+ " 'upper_lip_bottom': 13,\n",
119
+ " 'lower_lip_top': 14,\n",
120
+ " 'lower_lip_bottom': 17\n",
121
+ "}\n",
122
+ "\n",
123
+ "# AU26 - Jaw Drop (Surprise, Shock, Mouth Opening)\n",
124
+ "AU26_LANDMARKS = {\n",
125
+ " 'upper_lip': 13,\n",
126
+ " 'lower_lip': 14,\n",
127
+ " 'chin': 152,\n",
128
+ " 'nose': 1\n",
129
+ "}\n",
130
+ "\n",
131
+ "# ==================== UTILITY FUNCTIONS ====================\n",
132
+ "def calculate_distance(point1, point2):\n",
133
+ " return np.sqrt((point1[0] - point2[0])**2 + (point1[1] - point2[1])**2)\n",
134
+ "\n",
135
+ "def get_landmark_coords(landmarks, idx, frame_width, frame_height):\n",
136
+ " lm = landmarks[idx]\n",
137
+ " return np.array([lm.x * frame_width, lm.y * frame_height])\n",
138
+ "\n",
139
+ "# ==================== AU DETECTOR CLASSES ====================\n",
140
+ "\n",
141
+ "class AU01Detector:\n",
142
+ " \"\"\"AU01 - Inner Brow Raiser (Surprise, Fear, Worry)\"\"\"\n",
143
+ " def __init__(self, window_size=30):\n",
144
+ " self.name = \"AU01_InnerBrowRaise\"\n",
145
+ " self.activation_history = deque(maxlen=window_size)\n",
146
+ " self.intensity_history = deque(maxlen=window_size)\n",
147
+ " \n",
148
+ " def detect(self, landmarks, frame_width, frame_height):\n",
149
+ " left_inner = get_landmark_coords(landmarks, AU01_LANDMARKS['left_inner_brow'], frame_width, frame_height)\n",
150
+ " right_inner = get_landmark_coords(landmarks, AU01_LANDMARKS['right_inner_brow'], frame_width, frame_height)\n",
151
+ " left_outer = get_landmark_coords(landmarks, AU01_LANDMARKS['left_outer_brow'], frame_width, frame_height)\n",
152
+ " right_outer = get_landmark_coords(landmarks, AU01_LANDMARKS['right_outer_brow'], frame_width, frame_height)\n",
153
+ " nose = get_landmark_coords(landmarks, AU01_LANDMARKS['nose_bridge'], frame_width, frame_height)\n",
154
+ " \n",
155
+ " # Calculate inner vs outer brow height\n",
156
+ " inner_height = ((nose[1] - left_inner[1]) + (nose[1] - right_inner[1])) / 2\n",
157
+ " outer_height = ((nose[1] - left_outer[1]) + (nose[1] - right_outer[1])) / 2\n",
158
+ " \n",
159
+ " # AU01 active when inner brows raised MORE than outer (creates worried look)\n",
160
+ " raise_ratio = inner_height / (outer_height + 1e-6)\n",
161
+ " \n",
162
+ " is_active = raise_ratio > 1.15 # Inner brows 15% higher than outer\n",
163
+ " intensity = min(100, max(0, (raise_ratio - 1.0) * 500))\n",
164
+ " \n",
165
+ " self.activation_history.append(int(is_active))\n",
166
+ " self.intensity_history.append(intensity)\n",
167
+ " \n",
168
+ " return is_active, intensity\n",
169
+ "\n",
170
+ "\n",
171
+ "class AU04Detector:\n",
172
+ " \"\"\"AU04 - Brow Lowerer (Anger, Concentration, Stress)\"\"\"\n",
173
+ " def __init__(self, window_size=30):\n",
174
+ " self.name = \"AU04_BrowLower\"\n",
175
+ " self.activation_history = deque(maxlen=window_size)\n",
176
+ " self.intensity_history = deque(maxlen=window_size)\n",
177
+ " \n",
178
+ " def detect(self, landmarks, frame_width, frame_height):\n",
179
+ " left_inner_brow = get_landmark_coords(landmarks, AU04_LANDMARKS['left_inner_brow'], frame_width, frame_height)\n",
180
+ " right_inner_brow = get_landmark_coords(landmarks, AU04_LANDMARKS['right_inner_brow'], frame_width, frame_height)\n",
181
+ " left_eyelid = get_landmark_coords(landmarks, AU04_LANDMARKS['left_eyelid'], frame_width, frame_height)\n",
182
+ " right_eyelid = get_landmark_coords(landmarks, AU04_LANDMARKS['right_eyelid'], frame_width, frame_height)\n",
183
+ " nose_bridge = get_landmark_coords(landmarks, AU04_LANDMARKS['nose_bridge'], frame_width, frame_height)\n",
184
+ " \n",
185
+ " left_brow_eyelid_dist = left_inner_brow[1] - left_eyelid[1]\n",
186
+ " right_brow_eyelid_dist = right_inner_brow[1] - right_eyelid[1]\n",
187
+ " avg_brow_eyelid_dist = (left_brow_eyelid_dist + right_brow_eyelid_dist) / 2\n",
188
+ " \n",
189
+ " face_height = calculate_distance(left_inner_brow, nose_bridge)\n",
190
+ " normalized_distance = avg_brow_eyelid_dist / (face_height + 1e-6)\n",
191
+ " \n",
192
+ " inner_brow_distance = calculate_distance(left_inner_brow, right_inner_brow)\n",
193
+ " outer_eye_distance = calculate_distance(left_eyelid, right_eyelid)\n",
194
+ " brow_compression_ratio = inner_brow_distance / (outer_eye_distance + 1e-6)\n",
195
+ " \n",
196
+ " is_active = (normalized_distance > -0.30 or brow_compression_ratio < 0.95)\n",
197
+ " intensity = min(100, max(0, (normalized_distance + 0.40) / 0.40 * 100))\n",
198
+ " \n",
199
+ " self.activation_history.append(int(is_active))\n",
200
+ " self.intensity_history.append(intensity)\n",
201
+ " \n",
202
+ " return is_active, intensity\n",
203
+ "\n",
204
+ "\n",
205
+ "class AU06Detector:\n",
206
+ " \"\"\"AU06 - Cheek Raiser (Genuine Smile)\"\"\"\n",
207
+ " def __init__(self, window_size=30):\n",
208
+ " self.name = \"AU06_CheekRaise\"\n",
209
+ " self.activation_history = deque(maxlen=window_size)\n",
210
+ " self.intensity_history = deque(maxlen=window_size)\n",
211
+ " \n",
212
+ " def detect(self, landmarks, frame_width, frame_height):\n",
213
+ " left_cheek = get_landmark_coords(landmarks, AU06_LANDMARKS['left_cheek'], frame_width, frame_height)\n",
214
+ " right_cheek = get_landmark_coords(landmarks, AU06_LANDMARKS['right_cheek'], frame_width, frame_height)\n",
215
+ " left_lower_lid = get_landmark_coords(landmarks, AU06_LANDMARKS['left_lower_eyelid'], frame_width, frame_height)\n",
216
+ " right_lower_lid = get_landmark_coords(landmarks, AU06_LANDMARKS['right_lower_eyelid'], frame_width, frame_height)\n",
217
+ " \n",
218
+ " # When cheeks raise, distance between cheek and lower eyelid decreases\n",
219
+ " left_distance = calculate_distance(left_cheek, left_lower_lid)\n",
220
+ " right_distance = calculate_distance(right_cheek, right_lower_lid)\n",
221
+ " avg_distance = (left_distance + right_distance) / 2\n",
222
+ " \n",
223
+ " # Also check if lower eyelid moves up\n",
224
+ " left_eye_outer = get_landmark_coords(landmarks, AU06_LANDMARKS['left_eye_outer'], frame_width, frame_height)\n",
225
+ " eye_height = abs(left_eye_outer[1] - left_lower_lid[1])\n",
226
+ " \n",
227
+ " cheek_raise_score = eye_height / (avg_distance + 1e-6)\n",
228
+ " \n",
229
+ " is_active = cheek_raise_score > 0.8\n",
230
+ " intensity = min(100, max(0, (cheek_raise_score - 0.5) * 200))\n",
231
+ " \n",
232
+ " self.activation_history.append(int(is_active))\n",
233
+ " self.intensity_history.append(intensity)\n",
234
+ " \n",
235
+ " return is_active, intensity\n",
236
+ "\n",
237
+ "\n",
238
+ "class AU07Detector:\n",
239
+ " \"\"\"AU07 - Lid Tightener (Tension, Squinting)\"\"\"\n",
240
+ " def __init__(self, window_size=30):\n",
241
+ " self.name = \"AU07_LidTighten\"\n",
242
+ " self.activation_history = deque(maxlen=window_size)\n",
243
+ " self.intensity_history = deque(maxlen=window_size)\n",
244
+ " \n",
245
+ " def detect(self, landmarks, frame_width, frame_height):\n",
246
+ " left_upper = get_landmark_coords(landmarks, AU07_LANDMARKS['left_upper_lid'], frame_width, frame_height)\n",
247
+ " right_upper = get_landmark_coords(landmarks, AU07_LANDMARKS['right_upper_lid'], frame_width, frame_height)\n",
248
+ " left_lower = get_landmark_coords(landmarks, AU07_LANDMARKS['left_lower_lid'], frame_width, frame_height)\n",
249
+ " right_lower = get_landmark_coords(landmarks, AU07_LANDMARKS['right_lower_lid'], frame_width, frame_height)\n",
250
+ " \n",
251
+ " # Eye opening (smaller = more tightened)\n",
252
+ " left_eye_opening = abs(left_upper[1] - left_lower[1])\n",
253
+ " right_eye_opening = abs(right_upper[1] - right_lower[1])\n",
254
+ " avg_eye_opening = (left_eye_opening + right_eye_opening) / 2\n",
255
+ " \n",
256
+ " # Normalize by face height\n",
257
+ " face_ref = calculate_distance(left_upper, \n",
258
+ " get_landmark_coords(landmarks, 152, frame_width, frame_height))\n",
259
+ " normalized_opening = avg_eye_opening / (face_ref + 1e-6)\n",
260
+ " \n",
261
+ " is_active = normalized_opening < 0.025 # Eyes tightened/squinted\n",
262
+ " intensity = min(100, max(0, (0.035 - normalized_opening) / 0.035 * 100))\n",
263
+ " \n",
264
+ " self.activation_history.append(int(is_active))\n",
265
+ " self.intensity_history.append(intensity)\n",
266
+ " \n",
267
+ " return is_active, intensity\n",
268
+ "\n",
269
+ "\n",
270
+ "class AU12Detector:\n",
271
+ " \"\"\"AU12 - Lip Corner Puller (Smile)\"\"\"\n",
272
+ " def __init__(self, window_size=30):\n",
273
+ " self.name = \"AU12_LipCornerPull\"\n",
274
+ " self.activation_history = deque(maxlen=window_size)\n",
275
+ " self.intensity_history = deque(maxlen=window_size)\n",
276
+ " \n",
277
+ " def detect(self, landmarks, frame_width, frame_height):\n",
278
+ " left_corner = get_landmark_coords(landmarks, AU12_LANDMARKS['left_corner'], frame_width, frame_height)\n",
279
+ " right_corner = get_landmark_coords(landmarks, AU12_LANDMARKS['right_corner'], frame_width, frame_height)\n",
280
+ " upper_center = get_landmark_coords(landmarks, AU12_LANDMARKS['upper_center'], frame_width, frame_height)\n",
281
+ " lower_center = get_landmark_coords(landmarks, AU12_LANDMARKS['lower_center'], frame_width, frame_height)\n",
282
+ " \n",
283
+ " mouth_width = calculate_distance(left_corner, right_corner)\n",
284
+ " mouth_height = calculate_distance(upper_center, lower_center)\n",
285
+ " mouth_center_y = (upper_center[1] + lower_center[1]) / 2\n",
286
+ " \n",
287
+ " left_corner_lift = mouth_center_y - left_corner[1]\n",
288
+ " right_corner_lift = mouth_center_y - right_corner[1]\n",
289
+ " avg_corner_lift = (left_corner_lift + right_corner_lift) / 2\n",
290
+ " \n",
291
+ " mouth_ratio = mouth_width / (mouth_height + 1e-6)\n",
292
+ " normalized_lift = avg_corner_lift / mouth_height if mouth_height > 0 else 0\n",
293
+ " \n",
294
+ " lift_difference = abs(left_corner_lift - right_corner_lift)\n",
295
+ " symmetry_score = 1.0 - min(1.0, lift_difference / (mouth_height + 1e-6))\n",
296
+ " \n",
297
+ " is_active = (normalized_lift > 0.25 and mouth_ratio > 2.8 and symmetry_score > 0.6)\n",
298
+ " intensity = min(100, max(0, normalized_lift * 250))\n",
299
+ " \n",
300
+ " self.activation_history.append(int(is_active))\n",
301
+ " self.intensity_history.append(intensity)\n",
302
+ " \n",
303
+ " return is_active, intensity\n",
304
+ "\n",
305
+ "\n",
306
+ "class AU14Detector:\n",
307
+ " \"\"\"AU14 - Dimpler (Smile Depth Indicator)\"\"\"\n",
308
+ " def __init__(self, window_size=30):\n",
309
+ " self.name = \"AU14_Dimpler\"\n",
310
+ " self.activation_history = deque(maxlen=window_size)\n",
311
+ " self.intensity_history = deque(maxlen=window_size)\n",
312
+ " \n",
313
+ " def detect(self, landmarks, frame_width, frame_height):\n",
314
+ " left_dimple = get_landmark_coords(landmarks, AU14_LANDMARKS['left_dimple'], frame_width, frame_height)\n",
315
+ " right_dimple = get_landmark_coords(landmarks, AU14_LANDMARKS['right_dimple'], frame_width, frame_height)\n",
316
+ " left_corner = get_landmark_coords(landmarks, AU14_LANDMARKS['left_corner'], frame_width, frame_height)\n",
317
+ " right_corner = get_landmark_coords(landmarks, AU14_LANDMARKS['right_corner'], frame_width, frame_height)\n",
318
+ " \n",
319
+ " # Dimples appear when corners pull back and create indentation\n",
320
+ " left_depth = calculate_distance(left_dimple, left_corner)\n",
321
+ " right_depth = calculate_distance(right_dimple, right_corner)\n",
322
+ " avg_depth = (left_depth + right_depth) / 2\n",
323
+ " \n",
324
+ " # Check corner retraction\n",
325
+ " corner_distance = calculate_distance(left_corner, right_corner)\n",
326
+ " dimple_score = avg_depth / (corner_distance + 1e-6)\n",
327
+ " \n",
328
+ " is_active = dimple_score > 0.15\n",
329
+ " intensity = min(100, max(0, (dimple_score - 0.10) * 500))\n",
330
+ " \n",
331
+ " self.activation_history.append(int(is_active))\n",
332
+ " self.intensity_history.append(intensity)\n",
333
+ " \n",
334
+ " return is_active, intensity\n",
335
+ "\n",
336
+ "\n",
337
+ "class AU17Detector:\n",
338
+ " \"\"\"AU17 - Chin Raiser (Doubt, Pouting, Sadness)\"\"\"\n",
339
+ " def __init__(self, window_size=30):\n",
340
+ " self.name = \"AU17_ChinRaise\"\n",
341
+ " self.activation_history = deque(maxlen=window_size)\n",
342
+ " self.intensity_history = deque(maxlen=window_size)\n",
343
+ " \n",
344
+ " def detect(self, landmarks, frame_width, frame_height):\n",
345
+ " chin = get_landmark_coords(landmarks, AU17_LANDMARKS['chin_center'], frame_width, frame_height)\n",
346
+ " lower_lip = get_landmark_coords(landmarks, AU17_LANDMARKS['lower_lip'], frame_width, frame_height)\n",
347
+ " \n",
348
+ " # When chin raises, distance between chin and lower lip decreases\n",
349
+ " chin_lip_distance = calculate_distance(chin, lower_lip)\n",
350
+ " \n",
351
+ " # Normalize by face height\n",
352
+ " nose = get_landmark_coords(landmarks, 1, frame_width, frame_height)\n",
353
+ " face_height = calculate_distance(nose, chin)\n",
354
+ " normalized_distance = chin_lip_distance / (face_height + 1e-6)\n",
355
+ " \n",
356
+ " is_active = normalized_distance < 0.08 # Chin pushed up\n",
357
+ " intensity = min(100, max(0, (0.12 - normalized_distance) / 0.12 * 100))\n",
358
+ " \n",
359
+ " self.activation_history.append(int(is_active))\n",
360
+ " self.intensity_history.append(intensity)\n",
361
+ " \n",
362
+ " return is_active, intensity\n",
363
+ "\n",
364
+ "\n",
365
+ "class AU23Detector:\n",
366
+ " \"\"\"AU23 - Lip Tightener (Anger, Tension)\"\"\"\n",
367
+ " def __init__(self, window_size=30):\n",
368
+ " self.name = \"AU23_LipTighten\"\n",
369
+ " self.activation_history = deque(maxlen=window_size)\n",
370
+ " self.intensity_history = deque(maxlen=window_size)\n",
371
+ " \n",
372
+ " def detect(self, landmarks, frame_width, frame_height):\n",
373
+ " left_corner = get_landmark_coords(landmarks, AU23_LANDMARKS['left_corner'], frame_width, frame_height)\n",
374
+ " right_corner = get_landmark_coords(landmarks, AU23_LANDMARKS['right_corner'], frame_width, frame_height)\n",
375
+ " left_outer = get_landmark_coords(landmarks, AU23_LANDMARKS['left_outer'], frame_width, frame_height)\n",
376
+ " right_outer = get_landmark_coords(landmarks, AU23_LANDMARKS['right_outer'], frame_width, frame_height)\n",
377
+ " \n",
378
+ " corner_width = calculate_distance(left_corner, right_corner)\n",
379
+ " outer_width = calculate_distance(left_outer, right_outer)\n",
380
+ " tightness_ratio = corner_width / (outer_width + 1e-6)\n",
381
+ " \n",
382
+ " is_active = (tightness_ratio < 0.85)\n",
383
+ " intensity = min(100, max(0, (0.95 - tightness_ratio) / 0.20 * 100))\n",
384
+ " \n",
385
+ " self.activation_history.append(int(is_active))\n",
386
+ " self.intensity_history.append(intensity)\n",
387
+ " \n",
388
+ " return is_active, intensity\n",
389
+ "\n",
390
+ "\n",
391
+ "class AU24Detector:\n",
392
+ " \"\"\"AU24 - Lip Pressor (Stress, Tension)\"\"\"\n",
393
+ " def __init__(self, window_size=30):\n",
394
+ " self.name = \"AU24_LipPress\"\n",
395
+ " self.activation_history = deque(maxlen=window_size)\n",
396
+ " self.intensity_history = deque(maxlen=window_size)\n",
397
+ " \n",
398
+ " def detect(self, landmarks, frame_width, frame_height):\n",
399
+ " upper_lip_top = get_landmark_coords(landmarks, AU24_LANDMARKS['upper_lip_top'], frame_width, frame_height)\n",
400
+ " upper_lip_bottom = get_landmark_coords(landmarks, AU24_LANDMARKS['upper_lip_bottom'], frame_width, frame_height)\n",
401
+ " lower_lip_top = get_landmark_coords(landmarks, AU24_LANDMARKS['lower_lip_top'], frame_width, frame_height)\n",
402
+ " lower_lip_bottom = get_landmark_coords(landmarks, AU24_LANDMARKS['lower_lip_bottom'], frame_width, frame_height)\n",
403
+ " \n",
404
+ " upper_lip_thickness = calculate_distance(upper_lip_top, upper_lip_bottom)\n",
405
+ " lower_lip_thickness = calculate_distance(lower_lip_top, lower_lip_bottom)\n",
406
+ " total_lip_thickness = upper_lip_thickness + lower_lip_thickness\n",
407
+ " \n",
408
+ " mouth_opening = calculate_distance(upper_lip_bottom, lower_lip_top)\n",
409
+ " lip_press_score = mouth_opening / (total_lip_thickness + 1e-6)\n",
410
+ " \n",
411
+ " is_active = (lip_press_score < 0.4 and total_lip_thickness < 15)\n",
412
+ " intensity = min(100, max(0, (0.6 - lip_press_score) / 0.6 * 100))\n",
413
+ " \n",
414
+ " self.activation_history.append(int(is_active))\n",
415
+ " self.intensity_history.append(intensity)\n",
416
+ " \n",
417
+ " return is_active, intensity\n",
418
+ "\n",
419
+ "\n",
420
+ "class AU26Detector:\n",
421
+ " \"\"\"AU26 - Jaw Drop (Surprise, Shock)\"\"\"\n",
422
+ " def __init__(self, window_size=30):\n",
423
+ " self.name = \"AU26_JawDrop\"\n",
424
+ " self.activation_history = deque(maxlen=window_size)\n",
425
+ " self.intensity_history = deque(maxlen=window_size)\n",
426
+ " \n",
427
+ " def detect(self, landmarks, frame_width, frame_height):\n",
428
+ " upper_lip = get_landmark_coords(landmarks, AU26_LANDMARKS['upper_lip'], frame_width, frame_height)\n",
429
+ " lower_lip = get_landmark_coords(landmarks, AU26_LANDMARKS['lower_lip'], frame_width, frame_height)\n",
430
+ " chin = get_landmark_coords(landmarks, AU26_LANDMARKS['chin'], frame_width, frame_height)\n",
431
+ " nose = get_landmark_coords(landmarks, AU26_LANDMARKS['nose'], frame_width, frame_height)\n",
432
+ " \n",
433
+ " # Mouth opening\n",
434
+ " mouth_opening = calculate_distance(upper_lip, lower_lip)\n",
435
+ " \n",
436
+ " # Jaw drop (distance from nose to chin increases)\n",
437
+ " jaw_length = calculate_distance(nose, chin)\n",
438
+ " \n",
439
+ " # Normalize\n",
440
+ " mouth_opening_ratio = mouth_opening / (jaw_length + 1e-6)\n",
441
+ " \n",
442
+ " is_active = mouth_opening_ratio > 0.15 # Mouth significantly open\n",
443
+ " intensity = min(100, max(0, (mouth_opening_ratio - 0.10) / 0.20 * 100))\n",
444
+ " \n",
445
+ " self.activation_history.append(int(is_active))\n",
446
+ " self.intensity_history.append(intensity)\n",
447
+ " \n",
448
+ " return is_active, intensity\n",
449
+ "\n",
450
+ "\n",
451
+ "# ==================== FEATURE EXTRACTOR ====================\n",
452
+ "\n",
453
+ "class MultiAUFeatureExtractor:\n",
454
+ " def __init__(self, detectors):\n",
455
+ " self.detectors = detectors\n",
456
+ " self.feature_log = []\n",
457
+ " \n",
458
+ " def extract_features(self, timestamp):\n",
459
+ " features = {'timestamp': timestamp}\n",
460
+ " \n",
461
+ " for detector in self.detectors:\n",
462
+ " is_active = detector.activation_history[-1] if detector.activation_history else 0\n",
463
+ " intensity = detector.intensity_history[-1] if detector.intensity_history else 0\n",
464
+ " \n",
465
+ " activation_rate = sum(detector.activation_history) / len(detector.activation_history) if detector.activation_history else 0\n",
466
+ " avg_intensity = np.mean(detector.intensity_history) if detector.intensity_history else 0\n",
467
+ " max_intensity = np.max(detector.intensity_history) if detector.intensity_history else 0\n",
468
+ " intensity_std = np.std(detector.intensity_history) if detector.intensity_history else 0\n",
469
+ " \n",
470
+ " features[f'{detector.name}_active'] = is_active\n",
471
+ " features[f'{detector.name}_intensity'] = intensity\n",
472
+ " features[f'{detector.name}_activation_rate'] = activation_rate\n",
473
+ " features[f'{detector.name}_avg_intensity'] = avg_intensity\n",
474
+ " features[f'{detector.name}_max_intensity'] = max_intensity\n",
475
+ " features[f'{detector.name}_intensity_std'] = intensity_std\n",
476
+ " \n",
477
+ " self.feature_log.append(features)\n",
478
+ " return features\n",
479
+ " \n",
480
+ " def get_dataframe(self):\n",
481
+ " return pd.DataFrame(self.feature_log)\n",
482
+ " \n",
483
+ " def save_features(self, filename):\n",
484
+ " df = self.get_dataframe()\n",
485
+ " df.to_csv(filename, index=False)\n",
486
+ " print(f\"βœ“ Features saved to {filename}\")\n",
487
+ "\n",
488
+ "\n",
489
+ "# ==================== DETECTION SESSION ====================\n",
490
+ "\n",
491
+ "def run_detection_session(duration_seconds=15, save_data=True):\n",
492
+ " # Initialize all 10 AU detectors\n",
493
+ " au01 = AU01Detector()\n",
494
+ " au04 = AU04Detector()\n",
495
+ " au06 = AU06Detector()\n",
496
+ " au07 = AU07Detector()\n",
497
+ " au12 = AU12Detector()\n",
498
+ " au14 = AU14Detector()\n",
499
+ " au17 = AU17Detector()\n",
500
+ " au23 = AU23Detector()\n",
501
+ " au24 = AU24Detector()\n",
502
+ " au26 = AU26Detector()\n",
503
+ " \n",
504
+ " detectors = [au01, au04, au06, au07, au12, au14, au17, au23, au24, au26]\n",
505
+ " feature_extractor = MultiAUFeatureExtractor(detectors)\n",
506
+ " \n",
507
+ " cap = cv2.VideoCapture(0)\n",
508
+ " \n",
509
+ " print(f\"\\n{'='*70}\")\n",
510
+ " print(f\" COMPLETE 10-AU STRESS DETECTION SYSTEM\")\n",
511
+ " print(f\" Recording for {duration_seconds} seconds...\")\n",
512
+ " print(f\"{'='*70}\\n\")\n",
513
+ " \n",
514
+ " start_time = time.time()\n",
515
+ " frame_count = 0\n",
516
+ " \n",
517
+ " while True:\n",
518
+ " ret, frame = cap.read()\n",
519
+ " if not ret:\n",
520
+ " break\n",
521
+ " \n",
522
+ " current_time = time.time()\n",
523
+ " elapsed = current_time - start_time\n",
524
+ " \n",
525
+ " if elapsed >= duration_seconds:\n",
526
+ " break\n",
527
+ " \n",
528
+ " frame = cv2.flip(frame, 1)\n",
529
+ " frame_height, frame_width = frame.shape[:2]\n",
530
+ " rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n",
531
+ " \n",
532
+ " results = face_mesh.process(rgb_frame)\n",
533
+ " \n",
534
+ " if results.multi_face_landmarks:\n",
535
+ " landmarks = results.multi_face_landmarks[0].landmark\n",
536
+ " \n",
537
+ " # Detect all 10 AUs\n",
538
+ " au01_active, au01_intensity = au01.detect(landmarks, frame_width, frame_height)\n",
539
+ " au04_active, au04_intensity = au04.detect(landmarks, frame_width, frame_height)\n",
540
+ " au06_active, au06_intensity = au06.detect(landmarks, frame_width, frame_height)\n",
541
+ " au07_active, au07_intensity = au07.detect(landmarks, frame_width, frame_height)\n",
542
+ " au12_active, au12_intensity = au12.detect(landmarks, frame_width, frame_height)\n",
543
+ " au14_active, au14_intensity = au14.detect(landmarks, frame_width, frame_height)\n",
544
+ " au17_active, au17_intensity = au17.detect(landmarks, frame_width, frame_height)\n",
545
+ " au23_active, au23_intensity = au23.detect(landmarks, frame_width, frame_height)\n",
546
+ " au24_active, au24_intensity = au24.detect(landmarks, frame_width, frame_height)\n",
547
+ " au26_active, au26_intensity = au26.detect(landmarks, frame_width, frame_height)\n",
548
+ " \n",
549
+ " features = feature_extractor.extract_features(elapsed)\n",
550
+ " \n",
551
+ " # Display (2 columns)\n",
552
+ " y_offset = 25\n",
553
+ " col1_x = 10\n",
554
+ " col2_x = frame_width // 2 + 10\n",
555
+ " \n",
556
+ " # Header\n",
557
+ " cv2.putText(frame, f\"Time: {elapsed:.1f}s / {duration_seconds}s\", \n",
558
+ " (10, y_offset), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2)\n",
559
+ " \n",
560
+ " y_offset += 35\n",
561
+ " \n",
562
+ " # Column 1: Stress Indicators\n",
563
+ " cv2.putText(frame, \"STRESS INDICATORS:\", (col1_x, y_offset), \n",
564
+ " cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 2)\n",
565
+ " y_offset += 25\n",
566
+ " \n",
567
+ " stress_aus = [\n",
568
+ " (au01_active, au01_intensity, \"AU01-BrowRaise\"),\n",
569
+ " (au04_active, au04_intensity, \"AU04-BrowLower\"),\n",
570
+ " (au07_active, au07_intensity, \"AU07-LidTight\"),\n",
571
+ " (au17_active, au17_intensity, \"AU17-ChinRaise\"),\n",
572
+ " (au23_active, au23_intensity, \"AU23-LipTight\"),\n",
573
+ " (au24_active, au24_intensity, \"AU24-LipPress\")\n",
574
+ " ]\n",
575
+ " \n",
576
+ " for active, intensity, name in stress_aus:\n",
577
+ " color = (0, 0, 255) if active else (100, 100, 100)\n",
578
+ " cv2.putText(frame, f\"{name}: {intensity:.0f}%\", \n",
579
+ " (col1_x, y_offset), cv2.FONT_HERSHEY_SIMPLEX, 0.4, color, 1)\n",
580
+ " y_offset += 20\n",
581
+ " \n",
582
+ " # Column 2: Positive Indicators\n",
583
+ " y_offset = 60\n",
584
+ " cv2.putText(frame, \"POSITIVE INDICATORS:\", (col2_x, y_offset), \n",
585
+ " cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)\n",
586
+ " y_offset += 25\n",
587
+ " \n",
588
+ " positive_aus = [\n",
589
+ " (au06_active, au06_intensity, \"AU06-CheekRaise\"),\n",
590
+ " (au12_active, au12_intensity, \"AU12-SmilePull\"),\n",
591
+ " (au14_active, au14_intensity, \"AU14-Dimpler\"),\n",
592
+ " (au26_active, au26_intensity, \"AU26-JawDrop\")\n",
593
+ " ]\n",
594
+ " \n",
595
+ " for active, intensity, name in positive_aus:\n",
596
+ " color = (0, 255, 0) if active else (100, 100, 100)\n",
597
+ " cv2.putText(frame, f\"{name}: {intensity:.0f}%\", \n",
598
+ " (col2_x, y_offset), cv2.FONT_HERSHEY_SIMPLEX, 0.4, color, 1)\n",
599
+ " y_offset += 20\n",
600
+ " \n",
601
+ " # Bottom summary\n",
602
+ " stress_count = sum([au01_active, au04_active, au07_active, au17_active, au23_active, au24_active])\n",
603
+ " positive_count = sum([au06_active, au12_active, au14_active])\n",
604
+ " \n",
605
+ " cv2.rectangle(frame, (10, frame_height - 60), (frame_width - 10, frame_height - 10), (50, 50, 50), -1)\n",
606
+ " cv2.putText(frame, f\"Stress AUs: {stress_count}/6 | Positive AUs: {positive_count}/4\", \n",
607
+ " (20, frame_height - 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2)\n",
608
+ " \n",
609
+ " cv2.imshow('Complete 10-AU Stress Detection', frame)\n",
610
+ " frame_count += 1\n",
611
+ " \n",
612
+ " if cv2.waitKey(1) & 0xFF == ord('q'):\n",
613
+ " break\n",
614
+ " \n",
615
+ " cap.release()\n",
616
+ " cv2.destroyAllWindows()\n",
617
+ " \n",
618
+ " if save_data:\n",
619
+ " timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
620
+ " filename = f\"complete_au_features_{timestamp}.csv\"\n",
621
+ " feature_extractor.save_features(filename)\n",
622
+ " \n",
623
+ " print(f\"\\nβœ“ Session complete! Processed {frame_count} frames\")\n",
624
+ " print(f\"βœ“ Average FPS: {frame_count/duration_seconds:.1f}\")\n",
625
+ " \n",
626
+ " return feature_extractor.get_dataframe()\n",
627
+ "\n",
628
+ "\n",
629
+ "# ==================== ADVANCED ANALYSIS ====================\n",
630
+ "\n",
631
+ "def calculate_comprehensive_stress_score(df):\n",
632
+ " \"\"\"\n",
633
+ " Research-based stress scoring using all 10 AUs\n",
634
+ " Weights based on affective computing literature\n",
635
+ " \"\"\"\n",
636
+ " \n",
637
+ " # STRESS INDICATORS (weighted by research evidence)\n",
638
+ " # AU04 (Brow Lower) - Primary anger/stress indicator\n",
639
+ " au04_score = (df['AU04_BrowLower_intensity'].mean() / 100) * (df['AU04_BrowLower_activation_rate'].mean()) * 25\n",
640
+ " \n",
641
+ " # AU01 (Inner Brow Raise) - Worry/sadness indicator\n",
642
+ " au01_score = (df['AU01_InnerBrowRaise_intensity'].mean() / 100) * (df['AU01_InnerBrowRaise_activation_rate'].mean()) * 15\n",
643
+ " \n",
644
+ " # AU07 (Lid Tighten) - Tension indicator\n",
645
+ " au07_score = (df['AU07_LidTighten_intensity'].mean() / 100) * (df['AU07_LidTighten_activation_rate'].mean()) * 12\n",
646
+ " \n",
647
+ " # AU24 (Lip Press) - Stress/tension\n",
648
+ " au24_score = (df['AU24_LipPress_intensity'].mean() / 100) * (df['AU24_LipPress_activation_rate'].mean()) * 15\n",
649
+ " \n",
650
+ " # AU23 (Lip Tighten) - Anger/tension\n",
651
+ " au23_score = (df['AU23_LipTighten_intensity'].mean() / 100) * (df['AU23_LipTighten_activation_rate'].mean()) * 12\n",
652
+ " \n",
653
+ " # AU17 (Chin Raise) - Doubt/sadness\n",
654
+ " au17_score = (df['AU17_ChinRaise_intensity'].mean() / 100) * (df['AU17_ChinRaise_activation_rate'].mean()) * 8\n",
655
+ " \n",
656
+ " # POSITIVE INDICATORS (reduce stress score)\n",
657
+ " # AU06 + AU12 (Duchenne Smile - genuine happiness)\n",
658
+ " duchenne_smile = ((df['AU06_CheekRaise_active'] == 1) & (df['AU12_LipCornerPull_active'] == 1)).sum() / len(df)\n",
659
+ " positive_reduction = duchenne_smile * 15\n",
660
+ " \n",
661
+ " # AU12 alone (social smile - may mask stress)\n",
662
+ " social_smile = ((df['AU12_LipCornerPull_active'] == 1) & (df['AU06_CheekRaise_active'] == 0)).sum() / len(df)\n",
663
+ " masking_indicator = social_smile * 5 # Adds to stress if smiling without cheek raise\n",
664
+ " \n",
665
+ " # TEMPORAL PATTERNS\n",
666
+ " # Sustained stress (continuous activation 3+ seconds)\n",
667
+ " sustained_stress = 0\n",
668
+ " for au_name in ['AU04_BrowLower', 'AU24_LipPress', 'AU23_LipTighten']:\n",
669
+ " streak = 0\n",
670
+ " for val in df[f'{au_name}_active']:\n",
671
+ " if val == 1:\n",
672
+ " streak += 1\n",
673
+ " if streak >= 90: # 3 seconds at 30fps\n",
674
+ " sustained_stress += 1\n",
675
+ " break\n",
676
+ " else:\n",
677
+ " streak = 0\n",
678
+ " sustained_score = min(10, sustained_stress * 3)\n",
679
+ " \n",
680
+ " # Co-occurrence of multiple stress AUs\n",
681
+ " stress_cols = ['AU01_InnerBrowRaise_active', 'AU04_BrowLower_active', \n",
682
+ " 'AU07_LidTighten_active', 'AU23_LipTighten_active', \n",
683
+ " 'AU24_LipPress_active', 'AU17_ChinRaise_active']\n",
684
+ " co_occurrence = (df[stress_cols].sum(axis=1) >= 3).sum() / len(df)\n",
685
+ " co_occurrence_score = co_occurrence * 8\n",
686
+ " \n",
687
+ " # COMBINED STRESS SCORE\n",
688
+ " raw_stress = (au04_score + au01_score + au07_score + au24_score + \n",
689
+ " au23_score + au17_score + sustained_score + \n",
690
+ " co_occurrence_score + masking_indicator - positive_reduction)\n",
691
+ " \n",
692
+ " stress_score = min(100, max(0, raw_stress))\n",
693
+ " \n",
694
+ " # Classification\n",
695
+ " if stress_score < 25:\n",
696
+ " classification = \"NOT STRESSED\"\n",
697
+ " color = \"🟒\"\n",
698
+ " elif stress_score < 55:\n",
699
+ " classification = \"POSSIBLY STRESSED\"\n",
700
+ " color = \"🟑\"\n",
701
+ " else:\n",
702
+ " classification = \"STRESSED\"\n",
703
+ " color = \"πŸ”΄\"\n",
704
+ " \n",
705
+ " return {\n",
706
+ " 'classification': classification,\n",
707
+ " 'color': color,\n",
708
+ " 'stress_score': stress_score,\n",
709
+ " 'components': {\n",
710
+ " 'au04': au04_score,\n",
711
+ " 'au01': au01_score,\n",
712
+ " 'au07': au07_score,\n",
713
+ " 'au24': au24_score,\n",
714
+ " 'au23': au23_score,\n",
715
+ " 'au17': au17_score,\n",
716
+ " 'sustained': sustained_score,\n",
717
+ " 'co_occurrence': co_occurrence_score,\n",
718
+ " 'duchenne_smile': duchenne_smile,\n",
719
+ " 'social_smile_masking': masking_indicator\n",
720
+ " },\n",
721
+ " 'activation_percentages': {\n",
722
+ " 'AU01': (df['AU01_InnerBrowRaise_active'].sum() / len(df)) * 100,\n",
723
+ " 'AU04': (df['AU04_BrowLower_active'].sum() / len(df)) * 100,\n",
724
+ " 'AU06': (df['AU06_CheekRaise_active'].sum() / len(df)) * 100,\n",
725
+ " 'AU07': (df['AU07_LidTighten_active'].sum() / len(df)) * 100,\n",
726
+ " 'AU12': (df['AU12_LipCornerPull_active'].sum() / len(df)) * 100,\n",
727
+ " 'AU14': (df['AU14_Dimpler_active'].sum() / len(df)) * 100,\n",
728
+ " 'AU17': (df['AU17_ChinRaise_active'].sum() / len(df)) * 100,\n",
729
+ " 'AU23': (df['AU23_LipTighten_active'].sum() / len(df)) * 100,\n",
730
+ " 'AU24': (df['AU24_LipPress_active'].sum() / len(df)) * 100,\n",
731
+ " 'AU26': (df['AU26_JawDrop_active'].sum() / len(df)) * 100\n",
732
+ " }\n",
733
+ " }\n",
734
+ "\n",
735
+ "\n",
736
+ "# ==================== COMPREHENSIVE VISUALIZATION ====================\n",
737
+ "\n",
738
+ "def plot_comprehensive_analysis(df):\n",
739
+ " \"\"\"Create 10 publication-quality plots\"\"\"\n",
740
+ " \n",
741
+ " fig = plt.figure(figsize=(20, 16))\n",
742
+ " gs = GridSpec(5, 3, figure=fig, hspace=0.35, wspace=0.3)\n",
743
+ " \n",
744
+ " fig.suptitle('Comprehensive 10-AU Facial Expression Analysis\\nStress Detection System', \n",
745
+ " fontsize=18, fontweight='bold', y=0.995)\n",
746
+ " \n",
747
+ " # Plot 1: All AU Activations Over Time\n",
748
+ " ax1 = fig.add_subplot(gs[0, :2])\n",
749
+ " stress_aus = ['AU01_InnerBrowRaise', 'AU04_BrowLower', 'AU07_LidTighten', \n",
750
+ " 'AU17_ChinRaise', 'AU23_LipTighten', 'AU24_LipPress']\n",
751
+ " colors_stress = ['orange', 'red', 'darkred', 'brown', 'purple', 'magenta']\n",
752
+ " \n",
753
+ " for au, color in zip(stress_aus, colors_stress):\n",
754
+ " ax1.plot(df['timestamp'], df[f'{au}_active'], label=au.split('_')[1], \n",
755
+ " color=color, linewidth=1.5, alpha=0.7)\n",
756
+ " \n",
757
+ " ax1.set_xlabel('Time (seconds)', fontweight='bold')\n",
758
+ " ax1.set_ylabel('Activation (Binary)', fontweight='bold')\n",
759
+ " ax1.set_title('Stress-Related AU Temporal Patterns')\n",
760
+ " ax1.legend(loc='upper right', ncol=3, fontsize=8)\n",
761
+ " ax1.grid(True, alpha=0.3)\n",
762
+ " ax1.set_ylim(-0.1, 1.1)\n",
763
+ " \n",
764
+ " # Plot 2: Positive AUs Over Time\n",
765
+ " ax2 = fig.add_subplot(gs[0, 2])\n",
766
+ " positive_aus = ['AU06_CheekRaise', 'AU12_LipCornerPull', 'AU14_Dimpler', 'AU26_JawDrop']\n",
767
+ " colors_pos = ['lightgreen', 'green', 'darkgreen', 'blue']\n",
768
+ " \n",
769
+ " for au, color in zip(positive_aus, colors_pos):\n",
770
+ " ax2.plot(df['timestamp'], df[f'{au}_active'], label=au.split('_')[1], \n",
771
+ " color=color, linewidth=1.5, alpha=0.7)\n",
772
+ " \n",
773
+ " ax2.set_xlabel('Time (s)', fontweight='bold')\n",
774
+ " ax2.set_ylabel('Activation', fontweight='bold')\n",
775
+ " ax2.set_title('Positive AU Patterns')\n",
776
+ " ax2.legend(fontsize=7)\n",
777
+ " ax2.grid(True, alpha=0.3)\n",
778
+ " ax2.set_ylim(-0.1, 1.1)\n",
779
+ " \n",
780
+ " # Plot 3: Intensity Heatmap (All 10 AUs)\n",
781
+ " ax3 = fig.add_subplot(gs[1, :])\n",
782
+ " all_aus = ['AU01_InnerBrowRaise', 'AU04_BrowLower', 'AU06_CheekRaise', \n",
783
+ " 'AU07_LidTighten', 'AU12_LipCornerPull', 'AU14_Dimpler',\n",
784
+ " 'AU17_ChinRaise', 'AU23_LipTighten', 'AU24_LipPress', 'AU26_JawDrop']\n",
785
+ " \n",
786
+ " intensity_data = df[[f'{au}_intensity' for au in all_aus]].T\n",
787
+ " im = ax3.imshow(intensity_data, aspect='auto', cmap='RdYlGn_r', interpolation='nearest')\n",
788
+ " ax3.set_yticks(range(10))\n",
789
+ " ax3.set_yticklabels([au.split('_')[0] for au in all_aus])\n",
790
+ " ax3.set_xlabel('Frame Number', fontweight='bold')\n",
791
+ " ax3.set_title('Complete AU Intensity Heatmap (All 10 Action Units)')\n",
792
+ " plt.colorbar(im, ax=ax3, label='Intensity (%)')\n",
793
+ " \n",
794
+ " # Plot 4: AU Activation Frequency Bar Chart\n",
795
+ " ax4 = fig.add_subplot(gs[2, 0])\n",
796
+ " result = calculate_comprehensive_stress_score(df)\n",
797
+ " au_names = list(result['activation_percentages'].keys())\n",
798
+ " au_values = list(result['activation_percentages'].values())\n",
799
+ " colors = ['red' if 'AU04' in au or 'AU24' in au or 'AU23' in au \n",
800
+ " else 'orange' if 'AU01' in au or 'AU07' in au or 'AU17' in au\n",
801
+ " else 'green' for au in au_names]\n",
802
+ " \n",
803
+ " bars = ax4.barh(au_names, au_values, color=colors, alpha=0.7)\n",
804
+ " ax4.set_xlabel('Activation Percentage (%)', fontweight='bold')\n",
805
+ " ax4.set_title('AU Activation Frequencies')\n",
806
+ " ax4.grid(True, alpha=0.3, axis='x')\n",
807
+ " \n",
808
+ " # Plot 5: Duchenne vs Non-Duchenne Smile Detection\n",
809
+ " ax5 = fig.add_subplot(gs[2, 1])\n",
810
+ " duchenne = ((df['AU06_CheekRaise_active'] == 1) & (df['AU12_LipCornerPull_active'] == 1)).sum()\n",
811
+ " non_duchenne = ((df['AU12_LipCornerPull_active'] == 1) & (df['AU06_CheekRaise_active'] == 0)).sum()\n",
812
+ " no_smile = len(df) - duchenne - non_duchenne\n",
813
+ " \n",
814
+ " labels = ['Genuine\\n(Duchenne)', 'Social\\n(Masking)', 'No Smile']\n",
815
+ " sizes = [duchenne, non_duchenne, no_smile]\n",
816
+ " colors_pie = ['green', 'yellow', 'lightgray']\n",
817
+ " \n",
818
+ " ax5.pie(sizes, labels=labels, colors=colors_pie, autopct='%1.1f%%', startangle=90)\n",
819
+ " ax5.set_title('Smile Type Distribution')\n",
820
+ " \n",
821
+ " # Plot 6: Stress vs Positive Balance\n",
822
+ " ax6 = fig.add_subplot(gs[2, 2])\n",
823
+ " stress_intensity_avg = df[['AU01_InnerBrowRaise_intensity', 'AU04_BrowLower_intensity',\n",
824
+ " 'AU07_LidTighten_intensity', 'AU23_LipTighten_intensity',\n",
825
+ " 'AU24_LipPress_intensity', 'AU17_ChinRaise_intensity']].mean(axis=1)\n",
826
+ " positive_intensity_avg = df[['AU06_CheekRaise_intensity', 'AU12_LipCornerPull_intensity',\n",
827
+ " 'AU14_Dimpler_intensity']].mean(axis=1)\n",
828
+ " \n",
829
+ " ax6.plot(df['timestamp'], stress_intensity_avg, color='red', linewidth=2, label='Stress AUs')\n",
830
+ " ax6.plot(df['timestamp'], positive_intensity_avg, color='green', linewidth=2, label='Positive AUs')\n",
831
+ " ax6.fill_between(df['timestamp'], stress_intensity_avg, alpha=0.3, color='red')\n",
832
+ " ax6.fill_between(df['timestamp'], positive_intensity_avg, alpha=0.3, color='green')\n",
833
+ " ax6.set_xlabel('Time (seconds)', fontweight='bold')\n",
834
+ " ax6.set_ylabel('Average Intensity (%)', fontweight='bold')\n",
835
+ " ax6.set_title('Stress vs Positive Affect Balance')\n",
836
+ " ax6.legend()\n",
837
+ " ax6.grid(True, alpha=0.3)\n",
838
+ " \n",
839
+ " # Plot 7: Correlation Matrix (All AUs)\n",
840
+ " ax7 = fig.add_subplot(gs[3, :2])\n",
841
+ " correlation_cols = [f'{au}_intensity' for au in all_aus]\n",
842
+ " corr_matrix = df[correlation_cols].corr()\n",
843
+ " \n",
844
+ " im = ax7.imshow(corr_matrix, cmap='coolwarm', vmin=-1, vmax=1, aspect='auto')\n",
845
+ " ax7.set_xticks(range(10))\n",
846
+ " ax7.set_yticks(range(10))\n",
847
+ " ax7.set_xticklabels([au.split('_')[0] for au in all_aus], rotation=45, ha='right')\n",
848
+ " ax7.set_yticklabels([au.split('_')[0] for au in all_aus])\n",
849
+ " ax7.set_title('Complete AU Correlation Matrix')\n",
850
+ " \n",
851
+ " # Add correlation values\n",
852
+ " for i in range(10):\n",
853
+ " for j in range(10):\n",
854
+ " if abs(corr_matrix.iloc[i, j]) > 0.3: # Only show strong correlations\n",
855
+ " ax7.text(j, i, f'{corr_matrix.iloc[i, j]:.2f}',\n",
856
+ " ha=\"center\", va=\"center\", color=\"black\", fontsize=7)\n",
857
+ " \n",
858
+ " plt.colorbar(im, ax=ax7)\n",
859
+ " \n",
860
+ " # Plot 8: Time-Windowed Stress Evolution\n",
861
+ " ax8 = fig.add_subplot(gs[3, 2])\n",
862
+ " window_size = 90 # 3 seconds\n",
863
+ " windowed_stress = []\n",
864
+ " window_times = []\n",
865
+ " \n",
866
+ " for i in range(0, len(df) - window_size, window_size // 2):\n",
867
+ " window_df = df.iloc[i:i+window_size]\n",
868
+ " if len(window_df) > 0:\n",
869
+ " window_result = calculate_comprehensive_stress_score(window_df)\n",
870
+ " windowed_stress.append(window_result['stress_score'])\n",
871
+ " window_times.append(window_df['timestamp'].mean())\n",
872
+ " \n",
873
+ " ax8.plot(window_times, windowed_stress, color='red', linewidth=2, marker='o')\n",
874
+ " ax8.fill_between(window_times, windowed_stress, alpha=0.3, color='red')\n",
875
+ " ax8.axhline(y=25, color='green', linestyle='--', label='Low threshold', alpha=0.5)\n",
876
+ " ax8.axhline(y=55, color='orange', linestyle='--', label='High threshold', alpha=0.5)\n",
877
+ " ax8.set_xlabel('Time (seconds)', fontweight='bold')\n",
878
+ " ax8.set_ylabel('Stress Score', fontweight='bold')\n",
879
+ " ax8.set_title('Stress Score Evolution (3s windows)')\n",
880
+ " ax8.legend()\n",
881
+ " ax8.grid(True, alpha=0.3)\n",
882
+ " \n",
883
+ " # Plot 9: AU Co-occurrence Matrix\n",
884
+ " ax9 = fig.add_subplot(gs[4, 0])\n",
885
+ " stress_au_cols = [f'{au}_active' for au in stress_aus]\n",
886
+ " co_occur_matrix = np.zeros((6, 6))\n",
887
+ " \n",
888
+ " for i in range(6):\n",
889
+ " for j in range(6):\n",
890
+ " co_occur = ((df[stress_au_cols[i]] == 1) & (df[stress_au_cols[j]] == 1)).sum()\n",
891
+ " co_occur_matrix[i, j] = co_occur / len(df) * 100\n",
892
+ " \n",
893
+ " im = ax9.imshow(co_occur_matrix, cmap='Reds', aspect='auto')\n",
894
+ " ax9.set_xticks(range(6))\n",
895
+ " ax9.set_yticks(range(6))\n",
896
+ " ax9.set_xticklabels([au.split('_')[0] for au in stress_aus], rotation=45, ha='right')\n",
897
+ " ax9.set_yticklabels([au.split('_')[0] for au in stress_aus])\n",
898
+ " ax9.set_title('Stress AU Co-occurrence (%)')\n",
899
+ " plt.colorbar(im, ax=ax9)\n",
900
+ " \n",
901
+ " # Plot 10: Comprehensive Summary Report\n",
902
+ " ax10 = fig.add_subplot(gs[4, 1:])\n",
903
+ " ax10.axis('off')\n",
904
+ " \n",
905
+ " result = calculate_comprehensive_stress_score(df)\n",
906
+ " \n",
907
+ " summary_text = f\"\"\"\n",
908
+ "╔═══════════════════════════════════════════════════════════════════════════════════════════════╗\n",
909
+ "β•‘ COMPREHENSIVE STRESS ASSESSMENT REPORT β•‘\n",
910
+ "╠═══════════════════════════════════════════════════════════════════════════════════════════════╣\n",
911
+ "β•‘ β•‘\n",
912
+ "β•‘ CLASSIFICATION: {result['color']} {result['classification']:<25} | STRESS SCORE: {result['stress_score']:.1f}/100 β•‘\n",
913
+ "β•‘ β•‘\n",
914
+ "╠═══════════════════════════════════════════════════════════════════════════════════════════════╣\n",
915
+ "β•‘ COMPONENT CONTRIBUTIONS: β•‘\n",
916
+ "β•‘ β€’ AU04 (Brow Lower): {result['components']['au04']:.2f} / 25.0 [{result['activation_percentages']['AU04']:5.1f}% active] β•‘\n",
917
+ "β•‘ β€’ AU01 (Inner Brow): {result['components']['au01']:.2f} / 15.0 [{result['activation_percentages']['AU01']:5.1f}% active] β•‘\n",
918
+ "β•‘ β€’ AU07 (Lid Tighten): {result['components']['au07']:.2f} / 12.0 [{result['activation_percentages']['AU07']:5.1f}% active] β•‘\n",
919
+ "β•‘ β€’ AU24 (Lip Press): {result['components']['au24']:.2f} / 15.0 [{result['activation_percentages']['AU24']:5.1f}% active] β•‘\n",
920
+ "β•‘ β€’ AU23 (Lip Tighten): {result['components']['au23']:.2f} / 12.0 [{result['activation_percentages']['AU23']:5.1f}% active] β•‘\n",
921
+ "β•‘ β€’ AU17 (Chin Raise): {result['components']['au17']:.2f} / 8.0 [{result['activation_percentages']['AU17']:5.1f}% active] β•‘\n",
922
+ "β•‘ β€’ Sustained Activation: {result['components']['sustained']:.2f} / 10.0 β•‘\n",
923
+ "β•‘ β€’ Co-occurrence Pattern: {result['components']['co_occurrence']:.2f} / 8.0 β•‘\n",
924
+ "β•‘ β€’ Smile Masking Effect: {result['components']['social_smile_masking']:.2f} (adds stress if present) β•‘\n",
925
+ "β•‘ β€’ Duchenne Smile Bonus: -{result['components']['duchenne_smile']*15:.2f} (reduces stress) β•‘\n",
926
+ "β•‘ β•‘\n",
927
+ "╠═══════════════════════════════════════════════════════════════════════════════════════════════╣\n",
928
+ "β•‘ POSITIVE AFFECT INDICATORS: β•‘\n",
929
+ "β•‘ β€’ AU06 (Cheek Raise): {result['activation_percentages']['AU06']:5.1f}% active β•‘\n",
930
+ "β•‘ β€’ AU12 (Lip Pull): {result['activation_percentages']['AU12']:5.1f}% active β•‘\n",
931
+ "β•‘ β€’ AU14 (Dimpler): {result['activation_percentages']['AU14']:5.1f}% active β•‘\n",
932
+ "β•‘ β€’ Duchenne Smile Rate: {result['components']['duchenne_smile']*100:.1f}% β•‘\n",
933
+ "β•‘ β•‘\n",
934
+ "╠═══════════════════════════════════════════════════════════════════════════════════════════════╣\n",
935
+ "β•‘ RESEARCH BASIS: β•‘\n",
936
+ "β•‘ Weights based on Facial Action Coding System (FACS) research: β•‘\n",
937
+ "β•‘ β€’ AU04 highest weight (Ekman & Friesen, 1978) - primary anger/stress indicator β•‘\n",
938
+ "β•‘ β€’ AU06+AU12 combination identifies genuine happiness (Duchenne marker) β•‘\n",
939
+ "β•‘ β€’ Sustained activation and co-occurrence patterns enhance stress detection accuracy β•‘\n",
940
+ "β•‘ β€’ Temporal windowing allows detection of acute stress episodes vs chronic patterns β•‘\n",
941
+ "β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•\n",
942
+ " \"\"\"\n",
943
+ " \n",
944
+ " ax10.text(0.05, 0.5, summary_text, fontsize=8, family='monospace',\n",
945
+ " verticalalignment='center',\n",
946
+ " bbox=dict(boxstyle='round', facecolor='lightgray', alpha=0.2))\n",
947
+ " \n",
948
+ " plt.tight_layout()\n",
949
+ " return fig, result\n",
950
+ "\n",
951
+ "\n",
952
+ "# ==================== MAIN EXECUTION ====================\n",
953
+ "\n",
954
+ "if __name__ == \"__main__\":\n",
955
+ " print(\"\\n\" + \"=\"*70)\n",
956
+ " print(\" COMPLETE 10-AU STRESS DETECTION SYSTEM\")\n",
957
+ " print(\" Based on Facial Action Coding System (FACS)\")\n",
958
+ " print(\" Research Guide: Prof. Anup Nandy\")\n",
959
+ " print(\"=\"*70)\n",
960
+ " \n",
961
+ " print(\"\\n Action Units Detected:\")\n",
962
+ " print(\" STRESS: AU01, AU04, AU07, AU17, AU23, AU24\")\n",
963
+ " print(\" POSITIVE: AU06, AU12, AU14, AU26\")\n",
964
+ " print(\"\\n Press Enter to start 15-second recording...\")\n",
965
+ " \n",
966
+ " input()\n",
967
+ " \n",
968
+ " df = run_detection_session(duration_seconds=15, save_data=True)\n",
969
+ " \n",
970
+ " print(\"\\n\" + \"=\"*70)\n",
971
+ " print(\" Generating comprehensive analysis...\")\n",
972
+ " print(\"=\"*70 + \"\\n\")\n",
973
+ " \n",
974
+ " fig, result = plot_comprehensive_analysis(df)\n",
975
+ " \n",
976
+ " print(f\"\\n{result['color']} FINAL ASSESSMENT: {result['classification']}\")\n",
977
+ " print(f\" Stress Score: {result['stress_score']:.1f}/100\")\n",
978
+ " print(f\"\\n Data saved with {len(df)} frames\")\n",
979
+ " print(f\" Total features per frame: {len(df.columns) - 1}\")\n",
980
+ " print(\"\\n\" + \"=\"*70)\n",
981
+ " \n",
982
+ " plt.show()\n",
983
+ " \n",
984
+ " print(\"\\nβœ“ Analysis complete!\")"
985
+ ]
986
+ },
987
+ {
988
+ "cell_type": "code",
989
+ "execution_count": null,
990
+ "id": "a7551dd6",
991
+ "metadata": {},
992
+ "outputs": [],
993
+ "source": []
994
+ }
995
+ ],
996
+ "metadata": {
997
+ "kernelspec": {
998
+ "display_name": "Python 3.8 (Global)",
999
+ "language": "python",
1000
+ "name": "globalpython38"
1001
+ },
1002
+ "language_info": {
1003
+ "codemirror_mode": {
1004
+ "name": "ipython",
1005
+ "version": 3
1006
+ },
1007
+ "file_extension": ".py",
1008
+ "mimetype": "text/x-python",
1009
+ "name": "python",
1010
+ "nbconvert_exporter": "python",
1011
+ "pygments_lexer": "ipython3",
1012
+ "version": "3.8.10"
1013
+ }
1014
+ },
1015
+ "nbformat": 4,
1016
+ "nbformat_minor": 5
1017
+ }