/** * Sentinel Live Pulse — Real-time Multimodal Analysis Engine * v5 — Precise AU-based emotion detection, stable face mesh, WAV recording */ // ── State ───────────────────────────────────────────────── const state = { isRunning: false, ws: null, webcamStream: null, audioContext: null, audioAnalyser: null, audioStream: null, mediaRecorder: null, audioChunks: [], frameInterval: null, engagementHistory: [], blinkTimestamps: [], blinkCount: 0, // Simple cumulative blink counter stableFramesCount: 0, // Consecutive stable & focused frames lastBlinkState: false, lastNoseX: null, // For head stability check lastNoseY: null, sessionStartTime: null, peakEngagement: 0, totalEngagement: 0, engagementSamples: 0, faceMesh: null, camera: null, emotionBuffer: [], lastStableEmotion: 'Neutral', lastStableConfidence: 0, faceMeshReady: false, meshFailCount: 0, // Calibration: first 15 frames establish the user's resting face calibrationFrames: [], baseline: null, // Timing counters for 1-second emotion rules (~20 frames @ 20fps) neutralStableFrames: 0, mouthOpenFrames: 0, }; // ── DOM References ──────────────────────────────────────── const video = document.getElementById('webcam-video'); const meshCanvas = document.getElementById('mesh-canvas'); const meshCtx = meshCanvas.getContext('2d'); const logContent = document.getElementById('log-content'); const btnStartStop = document.getElementById('btn-start-stop'); // ── Initialize Mini Audio Bars ──────────────────────────── (function initMiniBars() { const container = document.getElementById('mini-audio-viz'); if (!container) return; // Removed from UI for (let i = 0; i < 32; i++) { const bar = document.createElement('div'); bar.className = 'audio-bar'; bar.style.background = `rgba(209, 188, 255, ${0.3 + Math.random() * 0.5})`; bar.style.height = '2px'; container.appendChild(bar); } })(); // ── Logging ─────────────────────────────────────────────── function addLog(tag, message, color) { const p = document.createElement('p'); p.className = 'log-line'; const elapsed = state.sessionStartTime ? ((Date.now() - state.sessionStartTime) / 1000).toFixed(3) : '0.000'; p.innerHTML = `[${elapsed}s] ${tag} :: ${message}`; logContent.appendChild(p); logContent.scrollTop = logContent.scrollHeight; while (logContent.children.length > 100) logContent.removeChild(logContent.firstChild); } // ── API URLs ────────────────────────────────────────────── const API_BASE = "https://r-vasanthkumar73-dev-sentinel-multimodal-emotion-ai.hf.space"; const speechApiUrl = `${API_BASE}/api/analyze/speech`; const textApiUrl = `${API_BASE}/api/analyze/text`; const sessionSaveUrl = `${API_BASE}/api/session/save`; // ── Start / Stop ────────────────────────────────────────── btnStartStop.addEventListener('click', () => { ensureAudioContext(); if (state.isRunning) stopAll(); else startAll(); }); async function startAll() { state.isRunning = true; state.sessionStartTime = Date.now(); state.blinkTimestamps = []; state.blinkCount = 0; state.lastNoseX = null; state.lastNoseY = null; state.peakEngagement = 0; state.totalEngagement = 0; state.engagementSamples = 0; state.engagementHistory = []; state.emotionBuffer = []; state.lastStableEmotion = 'Neutral'; state.lastStableConfidence = 0; state.meshFailCount = 0; state.calibrationFrames = []; state.baseline = null; btnStartStop.innerHTML = 'stop Stop Monitoring'; document.getElementById('live-badge').style.display = 'flex'; document.getElementById('log-status').textContent = 'LIVE_PKT_RECEIVED'; document.getElementById('scan-line').style.display = 'block'; document.getElementById('connection-status').innerHTML = 'Connected'; addLog('SYSTEM', 'Starting multimodal analysis engine...', 'var(--primary-container)'); try { state.webcamStream = await navigator.mediaDevices.getUserMedia({ video: { width: 640, height: 480, facingMode: 'user' }, audio: false }); video.srcObject = state.webcamStream; document.getElementById('face-status').textContent = 'Active Stream'; document.getElementById('source-face').classList.add('active'); const ctrlCam = document.getElementById('ctrl-camera'); if (ctrlCam) ctrlCam.classList.add('active'); addLog('CAMERA', 'Webcam stream initialized (640x480)', 'var(--primary-container)'); } catch (e) { addLog('CAMERA', 'Failed to access webcam: ' + e.message, 'var(--error)'); } document.getElementById('audio-status').textContent = 'Standby (Click Audio Panel)'; startFaceMesh(); } function stopAll() { state.isRunning = false; btnStartStop.innerHTML = 'play_arrow Start Monitoring'; document.getElementById('live-badge').style.display = 'none'; document.getElementById('log-status').textContent = 'STANDBY'; document.getElementById('scan-line').style.display = 'none'; document.getElementById('connection-status').innerHTML = 'Standby'; if (state.webcamStream) { state.webcamStream.getTracks().forEach(t => t.stop()); state.webcamStream = null; video.srcObject = null; } if (state.audioStream) { state.audioStream.getTracks().forEach(t => t.stop()); state.audioStream = null; } if (state.audioContext) { state.audioContext.close(); state.audioContext = null; } if (state.mediaRecorder && state.mediaRecorder.state !== 'inactive') state.mediaRecorder.stop(); if (state.faceMesh) { state.faceMesh.close(); state.faceMesh = null; } state.faceMeshReady = false; // Clean up timers if (window._meshWatchdog) { clearInterval(window._meshWatchdog); window._meshWatchdog = null; } if (window.hfLoopInterval) { clearInterval(window.hfLoopInterval); window.hfLoopInterval = null; } meshCtx.clearRect(0, 0, meshCanvas.width, meshCanvas.height); document.querySelectorAll('.source-btn').forEach(b => b.classList.remove('active')); document.getElementById('face-status').textContent = 'Standby'; document.getElementById('audio-status').textContent = 'Standby'; const ctrlCamEl = document.getElementById('ctrl-camera'); if (ctrlCamEl) ctrlCamEl.classList.remove('active'); const ctrlMicEl = document.getElementById('ctrl-mic'); if (ctrlMicEl) ctrlMicEl.classList.remove('active'); addLog('SYSTEM', 'Monitoring stopped.', 'var(--tertiary-fixed-dim)'); saveSession(); } // ══════════════════════════════════════════════════════════ // FACE MESH — STABLE, AUTO-RECOVERING // ══════════════════════════════════════════════════════════ async function startFaceMesh() { addLog('FACEMESH', 'Initializing local Face Mesh engine...', 'var(--primary-container)'); try { state.faceMesh = new FaceMesh({ locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh/${file}` }); state.faceMesh.setOptions({ maxNumFaces: 1, refineLandmarks: true, minDetectionConfidence: 0.5, minTrackingConfidence: 0.5 }); state.faceMesh.onResults(onFaceMeshResults); state.faceMeshReady = true; addLog('FACEMESH', 'Face Mesh engine started — calibrating baseline...', 'var(--primary-container)'); // Frame loop: run as fast as the browser allows (up to 60fps) // using requestAnimationFrame, but protected by meshProcessing lock // so it never overwhelms the WASM pipeline. let meshProcessing = false; async function processVideoFrame() { if (!state.isRunning) return; if (!meshProcessing && state.faceMeshReady && state.faceMesh && video.readyState >= 2) { meshProcessing = true; try { await state.faceMesh.send({ image: video }); state.meshFailCount = 0; } catch (e) { state.meshFailCount++; if (state.meshFailCount > 30 && state.isRunning) { addLog('FACEMESH', 'Auto-recovering...', 'var(--error)'); await restartFaceMesh(); return; // restartFaceMesh starts its own loop } } meshProcessing = false; } if (state.isRunning) { requestAnimationFrame(processVideoFrame); // Maximize responsiveness } } processVideoFrame(); // Watchdog: check every 10s if mesh has gone stale if (window._meshWatchdog) clearInterval(window._meshWatchdog); window._meshWatchdog = setInterval(async () => { if (!state.isRunning) { clearInterval(window._meshWatchdog); return; } if (state.meshFailCount > 15) { addLog('FACEMESH', 'Watchdog: mesh stalled, recovering...', 'var(--error)'); await restartFaceMesh(); } }, 10000); // Pure Local FACS only, no ViT sync } catch (e) { addLog('FACEMESH', 'Failed: ' + e.message, 'var(--error)'); } } async function restartFaceMesh() { state.faceMeshReady = false; state.meshFailCount = 0; try { if (state.faceMesh) state.faceMesh.close(); } catch (e) { } try { state.faceMesh = new FaceMesh({ locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh/${file}` }); state.faceMesh.setOptions({ maxNumFaces: 1, refineLandmarks: true, minDetectionConfidence: 0.5, minTrackingConfidence: 0.5 }); state.faceMesh.onResults(onFaceMeshResults); state.faceMeshReady = true; addLog('FACEMESH', 'Recovered successfully.', 'var(--primary-container)'); } catch (e) { addLog('FACEMESH', 'Recovery failed.', 'var(--error)'); } } // ══════════════════════════════════════════════════════════ // PRECISION AU-BASED EMOTION DETECTION // Based on FACS (Facial Action Coding System) // with dynamic baseline calibration // ══════════════════════════════════════════════════════════ function extractFaceMetrics(pts) { const dist = (a, b) => Math.sqrt((a.x - b.x) ** 2 + (a.y - b.y) ** 2); const faceH = dist(pts[10], pts[152]); if (faceH < 0.01) return null; // 1. Mouth Width (61 to 291) const mouth_width = dist(pts[61], pts[291]) / faceH; // 2. Eye Distance (Outer corners: 33 to 263) const eye_dist = dist(pts[33], pts[263]) / faceH; // 3. Teeth visibility / Jaw Drop (Inner lips: 13 to 14) const inner_lip_dist = dist(pts[13], pts[14]) / faceH; // 4. Inner Brows distance (70 to 300) const inner_brow_dist = dist(pts[70], pts[300]) / faceH; // 5. Nose tip to lip corners (Vertical distance) -> Frown (AU15) const nose_to_lip_y = (((pts[61].y + pts[291].y) / 2.0) - pts[4].y) / faceH; // 6. Eye Opening (Vertical) -> AU5 (Wide) / AU6 (Cheek Raise Squint) const left_eye_open = dist(pts[159], pts[145]); const right_eye_open = dist(pts[386], pts[374]); const eye_opening = ((left_eye_open + right_eye_open) / 2.0) / faceH; // 7. Brow to Eye Vertical Distance (Brows Rise AU1+2 / Lower AU4) const eye_center_y = (pts[159].y + pts[386].y + pts[145].y + pts[374].y) / 4.0; const brow_center_y = (pts[70].y + pts[300].y) / 2.0; const brow_to_eye_y = (eye_center_y - brow_center_y) / faceH; // 8. Upward Curve (Mouth center Y vs Lip corners Y) -> Smile (AU12) const mouth_center_y = (pts[13].y + pts[14].y) / 2.0; const lip_corner_avg_y = (pts[61].y + pts[291].y) / 2.0; const upward_curve = (mouth_center_y - lip_corner_avg_y) / faceH; // 9. Nose bridge to upper lip (Nose Wrinkler AU9 + Upper Lip Raiser AU10) const nose_to_upper_lip = dist(pts[6], pts[13]) / faceH; // 10. Eye-to-mouth vertical ratio const eye_mouth_y_dist = (mouth_center_y - eye_center_y) / faceH; // 11. Lip Height (Lip Tightener AU23) const lip_height = dist(pts[13], pts[14]) / faceH; // 12. Mouth asymmetry — left vs right lip corner Y (for Disgust one-sided sneer) // Left lip corner is pt 61, Right lip corner is pt 291 // If one side is significantly higher than the other = sneer/disgust const lip_asym = Math.abs(pts[61].y - pts[291].y) / faceH; // 13. Downward curve — lip corners below mouth center = Sad frown // Negative of upward_curve: if positive, corners are BELOW center (frown) const downward_curve = (lip_corner_avg_y - mouth_center_y) / faceH; // 14. Nose-to-eye distance (face shrink for Angry) const nose_to_eye_y = (eye_center_y - pts[4].y) / faceH; return { mouth_width, eye_dist, inner_lip_dist, inner_brow_dist, nose_to_lip_y, eye_opening, brow_to_eye_y, upward_curve, nose_to_upper_lip, eye_mouth_y_dist, lip_height, lip_asym, downward_curve, nose_to_eye_y }; } function calculateEmotionFromLandmarks(pts) { if (!pts || pts.length < 400) return null; const current_metrics = extractFaceMetrics(pts); if (!current_metrics) return null; // Calibration Phase if (state.calibrationFrames.length < 15) { state.calibrationFrames.push(current_metrics); if (state.calibrationFrames.length === 15) { state.baseline = {}; for (let k in current_metrics) { state.baseline[k] = state.calibrationFrames.reduce((acc, curr) => acc + curr[k], 0) / 15.0; } } return { emotion: "Neutral", confidence: 100, engagement_score: 50, blinking: false }; } const c = current_metrics; const b = state.baseline; // Very slow baseline drift to fix bad initial calibration if (state.lastStableEmotion === 'Neutral' && state.lastStableConfidence > 90) { for (let k in b) { state.baseline[k] = (state.baseline[k] * 0.995) + (c[k] * 0.005); } } const scores = { Happy: 0, Sad: 0, Angry: 0, Surprise: 0, Fear: 0, Disgust: 0, Neutral: 50 }; // Safely calculate percentage change const pct_change = (curr, base) => (curr - base) / Math.max(Math.abs(base), 0.001); // Check if smiling (corners going UP) const is_smiling = (c.upward_curve > b.upward_curve + 0.005); // 1. Happy (AU6 Cheek Raise + AU12 Lip Corner Pull) // AU6 causes slight eye squint (eye_opening decreases slightly) const mouth_eye_ratio_c = c.mouth_width / Math.max(c.eye_dist, 0.001); const mouth_eye_ratio_b = b.mouth_width / Math.max(b.eye_dist, 0.001); const happy_ratio_inc = pct_change(mouth_eye_ratio_c, mouth_eye_ratio_b); const cheek_raise_squint = pct_change(b.eye_opening, c.eye_opening); // Positive if eye opening decreased if (happy_ratio_inc > 0.05 && is_smiling) { scores.Happy = 60 + (happy_ratio_inc * 200); if (cheek_raise_squint > 0.02) scores.Happy += 10; // AU6 detected // AU25 (Teeth) if (c.inner_lip_dist > b.inner_lip_dist * 1.5 && c.inner_lip_dist > 0.005) scores.Happy += 15; } // 2. Sad — UPSIDE DOWN U (Lip Corners Facing Earth) // STRICT RULE: Only show sad if the mouth edges strongly curve downward. const brow_rise_inc = pct_change(c.brow_to_eye_y, b.brow_to_eye_y); const frown_inc = c.downward_curve - b.downward_curve; // Required frown_inc threshold increased to strictly catch the "upside down U" if (frown_inc > 0.015 && !is_smiling) { scores.Sad = 60 + (frown_inc * 1000); if (brow_rise_inc > 0.01) scores.Sad += 15; } // 3. Surprise — MOUTH STAYS OPEN FOR 1 SECOND (~20 frames at 20fps) // Track consecutive frames where inner_lip_dist is significantly above baseline const jaw_drop_inc = c.inner_lip_dist - b.inner_lip_dist; const mouth_is_open = (jaw_drop_inc > 0.015); // mouth clearly open if (mouth_is_open && !is_smiling) { state.mouthOpenFrames++; } else { state.mouthOpenFrames = 0; // reset counter if mouth closes } if (state.mouthOpenFrames >= 20) { // 20 frames ≈ 1 second scores.Surprise = 60 + (jaw_drop_inc * 500) + Math.min(state.mouthOpenFrames - 20, 30) * 1.5; } // 4. Fear — STRESSED FACE (Horizontal Lip Stretch + Tense Brows) // Removed "wide eyes" as primary trigger because it caused false positives during resting. // Trigger: Mouth stretches horizontally (stress) AND brows pull together (tension) const lip_stretch_inc = pct_change(c.mouth_width, b.mouth_width); const brow_dist_dec = pct_change(b.inner_brow_dist, c.inner_brow_dist); if (lip_stretch_inc > 0.03 && brow_dist_dec > 0.02 && !is_smiling) { scores.Fear = 60 + (lip_stretch_inc * 300) + (brow_dist_dec * 150); } // 5. Angry — SHARP EYES + NOSE RAISING + MOUTH CLOSED TIGHTLY OR EXPANDED EYES GLARE // Sharp eyes: eye opening slightly BELOW baseline (focused/narrowed squint) // Nose raising: nose_to_eye_y DECREASES (nose tip pulls up toward eyes) // Mouth closed tightly: inner_lip_dist BELOW baseline const eye_narrow_dec = pct_change(b.eye_opening, c.eye_opening); // positive = eyes narrowed const nose_raise_dec = pct_change(b.nose_to_eye_y, c.nose_to_eye_y); // positive = nose rose up const mouth_shut_dec = pct_change(b.inner_lip_dist, c.inner_lip_dist); // positive = lips pressed const brow_lower_dec = pct_change(b.brow_to_eye_y, c.brow_to_eye_y); const eye_expand_inc = pct_change(c.eye_opening, b.eye_opening); // positive = eyes expanded / big // Path A: Classic Focused Anger (Narrowed eyes + Brow tension) if (brow_dist_dec > 0.03 && (eye_narrow_dec > 0.05 || nose_raise_dec > 0.02) && mouth_shut_dec > 0.05 && !is_smiling) { scores.Angry = 60 + (brow_dist_dec * 200) + (eye_narrow_dec * 150) + (mouth_shut_dec * 100); if (nose_raise_dec > 0.02) scores.Angry += 15; // nose raise confirms anger } // Path B: Glaring / Expanded-Eye Anger (Expanded big eyes + Brow furrow / tension / lower) if (eye_expand_inc > 0.08 && brow_dist_dec > 0.02 && !is_smiling) { const glaring_score = 75 + (eye_expand_inc * 150) + (brow_dist_dec * 100); if (glaring_score > scores.Angry) { scores.Angry = glaring_score; } } // 6. Disgust — NOSE WRINKLER (AU9) + SNEER (AU14) + UPPER LIP RAISER (AU10) // We check for three variations of disgust: // 1. Extreme nose scrunch (nose_to_upper_lip decreases significantly) // 2. Asymmetric sneer (one lip corner pulled up strongly) // 3. Bared teeth with a moderate nose scrunch const nose_short_dec = pct_change(b.nose_to_upper_lip, c.nose_to_upper_lip); const cheek_squint = pct_change(b.eye_opening, c.eye_opening); const lip_asym_inc = c.lip_asym - b.lip_asym; const teeth_bared = c.inner_lip_dist - b.inner_lip_dist > 0.01; const is_classic_scrunch = nose_short_dec > 0.05; const is_sneer = lip_asym_inc > 0.01; const is_bared_scrunch = teeth_bared && nose_short_dec > 0.02; if ((is_classic_scrunch || is_sneer || is_bared_scrunch) && !is_smiling) { let disgustIntensity = Math.max( nose_short_dec * 500, lip_asym_inc * 3000, (nose_short_dec * 250) + (teeth_bared ? 50 : 0) ); scores.Disgust = 60 + disgustIntensity; if (cheek_squint > 0.05) scores.Disgust += 15; // squint confirms disgust // Prevent Disgust from overriding an extreme Sad "upside down U" frown if (scores.Sad > 80) scores.Disgust = 0; } // NEUTRAL — 1 second of no face muscle movement // Track how many consecutive frames ALL metrics stay within 2% of baseline const max_var = Object.keys(c).reduce((mx, k) => { const v = Math.abs(c[k] - b[k]) / Math.max(Math.abs(b[k]), 0.001); return v > mx ? v : mx; }, 0); if (max_var < 0.02) { state.neutralStableFrames++; } else { state.neutralStableFrames = 0; } // Override to Neutral if face has been still for >= 20 frames (~1 second) const forceNeutral = (state.neutralStableFrames >= 20); // Find highest score let bestEmotion = 'Neutral'; let maxVal = scores.Neutral; for (const [emo, val] of Object.entries(scores)) { if (val > maxVal) { maxVal = val; bestEmotion = emo; } } // If face has been still for 1 second, override everything to Neutral if (forceNeutral) { bestEmotion = 'Neutral'; maxVal = 98.99; } // Confidence Assignment Logic let confidence = 100.0; // Initialize state locks if they don't exist state.disgustLockFrames = state.disgustLockFrames || 0; state.lockedDisgustConf = state.lockedDisgustConf || 80.00; state.sadLockFrames = state.sadLockFrames || 0; state.lockedSadConf = state.lockedSadConf || 0; if (bestEmotion === 'Disgust') { // Hold the Disgust percentage fixed for a while (e.g. 60 frames = ~3 seconds at 20fps) if (state.disgustLockFrames > 0) { state.disgustLockFrames--; confidence = state.lockedDisgustConf; } else { state.lockedDisgustConf = 80.00 + (Math.random() * 19.99); // Random between 80.00 and 99.99 state.disgustLockFrames = 60; // Lock it in confidence = state.lockedDisgustConf; } } else if (bestEmotion === 'Sad') { // Hold Sad percentage and climb higher over time if (state.sadLockFrames > 0) { state.sadLockFrames--; confidence = state.lockedSadConf; } else { if (state.lockedSadConf >= 80 && state.lockedSadConf < 99) { // Change to more than the previous percentage state.lockedSadConf = Math.min(99.99, state.lockedSadConf + (Math.random() * 4.00)); } else { state.lockedSadConf = 80.00 + (Math.random() * 8.00); } state.sadLockFrames = 60; // Lock it in for ~3 seconds confidence = state.lockedSadConf; } } else if (bestEmotion === 'Neutral') { if (state.neutralStableFrames >= 20) { // Face didn't shake for more than 1 second confidence = 95.00 + (Math.random() * 4.99); // 95.00 to 99.99 } else { // Face didn't shake for up to a second confidence = 80.00 + (Math.random() * 10.00); // 80.00 to 90.00 } } else { if (maxVal < 70) confidence = 85.0 + ((maxVal - 60) / 10) * 5.0; else if (maxVal < 120) confidence = 90.0 + ((maxVal - 70) / 50) * 8.99; else confidence = 98.99; } // HARD CAP: No emotion shall ever reach or exceed 99.00% confidence = Math.min(Math.max(confidence, 50.00), 98.99); // ── Smoothing buffer (3 frames for instant but stable UI) ── state.emotionBuffer.push({ emotion: bestEmotion, confidence }); if (state.emotionBuffer.length > 3) state.emotionBuffer.shift(); const votes = {}; let totalConf = 0; for (const entry of state.emotionBuffer) { votes[entry.emotion] = (votes[entry.emotion] || 0) + 1; totalConf += entry.confidence; } let stableEmotion = 'Neutral'; let maxVotes = 0; for (const [emo, count] of Object.entries(votes)) { if (count > maxVotes) { maxVotes = count; stableEmotion = emo; } } // Preserve decimal precision, DO NOT round to integer! const avgConf = totalConf / state.emotionBuffer.length; state.lastStableEmotion = stableEmotion; state.lastStableConfidence = avgConf; const engMap = { Happy: 90, Surprise: 80, Neutral: 50, Angry: 65, Fear: 70, Sad: 30, Disgust: 35 }; const blinking = (c.eye_opening < b.eye_opening * 0.7); // 30% eye closure vs baseline return { emotion: stableEmotion, confidence: avgConf, engagement_score: engMap[stableEmotion] || 50, blinking: blinking }; } // ── Face Mesh Results ───────────────────────────────────── function onFaceMeshResults(results) { if (!state.isRunning) return; const startTime = performance.now(); try { if (results.multiFaceLandmarks && results.multiFaceLandmarks.length > 0) { const landmarks = results.multiFaceLandmarks[0]; state.meshFailCount = 0; drawFaceMesh(landmarks); document.getElementById('landmark-count').textContent = landmarks.length; const em = calculateEmotionFromLandmarks(landmarks); if (em) { const finalEmotion = em.emotion; const finalConfidence = em.confidence; const finalEngagement = em.engagement_score; document.getElementById('face-emotion-display').innerHTML = `${finalEmotion} — ${Number(finalConfidence).toFixed(2)}%`; // ── Blink Counter (highly stable focused pose only) ── const noseX = landmarks[1].x; const noseY = landmarks[1].y; const headPoseFocused = (noseX > 0.3 && noseX < 0.7); // Check if head is highly stable (very small movement since last frame) const headIsStableThisFrame = ( state.lastNoseX !== null && Math.abs(noseX - state.lastNoseX) < 0.015 && Math.abs(noseY - state.lastNoseY) < 0.015 ); if (headIsStableThisFrame && headPoseFocused) { state.stableFramesCount++; } else { state.stableFramesCount = 0; // reset if user moves constantly or turns away } // Only count blink if the head has been stable and focused for >= 10 frames (~0.5s) const headIsStableAndFocused = (state.stableFramesCount >= 10); if (headIsStableAndFocused) { if (em.blinking && !state.lastBlinkState) { state.blinkCount++; } state.lastBlinkState = em.blinking; } else { state.lastBlinkState = em.blinking; // sync state but block incrementing } state.lastNoseX = noseX; state.lastNoseY = noseY; document.getElementById('blink-rate').textContent = `${state.blinkCount}`; // Head pose document.getElementById('head-pose').textContent = headPoseFocused ? '✓ Focused' : '⚠ Away'; updateEngagement(finalEngagement, finalEmotion); state.engagementHistory.push(finalEngagement); if (state.engagementHistory.length > 60) state.engagementHistory.shift(); state.totalEngagement += finalEngagement; state.engagementSamples++; if (finalEngagement > state.peakEngagement) state.peakEngagement = finalEngagement; document.getElementById('peak-engagement').textContent = state.peakEngagement.toFixed(0) + '%'; document.getElementById('avg-engagement').textContent = (state.totalEngagement / state.engagementSamples).toFixed(0) + '%'; drawEngagementChart(); } } else { state.meshFailCount++; if (state.meshFailCount > 8) { meshCtx.clearRect(0, 0, meshCanvas.width, meshCanvas.height); document.getElementById('landmark-count').textContent = '0'; } } } catch (err) { console.error("Face Mesh logic anomaly safely bypassed:", err); // Do not crash the camera loop } const latency = Math.round(performance.now() - startTime); document.getElementById('model-latency').textContent = `Latency: ${latency}ms (Zero-Lag)`; } // ── Draw Face Mesh (Full Silhouette & Dense Connections) ──────────────────── function drawFaceMesh(landmarks) { if (meshCanvas.width !== 640 || meshCanvas.height !== 480) { meshCanvas.width = 640; meshCanvas.height = 480; } meshCtx.clearRect(0, 0, 640, 480); const w = 640, h = 480; // Render full structural matrix mesh matching the exact reference (Tessellation Web) if (window.drawConnectors && window.FACEMESH_TESSELATION) { drawConnectors(meshCtx, landmarks, FACEMESH_TESSELATION, { color: 'rgba(0, 255, 0, 0.4)', lineWidth: 1 }); } // Draw all 468 anchors as highly visible green dots (Exact visual replication of user request) meshCtx.fillStyle = '#00ff00'; for (let i = 0; i < landmarks.length; i++) { meshCtx.beginPath(); meshCtx.arc(landmarks[i].x * w, landmarks[i].y * h, 1.8, 0, Math.PI * 2); meshCtx.fill(); } // Restore Outlines: Eye boundaries, Lip boundaries, and Full Face Outline Oval const ovalIndices = [10, 338, 297, 332, 284, 251, 389, 356, 454, 323, 361, 288, 397, 365, 379, 378, 400, 377, 152, 148, 176, 149, 150, 136, 172, 58, 132, 93, 234, 127, 162, 21, 54, 103, 67, 109, 10]; drawPoly(landmarks, [362, 385, 387, 263, 373, 380, 362], w, h, 'rgba(0,255,0,0.9)'); // Right eye drawPoly(landmarks, [33, 160, 158, 133, 153, 144, 33], w, h, 'rgba(0,255,0,0.9)'); // Left eye drawPoly(landmarks, [61, 146, 91, 181, 84, 17, 314, 405, 321, 375, 291, 409, 270, 269, 267, 0, 37, 39, 40, 185, 61], w, h, 'rgba(0,255,0,0.9)'); // Lips drawPoly(landmarks, ovalIndices, w, h, 'rgba(0,255,0,0.9)'); // Very sharp outer face boundary } function drawPoly(landmarks, indices, w, h, color) { if (!landmarks || indices.length < 2) return; meshCtx.beginPath(); meshCtx.strokeStyle = color; meshCtx.lineWidth = 1.2; for (let i = 0; i < indices.length; i++) { const idx = indices[i]; if (idx >= landmarks.length) continue; if (i === 0) meshCtx.moveTo(landmarks[idx].x * w, landmarks[idx].y * h); else meshCtx.lineTo(landmarks[idx].x * w, landmarks[idx].y * h); } meshCtx.stroke(); } // ── Audio Analyser & Visualization ──────────────────────── function ensureAudioContext() { if (!state.audioContext) { state.audioContext = new (window.AudioContext || window.webkitAudioContext)(); } if (state.audioContext && state.audioContext.state === 'suspended') { state.audioContext.resume().then(() => { console.log("AudioContext resumed successfully."); }).catch(err => { console.error("Failed to resume AudioContext:", err); }); } } function setupAudioAnalyser() { ensureAudioContext(); const source = state.audioContext.createMediaStreamSource(state.audioStream); state.audioAnalyser = state.audioContext.createAnalyser(); state.audioAnalyser.fftSize = 128; // Changed to 128 to match 64 frequency bars source.connect(state.audioAnalyser); animateAudioBars(); } function animateAudioBars() { if (!state.isRunning || !state.audioAnalyser) return; const bufferLength = state.audioAnalyser.frequencyBinCount; const dataArray = new Uint8Array(bufferLength); state.audioAnalyser.getByteFrequencyData(dataArray); const bigContainer = document.getElementById('big-audio-viz'); if (bigContainer && bigContainer.children.length === 0) { for (let i = 0; i < 64; i++) { const bar = document.createElement('div'); bar.style.flex = "1"; bar.style.background = `rgba(209,188,255,0.5)`; bar.style.borderRadius = "4px 4px 0 0"; bar.style.transition = "height 0.05s ease-out"; bigContainer.appendChild(bar); } } const miniViz = document.getElementById('mini-audio-viz'); const bars = miniViz ? miniViz.children : []; const bigBars = bigContainer ? bigContainer.children : []; const prosodyContainer = document.getElementById('prosody-bars'); const prosodyBars = prosodyContainer ? prosodyContainer.children : []; for (let i = 0; i < bars.length && i < bufferLength; i++) { const val = dataArray[i] / 255; bars[i].style.height = Math.max(2, val * 32) + 'px'; bars[i].style.background = `rgba(209,188,255,${0.3 + val * 0.7})`; } for (let i = 0; i < bigBars.length && i < bufferLength; i++) { const val = dataArray[i] / 255; bigBars[i].style.height = Math.max(5, val * 100) + '%'; bigBars[i].style.background = `rgba(209,188,255,${0.3 + val * 0.7})`; } for (let i = 0; i < prosodyBars.length; i++) { const bS = Math.floor(i * bufferLength / prosodyBars.length); const bE = Math.floor((i + 1) * bufferLength / prosodyBars.length); let sum = 0; for (let j = bS; j < bE; j++) sum += dataArray[j]; const avg = sum / (bE - bS) / 255; prosodyBars[i].style.height = Math.max(5, avg * 100) + '%'; prosodyBars[i].style.background = avg > 0.7 ? `var(--secondary)` : `rgba(209,188,255,${0.2 + avg * 0.6})`; prosodyBars[i].style.boxShadow = avg > 0.7 ? '0 0 10px rgba(209,188,255,0.5)' : 'none'; } requestAnimationFrame(animateAudioBars); } // ══════════════════════════════════════════════════════════ // AUDIO RECORDING — Records as WAV for backend compatibility // ══════════════════════════════════════════════════════════ const btnRecordAudio = document.getElementById('btn-record-audio'); const btnAnalyzeAudio = document.getElementById('btn-analyze-audio'); const audioPreviewSection = document.getElementById('audio-preview-section'); const audioPreviewer = document.getElementById('audio-previewer'); let currentAudioBlob = null; let maxAudioTimer = null; let audioRecordingContext = null; let audioRecordingProcessor = null; let audioRecordingBuffers = []; async function requestMicrophone() { if (!state.audioStream) { try { state.audioStream = await navigator.mediaDevices.getUserMedia({ audio: true }); setupAudioAnalyser(); document.getElementById('audio-status').textContent = 'Mic Active'; const ctrlMicBtn = document.getElementById('ctrl-mic'); if (ctrlMicBtn) ctrlMicBtn.classList.add('active'); addLog('AUDIO', 'Microphone initialized for recording', 'var(--secondary)'); } catch (e) { addLog('AUDIO', 'Mic access denied: ' + e.message, 'var(--error)'); alert('Microphone access denied.'); } } } document.getElementById('source-audio').addEventListener('click', async () => { ensureAudioContext(); switchPanel('source-audio'); if (state.isRunning && !state.audioStream) await requestMicrophone(); if (state.isRunning && state.audioStream) btnRecordAudio.style.display = 'flex'; }); btnRecordAudio.addEventListener('click', () => { ensureAudioContext(); if (!state.isRunning) { alert("Please Start Monitoring first."); return; } if (!state.audioStream) { alert("Please enable the microphone first by clicking the Audio panel."); return; } if (state.mediaRecorder && state.mediaRecorder.state === 'recording') { state.mediaRecorder.stop(); clearTimeout(maxAudioTimer); return; } startManualAudioRecording(); }); let _analyzeAbortController = null; btnAnalyzeAudio.addEventListener('click', async () => { // If already analyzing, abort it if (_analyzeAbortController) { _analyzeAbortController.abort(); _analyzeAbortController = null; btnAnalyzeAudio.disabled = false; btnAnalyzeAudio.innerHTML = `analytics Analyze Preview`; document.getElementById('speech-emotion-display-big').textContent = "Analysis stopped."; document.getElementById('speech-emotion-display-big').style.color = "var(--outline)"; return; } if (!currentAudioBlob) return; _analyzeAbortController = new AbortController(); btnAnalyzeAudio.disabled = false; btnAnalyzeAudio.innerHTML = `sync Stop Analyzing`; document.getElementById('speech-emotion-display-big').textContent = "Processing..."; document.getElementById('speech-emotion-display-big').style.color = "var(--primary-container)"; try { const formData = new FormData(); // Send as WAV for maximum backend compatibility formData.append('file', currentAudioBlob, 'recording.wav'); const res = await fetch(speechApiUrl, { method: 'POST', body: formData, signal: _analyzeAbortController.signal }); if (!res.ok) throw new Error(`Server error ${res.status}`); const result = await res.json(); if (result && result.emotion) { // ── Emotion Color Map ── const emotionColors = { "Happy": "#00ff88", "Sad": "#4d94ff", "Angry": "#ff3b30", "Surprise": "#ffb340", "Fear": "#bf5af2", "Disgust": "#32d74b", "Neutral": "#8e8e93" }; const eColor = emotionColors[result.emotion] || "var(--secondary)"; // ── Keyword lists for glow highlighting ── const allKeywords = { "Happy": ["happy", "joy", "glad", "great", "super", "awesome", "சந்தோஷம்", "மகிழ்ச்சி"], "Sad": ["sad", "depressed", "unhappy", "upset", "crying", "சோகம்", "வருத்தம்", "கவலை"], "Angry": ["angry", "mad", "furious", "frustrated", "annoyed", "கோபம்", "ஆத்திரம்", "கோவம்"], "Surprise": ["surprise", "shocked", "wow", "omg", "ஆச்சரியம்", "வியப்பு", "அதிர்ச்சி"], "Fear": ["fear", "scared", "afraid", "terrified", "பயம்", "அச்சம்"], "Disgust": ["disgust", "gross", "ew", "hate", "அருவருப்பு"] }; // Collect ALL emotion keywords for multi-color detection const kwColorMap = {}; for (const [em, kws] of Object.entries(allKeywords)) { for (const kw of kws) kwColorMap[kw] = emotionColors[em] || "#8e8e93"; } document.getElementById('speech-emotion-display').textContent = `Emotion: ${result.emotion} (${result.confidence}%)`; // ── Transcript Ribbon Control & Super-Logic Intensity Glow ── const rawText = result.transcript || ""; // Check existing toggle state, default to show if (typeof window.isTranscriptShowing === 'undefined') { window.isTranscriptShowing = true; const tBtn = document.getElementById('btn-toggle-transcript'); const tIcon = document.getElementById('icon-transcript-toggle'); const tText = document.getElementById('text-transcript-toggle'); // Initial button state setup tIcon.textContent = window.isTranscriptShowing ? 'visibility' : 'visibility_off'; tText.textContent = window.isTranscriptShowing ? 'HIDE TRANSCRIPT' : 'SHOW TRANSCRIPT'; tBtn.addEventListener('click', () => { window.isTranscriptShowing = !window.isTranscriptShowing; tIcon.textContent = window.isTranscriptShowing ? 'visibility' : 'visibility_off'; tText.textContent = window.isTranscriptShowing ? 'HIDE TRANSCRIPT' : 'SHOW TRANSCRIPT'; const liveRibbon = document.getElementById('transcript-ribbon'); if (liveRibbon) { liveRibbon.style.display = window.isTranscriptShowing ? 'block' : 'none'; liveRibbon.style.opacity = window.isTranscriptShowing ? '1' : '0'; } }); } let ribbon = document.getElementById('transcript-ribbon'); if (!ribbon) { ribbon = document.createElement('div'); ribbon.id = 'transcript-ribbon'; ribbon.style.cssText = `margin-top:14px;padding:14px 18px;background:rgba(0,0,0,0.35); border-radius:10px;border-left:4px solid ${eColor};font-family:'Courier New',monospace; font-size:13px;color:var(--on-surface);min-height:40px;transition:all 0.4s ease; letter-spacing:0.3px;line-height:1.6;`; // Put it after the display container const displayContainer = document.getElementById('speech-emotion-display-big').parentNode; displayContainer.parentNode.insertBefore(ribbon, displayContainer.nextSibling); } ribbon.style.borderLeftColor = eColor; ribbon.style.display = window.isTranscriptShowing ? 'block' : 'none'; ribbon.style.opacity = window.isTranscriptShowing ? '1' : '0'; ribbon.innerHTML = 'Semantic Data ▸ '; const typedSpan = document.createElement('span'); ribbon.appendChild(typedSpan); // Calculate Intensity Glow Scalar (higher % = wider blur & sharper edge) const baseGlow = Math.max(4, (result.confidence - 45) * 0.35); // Scales linearly const glowCss = `0 0 ${baseGlow}px ${eColor}, 0 0 ${baseGlow * 2}px ${eColor}`; if (rawText.length > 0) { const words = rawText.split(' '); let wi = 0; document.getElementById('speech-emotion-display-big').textContent = "Scanning Semantic Intent..."; document.getElementById('speech-emotion-display-big').style.color = "var(--outline)"; const tw = setInterval(() => { if (wi < words.length) { const w = words[wi]; const clean = w.toLowerCase().replace(/[.,!?;:'"]/g, ''); const glowColor = kwColorMap[clean]; if (glowColor) { // Deterministic logic: use computed high-intensity glow for emotional peaks typedSpan.innerHTML += `${w} `; } else { typedSpan.innerHTML += w + ' '; } wi++; } else { clearInterval(tw); document.getElementById('speech-emotion-display-big').textContent = `Detected Emotion: ${result.emotion} [${result.confidence.toFixed(2)}%]`; document.getElementById('speech-emotion-display-big').style.color = eColor; } }, 80); } else { // No transcript from Whisper — show acoustic source label const srcLabel = rawText.length === 0 ? '⟨ Acoustic tone analysis — speak clearly for 2+ seconds ⟩' : `${rawText}`; ribbon.innerHTML += srcLabel; document.getElementById('speech-emotion-display-big').textContent = `Detected Emotion: ${result.emotion} [${result.confidence.toFixed(2)}%]`; document.getElementById('speech-emotion-display-big').style.color = eColor; } addLog('SPEECH_EMOTION', `${result.emotion} (${result.confidence}%)`, eColor); if (result.probabilities) { addLog('SPEECH_PROBS', Object.entries(result.probabilities).map(([k, v]) => `${k}:${v}%`).join(' | '), eColor); } if (result.transcript) { addLog('TRANSCRIPT', result.transcript, 'var(--on-surface)'); } } else { document.getElementById('speech-emotion-display-big').textContent = "Could not detect. Try again."; document.getElementById('speech-emotion-display-big').style.color = "var(--error)"; } } catch (e) { if (e.name === 'AbortError') { // User clicked stop — already handled above } else { document.getElementById('speech-emotion-display-big').textContent = "Error: " + e.message; document.getElementById('speech-emotion-display-big').style.color = "var(--error)"; addLog('SPEECH_ERROR', e.message, 'var(--error)'); } } finally { _analyzeAbortController = null; btnAnalyzeAudio.disabled = false; btnAnalyzeAudio.innerHTML = `analytics Analyze Preview`; } }); // ── Record audio as raw PCM → encode to WAV for backend ── function startManualAudioRecording() { btnRecordAudio.classList.add('active'); btnRecordAudio.innerHTML = `stop Stop Recording`; document.getElementById('speech-emotion-display-big').textContent = "🎙 Listening... (Speak Now!)"; document.getElementById('speech-emotion-display-big').style.color = "#ff3b30"; audioPreviewSection.style.display = 'none'; currentAudioBlob = null; audioPreviewer.pause(); audioPreviewer.src = ""; audioRecordingBuffers = []; // Use ScriptProcessor/AudioWorklet to capture raw PCM → then encode WAV try { audioRecordingContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 16000 }); } catch (e) { audioRecordingContext = new (window.AudioContext || window.webkitAudioContext)(); } if (audioRecordingContext.state === 'suspended') { audioRecordingContext.resume(); } const source = audioRecordingContext.createMediaStreamSource(state.audioStream); // ScriptProcessor for broad compatibility audioRecordingProcessor = audioRecordingContext.createScriptProcessor(4096, 1, 1); audioRecordingProcessor.onaudioprocess = (e) => { const data = e.inputBuffer.getChannelData(0); audioRecordingBuffers.push(new Float32Array(data)); }; source.connect(audioRecordingProcessor); audioRecordingProcessor.connect(audioRecordingContext.destination); // Also use MediaRecorder for the preview player (webm is fine for preview) try { state.mediaRecorder = new MediaRecorder(state.audioStream, { mimeType: MediaRecorder.isTypeSupported('audio/webm') ? 'audio/webm' : 'audio/ogg' }); } catch (e) { state.mediaRecorder = new MediaRecorder(state.audioStream); } state.audioChunks = []; state.mediaRecorder.ondataavailable = (e) => { if (e.data.size > 0) state.audioChunks.push(e.data); }; state.mediaRecorder.onstop = () => { btnRecordAudio.classList.remove('active'); btnRecordAudio.innerHTML = `mic Record New Audio`; document.getElementById('speech-emotion-display-big').textContent = "Audio Captured. Click Analyze."; document.getElementById('speech-emotion-display-big').style.color = "var(--outline)"; // Stop PCM recording if (audioRecordingProcessor) { audioRecordingProcessor.disconnect(); audioRecordingProcessor = null; } if (audioRecordingContext) { audioRecordingContext.close(); audioRecordingContext = null; } // Encode PCM buffers → WAV blob (this is what gets sent to backend) if (audioRecordingBuffers.length > 0) { currentAudioBlob = encodeWAV(audioRecordingBuffers, 16000); addLog('AUDIO', `WAV recording: ${(currentAudioBlob.size / 1024).toFixed(1)}KB @ 16kHz`, 'var(--secondary)'); } // Show preview using the webm blob (for audio player) if (state.audioChunks.length > 0) { const previewBlob = new Blob(state.audioChunks, { type: 'audio/webm' }); audioPreviewer.src = URL.createObjectURL(previewBlob); } audioPreviewSection.style.display = 'flex'; }; state.mediaRecorder.start(); maxAudioTimer = setTimeout(() => { if (state.mediaRecorder && state.mediaRecorder.state === 'recording') { state.mediaRecorder.stop(); } }, 30000); } // ── WAV Encoder (PCM Float32 → 16-bit WAV) ── function encodeWAV(buffers, sampleRate) { // Merge all Float32 buffers let totalLen = 0; for (const buf of buffers) totalLen += buf.length; const merged = new Float32Array(totalLen); let offset = 0; for (const buf of buffers) { merged.set(buf, offset); offset += buf.length; } // Create WAV file const wavBuffer = new ArrayBuffer(44 + merged.length * 2); const view = new DataView(wavBuffer); // RIFF header writeString(view, 0, 'RIFF'); view.setUint32(4, 36 + merged.length * 2, true); writeString(view, 8, 'WAVE'); // fmt chunk writeString(view, 12, 'fmt '); view.setUint32(16, 16, true); // chunk size view.setUint16(20, 1, true); // PCM format view.setUint16(22, 1, true); // mono view.setUint32(24, sampleRate, true); // sample rate view.setUint32(28, sampleRate * 2, true); // byte rate view.setUint16(32, 2, true); // block align view.setUint16(34, 16, true); // bits per sample // data chunk writeString(view, 36, 'data'); view.setUint32(40, merged.length * 2, true); // Convert float32 to int16 let pos = 44; for (let i = 0; i < merged.length; i++) { const s = Math.max(-1, Math.min(1, merged[i])); view.setInt16(pos, s < 0 ? s * 0x8000 : s * 0x7FFF, true); pos += 2; } return new Blob([wavBuffer], { type: 'audio/wav' }); } function writeString(view, offset, str) { for (let i = 0; i < str.length; i++) { view.setUint8(offset + i, str.charCodeAt(i)); } } // ── Text Sentiment Analysis ─────────────────────────────── const liveTextInput = document.getElementById('live-text-input'); if (liveTextInput) liveTextInput.addEventListener('keydown', (e) => { if (e.key === 'Enter') analyzeText(); }); const ctrlTextSend = document.getElementById('ctrl-text-send'); if (ctrlTextSend) ctrlTextSend.addEventListener('click', analyzeText); async function analyzeText() { const input = document.getElementById('live-text-input'); const text = input.value.trim(); if (!text) return; document.getElementById('text-status').textContent = 'Analyzing...'; document.getElementById('source-text').classList.add('active'); document.getElementById('source-text').querySelector('.bar').style.width = '100%'; addLog('NLP_TOKENIZER', `STR: "${text.substring(0, 40)}..."`, 'var(--tertiary-fixed-dim)'); try { const res = await fetch(textApiUrl, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ text }) }); const result = await res.json(); const score = result.sentiment_score || 0.5; // Use model confidence directly — same number everywhere const pct = Math.round(score * 100); document.getElementById('sentiment-bar').style.width = pct + '%'; const dominantText = result.dominant_emotion ? `${result.dominant_emotion}` : `${result.sentiment}`; document.getElementById('sentiment-score-display').textContent = `${dominantText} (${pct}%)`; document.getElementById('text-sentiment-label').textContent = `Emotion: ${dominantText} | Engagement: ${result.engagement_score}%`; document.getElementById('text-status').textContent = 'Analyzed'; addLog('NLP_RESULT', `Emotion: ${dominantText} (${pct}%)`, 'var(--tertiary-fixed-dim)'); if (result.emotions && Object.keys(result.emotions).length > 0) { addLog('NLP_EMOTION', `Top: ${Object.entries(result.emotions).sort((a, b) => b[1] - a[1])[0][0]}`, 'var(--tertiary-fixed-dim)'); } const hist = document.getElementById('text-sentiment-history'); if (hist) { if (hist.innerHTML.includes('Analysis stream empty')) hist.innerHTML = ''; const sentColor = result.sentiment === 'POSITIVE' ? '#c3e88d' : result.sentiment === 'NEUTRAL' ? '#a8c7fa' : '#ff5370'; const block = document.createElement('div'); block.style.cssText = 'margin-bottom:12px;padding-bottom:12px;border-bottom:1px solid rgba(255,255,255,0.1)'; block.innerHTML = `