| |
| |
| |
| |
| |
| import { useCallback, useEffect, useRef, useState } from "react"; |
| import { api } from "../api/client"; |
|
|
| const POSE_LABELS = { |
| GOOD: { color: "#48bb78", text: "✅ Good Posture" }, |
| SLOUCHING: { color: "#ed8936", text: "⚠️ Sit Up Straight" }, |
| HEAD_FORWARD: { color: "#ed8936", text: "⚠️ Head Forward" }, |
| LOOKING_AWAY: { color: "#fc8181", text: "👀 Looking Away" }, |
| FACE_NOT_VISIBLE: { color: "#fc8181", text: "👁 Face Not Visible" }, |
| NO_POSE: { color: "#a0a3b1", text: "📷 Loading Camera…" }, |
| }; |
|
|
| export default function PostureMonitor({ sessionId, stream }) { |
| const videoRef = useRef(null); |
| const canvasRef = useRef(null); |
| const poseRef = useRef(null); |
| const timerRef = useRef(null); |
| const sendRef = useRef(null); |
| const metricsRef = useRef([]); |
| const lastTsRef = useRef(0); |
| const baselineRef = useRef({ count: 0, noseShoulderOffset: 0 }); |
|
|
| const [status, setStatus] = useState("NO_POSE"); |
| const [poseReady, setPoseReady] = useState(false); |
| const [poseError, setPoseError] = useState(null); |
|
|
| |
| useEffect(() => { |
| if (!stream || !videoRef.current) return; |
| videoRef.current.srcObject = stream; |
| videoRef.current.play().catch(() => {}); |
| }, [stream]); |
|
|
| const analysePosture = useCallback((landmarks) => { |
| |
| |
| |
| |
| if (!landmarks || landmarks.length < 25) { |
| return { label: "FACE_NOT_VISIBLE", score: 0.1, torsoAngle: 0 }; |
| } |
|
|
| const NOSE = 0; |
| const LEFT_SHOULDER = 11; |
| const RIGHT_SHOULDER = 12; |
| const LEFT_HIP = 23; |
| const RIGHT_HIP = 24; |
|
|
| const nose = landmarks[NOSE]; |
| const lS = landmarks[LEFT_SHOULDER]; |
| const rS = landmarks[RIGHT_SHOULDER]; |
| const lH = landmarks[LEFT_HIP]; |
| const rH = landmarks[RIGHT_HIP]; |
|
|
| if (!nose || !lS || !rS || !lH || !rH) { |
| return { label: "FACE_NOT_VISIBLE", score: 0.1, torsoAngle: 0 }; |
| } |
| if (nose.visibility < 0.5) { |
| return { label: "LOOKING_AWAY", score: 0.1, torsoAngle: 0 }; |
| } |
|
|
| const shoulderMidX = (lS.x + rS.x) / 2; |
| const shoulderMidY = (lS.y + rS.y) / 2; |
| const hipMidY = (lH.y + rH.y) / 2; |
| const shoulderDiffY = Math.abs(lS.y - rS.y); |
| const noseForwardOffset = Math.abs(nose.x - shoulderMidX); |
| const torsoHeight = hipMidY - shoulderMidY; |
| const torsoAngle = Math.round(Math.abs(shoulderMidY - hipMidY) * 100); |
|
|
| |
| if (baselineRef.current.count < 3) { |
| baselineRef.current.noseShoulderOffset += noseForwardOffset; |
| baselineRef.current.count += 1; |
| return { label: "NO_POSE", score: 0.5, torsoAngle }; |
| } |
| const baselineOffset = baselineRef.current.noseShoulderOffset / baselineRef.current.count; |
|
|
| |
| const isHeadForward = (noseForwardOffset - baselineOffset) > 0.12; |
| const isSlouching = torsoHeight < 0.18 || shoulderDiffY > 0.12; |
| const isHeadUp = nose.y < shoulderMidY; |
| const shouldersLevel = shoulderDiffY < 0.10; |
| const headAligned = (noseForwardOffset - baselineOffset) <= 0.08; |
|
|
| if (isHeadForward) { |
| return { label: "HEAD_FORWARD", score: 0.4, torsoAngle }; |
| } |
| if (isSlouching) { |
| return { label: "SLOUCHING", score: 0.3, torsoAngle }; |
| } |
| if (headAligned && shouldersLevel && isHeadUp) { |
| return { label: "GOOD", score: 0.9, torsoAngle }; |
| } |
| return { label: "SLOUCHING", score: 0.3, torsoAngle }; |
| }, []); |
|
|
| const drawSkeleton = useCallback((canvas, landmarks) => { |
| const ctx = canvas.getContext("2d"); |
| ctx.clearRect(0, 0, canvas.width, canvas.height); |
| if (!landmarks) return; |
|
|
| const W = canvas.width, H = canvas.height; |
| const CONNECTIONS = [ |
| [11,12],[11,23],[12,24],[23,24], |
| [11,13],[13,15],[12,14],[14,16], |
| [0,11],[0,12], |
| ]; |
|
|
| ctx.strokeStyle = "rgba(102,126,234,0.85)"; |
| ctx.lineWidth = 2.5; |
| for (const [a, b] of CONNECTIONS) { |
| const pa = landmarks[a], pb = landmarks[b]; |
| if (!pa || !pb || pa.visibility < 0.45 || pb.visibility < 0.45) continue; |
| ctx.beginPath(); |
| ctx.moveTo(pa.x * W, pa.y * H); |
| ctx.lineTo(pb.x * W, pb.y * H); |
| ctx.stroke(); |
| } |
| for (const i of [0, 11, 12, 23, 24]) { |
| const p = landmarks[i]; |
| if (!p || p.visibility < 0.45) continue; |
| ctx.beginPath(); |
| ctx.arc(p.x * W, p.y * H, 5, 0, Math.PI * 2); |
| ctx.fillStyle = "#667eea"; |
| ctx.fill(); |
| } |
| }, []); |
|
|
| |
| useEffect(() => { |
| if (!stream) return; |
| let cancelled = false; |
|
|
| (async () => { |
| for (const delegate of ["GPU", "CPU"]) { |
| if (cancelled) return; |
| try { |
| const { PoseLandmarker, FilesetResolver } = |
| await import("@mediapipe/tasks-vision"); |
|
|
| const vision = await FilesetResolver.forVisionTasks( |
| "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.14/wasm" |
| ); |
|
|
| const landmarker = await PoseLandmarker.createFromOptions(vision, { |
| baseOptions: { |
| modelAssetPath: |
| "https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/1/pose_landmarker_lite.task", |
| delegate, |
| }, |
| runningMode: "VIDEO", |
| numPoses: 1, |
| }); |
|
|
| poseRef.current = landmarker; |
| setPoseReady(true); |
| setPoseError(null); |
| console.log(`✅ MediaPipe pose ready (delegate=${delegate})`); |
| return; |
| } catch (e) { |
| console.warn(`MediaPipe init failed (delegate=${delegate}):`, e); |
| if (delegate === "CPU") { |
| setPoseError("Pose detection unavailable — camera shown without skeleton."); |
| } |
| } |
| } |
| })(); |
|
|
| return () => { |
| cancelled = true; |
| |
| if (poseRef.current && poseRef.current.close) { |
| poseRef.current.close(); |
| poseRef.current = null; |
| } |
| }; |
| }, [stream]); |
|
|
| |
| useEffect(() => { |
| if (!poseReady || !poseRef.current) return; |
| baselineRef.current = { count: 0, noseShoulderOffset: 0 }; |
|
|
| timerRef.current = setInterval(() => { |
| const video = videoRef.current; |
| const canvas = canvasRef.current; |
| if (!video || !canvas || video.readyState < 2 || video.paused) return; |
|
|
| canvas.width = video.videoWidth || 320; |
| canvas.height = video.videoHeight || 240; |
|
|
| const now = performance.now(); |
| if (now <= lastTsRef.current) return; |
| lastTsRef.current = now; |
|
|
| try { |
| const result = poseRef.current.detectForVideo(video, now); |
| const lm = result.landmarks?.[0]; |
| drawSkeleton(canvas, lm); |
| const posture = analysePosture(lm); |
| setStatus(posture.label); |
| metricsRef.current.push({ |
| posture_score: posture.score, |
| posture_label: posture.label, |
| spine_height: posture.torsoAngle, |
| hands_visible: true, |
| timestamp: Date.now(), |
| }); |
| } catch (_) {} |
| }, 500); |
|
|
| |
| const metricsBuffer = []; |
| let consecutiveFailures = 0; |
| const MAX_BUFFER_SIZE = 100; |
|
|
| sendRef.current = setInterval(async () => { |
| if (!sessionId || metricsRef.current.length === 0) return; |
|
|
| |
| metricsBuffer.push(...metricsRef.current); |
| if (metricsBuffer.length > MAX_BUFFER_SIZE) { |
| |
| metricsBuffer.splice(0, metricsBuffer.length - MAX_BUFFER_SIZE); |
| } |
| metricsRef.current = []; |
|
|
| |
| const latest = metricsBuffer[metricsBuffer.length - 1]; |
| if (!latest) return; |
|
|
| try { |
| await api.sendPosture({ session_id: sessionId, metrics: latest }); |
| |
| metricsBuffer.length = 0; |
| consecutiveFailures = 0; |
| } catch (err) { |
| consecutiveFailures++; |
| console.warn(`Posture metrics send failed (${consecutiveFailures}x):`, err); |
|
|
| |
| if (consecutiveFailures >= 3 && metricsBuffer.length > 1) { |
| try { |
| |
| const avgScore = metricsBuffer.reduce((sum, m) => sum + (m.posture_score || 0), 0) / metricsBuffer.length; |
| const modeLabel = metricsBuffer |
| .map(m => m.posture_label) |
| .sort((a, b) => |
| metricsBuffer.filter(m => m.posture_label === a).length - |
| metricsBuffer.filter(m => m.posture_label === b).length |
| ).pop(); |
|
|
| const batchMetric = { |
| posture_score: avgScore, |
| posture_label: modeLabel, |
| spine_height: metricsBuffer[metricsBuffer.length - 1].spine_height, |
| hands_visible: true, |
| timestamp: Date.now(), |
| batch_size: metricsBuffer.length, |
| batch_aggregated: true |
| }; |
|
|
| await api.sendPosture({ session_id: sessionId, metrics: batchMetric }); |
| metricsBuffer.length = 0; |
| consecutiveFailures = 0; |
| } catch (batchErr) { |
| console.warn("Batch posture send also failed:", batchErr); |
| } |
| } |
| } |
| }, 30000); |
|
|
| return () => { |
| clearInterval(timerRef.current); |
| clearInterval(sendRef.current); |
| |
| |
| if (sessionId && metricsBuffer.length > 0) { |
| const latest = metricsBuffer[metricsBuffer.length - 1]; |
| const payload = JSON.stringify({ |
| session_id: sessionId, |
| metrics: { |
| ...latest, |
| flush_on_unmount: true, |
| timestamp: Date.now() |
| } |
| }); |
| |
| |
| if (navigator.sendBeacon) { |
| navigator.sendBeacon('/api/posture/report', new Blob([payload], { type: 'application/json' })); |
| } |
| |
| metricsBuffer.length = 0; |
| } |
| }; |
| }, [poseReady, sessionId, analysePosture, drawSkeleton]); |
|
|
| const badge = POSE_LABELS[status] || POSE_LABELS.GOOD; |
|
|
| return ( |
| <div style={S.wrap}> |
| <div style={S.videoWrap}> |
| <video ref={videoRef} muted playsInline style={S.video} /> |
| <canvas ref={canvasRef} style={S.canvas} /> |
| <div style={{ ...S.badge, background: badge.color + "22", border: `1px solid ${badge.color}55`, color: badge.color }}> |
| {badge.text} |
| </div> |
| {poseError && <div style={S.errorNote}>⚠️ {poseError}</div>} |
| </div> |
| </div> |
| ); |
| } |
|
|
| const S = { |
| wrap: { width: "100%" }, |
| videoWrap: { position: "relative", background: "#0f1117", borderRadius: "12px", overflow: "hidden", aspectRatio: "4/3" }, |
| video: { width: "100%", height: "100%", objectFit: "cover", transform: "scaleX(-1)" }, |
| canvas: { position: "absolute", inset: 0, width: "100%", height: "100%", transform: "scaleX(-1)", pointerEvents: "none" }, |
| badge: { position: "absolute", bottom: "8px", left: "8px", right: "8px", borderRadius: "6px", padding: "5px 10px", fontSize: "12px", fontWeight: 600, textAlign: "center" }, |
| errorNote: { position: "absolute", top: "8px", left: "8px", right: "8px", background: "rgba(0,0,0,0.75)", color: "#a0a3b1", fontSize: "11px", textAlign: "center", borderRadius: "4px", padding: "4px 8px" }, |
| }; |
|
|