// FaceScanner.jsx — MediaPipe FaceMesh + gradient mesh overlay + liveness challenges // Real client-side liveness: blink (EAR), head yaw/pitch, mouth open (MAR), smile // Draws animated gradient tessellation like immigration kiosk. import React, { useEffect, useRef, useState, useCallback } from "react"; import { FaceMesh, FACEMESH_TESSELATION, FACEMESH_RIGHT_EYE, FACEMESH_LEFT_EYE, FACEMESH_LIPS, FACEMESH_FACE_OVAL } from "@mediapipe/face_mesh"; import { motion, AnimatePresence } from "framer-motion"; import { Eye, ArrowLeft, ArrowRight, ArrowDown, Smile, Loader2, ShieldCheck, ShieldAlert, RotateCcw } from "lucide-react"; // MediaPipe landmark indexes const RIGHT_EYE = [33, 160, 158, 133, 153, 144]; const LEFT_EYE = [362, 385, 387, 263, 373, 380]; const UPPER_LIP_CENTER = 13; const LOWER_LIP_CENTER = 14; const MOUTH_LEFT = 61; const MOUTH_RIGHT = 291; const NOSE_TIP = 1; const LEFT_CHEEK = 234; const RIGHT_CHEEK = 454; const FOREHEAD = 10; const CHIN = 152; const CHALLENGE_META = { blink: { label: "Silakan Berkedip", subtitle: "Kedipkan mata Anda perlahan", Icon: Eye }, turn_left: { label: "Tengok ke Kiri", subtitle: "Putar kepala ke kiri", Icon: ArrowLeft }, turn_right: { label: "Tengok ke Kanan", subtitle: "Putar kepala ke kanan", Icon: ArrowRight }, nod: { label: "Anggukkan Kepala", subtitle: "Turunkan dagu ke bawah", Icon: ArrowDown }, smile: { label: "Silakan Senyum", subtitle: "Buka mulut atau senyum lebar", Icon: Smile }, }; function dist(a, b) { const dx = a.x - b.x; const dy = a.y - b.y; return Math.sqrt(dx * dx + dy * dy); } function eyeEAR(lm, idx) { const p1 = lm[idx[0]]; const p2 = lm[idx[1]]; const p3 = lm[idx[2]]; const p4 = lm[idx[3]]; const p5 = lm[idx[4]]; const p6 = lm[idx[5]]; return (dist(p2, p6) + dist(p3, p5)) / (2 * dist(p1, p4)); } function mouthOpenness(lm) { const v = dist(lm[UPPER_LIP_CENTER], lm[LOWER_LIP_CENTER]); const h = dist(lm[MOUTH_LEFT], lm[MOUTH_RIGHT]); return v / Math.max(0.0001, h); } function mouthWidthRatio(lm) { // measure width vs face width (cheeks) — smiling stretches this const mw = dist(lm[MOUTH_LEFT], lm[MOUTH_RIGHT]); const fw = dist(lm[LEFT_CHEEK], lm[RIGHT_CHEEK]); return mw / Math.max(0.0001, fw); } // yaw sign: nose x relative to face midline function estimateYaw(lm) { const nose = lm[NOSE_TIP]; const leftCheek = lm[LEFT_CHEEK]; const rightCheek = lm[RIGHT_CHEEK]; const midX = (leftCheek.x + rightCheek.x) / 2; const width = rightCheek.x - leftCheek.x; // normalized offset [-1..1] return (nose.x - midX) / Math.max(0.0001, width / 2); } function estimatePitch(lm) { const nose = lm[NOSE_TIP]; const forehead = lm[FOREHEAD]; const chin = lm[CHIN]; const midY = (forehead.y + chin.y) / 2; const height = chin.y - forehead.y; return (nose.y - midY) / Math.max(0.0001, height / 2); } /** * Draw a gradient tessellation mesh on canvas using MediaPipe connections. * Each frame we shift gradient phase so it feels alive. */ function drawGradientMesh(ctx, lm, width, height, phase) { const grad = ctx.createLinearGradient(0, 0, width, height); const p = (phase % 1); // rotate stops for animated color travel grad.addColorStop((0 + p) % 1, "#8A2BE2"); grad.addColorStop((0.33 + p) % 1, "#0057FF"); grad.addColorStop((0.66 + p) % 1, "#00F0FF"); grad.addColorStop((1 + p) % 1, "#39FF14"); ctx.strokeStyle = grad; ctx.lineWidth = 0.6; ctx.globalAlpha = 0.55; ctx.beginPath(); for (const [i, j] of FACEMESH_TESSELATION) { const a = lm[i]; const b = lm[j]; if (!a || !b) continue; ctx.moveTo(a.x * width, a.y * height); ctx.lineTo(b.x * width, b.y * height); } ctx.stroke(); // Emphasize eyes / mouth / face oval with brighter accent ctx.globalAlpha = 0.95; ctx.lineWidth = 1.4; const accents = [FACEMESH_LEFT_EYE, FACEMESH_RIGHT_EYE, FACEMESH_LIPS, FACEMESH_FACE_OVAL]; const strokes = ["#00F0FF", "#00F0FF", "#39FF14", "#8A2BE2"]; accents.forEach((set, k) => { ctx.strokeStyle = strokes[k]; ctx.beginPath(); for (const [i, j] of set) { const a = lm[i]; const b = lm[j]; if (!a || !b) continue; ctx.moveTo(a.x * width, a.y * height); ctx.lineTo(b.x * width, b.y * height); } ctx.stroke(); }); // Dot on key points ctx.globalAlpha = 0.9; const pts = [NOSE_TIP, FOREHEAD, CHIN, LEFT_CHEEK, RIGHT_CHEEK]; ctx.fillStyle = "#00F0FF"; pts.forEach((idx) => { const p = lm[idx]; if (!p) return; ctx.beginPath(); ctx.arc(p.x * width, p.y * height, 2.4, 0, Math.PI * 2); ctx.fill(); }); ctx.globalAlpha = 1; } export default function FaceScanner({ challenges, onCapture, onAllChallengesDone, autoCapture = false, captureOnDone = false, width = 640, height = 480, showChallenges = true, }) { const videoRef = useRef(null); const canvasRef = useRef(null); const meshRef = useRef(null); const streamRef = useRef(null); const rafRef = useRef(null); const lastLmRef = useRef(null); const phaseRef = useRef(0); // challenge state const [currentIdx, setCurrentIdx] = useState(0); const [passed, setPassed] = useState([]); const [faceDetected, setFaceDetected] = useState(false); const [statusText, setStatusText] = useState("Menunggu wajah…"); const [ready, setReady] = useState(false); // per-challenge internal states const stateRef = useRef({ blinkEyeClosed: false, blinkCount: 0, baselineYaw: null, baselineMouth: null, baselineSmile: null, baselinePitch: null, lastMoveT: 0, }); const currentChallenge = challenges?.[currentIdx]; const resetInternal = useCallback(() => { stateRef.current = { blinkEyeClosed: false, blinkCount: 0, baselineYaw: null, baselineMouth: null, baselineSmile: null, baselinePitch: null, lastMoveT: 0, }; }, []); const onResults = useCallback( (results) => { const canvas = canvasRef.current; const video = videoRef.current; if (!canvas || !video) return; const ctx = canvas.getContext("2d"); const w = canvas.width; const h = canvas.height; ctx.clearRect(0, 0, w, h); phaseRef.current = (phaseRef.current + 0.008) % 1; const lm = results.multiFaceLandmarks?.[0]; if (!lm) { setFaceDetected(false); setStatusText("Posisikan wajah di tengah kamera…"); lastLmRef.current = null; return; } setFaceDetected(true); lastLmRef.current = lm; // draw mesh drawGradientMesh(ctx, lm, w, h, phaseRef.current); // process current challenge (only if provided) if (!challenges || challenges.length === 0) return; const chal = challenges[currentIdx]; if (!chal) return; const s = stateRef.current; const earR = eyeEAR(lm, RIGHT_EYE); const earL = eyeEAR(lm, LEFT_EYE); const ear = (earR + earL) / 2; const mopen = mouthOpenness(lm); const smileRatio = mouthWidthRatio(lm); const yaw = estimateYaw(lm); const pitch = estimatePitch(lm); let done = false; let hint = ""; if (chal === "blink") { // baseline eye open ~0.28, closed ~<0.19 const CLOSED = 0.19; const OPEN = 0.26; if (!s.blinkEyeClosed && ear < CLOSED) { s.blinkEyeClosed = true; } else if (s.blinkEyeClosed && ear > OPEN) { s.blinkCount += 1; s.blinkEyeClosed = false; } hint = `Blink terdeteksi: ${s.blinkCount}/1`; if (s.blinkCount >= 1) done = true; } else if (chal === "turn_left" || chal === "turn_right") { if (s.baselineYaw == null) s.baselineYaw = yaw; // In mirrored video, "turn_left" from user's POV = camera sees nose move right const delta = yaw - s.baselineYaw; const THRESH = 0.35; if (chal === "turn_left" && delta >= THRESH) done = true; if (chal === "turn_right" && delta <= -THRESH) done = true; hint = `Yaw: ${delta.toFixed(2)}`; } else if (chal === "nod") { if (s.baselinePitch == null) s.baselinePitch = pitch; const delta = pitch - s.baselinePitch; if (delta > 0.3) done = true; hint = `Pitch: ${delta.toFixed(2)}`; } else if (chal === "smile") { if (s.baselineMouth == null) s.baselineMouth = mopen; if (s.baselineSmile == null) s.baselineSmile = smileRatio; const dOpen = mopen - s.baselineMouth; const dSmile = smileRatio - s.baselineSmile; if (dOpen > 0.06 || dSmile > 0.05 || mopen > 0.35 || smileRatio > 0.55) done = true; hint = `Mulut: ${(mopen).toFixed(2)} lebar: ${smileRatio.toFixed(2)}`; } setStatusText(hint); if (done) { const newPassed = [...passed, chal]; setPassed(newPassed); resetInternal(); const next = currentIdx + 1; if (next >= challenges.length) { setStatusText("Semua verifikasi terlewati"); if (onAllChallengesDone) onAllChallengesDone(newPassed); if (captureOnDone) { // capture immediately from current frame setTimeout(() => doCapture(), 250); } } else { setCurrentIdx(next); } } }, // eslint-disable-next-line react-hooks/exhaustive-deps [currentIdx, challenges, passed] ); const doCapture = useCallback(() => { const video = videoRef.current; if (!video) return null; const tmp = document.createElement("canvas"); tmp.width = video.videoWidth || width; tmp.height = video.videoHeight || height; const tctx = tmp.getContext("2d"); // Un-mirror capture (backend expects real orientation) tctx.save(); tctx.scale(-1, 1); tctx.drawImage(video, -tmp.width, 0, tmp.width, tmp.height); tctx.restore(); const dataUrl = tmp.toDataURL("image/jpeg", 0.85); if (onCapture) onCapture(dataUrl); return dataUrl; }, [onCapture, width, height]); // expose capture via ref-like callback useEffect(() => { // attach capture handler to window if needed? Actually parent uses onCapture }, []); // Initialize mediapipe + camera useEffect(() => { let cancelled = false; async function init() { const mesh = new FaceMesh({ locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh/${file}`, }); mesh.setOptions({ maxNumFaces: 1, refineLandmarks: true, minDetectionConfidence: 0.6, minTrackingConfidence: 0.6, }); mesh.onResults(onResults); meshRef.current = mesh; try { const stream = await navigator.mediaDevices.getUserMedia({ video: { width: { ideal: 640 }, height: { ideal: 480 }, facingMode: "user" }, audio: false, }); if (cancelled) { stream.getTracks().forEach((t) => t.stop()); return; } streamRef.current = stream; const video = videoRef.current; video.srcObject = stream; await video.play(); setReady(true); const loop = async () => { if (cancelled) return; if (meshRef.current && video.readyState >= 2) { try { await meshRef.current.send({ image: video }); } catch (_) { // ignore } } rafRef.current = requestAnimationFrame(loop); }; loop(); } catch (err) { setStatusText("Kamera tidak dapat diakses: " + err.message); } } init(); return () => { cancelled = true; if (rafRef.current) cancelAnimationFrame(rafRef.current); if (streamRef.current) streamRef.current.getTracks().forEach((t) => t.stop()); try { meshRef.current?.close?.(); } catch (_) {} }; // eslint-disable-next-line react-hooks/exhaustive-deps }, []); // update onResults binding when currentIdx changes (React closure) useEffect(() => { if (meshRef.current) meshRef.current.onResults(onResults); }, [onResults]); const progress = challenges?.length ? passed.length / challenges.length : 0; const currentMeta = currentChallenge ? CHALLENGE_META[currentChallenge] : null; return (