// MediaPipe Hand runtime — 21 landmarks per hand, in-browser, GPU→CPU fallback. // Lifted from abuelas-garden/web/pose/hands.js and upgraded to TWO hands so the // performer can run two decks at once. Camera never leaves the device. import { HandLandmarker, FilesetResolver } from "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.12"; const WASM = "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.12/wasm"; const MODEL = "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task"; export const HAND_CONN = [ [0,1],[1,2],[2,3],[3,4], // thumb [0,5],[5,6],[6,7],[7,8], // index [5,9],[9,10],[10,11],[11,12], // middle [9,13],[13,14],[14,15],[15,16], // ring [13,17],[17,18],[18,19],[19,20], // pinky [0,17], // palm base ]; export class HandEngine { constructor(numHands = 2) { this.landmarker = null; this.running = false; this.fps = 0; this._n = 0; this._t = 0; this.delegate = null; this.numHands = numHands; } async start(video, onFrame) { const stream = await navigator.mediaDevices.getUserMedia({ video: { facingMode: "user", width: { ideal: 960 }, height: { ideal: 720 } }, audio: false, }); video.srcObject = stream; await video.play(); const vision = await FilesetResolver.forVisionTasks(WASM); const opts = (delegate) => ({ baseOptions: { modelAssetPath: MODEL, delegate }, runningMode: "VIDEO", numHands: this.numHands, minHandDetectionConfidence: 0.4, minHandPresenceConfidence: 0.4, minTrackingConfidence: 0.4, }); try { this.landmarker = await HandLandmarker.createFromOptions(vision, opts("GPU")); this.delegate = "GPU"; } catch { this.landmarker = await HandLandmarker.createFromOptions(vision, opts("CPU")); this.delegate = "CPU"; } this.running = true; this._t = performance.now(); const loop = () => { if (!this.running) return; const now = performance.now(); if (video.readyState >= 2 && this.landmarker) { let hands = []; try { const res = this.landmarker.detectForVideo(video, now); const labels = res?.handednesses || res?.handedness || []; (res?.landmarks || []).forEach((lm, i) => { hands.push({ lm, handed: labels?.[i]?.[0]?.categoryName || null }); }); } catch (_) {} onFrame(hands, now, this.fps); } this._n++; if (now - this._t > 500) { this.fps = (this._n * 1000) / (now - this._t); this._n = 0; this._t = now; } requestAnimationFrame(loop); }; requestAnimationFrame(loop); } stop() { this.running = false; } } // Draw a mirrored hand skeleton. `hue` tints the deck (cyan vs magenta). export function drawHand(ctx, lm, w, h, hue = "rgba(80,220,255,") { if (!lm) return; ctx.save(); ctx.translate(w, 0); ctx.scale(-1, 1); ctx.lineWidth = 4; ctx.lineCap = "round"; ctx.strokeStyle = hue + ".85)"; ctx.shadowColor = hue + ".9)"; ctx.shadowBlur = 14; for (const [i, j] of HAND_CONN) { const p = lm[i], q = lm[j]; if (!p || !q) continue; ctx.beginPath(); ctx.moveTo(p.x * w, p.y * h); ctx.lineTo(q.x * w, q.y * h); ctx.stroke(); } ctx.fillStyle = hue + ".95)"; for (const p of lm) { ctx.beginPath(); ctx.arc(p.x * w, p.y * h, 5, 0, 7); ctx.fill(); } ctx.restore(); }