web/hands.js

// MediaPipe Hand runtime — 21 landmarks per hand, in-browser, GPU→CPU fallback.
// Lifted from abuelas-garden/web/pose/hands.js and upgraded to TWO hands so the
// performer can run two decks at once. Camera never leaves the device.
import { HandLandmarker, FilesetResolver } from "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.12";

const WASM = "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.12/wasm";
const MODEL = "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task";

export const HAND_CONN = [
  [0,1],[1,2],[2,3],[3,4],            // thumb
  [0,5],[5,6],[6,7],[7,8],            // index
  [5,9],[9,10],[10,11],[11,12],       // middle
  [9,13],[13,14],[14,15],[15,16],     // ring
  [13,17],[17,18],[18,19],[19,20],    // pinky
  [0,17],                             // palm base
];

export class HandEngine {
  constructor(numHands = 2) {
    this.landmarker = null; this.running = false;
    this.fps = 0; this._n = 0; this._t = 0; this.delegate = null;
    this.numHands = numHands;
  }

  async start(video, onFrame) {
    const stream = await navigator.mediaDevices.getUserMedia({
      video: { facingMode: "user", width: { ideal: 960 }, height: { ideal: 720 } }, audio: false,
    });
    video.srcObject = stream; await video.play();

    const vision = await FilesetResolver.forVisionTasks(WASM);
    const opts = (delegate) => ({
      baseOptions: { modelAssetPath: MODEL, delegate },
      runningMode: "VIDEO", numHands: this.numHands,
      minHandDetectionConfidence: 0.4, minHandPresenceConfidence: 0.4, minTrackingConfidence: 0.4,
    });
    try { this.landmarker = await HandLandmarker.createFromOptions(vision, opts("GPU")); this.delegate = "GPU"; }
    catch { this.landmarker = await HandLandmarker.createFromOptions(vision, opts("CPU")); this.delegate = "CPU"; }

    this.running = true; this._t = performance.now();
    const loop = () => {
      if (!this.running) return;
      const now = performance.now();
      if (video.readyState >= 2 && this.landmarker) {
        let hands = [];
        try {
          const res = this.landmarker.detectForVideo(video, now);
          const labels = res?.handednesses || res?.handedness || [];
          (res?.landmarks || []).forEach((lm, i) => {
            hands.push({ lm, handed: labels?.[i]?.[0]?.categoryName || null });
          });
        } catch (_) {}
        onFrame(hands, now, this.fps);
      }
      this._n++; if (now - this._t > 500) { this.fps = (this._n * 1000) / (now - this._t); this._n = 0; this._t = now; }
      requestAnimationFrame(loop);
    };
    requestAnimationFrame(loop);
  }
  stop() { this.running = false; }
}

// Draw a mirrored hand skeleton. `hue` tints the deck (cyan vs magenta).
export function drawHand(ctx, lm, w, h, hue = "rgba(80,220,255,") {
  if (!lm) return;
  ctx.save(); ctx.translate(w, 0); ctx.scale(-1, 1);
  ctx.lineWidth = 4; ctx.lineCap = "round";
  ctx.strokeStyle = hue + ".85)"; ctx.shadowColor = hue + ".9)"; ctx.shadowBlur = 14;
  for (const [i, j] of HAND_CONN) {
    const p = lm[i], q = lm[j]; if (!p || !q) continue;
    ctx.beginPath(); ctx.moveTo(p.x * w, p.y * h); ctx.lineTo(q.x * w, q.y * h); ctx.stroke();
  }
  ctx.fillStyle = hue + ".95)";
  for (const p of lm) { ctx.beginPath(); ctx.arc(p.x * w, p.y * h, 5, 0, 7); ctx.fill(); }
  ctx.restore();
}