js/handTracking.js

// MediaPipe Tasks Vision — HandLandmarker. Loaded from CDN (online delivery, per plan).
// The heavy library is imported LAZILY (dynamic import inside initHandTracking) so page load and
// the Start button never depend on the CDN; a CDN failure becomes a catchable startup error.
const LIB_URL = "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.14/vision_bundle.mjs";
const WASM_PATH = "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.14/wasm";
const MODEL_URL =
  "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task";

// Landmarks that sit around the palm; averaging them gives a stable anchor for the hand icon
// (steadier than a single fingertip, and matches a big palm-shaped cursor).
const PALM_LANDMARKS = [0, 5, 9, 13, 17];

let handLandmarker = null;

export async function initHandTracking() {
  const { HandLandmarker, FilesetResolver } = await import(LIB_URL);
  const vision = await FilesetResolver.forVisionTasks(WASM_PATH);

  const options = {
    baseOptions: { modelAssetPath: MODEL_URL, delegate: "GPU" },
    runningMode: "video",
    numHands: 1,
  };

  try {
    handLandmarker = await HandLandmarker.createFromOptions(vision, options);
  } catch (err) {
    // Some machines/browsers lack a usable GPU delegate — fall back to CPU.
    console.warn("HandLandmarker GPU init failed, falling back to CPU:", err);
    options.baseOptions.delegate = "CPU";
    handLandmarker = await HandLandmarker.createFromOptions(vision, options);
  }

  return handLandmarker;
}

// Returns { x, y } normalized [0,1] palm anchor in the UN-flipped video frame, or null if no hand.
// `timestampMs` must be monotonically increasing (the requestAnimationFrame timestamp works).
export function detectHand(video, timestampMs) {
  if (!handLandmarker || !video.videoWidth) return null;

  let result;
  try {
    result = handLandmarker.detectForVideo(video, timestampMs);
  } catch (err) {
    // A rare frame can throw (e.g. timestamp hiccup) — skip it rather than crash the loop.
    return null;
  }

  if (!result || !result.landmarks || result.landmarks.length === 0) return null;

  const lm = result.landmarks[0];
  let x = 0;
  let y = 0;
  for (const i of PALM_LANDMARKS) {
    x += lm[i].x;
    y += lm[i].y;
  }
  x /= PALM_LANDMARKS.length;
  y /= PALM_LANDMARKS.length;

  return { x, y };
}