Use look_at_image command instead of manual angle math

- App now sends pixel coordinates via lookAtImage(u, v) — daemon does
all the camera intrinsics, undistortion, and rotation math
- Removed gain/deadzone tuning (no longer needed)
- Added lookAtImage() to reachy-mini.js library
- Simplified control loop: detect hand → smooth → send pixels

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

app.js

@@ -20,20 +20,17 @@ let controlTimer = null;
 
 // Tracking state — shared between render loop and control loop
 let trackingEnabled = true;
-let handX = NaN, handY = NaN; // latest smoothed hand position (0-1), NaN = no hand
-let currentYaw = 0, currentPitch = 0;
-let noHandFrames = 0;
+let handPixelX = NaN, handPixelY = NaN; // latest smoothed hand position in pixels
 let lastDetectTs = -1;
 let frameCount = 0;
 let fpsTime = 0;
-const NO_HAND_RETURN_FRAMES = 30; // at 5Hz = 6 seconds
+let noHandTicks = 0;
 
 // Tuning params
-let gain = 60;
 let smoothing = 0.60;
-let deadzone = 0.03;
 const CONTROL_HZ = 5;
 const CONTROL_INTERVAL = 1000 / CONTROL_HZ; // 200ms
+const NO_HAND_RETURN_TICKS = 30; // at 5Hz = 6 seconds
 
 // Canvas sync cache
 let cachedVideoW = 0, cachedVideoH = 0;
@@ -192,7 +189,7 @@ function updateTrackBadge() {
     if (!trackingEnabled) {
         b.className = 'badge tracking-disabled';
         b.textContent = 'Tracking off';
-    } else if (isNaN(handX)) {
+    } else if (isNaN(handPixelX)) {
         b.className = 'badge tracking-searching';
         b.textContent = 'Searching...';
     } else {
@@ -205,8 +202,8 @@ function updateTrackBadge() {
 $('trackingToggle').addEventListener('change', (e) => {
     trackingEnabled = e.target.checked;
     if (trackingEnabled) {
-        handX = NaN;
-        handY = NaN;
+        handPixelX = NaN;
+        handPixelY = NaN;
     }
     updateTrackBadge();
     if (!trackingEnabled) clearCanvas();
@@ -218,10 +215,8 @@ $('tuneBtn').addEventListener('click', () => {
 });
 
 function initTuning() {
-    const gs = $('gainSlider'), ss = $('smoothSlider'), ds = $('dzSlider');
-    gs.addEventListener('input', () => { gain = +gs.value; $('gainVal').textContent = gain; });
+    const ss = $('smoothSlider');
     ss.addEventListener('input', () => { smoothing = +ss.value / 100; $('smoothVal').textContent = smoothing.toFixed(2); });
-    ds.addEventListener('input', () => { deadzone = +ds.value / 100; $('dzVal').textContent = deadzone.toFixed(2); });
 }
 
 // ─── MediaPipe detector ───
@@ -330,7 +325,7 @@ function renderFrame(now) {
         const hand = results.landmarks[0];
         drawHand(hand, w, h);
 
-        // Compute centroid
+        // Compute centroid in normalized coords (0-1)
         let cx = 0, cy = 0;
         for (let i = 0; i < hand.length; i++) { cx += hand[i].x; cy += hand[i].y; }
         cx /= hand.length;
@@ -342,28 +337,27 @@ function renderFrame(now) {
         ctx.fillStyle = 'rgba(255, 107, 53, 0.8)';
         ctx.fill();
 
-        // Update shared hand position with EMA smoothing
+        // Update shared hand position with EMA smoothing (in pixels)
         const alpha = 1 - smoothing;
-        if (isNaN(handX)) { handX = cx; handY = cy; }
-        else { handX = alpha * cx + smoothing * handX; handY = alpha * cy + smoothing * handY; }
+        const rawPixelX = cx * w;
+        const rawPixelY = cy * h;
+        if (isNaN(handPixelX)) { handPixelX = rawPixelX; handPixelY = rawPixelY; }
+        else { handPixelX = alpha * rawPixelX + smoothing * handPixelX; handPixelY = alpha * rawPixelY + smoothing * handPixelY; }
 
-        // Draw error line
+        // Draw error line from center to smoothed position
         ctx.beginPath();
         ctx.moveTo(0.5 * w, 0.5 * h);
-        ctx.lineTo(handX * w, handY * h);
+        ctx.lineTo(handPixelX, handPixelY);
         ctx.strokeStyle = 'rgba(255, 107, 53, 0.4)';
         ctx.lineWidth = 2;
         ctx.setLineDash([6, 4]);
         ctx.stroke();
         ctx.setLineDash([]);
-    } else {
-        // No hand detected this frame — don't clear handX/handY immediately,
-        // the control loop handles timeout
     }
 }
 
 // ═══════════════════════════════════════════════════════════════
-// CONTROL LOOP — fixed 5 Hz, reads handX/handY, sends commands
+// CONTROL LOOP — fixed 5 Hz, sends look_at_image commands
 // ═══════════════════════════════════════════════════════════════
 
 function startControl() {
@@ -378,44 +372,28 @@ function stopControl() {
 function controlTick() {
     if (robot.state !== 'streaming') { stopControl(); return; }
 
-    if (!trackingEnabled || isNaN(handX)) {
-        // No hand: count frames, slowly return to center
-        noHandFrames++;
-        if (noHandFrames > NO_HAND_RETURN_FRAMES) {
-            currentYaw *= 0.95;
-            currentPitch *= 0.95;
-            if (Math.abs(currentYaw) < 0.3) currentYaw = 0;
-            if (Math.abs(currentPitch) < 0.3) currentPitch = 0;
+    if (!trackingEnabled || isNaN(handPixelX)) {
+        noHandTicks++;
+        // After timeout with no hand, look at center
+        if (noHandTicks > NO_HAND_RETURN_TICKS) {
+            const w = video.videoWidth, h = video.videoHeight;
+            if (w && h) {
+                robot.lookAtImage(w / 2, h / 2);
+            }
         }
     } else {
-        noHandFrames = 0;
-
-        // Error from center (0.5, 0.5)
-        let errorX = handX - 0.5;
-        let errorY = handY - 0.5;
-
-        // Deadzone
-        if (Math.abs(errorX) < deadzone) errorX = 0;
-        if (Math.abs(errorY) < deadzone) errorY = 0;
-
-        // Simple proportional: error * gain = target angle
-        // gain=60, max error=0.5 → 30° max, good fit for robot limits
-        currentYaw = -gain * errorX;
-        currentPitch = gain * errorY;
-
-        // Clamp
-        currentYaw = Math.max(-45, Math.min(45, currentYaw));
-        currentPitch = Math.max(-30, Math.min(30, currentPitch));
+        noHandTicks = 0;
+        // Send pixel coordinates — the daemon handles all the math
+        robot.lookAtImage(handPixelX, handPixelY);
     }
 
-    // Send command
-    robot.setHeadPose(0, currentPitch, currentYaw);
-
     // Update HUD
     updateTrackBadge();
-    $('headBadge').textContent = `Y:${currentYaw.toFixed(1)} P:${currentPitch.toFixed(1)}`;
+    if (!isNaN(handPixelX)) {
+        $('headBadge').textContent = `px:${Math.round(handPixelX)},${Math.round(handPixelY)}`;
+    }
 
-    // Latency (every 2s = every 10 ticks)
+    // Latency (every 2s)
     const now = performance.now();
     if (now - lastLatencyTime >= 2000) {
         lastLatencyTime = now;
@@ -462,10 +440,9 @@ function clearCanvas() {
 }
 
 function resetTrackingState() {
-    handX = NaN; handY = NaN;
-    currentYaw = 0; currentPitch = 0;
+    handPixelX = NaN; handPixelY = NaN;
     lastDetectTs = -1;
-    noHandFrames = 0;
+    noHandTicks = 0;
 }
 
 // ─── Latency badge ───

index.html

@@ -78,21 +78,11 @@
             </div>
 
             <div class="tuning hidden" id="tuningPanel">
-                <label>
-                    Gain
-                    <input type="range" id="gainSlider" min="5" max="120" value="60">
-                    <span class="value" id="gainVal">60</span>
-                </label>
                 <label>
                     Smooth
                     <input type="range" id="smoothSlider" min="0" max="95" value="60">
                     <span class="value" id="smoothVal">0.60</span>
                 </label>
-                <label>
-                    Dead
-                    <input type="range" id="dzSlider" min="0" max="15" value="3">
-                    <span class="value" id="dzVal">0.03</span>
-                </label>
             </div>
         </main>
     </div>

reachy-mini.js

@@ -541,6 +541,17 @@ export class ReachyMini extends EventTarget {
         return this._sendCommand({ type: "set_target", head: rpyToMatrix(roll, pitch, yaw).flat() });
     }
 
+    /**
+     * Make the robot look at a pixel position in the camera frame.
+     * @param {number} u — horizontal pixel coordinate
+     * @param {number} v — vertical pixel coordinate
+     * @param {number} [duration=0] — movement duration in seconds (0 = immediate)
+     * @returns {boolean}
+     */
+    lookAtImage(u, v, duration = 0) {
+        return this._sendCommand({ type: "look_at_image", u, v, duration });
+    }
+
     /**
      * Set antenna positions.
      * @param {number} rightDeg  @param {number} leftDeg