Rewrite as 1Hz detect-then-move loop to fix feedback oscillation

The camera is on the robot's head, so moving the head changes where
the hand appears in the frame. At high frequency (5Hz+), this creates
a feedback loop of conflicting commands causing chaotic movement.

New architecture:
- Control loop at 1Hz: one fresh detection + one lookAtImage(u,v,0.8)
- Head smoothly interpolates over 0.8s, settles 0.2s before next tick
- Render loop (every frame) only draws hand skeleton overlay, no control
- No EMA smoothing needed — single sample per second is stable enough

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

app.js

@@ -19,7 +19,7 @@ function checkMinVersion(ver, min) {
         if ((v[i] || 0) > (m[i] || 0)) return true;
         if ((v[i] || 0) < (m[i] || 0)) return false;
     }
-    return true; // equal
+    return true;
 }
 
 // ─── State ───
@@ -29,23 +29,21 @@ let detachVideo = null;
 let selectedRobotId = null;
 let controlTimer = null;
 
-// ─── TEST MODE: click on video to send look_at_image ───
-const TEST_MODE = true; // set to false to re-enable hand tracking control
-let clickMarker = null; // {x, y} in video pixels
+// Control config
+const CONTROL_INTERVAL = 1000; // 1 Hz — one detect + one move per second
+const MOVE_DURATION = 0.8;     // smooth interpolation over 0.8s (leaves 0.2s settle)
+const NO_HAND_RETURN_TICKS = 5; // 5 seconds without hand → return to center
 
-// Tracking state — shared between render loop and control loop
+// Tracking state
 let trackingEnabled = true;
-let handPixelX = NaN, handPixelY = NaN; // latest smoothed hand position in pixels
-let lastDetectTs = -1;
-let frameCount = 0;
-let fpsTime = 0;
 let noHandTicks = 0;
+let lastTarget = null; // {u, v} — last sent target for HUD display
 
-// Tuning params
-let smoothing = 0.60;
-const CONTROL_HZ = 5;
-const CONTROL_INTERVAL = 1000 / CONTROL_HZ; // 200ms
-const NO_HAND_RETURN_TICKS = 30; // at 5Hz = 6 seconds
+// Render-only state (for visual overlay between control ticks)
+let lastRenderDetectTs = -1;
+let lastRenderResults = null;
+let frameCount = 0;
+let fpsTime = 0;
 
 // Canvas sync cache
 let cachedVideoW = 0, cachedVideoH = 0;
@@ -217,7 +215,7 @@ function updateTrackBadge() {
     if (!trackingEnabled) {
         b.className = 'badge tracking-disabled';
         b.textContent = 'Tracking off';
-    } else if (isNaN(handPixelX)) {
+    } else if (!lastTarget) {
         b.className = 'badge tracking-searching';
         b.textContent = 'Searching...';
     } else {
@@ -229,10 +227,6 @@ function updateTrackBadge() {
 // ─── Tracking toggle ───
 $('trackingToggle').addEventListener('change', (e) => {
     trackingEnabled = e.target.checked;
-    if (trackingEnabled) {
-        handPixelX = NaN;
-        handPixelY = NaN;
-    }
     updateTrackBadge();
     if (!trackingEnabled) clearCanvas();
 });
@@ -244,28 +238,9 @@ $('tuneBtn').addEventListener('click', () => {
 
 function initTuning() {
     const ss = $('smoothSlider');
-    ss.addEventListener('input', () => { smoothing = +ss.value / 100; $('smoothVal').textContent = smoothing.toFixed(2); });
+    ss.addEventListener('input', () => { $('smoothVal').textContent = (+ss.value / 100).toFixed(2); });
 }
 
-// ─── TEST MODE: click-to-look-at ───
-canvas.addEventListener('click', (e) => {
-    if (!TEST_MODE || robot.state !== 'streaming') return;
-    const vw = video.videoWidth, vh = video.videoHeight;
-    if (!vw || !vh) return;
-
-    // Convert click position (on canvas element) to video pixel coordinates
-    const rect = canvas.getBoundingClientRect();
-    const u = (e.clientX - rect.left) / rect.width * vw;
-    const v = (e.clientY - rect.top) / rect.height * vh;
-
-    clickMarker = { x: u, y: v };
-    console.log(`look_at_image(${Math.round(u)}, ${Math.round(v)}) — video: ${vw}x${vh}`);
-    robot.lookAtImage(u, v);
-
-    // Update HUD
-    $('headBadge').textContent = `click: ${Math.round(u)},${Math.round(v)}`;
-});
-
 // ─── MediaPipe detector ───
 async function initDetector() {
     if (detector) {
@@ -296,7 +271,8 @@ async function initDetector() {
 }
 
 // ═══════════════════════════════════════════════════════════════
-// RENDER LOOP — runs every video frame, does detection + drawing
+// RENDER LOOP — visual overlay only (no control, just drawing)
+// Runs every video frame for smooth FPS counter + hand skeleton
 // ═══════════════════════════════════════════════════════════════
 
 function startTracking() {
@@ -352,76 +328,56 @@ function renderFrame(now) {
         fpsTime = now;
     }
 
-    // Run detection
-    let results = null;
-    if (detector && trackingEnabled && now > lastDetectTs) {
+    // Run detection for visual overlay (doesn't affect control)
+    if (detector && trackingEnabled && now > lastRenderDetectTs) {
         try {
-            results = detector.detectForVideo(video, now);
-            lastDetectTs = now;
+            lastRenderResults = detector.detectForVideo(video, now);
+            lastRenderDetectTs = now;
         } catch (_) {}
     }
 
-    // Clear & draw
+    // Draw
     ctx.clearRect(0, 0, canvas.width, canvas.height);
     if (!trackingEnabled) return;
 
     const w = canvas.width, h = canvas.height;
     drawCrosshair(w, h);
 
-    // Draw click marker in test mode
-    if (TEST_MODE && clickMarker) {
+    // Draw last sent target
+    if (lastTarget) {
         ctx.beginPath();
-        ctx.arc(clickMarker.x, clickMarker.y, 12, 0, Math.PI * 2);
+        ctx.arc(lastTarget.u, lastTarget.v, 12, 0, Math.PI * 2);
         ctx.strokeStyle = 'rgba(255, 50, 50, 0.9)';
         ctx.lineWidth = 3;
         ctx.stroke();
-        ctx.beginPath();
-        ctx.moveTo(clickMarker.x - 18, clickMarker.y);
-        ctx.lineTo(clickMarker.x + 18, clickMarker.y);
-        ctx.moveTo(clickMarker.x, clickMarker.y - 18);
-        ctx.lineTo(clickMarker.x, clickMarker.y + 18);
-        ctx.strokeStyle = 'rgba(255, 50, 50, 0.6)';
-        ctx.lineWidth = 1.5;
-        ctx.stroke();
     }
 
-    if (results && results.landmarks && results.landmarks.length > 0) {
-        const hand = results.landmarks[0];
+    // Draw hand skeleton from latest render detection
+    if (lastRenderResults && lastRenderResults.landmarks && lastRenderResults.landmarks.length > 0) {
+        const hand = lastRenderResults.landmarks[0];
         drawHand(hand, w, h);
 
-        // Compute centroid in normalized coords (0-1)
         let cx = 0, cy = 0;
         for (let i = 0; i < hand.length; i++) { cx += hand[i].x; cy += hand[i].y; }
         cx /= hand.length;
         cy /= hand.length;
 
-        // Draw centroid
         ctx.beginPath();
         ctx.arc(cx * w, cy * h, 8, 0, Math.PI * 2);
         ctx.fillStyle = 'rgba(255, 107, 53, 0.8)';
         ctx.fill();
-
-        // Update shared hand position with EMA smoothing (in pixels)
-        const alpha = 1 - smoothing;
-        const rawPixelX = cx * w;
-        const rawPixelY = cy * h;
-        if (isNaN(handPixelX)) { handPixelX = rawPixelX; handPixelY = rawPixelY; }
-        else { handPixelX = alpha * rawPixelX + smoothing * handPixelX; handPixelY = alpha * rawPixelY + smoothing * handPixelY; }
-
-        // Draw error line from center to smoothed position
-        ctx.beginPath();
-        ctx.moveTo(0.5 * w, 0.5 * h);
-        ctx.lineTo(handPixelX, handPixelY);
-        ctx.strokeStyle = 'rgba(255, 107, 53, 0.4)';
-        ctx.lineWidth = 2;
-        ctx.setLineDash([6, 4]);
-        ctx.stroke();
-        ctx.setLineDash([]);
     }
 }
 
 // ═══════════════════════════════════════════════════════════════
-// CONTROL LOOP — fixed 5 Hz, sends look_at_image commands
+// CONTROL LOOP — 1 Hz: one detection + one movement per tick
+//
+// Why 1 Hz? The camera is on the robot's head. When we send
+// look_at_image, the head moves, which moves the camera, which
+// changes where the hand appears in the frame. At high frequency
+// this creates a feedback loop of conflicting commands. At 1 Hz
+// with duration=0.8s, the head smoothly moves to the target and
+// has 0.2s to settle before the next detection.
 // ═══════════════════════════════════════════════════════════════
 
 function startControl() {
@@ -435,40 +391,47 @@ function stopControl() {
 
 function controlTick() {
     if (robot.state !== 'streaming') { stopControl(); return; }
+    if (!trackingEnabled || !detector) return;
 
-    // In test mode, control loop only updates latency — clicks send commands directly
-    if (TEST_MODE) {
-        const now = performance.now();
-        if (now - lastLatencyTime >= 2000) {
-            lastLatencyTime = now;
-            updateLatencyBadge();
-        }
-        return;
-    }
+    const w = video.videoWidth, h = video.videoHeight;
+    if (!w || !h || video.readyState < 2) return;
+
+    // Run a fresh detection for control (independent of render loop)
+    let results = null;
+    const now = performance.now();
+    try {
+        results = detector.detectForVideo(video, now);
+    } catch (_) {}
+
+    if (results && results.landmarks && results.landmarks.length > 0) {
+        noHandTicks = 0;
+        const hand = results.landmarks[0];
+
+        // Compute centroid in pixels
+        let cx = 0, cy = 0;
+        for (let i = 0; i < hand.length; i++) { cx += hand[i].x; cy += hand[i].y; }
+        cx /= hand.length;
+        cy /= hand.length;
+        const u = cx * w;
+        const v = cy * h;
+
+        lastTarget = { u, v };
+        robot.lookAtImage(u, v, MOVE_DURATION);
 
-    if (!trackingEnabled || isNaN(handPixelX)) {
+        // HUD
+        $('headBadge').textContent = `px:${Math.round(u)},${Math.round(v)}`;
+    } else {
         noHandTicks++;
-        // After timeout with no hand, look at center
         if (noHandTicks > NO_HAND_RETURN_TICKS) {
-            const w = video.videoWidth, h = video.videoHeight;
-            if (w && h) {
-                robot.lookAtImage(w / 2, h / 2);
-            }
+            lastTarget = null;
+            robot.lookAtImage(w / 2, h / 2, MOVE_DURATION);
+            $('headBadge').textContent = 'center';
         }
-    } else {
-        noHandTicks = 0;
-        // Send pixel coordinates — the daemon handles all the math
-        robot.lookAtImage(handPixelX, handPixelY);
     }
 
-    // Update HUD
     updateTrackBadge();
-    if (!isNaN(handPixelX)) {
-        $('headBadge').textContent = `px:${Math.round(handPixelX)},${Math.round(handPixelY)}`;
-    }
 
-    // Latency (every 2s)
-    const now = performance.now();
+    // Latency (every ~2 ticks)
     if (now - lastLatencyTime >= 2000) {
         lastLatencyTime = now;
         updateLatencyBadge();
@@ -514,8 +477,9 @@ function clearCanvas() {
 }
 
 function resetTrackingState() {
-    handPixelX = NaN; handPixelY = NaN;
-    lastDetectTs = -1;
+    lastTarget = null;
+    lastRenderDetectTs = -1;
+    lastRenderResults = null;
     noHandTicks = 0;
 }
 

style.css

@@ -160,8 +160,7 @@ main {
 
 .video-container canvas {
     position: absolute;
-    pointer-events: auto;
-    cursor: crosshair;
+    pointer-events: none;
 }
 
 /* ── HUD badges ── */