Blame for processor/web/static/js/heatmap.js

/* Bionic Hand Sensor — synced with video playback */

let heatmapFps = 30;
let heatmapSyncing = false;
const heatmapSyncBound = new WeakSet();
let currentEpisodeUuid = null;
let lastFrameIdx = -1;
let totalFrames = 0;

// Sensor → hand mapping (populated by loadFrameData)
let sensorRight = null;   // sensor name for right hand (e.g. "sensors_right")
let sensorLeft = null;    // sensor name for left hand  (e.g. "sensors_left")

// Hand-pressure tiles placed on the video canvas by drag & drop.
// Each entry: {canvas: <canvas element>, hand: 'left' | 'right'}.
// frames-data already contains the complete sensor sequence, so the browser
// renders it locally instead of requesting one PNG per frame.
let canvasHandTiles = [];
let sensorFramesByIndex = new Map();

function registerHandTile(canvas, hand) {
    canvasHandTiles.push({ canvas, hand });
    const frame = typeof getCurrentFrame === 'function' ? getCurrentFrame() : 0;
    const sensor = hand === 'left' ? sensorLeft : sensorRight;
    const values = _sensorValue(frame, sensor);
    if (canvas && values) renderHandSensorCanvas(canvas, values, hand === 'left');
}

function renderGloveHeatmapCanvas(canvas, values) {
    // Plain 16x16 sensor heatmap drawn locally from the preloaded
    // frames-data payload — mirrors the backend PNG (Viridis ramp +
    // Gaussian-ish smoothing) without any per-frame HTTP request.
    if (!canvas) return;
    const size = 512, cell = size / 16;
    if (canvas.width !== size || canvas.height !== size) {
        canvas.width = size;
        canvas.height = size;
    }
    const ctx = canvas.getContext('2d');
    const matrix = Array.isArray(values) && Array.isArray(values[0])
        ? values : [];
    const flat = (Array.isArray(values) && !Array.isArray(values[0]))
        ? values : (matrix.flat ? matrix.flat() : []);
    if (!flat.length) return;
    const vmax = Math.max(1, ...flat.map(v => Number(v) || 0));
    const drawCells = () => {
        ctx.fillStyle = '#121212';
        ctx.fillRect(0, 0, size, size);
        for (let r = 0; r < 16; r++) {
            for (let c = 0; c < 16; c++) {
                const value = Number(matrix[r] ? matrix[r][c]
                    : flat[r * 16 + c]) || 0;
                ctx.fillStyle = _viridisSensorColor(value, vmax);
                ctx.fillRect(c * cell, r * cell, cell + 1, cell + 1);
            }
        }
    };
    if (typeof ctx.filter === 'string') {
        // ~Gaussian blur like the backend's 7px kernel (Chrome/Firefox).
        ctx.filter = 'blur(7px)';
        drawCells();
        ctx.filter = 'none';
    } else {
        drawCells();
    }
}

function registerGloveTile(canvas, sensor) {
    if (!canvas || !sensor) return;
    canvasHandTiles.push({ canvas, sensor });
    const frame = typeof getCurrentFrame === 'function' ? getCurrentFrame() : 0;
    const values = _sensorValue(frame, sensor);
    if (values) renderGloveHeatmapCanvas(canvas, values);
}

function unregisterHandTile(canvas) {
    canvasHandTiles = canvasHandTiles.filter(entry => entry.canvas !== canvas);
}

function clearHandTiles() {
    canvasHandTiles = [];
}

function hasHandSensor(hand) {
    return hand === 'left' ? Boolean(sensorLeft) : Boolean(sensorRight);
}

function handTileUrl(frameIndex, hand) {
    const sensor = hand === 'left' ? sensorLeft : sensorRight;
    const params = new URLSearchParams({ frame_index: frameIndex });
    if (sensor) params.set('sensor', sensor);
    if (hand === 'left') params.set('mirror', 'true');
    params.set('compact', 'true');
    return `/api/v1/video/${currentEpisodeUuid}/hand?${params.toString()}`;
}


function _resolveSensorMapping(sensors) {
    /* Given available sensor names, decide which to use for left/right hand.

       - Only columns containing "sensor" are considered as hand pressure sensors
         (columns like hand_angles / hand_joints are not 16×16 pressure grids)
       - Contains "right" / "left" in name → map accordingly
       - Single sensor with direction hint → only that hand, other hand hidden
       - Single sensor without hint → both hands use it (old hardware compat)
       - Multiple sensors without hints → index 0→right, index 1→left
    */
    sensorRight = null;
    sensorLeft = null;

    if (!sensors || sensors.length === 0) return;

    // Filter to pressure-sensor columns only (ignore hand_angles, hand_joints, etc.).
    // Accepts "sensors_left/right", SenseGlove "left_glove"/"right_glove" and
    // tactile "left"/"right" naming. IMU/valid telemetry columns must never
    // win the right/left assignment over a real pressure array.
    const pressureSensors = sensors.filter(s => {
        const n = s.toLowerCase();
        if (n.includes('imu') || n.endsWith('_valid')) return false;
        return n.includes('sensor') || n.includes('glove') || n.includes('tactile');
    });
    // If no pressure sensors found, fall back to all columns
    const candidates = pressureSensors.length > 0 ? pressureSensors : sensors;

    // Look for left/right hints in names
    for (const s of candidates) {
        const lower = s.toLowerCase();
        if (lower.includes('right') || lower.includes('_r')) sensorRight = s;
        if (lower.includes('left') || lower.includes('_l')) sensorLeft = s;
    }

    if (candidates.length === 1) {
        // Single sensor: only assign to the hand it names, if any
        if (!sensorRight && !sensorLeft) {
            // No direction hint → both hands same source (old hardware compat)
            sensorRight = candidates[0];
            sensorLeft = candidates[0];
        }
        // else: direction hint already set above, other hand stays null → hidden
        return;
    }

    // Multi-sensor: fill gaps with index-order fallback
    if (!sensorRight) sensorRight = candidates[0];
    if (!sensorLeft) sensorLeft = candidates.length >= 2 ? candidates[1] : candidates[0];
}

const HAND_SENSOR_REGIONS = [
    { rows: [0, 1, 2], cols: [14, 12, 13, 15], x: 250, y: 12, finger: true },
    { rows: [3, 4, 5], cols: [14, 12, 13, 15], x: 313, y: 12, finger: true },
    { rows: [6, 7, 8], cols: [14, 12, 13, 15], x: 376, y: 12, finger: true },
    { rows: [9, 10, 11], cols: [14, 12, 13, 15], x: 439, y: 12, finger: true },
    { rows: [12, 13, 14], cols: [14, 12, 13, 15], x: 502, y: 12, finger: true },
    { rows: Array.from({ length: 15 }, (_, i) => i), cols: [10, 9, 8, 6, 4], x: 280, y: 101, finger: false },
];

function _viridisSensorColor(value, vmax) {
    const stops = [
        [68, 1, 84], [59, 82, 139], [33, 145, 140],
        [94, 201, 98], [253, 231, 37],
    ];
    const ratio = Math.max(0, Math.min(1, Number(value || 0) / vmax));
    const pos = ratio * (stops.length - 1);
    const i = Math.min(stops.length - 2, Math.floor(pos));
    const t = pos - i;
    const a = stops[i], b = stops[i + 1];
    return `rgb(${Math.round(a[0] + (b[0] - a[0]) * t)},` +
        `${Math.round(a[1] + (b[1] - a[1]) * t)},` +
        `${Math.round(a[2] + (b[2] - a[2]) * t)})`;
}

function renderHandSensorCanvas(canvas, values, mirror) {
    if (!canvas) return;
    const width = 800, height = 200, cell = 16;
    if (canvas.width !== width || canvas.height !== height) {
        canvas.width = width;
        canvas.height = height;
    }
    canvas.style.aspectRatio = '4 / 1';
    const ctx = canvas.getContext('2d');
    const matrix = Array.isArray(values) && Array.isArray(values[0])
        ? values : [];
    const flat = matrix.flat ? matrix.flat() : [];
    const vmax = Math.max(1, ...flat.map(v => Number(v) || 0));
    ctx.save();
    if (mirror) {
        ctx.translate(width, 0);
        ctx.scale(-1, 1);
    }
    ctx.fillStyle = '#121212';
    ctx.fillRect(0, 0, width, height);
    ctx.font = '10px sans-serif';
    HAND_SENSOR_REGIONS.forEach(region => {
        const cs = cell;
        region.rows.forEach((row, i) => region.cols.forEach((col, j) => {
            const value = Number(matrix[row]?.[col]) || 0;
            const x = region.x + i * cs;
            const y = region.y + j * cs - 8;
            ctx.fillStyle = _viridisSensorColor(value, vmax);
            ctx.fillRect(x, y, cs, cs);
            ctx.strokeStyle = 'rgb(70,70,70)';
            ctx.strokeRect(x + 0.5, y + 0.5, cs - 1, cs - 1);
            ctx.fillStyle = vmax > 1 && value / vmax > 0.55 ? '#000' : '#fff';
            ctx.textAlign = 'center';
            ctx.textBaseline = 'middle';
            ctx.fillText(String(Math.round(value)), x + cs / 2, y + cs / 2);
        }));
    });
    ctx.restore();
}

function _sensorValue(frameIndex, sensor) {
    const row = sensorFramesByIndex.get(frameIndex);
    return row ? row[`observation_${sensor}`] : null;
}


async function loadFrameData(episodeId, sessionToken = null) {
    if (typeof isCurrentPlaybackSession === 'function' &&
        !isCurrentPlaybackSession(episodeId, sessionToken)) return;
    if (typeof currentEpisodeId !== 'undefined' && currentEpisodeId !== episodeId) return;  // 已切换批次
    currentEpisodeUuid = episodeId;
    lastFrameIdx = -1;
    _lastHandFrameIdx = -1;  // reset hand throttle on episode switch

    try {
        const signal = typeof getMediaLoadSignal === 'function'
            ? getMediaLoadSignal() : null;
        const cacheKey = `${episodeId}:glove:frames-data:v1`;
        let data = null;
        if (window.EgoMediaCache) {
            const cached = await window.EgoMediaCache.get(cacheKey);
            if (cached && cached.value) data = cached.value;
        }
        if (!data) {
            const res = await fetch(`/api/v1/episode/${episodeId}/frames-data`,
                signal ? { signal } : {});
            if ((typeof isCurrentPlaybackSession === 'function' &&
                 !isCurrentPlaybackSession(episodeId, sessionToken)) ||
                (typeof currentEpisodeId !== 'undefined' && currentEpisodeId !== episodeId)) return;  // stale — discard
            if (!res.ok) throw new Error(`frames-data request failed: ${res.status}`);
            data = await res.json();
            if (window.EgoMediaCache) {
                window.EgoMediaCache.put(cacheKey, data);
            }
        }
        if ((typeof isCurrentPlaybackSession === 'function' &&
             !isCurrentPlaybackSession(episodeId, sessionToken)) ||
            (typeof currentEpisodeId !== 'undefined' && currentEpisodeId !== episodeId)) return;  // stale — discard
        heatmapFps = data.fps || 30;

        // Resolve sensor → hand mapping from available sensors
        _resolveSensorMapping(data.sensors || []);

        // Keep the complete sensor payload indexed by the real frame number.
        // The source parquet may contain repeated/resampled rows, while the
        // API has already de-duplicated them to the video frame timeline.
        sensorFramesByIndex = new Map();
        (data.frames || []).forEach(frame => {
            if (frame && Number.isFinite(Number(frame.frame_index))) {
                sensorFramesByIndex.set(Number(frame.frame_index), frame);
            }
        });

        totalFrames = data.frame_count || 0;
        // Hand heatmaps are now draggable canvas tiles (top buttons), not a
        // fixed section under the video. Notify player.js to render the
        // drag-source buttons when sensor data exists.
        _updateHandLabels();
        updateHandImages(0);
        if (typeof renderSourceBar === 'function') renderSourceBar();

        // Notify player.js of frame info
        if (typeof setEpisodeFrameInfo === 'function') {
            setEpisodeFrameInfo(heatmapFps, totalFrames);
        }
    } catch (err) {
        const heatmapSection = document.getElementById('heatmap-section');
        if (heatmapSection) heatmapSection.classList.add('hidden');
    }
}

// 切换批次时清掉旧批次的帧状态:fps/总帧数在 loadFrameData 返回前保持
// "未知",避免同步换算(深度/手套 tile、热力图、seek)用到脏值。
// 同时把 uuid 立刻指向新批次,手部 tile 挂载时的首帧 URL 不会请求旧批次。
function resetHeatmapFrameState(episodeId) {
    heatmapFps = 0;
    totalFrames = 0;
    lastFrameIdx = -1;
    _lastHandFrameIdx = -1;
    sensorFramesByIndex = new Map();
    currentEpisodeUuid = episodeId || null;
}


function _updateHandLabels() {
    /* Show sensor name hints in hand labels. Hide hand when no sensor data available. */
    const leftContainer = document.querySelector('#heatmap-section .flex-1:first-child');
    const rightContainer = document.querySelector('#heatmap-section .flex-1:last-child');
    const leftLabel = leftContainer ? leftContainer.querySelector('.text-xs') : null;
    const rightLabel = rightContainer ? rightContainer.querySelector('.text-xs') : null;
    const imgL = document.getElementById('hand-left');
    const imgR = document.getElementById('hand-right');

    // 没有手部传感器数据 → 对应手部热力图容器保持隐藏,绝不显示空图
    if (leftContainer && imgL) {
        const baseLeft = (typeof t === 'function') ? t('left_hand') : 'Left Hand';
        if (sensorLeft) {
            leftLabel.textContent = baseLeft + ' (' + sensorLeft + ')';
            leftContainer.style.display = '';
            imgL.style.display = '';
        } else {
            leftLabel.textContent = baseLeft;
            leftContainer.style.display = 'none';
            imgL.style.display = 'none';
        }
    }
    if (rightContainer && imgR) {
        const baseRight = (typeof t === 'function') ? t('right_hand') : 'Right Hand';
        if (sensorRight) {
            rightLabel.textContent = baseRight + ' (' + sensorRight + ')';
            rightContainer.style.display = '';
            imgR.style.display = '';
        } else {
            rightLabel.textContent = baseRight;
            rightContainer.style.display = 'none';
            imgR.style.display = 'none';
        }
    }
}


let _lastHandFrameIdx = -1;

function updateHandImages(frameIndex) {
    if (!currentEpisodeUuid || frameIndex === lastFrameIdx) return;
    // Clamp to last frame to avoid 404 on out-of-range requests
    if (totalFrames > 0) frameIndex = Math.min(frameIndex, totalFrames - 1);
    lastFrameIdx = frameIndex;

    _lastHandFrameIdx = frameIndex;

    // Draw every frame from the already-loaded sensor sequence. No per-frame
    // HTTP request and no intentional frame skipping during playback.
    canvasHandTiles.forEach((entry) => {
        if (entry.hand) {
            const sensor = entry.hand === 'left' ? sensorLeft : sensorRight;
            const values = _sensorValue(frameIndex, sensor);
            if (entry.canvas && values) {
                renderHandSensorCanvas(entry.canvas, values, entry.hand === 'left');
            }
        } else if (entry.sensor) {
            const values = _sensorValue(frameIndex, entry.sensor);
            if (entry.canvas && values) {
                renderGloveHeatmapCanvas(entry.canvas, values);
            }
        }
    });

    const el = document.getElementById('heatmap-frame');
    if (el) el.textContent = frameIndex;
}


function _readFps() {
    /* Single source of truth — always read from player.js to avoid divergence. */
    return typeof getEpisodeFps === 'function' ? getEpisodeFps() : (heatmapFps || 30);
}

function _timeToFrame(currentTime) {
    /* Standard video frame→time inverse: frame N occupies [N/fps, (N+1)/fps).
       The +0.002 epsilon guards against float truncation when the browser seek
       lands a fraction of a millisecond short of the exact frame boundary. */
    return Math.floor(currentTime * _readFps() + 0.002);
}

function _tryConsumeSeekTarget() {
    /* Return the exact target frame from seekToFrame, or null if none pending.
       MUST only be called from seeked/pause — timeupdate must never consume it,
       otherwise timeupdate can steal the target before seeked fires, causing
       the display to jump target→recalculated→back. */
    return typeof consumeSeekTarget === 'function' ? consumeSeekTarget() : null;
}

function startHeatmapSync(plyrInstance) {
    if (!plyrInstance || !currentEpisodeUuid) return;
    heatmapSyncing = true;
    if (heatmapSyncBound.has(plyrInstance)) return;
    heatmapSyncBound.add(plyrInstance);

    plyrInstance.on('timeupdate', () => {
        if (!heatmapSyncing) return;
        // Ignore intermediate seek states while the shared timeline is being
        // scrubbed; the final pointer release performs one committed refresh.
        if (typeof isFrameScrubbing === 'function' && isFrameScrubbing()) return;
        // NEVER consume seek target — timeupdate fires asynchronously and
        // can race ahead of seeked, stealing the target and causing flicker.
        const frameIndex = _timeToFrame(plyrInstance.currentTime);
        updateHandImages(frameIndex);
        if (typeof updateFrameDisplay === 'function') updateFrameDisplay(frameIndex);
    });

    plyrInstance.on('seeked', () => {
        if (!heatmapSyncing) return;
        // Prefer exact target from seekToFrame (zero round-trip error).
        // Fall back to time→frame for user-dragged seeks.
        let frameIndex = _tryConsumeSeekTarget();
        if (frameIndex == null) {
            frameIndex = _timeToFrame(plyrInstance.currentTime);
        }
        updateHandImages(frameIndex);
        if (typeof updateFrameDisplay === 'function') updateFrameDisplay(frameIndex);
    });

    plyrInstance.on('pause', () => {
        if (!heatmapSyncing) return;
        // On pause: prefer pending seek target (stepFrame pauses-then-seeks),
        // otherwise compute exact paused frame so captureStart/End see it.
        let frameIndex = _tryConsumeSeekTarget();
        if (frameIndex == null) {
            frameIndex = _timeToFrame(plyrInstance.currentTime);
        }
        if (typeof updateFrameDisplay === 'function') updateFrameDisplay(frameIndex);
    });
}


function stopHeatmapSync() {
    heatmapSyncing = false;
}

// player.js owns the single rAF frame clock. Registering here keeps glove
// sensor canvases on exactly the same integer frame as RGB, depth and 3D.
if (typeof setOnFrameChange === 'function') {
    setOnFrameChange(frameIndex => {
        if (heatmapSyncing) updateHandImages(frameIndex);
    });
}