/* Plyr video player management */ console.log('[player.js] build 20260911-tactile-episode-color-scale'); const players = {}; // {camera_name: Plyr instance} let currentEpisodeId = null; let currentCameras = []; let _episodeMediaLoadController = null; // Every episode selection gets a new generation. Episode ids normally differ, // but reopening the same episode must also invalidate old async callbacks. let _playbackSessionToken = 0; let workspaceSources = []; let workspaceCameras = []; let currentHasSkeleton = false; let _defaultLayoutApplied = false; // balanced layout (videos + hand heatmaps) applied once // The review canvas always plays the raw camera video. Hand skeletons are // rendered by hand-overlay.js as an SVG layer on top of that video. async function pickVideoUrl(episodeId, camera, preferSkeleton = false) { return `/api/v1/video/${episodeId}/${camera}/preview-stream`; } function getMediaLoadSignal() { return _episodeMediaLoadController ? _episodeMediaLoadController.signal : null; } function getPlaybackSessionToken() { return _playbackSessionToken; } function isCurrentPlaybackSession(episodeId, token) { return currentEpisodeId === episodeId && (token == null || token === _playbackSessionToken); } function _destroyEpisodePlayers() { Object.values(players).forEach(player => { try { player.pause(); } catch (_) {} try { player.destroy(); } catch (_) {} }); Object.keys(players).forEach(key => delete players[key]); } function _cancelEpisodeMediaLoad(clearCanvas = true) { // Invalidate callbacks before aborting requests. Abort is not guaranteed // to prevent a response already resolved from reaching its .then(). _playbackSessionToken += 1; if (_episodeMediaLoadController) { try { _episodeMediaLoadController.abort(); } catch (_) {} _episodeMediaLoadController = null; } clearTimeout(_loadWatchdog); masterPlaying = false; try { pauseAll(); } catch (_) {} if (typeof stopHeatmapSync === 'function') stopHeatmapSync(); if (typeof cancelSegmentPlayback === 'function') cancelSegmentPlayback(); if (typeof disconnectAnnotationSocket === 'function') disconnectAnnotationSocket(); _destroyEpisodePlayers(); currentImageTiles = []; currentDepthPreviewTiles = []; currentHand3dTiles = []; currentSlamTiles = []; currentTactileTiles = []; workspaceGrid = null; workspaceMainRow = null; workspaceHandFooter = null; workspaceHandSlots = { left: null, right: null }; workspaceBottomExtra = null; if (typeof clearHandTiles === 'function') clearHandTiles(); if (typeof destroyAllHandOverlays === 'function') destroyAllHandOverlays(); if (clearCanvas) { const grid = document.getElementById('video-grid'); if (grid) { grid.innerHTML = '
' + '
' + '
Select an episode to review →
'; } } } function clearEpisodeWorkspace() { _cancelEpisodeMediaLoad(true); currentEpisodeId = null; currentCameras = []; currentMediaGroups = null; workspaceSources = []; workspaceCameras = []; workspaceGrid = null; workspaceMainRow = null; workspaceHandFooter = null; workspaceHandSlots = { left: null, right: null }; workspaceBottomExtra = null; if (typeof clearAnnotationsNow === 'function') clearAnnotationsNow(); hand3dData = null; hand3dDataBySource = {}; hand3dFrameCache = { frame: -1, data: null, inflight: -1 }; hand3dFrameCacheBySource = {}; hand3dWindow = { start: -1, end: -1, frames: {}, inflight: false }; hand3dWindowBySource = {}; episodeTotalFrames = 0; episodeFps = 0; frameDataReady = false; const frameLabel = document.getElementById('annotation-timeline-frame'); if (frameLabel) frameLabel.textContent = '0 / 0'; disableFrameControls(); hideVideoLoading(); resetSourceBar(); } function initPlayer(containerId, videoUrl) { const container = document.getElementById(containerId); if (!container) return null; // Remove existing video element container.innerHTML = ` `; const player = new Plyr(`#${containerId}-video`, { // Frame-aligned mode: individual video controls are hidden. A single // master play/pause button + seek bar above the canvas drives every // video at exactly the same frame (see bindMasterSync). controls: [], keyboard: { focused: true, global: true }, }); player.source = { type: 'video', sources: [{ src: videoUrl, type: 'video/mp4' }] }; // Notify when player is ready (metadata loaded, seekable) player.on('ready', () => { if (typeof onPlayerReady === 'function') onPlayerReady(); }); // 禁用双击全屏 — Plyr 3.7.8 无 doubleClick 配置项, // 其 dblclick 绑在 .plyr 容器 bubble 阶段,无法移除。 // 方案:capture 阶段设标记 → 覆写 fullscreen.toggle 拦截。 let dblClicked = false; player.elements.container.addEventListener('dblclick', () => { dblClicked = true; }, true); // capture 阶段,先于 Plyr 的 bubble handler 触发 const _origToggle = player.fullscreen.toggle.bind(player.fullscreen); player.fullscreen.toggle = function () { if (dblClicked) { dblClicked = false; return; } return _origToggle(); }; return player; } /* ── 媒体组(双目兼容)──────────────────────────── 素材清单由后端 /media-groups API 驱动:双目组(stereo_left+stereo_right)、 辅助流、单目、骨骼视频、手套传感器。前端按 kind 渲染,新增素材类型 无需改动前端代码。主要交互是"拖拽素材到画布"自由组合;点击素材项 或顶部标签只是便捷方式。 */ let currentMediaGroups = null; // /media-groups 返回的 {groups, singles, sources} let currentImageTiles = []; // 画布中的图像素材 tile {img, source}(手套热力图/深度图) let workspaceGrid = null; let workspaceMainRow = null; let workspaceHandFooter = null; let workspaceHandSlots = { left: null, right: null }; let workspaceBottomExtra = null; const _gloveSyncBound = new WeakSet(); let _suppressMasterEvents = false; let hand3dData = null; // /hand-3d 接口返回 {frames:[{fi,h0,h1}], baseline_m, ...} let hand3dDataBySource = {}; // source_key -> per-device world metadata let hand3dFrameCache = { frame: -1, data: null, inflight: -1 }; // 按帧懒加载缓存 let hand3dFrameCacheBySource = {}; let _hand3dTileEntry = null; // 3D tile 引用,懒加载回调重绘用 let currentHand3dTiles = []; // 画布中的 3D 骨骼 tile {canvas, source, rotX, rotY, ...} let currentDepthPreviewTiles = []; // raw uint16 depth codes + frontend Canvas let currentTactileTiles = []; // 左右力矩阵合并预览,按统一帧号刷新 let slamData = null; // /slam-trajectory response for current episode let slamFramesByIndex = {}; let slamWindowInflight = null; let currentSlamTiles = []; // 画布中的 SLAM 轨迹 tile let tactileMatrixData = null; // /tactile-matrix-meta response const SLAM_DISPLAY_SMOOTH_ALPHA = 0.35; const SLAM_DISPLAY_JOIN_GAP_FRAMES = 3; const SLAM_DISPLAY_CAMERA_DISTANCE_SCALE = 1.55; const _depthLittleEndian = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1; // Keep one recently-used complete raw-code buffer across episode clicks. The // decoded D435 stream is large, so bound this cache to one typical episode; // the cache contains only uint16 code values, never a JET/RGB image. const _depthFullCache = new Map(); const DEPTH_FULL_CACHE_MAX_BYTES = 420 * 1024 * 1024; function _depthFullCacheKey(source) { return `${currentEpisodeId || ''}:${source?.source_key || ''}`; } function _rememberDepthFull(entry) { if (!entry?.fullCodes || !entry?.source) return; const key = _depthFullCacheKey(entry.source); const bytes = entry.fullCodes.byteLength || 0; _depthFullCache.delete(key); _depthFullCache.set(key, { codes: entry.fullCodes, width: entry.width, height: entry.height, pixelCount: entry.pixelCount, frameBytes: entry.frameBytes, frameCount: entry.frameCount, bytes, }); let total = 0; for (const [cacheKey, value] of [..._depthFullCache.entries()].reverse()) { total += value.bytes || 0; if (total > DEPTH_FULL_CACHE_MAX_BYTES) _depthFullCache.delete(cacheKey); } } function _restoreDepthFull(entry) { if (!entry?.source) return false; const value = _depthFullCache.get(_depthFullCacheKey(entry.source)); if (!value?.codes) return false; _depthFullCache.delete(_depthFullCacheKey(entry.source)); _depthFullCache.set(_depthFullCacheKey(entry.source), value); Object.assign(entry, value, { fullReady: true, initialReady: true }); entry.frames.clear(); return true; } function _decodeDepthCodeBuffer(buffer) { if (_depthLittleEndian) return new Uint16Array(buffer); const view = new DataView(buffer); const codes = new Uint16Array(buffer.byteLength / 2); for (let i = 0; i < codes.length; i++) { codes[i] = view.getUint16(i * 2, true); } return codes; } function _depthCodesUrl(source, frame) { const template = source && (source.depth_codes_url || source.depth_preview_url); return template ? _frameUrl(template, frame) : ''; } // A depth request contains raw uint16 codes, not a colorized image. Larger // sequential windows remove the per-request decoder/HTTP overhead that made // the depth canvas fall behind the RGB master. // Four seconds at the canonical 30 FPS. This is the bounded fallback for // long episodes; unlike the old 60-frame window it leaves enough time for a // remote FFmpeg read to finish before the playhead reaches the boundary. const DEPTH_WINDOW_SIZE = 120; const DEPTH_PREFETCH_MARGIN = 20; // Frames buffered before depth playback may start. With 2x-transported // codes one frame is ~200KB, so 10 frames (~2MB) paints quickly while the // full stream keeps preloading in the background. const DEPTH_INITIAL_BUFFER_FRAMES = 10; // At 848x480 uint16, 600 frames are about 465 MiB. Below this boundary it is // safer to keep the complete decoded window set resident and guarantee zero // mid-play disk/cache reads; above it use the bounded three-window runway. const DEPTH_VERY_LONG_FRAMES = 600; const DEPTH_VERY_LONG_INITIAL_WINDOWS = 3; // Depth is display-only raw code data. Normal clips wait for all windows; // very long clips wait for three windows (~12s at 30 FPS), then continue in // the background without allowing the playhead to cross an empty window. const DEPTH_FULL_PRELOAD = true; // A 848x480 stream costs about 0.78 MiB per raw frame at native scale. With // the 2x-downsampled transport one frame is ~0.19 MiB, so 2000 frames fit // ~400 MiB — the budget that previously limited full preload to short clips // now covers every episode recorded so far. Clips beyond this fall back to // the strict all-windows preload barrier instead. const DEPTH_FULL_PRELOAD_MAX_FRAMES = 2000; // Complete browser-side video blobs make arbitrary progress-bar seeks local // and keep the rendered depth preview on the same seek timeline as RGB. const _browserVideoAssetCache = new Map(); const BROWSER_VIDEO_ASSET_CACHE_MAX = 4; function _browserVideoAssetUrl(source, fallbackUrl) { const url = String(fallbackUrl || ''); if (!url) return ''; const revision = source && ( source.depth_cache_key || source.video_cache_key || source.cache_key); if (!revision || /(?:^|[?&])v=/.test(url)) return url; return `${url}${url.includes('?') ? '&' : '?'}v=${encodeURIComponent(revision)}`; } async function _preloadBrowserVideoAsset(source, fallbackUrl, sessionToken) { const episodeAtRequest = currentEpisodeId; const tokenAtRequest = sessionToken == null ? _playbackSessionToken : sessionToken; const isActive = () => isCurrentPlaybackSession( episodeAtRequest, tokenAtRequest); if (!isActive()) return null; const url = _browserVideoAssetUrl(source, fallbackUrl); if (!url) return null; const key = `${episodeAtRequest}:${source?.kind || 'video'}:` + `${source?.source_key || ''}:${url}`; const existing = _browserVideoAssetCache.get(key); if (existing?.url) return existing.url; if (existing?.promise) return existing.promise; const promise = (async () => { try { const response = await fetch(url, { cache: 'force-cache', signal: getMediaLoadSignal() || undefined, }); if (!response.ok) throw new Error(`video preload failed: ${response.status}`); const blob = await response.blob(); if (!isActive()) return null; const objectUrl = URL.createObjectURL(blob); _browserVideoAssetCache.set(key, { url: objectUrl, bytes: blob.size, ready: true, }); while (_browserVideoAssetCache.size > BROWSER_VIDEO_ASSET_CACHE_MAX) { const oldestKey = _browserVideoAssetCache.keys().next().value; const oldest = _browserVideoAssetCache.get(oldestKey); if (oldest?.url) URL.revokeObjectURL(oldest.url); _browserVideoAssetCache.delete(oldestKey); } return objectUrl; } catch (error) { if (isActive()) { console.warn('[player] complete video preload failed; fallback to range stream', error); } return null; } finally { const state = _browserVideoAssetCache.get(key); if (state?.promise) { if (state.url) _browserVideoAssetCache.set(key, { url: state.url, bytes: state.bytes, ready: true, }); else _browserVideoAssetCache.delete(key); } } })(); _browserVideoAssetCache.set(key, { promise }); return promise; } let _depthPlaybackStall = null; function _depthFrameReady(frame) { return currentDepthPreviewTiles.length > 0 && currentDepthPreviewTiles.every(entry => (entry.fullReady && frame < entry.frameCount) || (!entry.fullReady && entry.fullCodes && frame < entry.frameCount) || entry.frames.has(frame)); } function _stallPlaybackForDepth(frame) { if (_depthPlaybackStall || !masterPlaying) return; _depthPlaybackStall = { episodeId: currentEpisodeId, frame }; masterPlaying = false; _suppressMasterEvents = true; pauseAll(); _suppressMasterEvents = false; refreshPlayButton(false); } function _maybeResumeAfterDepth() { const stall = _depthPlaybackStall; if (!stall || stall.episodeId !== currentEpisodeId || !_depthFrameReady(stall.frame)) return; _depthPlaybackStall = null; currentFrameTarget = stall.frame; seekToFrame(stall.frame); masterPlaying = true; playAll(); } function _depthCodesWindowUrl(source, start, end) { let template = source && source.depth_codes_window_url; // A media-groups response can come from the browser cache and predate // the window URL field. Derive the endpoint from the canonical source // identity so an old response cannot regress playback to one request per // frame. This still transfers raw display codes only; JET stays in the // browser. if (!template && source && source.source_key && currentEpisodeId) { template = `/api/v1/video/${encodeURIComponent(currentEpisodeId)}` + `/depth-codes-window/${encodeURIComponent(source.source_key)}` + '?start_frame={start}&end_frame={end}&scale=2'; } if (template) { let url = template.replace('{start}', Math.max(0, start)) .replace('{end}', Math.max(start, end)); // Server-provided templates may be cached by media-groups without // the downsampled transport; force it so windows stay ~23MB instead // of ~95MB (a stale cache must never regress playback). if (!url.includes('scale=')) { url += `${url.includes('?') ? '&' : '?'}scale=2`; } const revision = source && source.depth_cache_key; return revision ? `${url}${url.includes('?') ? '&' : '?'}v=${encodeURIComponent(revision)}` : url; } return ''; } function _depthCodesFullUrl(source) { let template = source && source.depth_codes_full_url; if (!template && source && source.source_key && currentEpisodeId) { template = `/api/v1/video/${encodeURIComponent(currentEpisodeId)}` + `/depth-codes-full/${encodeURIComponent(source.source_key)}` + '?scale=2'; } if (!template) return ''; // Same guard as the window URL: a cached media-groups payload must not // drop the downsampled transport. if (!template.includes('scale=')) { template += `${template.includes('?') ? '&' : '?'}scale=2`; } const revision = source && source.depth_cache_key; return revision ? `${template}${template.includes('?') ? '&' : '?'}v=${encodeURIComponent(revision)}` : template; } function _fullDepthCodesForFrame(entry, frame) { if (!entry || !entry.fullCodes || frame < 0 || frame >= entry.frameCount) return null; const begin = frame * entry.pixelCount; return entry.fullCodes.subarray(begin, begin + entry.pixelCount); } async function _preloadDepthCodes(entry) { if (!entry || !entry.source || entry.fullReady) return true; if (entry.fullInflight) return entry.fullInflight; if (_restoreDepthFull(entry)) { const codes = _fullDepthCodesForFrame(entry, 0); if (codes) DepthRenderer.render(entry.canvas, codes, entry.width, entry.height); return true; } const episodeAtRequest = currentEpisodeId; entry.initialReady = false; entry.fullInflight = (async () => { try { const response = await fetch(_depthCodesFullUrl(entry.source), { // The URL includes the source file revision. force-cache // therefore gives repeat clicks a disk-cache hit, while a // reprocessed MP4 receives a new URL and cannot show stale // depth codes. cache: 'force-cache', signal: getMediaLoadSignal() || undefined, }); if (!response.ok) { throw new Error(`full depth code request failed: ${response.status}`); } const width = Number(response.headers.get('X-Depth-Width')) || 0; const height = Number(response.headers.get('X-Depth-Height')) || 0; const frameBytes = Number(response.headers.get('X-Depth-Frame-Bytes')) || width * height * 2; const declaredFrames = Number(response.headers.get('X-Depth-Frames')) || Number(entry.source.frame_count) || 0; if (!width || !height || !frameBytes) { throw new Error('invalid depth stream metadata'); } // Allocate one contiguous raw-code buffer. It is intentionally // not a canvas/image buffer: no pseudo-colour data is created or // persisted. let capacity = Math.max(1, declaredFrames) * frameBytes; let raw = new Uint8Array(capacity); let offset = 0; let publishedBuffer = null; let publishedFrames = 0; const publishBufferedFrames = () => { const availableBytes = Math.floor(offset / frameBytes) * frameBytes; const availableFrames = Math.floor(availableBytes / frameBytes); if (availableFrames <= publishedFrames) return availableFrames; if (!publishedBuffer || publishedBuffer.buffer !== raw.buffer) { publishedBuffer = _decodeDepthCodeBuffer(raw.buffer); } entry.width = width; entry.height = height; entry.pixelCount = width * height; entry.frameBytes = frameBytes; entry.frameCount = availableFrames; entry.fullCodes = publishedBuffer; publishedFrames = availableFrames; // Paint only the first available frame during the sequential // preload. Rendering once per network chunk can mean hundreds // of 848x480 texture uploads before playback even starts and // is a major source of the apparent "stuck" loading state. // The completed stream is painted at the current frame below. const active = getActivePlayer(); const displayTarget = active && Number.isFinite(active.currentTime) ? Math.floor(active.currentTime * getEpisodeFps() + 0.002) : (typeof currentFrameTarget === 'number' ? currentFrameTarget : 0); const firstAvailable = Math.min( Math.max(0, displayTarget), availableFrames - 1); const displayCodes = publishedBuffer.subarray( firstAvailable * entry.pixelCount, (firstAvailable + 1) * entry.pixelCount); if (!entry.initialPainted && displayCodes.length === entry.pixelCount) { entry.lastRenderedFrame = firstAvailable; DepthRenderer.render(entry.canvas, displayCodes, width, height); entry.initialPainted = true; } if (!entry.initialReady && availableFrames >= Math.min(DEPTH_INITIAL_BUFFER_FRAMES, declaredFrames || availableFrames)) { entry.initialReady = true; } _maybeResumeAfterDepth(); return availableFrames; }; const consume = (value) => { if (!value || !value.byteLength) return; if (offset + value.byteLength > raw.byteLength) { const next = Math.max(offset + value.byteLength, Math.ceil(raw.byteLength * 1.25)); const grown = new Uint8Array(next); grown.set(raw.subarray(0, offset)); raw = grown; publishedBuffer = null; } raw.set(value, offset); offset += value.byteLength; publishBufferedFrames(); }; const reader = response.body && response.body.getReader ? response.body.getReader() : null; if (!reader) { consume(new Uint8Array(await response.arrayBuffer())); } else { let chunks = 0; while (true) { const part = await reader.read(); if (part.done) break; consume(part.value || new Uint8Array()); // Keep RGB controls/layout responsive during a large // background fill without rAF throttling in hidden tabs. if ((++chunks % 64) === 0) { await new Promise(resolve => setTimeout(resolve, 0)); } } } if (currentEpisodeId !== episodeAtRequest) return false; const usableBytes = Math.floor(offset / frameBytes) * frameBytes; if (usableBytes < frameBytes) throw new Error('empty depth code stream'); const exact = raw.byteLength === usableBytes ? raw.buffer : raw.slice(0, usableBytes).buffer; entry.fullCodes = _decodeDepthCodeBuffer(exact); entry.frameCount = Math.floor(usableBytes / frameBytes); entry.fullReady = true; entry.initialReady = true; entry.frames.clear(); _rememberDepthFull(entry); const active = getActivePlayer(); const displayTarget = active && Number.isFinite(active.currentTime) ? Math.floor(active.currentTime * getEpisodeFps() + 0.002) : (typeof currentFrameTarget === 'number' ? currentFrameTarget : 0); const frameCodes = _fullDepthCodesForFrame(entry, displayTarget); if (frameCodes) DepthRenderer.render(entry.canvas, frameCodes, width, height); _maybeResumeAfterDepth(); return true; } catch (error) { entry.fullError = error; console.warn('[player] background depth preload failed:', error); return false; } })().finally(() => { entry.fullInflight = null; }); // Playback must not start while the full buffer is still arriving: doing // so makes the central clock repeatedly pause/resume at the buffer edge. // A subsequent click can return immediately through _restoreDepthFull(). return entry.fullInflight; } // Normal/medium clips are completely fetched before playback. They are // fetched as 120-frame responses so the browser can persist them in its HTTP // disk cache without creating one huge 900MB Uint16Array. Only a few decoded // windows remain in RAM; later playback requests are local cache hits. async function _preloadDepthWindows(entry) { if (!entry || !entry.source) return false; if (entry.allPreloaded) return true; if (entry.allPreloadInflight) return entry.allPreloadInflight; const sourceFrameCount = Number(entry.source.frame_count) || 0; if (!sourceFrameCount) return false; const episodeAtRequest = currentEpisodeId; entry.allPreloadInflight = (async () => { try { for (let start = 0; start < sourceFrameCount; start += DEPTH_WINDOW_SIZE) { if (currentEpisodeId !== episodeAtRequest) return false; entry.preloadProgress = Math.min(1, start / sourceFrameCount); await _fetchDepthCodes(entry, start); if (!entry.frames.has(start)) { throw new Error(`depth preload window missing: ${start}`); } } if (currentEpisodeId !== episodeAtRequest) return false; // The sequential pass evicts old decoded windows from RAM. Put // frame zero back from the browser cache before releasing the // playback barrier. await _fetchDepthCodes(entry, 0); entry.allPreloaded = true; entry.initialReady = true; entry.preloadProgress = 1; return true; } catch (error) { entry.fullError = error; console.warn('[player] complete depth window preload failed:', error); return false; } })().finally(() => { entry.allPreloadInflight = null; }); return entry.allPreloadInflight; } async function _preloadDepthInitialWindows(entry, windowCount = DEPTH_VERY_LONG_INITIAL_WINDOWS) { if (!entry || !entry.source) return false; if (entry.initialBufferReady) return true; if (entry.initialPreloadInflight) return entry.initialPreloadInflight; const sourceFrameCount = Number(entry.source.frame_count) || 0; if (!sourceFrameCount) return false; const episodeAtRequest = currentEpisodeId; entry.initialPreloadInflight = (async () => { try { const limit = Math.min(sourceFrameCount, Math.max(1, windowCount) * DEPTH_WINDOW_SIZE); for (let start = 0; start < limit; start += DEPTH_WINDOW_SIZE) { if (currentEpisodeId !== episodeAtRequest) return false; await _fetchDepthCodes(entry, start); } entry.initialBufferReady = entry.frames.has(0); return entry.initialBufferReady; } catch (error) { console.warn('[player] initial depth buffer failed:', error); return false; } finally { entry.initialPreloadInflight = null; } })(); return entry.initialPreloadInflight; } async function _fetchDepthCodes(entry, frame) { if (!entry || !entry.source || !entry.canvas || frame < 0) return; if (entry.fullReady) { const codes = _fullDepthCodesForFrame(entry, frame); if (codes) { if (entry.lastRenderedFrame === frame) return; entry.lastRenderedFrame = frame; DepthRenderer.render(entry.canvas, codes, entry.width, entry.height); } return; } if (entry.fullInflight) return; if (entry.frames.has(frame)) { const cached = entry.frames.get(frame); if (entry.lastRenderedFrame !== frame) { entry.lastRenderedFrame = frame; DepthRenderer.render(entry.canvas, cached.codes, cached.width, cached.height); } return; } if (entry.windowInflight && frame >= entry.windowInflight.start && frame <= entry.windowInflight.end) return; entry.pendingFrame = frame; if (entry.windowInflight != null) return; const target = entry.pendingFrame; entry.pendingFrame = null; const sourceFrameCount = Number(entry.source.frame_count) || 0; // Once the source metadata is known, never ask the backend for a window // beyond the actual clip. Besides avoiding noisy 404s, this prevents a // failed tail request from looking like a playback stall on short clips. if (sourceFrameCount && target >= sourceFrameCount) return; const start = Math.floor(target / DEPTH_WINDOW_SIZE) * DEPTH_WINDOW_SIZE; if (sourceFrameCount && start >= sourceFrameCount) return; const end = sourceFrameCount ? Math.min(start + DEPTH_WINDOW_SIZE - 1, sourceFrameCount - 1) : start + DEPTH_WINDOW_SIZE - 1; entry.windowInflight = { start, end }; const episodeAtRequest = currentEpisodeId; try { const windowUrl = _depthCodesWindowUrl(entry.source, start, end); const response = await fetch(windowUrl || _depthCodesUrl(entry.source, target), { cache: 'force-cache', signal: getMediaLoadSignal() || undefined, }); if (!response.ok) throw new Error(`depth code request failed: ${response.status}`); const width = Number(response.headers.get('X-Depth-Width')) || 0; const height = Number(response.headers.get('X-Depth-Height')) || 0; const buffer = await response.arrayBuffer(); if (currentEpisodeId !== episodeAtRequest || buffer.byteLength % 2) return; // The API declares uint16 little-endian. Common browsers can use a // zero-copy view; the fallback keeps the wire contract strict on an // unusual big-endian host. const codes = _decodeDepthCodeBuffer(buffer); const expected = width * height; const count = Number(response.headers.get('X-Depth-Frames')) || 1; const first = Number(response.headers.get('X-Depth-Start')) || start; if (!width || !height || codes.length < expected) return; for (let offset = 0; offset < count; offset++) { const begin = offset * expected; const frameCodes = codes.subarray(begin, begin + expected); if (frameCodes.length !== expected) break; entry.frames.set(first + offset, { codes: frameCodes, width, height }); } if (entry.keepAllFrames) { entry.initialReady = true; } else { // Keep the current window plus several future windows. A FIFO // eviction could remove frame 0 while the long-clip prefetcher // is still filling frame 360, causing an unnecessary pause. const activeFrame = Math.max(0, Number(currentFrameTarget) || 0); const minFrame = Math.max(0, activeFrame - DEPTH_WINDOW_SIZE); const maxFrame = activeFrame + DEPTH_WINDOW_SIZE * 5 - 1; for (const frameKey of entry.frames.keys()) { if (frameKey < minFrame || frameKey > maxFrame) { entry.frames.delete(frameKey); } } } if (!entry.keepAllFrames && expected && entry.source.frame_count) { const residentBytes = Number(entry.source.frame_count) * expected * 2; entry.keepAllFrames = residentBytes <= 420 * 1024 * 1024; } // Playback may have advanced while the request was in flight. Render // the actual master frame if it is in this window; a prefetch must // never jump the depth canvas ahead to the first prefetched frame. const active = getActivePlayer(); const displayTarget = active && !active.paused && Number.isFinite(active.currentTime) ? Math.floor(active.currentTime * getEpisodeFps() + 0.002) : (typeof currentFrameTarget === 'number' ? currentFrameTarget : target); const cached = entry.frames.get(displayTarget) || (displayTarget === target ? entry.frames.get(target) : null); if (cached) { if (entry.lastRenderedFrame === displayTarget) return; entry.lastRenderedFrame = displayTarget; DepthRenderer.render(entry.canvas, cached.codes, cached.width, cached.height); } _maybeResumeAfterDepth(); } catch (_) { // A stale request or a temporary seek race must not break playback. } finally { entry.windowInflight = null; if (entry.pendingFrame != null && currentEpisodeId === episodeAtRequest) { _fetchDepthCodes(entry, entry.pendingFrame); } } } function _queueDepthWindow(entry, frame) { // keepAllFrames only controls eviction (RAM residency), NOT fetching: // with the 2x-downsampled transport, long clips now qualify as // keepAllFrames and must still be prefetched by this queue — otherwise // every 120-frame boundary stalls. if (!entry || entry.fullReady || frame < 0) return; const start = Math.floor(frame / DEPTH_WINDOW_SIZE) * DEPTH_WINDOW_SIZE; const count = Number(entry.source?.frame_count) || 0; if (count && start >= count) return; if (entry.frames.has(start) || (entry.windowInflight && entry.windowInflight.start === start) || entry.windowQueue.includes(start)) return; entry.windowQueue.push(start); _pumpDepthWindowQueue(entry); } async function _pumpDepthWindowQueue(entry) { if (!entry || entry.windowPump) return; entry.windowPump = true; try { while (entry.windowQueue.length && currentEpisodeId) { if (entry.windowInflight) { await new Promise(resolve => setTimeout(resolve, 20)); continue; } const start = entry.windowQueue.shift(); await _fetchDepthCodes(entry, start); } } finally { entry.windowPump = false; } } function _refreshDepthTilesAt(frame) { if (!Number.isFinite(frame) || frame < 0) return; currentDepthPreviewTiles.forEach(entry => { _fetchDepthCodes(entry, frame); if (entry.fullReady) return; // Fetch the next sequential window before the current one is empty. // The server-side reader stays sequential, so this is substantially // cheaper than waiting for a random frame request at the boundary. const windowStart = Math.floor(frame / DEPTH_WINDOW_SIZE) * DEPTH_WINDOW_SIZE; const prefetchFrame = windowStart + DEPTH_WINDOW_SIZE; const sourceFrameCount = Number(entry.source.frame_count) || 0; if (sourceFrameCount > DEPTH_VERY_LONG_FRAMES) { // Maintain at least three future windows in the queue. They are // fetched one at a time so the server decoder remains sequential. for (let i = 1; i <= 3; i++) { const queuedFrame = windowStart + i * DEPTH_WINDOW_SIZE; if (!sourceFrameCount || queuedFrame < sourceFrameCount) { _queueDepthWindow(entry, queuedFrame); } } } else if (frame >= windowStart + DEPTH_PREFETCH_MARGIN * 2 && (!sourceFrameCount || prefetchFrame < sourceFrameCount) && !entry.frames.has(prefetchFrame) && !entry.windowInflight) { _fetchDepthCodes(entry, prefetchFrame); } }); } // 3D 世界坐标窗口缓存:窗口拉取(±HAND3D_WINDOW 帧),播放期零逐帧请求 let hand3dWindow = { start: -1, end: -1, frames: {}, inflight: false }; let hand3dWindowBySource = {}; const HAND3D_WINDOW = 250; const HAND3D_INITIAL_BUFFER_FRAMES = 450; const HAND3D_MAX_CACHED_FRAMES = 2500; // 3D 世界窗口在接近边界前提前请求下一段。严格同步模式下不使用 // 上一帧补位,因此必须在真正越过边界前完成预取。 const HAND3D_PREFETCH_MARGIN = 60; // Review episodes are normally short (hundreds/thousands of frames). Load // those point sequences once before playback so a window boundary can never // leave the spatial canvas one or more frames behind RGB. Very long episodes // keep the bounded-window fallback to avoid turning a browser tab into a // multi-hundred-megabyte JSON allocation. const HAND3D_FULL_PRELOAD = true; const HAND3D_FULL_PRELOAD_MAX_FRAMES = 5000; const HAND3D_FULL_CACHE_MAX_EPISODES = 2; const _hand3dFullCache = new Map(); // episode/source -> {frames, count} // A workflow rerun replaces the canonical parquet. Drop only that episode's // browser-side point/depth buffers so a later click cannot show the old // derived result; unrelated episode caches remain warm. window.invalidateEpisodePlaybackCache = function (episodeId) { const prefix = `${episodeId || ''}:`; for (const key of _depthFullCache.keys()) { if (key.startsWith(prefix)) _depthFullCache.delete(key); } for (const key of _hand3dFullCache.keys()) { if (key.startsWith(prefix)) _hand3dFullCache.delete(key); } if (currentEpisodeId === episodeId) { slamData = null; slamFramesByIndex = {}; slamWindowInflight = null; } if (window.EgoMediaCache && episodeId) { window.EgoMediaCache.removeEpisode(episodeId).catch(() => {}); } }; // 世界坐标使用稳定的空间锚点和安全视距。世界坐标中的点不能根据 // 当前帧的单手/双手数量重新居中或缩放,否则检测状态变化会被误显示 // 成相机跳动。2m 同时为固定网格保留足够的可视边界,用户仍可滚轮缩放。 const HAND3D_WORLD_DEFAULT_CENTER = [0, 0, 0.7]; const HAND3D_WORLD_DEFAULT_DISTANCE = 2.0; // World-preview sizing is screen-space only. The target is intentionally // shared by depth and RGB-estimated 3D so different coordinate sources have // the same visual scale without changing parquet/API coordinates. const HAND3D_DISPLAY_SPAN_RATIO = 0.18; const HAND3D_DISPLAY_SPAN_MIN_PX = 70; const HAND3D_DISPLAY_SPAN_MAX_PX = 130; const HAND3D_DISPLAY_SCALE_MIN = 0.70; const HAND3D_DISPLAY_SCALE_MAX = 3.0; // RGB/深度 3D 坐标统一为 X 右、Y 上、Z 向前;网格放在手的下方。 const HAND3D_WORLD_DEFAULT_GRID_Y = -0.25; // Root-anchored preview: MediaPipe landmark 0 (Wrist) is placed on the // ground grid at two stable left/right display anchors. The stored camera // coordinates remain untouched; this is only a presentation transform. // Fixed preview positions. Keep enough horizontal clearance so the two // skeletons cannot visually merge on the shared 3D canvas. const HAND3D_ROOT_ANCHOR_X = 0.22; const HAND3D_ROOT_ANCHOR_LIFT = 0.01; // 默认正面视角与左侧 RGB 画面一致;拖拽仍可改变俯仰角查看空间深度。 const HAND3D_DEFAULT_ELEVATION = 0; const HAND3D_MAX_CAMERA_DISTANCE = 6.0; // RGB-only 3D is relative/estimated rather than metric. Its current image // model is smaller than a physical hand, so normalize only the preview // geometry to a stable hand span comparable with the depth view. const RGB_ESTIMATED_DISPLAY_HAND_SPAN = 0.14; const RGB_ESTIMATED_DISPLAY_SCALE_MIN = 1.0; const RGB_ESTIMATED_DISPLAY_SCALE_MAX = 5.0; function _hand3dCacheFor(sourceKey) { const source = sourceKey || 'default'; if (!hand3dWindowBySource[source]) { hand3dWindowBySource[source] = { start: -1, end: -1, frames: {}, inflight: false, inflightPromise: null, }; } return hand3dWindowBySource[source]; } function _hand3dFrameCacheFor(sourceKey) { const source = sourceKey || 'default'; if (!hand3dFrameCacheBySource[source]) { hand3dFrameCacheBySource[source] = { frame: -1, data: null, inflight: -1 }; } return hand3dFrameCacheBySource[source]; } function _hand3dCached(frame, sourceKey) { const cache = _hand3dCacheFor(sourceKey); const fr = cache.frames[frame]; return fr ? { h0: fr.h0 || null, h1: fr.h1 || null } : null; } function _hand3dArtifactRevision(sourceKey) { const source = sourceKey || 'default'; const data = hand3dDataBySource[source] || hand3dData; return String(data && data.artifact_revision || 'unknown'); } function _hand3dFullCacheKey(sourceKey, artifactRevision = null) { const revision = artifactRevision == null ? _hand3dArtifactRevision(sourceKey) : String(artifactRevision); return `${currentEpisodeId || ''}:${sourceKey || 'default'}:${revision}`; } function _restoreHand3DFull(cache, sourceKey, artifactRevision) { const saved = _hand3dFullCache.get( _hand3dFullCacheKey(sourceKey, artifactRevision)); if (!saved || !saved.frames || String(saved.artifactRevision || 'unknown') !== String(artifactRevision)) { return false; } cache.frames = saved.frames; cache.start = 0; cache.end = Math.max(0, Number(saved.count || 0) - 1); cache.fullReady = true; cache.fullArtifactRevision = String(artifactRevision); return true; } function _rememberHand3DFull(cache, sourceKey, count, artifactRevision) { const key = _hand3dFullCacheKey(sourceKey, artifactRevision); _hand3dFullCache.delete(key); _hand3dFullCache.set(key, { frames: cache.frames, count, artifactRevision: String(artifactRevision), }); while (_hand3dFullCache.size > HAND3D_FULL_CACHE_MAX_EPISODES) { _hand3dFullCache.delete(_hand3dFullCache.keys().next().value); } } async function fetchHand3DFull(sourceKey, sessionToken = _playbackSessionToken) { const source = sourceKey || 'default'; const episodeAtRequest = currentEpisodeId; const tokenAtRequest = sessionToken; const isActive = () => isCurrentPlaybackSession( episodeAtRequest, tokenAtRequest); if (!isActive()) return false; const cache = _hand3dCacheFor(source); const data = hand3dDataBySource[source] || hand3dData; const artifactRevision = String( data && data.artifact_revision || 'unknown'); if (cache.fullReady && String(cache.fullArtifactRevision || 'unknown') === artifactRevision) { return true; } if (cache.fullReady) cache.fullReady = false; if (_restoreHand3DFull(cache, source, artifactRevision)) return true; if (cache.fullInflight) return cache.fullInflight; const count = Number(data && data.count || 0); if (!count || count > HAND3D_FULL_PRELOAD_MAX_FRAMES) return false; const signal = getMediaLoadSignal(); const persistentKey = `${episodeAtRequest}:hand3d:${source}:${count}:${artifactRevision}`; cache.fullInflight = (async () => { try { if (window.EgoMediaCache) { const saved = await window.EgoMediaCache.get(persistentKey); if (!isActive()) return false; const value = saved && saved.value; if (value && Array.isArray(value.frames) && Number(value.count || 0) === count && String(value.artifact_revision || 'unknown') === artifactRevision) { const frames = {}; value.frames.forEach(fr => { frames[fr.f] = fr; }); if (!isActive()) return false; cache.frames = frames; cache.start = 0; cache.end = Math.max(0, count - 1); cache.fullReady = true; cache.fullArtifactRevision = artifactRevision; _rememberHand3DFull( cache, source, count, artifactRevision); return true; } } const res = await fetch( `/api/v1/video/${episodeAtRequest}/hand-3d?source_key=` + `${encodeURIComponent(source)}&start_frame=0&end_frame=${count - 1}`, signal ? { signal } : {}); if (!res.ok || !isActive()) return false; const payload = await res.json(); if (!isActive()) return false; if (String(payload && payload.artifact_revision || 'unknown') !== artifactRevision) return false; const frames = {}; (payload.frames || []).forEach(fr => { frames[fr.f] = fr; }); if (!isActive()) return false; cache.frames = frames; cache.start = 0; cache.end = Math.max(0, count - 1); cache.fullReady = true; cache.fullArtifactRevision = artifactRevision; _rememberHand3DFull( cache, source, count, artifactRevision); if (window.EgoMediaCache && isActive()) { window.EgoMediaCache.put(persistentKey, { frames: payload.frames || [], count, artifact_revision: artifactRevision, }); } return true; } catch (_) { return false; } finally { cache.fullInflight = null; } })(); return cache.fullInflight; } function _hand3dPlaybackFrame(fallback = 0) { const player = getActivePlayer(); if (player && !player.paused && !player.ended && Number.isFinite(player.currentTime)) { return Math.max(0, Math.floor( player.currentTime * getEpisodeFps() + 0.002)); } return typeof currentFrameTarget === 'number' ? currentFrameTarget : fallback; } function _mergeHand3DWindow(cache, frames, start, end) { const merged = cache.frames || {}; (frames || []).forEach(fr => { merged[fr.f] = fr; }); cache.frames = merged; cache.start = cache.start < 0 ? start : Math.min(cache.start, start); cache.end = Math.max(cache.end, end); const keys = Object.keys(merged).map(Number).sort((a, b) => a - b); while (keys.length > HAND3D_MAX_CACHED_FRAMES) { delete merged[keys.shift()]; } const remaining = Object.keys(merged).map(Number); cache.start = remaining.length ? Math.min(...remaining) : -1; cache.end = remaining.length ? Math.max(...remaining) : -1; } async function fetchHand3DWindow(center, sourceKey) { const source = sourceKey || 'default'; const cache = _hand3dCacheFor(source); if (cache.fullReady) return true; if (cache.inflight) return cache.inflightPromise || false; // 该源没有任何有效手部帧(如深度抬升全空的批次)→ 不发起 // 20s+ 的空拉取;标记 fullReady 后后续窗口请求全部短路。 // 仅当后端确实统计过(counted=true)且为 0 才跳过 —— 统计失败的 // 默认 0 不能误杀有数据的批次。 const handData = hand3dDataBySource[source] || hand3dData; const handOption = handData && Array.isArray(handData.available_sources) ? handData.available_sources.find( item => (item.source_key || 'default') === source) : null; const handMeta = handOption || handData; if (handMeta && handMeta.counted === true && Number(handMeta.valid_hand_frames ?? -1) === 0) { cache.fullReady = true; return false; } const insideWindow = center >= cache.start && center <= cache.end; const insidePrefetchRange = center >= cache.start + HAND3D_PREFETCH_MARGIN && center <= cache.end - HAND3D_PREFETCH_MARGIN; if (insideWindow && insidePrefetchRange) return; const start = Math.max(0, center - HAND3D_WINDOW); const end = center + HAND3D_WINDOW; const episodeAtRequest = currentEpisodeId; const sessionToken = _playbackSessionToken; const signal = getMediaLoadSignal(); cache.inflight = true; cache.inflightPromise = (async () => { try { const persistentKey = `${episodeAtRequest}:hand3d:${source}:${start}:${end}`; if (window.EgoMediaCache) { const saved = await window.EgoMediaCache.get(persistentKey); if (!isCurrentPlaybackSession(episodeAtRequest, sessionToken)) return false; const value = saved && saved.value; if (value && Array.isArray(value.frames)) { _mergeHand3DWindow(cache, value.frames, Number(value.start ?? start), Number(value.end ?? end)); for (const entry of currentHand3dTiles) { if ((entry.source.source_key || 'default') === source) { renderHand3DTile(entry, _hand3dPlaybackFrame()); } } return true; } } const res = await fetch( `/api/v1/video/${episodeAtRequest}/hand-3d?source_key=${encodeURIComponent(source)}&start_frame=${start}&end_frame=${end}`, signal ? { signal } : {}); if (!isCurrentPlaybackSession(episodeAtRequest, sessionToken)) return false; if (!res.ok) return false; const data = await res.json(); if (!isCurrentPlaybackSession(episodeAtRequest, sessionToken)) return false; _mergeHand3DWindow(cache, data.frames, data.start, data.end); if (window.EgoMediaCache) { window.EgoMediaCache.put(persistentKey, { frames: data.frames || [], start: data.start, end: data.end, }); } // 只在完整响应成功后替换窗口,避免半更新状态被渲染。 for (const entry of currentHand3dTiles) { if ((entry.source.source_key || 'default') === source) { renderHand3DTile(entry, _hand3dPlaybackFrame()); } } return true; } catch (e) { /* 拉取失败保持旧缓存 */ return false; } })(); try { return await cache.inflightPromise; } finally { cache.inflight = false; cache.inflightPromise = null; } } async function _ensureHand3DInitialWindow(sourceKey) { const source = sourceKey || 'default'; const data = hand3dDataBySource[source] || hand3dData; if (!data || !data.hasPreview) return true; const cache = _hand3dCacheFor(source); if (HAND3D_FULL_PRELOAD) { const loaded = await fetchHand3DFull(source); if (loaded) return true; } if (cache.start <= 0 && cache.end >= 0) return true; const count = Number(data.count || 0); const initialCenter = count > HAND3D_INITIAL_BUFFER_FRAMES ? Math.min(Math.floor(HAND3D_INITIAL_BUFFER_FRAMES / 2), count - 1) : 0; const loaded = Boolean(await fetchHand3DWindow(initialCenter, source)); if (loaded && count > HAND3D_INITIAL_BUFFER_FRAMES) { // Keep three future windows queued without blocking the first frame. // They merge into the bounded 3D cache and are usually served from // IndexedDB after the first visit. (async () => { let center = initialCenter + HAND3D_WINDOW * 2; for (let i = 0; i < 3 && center < count; i++, center += HAND3D_WINDOW * 2) { await fetchHand3DWindow(center, source); } })(); } return loaded; } // The hand endpoint returns a display-only compact crop of its 800x220 // canvas. The raw sensor values are not changed by this presentation crop. const HAND_PREVIEW_ASPECT = 800 / 200; const HAND_FOOTER_HEIGHT = 144; const HAND_FOOTER_WITH_EXTRA_HEIGHT = 300; function groupedSourceKey(source) { if (source.kind === 'stereo_group') return `stereo:${source.id}`; return `${source.kind}:${source.source_key || source.camera || source.id}`; } function groupedSourceDomId(source) { return groupedSourceKey(source).replace(/[^a-zA-Z0-9_-]/g, '_'); } function groupedSourceLabel(source) { if (source.kind === 'skeleton') return `${source.label || source.source_key} · ${t('skeleton_suffix')}`; if (source.kind === 'glove') return source.label || `${source.source_key} · ${t('glove_suffix')}`; if (source.kind === 'depth') return source.label || `${source.source_key} · ${t('depth_suffix')}`; if (source.kind === 'slam_trajectory') return source.label || 'SLAM'; if (source.kind === 'tactile_force_left') return 'Left'; if (source.kind === 'tactile_force_right') return 'Right'; if (source.kind === 'stereo_group') return `${source.label} · ${t('stereo_pair')}`; return source.label || source.source_key || t('video_suffix'); } function _isBottomSource(source) { return ['hand', 'depth', 'glove', 'tactile_force_left', 'tactile_force_right'].includes(source.kind); } function groupedSource(kind, source_key, meta) { return { kind, source_key, ...(meta || {}) }; } /* 素材 → 画布条目:双目组展开为左右目并排,其余一对一 */ function expandWorkspaceItems(source) { if (source.kind === 'stereo_group') { return (source.members || []).map(m => ({ kind: 'video', source_key: m.source_key, label: m.label, stream_url: m.stream_url, video_cache_key: m.video_cache_key, frame_count: m.frame_count, fps: m.fps, })); } return [{ ...source }]; } function _sourceIsPlaced(source) { const placed = new Set(workspaceSources.map(item => groupedSourceKey(item))); if (source.kind === 'stereo_group') { const members = source.members || []; return members.length > 0 && members.every(member => placed.has(`video:${member.source_key}`)); } return placed.has(groupedSourceKey(source)); } function _syncWorkspacePlayers() { const active = getActivePlayer(); if (!active) return; Object.values(players).forEach(player => bindMasterSync(player)); bindGloveSync(active); bindFrameDrift(active); if (typeof startHeatmapSync === 'function') startHeatmapSync(active); if (masterPlaying) { Object.values(players).forEach(player => { // play() returns a promise: a try/catch cannot see its rejection, // and the load chain right after this call runs pauseAll(), which // aborts the pending play → uncaught AbortError in the console. try { const result = player.play(); if (result && typeof result.catch === 'function') { result.catch(() => {}); } } catch (e) {} }); } } function addGroupedSource(source) { const added = []; expandWorkspaceItems(source).forEach(item => { const key = groupedSourceKey(item); if (!workspaceSources.some(existing => groupedSourceKey(existing) === key)) { workspaceSources.push(item); added.push(item); } }); renderSourceBar(); renderPreviewVideoSources(); if (!added.length) return; // When an episode is already mounted, append only the new source. This // keeps existing Plyr instances and playback position untouched. if (workspaceMainRow && workspaceGrid) { const addedDepth = added.some(item => item.kind === 'depth'); const resumeAfterDepth = addedDepth && masterPlaying; if (addedDepth) { // Depth is a display-only stream and must be buffered before it // joins an already playing workspace; otherwise RGB advances // while the first depth window is still decoding. masterPlaying = false; _suppressMasterEvents = true; pauseAll(); _suppressMasterEvents = false; disableFrameControls(); } const pending = added.map(item => _appendWorkspaceSource(item)); _refreshWorkspaceLayout(); Promise.all(pending).then(() => { if (addedDepth && currentEpisodeId) { enableFrameControls(); if (resumeAfterDepth) { masterPlaying = true; playAll(); } } else { _syncWorkspacePlayers(); } }); } else { renderGroupedWorkspace(); } } function removeGroupedSource(source) { const key = groupedSourceKey(source); if (typeof destroyHandOverlay === 'function') destroyHandOverlay(key); const tile = workspaceGrid ? workspaceGrid.querySelector(`[data-source-key="${CSS.escape(key)}"]`) : null; const image = tile ? tile.querySelector('img,canvas') : null; const player = players[key]; const wasPlaying = masterPlaying; if (player) { // Plyr can emit pause while destroy() is tearing down the removed // player. Suppress that transient event so remaining players do not // get paused as a side effect of removing one tile. _suppressMasterEvents = true; try { player.destroy(); } catch (e) {} finally { _suppressMasterEvents = false; } delete players[key]; } if (image && source.kind === 'hand' && typeof unregisterHandTile === 'function') { unregisterHandTile(image); } currentImageTiles = currentImageTiles.filter(entry => groupedSourceKey(entry.source) !== key); currentDepthPreviewTiles = currentDepthPreviewTiles.filter(entry => groupedSourceKey(entry.source) !== key); currentSlamTiles = currentSlamTiles.filter(entry => groupedSourceKey(entry.source) !== key); currentTactileTiles = currentTactileTiles.filter(entry => groupedSourceKey(entry.source) !== key); workspaceSources = workspaceSources.filter(item => groupedSourceKey(item) !== key); renderSourceBar(); renderPreviewVideoSources(); if (!workspaceGrid || !workspaceMainRow) { renderGroupedWorkspace(); if (window.__episodeOpen === true) { const previewOptions = document.getElementById('preview-options'); if (previewOptions) previewOptions.classList.remove('hidden'); } return; } if (tile) tile.remove(); _refreshWorkspaceLayout(); const activeAfterRemove = getActivePlayer(); if (!activeAfterRemove) { if (typeof stopHeatmapSync === 'function') stopHeatmapSync(); masterPlaying = false; refreshPlayButton(false); } else { masterPlaying = wasPlaying; _syncWorkspacePlayers(); } // 删除深度/3D/手套 tile 只影响工作区素材,不应把批次详情的 // Preview Options 一并隐藏。 if (window.__episodeOpen === true) { const previewOptions = document.getElementById('preview-options'); if (previewOptions) previewOptions.classList.remove('hidden'); } } /* ── Source button bar ─────────────────────────────────────── Every asset (stereo group / single cameras / aux streams / glove heatmaps / depth / hand-pressure tiles) appears as a button above the video canvas. Click or drag to add it to the canvas; the button disappears once placed and returns when the tile is removed. No palette drawer, no tabs. */ function _sourceBarButton(label, icon, key, source, accent, parent) { const bar = parent || document.getElementById('source-bar'); if (!bar || _sourceIsPlaced(source)) return; const btn = document.createElement('button'); btn.type = 'button'; btn.draggable = true; btn.dataset.sourceKey = key; btn.title = label; btn.className = 'text-xs px-2 py-1 rounded border cursor-grab flex-shrink-0 flex items-center gap-1 ' + (accent ? 'border-cyan-700/60 bg-gray-800 text-gray-300 hover:border-cyan-500 hover:text-white' : 'border-gray-700 bg-gray-800 text-gray-300 hover:border-blue-500 hover:text-white'); btn.innerHTML = ` ${label}`; let dragging = false; btn.addEventListener('click', () => { if (!dragging) addGroupedSource(source); }); btn.addEventListener('dragstart', event => { dragging = true; event.dataTransfer.setData('application/x-egodata-source', JSON.stringify(_serializeSource(source))); event.dataTransfer.effectAllowed = 'copy'; btn.classList.add('opacity-50'); }); btn.addEventListener('dragend', () => { btn.classList.remove('opacity-50'); setTimeout(() => { dragging = false; }, 0); }); bar.appendChild(btn); } function _videoSource(item) { return { kind: 'video', source_key: item.source_key, label: item.label, stream_url: item.stream_url, frame_count: item.frame_count, fps: item.fps, }; } function addSource(source, key) { addGroupedSource(source); } function addHandTile(hand) { addSource({ kind: 'hand', hand, source_key: `hand_${hand}`, label: t(hand === 'left' ? 'left_hand' : 'right_hand'), }, `hand:hand_${hand}`); } /* Swap two video tiles in place — DOM-level reorder only, so the Plyr players are NOT rebuilt and playback continues uninterrupted. */ function reorderTiles(fromKey, toKey, container) { const fromIdx = workspaceSources.findIndex(s => groupedSourceKey(s) === fromKey); const toIdx = workspaceSources.findIndex(s => groupedSourceKey(s) === toKey); if (fromIdx < 0 || toIdx < 0 || fromIdx === toIdx) return; const [item] = workspaceSources.splice(fromIdx, 1); workspaceSources.splice(toIdx, 0, item); const fromTile = container.querySelector(`[data-source-key="${CSS.escape(fromKey)}"]`); const toTile = container.querySelector(`[data-source-key="${CSS.escape(toKey)}"]`); if (fromTile && toTile) { container.insertBefore(fromTile, fromIdx < toIdx ? toTile.nextSibling : toTile); } } function renderSourceBar() { const bar = document.getElementById('source-bar'); if (!bar) return; bar.innerHTML = ''; // All preview sources are now controlled by Preview Options. Keep this // function as a compatibility no-op because older code still calls it. bar.classList.add('hidden'); } function _previewVideoEntries() { const entries = []; const add = item => { if (!item || !item.source_key || entries.some(e => e.source_key === item.source_key)) return; entries.push(_videoSource(item)); }; if (currentMediaGroups) { (currentMediaGroups.groups || []).forEach(group => { (group.members || []).forEach(add); (group.aux || []).forEach(add); }); (currentMediaGroups.singles || []).forEach(add); } if (!entries.length) { currentCameras.forEach(camera => add({ source_key: camera, label: camera, stream_url: `/api/v1/video/${currentEpisodeId}/${camera}/preview-stream`, })); } return entries; } function _previewDeviceLabel(source) { const raw = String(source.device_name || source.source_key || source.label || '').trim(); // D405_depth_rgb / D435_depth_rgb are RGB stream slots; retain the // physical device token (D405_depth / D435_depth) shown to the reviewer. const device = raw.replace(/_rgb$/i, '').replace(/_color$/i, ''); const role = source.label && source.label !== 'Primary' && source.label !== device ? ` · ${source.label}` : ''; return `${device || raw}${role}`; } function renderPreviewVideoSources() { const row = document.getElementById('po-video-row'); const list = document.getElementById('po-video-list'); const all = document.getElementById('po-video-all'); if (!row || !list) return; const entries = _previewVideoEntries(); list.innerHTML = ''; row.classList.toggle('hidden', entries.length === 0); row.classList.toggle('block', entries.length > 0); if (!entries.length) return; const isPlaced = source => workspaceSources.some(item => groupedSourceKey(item) === groupedSourceKey(source)); entries.forEach(source => { const label = document.createElement('label'); label.className = 'flex items-center gap-2 px-1 py-1 rounded hover:bg-gray-700/70 cursor-pointer'; const input = document.createElement('input'); input.type = 'checkbox'; input.className = 'w-3 h-3 accent-cyan-500'; input.checked = isPlaced(source); input.dataset.sourceKey = source.source_key; const text = document.createElement('span'); text.className = 'truncate text-[11px] text-gray-300'; text.title = source.source_key; text.textContent = _previewDeviceLabel(source); input.addEventListener('change', () => { if (input.checked) addGroupedSource(source); else removeGroupedSource(source); }); label.append(input, text); list.appendChild(label); }); if (all) { const selected = entries.filter(isPlaced).length; all.textContent = selected === entries.length ? 'None' : 'All'; all.onclick = () => { const shouldAdd = selected !== entries.length; entries.forEach(source => { if (shouldAdd && !isPlaced(source)) addGroupedSource(source); if (!shouldAdd && isPlaced(source)) removeGroupedSource(source); }); renderPreviewVideoSources(); }; } } /* ── Source bar refresh ── */ function resetSourceBar() { renderSourceBar(); } function _serializeSource(source) { if (source.kind === 'stereo_group') { return { kind: 'stereo_group', id: source.id, label: source.label, members: source.members }; } return { kind: source.kind, source_key: source.source_key, label: source.label, hand: source.hand, stream_url: source.stream_url, heatmap_url: source.heatmap_url, depth_url: source.depth_url, depth_preview_url: source.depth_preview_url, depth_codes_url: source.depth_codes_url, raw_depth_url: source.raw_depth_url, depth_cache_key: source.depth_cache_key, video_cache_key: source.video_cache_key, // Optional grayscale clock stream; never used as depth pixels. depth_video_url: source.depth_video_url, missing_frames: source.missing_frames, depth_scale: source.depth_scale, frame_count: source.frame_count, fps: source.fps, // 并排拼接标记(3D 拼接等宽画面)→ 画布 tile 跨两列 stereo_side_by_side: source.stereo_side_by_side, }; } /* 画布渲染:video/skeleton → Plyr 播放器,glove/depth → 图像 tile */ function _frameUrl(urlTemplate, frameIndex) { return urlTemplate.replace('{frame}', Math.max(0, frameIndex)); } function _tactileForceAt(frame, side) { const map = tactileMatrixData?._forceByFrame; const value = map && map[String(Math.max(0, Number(frame) || 0))]?.[side]; if (!Array.isArray(value) || value.length < 3 || !value.slice(0, 3).every(item => Number.isFinite(Number(item)))) { return null; } const fx = Number(value[0]); const fy = Number(value[1]); const fz = Number(value[2]); return { fx, fy, fz, magnitude: Math.hypot(fx, fy, fz) }; } function _updateTactileForceTile(entry, frame) { if (!entry) return; const force = _tactileForceAt(frame, entry.source.side); if (entry.forceValue) { if (!force) { entry.forceValue.textContent = 'Fx — · Fy — · Fz —'; entry.forceValue.title = 'Force data unavailable'; } else { entry.forceValue.textContent = `Fx ${Math.round(force.fx).toLocaleString()} · ` + `Fy ${Math.round(force.fy).toLocaleString()} · ` + `Fz ${Math.round(force.fz).toLocaleString()}`; entry.forceValue.title = `fx ${force.fx.toFixed(1)} · fy ${force.fy.toFixed(1)} · ` + `fz ${force.fz.toFixed(1)} mN`; } } _renderTactileForceCurve(entry, frame); } // Fx/Fy/Fz history for one side across the whole episode, with a playhead at // the current frame. Colours follow the acquisition-side ForceCurveWidget. const TACTILE_FORCE_COLORS = ['#4fc3f7', '#aed581', '#ff8a65']; function _forceSeriesIndex(frame) { const series = tactileMatrixData?.force_series; if (!series || !Array.isArray(series.frames)) return null; if (!tactileMatrixData._frameIndex) { const map = new Map(); series.frames.forEach((value, index) => map.set(Number(value), index)); tactileMatrixData._frameIndex = map; } const index = tactileMatrixData._frameIndex.get(Math.round(Number(frame))); return index === undefined ? null : index; } function _renderTactileForceCurve(entry, frame) { const canvas = entry?.curveCanvas; if (!canvas) return; const rect = canvas.getBoundingClientRect(); const width = Math.max(1, rect.width), height = Math.max(1, rect.height); if (rect.width < 2 || rect.height < 2) { // The tile is built before it is attached to the workspace, so the // first call can run before layout. Retry once the layout exists. if (!entry._curveRetry) { entry._curveRetry = true; requestAnimationFrame(() => { entry._curveRetry = false; _renderTactileForceCurve(entry, frame); }); } return; } const dpr = window.devicePixelRatio || 1; if (canvas.width !== Math.round(width * dpr) || canvas.height !== Math.round(height * dpr)) { canvas.width = Math.round(width * dpr); canvas.height = Math.round(height * dpr); } const ctx = canvas.getContext('2d'); ctx.setTransform(dpr, 0, 0, dpr, 0, 0); ctx.clearRect(0, 0, width, height); const series = tactileMatrixData?.force_series?.[entry.source.side]; if (!Array.isArray(series) || !series.length) return; const pad = { l: 4, r: 4, t: 8, b: 4 }; const innerW = width - pad.l - pad.r; const innerH = height - pad.t - pad.b; // Symmetric auto range: p99.5 of |value| rounded up to 500 mN, so the // fixed +/-3000 mN range of the acquisition widget does not clip our data. if (!entry._curveRange) { const samples = []; series.forEach(value => { if (!value) return; for (let axis = 0; axis < 3; axis++) { const item = Math.abs(Number(value[axis])); if (Number.isFinite(item)) samples.push(item); } }); samples.sort((a, b) => a - b); const p995 = samples.length ? samples[Math.min(samples.length - 1, Math.floor(samples.length * 0.995))] : 0; entry._curveRange = Math.max(500, Math.ceil(p995 / 500) * 500); } const range = entry._curveRange; const mid = pad.t + innerH / 2; const Y = value => mid - (value / range) * (innerH / 2) * 0.92; const X = index => pad.l + (series.length <= 1 ? 0 : (index / (series.length - 1)) * innerW); ctx.lineWidth = 1; ctx.strokeStyle = 'rgba(148,163,184,0.16)'; [0.5, -0.5].forEach(frac => { const y = Y(frac * range); ctx.beginPath(); ctx.moveTo(pad.l, y); ctx.lineTo(width - pad.r, y); ctx.stroke(); }); ctx.strokeStyle = 'rgba(148,163,184,0.38)'; ctx.beginPath(); ctx.moveTo(pad.l, mid); ctx.lineTo(width - pad.r, mid); ctx.stroke(); for (let axis = 0; axis < 3; axis++) { ctx.strokeStyle = TACTILE_FORCE_COLORS[axis]; ctx.lineWidth = 1.2; ctx.beginPath(); let started = false; series.forEach((value, index) => { const item = value ? Number(value[axis]) : NaN; if (!Number.isFinite(item)) { started = false; return; } const x = X(index), y = Y(item); if (!started) { ctx.moveTo(x, y); started = true; } else { ctx.lineTo(x, y); } }); ctx.stroke(); } const headIndex = _forceSeriesIndex(frame); if (headIndex !== null) { const head = X(headIndex); ctx.strokeStyle = 'rgba(248,250,252,0.85)'; ctx.lineWidth = 1; ctx.beginPath(); ctx.moveTo(head, pad.t - 3); ctx.lineTo(head, height - pad.b); ctx.stroke(); } } function _createTactileWebGLRenderer(canvas) { const gl = canvas.getContext('webgl2', { alpha: false, antialias: false, premultipliedAlpha: false, }); if (!gl) return null; const vertexSource = `#version 300 es in vec2 a_position; out vec2 v_uv; void main() { v_uv = a_position * 0.5 + 0.5; gl_Position = vec4(a_position, 0.0, 1.0); }`; const fragmentSource = `#version 300 es precision highp float; uniform sampler2D u_force_field; uniform float u_zoom; uniform vec2 u_pan; in vec2 v_uv; out vec4 out_color; vec3 jet(float x) { return clamp(vec3( 1.5 - abs(4.0 * x - 3.0), 1.5 - abs(4.0 * x - 2.0), 1.5 - abs(4.0 * x - 1.0) ), 0.0, 1.0); } void main() { // Zoom/pan happen in screen-uv space; the field itself is 250x250. vec2 uv = (v_uv - 0.5) / u_zoom + 0.5 + u_pan; if (uv.x < 0.0 || uv.x > 1.0 || uv.y < 0.0 || uv.y > 1.0) { out_color = vec4(0.0039, 0.0039, 0.502, 1.0); return; } // Matrix row 0 is the top of the sensor; WebGL texture v=0 is // the bottom, so flip only the sampling coordinate. The field is // uploaded already normalised to 0..1 (R8), so no rescaling here. float value = texture(u_force_field, vec2(uv.x, 1.0 - uv.y)).r; out_color = vec4(jet(clamp(value, 0.0, 1.0)), 1.0); }`; const compile = (type, source) => { const shader = gl.createShader(type); gl.shaderSource(shader, source); gl.compileShader(shader); if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { console.warn('[player] tactile WebGL shader failed', gl.getShaderInfoLog(shader)); gl.deleteShader(shader); return null; } return shader; }; const vertex = compile(gl.VERTEX_SHADER, vertexSource); const fragment = compile(gl.FRAGMENT_SHADER, fragmentSource); if (!vertex || !fragment) return null; const program = gl.createProgram(); gl.attachShader(program, vertex); gl.attachShader(program, fragment); gl.linkProgram(program); gl.deleteShader(vertex); gl.deleteShader(fragment); if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { console.warn('[player] tactile WebGL program failed', gl.getProgramInfoLog(program)); gl.deleteProgram(program); return null; } const position = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, position); gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([ -1, -1, 1, -1, -1, 1, 1, 1, ]), gl.STATIC_DRAW); const texture = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, texture); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); return { gl, program, position, texture, positionLocation: gl.getAttribLocation(program, 'a_position'), fieldLocation: gl.getUniformLocation(program, 'u_force_field'), zoomLocation: gl.getUniformLocation(program, 'u_zoom'), panLocation: gl.getUniformLocation(program, 'u_pan'), field: null, zoom: 1, pan: [0, 0], filter: null, render(field = null, min = 0, max = 1) { if (field) { // R32F float textures are not filterable without // OES_texture_float_linear; without it every sample returns 0 // and the whole tile renders as JET(0) - a flat dark blue that // is indistinguishable from the background. Normalising to // 8-bit here keeps the texture in the always-filterable R8 // format, so LINEAR/NEAREST both work on every driver. if (field instanceof Float32Array) { const lo = Number.isFinite(Number(min)) ? Number(min) : 0; const hi = Number.isFinite(Number(max)) ? Number(max) : 1; const span = Math.max(hi - lo, 1e-6); const bytes = new Uint8Array(field.length); for (let i = 0; i < field.length; i++) { const value = (field[i] - lo) / span; bytes[i] = value <= 0 ? 0 : value >= 1 ? 255 : Math.round(value * 255); } this.field = bytes; } else { this.field = field; } } const rect = canvas.getBoundingClientRect(); const dpr = window.devicePixelRatio || 1; const width = Math.max(1, Math.round(rect.width * dpr)); const height = Math.max(1, Math.round(rect.height * dpr)); if (canvas.width !== width || canvas.height !== height) { canvas.width = width; canvas.height = height; } // 传感器是 250x250 的方格阵列,必须等比显示 —— 铺满 canvas 会按 // tile 的宽高比横向拉伸(实测宽窗约 2.9:1),格子就不再是正方形, // 接触位置的形状判断会失真。取短边做居中正方形 viewport,多出来的 // 两侧留背景色(gl.clear 不受 viewport 限制,会整块清成底色)。 const side = Math.min(width, height); gl.viewport(Math.round((width - side) / 2), Math.round((height - side) / 2), side, side); gl.clearColor(0.0039, 0.0039, 0.502, 1.0); gl.clear(gl.COLOR_BUFFER_BIT); if (!this.field) return; gl.useProgram(program); gl.bindBuffer(gl.ARRAY_BUFFER, position); gl.enableVertexAttribArray(this.positionLocation); gl.vertexAttribPointer(this.positionLocation, 2, gl.FLOAT, false, 0, 0); gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, texture); // One texel is one sensor cell. Sample NEAREST as soon as a cell // covers at least one screen pixel (crisp cells, like the // acquisition demo), and LINEAR only when shrinking - a NEAREST // minification would drop isolated contact cells entirely. const texelsPerPixel = (width / 250) * (this.zoom || 1); const filter = texelsPerPixel >= 1 ? gl.NEAREST : gl.LINEAR; if (this.filter !== filter) { gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, filter); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, filter); this.filter = filter; } gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false); gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1); // 250-wide R8 rows gl.texImage2D(gl.TEXTURE_2D, 0, gl.R8, 250, 250, 0, gl.RED, gl.UNSIGNED_BYTE, this.field); gl.uniform1i(this.fieldLocation, 0); gl.uniform1f(this.zoomLocation, this.zoom || 1); gl.uniform2f(this.panLocation, this.pan ? this.pan[0] : 0, this.pan ? this.pan[1] : 0); gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4); }, }; } // The tactile force matrix tile is display-only. Zoom/pan were removed on // purpose: the review page is read-only evidence, and a stray wheel or drag // could leave the matrix zoomed into a few cells and look like a sensor // defect. The whole 250x250 field stays fixed, one texture texel per cell. // The renderer keeps supporting zoom/pan (uniforms u_zoom/u_pan) so the // acquisition-side demo viewer and any future opt-in can reuse it; here the // view is pinned to identity. function _resetTactileView(renderer) { if (!renderer) return; renderer.zoom = 1; renderer.pan = [0, 0]; if (renderer.canvas) renderer.canvas.style.cursor = 'default'; renderer.render(); } // One chunk = 96 frames of 250x250 R8 = 6 MB per request. The old code issued // one 250 KB request per frame per side (15 MB/s during playback), which // saturated the browser's socket buffers (net::ERR_NO_BUFFER_SPACE). Chunks // are cached in memory, so playback and scrubbing need no network at all. const TACTILE_CHUNK_FRAMES = 96; const TACTILE_FIELD_STRIDE = 250 * 250; function _tactileChunkKey(frame) { return Math.floor(Math.max(0, frame) / TACTILE_CHUNK_FRAMES) * TACTILE_CHUNK_FRAMES; } function _tactileFrameField(entry, frame) { const chunk = entry.chunks.get(_tactileChunkKey(frame)); if (!chunk) return null; const index = Math.round(Number(frame)) - chunk.first; if (index < 0 || index >= chunk.count) return null; return chunk.bytes.subarray( index * TACTILE_FIELD_STRIDE, (index + 1) * TACTILE_FIELD_STRIDE); } function _fetchTactileChunk(entry, chunkStart) { if (!entry?.renderer || entry.chunks.has(chunkStart) || entry.chunkInflight.has(chunkStart)) return; entry.chunkInflight.add(chunkStart); const episodeAtRequest = currentEpisodeId; const tokenAtRequest = _playbackSessionToken; const signal = getMediaLoadSignal(); const url = `/api/v1/video/${currentEpisodeId}/tactile-matrix-range` + `?side=${entry.source.side}&start=${chunkStart}` + `&limit=${TACTILE_CHUNK_FRAMES}&v=20260911-tactile-episode-color-scale`; const options = { credentials: 'same-origin', cache: 'force-cache' }; if (signal) options.signal = signal; fetch(url, options).then(response => { if (!response.ok) throw new Error(`tactile range failed: ${response.status}`); const first = Number(response.headers.get('X-Force-First')); const count = Number(response.headers.get('X-Force-Frames')); return response.arrayBuffer().then(buffer => ({ buffer, first, count })); }).then(({ buffer, first, count }) => { entry.chunkInflight.delete(chunkStart); if (!isCurrentPlaybackSession(episodeAtRequest, tokenAtRequest) || !entry.canvas.isConnected) return; if (!Number.isFinite(first) || !Number.isFinite(count) || count <= 0 || buffer.byteLength < count * TACTILE_FIELD_STRIDE) return; entry.chunks.set(chunkStart, { first, count, bytes: new Uint8Array(buffer, 0, count * TACTILE_FIELD_STRIDE), }); // Paint the frame the player is showing if it just arrived. if (entry.pendingFrame !== null && entry.pendingFrame !== undefined) { const field = _tactileFrameField(entry, entry.pendingFrame); if (field) { entry.renderer.render(field); entry.loadedFrame = entry.pendingFrame; } } // Keep one chunk of look-ahead warm so playback never waits on the // network; the server caches each chunk, so repeats are cheap. const next = chunkStart + TACTILE_CHUNK_FRAMES; if (!entry.frameCount || next < entry.frameCount) { _fetchTactileChunk(entry, next); } }).catch(error => { entry.chunkInflight.delete(chunkStart); if (error?.name !== 'AbortError' && isCurrentPlaybackSession(episodeAtRequest, tokenAtRequest)) { console.warn('[player] tactile chunk unavailable', error); } }); } function _loadTactileWebGLFrame(entry, frame) { if (!entry?.canvas || !entry.renderer || !entry.source) return; const targetFrame = Math.max(0, Math.round(Number(frame) || 0)); entry.pendingFrame = targetFrame; const field = _tactileFrameField(entry, targetFrame); if (field) { entry.renderer.render(field); entry.loadedFrame = targetFrame; const next = _tactileChunkKey(targetFrame) + TACTILE_CHUNK_FRAMES; if (!entry.frameCount || next < entry.frameCount) { _fetchTactileChunk(entry, next); } return; } _fetchTactileChunk(entry, _tactileChunkKey(targetFrame)); } async function mountGroupedSource(source, tile) { const episodeAtMount = currentEpisodeId; const sessionTokenAtMount = _playbackSessionToken; if (!tile || currentEpisodeId !== episodeAtMount) return; if (source.kind === 'slam_trajectory') { const canvas = document.createElement('canvas'); canvas.className = 'w-full h-full block bg-black'; tile.appendChild(canvas); const entry = { canvas, source, rotY: 0, rotX: 0.35, zoom: 1, pan: [0, 0, 0], dragging: false, dragMode: '', lastX: 0, lastY: 0, fit: null, _lastCam: null, }; currentSlamTiles.push(entry); initSlamDrag(entry); if (!isCurrentPlaybackSession(episodeAtMount, sessionTokenAtMount) || !tile.isConnected) return; renderSlamTile(entry, 0); return; } if (source.kind === 'hand3d_world') { // 3D 手部世界坐标交互窗口:canvas 渲染 /hand-3d 逐帧数据 // (worker 只产 parquet,前端实时投影;拖拽旋转 + 滚轮缩放) const canvas = document.createElement('canvas'); canvas.className = 'w-full h-full block bg-black'; tile.appendChild(canvas); const entry = { canvas, source, rotY: 0, rotX: 0, zoom: 1, pan: [0, 0, 0], dragging: false, dragMode: '', lastX: 0, lastY: 0, _lastCam: null, _hasCommittedFrame: false }; currentHand3dTiles.push(entry); initHand3dDrag(entry); // Do not reveal the spatial canvas before its initial frame window is // available. RGB/2D and 3D then enter the workspace together. await _ensureHand3DInitialWindow(source.source_key || 'default'); if (!isCurrentPlaybackSession(episodeAtMount, sessionTokenAtMount) || !tile.isConnected) return; renderHand3DTile(entry, 0); return; } if (source.kind === 'tactile_force_left' || source.kind === 'tactile_force_right') { // One tile per side: pressure field on top, its Fx/Fy/Fz history // underneath. The acquisition-side GripperSideWidget stacks the same // way (heatmap above, force curve below). const holder = document.createElement('div'); holder.className = 'w-full h-full min-h-0 flex flex-col'; const matrixBox = document.createElement('div'); matrixBox.className = 'relative flex-1 min-h-0'; matrixBox.style.backgroundColor = 'rgb(1, 1, 128)'; const canvas = document.createElement('canvas'); canvas.className = 'w-full h-full block'; canvas.setAttribute('aria-label', source.side === 'right' ? 'Right force matrix' : 'Left force matrix'); matrixBox.appendChild(canvas); const forceValue = document.createElement('div'); forceValue.className = 'absolute z-10 rounded text-white pointer-events-none'; forceValue.style.cssText = 'top:6px;left:6px;padding:2px 6px;font-size:11px;' + 'background:rgba(0,0,0,.6);font-variant-numeric:tabular-nums;'; forceValue.textContent = 'Fx — · Fy — · Fz —'; matrixBox.appendChild(forceValue); const curveBox = document.createElement('div'); curveBox.className = 'relative shrink-0'; curveBox.style.cssText = 'height:92px;background:#0b1220;' + 'border-top:1px solid rgba(51,65,85,.5);'; const curveCanvas = document.createElement('canvas'); curveCanvas.className = 'w-full h-full block'; curveBox.appendChild(curveCanvas); const curveCaption = document.createElement('div'); curveCaption.className = 'absolute pointer-events-none'; curveCaption.style.cssText = 'top:2px;left:6px;font-size:10px;color:#64748b;'; curveCaption.textContent = t('force_unit_mn'); curveBox.appendChild(curveCaption); holder.appendChild(matrixBox); holder.appendChild(curveBox); tile.appendChild(holder); const renderer = _createTactileWebGLRenderer(canvas); const entry = { canvas, renderer, source, lastFrame: -1, forceValue, curveCanvas, loadedFrame: -1, pendingFrame: null, chunks: new Map(), chunkInflight: new Set(), frameCount: Number(tactileMatrixData?.frame_count) || 0 }; _resetTactileView(renderer); canvas.title = t('force_field_title'); currentTactileTiles.push(entry); const frame = typeof currentFrameTarget === 'number' ? currentFrameTarget : 0; _updateTactileForceTile(entry, frame); if (renderer) { renderer.render(); _loadTactileWebGLFrame(entry, frame); } else { forceValue.textContent = 'WebGL2 unavailable'; } return; } if (source.kind === 'depth' && source.depth_video_url) { // Depth previews are browser-decodable 8-bit Turbo MP4s generated // once per source from the canonical 12-bit stream (identical // clamp/scale/colormap math as the former canvas renderer). Play // them as native videos: instant open, native seek, and frame sync // through the same master-drift loop as every other camera. const playbackUrl = await _preloadBrowserVideoAsset( source, source.depth_video_url, sessionTokenAtMount); if (!isCurrentPlaybackSession(episodeAtMount, sessionTokenAtMount) || !tile.isConnected) return; const holder = document.createElement('div'); holder.id = `player-container-${groupedSourceDomId(source)}`; holder.className = 'bg-black rounded overflow-hidden w-full h-full min-h-0 relative'; tile.appendChild(holder); const player = initPlayer( holder.id, playbackUrl || _browserVideoAssetUrl( source, source.depth_video_url)); if (player) players[groupedSourceKey(source)] = player; return; } if (source.kind === 'depth' || source.kind === 'hand') { const img = source.kind === 'hand' ? document.createElement('canvas') : document.createElement('img'); // Hand-pressure panels are generated on a wide 800x220 canvas. Keep // that native ratio instead of stretching the heatmap vertically. img.className = source.kind === 'hand' ? 'w-full block bg-gray-900 hand-heatmap-image' : 'w-full h-full object-contain bg-black'; if (source.kind === 'hand') { img.style.objectFit = 'contain'; img.style.width = '100%'; img.style.height = 'auto'; img.style.maxHeight = '100%'; img.style.aspectRatio = String(HAND_PREVIEW_ASPECT); tile.style.alignItems = 'center'; tile.style.justifyContent = 'center'; } if (source.kind !== 'hand') img.alt = source.label || 'Sensor / Depth'; tile.appendChild(img); const entry = { img, source }; currentImageTiles.push(entry); const frame = typeof currentFrameTarget === 'number' ? currentFrameTarget : 0; if (source.kind === 'hand') { // Hand-pressure tile: use the full frames-data payload and draw // locally; this avoids one PNG request per video frame. if (typeof registerHandTile === 'function') registerHandTile(img, source.hand); } else if (source.kind === 'depth') { // Depth streams are mounted by the Canvas/code path above. } else { img.src = _frameUrl( source.heatmap_url || source.depth_preview_url || source.depth_url, frame, ); } return; } const holder = document.createElement('div'); holder.id = `player-container-${groupedSourceDomId(source)}`; holder.className = 'bg-black rounded overflow-hidden w-full h-full min-h-0'; tile.appendChild(holder); const sourceUrl = source.stream_url || await pickVideoUrl(episodeAtMount, source.source_key); const playbackUrl = await _preloadBrowserVideoAsset( source, sourceUrl, sessionTokenAtMount); if (!isCurrentPlaybackSession(episodeAtMount, sessionTokenAtMount) || !tile.isConnected) return; const player = initPlayer(holder.id, playbackUrl || sourceUrl); if (player) players[groupedSourceKey(source)] = player; // 手部骨骼 SVG 叠加层(只加在原始视频上;skeleton/渲染 tile 不加) // The first keypoint window is part of the shared initial-frame barrier. if (typeof initHandOverlay === 'function') { await initHandOverlay(source, tile, holder); } } /* 图像素材(手套热力图/深度图)跟随画布内任一 player 的播放进度 */ function bindGloveSync(player) { if (!player || _gloveSyncBound.has(player)) return; _gloveSyncBound.add(player); player.on('timeupdate', () => { if (isFrameScrubbing()) return; // 暂停态下的 seek(逐帧步进)也走这里:timeupdate 会在 seek 后触发一次 _refreshImageTilesAt(Math.floor((player.currentTime || 0) * getEpisodeFps() + 0.002)); }); } let _lastImageTileFrame = -1; function _refreshImageTilesAt(frame) { if (!currentImageTiles.length && !currentDepthPreviewTiles.length && !currentTactileTiles.length) return; _refreshDepthTilesAt(frame); if (currentImageTiles.length && frame !== _lastImageTileFrame) { _lastImageTileFrame = frame; currentImageTiles.forEach((entry) => { const { img, source } = entry; // Hand/glove tiles are driven by heatmap.js from the preloaded // frames-data payload; only template-based non-depth tiles are // refreshed here. let url; if (source.kind === 'depth' || source.kind === 'hand' || source.kind === 'glove') { return; } else { url = source.heatmap_url || source.depth_preview_url || source.depth_url; } if (url) img.src = _frameUrl(url, frame); }); } currentTactileTiles.forEach(entry => { if (!entry.canvas || entry.lastFrame === frame) return; entry.lastFrame = frame; _updateTactileForceTile(entry, frame); _loadTactileWebGLFrame(entry, frame); }); } /* 播放期逐帧刷新图像素材。 不能用 rAF 作为视频时钟: rAF 跟显示器刷新(通常 60Hz),而视频可能是 30FPS/变速/掉帧,于是深度画布会在同一个视频帧上重复上传,或先于视频 画面切到下一帧。优先使用 HTMLVideoElement.requestVideoFrameCallback, 让 RGB 视频真正送入 compositor 的那一帧驱动所有派生图层。回调内部 只发布一个整数帧号,深度 renderer 采用 latest-only,不创建伪彩色文件。 旧浏览器才走低频 rAF fallback。 */ let _imageRafStarted = false; let _centralLastFrame = -1; let _presentationClockToken = 0; let _presentationClockVideo = null; let _presentationClockHandle = null; function _rawVideoElement(player) { if (!player) return null; return player.media || player.elements?.media || player.elements?.container?.querySelector('video') || null; } function _publishPresentedFrame(video, metadata) { if (!video || video.paused || video.ended) return; const mediaTime = Number.isFinite(metadata?.mediaTime) ? metadata.mediaTime : (video.currentTime || 0); const frame = _clampFrameIndex(Math.floor( mediaTime * getEpisodeFps() + 0.002)); // currentTime assignment is a seek and can make a healthy slave stutter. // Only correct a real multi-frame drift; initial play/seek alignment is // handled by playAll/seekToFrame. const maxDrift = Math.max(0.08, 2.5 / getEpisodeFps()); Object.values(players).forEach(player => { const slaveVideo = _rawVideoElement(player); if (!slaveVideo || slaveVideo === video) return; if (Math.abs((slaveVideo.currentTime || 0) - mediaTime) > maxDrift) { try { slaveVideo.currentTime = mediaTime; } catch (_) {} } }); // 深度帧未到时暂停整组播放器,避免 RGB 继续前进而深度画面停在旧帧。 if (currentDepthPreviewTiles.length && !_depthFrameReady(frame)) { _stallPlaybackForDepth(frame); _refreshDepthTilesAt(frame); return; } if (frame !== _centralLastFrame) { _centralLastFrame = frame; updateFrameDisplay(frame); } _refreshImageTilesAt(frame); } function _schedulePresentationFrame(video, token) { if (token !== _presentationClockToken || video !== _presentationClockVideo || video.paused || video.ended) return; if (typeof video.requestVideoFrameCallback === 'function') { _presentationClockHandle = video.requestVideoFrameCallback((_, metadata) => { _presentationClockHandle = null; if (token !== _presentationClockToken || video !== _presentationClockVideo) return; _publishPresentedFrame(video, metadata); _schedulePresentationFrame(video, token); }); return; } // Compatibility fallback: one rAF still publishes at most one integer // video frame, so a 30FPS source is never rendered twice per frame. _presentationClockHandle = requestAnimationFrame(() => { _presentationClockHandle = null; if (token !== _presentationClockToken || video !== _presentationClockVideo) return; _publishPresentedFrame(video, null); _schedulePresentationFrame(video, token); }); } function _ensurePresentationClock() { const video = _rawVideoElement(getActivePlayer()); if (!video) return; if (video !== _presentationClockVideo) { _presentationClockToken += 1; _presentationClockVideo = video; _presentationClockHandle = null; } if (!video.paused && !video.ended && !_presentationClockHandle) { _schedulePresentationFrame(video, _presentationClockToken); } } function _startImageTileRaf() { if (_imageRafStarted) return; _imageRafStarted = true; window.__egodataCentralFrameClock = true; // This is only a cheap watchdog for play/pause and player replacement. // Actual frame work is scheduled by requestVideoFrameCallback above. const watchdog = () => { _ensurePresentationClock(); setTimeout(watchdog, 100); }; watchdog(); } function _setPlayerToCurrentFrame(player) { if (!player) return; const apply = () => { const time = Math.max(0, currentFrameTarget / (episodeFps || 30)); try { player.currentTime = time; } catch (e) {} }; // Set immediately when metadata is already available, and set again on // ready for newly-added videos whose media element is not seekable yet. apply(); if (typeof player.on === 'function' && typeof player.off === 'function') { const onReady = () => { player.off('ready', onReady); apply(); }; player.on('ready', onReady); } } function _restorePlayersToCurrentFrame() { Object.values(players).forEach(_setPlayerToCurrentFrame); } /* 并排拼接类源(3D 拼接等):内容是左右两路视频拼成的宽画面 (2560×800),tile 应横跨两列显示,而不是挤在单格里。 hand3d 是当前的 3D 拼接成品;未来任何带 stereo_side_by_side 标记的双目骨骼拼接视频自动同样处理。 */ function _isWideSource(source) { return Boolean(source) && ( source.kind === 'hand3d' || source.stereo_side_by_side === true ); } function _createWorkspaceTile(source, movable, reorderContainer) { const tile = document.createElement('div'); tile.className = 'relative bg-black rounded overflow-hidden min-w-0 min-h-0 flex flex-col'; tile.dataset.sourceKey = groupedSourceKey(source); tile.style.minWidth = '0'; tile.style.width = '100%'; tile.style.height = '100%'; if (_isWideSource(source)) { // 横跨两列:与"两路原图各占一格"的总宽度一致,宽画面不再被压成 // 单格(3D 拼接在 2 列布局下原本只占左侧格,右侧空置) tile.style.gridColumn = 'span 2'; } if (source.kind === 'hand') { // Match the generated 800x220 hand canvas so the card itself does not // introduce letterbox space after the image returns to contain mode. tile.classList.remove('bg-black'); tile.classList.add('bg-gray-900'); tile.style.height = 'auto'; tile.style.maxHeight = '100%'; tile.style.aspectRatio = String(HAND_PREVIEW_ASPECT); tile.style.alignSelf = 'center'; } if (!source.kind.startsWith('tactile_force_')) { const label = document.createElement('span'); label.className = 'absolute z-10 top-1 left-1 bg-black/70 text-gray-200 text-[10px] px-1.5 py-0.5 rounded'; label.textContent = groupedSourceLabel(source); tile.appendChild(label); } // \u5361\u7247\u53f3\u4e0a\u89d2\u7684 \u00d7 \u5df2\u79fb\u9664:\u5f53\u5361\u7247\u6bd4\u753b\u9762\u5bbd(\u5982 D435_depth \u4e24\u4fa7\u7559\u9ed1\u8fb9) // \u65f6\u5b83\u6302\u5728\u5361\u7247\u53f3\u4e0a\u89d2\u800c\u975e\u753b\u9762\u53f3\u4e0a\u89d2,\u79bb\u5185\u5bb9\u5f88\u8fdc\u3001\u770b\u7740\u50cf\u98d8\u5728\u5916\u9762\u3002 // \u589e\u5220\u4e00\u8def\u753b\u9762\u7684\u5165\u53e3\u4ecd\u5728\u6e90\u5217\u8868\u7684\u590d\u9009\u6846\u91cc(\u89c1 renderPreviewVideoSources // \u7684 input[type=checkbox] \u4e0e All/None \u6309\u94ae),\u529f\u80fd\u6ca1\u6709\u88ab\u5835\u6b7b\u3002 if (!movable) return tile; tile.draggable = true; tile.addEventListener('dragstart', event => { event.dataTransfer.setData('application/x-egodata-reorder', groupedSourceKey(source)); event.dataTransfer.effectAllowed = 'move'; tile.classList.add('opacity-50'); }); tile.addEventListener('dragend', () => tile.classList.remove('opacity-50', 'ring-2', 'ring-blue-500')); tile.addEventListener('dragover', event => { const types = event.dataTransfer.types || []; if (types.includes('application/x-egodata-reorder') || types.includes('application/x-egodata-source')) { event.preventDefault(); event.stopPropagation(); event.dataTransfer.dropEffect = types.includes('application/x-egodata-reorder') ? 'move' : 'copy'; tile.classList.add('ring-2', 'ring-blue-500'); } }); tile.addEventListener('dragleave', () => tile.classList.remove('ring-2', 'ring-blue-500')); tile.addEventListener('drop', event => { event.preventDefault(); event.stopPropagation(); tile.classList.remove('ring-2', 'ring-blue-500'); const fromKey = event.dataTransfer.getData('application/x-egodata-reorder'); if (fromKey && fromKey !== groupedSourceKey(source)) { reorderTiles(fromKey, groupedSourceKey(source), reorderContainer); return; } const raw = event.dataTransfer.getData('application/x-egodata-source'); if (raw) { try { addGroupedSource(JSON.parse(raw)); } catch (e) {} } }); return tile; } function _createHandFooter() { const footer = document.createElement('div'); footer.className = 'min-w-0 min-h-0 overflow-hidden grid grid-cols-2 gap-3'; footer.dataset.role = 'hand-footer'; const extra = document.createElement('div'); extra.className = 'min-w-0 min-h-0 overflow-hidden grid gap-3'; extra.dataset.role = 'bottom-extra'; extra.style.gridColumn = '1 / -1'; workspaceBottomExtra = extra; footer.appendChild(extra); ['left', 'right'].forEach(hand => { const slot = document.createElement('div'); slot.className = 'min-w-0 min-h-0 overflow-hidden bg-gray-900 rounded'; slot.dataset.handSlot = hand; footer.appendChild(slot); workspaceHandSlots[hand] = slot; }); return footer; } function _refreshWorkspaceLayout() { if (!workspaceGrid || !workspaceMainRow) return; const mainSources = workspaceSources.filter(source => !_isBottomSource(source)); const bottomSources = workspaceSources.filter(_isBottomSource); const handSources = workspaceSources.filter(source => source.kind === 'hand'); const columns = mainSources.length <= 1 ? 1 : 2; // 行数按"占用的格子数"算:并排拼接类(3D 拼接等)跨两列,占 2 格; // 单列布局下(只有一个源)跨列无意义,按 1 格计 const slots = mainSources.reduce( (n, source) => n + (columns >= 2 && _isWideSource(source) ? 2 : 1), 0); const rows = Math.max(1, Math.ceil(slots / columns)); const hasBottomExtras = bottomSources.some(source => source.kind !== 'hand'); const bottomHeight = handSources.length ? (hasBottomExtras ? HAND_FOOTER_WITH_EXTRA_HEIGHT : HAND_FOOTER_HEIGHT) : 260; workspaceGrid.style.gridTemplateRows = bottomSources.length ? `minmax(0, 1fr) ${bottomHeight}px` : 'minmax(0, 1fr)'; if (!bottomSources.length && workspaceHandFooter) { workspaceHandFooter.remove(); workspaceHandFooter = null; workspaceHandSlots = { left: null, right: null }; workspaceBottomExtra = null; } workspaceMainRow.style.gridTemplateColumns = `repeat(${columns}, minmax(0, 1fr))`; workspaceMainRow.style.gridTemplateRows = `repeat(${rows}, minmax(0, 1fr))`; if (!mainSources.length) { if (!workspaceMainRow.querySelector('[data-role="empty-workspace"]')) { workspaceMainRow.innerHTML = `
${t('drag_here')}
`; } } else { const empty = workspaceMainRow.querySelector('[data-role="empty-workspace"]'); if (empty) empty.remove(); mainSources.forEach(source => { const tile = workspaceMainRow.querySelector( `[data-source-key="${CSS.escape(groupedSourceKey(source))}"]`); if (tile) { // 跨列状态统一在这里兜底修正:只有多列布局下并排拼接类 // 才跨两列(单列时 span 2 会创建隐式列导致布局错位) tile.style.gridColumn = columns >= 2 && _isWideSource(source) ? 'span 2' : ''; workspaceMainRow.appendChild(tile); } }); } ['left', 'right'].forEach(hand => { const slot = workspaceHandSlots[hand]; if (!slot) return; const source = handSources.find(item => item.hand === hand); const tile = source ? workspaceGrid.querySelector(`[data-source-key="${CSS.escape(groupedSourceKey(source))}"]`) : null; if (tile && tile.parentNode !== slot) slot.appendChild(tile); slot.style.display = source ? '' : 'none'; }); if (workspaceBottomExtra) { const extras = bottomSources.filter(source => source.kind !== 'hand'); workspaceBottomExtra.style.display = extras.length ? 'grid' : 'none'; workspaceBottomExtra.style.gridTemplateColumns = `repeat(${Math.min(2, Math.max(1, extras.length))}, minmax(0, 1fr))`; extras.forEach(source => { const tile = workspaceGrid.querySelector( `[data-source-key="${CSS.escape(groupedSourceKey(source))}"]`); if (tile && tile.parentNode !== workspaceBottomExtra) { workspaceBottomExtra.appendChild(tile); } }); } if (workspaceHandFooter) { const hasExtras = bottomSources.some(source => source.kind !== 'hand'); const hasHands = handSources.length > 0; workspaceHandFooter.style.gridTemplateRows = hasExtras && hasHands ? 'minmax(0, 1fr) minmax(0, 1fr)' : 'minmax(0, 1fr)'; } } async function _appendWorkspaceSource(source) { if (!workspaceGrid || !workspaceMainRow) return; const movable = !_isBottomSource(source); const tile = _createWorkspaceTile(source, movable, workspaceMainRow); if (_isBottomSource(source)) { if (!workspaceHandFooter) { workspaceHandFooter = _createHandFooter(); workspaceGrid.appendChild(workspaceHandFooter); } if (source.kind === 'hand') { const slot = workspaceHandSlots[source.hand]; if (slot) slot.appendChild(tile); } else if (workspaceBottomExtra) { workspaceBottomExtra.appendChild(tile); } } else { const empty = workspaceMainRow.querySelector('[data-role="empty-workspace"]'); if (empty) empty.remove(); workspaceMainRow.appendChild(tile); } _refreshWorkspaceLayout(); await mountGroupedSource(source, tile); const player = players[groupedSourceKey(source)]; _setPlayerToCurrentFrame(player); _syncWorkspacePlayers(); } function renderGroupedWorkspace() { const grid = document.getElementById('video-grid'); if (!grid) return; const renderEpoch = currentEpisodeId; // 防快速切换批次:过期渲染不得启用控件/揭遮罩 const renderToken = _playbackSessionToken; masterPlaying = false; refreshPlayButton(false); if (typeof stopHeatmapSync === 'function') stopHeatmapSync(); Object.values(players).forEach(player => { try { player.destroy(); } catch (e) {} }); Object.keys(players).forEach(key => delete players[key]); currentImageTiles = []; currentDepthPreviewTiles = []; currentHand3dTiles = []; if (typeof clearHandTiles === 'function') clearHandTiles(); workspaceGrid = grid; workspaceMainRow = null; workspaceHandFooter = null; workspaceHandSlots = { left: null, right: null }; workspaceBottomExtra = null; grid.innerHTML = ''; grid.style.position = 'relative'; grid.className = 'flex-1 min-w-0 min-h-0 p-3 overflow-hidden grid gap-3'; grid.ondragover = event => { event.preventDefault(); grid.classList.add('ring-1', 'ring-blue-500/50'); }; grid.ondragleave = () => grid.classList.remove('ring-1', 'ring-blue-500/50'); grid.ondrop = event => { event.preventDefault(); grid.classList.remove('ring-1', 'ring-blue-500/50'); const raw = event.dataTransfer.getData('application/x-egodata-source'); if (raw) { try { addGroupedSource(JSON.parse(raw)); } catch (e) {} } else { const hand = event.dataTransfer.getData('application/x-egodata-hand'); if (hand) addHandTile(hand); } }; const mainSources = workspaceSources.filter(source => !_isBottomSource(source)); const bottomSources = workspaceSources.filter(_isBottomSource); const handSources = workspaceSources.filter(source => source.kind === 'hand'); const hasBottomExtras = bottomSources.some(source => source.kind !== 'hand'); grid.style.gridTemplateRows = bottomSources.length ? `minmax(0, 1fr) ${handSources.length ? (hasBottomExtras ? HAND_FOOTER_WITH_EXTRA_HEIGHT : HAND_FOOTER_HEIGHT) : 260}px` : 'minmax(0, 1fr)'; const mainRow = document.createElement('div'); mainRow.className = 'min-w-0 min-h-0 overflow-hidden grid gap-3'; mainRow.dataset.role = 'main-workspace'; workspaceMainRow = mainRow; grid.appendChild(mainRow); if (!mainSources.length) { mainRow.innerHTML = `
${t('drag_here')}
`; } if (bottomSources.length) { const footer = _createHandFooter(); workspaceHandFooter = footer; grid.appendChild(footer); } _refreshWorkspaceLayout(); renderSourceBar(); const mounted = []; mainSources.forEach(source => { const tile = _createWorkspaceTile(source, true, mainRow); mainRow.appendChild(tile); mounted.push(mountGroupedSource(source, tile)); }); bottomSources.forEach(source => { const tile = _createWorkspaceTile(source, false, null); if (source.kind === 'hand') { const slot = workspaceHandSlots[source.hand]; if (slot) slot.appendChild(tile); } else if (workspaceBottomExtra) { workspaceBottomExtra.appendChild(tile); } mounted.push(mountGroupedSource(source, tile)); }); Promise.all(mounted).then(async () => { if (!isCurrentPlaybackSession(renderEpoch, renderToken)) return; // 本次渲染作废 _restorePlayersToCurrentFrame(); const active = getActivePlayer(); if (!active) { hideVideoLoading(); return; } // A freshly mounted episode always starts paused. Clear the state and // sync before wiring the players: _syncWorkspacePlayers() re-issues // play() for a workspace still marked as playing, and the pauseAll() // further down then aborts it — a play/pause race that fires an // uncaught AbortError and leaves the transport button flapping. masterPlaying = false; _syncWorkspacePlayers(); refreshPlayButton(false); try { await loadFrameData(renderEpoch, renderToken); } catch (e) { if (isCurrentPlaybackSession(renderEpoch, renderToken)) hideVideoLoading(); return; } if (!isCurrentPlaybackSession(renderEpoch, renderToken)) return; if (!_defaultLayoutApplied && typeof hasHandSensor === 'function') { _defaultLayoutApplied = true; ['left', 'right'].forEach(hand => { if (hasHandSensor(hand) && !workspaceSources.some(source => source.kind === 'hand' && source.hand === hand)) { addHandTile(hand); } }); } // A newly opened episode always starts at its own frame zero. Do this // after the real FPS/frame count arrive so all overlays and media use // the same exact time origin, regardless of the previous episode. pauseAll(); seekToFrame(0); updateFrameDisplay(0); _refreshDepthTilesAt(0); if (typeof startHeatmapSync === 'function') startHeatmapSync(getActivePlayer()); if (typeof loadAnnotations === 'function') { await loadAnnotations(renderEpoch).catch(() => {}); } if (!isCurrentPlaybackSession(renderEpoch, renderToken)) return; // 深度首帧必须先落到 Canvas,再放开播放控件;否则用户会先 // 看到黑色深度窗,RGB 已经开始跑,随后每到深度边界就暂停。 // 短批次在这里等待整段 raw code preload,长批次等待首个 120 // 帧窗口,后续窗口由播放时钟提前预取。 // 全量加载契约:深度帧全部驻留才放开播放控件(手套/仿生手掌 // 数据随 loadFrameData 已在上面 await 完成并本地渲染,播放中 // 两类 tile 均无逐帧网络请求)。 const depthReady = await _awaitDepthInitialReady(renderEpoch); if (!isCurrentPlaybackSession(renderEpoch, renderToken)) return; if (!depthReady) { console.warn('[player] depth data not ready; keep playback gated'); return; } // 全量流解码时已顺带算出真实 q01/q99(服务端缓存),重拉一次 // 让 Turbo 色带从兜底全范围切到自适应范围。 _refreshDepthRanges(); // 全部就绪才放开控件/揭遮罩:消除「第二个视频晚 ready → 开头不同步」 // 与「帧数据返回前用旧 fps 交互 → 帧不对齐」两个竞态窗口 await _awaitPlayersReady(); if (!isCurrentPlaybackSession(renderEpoch, renderToken)) return; enableFrameControls(); hideVideoLoading(); }).catch(err => { // 兜底:任何未预期异常都不能让遮罩卡死/控件锁死(仅当前批次生效) console.error('[player] workspace load failed:', err); if (isCurrentPlaybackSession(renderEpoch, renderToken)) { enableFrameControls(); hideVideoLoading(); } }); } /* 默认画布:双目组 → 左右目并排;多物理设备单目流 → 全部显示 */ function _defaultWorkspaceFromGroups() { // 保留已自动挂载的 3D 世界窗口:loadHand3D 可能先于本函数完成 // 并已把 hand3d_world 加入 workspaceSources,整表重建不能冲掉 // (否则开关显示开启、窗口实际不存在 —— 竞态脱节 bug) const keep = workspaceSources.filter(s => s.kind === 'hand3d_world' || s.kind === 'slam_trajectory' || s.kind === 'depth' || s.kind === 'tactile_force_left' || s.kind === 'tactile_force_right'); // Depth is a first-class default preview source. It must be present in // workspaceSources before renderGroupedWorkspace takes its mount Promise // snapshot; adding it immediately after render creates a race where the // RGB tiles are ready but the depth canvas remains black with no request. const depthSources = (currentMediaGroups.sources || []) .filter(s => s.kind === 'depth') .filter(() => typeof window.isDepthOn !== 'function' || window.isDepthOn()) .map(_serializeSource); const retained = [...keep, ...depthSources.filter(depth => !keep.some(item => groupedSourceKey(item) === groupedSourceKey(depth)))]; workspaceSources = []; const grp = (currentMediaGroups.groups || [])[0]; if (grp && (grp.members || []).length) { workspaceSources = grp.members.map(m => ({ kind: 'video', source_key: m.source_key, label: m.label, stream_url: m.stream_url, video_cache_key: m.video_cache_key, frame_count: m.frame_count, fps: m.fps, })); workspaceSources.push(...retained); return; } const singles = currentMediaGroups.singles || []; if (singles.length) { workspaceSources = singles.map(single => ({ kind: 'video', source_key: single.source_key, label: single.label, stream_url: single.stream_url, video_cache_key: single.video_cache_key, frame_count: single.frame_count, fps: single.fps, })); } workspaceSources.push(...retained); } async function loadGroupedEpisodeVideo(episodeId, cameras, options = {}) { _cancelEpisodeMediaLoad(true); _episodeLoadInFlight = true; const loadToken = _playbackSessionToken; _episodeMediaLoadController = new AbortController(); const mediaSignal = _episodeMediaLoadController.signal; currentEpisodeId = episodeId; currentCameras = cameras || []; currentHasSkeleton = Boolean(options.hasSkeleton); _suppressMasterEvents = true; pauseAll(); _suppressMasterEvents = false; masterPlaying = false; refreshPlayButton(false); currentMediaGroups = null; workspaceSources = []; workspaceCameras = currentCameras.slice(); currentImageTiles = []; currentDepthPreviewTiles = []; currentHand3dTiles = []; currentSlamTiles = []; currentTactileTiles = []; hand3dData = null; hand3dDataBySource = {}; hand3dFrameCache = { frame: -1, data: null, inflight: -1 }; hand3dFrameCacheBySource = {}; hand3dWindow = { start: -1, end: -1, frames: {}, inflight: false }; hand3dWindowBySource = {}; slamData = null; slamFramesByIndex = {}; slamWindowInflight = null; tactileMatrixData = null; _depthPlaybackStall = null; _hand3dViewAnchor = null; _h3dBaseDist = null; _h3dGridY = null; _h3dCentroid.filters = [null, null]; _h3dCentroid.prevLabels = [null, null]; _h3dCentroid.prevC = [null, null]; _h3dCentroid.t0 = null; currentFrameTarget = 0; _centralLastFrame = -1; _lastDisplayedFrame = -1; _pendingSeekFrame = null; // Seed the UI with the selected episode's canonical metadata immediately. // frameDataReady remains false until frames-data has been validated. episodeTotalFrames = Math.max(0, Number(options.frameCount) || 0); episodeFps = Math.max(0, Number(options.fps) || 0); frameDataReady = false; _resetFrameUiForEpisode(0, episodeTotalFrames); _watchdogRetried = false; // 每个批次允许一次自愈重渲染 cancelSegmentPlayback(); // 切批次即终止段播放会话(旧会话的播放器即将被销毁) _defaultLayoutApplied = false; if (typeof resetHeatmapFrameState === 'function') resetHeatmapFrameState(episodeId); if (typeof clearAnnotationsNow === 'function') clearAnnotationsNow(); // 立即清掉上一批次残留的切片/overlay if (typeof stopHeatmapSync === 'function') stopHeatmapSync(); if (typeof clearHandTiles === 'function') clearHandTiles(); if (typeof destroyAllHandOverlays === 'function') destroyAllHandOverlays(); resetSourceBar(); disableFrameControls(); showVideoLoading(); // 自愈保险:加载链任何一环卡住,10 秒后强制启用控件并揭开遮罩 —— // 控件宁可可用,不可锁死("点了没反应"必须被兜底修掉)。 // 若此时还没有任何活跃播放器(挂载失败/被冲掉),重渲染工作区一次。 // 加载请求仍在进行中(如 hand-3d 元数据冷读 20s+)时不判定为故障, // 重新武装 watchdog 下一轮再查,避免误报"无活跃播放器"。 clearTimeout(_loadWatchdog); const watchdogTick = () => { if (!isCurrentPlaybackSession(episodeId, loadToken)) return; if (_episodeLoadInFlight) { _loadWatchdog = setTimeout(watchdogTick, 10000); return; } const depthStillBuffering = currentDepthPreviewTiles.some(entry => (entry.fullInflight && !entry.fullReady) || (entry.allPreloadInflight && !entry.allPreloaded)); if (!_controlsEnabled && !depthStillBuffering) { console.warn('[player] 加载自愈:强制启用控件'); enableFrameControls(); hideVideoLoading(); } if (!getActivePlayer() && !_watchdogRetried) { _watchdogRetried = true; console.warn('[player] 加载自愈:无活跃播放器 → 重渲染工作区'); try { renderGroupedWorkspace(); } catch (e) { console.error('[player] 自愈重渲染失败:', e); } } }; _loadWatchdog = setTimeout(watchdogTick, 10000); // Fetch 3D metadata in parallel with media-groups, but await both before // building the workspace. The 3D tile itself then awaits its first data // window together with each RGB overlay's first keypoint window. const hand3dReady = loadHand3D(episodeId, loadToken); const slamReady = loadSlamTrajectory(episodeId, loadToken); const tactileReady = loadTactileMatrixMeta(episodeId, loadToken); const grid = document.getElementById('video-grid'); if (!grid) { hideVideoLoading(); return; } // 媒体组 API:双目分组 + 素材清单(新/旧数据都兼容) try { // Media groups include transport URLs. They must not be served from // the browser's GET cache after a backend/frontend rollout. const res = await fetch( `/api/v1/episode/${episodeId}/media-groups?_v=202609022200`, { signal: mediaSignal, cache: 'no-store' }); if (!isCurrentPlaybackSession(episodeId, loadToken)) return; // 丢弃过期响应 if (res.ok) { currentMediaGroups = await res.json(); await Promise.all([hand3dReady, slamReady, tactileReady]); if (!isCurrentPlaybackSession(episodeId, loadToken)) return; renderSourceBar(); _defaultWorkspaceFromGroups(); // SLAM is a first-class spatial source. Add it before the initial // workspace render so it enters the same mount barrier as RGB. if (typeof window.ensureSlamTrajectoryTile === 'function') { window.ensureSlamTrajectoryTile(); } if (typeof window.ensureTactileMatrixTile === 'function') { window.ensureTactileMatrixTile(); } // Add the selected spatial source before renderGroupedWorkspace // takes its Promise.all(mounted) snapshot. if (typeof ensureHand3dWorldTile === 'function' && hand3dData && hand3dData.hasPreview && (typeof window.isHand3dWorldOn !== 'function' || window.isHand3dWorldOn())) { ensureHand3dWorldTile(); } renderGroupedWorkspace(); _episodeLoadInFlight = false; renderPreviewVideoSources(); // 3D 世界窗口兜底:开关开且有产物 → 确保已挂载 if (typeof ensureHand3dWorldTile === 'function' && hand3dData && hand3dData.hasPreview && (typeof window.isHand3dWorldOn !== 'function' || window.isHand3dWorldOn())) { ensureHand3dWorldTile(); } // 深度窗口兜底:☷ 深度开关开且有深度素材 → 自动挂载 // (素材栏已无深度按钮,挂载入口统一走 ☷) if (typeof ensureDepthTile === 'function' && (typeof window.isDepthOn !== 'function' || window.isDepthOn())) { ensureDepthTile(); } updateInfoBar(); if (typeof updatePreviewMenuData === 'function') updatePreviewMenuData(); return; } } catch (e) { if (e?.name === 'AbortError' || !isCurrentPlaybackSession(episodeId, loadToken)) return; /* fall through → 旧逻辑 */ } if (!isCurrentPlaybackSession(episodeId, loadToken)) return; // 兜底:API 不可用时按 camera_names 平铺到素材按钮条 renderSourceBar(); if (!currentCameras.length) { grid.innerHTML = '
' + t('no_video') + '
'; hideVideoLoading(); _episodeLoadInFlight = false; return; } workspaceSources = currentCameras.slice(0, 2).map(camera => groupedSource('video', camera, { label: camera })); renderGroupedWorkspace(); _episodeLoadInFlight = false; renderPreviewVideoSources(); updateInfoBar(); if (typeof updatePreviewMenuData === 'function') updatePreviewMenuData(); } function loadEpisodeVideo(episodeId, cameras, options = {}) { _cancelEpisodeMediaLoad(true); const loadToken = _playbackSessionToken; _episodeMediaLoadController = new AbortController(); currentEpisodeId = episodeId; currentCameras = cameras || []; _suppressMasterEvents = true; pauseAll(); _suppressMasterEvents = false; masterPlaying = false; refreshPlayButton(false); currentFrameTarget = 0; _centralLastFrame = -1; _lastDisplayedFrame = -1; _pendingSeekFrame = null; // reset — prevent stale seek target from previous episode episodeTotalFrames = Math.max(0, Number(options.frameCount) || 0); episodeFps = Math.max(0, Number(options.fps) || 0); frameDataReady = false; _resetFrameUiForEpisode(0, episodeTotalFrames); cancelSegmentPlayback(); // 切批次即终止段播放会话(旧会话的播放器即将被销毁) if (typeof resetHeatmapFrameState === 'function') resetHeatmapFrameState(episodeId); if (typeof clearAnnotationsNow === 'function') clearAnnotationsNow(); // 立即清掉上一批次残留的切片/overlay _defaultLayoutApplied = false; showVideoLoading(); stopHeatmapSync(); if (typeof clearHandTiles === 'function') clearHandTiles(); // The legacy single-video fallback uses the same hand-overlay state as // the grouped workspace. Clear it here too, otherwise a no-data episode // can leave the global skeleton switch disabled for the next episode. if (typeof destroyAllHandOverlays === 'function') destroyAllHandOverlays(); currentHand3dTiles = []; currentSlamTiles = []; hand3dData = null; hand3dDataBySource = {}; hand3dFrameCache = { frame: -1, data: null, inflight: -1 }; hand3dFrameCacheBySource = {}; hand3dWindow = { start: -1, end: -1, frames: {}, inflight: false }; hand3dWindowBySource = {}; slamData = null; slamFramesByIndex = {}; slamWindowInflight = null; _depthPlaybackStall = null; _hand3dViewAnchor = null; _h3dBaseDist = null; _h3dGridY = null; _h3dCentroid.filters = [null, null]; _h3dCentroid.prevLabels = [null, null]; _h3dCentroid.prevC = [null, null]; _h3dCentroid.t0 = null; loadHand3D(episodeId, loadToken); resetSourceBar(); disableFrameControls(); // disabled until player fires 'ready' const grid = document.getElementById('video-grid'); const tabs = document.getElementById('camera-tabs'); if (!grid || !tabs) { hideVideoLoading(); return; } // Clear existing grid.innerHTML = ''; tabs.innerHTML = 'Cameras:'; // Destroy old players Object.values(players).forEach(p => p.destroy()); Object.keys(players).forEach(k => delete players[k]); if (!cameras || cameras.length === 0) { grid.innerHTML = '
' + t('no_video') + '
'; hideVideoLoading(); return; } // Create tabs cameras.forEach((cam, i) => { const btn = document.createElement('button'); btn.textContent = cam; btn.className = i === 0 ? 'px-3 py-1 text-sm rounded-t bg-blue-600 text-white' : 'px-3 py-1 text-sm rounded-t bg-gray-800 text-gray-400 hover:bg-gray-700'; btn.onclick = () => switchTab(cam); tabs.appendChild(btn); }); // Create video container(s) const fallbackColumns = cameras.length <= 1 ? 1 : 2; const fallbackRows = Math.max(1, Math.ceil(cameras.length / fallbackColumns)); grid.className = 'flex-1 min-w-0 min-h-0 p-3 overflow-hidden grid gap-3'; grid.style.gridTemplateColumns = `repeat(${fallbackColumns}, minmax(0, 1fr))`; grid.style.gridTemplateRows = `repeat(${fallbackRows}, minmax(0, 1fr))`; grid.style.position = 'relative'; // anchor for skeleton canvas overlay cameras.forEach(async cam => { const div = document.createElement('div'); div.id = `player-container-${cam}`; div.className = 'bg-black rounded overflow-hidden min-w-0 min-h-0 w-full h-full'; grid.appendChild(div); const url = await pickVideoUrl(episodeId, cam); if (!isCurrentPlaybackSession(episodeId, loadToken)) return; players[cam] = initPlayer(`player-container-${cam}`, url); if (players[cam] && typeof initHandOverlay === 'function') { const source = { kind: 'video', source_key: cam, label: cam }; initHandOverlay(source, div, div); } }); // Start heatmap sync with first camera's player const firstCam = cameras[0]; const loadEpoch = episodeId; // capture for guard — discard if episode changed if (firstCam && players[firstCam]) { Object.values(players).forEach(p => bindMasterSync(p)); bindFrameDrift(players[firstCam]); loadFrameData(episodeId, loadToken).then(async () => { if (!isCurrentPlaybackSession(loadEpoch, loadToken)) return; // stale startHeatmapSync(players[firstCam]); // Now FPS + frameCount are correct — safe to render annotation overlay if (typeof loadAnnotations === 'function') { await loadAnnotations(episodeId).catch(() => {}); } if (!isCurrentPlaybackSession(loadEpoch, loadToken)) return; // 全部就绪才放开控件/揭遮罩(与 grouped 路径同一门控) await _awaitPlayersReady(); if (!isCurrentPlaybackSession(loadEpoch, loadToken)) return; enableFrameControls(); hideVideoLoading(); }); } // Update info bar updateInfoBar(); } function switchTab(camera) { // Highlight the active tab const tabs = document.getElementById('camera-tabs'); if (tabs) { tabs.querySelectorAll('button').forEach(btn => { if (btn.textContent === camera) { btn.className = 'px-3 py-1 text-sm rounded-t bg-blue-600 text-white'; } else { btn.className = 'px-3 py-1 text-sm rounded-t bg-gray-800 text-gray-400 hover:bg-gray-700'; } }); } // Show only selected camera const grid = document.getElementById('video-grid'); if (grid) { grid.className = 'flex-1 min-w-0 min-h-0 p-3 overflow-hidden grid grid-cols-1 gap-3'; if (typeof destroyAllHandOverlays === 'function') destroyAllHandOverlays(); grid.innerHTML = ''; const oldContainer = document.getElementById(`player-container-${camera}`); const div = document.createElement('div'); div.id = `player-container-${camera}`; div.className = 'bg-black rounded overflow-hidden min-w-0 min-h-0 w-full h-full'; grid.appendChild(div); // Re-init player if needed if (players[camera]) { players[camera].destroy(); } pickVideoUrl(currentEpisodeId, camera).then(url => { players[camera] = initPlayer(`player-container-${camera}`, url); if (players[camera] && typeof initHandOverlay === 'function') { const source = { kind: 'video', source_key: camera, label: camera }; initHandOverlay(source, div, div); } // Re-bind heatmap/frame sync to the new player instance Object.values(players).forEach(p => bindMasterSync(p)); bindFrameDrift(players[camera]); startHeatmapSync(players[camera]); // Re-render annotation overlay on the rebuilt player if (typeof annotations !== 'undefined' && annotations.length > 0) { renderAnnotationOverlay(annotations, episodeTotalFrames); } }); } } function playAll() { // 已播完(ended)后再次点击:HTML5 在 ended 状态 play() 是 no-op, // 必须先把所有播放器 seek 回第 0 帧才能从头重播。 const master = getActivePlayer(); if (master) { const dur = master.duration || 0; if (master.ended || (dur > 0 && (master.currentTime || 0) >= dur - 0.1)) { seekToFrame(0); } } // HTMLMediaElement.play() on several videos is not a barrier: each // element can start on a different refresh tick. Snap the complete set // to the master's integer frame before starting, so RGB, overlays and // the display-only 3D canvas all share one time origin. This does not // touch the stored parquet/video data. const syncMaster = getActivePlayer(); if (syncMaster && !syncMaster.ended) { const frame = _clampFrameIndex(Math.floor( (syncMaster.currentTime || 0) * getEpisodeFps() + 0.002)); const time = frame / getEpisodeFps(); currentFrameTarget = frame; Object.values(players).forEach(p => { try { if (Math.abs((p.currentTime || 0) - time) > 0.001) { p.currentTime = time; } } catch (e) {} }); if (_centralLastFrame !== frame) { _centralLastFrame = frame; updateFrameDisplay(frame); } } Object.values(players).forEach(p => { // play() 是异步 promise:中断(如被 pause 打断)产生的 AbortError // 拒绝必须吞掉,否则控制台报红且可能破坏后续播放状态。 try { const result = p.play(); if (result && typeof result.catch === 'function') { result.catch(() => {}); } } catch (e) {} }); // play() enters the native media state asynchronously. Arm the // presentation clock on the next task so the first displayed RGB frame // also becomes the first frame for depth/3D/sensor layers. setTimeout(_ensurePresentationClock, 0); } function pauseAll() { // Keep the transport state consistent with reality: callers that pause // without flipping the flag left the button reading "Pause" while every // player was stopped, so the next click replayed instead of resuming. masterPlaying = false; Object.values(players).forEach(p => { try { p.pause(); } catch(e) {} }); } /* ── Frame-aligned master control ──────────────────────── One play/pause button drives every video on the canvas. The master player's timeupdate forces slaves to the same time (±50ms drift correction); the master seek bar seeks all videos to the exact same frame. */ let masterPlaying = false; const _syncedPlayers = new WeakSet(); // play/pause broadcast bound const _driftBound = new WeakSet(); // frame-drift correction bound (master only) function refreshPlayButton(playing) { const btn = document.getElementById('btn-play-all'); if (!btn) return; const icon = btn.querySelector('[data-play-icon]'); const label = btn.querySelector('.play-label'); if (icon) { icon.setAttribute('icon', playing ? 'ant-design:pause-circle-filled' : 'ant-design:caret-right-filled'); } if (label) label.textContent = playing ? t('pause_btn') : t('play_btn'); btn.setAttribute('aria-pressed', String(Boolean(playing))); } function _lastValidFrame() { return Math.max(0, (episodeTotalFrames || 0) - 1); } function _clampFrameIndex(frameIndex) { const numeric = Number(frameIndex); const safeFrame = Number.isFinite(numeric) ? Math.floor(numeric) : 0; return episodeTotalFrames > 0 ? Math.max(0, Math.min(_lastValidFrame(), safeFrame)) : Math.max(0, safeFrame); } function stopPlaybackAtLastFrame() { cancelSegmentPlayback(); const lastFrame = _lastValidFrame(); masterPlaying = false; // Prevent a pause event from one view from starting another pause cascade // while all synchronized views are being snapped together. _suppressMasterEvents = true; pauseAll(); _suppressMasterEvents = false; seekToFrame(lastFrame); updateFrameDisplay(lastFrame); refreshPlayButton(false); } function togglePlayAll() { const master = getActivePlayer(); if (!master) { // 诊断:打印具体原因到控制台,不再"静默没反应" console.warn('[player] 播放不可用:无活跃播放器', { players: Object.keys(players), workspaceSources: workspaceSources.map(s => groupedSourceKey(s)) }); return; } // 只挡"播放器未就绪"。帧数据缺失(如 frames-data 接口瞬时失败)不 // 挡播放:按时间同步不依赖 fps,图像 tile 短暂用默认 30fps 可接受; // 若用 frameDataReady 一起挡,接口抖动会导致播放按钮"点了没反应"。 if (!playerReady) { console.warn('[player] 播放不可用:播放器未就绪(ready 事件未触发)'); return; } cancelSegmentPlayback(); if (masterPlaying) { pauseAll(); } else { // Set the broadcast lock before calling play(). Otherwise the first // native play event sees masterPlaying=false and recursively calls // playAll(), which can seek the video backwards by one frame/time tick. masterPlaying = true; playAll(); } // button state follows the players' play/pause events } /* Bind play/pause broadcast to ANY player — clicking the play button on any view (left, right, aux) plays/pauses the whole frame-aligned set. */ function bindMasterSync(player) { if (!player || _syncedPlayers.has(player)) return; _syncedPlayers.add(player); player.on('play', () => { if (_suppressMasterEvents) return; if (!masterPlaying) { masterPlaying = true; // broadcast lock (prevents re-entry) playAll(); } refreshPlayButton(true); }); player.on('pause', () => { if (_suppressMasterEvents) return; if (masterPlaying) { masterPlaying = false; // broadcast lock pauseAll(); } refreshPlayButton(false); }); player.on('ended', () => { if (_suppressMasterEvents) return; stopPlaybackAtLastFrame(); }); } /* Frame-drift correction on the master player: while playing, every slave is snapped back within half a frame; on pause all slaves are snapped to the exact master time so every view shows the identical frame. */ function bindFrameDrift(master) { if (!master || _driftBound.has(master)) return; _driftBound.add(master); master.on('timeupdate', () => { if (!masterPlaying || _frameScrubbing) return; const t = master.currentTime || 0; const maxDrift = Math.max(0.08, 2.5 / getEpisodeFps()); Object.values(players).forEach(p => { if (p === master) return; try { if (Math.abs(p.currentTime - t) > maxDrift) p.currentTime = t; } catch (e) {} }); }); master.on('pause', () => { const t = master.currentTime || 0; Object.values(players).forEach(p => { if (p === master) return; try { if (Math.abs(p.currentTime - t) > 0.005) p.currentTime = t; } catch (e) {} }); }); } function updateInfoBar() { const bar = document.getElementById('episode-info'); if (!bar) return; // 顶部信息条:组名 / fps / 帧数 / 同步状态 let groupText = ''; if (currentMediaGroups) { groupText = (currentMediaGroups.groups || []).map(g => g.label).join(' / '); if (!groupText && (currentMediaGroups.singles || []).length) groupText = t('single_mono'); } groupText = groupText || (currentCameras.length > 0 ? currentCameras.length + ' Cameras' : ''); const depthMissing = currentMediaGroups ? (currentMediaGroups.sources || []) .filter(source => source.kind === 'depth') .reduce((total, source) => total + (source.missing_frames || []).length, 0) : 0; const syncMarkup = depthMissing > 0 ? ` Depth missing ${depthMissing}` : ` ${t('synced_ok')}`; bar.innerHTML = ` ${groupText} | ${getEpisodeFps()} FPS · ${episodeTotalFrames || 0} ${t('frame')} | ${syncMarkup} `; bar.classList.remove('hidden'); bar.classList.add('flex'); updateEpisodeDetailSyncStatus(); } function updateEpisodeDetailSyncStatus() { const el = document.getElementById('episode-detail-sync-status'); if (!el) return; const depthSources = (currentMediaGroups?.sources || []) .filter(source => source.kind === 'depth'); if (!depthSources.length) { el.classList.add('hidden'); el.innerHTML = ''; return; } const missingFrames = [...new Set(depthSources.flatMap(source => source.missing_frames || []))] .sort((a, b) => a - b); if (!missingFrames.length) { el.className = 'flex items-center gap-1 text-green-400'; el.innerHTML = ' Depth data complete'; return; } const preview = missingFrames.slice(0, 4).join(', '); const suffix = missingFrames.length > 4 ? ', …' : ''; el.className = 'flex items-start gap-1 text-yellow-400'; el.innerHTML = ` Depth data missing ${missingFrames.length} frame${missingFrames.length === 1 ? '' : 's'} (${preview}${suffix})`; } // ═══════════════════════════════════════════════════════════ // Frame-level controls // ═══════════════════════════════════════════════════════════ const _frameChangeCallbacks = new Set(); let episodeFps = 30; let episodeTotalFrames = 0; function _syncFrameLabels(frameIndex) { const frame = Math.max(0, Math.floor(Number(frameIndex) || 0)); const detailEl = document.getElementById('detail-current-frame'); if (detailEl) detailEl.textContent = frame; const heatmapEl = document.getElementById('heatmap-frame'); if (heatmapEl) heatmapEl.textContent = frame; const timelineEl = document.getElementById('annotation-timeline-frame'); if (timelineEl) { timelineEl.textContent = episodeTotalFrames > 0 ? frame + ' / ' + episodeTotalFrames : '0 / 0'; } } function _resetFrameUiForEpisode(frameIndex = 0, totalFrames = 0) { currentFrameTarget = Math.max(0, Math.floor(Number(frameIndex) || 0)); _lastDisplayedFrame = -1; const safeTotal = Math.max(0, Math.floor(Number(totalFrames) || 0)); const detailTotal = document.getElementById('detail-total-frames'); if (detailTotal && safeTotal > 0) detailTotal.textContent = safeTotal; if (typeof resetAnnotationTimelineForEpisode === 'function') { resetAnnotationTimelineForEpisode(safeTotal); } _syncFrameLabels(currentFrameTarget); } function setEpisodeFrameInfo(fps, totalFrames) { const prevFps = episodeFps; episodeFps = fps || 30; episodeTotalFrames = Math.max(0, Number(totalFrames) || 0); frameDataReady = true; if (episodeTotalFrames > 0) { currentFrameTarget = _clampFrameIndex(currentFrameTarget); const detailTotal = document.getElementById('detail-total-frames'); if (detailTotal) detailTotal.textContent = episodeTotalFrames; } _syncFrameLabels(currentFrameTarget); updateInfoBar(); // fps 变化且已有非零目标帧 → 重对齐:此前用旧 fps 换算的 seek 停在 // 错误时间(loadFrameData 返回前点过 step/skip 的兜底修复) if (currentFrameTarget > 0 && episodeFps !== prevFps && !_frameScrubbing) { Object.values(players).forEach(_setPlayerToCurrentFrame); } } function setOnFrameChange(fn) { /* 多回调注册(标注高亮/手部骨骼叠加层各自挂一个,互不覆盖) */ if (typeof fn === 'function') _frameChangeCallbacks.add(fn); } let currentFrameTarget = 0; // integer frame index, avoids float error let _pendingSeekFrame = null; // target frame for in-flight seek, consumed by seeked handler let _lastDisplayedFrame = -1; // guard against redundant DOM updates on unchanged frame let _frameScrubbing = false; let _frameScrubWasPlaying = false; function isFrameScrubbing() { return _frameScrubbing; } function beginFrameScrub() { if (_frameScrubbing) return _frameScrubWasPlaying; _frameScrubWasPlaying = masterPlaying; _frameScrubbing = true; // Pause the synchronized set while the pointer previews frames. This // prevents drift correction from competing with the user's drag. if (_frameScrubWasPlaying) { masterPlaying = false; _suppressMasterEvents = true; pauseAll(); _suppressMasterEvents = false; refreshPlayButton(false); } return _frameScrubWasPlaying; } function previewFrame(frameIndex) { currentFrameTarget = _clampFrameIndex(frameIndex); updateFrameDisplay(currentFrameTarget); _refreshDepthTilesAt(currentFrameTarget); } function commitFrameScrub(frameIndex) { const resume = _frameScrubWasPlaying; _frameScrubbing = false; currentFrameTarget = _clampFrameIndex(frameIndex); const master = getActivePlayer(); if (!master) { seekToFrame(currentFrameTarget); return; } if (!resume) { seekToFrame(currentFrameTarget); return; } const resumeAfterSeek = () => { master.off('seeked', resumeAfterSeek); masterPlaying = true; playAll(); }; master.on('seeked', resumeAfterSeek); seekToFrame(currentFrameTarget); // Some media backends do not emit `seeked` when the target time is // already buffered/current. Keep the resumed-play behavior deterministic. setTimeout(() => { if (!_frameScrubbing && _frameScrubWasPlaying && !masterPlaying) { master.off('seeked', resumeAfterSeek); masterPlaying = true; playAll(); } }, 120); } function getActivePlayer() { for (const source of workspaceSources) { if (players[groupedSourceKey(source)]) return players[groupedSourceKey(source)]; } for (const cam of currentCameras) { if (players[cam]) return players[cam]; } return null; } function getCurrentFrame() { // During stepping, use the tracked integer target to avoid float precision loss. // During playback, timeupdate will sync currentFrameTarget from actual time. return currentFrameTarget; } function getCurrentTime() { const p = getActivePlayer(); if (!p) return 0; return p.currentTime || 0; } function getEpisodeFps() { /* Single source of truth for FPS — use this everywhere instead of a local copy. */ return episodeFps || 30; } function consumeSeekTarget() { /* Return the exact target frame from the most recent seekToFrame call, then clear it. This avoids the round-trip error where seeked recalculates a different frame from currentTime due to browser seek imprecision / millisecond truncation. */ if (_pendingSeekFrame != null) { const f = _pendingSeekFrame; _pendingSeekFrame = null; return f; } return null; } function seekToFrame(frameIndex) { frameIndex = _clampFrameIndex(frameIndex); currentFrameTarget = frameIndex; // 不提前返回:播放器尚未挂载时目标帧已记录在 currentFrameTarget, // 新播放器挂载(_setPlayerToCurrentFrame)或就绪后会自动补 seek, // 避免"刚切换视频时点切片静默失败"。 const time = frameIndex / (episodeFps || 30); // Only remember a pending exact-frame target when some player will // actually seek. A mount-time seekToFrame(0) on a fresh video never // fires 'seeked', leaving a stale 0 that the pause handler consumed and // reset the frame display to 0 after the first play. let willSeek = false; Object.values(players).forEach(player => { try { if (Math.abs((player.currentTime || 0) - time) > 0.001) { willSeek = true; } } catch (e) { /* ignore */ } }); if (willSeek) _pendingSeekFrame = frameIndex; Object.values(players).forEach(player => { try { player.currentTime = Math.max(0, time); } catch (e) {} }); } // ── Annotation segment auto-play ────────────────────── let _segmentPlayback = null; let _segAutoStopHandler = null; // compatibility for stale callers function cancelSegmentPlayback() { const session = _segmentPlayback; if (!session) return; _segmentPlayback = null; // 先清引用,off 失败也不影响后续段播放 if (session.player && session.handler) { // 会话的播放器可能已在切批次时被 destroy(Plyr 销毁后其内部 // container 为 null,off() 会抛 TypeError)—— 兜底吞掉。 try { session.player.off('timeupdate', session.handler); } catch (e) {} } if (session.player && session.startPlayback) { try { session.player.off('seeked', session.startPlayback); } catch (e) {} } } function legacyPlaySegment(startFrame, endFrame) { /* Legacy implementation kept only for compatibility with old callers. */ const p = getActivePlayer(); if (!p) return; // Remove any previous auto-stop handler(播放器可能已被销毁,兜底吞掉) if (_segAutoStopHandler) { try { p.off('timeupdate', _segAutoStopHandler); } catch (e) {} _segAutoStopHandler = null; } // Seek to start and play seekToFrame(startFrame); p.play().catch(() => {}); // Register one-shot auto-pause after endFrame _segAutoStopHandler = () => { const cf = Math.floor(p.currentTime * getEpisodeFps() + 0.002); if (cf >= endFrame) { p.pause(); seekToFrame(endFrame); // snap to exact endFrame — timeupdate may overshoot p.off('timeupdate', _segAutoStopHandler); _segAutoStopHandler = null; } }; p.on('timeupdate', _segAutoStopHandler); } // Final segment-playback implementation. It intentionally sits after the // legacy handler above so older cached markup cannot bypass synchronized // start/end handling. function playSegment(startFrame, endFrame) { const p = getActivePlayer(); if (!p) return; const start = _clampFrameIndex(startFrame); const end = Math.max(start, Math.min(_lastValidFrame(), Math.floor(Number(endFrame) || start))); cancelSegmentPlayback(); const session = { player: p, endFrame: end, started: false, handler: null, startPlayback: null, }; _segmentPlayback = session; const finish = () => { if (_segmentPlayback !== session) return; cancelSegmentPlayback(); masterPlaying = false; _suppressMasterEvents = true; pauseAll(); _suppressMasterEvents = false; seekToFrame(end); updateFrameDisplay(end); refreshPlayButton(false); }; session.handler = () => { if (_segmentPlayback !== session || !session.started) return; const frame = Math.floor((p.currentTime || 0) * getEpisodeFps() + 0.002); if (frame >= end) finish(); }; session.startPlayback = () => { if (_segmentPlayback !== session || session.started) return; session.started = true; p.off('seeked', session.startPlayback); masterPlaying = true; playAll(); }; p.on('timeupdate', session.handler); p.on('seeked', session.startPlayback); masterPlaying = false; _suppressMasterEvents = true; pauseAll(); _suppressMasterEvents = false; seekToFrame(start); // Buffered media can seek before emitting `seeked`. setTimeout(session.startPlayback, 120); } function stepFrame(delta) { const p = getActivePlayer(); if (!p) return; p.pause(); // Integer arithmetic — no float precision loss const newFrame = Math.max(0, Math.min(episodeTotalFrames - 1, currentFrameTarget + delta)); seekToFrame(newFrame); } function skipFrames(delta) { stepFrame(delta); } // Update frame display in UI function updateFrameDisplay(frameIndex) { // A media element may report its duration boundary as one frame past the // last actual image. Keep every UI surface on the valid frame range. frameIndex = _clampFrameIndex(frameIndex); // Skip if frame hasn't changed — prevents redundant DOM writes // that cause visual jitter on every timeupdate tick (~4 Hz) // but still refresh the timeline cursor. The timeline can be rebuilt after // the frame clock has already reached 0 (episode switch / annotation load), // so an early return must not leave the green cursor at the old position. if (frameIndex === _lastDisplayedFrame) { _syncFrameLabels(frameIndex); if (typeof updateAnnotationTimelineCursor === 'function') { updateAnnotationTimelineCursor(frameIndex); } return; } _lastDisplayedFrame = frameIndex; currentFrameTarget = frameIndex; _syncFrameLabels(frameIndex); const displayEl = document.getElementById('current-frame-display'); if (displayEl) displayEl.textContent = 'Frame ' + frameIndex; _frameChangeCallbacks.forEach(fn => { try { fn(frameIndex); } catch (e) { /* 单个回调异常不影响其他 */ } }); } // ═══════════════════════════════════════════════════════════ // Annotation overlay on Plyr progress bar // ═══════════════════════════════════════════════════════════ function renderAnnotationOverlay(annotations, totalFrames) { // Remove existing overlays from ALL players (not just first) document.querySelectorAll('.anno-progress-overlay').forEach(el => el.remove()); if (!annotations || annotations.length === 0) return; if (!totalFrames || totalFrames <= 0) return; // guard: frame data not yet loaded const maxFrame = totalFrames || (annotations.length > 0 ? Math.max(...annotations.map(a => a.end_frame_index)) : 1); if (maxFrame <= 0) return; // Build segment elements once, clone into each player's progress bar const buildSegments = () => annotations.map(seg => { const leftPct = (seg.start_frame_index / maxFrame) * 100; const widthPct = ((seg.end_frame_index - seg.start_frame_index + 1) / maxFrame) * 100; if (widthPct <= 0) return null; const segEl = document.createElement('div'); segEl.style.cssText = ` position: absolute; left: ${leftPct}%; width: ${widthPct}%; top: 0; height: 100%; background: ${seg.color || '#3B82F6'}; opacity: 0.55; pointer-events: auto; cursor: pointer; transition: opacity 0.15s; `; segEl.title = seg.label + ' (frame ' + seg.start_frame_index + '–' + seg.end_frame_index + ')'; segEl.onmouseenter = () => { segEl.style.opacity = '0.85'; }; segEl.onmouseleave = () => { segEl.style.opacity = '0.55'; }; segEl.onclick = (e) => { e.stopPropagation(); seekToFrame(seg.start_frame_index); }; return segEl; }).filter(Boolean); // Apply overlay to every player's progress bar (multi-camera support) document.querySelectorAll('.plyr__progress').forEach(progressContainer => { const overlay = document.createElement('div'); overlay.className = 'anno-progress-overlay'; overlay.style.cssText = ` position: absolute; top: 0; left: 0; width: 100%; height: 100%; pointer-events: none; z-index: 1; display: flex; border-radius: inherit; overflow: hidden; `; buildSegments().forEach(segEl => overlay.appendChild(segEl)); const parentStyle = window.getComputedStyle(progressContainer); if (parentStyle.position === 'static') { progressContainer.style.position = 'relative'; } progressContainer.appendChild(overlay); }); } function clearAnnotationOverlay() { document.querySelectorAll('.anno-progress-overlay').forEach(el => el.remove()); } // ═══════════════════════════════════════════════════════════ // Player ready state — controls enabled only after video loaded // ═══════════════════════════════════════════════════════════ let playerReady = false; let frameDataReady = false; // fps/总帧数已就绪;未就绪时禁止 seek/播放(旧 fps 是脏值) let _controlsEnabled = false; // 控件当前是否已启用(自愈看门狗用) let _loadWatchdog = null; let _episodeLoadInFlight = false; // 加载链(媒体组+hand3d)进行中;watchdog 不误判 let _watchdogRetried = false; // 自愈重渲染只试一次,防循环 function onPlayerReady() { playerReady = true; // 帧数据就绪前不放行控件:此时 fps 可能还是旧批次的脏值, // 提前放行会让 step/skip/播放用错误 fps 换算 → 帧不对齐。 // 控件最终由渲染完成链(renderGroupedWorkspace/loadEpisodeVideo)启用。 if (frameDataReady) enableFrameControls(); } function disableFrameControls() { playerReady = false; _controlsEnabled = false; const controls = document.getElementById('frame-controls'); if (controls) controls.classList.add('hidden'); const btns = ['btn-skip-back-10', 'btn-step-back', 'btn-play-all', 'btn-step-forward', 'btn-skip-forward-10']; btns.forEach(id => { const el = document.getElementById(id); if (el) { el.disabled = true; el.classList.add('opacity-40', 'cursor-not-allowed'); } }); } function enableFrameControls() { playerReady = true; _controlsEnabled = true; const controls = document.getElementById('frame-controls'); if (controls) controls.classList.remove('hidden'); const track = document.getElementById('annotation-timeline-track'); if (track && typeof bindFrameTimelineSeek === 'function') { bindFrameTimelineSeek(track, episodeTotalFrames || 1); } const btns = ['btn-skip-back-10', 'btn-step-back', 'btn-play-all', 'btn-step-forward', 'btn-skip-forward-10']; btns.forEach(id => { const el = document.getElementById(id); if (el) { el.disabled = false; el.classList.remove('opacity-40', 'cursor-not-allowed'); } }); // frame-controls 刚显示,其高度已参与工具条偏移 —— 重新定位播放工具条, // 否则工具条会按 controlsH=0 下沉,压到标注时间轴/切片块上 if (typeof _positionPlaybackToolbar === 'function') { const markers = document.getElementById('annotation-timeline-markers'); _positionPlaybackToolbar(markers ? markers.offsetHeight : 0); } } // Override stepFrame / seekToFrame to guard against unready state const _origStepFrame = stepFrame; stepFrame = function(delta) { if (!playerReady || !frameDataReady) return; _origStepFrame(delta); }; const _origSeekToFrame = seekToFrame; seekToFrame = function(frameIndex) { if (!playerReady || !frameDataReady) return; _origSeekToFrame(frameIndex); }; /* ── 加载遮罩 + 就绪门控 ───────────────────────────── 批次打开到「媒体组 + 全部播放器 + 帧数据 + 标注」就绪前, 画布盖遮罩、控件禁用,防止用旧批次 fps 提前交互。 */ function showVideoLoading() { const ov = document.getElementById('video-loading-overlay'); if (ov) ov.classList.remove('hidden'); } function hideVideoLoading() { const ov = document.getElementById('video-loading-overlay'); if (ov) ov.classList.add('hidden'); } function _awaitPlayersReady(timeoutMs = 45000) { /* 等待当前全部播放器 ready(元数据可 seek)且 canplay(首帧可画)。 只等 ready 会让深度预览在控制解锁时还没画出第一帧(黑窗)—— canplay 保证点播放时每个 tile 立即出画面。单播放器与整体都有 超时兜底,防止遮罩卡死。深度预览首次生成需要服务端转码 (约 4-10s),所以超时给足;纯 RGB 批次仍会在 1-2 秒内放行。 */ const list = Object.values(players); if (!list.length) return Promise.resolve(); const checks = list.map(player => new Promise(resolve => { let done = false; const finish = () => { if (!done) { done = true; resolve(); } }; const tryFinish = () => { if (!player.ready) return false; try { const media = player.media; if (media && media.readyState >= 3) { finish(); return true; } } catch (e) { /* fall through to event wait */ } return false; }; if (tryFinish()) return; try { const on = typeof player.once === 'function' ? player.once.bind(player) : player.on.bind(player); on('ready', () => tryFinish()); on('canplay', () => tryFinish()); } catch (e) { finish(); } setTimeout(finish, 30000); })); return Promise.race([ Promise.all(checks), new Promise(resolve => setTimeout(resolve, timeoutMs)), ]); } function _refreshDepthRanges() { // Fetch the adaptive q01/q99 range for every mounted depth tile. // Called at mount (fallback range arrives instantly) and again after // the all-loaded barrier (real bounds cached by the full-stream pass). if (!currentEpisodeId) return; const episodeAtRequest = currentEpisodeId; currentDepthPreviewTiles.forEach(entry => { if (!entry.source || !entry.canvas) return; fetch(`/api/v1/video/${encodeURIComponent(currentEpisodeId)}` + `/depth-stats/${encodeURIComponent(entry.source.source_key || '')}`) .then(response => (response.ok ? response.json() : null)) .then(stats => { if (stats && currentEpisodeId === episodeAtRequest && Number.isFinite(Number(stats.q01)) && Number.isFinite(Number(stats.q99)) && Number(stats.q99) > Number(stats.q01)) { entry.canvas._depthRange = { q01: Number(stats.q01), q99: Number(stats.q99), }; } }) .catch(() => { /* 统计失败 → 渲染器回退固定 0-4095 */ }); }); } async function _awaitDepthInitialReady(episodeId, timeoutMs = 60000) { const entries = currentDepthPreviewTiles.slice(); if (!entries.length) return true; // Strict barrier: playback may start only after EVERY depth frame is // resident (full stream buffered, or every aligned window preloaded). // The progressive early-release branches were removed on purpose — // releasing early caused boundary stalls that read as stutter. const isReady = (entry) => (entry.fullReady && entry.frameCount > 0) || (entry.allPreloaded && entry.frameCount > 0) || (Number(entry.source?.frame_count) <= 0 && entry.frames.has(0)); const deadline = performance.now() + timeoutMs; while (currentEpisodeId === episodeId && performance.now() < deadline) { if (entries.every(isReady)) return true; await new Promise(resolve => setTimeout(resolve, 50)); } return currentEpisodeId === episodeId && entries.every(isReady); } /* ══ 手部骨骼 3D 视图 ══════════════════════════════════ 数据来自 /api/v1/video/{episode}/hand-3d(hand_3d 产物, 可能是相对 3D 或相机米制坐标)。Canvas 透视投影,可拖拽旋转, 随播放帧同步;手 0 = 青色,手 1 = 橙色,label 标记左右。 */ // MediaPipe 21 点连接拓扑(官方 HAND_CONNECTIONS) const HAND3D_CONNECTIONS = [ [0, 1], [1, 2], [2, 3], [3, 4], // 拇指 [0, 5], [5, 6], [6, 7], [7, 8], // 食指 [5, 9], [9, 10], [10, 11], [11, 12], // 中指 [9, 13], [13, 14], [14, 15], [15, 16], // 无名指 [13, 17], [17, 18], [18, 19], [19, 20], // 小指 [0, 17], // 掌根 ]; async function loadSlamTrajectory(episodeId, sessionToken = null) { try { const signal = getMediaLoadSignal(); const res = await fetch( `/api/v1/video/${episodeId}/slam-trajectory?max_points=10000`, signal ? { signal, cache: 'no-store' } : { cache: 'no-store' }); if (!isCurrentPlaybackSession(episodeId, sessionToken)) return null; if (!res.ok) return null; const data = await res.json(); if (!isCurrentPlaybackSession(episodeId, sessionToken)) return null; if (!data || data.available !== true) return null; slamData = data; slamFramesByIndex = {}; (data.frames || data.trajectory || []).forEach(row => { const frame = Number(row && row.f); const pose = row && row.p; const valid = Array.isArray(pose) && pose.length >= 7 && pose.slice(0, 7).every(value => Number.isFinite(Number(value))) && Math.hypot(...pose.slice(3, 7).map(Number)) >= 0.5; if (Number.isFinite(frame) && valid) slamFramesByIndex[frame] = row; }); return data; } catch (_) { return null; } } async function loadTactileMatrixMeta(episodeId, sessionToken = null) { try { const signal = getMediaLoadSignal(); const res = await fetch( `/api/v1/video/${episodeId}/tactile-matrix-meta?v=202609091720-tactile-fetch-slam-axis`, signal ? { signal, cache: 'no-store' } : { cache: 'no-store' }); if (!isCurrentPlaybackSession(episodeId, sessionToken)) return null; if (!res.ok) return null; const data = await res.json(); if (!isCurrentPlaybackSession(episodeId, sessionToken)) return null; if (!data || data.available !== true) return null; // Scalar fx/fy/fz values are small enough to preload once per // episode. Keep them indexed by frame so scrubbing does not issue a // second parquet request for every tactile image. const series = data.force_series; const forceByFrame = Object.create(null); if (series && Array.isArray(series.frames)) { series.frames.forEach((frame, index) => { const key = String(Number(frame)); forceByFrame[key] = { left: Array.isArray(series.left) ? series.left[index] : null, right: Array.isArray(series.right) ? series.right[index] : null, }; }); } data._forceByFrame = forceByFrame; tactileMatrixData = data; return data; } catch (_) { return null; } } window.ensureSlamTrajectoryTile = function () { if (!slamData || slamData.available !== true) return; const source = { kind: 'slam_trajectory', source_key: 'slam_trajectory', label: 'SLAM', }; const key = groupedSourceKey(source); if (workspaceSources.some(item => groupedSourceKey(item) === key)) return; workspaceSources.push(source); }; window.ensureTactileMatrixTile = function () { if (!tactileMatrixData || tactileMatrixData.available !== true) return; const sources = [ { kind: 'tactile_force_left', source_key: 'tactile_force_left', side: 'left', label: 'Left' }, { kind: 'tactile_force_right', source_key: 'tactile_force_right', side: 'right', label: 'Right' }, ]; sources.forEach(source => { const key = groupedSourceKey(source); if (!workspaceSources.some(item => groupedSourceKey(item) === key)) { workspaceSources.push(source); } }); }; function _slamRows() { if (!slamData) return []; const rows = Array.isArray(slamData.trajectory) && slamData.trajectory.length ? slamData.trajectory : slamData.frames; return (rows || []).filter(row => row && Array.isArray(row.p) && row.p.length >= 7 && row.p.slice(0, 3).every(value => Number.isFinite(Number(value)))); } function _slamSmoothRows() { if (!slamData) return []; if (Array.isArray(slamData._displayRows)) return slamData._displayRows; const rows = _slamRows(); let previous = null; let previousFrame = null; const smoothed = rows.map(row => { const raw = row.p.slice(0, 3).map(Number); const frame = Number(row.f); const frameGap = previousFrame === null ? Infinity : frame - previousFrame; const canSmooth = previous && frameGap <= SLAM_DISPLAY_JOIN_GAP_FRAMES; const point = canSmooth ? raw.map((value, axis) => previous[axis] + SLAM_DISPLAY_SMOOTH_ALPHA * (value - previous[axis])) : raw; row._displayPosition = point; previous = point; previousFrame = frame; return row; }); slamData._displayRows = smoothed; return smoothed; } // The acquisition side records the S80M camera pose in a gravity-aligned // world frame: +Z is up (measured on real data - a physical turn of the // gripper rotates about world +Z, and the gripper's own up axis sits within // ~12 deg of it for the whole episode). The browser camera uses a right-handed // frame with Y up, so the display mapping puts world up on screen up. // // 源 +X 指向设备的**左侧**(S80M R_corr 之后源系是"左、后、上"), // 直接透传会让整个空间视图相对真实动作左右镜像:设备往左绕圈, // 画面里往右走。取反 X 修正左右;Y/Z 不动,前后与上下仍与实测一致。 function _slamDisplayPoint(sourcePoint) { return [-Number(sourcePoint[0]), Number(sourcePoint[2]), -Number(sourcePoint[1])]; } function _slamSourcePoint(row) { const point = Array.isArray(row && row._displayPosition) ? row._displayPosition : row && row.p; return point ? point.slice(0, 3).map(Number) : null; } function _slamCanJoinRows(previousRow, row) { if (!previousRow || !row) return false; const gap = Number(row.f) - Number(previousRow.f); return Number.isFinite(gap) && gap <= SLAM_DISPLAY_JOIN_GAP_FRAMES; } function _slamNearestRow(frame) { const target = Math.max(0, Math.round(Number(frame) || 0)); const rows = _slamSmoothRows(); if (!rows.length) return null; // If the current video frame has no valid pose, keep the last pose that // has already happened. Never select a future pose for the live marker. let best = null; rows.forEach(row => { const rowFrame = Number(row.f); if (rowFrame <= target && (!best || rowFrame > Number(best.f))) { best = row; } }); return best; } function _slamFit(rows) { const points = rows.map(row => _slamDisplayPoint(_slamSourcePoint(row))); const min = [Infinity, Infinity, Infinity]; const max = [-Infinity, -Infinity, -Infinity]; points.forEach(point => point.forEach((value, axis) => { min[axis] = Math.min(min[axis], value); max[axis] = Math.max(max[axis], value); })); const span = Math.max(max[0] - min[0], max[1] - min[1], max[2] - min[2], 0.4); const center = min.map((value, axis) => (value + max[axis]) / 2); const gridStep = 0.2; const gridY = Math.floor((min[1] - 0.08) / gridStep) * gridStep; const half = Math.max(0.6, Math.ceil((span * 0.7) / gridStep) * gridStep); const fov = 45 * Math.PI / 180; const baseDist = Math.max( 1.2, Math.min( 6, span * 1.25 * SLAM_DISPLAY_CAMERA_DISTANCE_SCALE / Math.tan(fov / 2), ), ); return { points, min, max, center, gridY, half, baseDist }; } function _slamRotateVector(pose, vector) { let qx = Number(pose[3]), qy = Number(pose[4]); let qz = Number(pose[5]), qw = Number(pose[6]); const norm = Math.hypot(qx, qy, qz, qw); if (![qx, qy, qz, qw].every(Number.isFinite) || norm < 1e-6) return vector; qx /= norm; qy /= norm; qz /= norm; qw /= norm; const tx = 2 * (qy * vector[2] - qz * vector[1]); const ty = 2 * (qz * vector[0] - qx * vector[2]); const tz = 2 * (qx * vector[1] - qy * vector[0]); return [ vector[0] + qw * tx + qy * tz - qz * ty, vector[1] + qw * ty + qz * tx - qx * tz, vector[2] + qw * tz + qx * ty - qy * tx, ]; } function _slamGripperState(row) { const values = row && Array.isArray(row.g) ? row.g : null; if (!values || values.length < 1) return null; const percent = Number(values[0]); if (!Number.isFinite(percent)) return null; return { percent: Math.max(0, Math.min(100, percent)), gripped: Number(values[1]) >= 0.5, }; } function renderSlamTile(entry, frameOverride) { if (!entry || !entry.canvas) return; const canvas = entry.canvas; const rect = canvas.getBoundingClientRect(); const dpr = window.devicePixelRatio || 1; const w = Math.max(1, rect.width), h = Math.max(1, rect.height); if (canvas.width !== Math.round(w * dpr) || canvas.height !== Math.round(h * dpr)) { canvas.width = Math.round(w * dpr); canvas.height = Math.round(h * dpr); } const ctx = canvas.getContext('2d'); ctx.setTransform(dpr, 0, 0, dpr, 0, 0); ctx.clearRect(0, 0, w, h); ctx.fillStyle = 'rgba(3,7,18,0.95)'; ctx.fillRect(0, 0, w, h); const rows = _slamSmoothRows(); if (!rows.length) return; if (!entry.fit) entry.fit = _slamFit(rows); const fit = entry.fit; const frame = typeof frameOverride === 'number' ? frameOverride : (typeof currentFrameTarget === 'number' ? currentFrameTarget : 0); const cam = _hand3dCam( fit.points, w, h, entry.rotY || 0, entry.rotX || 0.35, entry.zoom || 1, fit.baseDist, entry.pan || [0, 0, 0]); entry._lastCam = cam; const project = point => _projectHand3D(point, cam); const line = (a, b, color, width = 1) => { const pa = project(a), pb = project(b); if (!Number.isFinite(pa[0]) || !Number.isFinite(pa[1]) || !Number.isFinite(pb[0]) || !Number.isFinite(pb[1])) return; ctx.strokeStyle = color; ctx.lineWidth = width; ctx.beginPath(); ctx.moveTo(pa[0], pa[1]); ctx.lineTo(pb[0], pb[1]); ctx.stroke(); }; // Ground grid: every square is exactly 0.2 m. Labels are intentionally // kept to one small unit marker so the spatial view stays uncluttered. const minX = fit.center[0] - fit.half; const maxX = fit.center[0] + fit.half; const minZ = fit.center[2] - fit.half; const maxZ = fit.center[2] + fit.half; for (let x = minX; x <= maxX + 0.001; x += 0.2) { line([x, fit.gridY, minZ], [x, fit.gridY, maxZ], 'rgba(75,85,99,0.55)'); } for (let z = minZ; z <= maxZ + 0.001; z += 0.2) { line([minX, fit.gridY, z], [maxX, fit.gridY, z], 'rgba(75,85,99,0.55)'); } // Small axis marker at the fitted area: the recorded world axes, drawn in // the display frame that _slamDisplayPoint produces (world +Z is up). // The X tip follows the same negation as _slamDisplayPoint: display +X is // source -X, so the recorded +X axis points toward display -X. const axisOrigin = [fit.center[0], fit.gridY, fit.center[2]]; const axisTips = [ // World axes: X and Y horizontal, Z up. [[axisOrigin[0] - 0.2, axisOrigin[1], axisOrigin[2]], 'rgb(248,90,90)', 'X'], [[axisOrigin[0], axisOrigin[1], axisOrigin[2] - 0.2], 'rgb(80,220,130)', 'Y'], [[axisOrigin[0], axisOrigin[1] + 0.2, axisOrigin[2]], 'rgb(90,150,255)', 'Z'], ]; axisTips.forEach(([tip, color, label]) => { line(axisOrigin, tip, color, 1.5); const projected = project(tip); if (!Number.isFinite(projected[0]) || !Number.isFinite(projected[1])) return; ctx.font = '10px sans-serif'; ctx.fillStyle = color; ctx.fillText(label, projected[0] + 3, projected[1] - 3); }); const axis = project(axisOrigin); if (Number.isFinite(axis[0]) && Number.isFinite(axis[1])) { ctx.font = '9px sans-serif'; ctx.fillStyle = 'rgba(148,163,184,0.8)'; ctx.fillText('0.2m', axis[0] + 5, axis[1] + 12); } const current = Number(frame); let previous = null; let previousRow = null; rows.forEach(row => { const rowFrame = Number(row.f); if (rowFrame > current) return; const sourcePoint = _slamSourcePoint(row); const point = _slamDisplayPoint(sourcePoint); if (previous && _slamCanJoinRows(previousRow, row)) { line(previous, point, 'rgba(34,211,238,0.95)', 2); } previous = point; previousRow = row; }); const currentRow = _slamNearestRow(current); if (currentRow) { const pose = currentRow.p.map(Number); const sourcePoint = _slamSourcePoint(currentRow); const point = _slamDisplayPoint(sourcePoint); const projected = project(point); if (Number.isFinite(projected[0]) && Number.isFinite(projected[1])) { ctx.fillStyle = '#f8fafc'; ctx.beginPath(); ctx.arc(projected[0], projected[1], 4, 0, Math.PI * 2); ctx.fill(); ctx.strokeStyle = '#06b6d4'; ctx.lineWidth = 2; ctx.stroke(); } const axisLength = 0.08; const colors = ['rgb(248,90,90)', 'rgb(80,220,130)', 'rgb(90,150,255)']; [[1, 0, 0], [0, 1, 0], [0, 0, 1]].forEach((vector, index) => { const rotated = _slamRotateVector(pose, vector); const displayVector = _slamDisplayPoint(rotated); line(point, [ point[0] + displayVector[0] * axisLength, point[1] + displayVector[1] * axisLength, point[2] + displayVector[2] * axisLength, ], colors[index], 2); }); // Simplified UMI gripper model. The recorded pose frame is the S80M // camera frame; measured on the uploaded UMIGripper_AI episodes (every // frame of every batch) the device axes sit at // +X = gripper right (jaw separation, horizontal), // +Y = gripper forward (the optical axis, horizontal) and // +Z = gripper up (vertical, within ~12 deg of world +Z — the world // frame is gravity-aligned, so this is a fixed sensor-frame fact, not // a per-episode pose). So the body extends along local +Y, the two // plates separate along local X, and each plate is 0.20m long // (forward) by 0.05m tall (up) with a 0.12m maximum opening. const grip = _slamGripperState(currentRow); if (grip) { const toDisplay = local => { const rotated = _slamRotateVector(pose, local); return _slamDisplayPoint([ sourcePoint[0] + rotated[0], sourcePoint[1] + rotated[1], sourcePoint[2] + rotated[2], ]); }; const plateLength = 0.20; const plateHeight = 0.05; const maxHalfGap = 0.06; const baseOffset = 0.10; const halfGap = maxHalfGap * (1 - grip.percent / 100); const jawColor = grip.gripped ? 'rgba(248,90,90,0.95)' : 'rgba(88,168,232,0.95)'; const jawFill = grip.gripped ? 'rgba(248,90,90,0.16)' : 'rgba(88,168,232,0.12)'; const relativeCorners = [ [0, -plateLength / 2, -plateHeight / 2], [0, plateLength / 2, -plateHeight / 2], [0, plateLength / 2, plateHeight / 2], [0, -plateLength / 2, plateHeight / 2], ]; for (const sign of [1, -1]) { const centerX = sign * halfGap; const corners = relativeCorners.map(corner => project(toDisplay([ centerX + corner[0], baseOffset + corner[1], corner[2], ]))); if (corners.every(item => Number.isFinite(item[0]) && Number.isFinite(item[1]))) { ctx.fillStyle = jawFill; ctx.strokeStyle = jawColor; ctx.lineWidth = 2; ctx.beginPath(); ctx.moveTo(corners[0][0], corners[0][1]); corners.slice(1).forEach(item => ctx.lineTo(item[0], item[1])); ctx.closePath(); ctx.fill(); ctx.stroke(); } line( toDisplay([centerX, baseOffset - plateLength / 2, 0]), toDisplay([centerX, baseOffset + plateLength / 2, 0]), jawColor, 1); } // Rear crossbar plus the two supports that tie each plate back to // the body, so the opening stays readable when it is small. line(toDisplay([-maxHalfGap, baseOffset, 0]), toDisplay([maxHalfGap, baseOffset, 0]), 'rgba(148,163,184,0.8)', 1.5); for (const sign of [1, -1]) { for (const cornerY of [-plateLength / 2, plateLength / 2]) { line( toDisplay([sign * halfGap, baseOffset + cornerY, 0]), toDisplay([0, baseOffset + cornerY, 0]), 'rgba(148,163,184,0.8)', 1); } } } } } function initSlamDrag(entry) { if (!entry || !entry.canvas) return; const canvas = entry.canvas; canvas.style.cursor = 'grab'; canvas.addEventListener('mousedown', event => { if (event.button !== 0 && event.button !== 1) return; entry.dragging = true; entry.dragMode = event.button === 1 ? 'pan' : 'rotate'; entry.lastX = event.clientX; entry.lastY = event.clientY; canvas.style.cursor = entry.dragMode === 'pan' ? 'move' : 'grabbing'; event.preventDefault(); }); window.addEventListener('mousemove', event => { if (!entry.dragging) return; const dx = event.clientX - entry.lastX; const dy = event.clientY - entry.lastY; if (entry.dragMode === 'pan' && entry._lastCam) { const distance = Math.max(0.2, Math.hypot( entry._lastCam.eye[0] - entry._lastCam.centroid[0], entry._lastCam.eye[1] - entry._lastCam.centroid[1], entry._lastCam.eye[2] - entry._lastCam.centroid[2])); const worldPerPixel = distance / Math.max(1, entry._lastCam.f); const pan = entry.pan || (entry.pan = [0, 0, 0]); for (let i = 0; i < 3; i++) { pan[i] += (-dx * entry._lastCam.right[i] + dy * entry._lastCam.up[i]) * worldPerPixel; } } else { entry.rotY += dx * 0.01; entry.rotX = Math.max(-1.2, Math.min(1.2, entry.rotX + dy * 0.01)); } entry.lastX = event.clientX; entry.lastY = event.clientY; renderSlamTile(entry); }); window.addEventListener('mouseup', () => { if (!entry.dragging) return; entry.dragging = false; entry.dragMode = ''; canvas.style.cursor = 'grab'; }); canvas.addEventListener('contextmenu', event => event.preventDefault()); canvas.addEventListener('wheel', event => { event.preventDefault(); entry.zoom = Math.min(6, Math.max(0.3, (entry.zoom || 1) * Math.exp(-event.deltaY * 0.0015))); renderSlamTile(entry); }, { passive: false }); } async function loadHand3D(episodeId, sessionToken = null) { try { const signal = getMediaLoadSignal(); const res = await fetch( `/api/v1/video/${episodeId}/hand-3d`, signal ? { signal, cache: 'no-store' } : { cache: 'no-store' }); if (!isCurrentPlaybackSession(episodeId, sessionToken)) return; // 丢弃过期响应 if (!res.ok) return; const data = await res.json(); if (!isCurrentPlaybackSession(episodeId, sessionToken)) return; if (!data) return; const availableOptions = data.available_sources && data.available_sources.length ? data.available_sources : [{ ...data, source_key: data.source_key || 'hand3d' }]; // Stereo produces one 3D sequence per RGB view. The server scores // valid finite landmarks/frames and exposes the winning source. The // spatial preview intentionally mounts only that source; hand_0 and // hand_1 inside it remain available, so both physical hands are kept. const selectedKey = data.selected_source_key || data.source_key; const selectedOption = availableOptions.find(option => option.source_key === selectedKey); const options = selectedOption ? [selectedOption] : [ [...availableOptions].sort((a, b) => (Number(b.valid_landmark_points || 0) - Number(a.valid_landmark_points || 0)) || (Number(b.valid_hand_frames || 0) - Number(a.valid_hand_frames || 0)) || (a.source_key === 'stereo_left' ? -1 : 1) )[0], ]; hand3dDataBySource = {}; options.forEach(option => { const item = { ...data, ...option }; const relativeMode = item.unit === 'mediapipe_world_relative' || String(item.mode || '').toLowerCase().includes('relative'); item.relativeMode = relativeMode; item.worldMode = item.world_preview === true && item.depth_available === true && item.source === 'depth_camera_meters' && item.unit === 'camera_meters'; item.rgbEstimatedMode = item.rgb_estimated_preview === true || item.unit === 'rgb_estimated_meters' || String(item.mode || '').toLowerCase() === 'rgb_estimated_3d'; item.spaceMode = item.worldMode || item.rgbEstimatedMode; // The depth lift stores OpenCV camera coordinates (Y down), while // RGBWorldTracker stores display coordinates (Y up). Normalize // only the depth path before projection; do not flip RGB points. const frameName = String(item.coordinate_frame || '').toLowerCase(); item.cameraYDown = item.source === 'depth_camera_meters' && !frameName.includes('y_up'); item.hasPreview = item.spaceMode; hand3dDataBySource[item.source_key] = item; }); const firstSource = options[0].source_key || data.source_key || 'hand3d'; hand3dData = hand3dDataBySource[firstSource] || data; // Mounting is intentionally done by loadGroupedEpisodeVideo after // media-groups is ready. That keeps this metadata request from // creating a late tile outside the initial loading barrier. if (typeof updatePreviewMenuData === 'function') updatePreviewMenuData(); return hand3dData; } catch (e) { /* 无 3D 产物 → 不显示 3D 开关 */ return null; } } /* ☷ "3D Hand World" 开关联动:开启且工作区未挂载时自动加 3D canvas 窗口。 幂等性校验必须落到 DOM:源在 workspaceSources 但 tile 被重建冲掉时 (加载竞态),列表存在 ≠ 窗口存在 —— 会造成"开关显示开实际关"。 */ window.ensureHand3dWorldTile = function () { const sources = Object.values(hand3dDataBySource || {}) .filter(item => item && item.hasPreview); if (!sources.length && hand3dData && hand3dData.hasPreview) sources.push(hand3dData); sources.forEach(item => { const sourceKey = item.source_key || 'hand3d'; const src = { kind: 'hand3d_world', source_key: sourceKey, label: `${item.device_name || sourceKey} · ${item.rgbEstimatedMode ? 'RGB Estimated 3D' : t('hand3d_world')}`, }; const key = groupedSourceKey(src); const present = workspaceSources.some(s => groupedSourceKey(s) === key); if (present) { const tile = workspaceGrid ? workspaceGrid.querySelector(`[data-source-key="${CSS.escape(key)}"]`) : null; if (tile) return; workspaceSources = workspaceSources.filter(s => groupedSourceKey(s) !== key); } // During the first episode load the metadata can finish before the // workspace DOM exists. Queue the source in the source list; mounting // it here would recursively render an incomplete workspace. if (!workspaceGrid || !workspaceMainRow) { workspaceSources.push(src); return; } addGroupedSource(src); }); }; /* ☷ 开关关闭 → 从 workspaceSources 卸除 source(复用 removeGroupedSource 的完整清理:销毁 Plyr、删 tile、布局回收)。display:none 只藏 tile, _refreshWorkspaceLayout 仍按源列表预留 260px 底部条 = "位置被占用", 且再开时无可见变化 —— 必须卸源。再开由 ensure*Tile 重新挂载。 */ window.removeHand3dWorldTile = function () { workspaceSources.filter(s => s.kind === 'hand3d_world').forEach(src => { removeGroupedSource(src); }); currentHand3dTiles = currentHand3dTiles.filter( e => e.source.kind !== 'hand3d_world'); }; window.removeDepthTile = function () { workspaceSources.filter(s => s.kind === 'depth').forEach(src => { removeGroupedSource(src); }); }; /* ☷ 开关统一入口(hand-overlay.js _applyTileVisibility 调用): on → 自动挂载(幂等);off → 卸除 source(布局回收)。 */ window.setHand3dWorldVisible = function (on) { if (on) window.ensureHand3dWorldTile(); else window.removeHand3dWorldTile(); }; window.setDepthTileVisible = function (on) { if (on) window.ensureDepthTile(); else window.removeDepthTile(); }; /* ☷ 深度开关联动:开启且批次有深度素材 → 自动挂载深度视频 tile (与 3D Hand World 同机制:素材栏不再提供深度按钮,唯一入口是 ☷)。 同一幂等校验:源在列表但 DOM 无 tile 时重建。 */ window.ensureDepthTile = function () { const sources = (typeof currentMediaGroups !== 'undefined' && currentMediaGroups) ? (currentMediaGroups.sources || []) : []; const depths = sources.filter(s => s.kind === 'depth'); depths.forEach(depth => { const src = _serializeSource(depth); const key = groupedSourceKey(src); const present = workspaceSources.some(s => groupedSourceKey(s) === key); if (present) { const tile = workspaceGrid ? workspaceGrid.querySelector(`[data-source-key="${CSS.escape(key)}"]`) : null; if (tile) return; workspaceSources = workspaceSources.filter(s => groupedSourceKey(s) !== key); } addGroupedSource(src); }); }; /* 按帧懒加载 3D 手部点:播放/拖动到哪一帧就取哪一帧(全量返回在 60 分钟级批次上达 226MB/35s,会把页面拖死) */ async function fetchHand3DFrame(frame, sourceKey) { const source = sourceKey || 'default'; const cache = _hand3dFrameCacheFor(source); if (!currentEpisodeId || cache.inflight === frame) return; const episodeAtRequest = currentEpisodeId; const sessionToken = _playbackSessionToken; const signal = getMediaLoadSignal(); cache.inflight = frame; try { const res = await fetch( `/api/v1/video/${episodeAtRequest}/hand-3d?source_key=${encodeURIComponent(source)}&frame=${frame}`, signal ? { signal } : {}); if (!isCurrentPlaybackSession(episodeAtRequest, sessionToken)) return; if (!res.ok) { cache.inflight = -1; return; } const data = await res.json(); if (!isCurrentPlaybackSession(episodeAtRequest, sessionToken)) return; if (cache.inflight !== frame) return; // 帧已跳走,丢弃过期响应 hand3dFrameCacheBySource[source] = { frame, data, inflight: -1 }; currentHand3dTiles .filter(entry => (entry.source.source_key || 'default') === source) .forEach(entry => renderHand3DTile(entry)); } catch (e) { cache.inflight = -1; } } /* 米制 3D 点 → 绕 Y/X 轴旋转 → 屏幕坐标(等距投影) */ // ── 3D 世界坐标渲染(d435 demo RotatingSkeletonRenderer 的同款数学)── // 五指分色 + 掌心灰线 + 腕部白点 + 地面网格 + 相机系坐标轴 + 腕部 // 深度标注;透视投影 f=(h/2)/tan(fov/2),静态视角 θ=π(正面)+ 仰角 25°, // 世界坐标相机使用固定安全视距,不随当前可见手数量变化。用户拖拽 = // 相对静态视角的偏转;滚轮缩放 = 相机距离。 const HAND3D_FINGERS = [ // Keep the 3D tile identical to hand-overlay.js (the visible RGB video // overlay): thumb blue and pinky orange. { name: 'Thumb', ids: [1, 2, 3, 4], color: [0, 128, 255] }, { name: 'Index', ids: [5, 6, 7, 8], color: [0, 255, 0] }, { name: 'Middle', ids: [9, 10, 11, 12], color: [0, 255, 255] }, { name: 'Ring', ids: [13, 14, 15, 16], color: [255, 0, 255] }, { name: 'Pinky', ids: [17, 18, 19, 20], color: [255, 128, 0] }, ]; const HAND3D_PALM = [[0, 1], [0, 5], [5, 9], [9, 13], [13, 17], [0, 17]]; let _hand3dViewAnchor = null; // 世界坐标模式:统一的固定相机目标/世界原点 let _h3dBaseDist = null; // 世界坐标模式:首帧锁定的相机距离(仅滚轮缩放改变) let _h3dGridY = null; // 世界坐标模式:首帧锁定的地面网格高度(视角固定) // M1 质心锚定(demo _CentroidAnchor 展示路径的 JS 移植):每槽质心 // 强 One-Euro(freq 3.0/beta 0.3/dcutoff 0.3)+ 共模平移校正 —— // 静态视角下手稳,"跟随乱动"的抖动主要来自逐帧质心漂移,这里吸收。 const _h3dCentroid = { filters: [null, null], prevLabels: [null, null], prevC: [null, null], t0: null }; function _h3dMakeOneEuro(freqMin, beta, dcutoff) { let prev = null, prevDx = [0, 0, 0], prevTs = null; const alpha = (cutoff, dt) => { const tau = 1 / (2 * Math.PI * cutoff); return dt / (dt + tau); }; return (x, ts) => { if (prev === null || prevTs === null) { prev = x.slice(); prevDx = [0, 0, 0]; prevTs = ts; return x.slice(); } const dt = (ts - prevTs) / 1000; if (dt <= 1e-9) return prev.slice(); const dx = x.map((v, i) => (v - prev[i]) / dt); const ad = alpha(dcutoff, dt); prevDx = dx.map((v, i) => ad * v + (1 - ad) * prevDx[i]); const fc = freqMin + beta * Math.hypot(prevDx[0], prevDx[1], prevDx[2]); const a = alpha(fc, dt); const out = x.map((v, i) => a * v + (1 - a) * prev[i]); prev = out.slice(); prevTs = ts; return out.slice(); }; } function _h3dAnchorApply(ptsBySlot, labels, state = _h3dCentroid) { if (state.t0 === null) state.t0 = performance.now(); const ts = performance.now() - state.t0; const out = ptsBySlot.map(arr => arr.map(p => (p ? p.slice() : null))); for (let s = 0; s < 2; s++) { const finIdx = []; out[s].forEach((p, i) => { if (p) finIdx.push(i); }); if (finIdx.length < 4) { state.filters[s] = null; state.prevC[s] = null; state.prevLabels[s] = labels[s]; continue; } let c = [0, 0, 0]; finIdx.forEach(i => { c[0] += out[s][i][0]; c[1] += out[s][i][1]; c[2] += out[s][i][2]; }); c = c.map(v => v / finIdx.length); if (labels[s] !== state.prevLabels[s]) { // 重建帧且几何近(<0.1m)软衔接,否则硬重置(防跨手污染) const pc = state.prevC[s]; if (state.filters[s] && pc && pc.every(Number.isFinite) && Math.hypot(pc[0] - c[0], pc[1] - c[1], pc[2] - c[2]) < 0.10) { c = c.map((v, k) => 0.5 * pc[k] + 0.5 * v); } state.filters[s] = _h3dMakeOneEuro(3.0, 0.3, 0.3); } if (!state.filters[s]) state.filters[s] = _h3dMakeOneEuro(3.0, 0.3, 0.3); const ch = state.filters[s](c, ts); const delta = [ch[0] - c[0], ch[1] - c[1], ch[2] - c[2]]; finIdx.forEach(i => { out[s][i][0] += delta[0]; out[s][i][1] += delta[1]; out[s][i][2] += delta[2]; }); state.prevC[s] = ch; state.prevLabels[s] = labels[s]; } return out; } function _hand3dCam(valid, w, h, rotY, rotX, zoom, baseDistOverride = null, targetOffset = null) { // demo RotatingSkeletonRenderer.render 同款相机布置:质心为目标、 // fov 45°、静态正面视角 θ=π + 默认仰角 0°。世界坐标模式传入统一的 // baseDist,播放期不随手部跨度自适应;默认视角静止,滚轮可调 zoom。 const n = valid.length; const baseCentroid = valid.reduce( (a, p) => [a[0] + p[0], a[1] + p[1], a[2] + p[2]], [0, 0, 0]) .map(v => v / n); const centroid = baseCentroid.map((value, index) => value + (targetOffset && Number.isFinite(targetOffset[index]) ? targetOffset[index] : 0)); const fov = 45 * Math.PI / 180; if (_h3dBaseDist === null && !Number.isFinite(baseDistOverride)) { const span = Math.max( Math.max(...valid.map(p => p[0])) - Math.min(...valid.map(p => p[0])), Math.max(...valid.map(p => p[1])) - Math.min(...valid.map(p => p[1])), Math.max(...valid.map(p => p[2])) - Math.min(...valid.map(p => p[2]))); const half = span > 1e-6 ? span / 2 : 0.3; // 5.0 倍系数:手约占画面高度 1/5(默认视角按需求再拉远); // 距离下限 0.5m:手离相机远(跨度小)时默认视角也不会怼脸, // 需要更近/更远仍可用滚轮缩放 _h3dBaseDist = Math.min(2.0, Math.max(0.5, 6.0 * half / Math.tan(fov / 2))); } const baseDist = Number.isFinite(baseDistOverride) ? baseDistOverride : _h3dBaseDist; // Keep the default at 2m, but allow the user to zoom farther out when // inspecting a handless/world-only frame. const dist = Math.min(HAND3D_MAX_CAMERA_DISTANCE, Math.max(0.2, baseDist / (zoom || 1))); const theta = Math.PI + (rotY || 0); // 静态正面 + 拖拽偏转 // 仰角钳制 ±80°(标准轨道相机 pitch clamp):越过 ±90° 时 // right = upW×fwd 与 up 基同时反向,画面绕屏幕中心 180° 翻转 // —— 即"拖拽到达中心点后手部位置突变"的来源 const elev = Math.max(-80 * Math.PI / 180, Math.min(80 * Math.PI / 180, HAND3D_DEFAULT_ELEVATION * Math.PI / 180 + (rotX || 0))); const ce = Math.cos(elev), se = Math.sin(elev); const eye = [ centroid[0] + dist * Math.sin(theta) * ce, // 世界 Y 已统一为向上,因此正仰角时相机应位于目标上方。 centroid[1] + dist * se, centroid[2] + dist * Math.cos(theta) * ce, ]; // look-at 基:处理链路输出的世界坐标是 X 右、Y 上、Z 向前。 // Canvas 的屏幕 Y 由投影公式负责翻转,因此这里不能再把世界上方向 // 写成 -Y,否则 X/Y 轴都会被镜像,右侧空间视图会和原图相反。 let fwd = [centroid[0] - eye[0], centroid[1] - eye[1], centroid[2] - eye[2]]; const fn = Math.hypot(fwd[0], fwd[1], fwd[2]) + 1e-9; fwd = fwd.map(v => v / fn); const upW = [0, 1, 0]; let right = [ upW[1] * fwd[2] - upW[2] * fwd[1], upW[2] * fwd[0] - upW[0] * fwd[2], upW[0] * fwd[1] - upW[1] * fwd[0], ]; const rn = Math.hypot(right[0], right[1], right[2]) + 1e-9; right = right.map(v => v / rn); const up = [ fwd[1] * right[2] - fwd[2] * right[1], fwd[2] * right[0] - fwd[0] * right[2], fwd[0] * right[1] - fwd[1] * right[0], ]; return { centroid, eye, fwd, right, up, f: (h / 2) / Math.tan(fov / 2), w, h }; } function _projectHand3D(p, cam) { // 透视投影(demo _project 同款):u = cx + f·(d·right)/z,v = cy − f·(d·up)/z if (!p || !Number.isFinite(p[0]) || !Number.isFinite(p[1]) || !Number.isFinite(p[2])) { return [NaN, NaN]; } const d = [p[0] - cam.eye[0], p[1] - cam.eye[1], p[2] - cam.eye[2]]; const z = d[0] * cam.fwd[0] + d[1] * cam.fwd[1] + d[2] * cam.fwd[2]; if (z <= 1e-4) return [NaN, NaN]; const u = cam.w / 2 + cam.f * (d[0] * cam.right[0] + d[1] * cam.right[1] + d[2] * cam.right[2]) / z; const v = cam.h / 2 - cam.f * (d[0] * cam.up[0] + d[1] * cam.up[1] + d[2] * cam.up[2]) / z; return [u, v]; } // 透视视图中的骨骼长度已经包含了距离缩放。使用它计算画笔尺寸, // 避免手移动到远处后,固定像素半径的关节点互相盖住而看起来像圆球。 function _hand3dProjectedLength(a, b) { if (!a || !b || !Number.isFinite(a[0]) || !Number.isFinite(a[1]) || !Number.isFinite(b[0]) || !Number.isFinite(b[1])) return 0; return Math.hypot(a[0] - b[0], a[1] - b[1]); } function _hand3dStrokeWidth(a, b, maxWidth) { const length = _hand3dProjectedLength(a, b); // 远处骨骼最小保留约 0.5px,近处不超过原来的视觉宽度。 return Math.max(0.5, Math.min(maxWidth, length * 0.14)); } function _hand3dStyleScale(proj) { const valid = (proj || []).filter(p => p && Number.isFinite(p[0]) && Number.isFinite(p[1])); if (valid.length < 2) return 1; const xs = valid.map(p => p[0]), ys = valid.map(p => p[1]); const span = Math.max(Math.max(...xs) - Math.min(...xs), Math.max(...ys) - Math.min(...ys)); // Same reference size as the left SVG/hand_render style. return Math.max(0.45, Math.min(1, span / 160)); } function _normalizeRgbEstimatedDisplayScale(slots, entry) { const handSpans = []; const all = []; for (const slot of slots) { const valid = slot.valid || []; if (valid.length < 4) continue; const xs = valid.map(p => p[0]), ys = valid.map(p => p[1]); handSpans.push(Math.max(Math.max(...xs) - Math.min(...xs), Math.max(...ys) - Math.min(...ys))); all.push(...valid); } if (!handSpans.length || !all.length) return 1; const measured = handSpans.reduce((sum, value) => sum + value, 0) / handSpans.length; if (!(measured > 1e-6)) return 1; const targetScale = Math.max(RGB_ESTIMATED_DISPLAY_SCALE_MIN, Math.min(RGB_ESTIMATED_DISPLAY_SCALE_MAX, RGB_ESTIMATED_DISPLAY_HAND_SPAN / measured)); // Do not resize the whole RGB skeleton independently on every frame. // Small 2D/PnP scale changes otherwise become visible as breathing and // jitter after the wrist root is anchored in the world tile. let scale = targetScale; if (entry) { const previous = Number(entry._rgbDisplayScale); scale = Number.isFinite(previous) ? previous * 0.92 + targetScale * 0.08 : targetScale; entry._rgbDisplayScale = scale; } if (Math.abs(scale - 1) < 1e-3) return 1; const center = all.reduce((sum, p) => [ sum[0] + p[0], sum[1] + p[1], sum[2] + p[2], ], [0, 0, 0]).map(value => value / all.length); for (const slot of slots) { slot.pts = slot.pts.map(p => p ? [ center[0] + (p[0] - center[0]) * scale, center[1] + (p[1] - center[1]) * scale, center[2] + (p[2] - center[2]) * scale, ] : null); slot.fin = slot.pts.map(Boolean); slot.valid = slot.pts.filter(Boolean); } return scale; } function _normalizeDepthWorldDisplayScale(slots, entry, w, h) { // Calculate once from the first usable frame. Recomputing this per frame // would make the skeleton visibly breathe while playing. let scale = Number(entry?._worldDisplayScale); if (!Number.isFinite(scale)) { const spans = []; for (const slot of slots) { const valid = slot.valid || []; if (valid.length < 4) continue; // Ignore isolated depth outliers when choosing the fixed preview // scale. A single bad landmark in #3's first frame was metres // away from the hand and permanently made later frames tiny. const robustSpan = values => { const sorted = values.filter(Number.isFinite).sort((a, b) => a - b); if (!sorted.length) return 0; const quantile = fraction => sorted[Math.floor( (sorted.length - 1) * fraction)]; return quantile(0.95) - quantile(0.05); }; spans.push(Math.max( robustSpan(valid.map(p => p[0])), robustSpan(valid.map(p => p[1])), robustSpan(valid.map(p => p[2])), )); } const measured = spans.length ? spans.reduce((sum, value) => sum + value, 0) / spans.length : 0; const fov = 45 * Math.PI / 180; const focal = (Math.max(1, h) / 2) / Math.tan(fov / 2); const targetPx = Math.max(HAND3D_DISPLAY_SPAN_MIN_PX, Math.min(HAND3D_DISPLAY_SPAN_MAX_PX, Math.min(w, h) * HAND3D_DISPLAY_SPAN_RATIO)); // At the fixed world camera distance, convert the target screen span // into a metric display span. This is only a preview transform. const targetWorldSpan = targetPx * HAND3D_WORLD_DEFAULT_DISTANCE / focal; scale = measured > 1e-6 ? Math.max(HAND3D_DISPLAY_SCALE_MIN, Math.min(HAND3D_DISPLAY_SCALE_MAX, targetWorldSpan / measured)) : 1; if (entry) entry._worldDisplayScale = scale; } if (Math.abs(scale - 1) < 1e-3) return scale; slots.forEach(slot => { const wrist = slot.pts[0] || slot.valid[0]; if (!wrist) return; slot.pts = slot.pts.map(point => point ? [ wrist[0] + (point[0] - wrist[0]) * scale, wrist[1] + (point[1] - wrist[1]) * scale, wrist[2] + (point[2] - wrist[2]) * scale, ] : null); slot.fin = slot.pts.map(Boolean); slot.valid = slot.pts.filter(Boolean); }); return scale; } // A few D435 frames contain landmark coordinates that jump metres away from // the wrist. Keep those points out of the 3D preview only. The parquet/API // values remain untouched, and a real hand landmark cannot be this far from // its wrist in the camera coordinate system. const HAND3D_WORLD_DISPLAY_MAX_HAND_RADIUS = 0.35; function _filterDepthWorldDisplayOutliers(slots) { slots.forEach(slot => { const points = slot.pts || []; const wrist = points[0] || points.find(Boolean); if (!wrist) return; let validCount = 0; let compactCount = 0; const maxRadius = HAND3D_WORLD_DISPLAY_MAX_HAND_RADIUS; const keep = points.map(point => { if (!point) return false; validCount += 1; const dx = point[0] - wrist[0]; const dy = point[1] - wrist[1]; const dz = point[2] - wrist[2]; const isCompact = Math.hypot(dx, dy, dz) <= maxRadius; if (isCompact) compactCount += 1; return isCompact; }); // Do not blank a sparse hand just because it has one questionable // point. Only apply the filter when a usable compact skeleton remains. if (compactCount < 4 || compactCount === validCount) return; slot.pts = points.map((point, index) => point && keep[index] ? point : null); slot.fin = slot.pts.map(Boolean); slot.valid = slot.pts.filter(Boolean); }); } function _hand3dJointRadius(proj, index, tip = false, styleScale = 1) { if (!proj[index] || !Number.isFinite(proj[index][0]) || !Number.isFinite(proj[index][1])) return 0; const lengths = []; for (const [a, b] of HAND3D_CONNECTIONS) { if (a !== index && b !== index) continue; const length = _hand3dProjectedLength(proj[a], proj[b]); if (length > 0) lengths.push(length); } const preferred = (tip ? 7 : 5) * styleScale; if (!lengths.length) return Math.max(0.65, preferred); const mean = lengths.reduce((sum, value) => sum + value, 0) / lengths.length; // Prefer the canonical overlay size, but cap it by the projected local // bone size so distant joints never cover one another. return Math.max(0.65, Math.min(preferred, mean * (tip ? 0.18 : 0.16))); } function _drawHand3DWorldReference(ctx, w, h, state) { if (!state || !state.cam || !state.anchor) return; const { cam, anchor } = state; const px = p => _projectHand3D(p, cam); const line = (a, b, color, width) => { if (!Number.isFinite(a[0]) || !Number.isFinite(a[1]) || !Number.isFinite(b[0]) || !Number.isFinite(b[1])) return; ctx.strokeStyle = color; ctx.lineWidth = _hand3dStrokeWidth(a, b, width); ctx.beginPath(); ctx.moveTo(a[0], a[1]); ctx.lineTo(b[0], b[1]); ctx.stroke(); }; const yGrid = Number.isFinite(state.gridY) ? state.gridY : -0.25; // 网格是世界坐标的一部分,平移视角时不能跟着相机目标重新居中。 const zc = anchor[2], xc = anchor[0]; for (let x = xc - 0.4; x <= xc + 0.401; x += 0.05) { line(px([x, yGrid, zc - 0.45]), px([x, yGrid, zc + 0.45]), 'rgba(58,58,58,0.9)', 1); } for (let z = zc - 0.45; z <= zc + 0.451; z += 0.05) { line(px([xc - 0.4, yGrid, z]), px([xc + 0.4, yGrid, z]), 'rgba(58,58,58,0.9)', 1); } const axes = [[[0.1, 0, 0], 'rgb(60,60,255)', 'X'], [[0, 0.1, 0], 'rgb(60,200,60)', 'Y'], [[0, 0, 0.1], 'rgb(255,120,60)', 'Z']]; for (const [v, color, name] of axes) { const o = px(anchor); const e = px([anchor[0] + v[0], anchor[1] + v[1], anchor[2] + v[2]]); line(o, e, color, 1); if (Number.isFinite(e[0])) { ctx.fillStyle = color; ctx.font = '9px sans-serif'; ctx.fillText(name, e[0] + 3, e[1] - 3); } } } function _worldReferenceCam(state, w, h, entry) { if (!state) return null; if (state.center && Number.isFinite(state.baseDist)) { return _hand3dCam( [state.center], w, h, entry.rotY || 0, entry.rotX || 0, entry.zoom || 1, state.baseDist, entry.pan || [0, 0, 0]); } return state.cam || null; } function _defaultHand3DWorldReference(w, h, entry) { const anchor = HAND3D_WORLD_DEFAULT_CENTER.slice(); const gridY = HAND3D_WORLD_DEFAULT_GRID_Y; return { center: anchor.slice(), baseDist: HAND3D_WORLD_DEFAULT_DISTANCE, // 视距固定在安全默认值。不能用首个有效帧拟合:首帧可能只有 // 一只手或只有部分关键点,会让网格和后续双手画面突然缩放。 fitted: true, anchor, gridY, rootAnchors: { left: [anchor[0] - HAND3D_ROOT_ANCHOR_X, gridY + HAND3D_ROOT_ANCHOR_LIFT, anchor[2]], right: [anchor[0] + HAND3D_ROOT_ANCHOR_X, gridY + HAND3D_ROOT_ANCHOR_LIFT, anchor[2]], }, }; } function _h3dRootAnchorKey(label, slotIndex) { const value = String(label || '').toLowerCase(); if (value === 'left') return 'left'; if (value === 'right') return 'right'; // Older artifacts may not carry handedness. Keep a deterministic // fallback without changing the anchor when the other hand appears. return slotIndex === 0 ? 'left' : 'right'; } function _h3dStableAnchorSide(entry, label, slotIndex) { if (!entry._anchorSides) entry._anchorSides = [null, null]; const value = String(label || '').toLowerCase(); const reported = value === 'left' || value === 'right' ? value : null; if (!entry._anchorSides[slotIndex] && reported) { const other = entry._anchorSides[slotIndex === 0 ? 1 : 0]; // If two unstable labels claim the same side, preserve two distinct // display anchors rather than letting the hands overlap. entry._anchorSides[slotIndex] = (other && other === reported) ? (reported === 'left' ? 'right' : 'left') : reported; } return entry._anchorSides[slotIndex] || (slotIndex === 0 ? 'left' : 'right'); } function _h3dRootAnchorApply(ptsBySlot, labels, state) { const rootAnchors = state && state.rootAnchors; if (!rootAnchors) return ptsBySlot; if (!state.lastRawRoots) state.lastRawRoots = [null, null]; return ptsBySlot.map((points, slotIndex) => { const root = points && points[0] ? points[0] : state.lastRawRoots[slotIndex]; if (!root || !root.every(Number.isFinite)) { return points.map(point => point ? point.slice() : null); } if (points && points[0]) state.lastRawRoots[slotIndex] = points[0].slice(); const anchor = rootAnchors[ _h3dRootAnchorKey(labels[slotIndex], slotIndex) ]; return points.map(point => point ? [ point[0] - root[0] + anchor[0], point[1] - root[1] + anchor[1], point[2] - root[2] + anchor[2], ] : null); }); } function _h3dApplyFixedDisplayAnchors(slots, spaceMode, entry) { // The preview is intentionally a fixed layout. RGB-only coordinates are // camera-relative estimates, and MediaPipe-relative coordinates are // hand-local, so neither source can reliably define the two-hand layout. // Apply the same explicit wrist targets on every render path. const centerZ = spaceMode ? HAND3D_WORLD_DEFAULT_CENTER[2] : 0; const centerY = spaceMode ? HAND3D_WORLD_DEFAULT_GRID_Y + HAND3D_ROOT_ANCHOR_LIFT : 0; // Old RGB artifacts may contain duplicate handedness labels (commonly // both hands reported as Right). The original 2D overlay is slot-based, // so it still looks correct, but label-only anchoring mirrors the two // hands in the 3D tile. For that legacy case, initialize the semantic // sides from the raw camera-space wrist X order and then keep them fixed. const reported = slots.map(slot => String(slot.label || '').toLowerCase()); const duplicateLabels = reported.length === 2 && reported[0] && reported[0] === reported[1]; if (duplicateLabels && !entry._duplicateAnchorSides) { const ordered = slots .map((slot, index) => ({ index, wrist: slot.pts[0] || slot.valid[0] })) .filter(item => item.wrist && Number.isFinite(item.wrist[0])) .sort((a, b) => a.wrist[0] - b.wrist[0]); if (ordered.length === 2) { entry._duplicateAnchorSides = []; entry._duplicateAnchorSides[ordered[0].index] = 'left'; entry._duplicateAnchorSides[ordered[1].index] = 'right'; } } slots.forEach((slot, slotIndex) => { const wrist = slot.pts[0] || slot.valid[0]; if (!wrist) return; // Lock the semantic side once per tile. A transient per-frame // Left/Right classification must never move a whole hand to the // other side of the canvas, while the initial semantic orientation // remains compatible with both black-glove and bare-hand outputs. const side = duplicateLabels && entry._duplicateAnchorSides ? entry._duplicateAnchorSides[slotIndex] : _h3dStableAnchorSide(entry, slot.label, slotIndex); const target = [ side === 'left' ? -HAND3D_ROOT_ANCHOR_X : HAND3D_ROOT_ANCHOR_X, centerY, centerZ, ]; slot.pts = slot.pts.map(point => point ? [ point[0] - wrist[0] + target[0], point[1] - wrist[1] + target[1], point[2] - wrist[2] + target[2], ] : null); slot.fin = slot.pts.map(Boolean); slot.valid = slot.pts.filter(Boolean); }); } function _ensureHand3DWorldReference(w, h, entry) { const sourceData = hand3dDataBySource[entry.source.source_key] || hand3dData; if (!sourceData || !sourceData.spaceMode) return null; if (!entry._worldReference) entry._worldReference = _defaultHand3DWorldReference(w, h, entry); // Use the same world-space anchor for every episode. Hand data is shifted // relative to this stable reference, so its arrival cannot move the view. if (_hand3dViewAnchor === null) _hand3dViewAnchor = entry._worldReference.anchor.slice(); if (_h3dGridY === null) _h3dGridY = HAND3D_WORLD_DEFAULT_GRID_Y; entry._worldReference.cam = _worldReferenceCam(entry._worldReference, w, h, entry); return entry._worldReference; } function renderHand3DTile(entry, frameOverride) { if (!entry || !entry.canvas) return; const canvas = entry.canvas; const rect = canvas.getBoundingClientRect(); const dpr = window.devicePixelRatio || 1; const w = Math.max(1, rect.width), h = Math.max(1, rect.height); if (canvas.width !== Math.round(w * dpr) || canvas.height !== Math.round(h * dpr)) { canvas.width = Math.round(w * dpr); canvas.height = Math.round(h * dpr); } const ctx = canvas.getContext('2d'); ctx.setTransform(dpr, 0, 0, dpr, 0, 0); const frame = typeof frameOverride === 'number' ? frameOverride : (typeof currentFrameTarget === 'number' ? currentFrameTarget : 0); _hand3dTileEntry = entry; const sourceKey = entry.source.source_key || 'default'; const sourceData = hand3dDataBySource[sourceKey] || hand3dData; const worldMode = !!(sourceData && sourceData.worldMode); const spaceMode = !!(sourceData && sourceData.spaceMode); const cameraYDown = !!(sourceData && sourceData.cameraYDown); let fd = _hand3dCached(frame, sourceKey); const frameCache = _hand3dFrameCacheFor(sourceKey); if (!spaceMode && frameCache.frame === frame && frameCache.data) { fd = { h0: frameCache.data.h0 || null, h1: frameCache.data.h1 || null, }; } if (sourceData && (sourceData.hasPreview || spaceMode)) { if (spaceMode) fetchHand3DWindow(frame, sourceKey); else fetchHand3DFrame(frame, sourceKey); } // 缓存窗口切换期间保留上一张已提交的 3D 画面,不清空画布。 // 数据返回后本函数再次调用,并在下面一次性提交新帧,避免闪烁。 // 若某帧确实没有手部数据,fd 是非 null 的空对象,仍会正常清空并 // 绘制参考系;因此这里只处理“尚未拿到当前帧”的网络等待状态。 if (fd === null && entry._hasCommittedFrame) return; ctx.clearRect(0, 0, w, h); ctx.fillStyle = 'rgba(3,7,18,0.95)'; ctx.fillRect(0, 0, w, h); if (!fd || (!fd.h0 && !fd.h1)) { if (spaceMode) { _ensureHand3DWorldReference(w, h, entry); _drawHand3DWorldReference(ctx, w, h, entry._worldReference); return; } return; } entry._hasCommittedFrame = true; // 收集有效点(世界坐标模式:d435 相机系米制) const slots = [ { key: 'h0', label: fd.h0 && fd.h0.label ? String(fd.h0.label) : '' }, { key: 'h1', label: fd.h1 && fd.h1.label ? String(fd.h1.label) : '' }, ]; const all = []; for (const s of slots) { const hand = fd[s.key]; s.pts = hand && hand.lm ? hand.lm.map(p => ( p && Number.isFinite(p[0]) && Number.isFinite(p[1]) && Number.isFinite(p[2]) ? [p[0], cameraYDown ? -p[1] : p[1], p[2]] : null)) : []; s.fin = s.pts.map(Boolean); s.valid = s.pts.filter(Boolean); all.push(...s.valid); } if (all.length < 4) { if (spaceMode) { _ensureHand3DWorldReference(w, h, entry); } if (spaceMode && entry._worldReference) { entry._worldReference.cam = _worldReferenceCam(entry._worldReference, w, h, entry); _drawHand3DWorldReference(ctx, w, h, entry._worldReference); return; } return; } // Keep the stored RGB estimate untouched. This is a presentation-only // normalization so RGB and depth tiles occupy a comparable visual scale. if (sourceData && sourceData.rgbEstimatedMode) { _normalizeRgbEstimatedDisplayScale(slots, entry); all.length = 0; slots.forEach(s => all.push(...s.valid)); } if (sourceData && sourceData.worldMode) { _filterDepthWorldDisplayOutliers(slots); all.length = 0; slots.forEach(s => all.push(...s.valid)); _normalizeDepthWorldDisplayScale(slots, entry, w, h); all.length = 0; slots.forEach(s => all.push(...s.valid)); } if (spaceMode) _ensureHand3DWorldReference(w, h, entry); if (spaceMode && entry._worldReference && entry._worldReference.fitted !== true) { // 兼容旧的 tile 状态,但不再依据当前帧的关键点尺度拟合。 // 尤其不能让首个单手帧把相机拉近,否则第二只手出现时视角会跳。 entry._worldReference.baseDist = HAND3D_WORLD_DEFAULT_DISTANCE; entry._worldReference.fitted = true; } // All 3D preview modes use the same fixed wrist layout. This is a // display-only transform; the original camera-relative coordinates in // parquet/API remain unchanged. _h3dApplyFixedDisplayAnchors(slots, spaceMode, entry); all.length = 0; slots.forEach(s => all.push(...s.valid)); // 视图锚点:世界坐标模式使用真实的固定相机坐标,不按当前可见手的 // 质心平移。否则双手变单手时 c0 会改变,剩余手会被人为挪动。 // 相对 3D 仍按腕点独立归一化。 if (!spaceMode && _hand3dViewAnchor === null) { const c0 = all.reduce((a, p) => [a[0] + p[0], a[1] + p[1], a[2] + p[2]], [0, 0, 0]) .map(v => v / all.length); _hand3dViewAnchor = c0.slice(); } // 世界坐标保留处理链路输出的绝对相机坐标。anchor 只用于 // 相机/网格参考,不再把当前帧整体搬到 anchor。相对 3D 已在 // 上面的腕点归一化阶段完成布局。 const shifted = all.map(p => p.slice()); let cam; if (spaceMode) { // The current hand points move and render, but never redefine the // shared initial world-coordinate viewpoint. entry._worldReference.cam = _worldReferenceCam(entry._worldReference, w, h, entry); cam = entry._worldReference.cam; } else { cam = _hand3dCam(shifted, w, h, entry.rotY || 0, entry.rotX || 0, entry.zoom || 1, null, entry.pan || [0, 0, 0]); } entry._lastCam = cam; const px = p => _projectHand3D(p, cam); const line = (a, b, color, width, styleScale = 1) => { if (!Number.isFinite(a[0]) || !Number.isFinite(a[1]) || !Number.isFinite(b[0]) || !Number.isFinite(b[1])) return; ctx.strokeStyle = color; ctx.lineWidth = _hand3dStrokeWidth(a, b, width * styleScale); ctx.beginPath(); ctx.moveTo(a[0], a[1]); ctx.lineTo(b[0], b[1]); ctx.stroke(); }; if (spaceMode) { // 世界坐标参考系独立于当前帧关键点,关键点缺失时仍保留。 _drawHand3DWorldReference(ctx, w, h, entry._worldReference); } // painter 算法:远手先画(demo order.sort(key=-depth)) const order = slots .filter(s => s.valid.length) .map(s => ({ s, depth: s.valid.reduce((a, p) => a + ((p[0] - cam.eye[0]) * cam.fwd[0] + (p[1] - cam.eye[1]) * cam.fwd[1] + (p[2] - cam.eye[2]) * cam.fwd[2]), 0) / s.valid.length, })) .sort((a, b) => b.depth - a.depth); for (const { s } of order) { const proj = s.pts.map(p => px(p)); const styleScale = _hand3dStyleScale(proj); // 掌心灰线 for (const [a, b] of HAND3D_PALM) { if (s.fin[a] && s.fin[b]) { line(proj[a], proj[b], 'rgb(200,200,200)', 2, styleScale); } } // 五指分色链(拇指 1→2→3→4;其余 0→MCP→…→指尖) for (const f of HAND3D_FINGERS) { const chain = f.name === 'Thumb' ? f.ids : [0, ...f.ids]; for (let i = 0; i < chain.length - 1; i++) { if (s.fin[chain[i]] && s.fin[chain[i + 1]]) { line(proj[chain[i]], proj[chain[i + 1]], `rgb(${f.color[0]},${f.color[1]},${f.color[2]})`, 3, styleScale); } } for (const idx of f.ids) { if (!s.fin[idx]) continue; const r = _hand3dJointRadius( proj, idx, idx === f.ids[f.ids.length - 1], styleScale); if (r <= 0) continue; ctx.fillStyle = `rgb(${f.color[0]},${f.color[1]},${f.color[2]})`; ctx.beginPath(); ctx.arc(proj[idx][0], proj[idx][1], r, 0, Math.PI * 2); ctx.fill(); ctx.strokeStyle = 'rgb(30,30,30)'; ctx.lineWidth = Math.max(0.5, Math.min(1, r * 0.22)); ctx.stroke(); } } // 腕部白点 if (s.fin[0]) { const wristRadius = Math.max(0.65, Math.min( 9 * styleScale, _hand3dJointRadius(proj, 0, false, styleScale) * 1.45)); ctx.fillStyle = '#fff'; ctx.beginPath(); ctx.arc(proj[0][0], proj[0][1], Math.max(1.2, Math.min(7, wristRadius)), 0, Math.PI * 2); ctx.fill(); ctx.strokeStyle = 'rgb(40,40,40)'; ctx.lineWidth = Math.max(0.5, Math.min(1.5, wristRadius * 0.22)); ctx.stroke(); } // 腕部深度标注只在相机米制世界坐标中显示。 if (worldMode && s.fin[0] && Number.isFinite(proj[0][0])) { ctx.fillStyle = s.label === 'Right' ? 'rgb(0,255,0)' : 'rgb(255,200,0)'; ctx.font = '11px sans-serif'; ctx.fillText(`${s.label} z=${s.pts[0][2].toFixed(2)}m`, proj[0][0] + 12, proj[0][1] - 12); } } // HUD ctx.fillStyle = 'rgba(235,235,235,0.9)'; ctx.font = '11px sans-serif'; ctx.textAlign = 'left'; ctx.fillText(`frame ${frame}`, 12, 22); // The interaction/mode hint was only a debug HUD and is intentionally // omitted from the spatial preview. } function initHand3dDrag(entry) { if (!entry || !entry.canvas) return; const canvas = entry.canvas; canvas.style.cursor = 'grab'; canvas.addEventListener('mousedown', e => { if (e.button !== 0 && e.button !== 1) return; entry.dragging = true; entry.dragMode = e.button === 1 ? 'pan' : 'rotate'; entry.lastX = e.clientX; entry.lastY = e.clientY; canvas.style.cursor = entry.dragMode === 'pan' ? 'move' : 'grabbing'; e.preventDefault(); }); window.addEventListener('mousemove', e => { if (!entry.dragging) return; // 左键是轨道旋转:水平向右看右侧,垂直向上看上方。 const dx = e.clientX - entry.lastX; const dy = e.clientY - entry.lastY; if (entry.dragMode === 'pan') { // 让空间像被鼠标抓住一样移动:拖右→内容右移,拖下→内容下移。 // 使用当前相机 right/up 基向量,旋转后平移仍符合屏幕方向。 const cam = entry._lastCam; if (cam) { const distance = Math.max(0.2, Math.hypot( cam.eye[0] - cam.centroid[0], cam.eye[1] - cam.centroid[1], cam.eye[2] - cam.centroid[2])); const worldPerPixel = distance / Math.max(1, cam.f); const pan = entry.pan || (entry.pan = [0, 0, 0]); for (let i = 0; i < 3; i++) { pan[i] += (-dx * cam.right[i] + dy * cam.up[i]) * worldPerPixel; } } } else { // 水平向右拖动查看右侧;向上拖动查看上方。 entry.rotY += dx * 0.01; entry.rotX += dy * 0.01; } entry.lastX = e.clientX; entry.lastY = e.clientY; renderHand3DTile(entry); }); window.addEventListener('mouseup', () => { if (entry.dragging) { entry.dragging = false; entry.dragMode = ''; canvas.style.cursor = 'grab'; } }); canvas.addEventListener('contextmenu', e => e.preventDefault()); // 滚轮缩放(世界坐标模式:0.3x – 6x) canvas.addEventListener('wheel', e => { e.preventDefault(); entry.zoom = Math.min(6, Math.max(0.3, (entry.zoom || 1) * Math.exp(-e.deltaY * 0.0015))); renderHand3DTile(entry); }, { passive: false }); } /* 逐帧步进回调:3D 世界窗口随统一帧号重绘(多回调机制,与骨骼叠加层/ 标注高亮共用;闭包在调用时遍历,脚本加载时 tile 列表为空无影响) */ setOnFrameChange((f) => { for (const entry of currentHand3dTiles) renderHand3DTile(entry, f); for (const entry of currentSlamTiles) renderSlamTile(entry, f); }); /* 图像素材(深度图/手套热力图)的逐帧 rAF 刷新循环(见 _startImageTileRaf) */ if (document.readyState !== 'loading') _startImageTileRaf(); else document.addEventListener('DOMContentLoaded', _startImageTileRaf); /* 漂浮预览控件的显隐:点开批次详情后才显示(☷ 与 Slice Preview 在标注/审核/审核通过页面可用);返回批次列表时隐藏。 */ function setPreviewPanelsVisible(open) { window.__episodeOpen = !!open; const po = document.getElementById('preview-options'); if (po) po.classList.toggle('hidden', !open); const menu = document.getElementById('preview-options-menu'); if (!open && menu) menu.classList.add('hidden'); if (open) { renderPreviewVideoSources(); if (typeof updatePreviewMenuData === 'function') updatePreviewMenuData(); } const pt = document.getElementById('playback-toolbar'); if (pt) pt.classList.toggle('hidden', !open); if (!open && typeof resetSlicePreview === 'function') resetSlicePreview(''); if (typeof refreshSlicePreview === 'function') refreshSlicePreview(); } window.ensurePreviewOptionsVisible = function () { if (window.__episodeOpen !== true) return; const box = document.getElementById('preview-options'); if (box) box.classList.remove('hidden'); }; /* 诊断入口:播放按钮"点了没反应"时,在浏览器控制台(F12 → Console)输入 __playerDebug() 回车,把输出发过来 —— 一次定位卡在哪个状态。 */ window.__playerDebug = function () { return { episode: currentEpisodeId, playerReady: playerReady, frameDataReady: frameDataReady, controlsEnabled: _controlsEnabled, players: Object.keys(players), activePlayer: Boolean(getActivePlayer()), workspaceSources: workspaceSources.map(s => groupedSourceKey(s)), overlayHidden: (document.getElementById('video-loading-overlay') || {}) .classList.contains('hidden'), }; };