Spaces:
Running
Running
stouchinglhh
Claude
sync processor from GitLab c697b6e: 版本 1.19.4 / 前端 1.12.4 —— 服务端 i18n(首屏不再闪英文)、推理模块并入模型页、前端抽出 shared/、缓存键改用前端号
bb59211 | /* Plyr video player management */ | |
| console.log('[player.js] build 20260911-tactile-episode-color-scale'); | |
| const players = {}; // {camera_name: Plyr instance} | |
| let currentEpisodeId = null; | |
| let currentCameras = []; | |
| let _episodeMediaLoadController = null; | |
| // Every episode selection gets a new generation. Episode ids normally differ, | |
| // but reopening the same episode must also invalidate old async callbacks. | |
| let _playbackSessionToken = 0; | |
| let workspaceSources = []; | |
| let workspaceCameras = []; | |
| let currentHasSkeleton = false; | |
| let _defaultLayoutApplied = false; // balanced layout (videos + hand heatmaps) applied once | |
| // The review canvas always plays the raw camera video. Hand skeletons are | |
| // rendered by hand-overlay.js as an SVG layer on top of that video. | |
| async function pickVideoUrl(episodeId, camera, preferSkeleton = false) { | |
| return `/api/v1/video/${episodeId}/${camera}/preview-stream`; | |
| } | |
| function getMediaLoadSignal() { | |
| return _episodeMediaLoadController ? _episodeMediaLoadController.signal : null; | |
| } | |
| function getPlaybackSessionToken() { | |
| return _playbackSessionToken; | |
| } | |
| function isCurrentPlaybackSession(episodeId, token) { | |
| return currentEpisodeId === episodeId && | |
| (token == null || token === _playbackSessionToken); | |
| } | |
| function _destroyEpisodePlayers() { | |
| Object.values(players).forEach(player => { | |
| try { player.pause(); } catch (_) {} | |
| try { player.destroy(); } catch (_) {} | |
| }); | |
| Object.keys(players).forEach(key => delete players[key]); | |
| } | |
| function _cancelEpisodeMediaLoad(clearCanvas = true) { | |
| // Invalidate callbacks before aborting requests. Abort is not guaranteed | |
| // to prevent a response already resolved from reaching its .then(). | |
| _playbackSessionToken += 1; | |
| if (_episodeMediaLoadController) { | |
| try { _episodeMediaLoadController.abort(); } catch (_) {} | |
| _episodeMediaLoadController = null; | |
| } | |
| clearTimeout(_loadWatchdog); | |
| masterPlaying = false; | |
| try { pauseAll(); } catch (_) {} | |
| if (typeof stopHeatmapSync === 'function') stopHeatmapSync(); | |
| if (typeof cancelSegmentPlayback === 'function') cancelSegmentPlayback(); | |
| if (typeof disconnectAnnotationSocket === 'function') disconnectAnnotationSocket(); | |
| _destroyEpisodePlayers(); | |
| currentImageTiles = []; | |
| currentDepthPreviewTiles = []; | |
| currentHand3dTiles = []; | |
| currentSlamTiles = []; | |
| currentTactileTiles = []; | |
| workspaceGrid = null; | |
| workspaceMainRow = null; | |
| workspaceHandFooter = null; | |
| workspaceHandSlots = { left: null, right: null }; | |
| workspaceBottomExtra = null; | |
| if (typeof clearHandTiles === 'function') clearHandTiles(); | |
| if (typeof destroyAllHandOverlays === 'function') destroyAllHandOverlays(); | |
| if (clearCanvas) { | |
| const grid = document.getElementById('video-grid'); | |
| if (grid) { | |
| grid.innerHTML = '<div class="flex items-center justify-center h-64 text-gray-600">' + | |
| '<div><div class="text-4xl mb-3 text-center"><iconify-icon icon="ant-design:video-camera-outlined"></iconify-icon></div>' + | |
| '<div>Select an episode to review →</div></div></div>'; | |
| } | |
| } | |
| } | |
| function clearEpisodeWorkspace() { | |
| _cancelEpisodeMediaLoad(true); | |
| currentEpisodeId = null; | |
| currentCameras = []; | |
| currentMediaGroups = null; | |
| workspaceSources = []; | |
| workspaceCameras = []; | |
| workspaceGrid = null; | |
| workspaceMainRow = null; | |
| workspaceHandFooter = null; | |
| workspaceHandSlots = { left: null, right: null }; | |
| workspaceBottomExtra = null; | |
| if (typeof clearAnnotationsNow === 'function') clearAnnotationsNow(); | |
| hand3dData = null; | |
| hand3dDataBySource = {}; | |
| hand3dFrameCache = { frame: -1, data: null, inflight: -1 }; | |
| hand3dFrameCacheBySource = {}; | |
| hand3dWindow = { start: -1, end: -1, frames: {}, inflight: false }; | |
| hand3dWindowBySource = {}; | |
| episodeTotalFrames = 0; | |
| episodeFps = 0; | |
| frameDataReady = false; | |
| const frameLabel = document.getElementById('annotation-timeline-frame'); | |
| if (frameLabel) frameLabel.textContent = '0 / 0'; | |
| disableFrameControls(); | |
| hideVideoLoading(); | |
| resetSourceBar(); | |
| } | |
| function initPlayer(containerId, videoUrl) { | |
| const container = document.getElementById(containerId); | |
| if (!container) return null; | |
| // Remove existing video element | |
| container.innerHTML = ` | |
| <video id="${containerId}-video" controls crossorigin playsinline preload="auto"></video> | |
| `; | |
| const player = new Plyr(`#${containerId}-video`, { | |
| // Frame-aligned mode: individual video controls are hidden. A single | |
| // master play/pause button + seek bar above the canvas drives every | |
| // video at exactly the same frame (see bindMasterSync). | |
| controls: [], | |
| keyboard: { focused: true, global: true }, | |
| }); | |
| player.source = { | |
| type: 'video', | |
| sources: [{ src: videoUrl, type: 'video/mp4' }] | |
| }; | |
| // Notify when player is ready (metadata loaded, seekable) | |
| player.on('ready', () => { | |
| if (typeof onPlayerReady === 'function') onPlayerReady(); | |
| }); | |
| // 禁用双击全屏 — Plyr 3.7.8 无 doubleClick 配置项， | |
| // 其 dblclick 绑在 .plyr 容器 bubble 阶段，无法移除。 | |
| // 方案：capture 阶段设标记 → 覆写 fullscreen.toggle 拦截。 | |
| let dblClicked = false; | |
| player.elements.container.addEventListener('dblclick', () => { | |
| dblClicked = true; | |
| }, true); // capture 阶段，先于 Plyr 的 bubble handler 触发 | |
| const _origToggle = player.fullscreen.toggle.bind(player.fullscreen); | |
| player.fullscreen.toggle = function () { | |
| if (dblClicked) { | |
| dblClicked = false; | |
| return; | |
| } | |
| return _origToggle(); | |
| }; | |
| return player; | |
| } | |
| /* ── 媒体组(双目兼容)──────────────────────────── | |
| 素材清单由后端 /media-groups API 驱动:双目组(stereo_left+stereo_right)、 | |
| 辅助流、单目、骨骼视频、手套传感器。前端按 kind 渲染,新增素材类型 | |
| 无需改动前端代码。主要交互是"拖拽素材到画布"自由组合;点击素材项 | |
| 或顶部标签只是便捷方式。 */ | |
| let currentMediaGroups = null; // /media-groups 返回的 {groups, singles, sources} | |
| let currentImageTiles = []; // 画布中的图像素材 tile {img, source}(手套热力图/深度图) | |
| let workspaceGrid = null; | |
| let workspaceMainRow = null; | |
| let workspaceHandFooter = null; | |
| let workspaceHandSlots = { left: null, right: null }; | |
| let workspaceBottomExtra = null; | |
| const _gloveSyncBound = new WeakSet(); | |
| let _suppressMasterEvents = false; | |
| let hand3dData = null; // /hand-3d 接口返回 {frames:[{fi,h0,h1}], baseline_m, ...} | |
| let hand3dDataBySource = {}; // source_key -> per-device world metadata | |
| let hand3dFrameCache = { frame: -1, data: null, inflight: -1 }; // 按帧懒加载缓存 | |
| let hand3dFrameCacheBySource = {}; | |
| let _hand3dTileEntry = null; // 3D tile 引用,懒加载回调重绘用 | |
| let currentHand3dTiles = []; // 画布中的 3D 骨骼 tile {canvas, source, rotX, rotY, ...} | |
| let currentDepthPreviewTiles = []; // raw uint16 depth codes + frontend Canvas | |
| let currentTactileTiles = []; // 左右力矩阵合并预览,按统一帧号刷新 | |
| let slamData = null; // /slam-trajectory response for current episode | |
| let slamFramesByIndex = {}; | |
| let slamWindowInflight = null; | |
| let currentSlamTiles = []; // 画布中的 SLAM 轨迹 tile | |
| let tactileMatrixData = null; // /tactile-matrix-meta response | |
| const SLAM_DISPLAY_SMOOTH_ALPHA = 0.35; | |
| const SLAM_DISPLAY_JOIN_GAP_FRAMES = 3; | |
| const SLAM_DISPLAY_CAMERA_DISTANCE_SCALE = 1.55; | |
| const _depthLittleEndian = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1; | |
| // Keep one recently-used complete raw-code buffer across episode clicks. The | |
| // decoded D435 stream is large, so bound this cache to one typical episode; | |
| // the cache contains only uint16 code values, never a JET/RGB image. | |
| const _depthFullCache = new Map(); | |
| const DEPTH_FULL_CACHE_MAX_BYTES = 420 * 1024 * 1024; | |
| function _depthFullCacheKey(source) { | |
| return `${currentEpisodeId || ''}:${source?.source_key || ''}`; | |
| } | |
| function _rememberDepthFull(entry) { | |
| if (!entry?.fullCodes || !entry?.source) return; | |
| const key = _depthFullCacheKey(entry.source); | |
| const bytes = entry.fullCodes.byteLength || 0; | |
| _depthFullCache.delete(key); | |
| _depthFullCache.set(key, { | |
| codes: entry.fullCodes, | |
| width: entry.width, | |
| height: entry.height, | |
| pixelCount: entry.pixelCount, | |
| frameBytes: entry.frameBytes, | |
| frameCount: entry.frameCount, | |
| bytes, | |
| }); | |
| let total = 0; | |
| for (const [cacheKey, value] of [..._depthFullCache.entries()].reverse()) { | |
| total += value.bytes || 0; | |
| if (total > DEPTH_FULL_CACHE_MAX_BYTES) _depthFullCache.delete(cacheKey); | |
| } | |
| } | |
| function _restoreDepthFull(entry) { | |
| if (!entry?.source) return false; | |
| const value = _depthFullCache.get(_depthFullCacheKey(entry.source)); | |
| if (!value?.codes) return false; | |
| _depthFullCache.delete(_depthFullCacheKey(entry.source)); | |
| _depthFullCache.set(_depthFullCacheKey(entry.source), value); | |
| Object.assign(entry, value, { fullReady: true, initialReady: true }); | |
| entry.frames.clear(); | |
| return true; | |
| } | |
| function _decodeDepthCodeBuffer(buffer) { | |
| if (_depthLittleEndian) return new Uint16Array(buffer); | |
| const view = new DataView(buffer); | |
| const codes = new Uint16Array(buffer.byteLength / 2); | |
| for (let i = 0; i < codes.length; i++) { | |
| codes[i] = view.getUint16(i * 2, true); | |
| } | |
| return codes; | |
| } | |
| function _depthCodesUrl(source, frame) { | |
| const template = source && (source.depth_codes_url || source.depth_preview_url); | |
| return template ? _frameUrl(template, frame) : ''; | |
| } | |
| // A depth request contains raw uint16 codes, not a colorized image. Larger | |
| // sequential windows remove the per-request decoder/HTTP overhead that made | |
| // the depth canvas fall behind the RGB master. | |
| // Four seconds at the canonical 30 FPS. This is the bounded fallback for | |
| // long episodes; unlike the old 60-frame window it leaves enough time for a | |
| // remote FFmpeg read to finish before the playhead reaches the boundary. | |
| const DEPTH_WINDOW_SIZE = 120; | |
| const DEPTH_PREFETCH_MARGIN = 20; | |
| // Frames buffered before depth playback may start. With 2x-transported | |
| // codes one frame is ~200KB, so 10 frames (~2MB) paints quickly while the | |
| // full stream keeps preloading in the background. | |
| const DEPTH_INITIAL_BUFFER_FRAMES = 10; | |
| // At 848x480 uint16, 600 frames are about 465 MiB. Below this boundary it is | |
| // safer to keep the complete decoded window set resident and guarantee zero | |
| // mid-play disk/cache reads; above it use the bounded three-window runway. | |
| const DEPTH_VERY_LONG_FRAMES = 600; | |
| const DEPTH_VERY_LONG_INITIAL_WINDOWS = 3; | |
| // Depth is display-only raw code data. Normal clips wait for all windows; | |
| // very long clips wait for three windows (~12s at 30 FPS), then continue in | |
| // the background without allowing the playhead to cross an empty window. | |
| const DEPTH_FULL_PRELOAD = true; | |
| // A 848x480 stream costs about 0.78 MiB per raw frame at native scale. With | |
| // the 2x-downsampled transport one frame is ~0.19 MiB, so 2000 frames fit | |
| // ~400 MiB — the budget that previously limited full preload to short clips | |
| // now covers every episode recorded so far. Clips beyond this fall back to | |
| // the strict all-windows preload barrier instead. | |
| const DEPTH_FULL_PRELOAD_MAX_FRAMES = 2000; | |
| // Complete browser-side video blobs make arbitrary progress-bar seeks local | |
| // and keep the rendered depth preview on the same seek timeline as RGB. | |
| const _browserVideoAssetCache = new Map(); | |
| const BROWSER_VIDEO_ASSET_CACHE_MAX = 4; | |
| function _browserVideoAssetUrl(source, fallbackUrl) { | |
| const url = String(fallbackUrl || ''); | |
| if (!url) return ''; | |
| const revision = source && ( | |
| source.depth_cache_key || source.video_cache_key || source.cache_key); | |
| if (!revision || /(?:^|[?&])v=/.test(url)) return url; | |
| return `${url}${url.includes('?') ? '&' : '?'}v=${encodeURIComponent(revision)}`; | |
| } | |
| async function _preloadBrowserVideoAsset(source, fallbackUrl, sessionToken) { | |
| const episodeAtRequest = currentEpisodeId; | |
| const tokenAtRequest = sessionToken == null | |
| ? _playbackSessionToken : sessionToken; | |
| const isActive = () => isCurrentPlaybackSession( | |
| episodeAtRequest, tokenAtRequest); | |
| if (!isActive()) return null; | |
| const url = _browserVideoAssetUrl(source, fallbackUrl); | |
| if (!url) return null; | |
| const key = `${episodeAtRequest}:${source?.kind || 'video'}:` + | |
| `${source?.source_key || ''}:${url}`; | |
| const existing = _browserVideoAssetCache.get(key); | |
| if (existing?.url) return existing.url; | |
| if (existing?.promise) return existing.promise; | |
| const promise = (async () => { | |
| try { | |
| const response = await fetch(url, { | |
| cache: 'force-cache', | |
| signal: getMediaLoadSignal() || undefined, | |
| }); | |
| if (!response.ok) throw new Error(`video preload failed: ${response.status}`); | |
| const blob = await response.blob(); | |
| if (!isActive()) return null; | |
| const objectUrl = URL.createObjectURL(blob); | |
| _browserVideoAssetCache.set(key, { | |
| url: objectUrl, bytes: blob.size, ready: true, | |
| }); | |
| while (_browserVideoAssetCache.size > BROWSER_VIDEO_ASSET_CACHE_MAX) { | |
| const oldestKey = _browserVideoAssetCache.keys().next().value; | |
| const oldest = _browserVideoAssetCache.get(oldestKey); | |
| if (oldest?.url) URL.revokeObjectURL(oldest.url); | |
| _browserVideoAssetCache.delete(oldestKey); | |
| } | |
| return objectUrl; | |
| } catch (error) { | |
| if (isActive()) { | |
| console.warn('[player] complete video preload failed; fallback to range stream', error); | |
| } | |
| return null; | |
| } finally { | |
| const state = _browserVideoAssetCache.get(key); | |
| if (state?.promise) { | |
| if (state.url) _browserVideoAssetCache.set(key, { | |
| url: state.url, bytes: state.bytes, ready: true, | |
| }); | |
| else _browserVideoAssetCache.delete(key); | |
| } | |
| } | |
| })(); | |
| _browserVideoAssetCache.set(key, { promise }); | |
| return promise; | |
| } | |
| let _depthPlaybackStall = null; | |
| function _depthFrameReady(frame) { | |
| return currentDepthPreviewTiles.length > 0 && | |
| currentDepthPreviewTiles.every(entry => | |
| (entry.fullReady && frame < entry.frameCount) || | |
| (!entry.fullReady && entry.fullCodes && frame < entry.frameCount) || | |
| entry.frames.has(frame)); | |
| } | |
| function _stallPlaybackForDepth(frame) { | |
| if (_depthPlaybackStall || !masterPlaying) return; | |
| _depthPlaybackStall = { episodeId: currentEpisodeId, frame }; | |
| masterPlaying = false; | |
| _suppressMasterEvents = true; | |
| pauseAll(); | |
| _suppressMasterEvents = false; | |
| refreshPlayButton(false); | |
| } | |
| function _maybeResumeAfterDepth() { | |
| const stall = _depthPlaybackStall; | |
| if (!stall || stall.episodeId !== currentEpisodeId || | |
| !_depthFrameReady(stall.frame)) return; | |
| _depthPlaybackStall = null; | |
| currentFrameTarget = stall.frame; | |
| seekToFrame(stall.frame); | |
| masterPlaying = true; | |
| playAll(); | |
| } | |
| function _depthCodesWindowUrl(source, start, end) { | |
| let template = source && source.depth_codes_window_url; | |
| // A media-groups response can come from the browser cache and predate | |
| // the window URL field. Derive the endpoint from the canonical source | |
| // identity so an old response cannot regress playback to one request per | |
| // frame. This still transfers raw display codes only; JET stays in the | |
| // browser. | |
| if (!template && source && source.source_key && currentEpisodeId) { | |
| template = `/api/v1/video/${encodeURIComponent(currentEpisodeId)}` + | |
| `/depth-codes-window/${encodeURIComponent(source.source_key)}` + | |
| '?start_frame={start}&end_frame={end}&scale=2'; | |
| } | |
| if (template) { | |
| let url = template.replace('{start}', Math.max(0, start)) | |
| .replace('{end}', Math.max(start, end)); | |
| // Server-provided templates may be cached by media-groups without | |
| // the downsampled transport; force it so windows stay ~23MB instead | |
| // of ~95MB (a stale cache must never regress playback). | |
| if (!url.includes('scale=')) { | |
| url += `${url.includes('?') ? '&' : '?'}scale=2`; | |
| } | |
| const revision = source && source.depth_cache_key; | |
| return revision | |
| ? `${url}${url.includes('?') ? '&' : '?'}v=${encodeURIComponent(revision)}` | |
| : url; | |
| } | |
| return ''; | |
| } | |
| function _depthCodesFullUrl(source) { | |
| let template = source && source.depth_codes_full_url; | |
| if (!template && source && source.source_key && currentEpisodeId) { | |
| template = `/api/v1/video/${encodeURIComponent(currentEpisodeId)}` + | |
| `/depth-codes-full/${encodeURIComponent(source.source_key)}` + | |
| '?scale=2'; | |
| } | |
| if (!template) return ''; | |
| // Same guard as the window URL: a cached media-groups payload must not | |
| // drop the downsampled transport. | |
| if (!template.includes('scale=')) { | |
| template += `${template.includes('?') ? '&' : '?'}scale=2`; | |
| } | |
| const revision = source && source.depth_cache_key; | |
| return revision | |
| ? `${template}${template.includes('?') ? '&' : '?'}v=${encodeURIComponent(revision)}` | |
| : template; | |
| } | |
| function _fullDepthCodesForFrame(entry, frame) { | |
| if (!entry || !entry.fullCodes || | |
| frame < 0 || frame >= entry.frameCount) return null; | |
| const begin = frame * entry.pixelCount; | |
| return entry.fullCodes.subarray(begin, begin + entry.pixelCount); | |
| } | |
| async function _preloadDepthCodes(entry) { | |
| if (!entry || !entry.source || entry.fullReady) return true; | |
| if (entry.fullInflight) return entry.fullInflight; | |
| if (_restoreDepthFull(entry)) { | |
| const codes = _fullDepthCodesForFrame(entry, 0); | |
| if (codes) DepthRenderer.render(entry.canvas, codes, entry.width, entry.height); | |
| return true; | |
| } | |
| const episodeAtRequest = currentEpisodeId; | |
| entry.initialReady = false; | |
| entry.fullInflight = (async () => { | |
| try { | |
| const response = await fetch(_depthCodesFullUrl(entry.source), { | |
| // The URL includes the source file revision. force-cache | |
| // therefore gives repeat clicks a disk-cache hit, while a | |
| // reprocessed MP4 receives a new URL and cannot show stale | |
| // depth codes. | |
| cache: 'force-cache', signal: getMediaLoadSignal() || undefined, | |
| }); | |
| if (!response.ok) { | |
| throw new Error(`full depth code request failed: ${response.status}`); | |
| } | |
| const width = Number(response.headers.get('X-Depth-Width')) || 0; | |
| const height = Number(response.headers.get('X-Depth-Height')) || 0; | |
| const frameBytes = Number(response.headers.get('X-Depth-Frame-Bytes')) || | |
| width * height * 2; | |
| const declaredFrames = Number(response.headers.get('X-Depth-Frames')) || | |
| Number(entry.source.frame_count) || 0; | |
| if (!width || !height || !frameBytes) { | |
| throw new Error('invalid depth stream metadata'); | |
| } | |
| // Allocate one contiguous raw-code buffer. It is intentionally | |
| // not a canvas/image buffer: no pseudo-colour data is created or | |
| // persisted. | |
| let capacity = Math.max(1, declaredFrames) * frameBytes; | |
| let raw = new Uint8Array(capacity); | |
| let offset = 0; | |
| let publishedBuffer = null; | |
| let publishedFrames = 0; | |
| const publishBufferedFrames = () => { | |
| const availableBytes = Math.floor(offset / frameBytes) * frameBytes; | |
| const availableFrames = Math.floor(availableBytes / frameBytes); | |
| if (availableFrames <= publishedFrames) return availableFrames; | |
| if (!publishedBuffer || publishedBuffer.buffer !== raw.buffer) { | |
| publishedBuffer = _decodeDepthCodeBuffer(raw.buffer); | |
| } | |
| entry.width = width; | |
| entry.height = height; | |
| entry.pixelCount = width * height; | |
| entry.frameBytes = frameBytes; | |
| entry.frameCount = availableFrames; | |
| entry.fullCodes = publishedBuffer; | |
| publishedFrames = availableFrames; | |
| // Paint only the first available frame during the sequential | |
| // preload. Rendering once per network chunk can mean hundreds | |
| // of 848x480 texture uploads before playback even starts and | |
| // is a major source of the apparent "stuck" loading state. | |
| // The completed stream is painted at the current frame below. | |
| const active = getActivePlayer(); | |
| const displayTarget = active && Number.isFinite(active.currentTime) | |
| ? Math.floor(active.currentTime * getEpisodeFps() + 0.002) | |
| : (typeof currentFrameTarget === 'number' ? currentFrameTarget : 0); | |
| const firstAvailable = Math.min( | |
| Math.max(0, displayTarget), availableFrames - 1); | |
| const displayCodes = publishedBuffer.subarray( | |
| firstAvailable * entry.pixelCount, | |
| (firstAvailable + 1) * entry.pixelCount); | |
| if (!entry.initialPainted && displayCodes.length === entry.pixelCount) { | |
| entry.lastRenderedFrame = firstAvailable; | |
| DepthRenderer.render(entry.canvas, displayCodes, width, height); | |
| entry.initialPainted = true; | |
| } | |
| if (!entry.initialReady && availableFrames >= | |
| Math.min(DEPTH_INITIAL_BUFFER_FRAMES, declaredFrames || availableFrames)) { | |
| entry.initialReady = true; | |
| } | |
| _maybeResumeAfterDepth(); | |
| return availableFrames; | |
| }; | |
| const consume = (value) => { | |
| if (!value || !value.byteLength) return; | |
| if (offset + value.byteLength > raw.byteLength) { | |
| const next = Math.max(offset + value.byteLength, | |
| Math.ceil(raw.byteLength * 1.25)); | |
| const grown = new Uint8Array(next); | |
| grown.set(raw.subarray(0, offset)); | |
| raw = grown; | |
| publishedBuffer = null; | |
| } | |
| raw.set(value, offset); | |
| offset += value.byteLength; | |
| publishBufferedFrames(); | |
| }; | |
| const reader = response.body && response.body.getReader | |
| ? response.body.getReader() : null; | |
| if (!reader) { | |
| consume(new Uint8Array(await response.arrayBuffer())); | |
| } else { | |
| let chunks = 0; | |
| while (true) { | |
| const part = await reader.read(); | |
| if (part.done) break; | |
| consume(part.value || new Uint8Array()); | |
| // Keep RGB controls/layout responsive during a large | |
| // background fill without rAF throttling in hidden tabs. | |
| if ((++chunks % 64) === 0) { | |
| await new Promise(resolve => setTimeout(resolve, 0)); | |
| } | |
| } | |
| } | |
| if (currentEpisodeId !== episodeAtRequest) return false; | |
| const usableBytes = Math.floor(offset / frameBytes) * frameBytes; | |
| if (usableBytes < frameBytes) throw new Error('empty depth code stream'); | |
| const exact = raw.byteLength === usableBytes | |
| ? raw.buffer : raw.slice(0, usableBytes).buffer; | |
| entry.fullCodes = _decodeDepthCodeBuffer(exact); | |
| entry.frameCount = Math.floor(usableBytes / frameBytes); | |
| entry.fullReady = true; | |
| entry.initialReady = true; | |
| entry.frames.clear(); | |
| _rememberDepthFull(entry); | |
| const active = getActivePlayer(); | |
| const displayTarget = active && Number.isFinite(active.currentTime) | |
| ? Math.floor(active.currentTime * getEpisodeFps() + 0.002) | |
| : (typeof currentFrameTarget === 'number' ? currentFrameTarget : 0); | |
| const frameCodes = _fullDepthCodesForFrame(entry, displayTarget); | |
| if (frameCodes) DepthRenderer.render(entry.canvas, frameCodes, width, height); | |
| _maybeResumeAfterDepth(); | |
| return true; | |
| } catch (error) { | |
| entry.fullError = error; | |
| console.warn('[player] background depth preload failed:', error); | |
| return false; | |
| } | |
| })().finally(() => { | |
| entry.fullInflight = null; | |
| }); | |
| // Playback must not start while the full buffer is still arriving: doing | |
| // so makes the central clock repeatedly pause/resume at the buffer edge. | |
| // A subsequent click can return immediately through _restoreDepthFull(). | |
| return entry.fullInflight; | |
| } | |
| // Normal/medium clips are completely fetched before playback. They are | |
| // fetched as 120-frame responses so the browser can persist them in its HTTP | |
| // disk cache without creating one huge 900MB Uint16Array. Only a few decoded | |
| // windows remain in RAM; later playback requests are local cache hits. | |
| async function _preloadDepthWindows(entry) { | |
| if (!entry || !entry.source) return false; | |
| if (entry.allPreloaded) return true; | |
| if (entry.allPreloadInflight) return entry.allPreloadInflight; | |
| const sourceFrameCount = Number(entry.source.frame_count) || 0; | |
| if (!sourceFrameCount) return false; | |
| const episodeAtRequest = currentEpisodeId; | |
| entry.allPreloadInflight = (async () => { | |
| try { | |
| for (let start = 0; start < sourceFrameCount; start += DEPTH_WINDOW_SIZE) { | |
| if (currentEpisodeId !== episodeAtRequest) return false; | |
| entry.preloadProgress = Math.min(1, start / sourceFrameCount); | |
| await _fetchDepthCodes(entry, start); | |
| if (!entry.frames.has(start)) { | |
| throw new Error(`depth preload window missing: ${start}`); | |
| } | |
| } | |
| if (currentEpisodeId !== episodeAtRequest) return false; | |
| // The sequential pass evicts old decoded windows from RAM. Put | |
| // frame zero back from the browser cache before releasing the | |
| // playback barrier. | |
| await _fetchDepthCodes(entry, 0); | |
| entry.allPreloaded = true; | |
| entry.initialReady = true; | |
| entry.preloadProgress = 1; | |
| return true; | |
| } catch (error) { | |
| entry.fullError = error; | |
| console.warn('[player] complete depth window preload failed:', error); | |
| return false; | |
| } | |
| })().finally(() => { | |
| entry.allPreloadInflight = null; | |
| }); | |
| return entry.allPreloadInflight; | |
| } | |
| async function _preloadDepthInitialWindows(entry, windowCount = DEPTH_VERY_LONG_INITIAL_WINDOWS) { | |
| if (!entry || !entry.source) return false; | |
| if (entry.initialBufferReady) return true; | |
| if (entry.initialPreloadInflight) return entry.initialPreloadInflight; | |
| const sourceFrameCount = Number(entry.source.frame_count) || 0; | |
| if (!sourceFrameCount) return false; | |
| const episodeAtRequest = currentEpisodeId; | |
| entry.initialPreloadInflight = (async () => { | |
| try { | |
| const limit = Math.min(sourceFrameCount, | |
| Math.max(1, windowCount) * DEPTH_WINDOW_SIZE); | |
| for (let start = 0; start < limit; start += DEPTH_WINDOW_SIZE) { | |
| if (currentEpisodeId !== episodeAtRequest) return false; | |
| await _fetchDepthCodes(entry, start); | |
| } | |
| entry.initialBufferReady = entry.frames.has(0); | |
| return entry.initialBufferReady; | |
| } catch (error) { | |
| console.warn('[player] initial depth buffer failed:', error); | |
| return false; | |
| } finally { | |
| entry.initialPreloadInflight = null; | |
| } | |
| })(); | |
| return entry.initialPreloadInflight; | |
| } | |
| async function _fetchDepthCodes(entry, frame) { | |
| if (!entry || !entry.source || !entry.canvas || frame < 0) return; | |
| if (entry.fullReady) { | |
| const codes = _fullDepthCodesForFrame(entry, frame); | |
| if (codes) { | |
| if (entry.lastRenderedFrame === frame) return; | |
| entry.lastRenderedFrame = frame; | |
| DepthRenderer.render(entry.canvas, codes, entry.width, entry.height); | |
| } | |
| return; | |
| } | |
| if (entry.fullInflight) return; | |
| if (entry.frames.has(frame)) { | |
| const cached = entry.frames.get(frame); | |
| if (entry.lastRenderedFrame !== frame) { | |
| entry.lastRenderedFrame = frame; | |
| DepthRenderer.render(entry.canvas, cached.codes, cached.width, cached.height); | |
| } | |
| return; | |
| } | |
| if (entry.windowInflight && frame >= entry.windowInflight.start | |
| && frame <= entry.windowInflight.end) return; | |
| entry.pendingFrame = frame; | |
| if (entry.windowInflight != null) return; | |
| const target = entry.pendingFrame; | |
| entry.pendingFrame = null; | |
| const sourceFrameCount = Number(entry.source.frame_count) || 0; | |
| // Once the source metadata is known, never ask the backend for a window | |
| // beyond the actual clip. Besides avoiding noisy 404s, this prevents a | |
| // failed tail request from looking like a playback stall on short clips. | |
| if (sourceFrameCount && target >= sourceFrameCount) return; | |
| const start = Math.floor(target / DEPTH_WINDOW_SIZE) * DEPTH_WINDOW_SIZE; | |
| if (sourceFrameCount && start >= sourceFrameCount) return; | |
| const end = sourceFrameCount | |
| ? Math.min(start + DEPTH_WINDOW_SIZE - 1, sourceFrameCount - 1) | |
| : start + DEPTH_WINDOW_SIZE - 1; | |
| entry.windowInflight = { start, end }; | |
| const episodeAtRequest = currentEpisodeId; | |
| try { | |
| const windowUrl = _depthCodesWindowUrl(entry.source, start, end); | |
| const response = await fetch(windowUrl || _depthCodesUrl(entry.source, target), { | |
| cache: 'force-cache', signal: getMediaLoadSignal() || undefined, | |
| }); | |
| if (!response.ok) throw new Error(`depth code request failed: ${response.status}`); | |
| const width = Number(response.headers.get('X-Depth-Width')) || 0; | |
| const height = Number(response.headers.get('X-Depth-Height')) || 0; | |
| const buffer = await response.arrayBuffer(); | |
| if (currentEpisodeId !== episodeAtRequest || buffer.byteLength % 2) return; | |
| // The API declares uint16 little-endian. Common browsers can use a | |
| // zero-copy view; the fallback keeps the wire contract strict on an | |
| // unusual big-endian host. | |
| const codes = _decodeDepthCodeBuffer(buffer); | |
| const expected = width * height; | |
| const count = Number(response.headers.get('X-Depth-Frames')) || 1; | |
| const first = Number(response.headers.get('X-Depth-Start')) || start; | |
| if (!width || !height || codes.length < expected) return; | |
| for (let offset = 0; offset < count; offset++) { | |
| const begin = offset * expected; | |
| const frameCodes = codes.subarray(begin, begin + expected); | |
| if (frameCodes.length !== expected) break; | |
| entry.frames.set(first + offset, { codes: frameCodes, width, height }); | |
| } | |
| if (entry.keepAllFrames) { | |
| entry.initialReady = true; | |
| } else { | |
| // Keep the current window plus several future windows. A FIFO | |
| // eviction could remove frame 0 while the long-clip prefetcher | |
| // is still filling frame 360, causing an unnecessary pause. | |
| const activeFrame = Math.max(0, Number(currentFrameTarget) || 0); | |
| const minFrame = Math.max(0, activeFrame - DEPTH_WINDOW_SIZE); | |
| const maxFrame = activeFrame + DEPTH_WINDOW_SIZE * 5 - 1; | |
| for (const frameKey of entry.frames.keys()) { | |
| if (frameKey < minFrame || frameKey > maxFrame) { | |
| entry.frames.delete(frameKey); | |
| } | |
| } | |
| } | |
| if (!entry.keepAllFrames && expected && entry.source.frame_count) { | |
| const residentBytes = Number(entry.source.frame_count) * expected * 2; | |
| entry.keepAllFrames = residentBytes <= 420 * 1024 * 1024; | |
| } | |
| // Playback may have advanced while the request was in flight. Render | |
| // the actual master frame if it is in this window; a prefetch must | |
| // never jump the depth canvas ahead to the first prefetched frame. | |
| const active = getActivePlayer(); | |
| const displayTarget = active && !active.paused && | |
| Number.isFinite(active.currentTime) | |
| ? Math.floor(active.currentTime * getEpisodeFps() + 0.002) | |
| : (typeof currentFrameTarget === 'number' ? currentFrameTarget : target); | |
| const cached = entry.frames.get(displayTarget) || | |
| (displayTarget === target ? entry.frames.get(target) : null); | |
| if (cached) { | |
| if (entry.lastRenderedFrame === displayTarget) return; | |
| entry.lastRenderedFrame = displayTarget; | |
| DepthRenderer.render(entry.canvas, cached.codes, cached.width, cached.height); | |
| } | |
| _maybeResumeAfterDepth(); | |
| } catch (_) { | |
| // A stale request or a temporary seek race must not break playback. | |
| } finally { | |
| entry.windowInflight = null; | |
| if (entry.pendingFrame != null && currentEpisodeId === episodeAtRequest) { | |
| _fetchDepthCodes(entry, entry.pendingFrame); | |
| } | |
| } | |
| } | |
| function _queueDepthWindow(entry, frame) { | |
| // keepAllFrames only controls eviction (RAM residency), NOT fetching: | |
| // with the 2x-downsampled transport, long clips now qualify as | |
| // keepAllFrames and must still be prefetched by this queue — otherwise | |
| // every 120-frame boundary stalls. | |
| if (!entry || entry.fullReady || frame < 0) return; | |
| const start = Math.floor(frame / DEPTH_WINDOW_SIZE) * DEPTH_WINDOW_SIZE; | |
| const count = Number(entry.source?.frame_count) || 0; | |
| if (count && start >= count) return; | |
| if (entry.frames.has(start) || | |
| (entry.windowInflight && entry.windowInflight.start === start) || | |
| entry.windowQueue.includes(start)) return; | |
| entry.windowQueue.push(start); | |
| _pumpDepthWindowQueue(entry); | |
| } | |
| async function _pumpDepthWindowQueue(entry) { | |
| if (!entry || entry.windowPump) return; | |
| entry.windowPump = true; | |
| try { | |
| while (entry.windowQueue.length && currentEpisodeId) { | |
| if (entry.windowInflight) { | |
| await new Promise(resolve => setTimeout(resolve, 20)); | |
| continue; | |
| } | |
| const start = entry.windowQueue.shift(); | |
| await _fetchDepthCodes(entry, start); | |
| } | |
| } finally { | |
| entry.windowPump = false; | |
| } | |
| } | |
| function _refreshDepthTilesAt(frame) { | |
| if (!Number.isFinite(frame) || frame < 0) return; | |
| currentDepthPreviewTiles.forEach(entry => { | |
| _fetchDepthCodes(entry, frame); | |
| if (entry.fullReady) return; | |
| // Fetch the next sequential window before the current one is empty. | |
| // The server-side reader stays sequential, so this is substantially | |
| // cheaper than waiting for a random frame request at the boundary. | |
| const windowStart = Math.floor(frame / DEPTH_WINDOW_SIZE) * DEPTH_WINDOW_SIZE; | |
| const prefetchFrame = windowStart + DEPTH_WINDOW_SIZE; | |
| const sourceFrameCount = Number(entry.source.frame_count) || 0; | |
| if (sourceFrameCount > DEPTH_VERY_LONG_FRAMES) { | |
| // Maintain at least three future windows in the queue. They are | |
| // fetched one at a time so the server decoder remains sequential. | |
| for (let i = 1; i <= 3; i++) { | |
| const queuedFrame = windowStart + i * DEPTH_WINDOW_SIZE; | |
| if (!sourceFrameCount || queuedFrame < sourceFrameCount) { | |
| _queueDepthWindow(entry, queuedFrame); | |
| } | |
| } | |
| } else if (frame >= windowStart + DEPTH_PREFETCH_MARGIN * 2 && | |
| (!sourceFrameCount || prefetchFrame < sourceFrameCount) && | |
| !entry.frames.has(prefetchFrame) && !entry.windowInflight) { | |
| _fetchDepthCodes(entry, prefetchFrame); | |
| } | |
| }); | |
| } | |
| // 3D 世界坐标窗口缓存:窗口拉取(±HAND3D_WINDOW 帧),播放期零逐帧请求 | |
| let hand3dWindow = { start: -1, end: -1, frames: {}, inflight: false }; | |
| let hand3dWindowBySource = {}; | |
| const HAND3D_WINDOW = 250; | |
| const HAND3D_INITIAL_BUFFER_FRAMES = 450; | |
| const HAND3D_MAX_CACHED_FRAMES = 2500; | |
| // 3D 世界窗口在接近边界前提前请求下一段。严格同步模式下不使用 | |
| // 上一帧补位，因此必须在真正越过边界前完成预取。 | |
| const HAND3D_PREFETCH_MARGIN = 60; | |
| // Review episodes are normally short (hundreds/thousands of frames). Load | |
| // those point sequences once before playback so a window boundary can never | |
| // leave the spatial canvas one or more frames behind RGB. Very long episodes | |
| // keep the bounded-window fallback to avoid turning a browser tab into a | |
| // multi-hundred-megabyte JSON allocation. | |
| const HAND3D_FULL_PRELOAD = true; | |
| const HAND3D_FULL_PRELOAD_MAX_FRAMES = 5000; | |
| const HAND3D_FULL_CACHE_MAX_EPISODES = 2; | |
| const _hand3dFullCache = new Map(); // episode/source -> {frames, count} | |
| // A workflow rerun replaces the canonical parquet. Drop only that episode's | |
| // browser-side point/depth buffers so a later click cannot show the old | |
| // derived result; unrelated episode caches remain warm. | |
| window.invalidateEpisodePlaybackCache = function (episodeId) { | |
| const prefix = `${episodeId || ''}:`; | |
| for (const key of _depthFullCache.keys()) { | |
| if (key.startsWith(prefix)) _depthFullCache.delete(key); | |
| } | |
| for (const key of _hand3dFullCache.keys()) { | |
| if (key.startsWith(prefix)) _hand3dFullCache.delete(key); | |
| } | |
| if (currentEpisodeId === episodeId) { | |
| slamData = null; | |
| slamFramesByIndex = {}; | |
| slamWindowInflight = null; | |
| } | |
| if (window.EgoMediaCache && episodeId) { | |
| window.EgoMediaCache.removeEpisode(episodeId).catch(() => {}); | |
| } | |
| }; | |
| // 世界坐标使用稳定的空间锚点和安全视距。世界坐标中的点不能根据 | |
| // 当前帧的单手/双手数量重新居中或缩放,否则检测状态变化会被误显示 | |
| // 成相机跳动。2m 同时为固定网格保留足够的可视边界,用户仍可滚轮缩放。 | |
| const HAND3D_WORLD_DEFAULT_CENTER = [0, 0, 0.7]; | |
| const HAND3D_WORLD_DEFAULT_DISTANCE = 2.0; | |
| // World-preview sizing is screen-space only. The target is intentionally | |
| // shared by depth and RGB-estimated 3D so different coordinate sources have | |
| // the same visual scale without changing parquet/API coordinates. | |
| const HAND3D_DISPLAY_SPAN_RATIO = 0.18; | |
| const HAND3D_DISPLAY_SPAN_MIN_PX = 70; | |
| const HAND3D_DISPLAY_SPAN_MAX_PX = 130; | |
| const HAND3D_DISPLAY_SCALE_MIN = 0.70; | |
| const HAND3D_DISPLAY_SCALE_MAX = 3.0; | |
| // RGB/深度 3D 坐标统一为 X 右、Y 上、Z 向前；网格放在手的下方。 | |
| const HAND3D_WORLD_DEFAULT_GRID_Y = -0.25; | |
| // Root-anchored preview: MediaPipe landmark 0 (Wrist) is placed on the | |
| // ground grid at two stable left/right display anchors. The stored camera | |
| // coordinates remain untouched; this is only a presentation transform. | |
| // Fixed preview positions. Keep enough horizontal clearance so the two | |
| // skeletons cannot visually merge on the shared 3D canvas. | |
| const HAND3D_ROOT_ANCHOR_X = 0.22; | |
| const HAND3D_ROOT_ANCHOR_LIFT = 0.01; | |
| // 默认正面视角与左侧 RGB 画面一致；拖拽仍可改变俯仰角查看空间深度。 | |
| const HAND3D_DEFAULT_ELEVATION = 0; | |
| const HAND3D_MAX_CAMERA_DISTANCE = 6.0; | |
| // RGB-only 3D is relative/estimated rather than metric. Its current image | |
| // model is smaller than a physical hand, so normalize only the preview | |
| // geometry to a stable hand span comparable with the depth view. | |
| const RGB_ESTIMATED_DISPLAY_HAND_SPAN = 0.14; | |
| const RGB_ESTIMATED_DISPLAY_SCALE_MIN = 1.0; | |
| const RGB_ESTIMATED_DISPLAY_SCALE_MAX = 5.0; | |
| function _hand3dCacheFor(sourceKey) { | |
| const source = sourceKey || 'default'; | |
| if (!hand3dWindowBySource[source]) { | |
| hand3dWindowBySource[source] = { | |
| start: -1, end: -1, frames: {}, inflight: false, | |
| inflightPromise: null, | |
| }; | |
| } | |
| return hand3dWindowBySource[source]; | |
| } | |
| function _hand3dFrameCacheFor(sourceKey) { | |
| const source = sourceKey || 'default'; | |
| if (!hand3dFrameCacheBySource[source]) { | |
| hand3dFrameCacheBySource[source] = { frame: -1, data: null, inflight: -1 }; | |
| } | |
| return hand3dFrameCacheBySource[source]; | |
| } | |
| function _hand3dCached(frame, sourceKey) { | |
| const cache = _hand3dCacheFor(sourceKey); | |
| const fr = cache.frames[frame]; | |
| return fr ? { h0: fr.h0 || null, h1: fr.h1 || null } : null; | |
| } | |
| function _hand3dArtifactRevision(sourceKey) { | |
| const source = sourceKey || 'default'; | |
| const data = hand3dDataBySource[source] || hand3dData; | |
| return String(data && data.artifact_revision || 'unknown'); | |
| } | |
| function _hand3dFullCacheKey(sourceKey, artifactRevision = null) { | |
| const revision = artifactRevision == null | |
| ? _hand3dArtifactRevision(sourceKey) : String(artifactRevision); | |
| return `${currentEpisodeId || ''}:${sourceKey || 'default'}:${revision}`; | |
| } | |
| function _restoreHand3DFull(cache, sourceKey, artifactRevision) { | |
| const saved = _hand3dFullCache.get( | |
| _hand3dFullCacheKey(sourceKey, artifactRevision)); | |
| if (!saved || !saved.frames | |
| || String(saved.artifactRevision || 'unknown') !== String(artifactRevision)) { | |
| return false; | |
| } | |
| cache.frames = saved.frames; | |
| cache.start = 0; | |
| cache.end = Math.max(0, Number(saved.count || 0) - 1); | |
| cache.fullReady = true; | |
| cache.fullArtifactRevision = String(artifactRevision); | |
| return true; | |
| } | |
| function _rememberHand3DFull(cache, sourceKey, count, artifactRevision) { | |
| const key = _hand3dFullCacheKey(sourceKey, artifactRevision); | |
| _hand3dFullCache.delete(key); | |
| _hand3dFullCache.set(key, { | |
| frames: cache.frames, count, | |
| artifactRevision: String(artifactRevision), | |
| }); | |
| while (_hand3dFullCache.size > HAND3D_FULL_CACHE_MAX_EPISODES) { | |
| _hand3dFullCache.delete(_hand3dFullCache.keys().next().value); | |
| } | |
| } | |
| async function fetchHand3DFull(sourceKey, sessionToken = _playbackSessionToken) { | |
| const source = sourceKey || 'default'; | |
| const episodeAtRequest = currentEpisodeId; | |
| const tokenAtRequest = sessionToken; | |
| const isActive = () => isCurrentPlaybackSession( | |
| episodeAtRequest, tokenAtRequest); | |
| if (!isActive()) return false; | |
| const cache = _hand3dCacheFor(source); | |
| const data = hand3dDataBySource[source] || hand3dData; | |
| const artifactRevision = String( | |
| data && data.artifact_revision || 'unknown'); | |
| if (cache.fullReady | |
| && String(cache.fullArtifactRevision || 'unknown') | |
| === artifactRevision) { | |
| return true; | |
| } | |
| if (cache.fullReady) cache.fullReady = false; | |
| if (_restoreHand3DFull(cache, source, artifactRevision)) return true; | |
| if (cache.fullInflight) return cache.fullInflight; | |
| const count = Number(data && data.count || 0); | |
| if (!count || count > HAND3D_FULL_PRELOAD_MAX_FRAMES) return false; | |
| const signal = getMediaLoadSignal(); | |
| const persistentKey = `${episodeAtRequest}:hand3d:${source}:${count}:${artifactRevision}`; | |
| cache.fullInflight = (async () => { | |
| try { | |
| if (window.EgoMediaCache) { | |
| const saved = await window.EgoMediaCache.get(persistentKey); | |
| if (!isActive()) return false; | |
| const value = saved && saved.value; | |
| if (value && Array.isArray(value.frames) | |
| && Number(value.count || 0) === count | |
| && String(value.artifact_revision || 'unknown') | |
| === artifactRevision) { | |
| const frames = {}; | |
| value.frames.forEach(fr => { frames[fr.f] = fr; }); | |
| if (!isActive()) return false; | |
| cache.frames = frames; | |
| cache.start = 0; | |
| cache.end = Math.max(0, count - 1); | |
| cache.fullReady = true; | |
| cache.fullArtifactRevision = artifactRevision; | |
| _rememberHand3DFull( | |
| cache, source, count, artifactRevision); | |
| return true; | |
| } | |
| } | |
| const res = await fetch( | |
| `/api/v1/video/${episodeAtRequest}/hand-3d?source_key=` | |
| + `${encodeURIComponent(source)}&start_frame=0&end_frame=${count - 1}`, | |
| signal ? { signal } : {}); | |
| if (!res.ok || !isActive()) return false; | |
| const payload = await res.json(); | |
| if (!isActive()) return false; | |
| if (String(payload && payload.artifact_revision || 'unknown') | |
| !== artifactRevision) return false; | |
| const frames = {}; | |
| (payload.frames || []).forEach(fr => { frames[fr.f] = fr; }); | |
| if (!isActive()) return false; | |
| cache.frames = frames; | |
| cache.start = 0; | |
| cache.end = Math.max(0, count - 1); | |
| cache.fullReady = true; | |
| cache.fullArtifactRevision = artifactRevision; | |
| _rememberHand3DFull( | |
| cache, source, count, artifactRevision); | |
| if (window.EgoMediaCache && isActive()) { | |
| window.EgoMediaCache.put(persistentKey, { | |
| frames: payload.frames || [], | |
| count, | |
| artifact_revision: artifactRevision, | |
| }); | |
| } | |
| return true; | |
| } catch (_) { | |
| return false; | |
| } finally { | |
| cache.fullInflight = null; | |
| } | |
| })(); | |
| return cache.fullInflight; | |
| } | |
| function _hand3dPlaybackFrame(fallback = 0) { | |
| const player = getActivePlayer(); | |
| if (player && !player.paused && !player.ended | |
| && Number.isFinite(player.currentTime)) { | |
| return Math.max(0, Math.floor( | |
| player.currentTime * getEpisodeFps() + 0.002)); | |
| } | |
| return typeof currentFrameTarget === 'number' ? currentFrameTarget : fallback; | |
| } | |
| function _mergeHand3DWindow(cache, frames, start, end) { | |
| const merged = cache.frames || {}; | |
| (frames || []).forEach(fr => { merged[fr.f] = fr; }); | |
| cache.frames = merged; | |
| cache.start = cache.start < 0 ? start : Math.min(cache.start, start); | |
| cache.end = Math.max(cache.end, end); | |
| const keys = Object.keys(merged).map(Number).sort((a, b) => a - b); | |
| while (keys.length > HAND3D_MAX_CACHED_FRAMES) { | |
| delete merged[keys.shift()]; | |
| } | |
| const remaining = Object.keys(merged).map(Number); | |
| cache.start = remaining.length ? Math.min(...remaining) : -1; | |
| cache.end = remaining.length ? Math.max(...remaining) : -1; | |
| } | |
| async function fetchHand3DWindow(center, sourceKey) { | |
| const source = sourceKey || 'default'; | |
| const cache = _hand3dCacheFor(source); | |
| if (cache.fullReady) return true; | |
| if (cache.inflight) return cache.inflightPromise || false; | |
| // 该源没有任何有效手部帧(如深度抬升全空的批次)→ 不发起 | |
| // 20s+ 的空拉取;标记 fullReady 后后续窗口请求全部短路。 | |
| // 仅当后端确实统计过(counted=true)且为 0 才跳过 —— 统计失败的 | |
| // 默认 0 不能误杀有数据的批次。 | |
| const handData = hand3dDataBySource[source] || hand3dData; | |
| const handOption = handData && Array.isArray(handData.available_sources) | |
| ? handData.available_sources.find( | |
| item => (item.source_key || 'default') === source) : null; | |
| const handMeta = handOption || handData; | |
| if (handMeta && handMeta.counted === true | |
| && Number(handMeta.valid_hand_frames ?? -1) === 0) { | |
| cache.fullReady = true; | |
| return false; | |
| } | |
| const insideWindow = center >= cache.start && center <= cache.end; | |
| const insidePrefetchRange = center >= cache.start + HAND3D_PREFETCH_MARGIN | |
| && center <= cache.end - HAND3D_PREFETCH_MARGIN; | |
| if (insideWindow && insidePrefetchRange) return; | |
| const start = Math.max(0, center - HAND3D_WINDOW); | |
| const end = center + HAND3D_WINDOW; | |
| const episodeAtRequest = currentEpisodeId; | |
| const sessionToken = _playbackSessionToken; | |
| const signal = getMediaLoadSignal(); | |
| cache.inflight = true; | |
| cache.inflightPromise = (async () => { | |
| try { | |
| const persistentKey = `${episodeAtRequest}:hand3d:${source}:${start}:${end}`; | |
| if (window.EgoMediaCache) { | |
| const saved = await window.EgoMediaCache.get(persistentKey); | |
| if (!isCurrentPlaybackSession(episodeAtRequest, sessionToken)) return false; | |
| const value = saved && saved.value; | |
| if (value && Array.isArray(value.frames)) { | |
| _mergeHand3DWindow(cache, value.frames, | |
| Number(value.start ?? start), Number(value.end ?? end)); | |
| for (const entry of currentHand3dTiles) { | |
| if ((entry.source.source_key || 'default') === source) { | |
| renderHand3DTile(entry, _hand3dPlaybackFrame()); | |
| } | |
| } | |
| return true; | |
| } | |
| } | |
| const res = await fetch( | |
| `/api/v1/video/${episodeAtRequest}/hand-3d?source_key=${encodeURIComponent(source)}&start_frame=${start}&end_frame=${end}`, | |
| signal ? { signal } : {}); | |
| if (!isCurrentPlaybackSession(episodeAtRequest, sessionToken)) return false; | |
| if (!res.ok) return false; | |
| const data = await res.json(); | |
| if (!isCurrentPlaybackSession(episodeAtRequest, sessionToken)) return false; | |
| _mergeHand3DWindow(cache, data.frames, data.start, data.end); | |
| if (window.EgoMediaCache) { | |
| window.EgoMediaCache.put(persistentKey, { | |
| frames: data.frames || [], start: data.start, end: data.end, | |
| }); | |
| } | |
| // 只在完整响应成功后替换窗口，避免半更新状态被渲染。 | |
| for (const entry of currentHand3dTiles) { | |
| if ((entry.source.source_key || 'default') === source) { | |
| renderHand3DTile(entry, _hand3dPlaybackFrame()); | |
| } | |
| } | |
| return true; | |
| } catch (e) { /* 拉取失败保持旧缓存 */ | |
| return false; | |
| } | |
| })(); | |
| try { | |
| return await cache.inflightPromise; | |
| } finally { | |
| cache.inflight = false; | |
| cache.inflightPromise = null; | |
| } | |
| } | |
| async function _ensureHand3DInitialWindow(sourceKey) { | |
| const source = sourceKey || 'default'; | |
| const data = hand3dDataBySource[source] || hand3dData; | |
| if (!data || !data.hasPreview) return true; | |
| const cache = _hand3dCacheFor(source); | |
| if (HAND3D_FULL_PRELOAD) { | |
| const loaded = await fetchHand3DFull(source); | |
| if (loaded) return true; | |
| } | |
| if (cache.start <= 0 && cache.end >= 0) return true; | |
| const count = Number(data.count || 0); | |
| const initialCenter = count > HAND3D_INITIAL_BUFFER_FRAMES | |
| ? Math.min(Math.floor(HAND3D_INITIAL_BUFFER_FRAMES / 2), count - 1) : 0; | |
| const loaded = Boolean(await fetchHand3DWindow(initialCenter, source)); | |
| if (loaded && count > HAND3D_INITIAL_BUFFER_FRAMES) { | |
| // Keep three future windows queued without blocking the first frame. | |
| // They merge into the bounded 3D cache and are usually served from | |
| // IndexedDB after the first visit. | |
| (async () => { | |
| let center = initialCenter + HAND3D_WINDOW * 2; | |
| for (let i = 0; i < 3 && center < count; i++, center += HAND3D_WINDOW * 2) { | |
| await fetchHand3DWindow(center, source); | |
| } | |
| })(); | |
| } | |
| return loaded; | |
| } | |
| // The hand endpoint returns a display-only compact crop of its 800x220 | |
| // canvas. The raw sensor values are not changed by this presentation crop. | |
| const HAND_PREVIEW_ASPECT = 800 / 200; | |
| const HAND_FOOTER_HEIGHT = 144; | |
| const HAND_FOOTER_WITH_EXTRA_HEIGHT = 300; | |
| function groupedSourceKey(source) { | |
| if (source.kind === 'stereo_group') return `stereo:${source.id}`; | |
| return `${source.kind}:${source.source_key || source.camera || source.id}`; | |
| } | |
| function groupedSourceDomId(source) { | |
| return groupedSourceKey(source).replace(/[^a-zA-Z0-9_-]/g, '_'); | |
| } | |
| function groupedSourceLabel(source) { | |
| if (source.kind === 'skeleton') return `${source.label || source.source_key} · ${t('skeleton_suffix')}`; | |
| if (source.kind === 'glove') return source.label || `${source.source_key} · ${t('glove_suffix')}`; | |
| if (source.kind === 'depth') return source.label || `${source.source_key} · ${t('depth_suffix')}`; | |
| if (source.kind === 'slam_trajectory') return source.label || 'SLAM'; | |
| if (source.kind === 'tactile_force_left') return 'Left'; | |
| if (source.kind === 'tactile_force_right') return 'Right'; | |
| if (source.kind === 'stereo_group') return `${source.label} · ${t('stereo_pair')}`; | |
| return source.label || source.source_key || t('video_suffix'); | |
| } | |
| function _isBottomSource(source) { | |
| return ['hand', 'depth', 'glove', 'tactile_force_left', | |
| 'tactile_force_right'].includes(source.kind); | |
| } | |
| function groupedSource(kind, source_key, meta) { | |
| return { kind, source_key, ...(meta || {}) }; | |
| } | |
| /* 素材 → 画布条目:双目组展开为左右目并排,其余一对一 */ | |
| function expandWorkspaceItems(source) { | |
| if (source.kind === 'stereo_group') { | |
| return (source.members || []).map(m => ({ | |
| kind: 'video', source_key: m.source_key, label: m.label, | |
| stream_url: m.stream_url, video_cache_key: m.video_cache_key, | |
| frame_count: m.frame_count, fps: m.fps, | |
| })); | |
| } | |
| return [{ ...source }]; | |
| } | |
| function _sourceIsPlaced(source) { | |
| const placed = new Set(workspaceSources.map(item => groupedSourceKey(item))); | |
| if (source.kind === 'stereo_group') { | |
| const members = source.members || []; | |
| return members.length > 0 && members.every(member => | |
| placed.has(`video:${member.source_key}`)); | |
| } | |
| return placed.has(groupedSourceKey(source)); | |
| } | |
| function _syncWorkspacePlayers() { | |
| const active = getActivePlayer(); | |
| if (!active) return; | |
| Object.values(players).forEach(player => bindMasterSync(player)); | |
| bindGloveSync(active); | |
| bindFrameDrift(active); | |
| if (typeof startHeatmapSync === 'function') startHeatmapSync(active); | |
| if (masterPlaying) { | |
| Object.values(players).forEach(player => { | |
| // play() returns a promise: a try/catch cannot see its rejection, | |
| // and the load chain right after this call runs pauseAll(), which | |
| // aborts the pending play → uncaught AbortError in the console. | |
| try { | |
| const result = player.play(); | |
| if (result && typeof result.catch === 'function') { | |
| result.catch(() => {}); | |
| } | |
| } catch (e) {} | |
| }); | |
| } | |
| } | |
| function addGroupedSource(source) { | |
| const added = []; | |
| expandWorkspaceItems(source).forEach(item => { | |
| const key = groupedSourceKey(item); | |
| if (!workspaceSources.some(existing => groupedSourceKey(existing) === key)) { | |
| workspaceSources.push(item); | |
| added.push(item); | |
| } | |
| }); | |
| renderSourceBar(); | |
| renderPreviewVideoSources(); | |
| if (!added.length) return; | |
| // When an episode is already mounted, append only the new source. This | |
| // keeps existing Plyr instances and playback position untouched. | |
| if (workspaceMainRow && workspaceGrid) { | |
| const addedDepth = added.some(item => item.kind === 'depth'); | |
| const resumeAfterDepth = addedDepth && masterPlaying; | |
| if (addedDepth) { | |
| // Depth is a display-only stream and must be buffered before it | |
| // joins an already playing workspace; otherwise RGB advances | |
| // while the first depth window is still decoding. | |
| masterPlaying = false; | |
| _suppressMasterEvents = true; | |
| pauseAll(); | |
| _suppressMasterEvents = false; | |
| disableFrameControls(); | |
| } | |
| const pending = added.map(item => _appendWorkspaceSource(item)); | |
| _refreshWorkspaceLayout(); | |
| Promise.all(pending).then(() => { | |
| if (addedDepth && currentEpisodeId) { | |
| enableFrameControls(); | |
| if (resumeAfterDepth) { | |
| masterPlaying = true; | |
| playAll(); | |
| } | |
| } else { | |
| _syncWorkspacePlayers(); | |
| } | |
| }); | |
| } else { | |
| renderGroupedWorkspace(); | |
| } | |
| } | |
| function removeGroupedSource(source) { | |
| const key = groupedSourceKey(source); | |
| if (typeof destroyHandOverlay === 'function') destroyHandOverlay(key); | |
| const tile = workspaceGrid | |
| ? workspaceGrid.querySelector(`[data-source-key="${CSS.escape(key)}"]`) | |
| : null; | |
| const image = tile ? tile.querySelector('img,canvas') : null; | |
| const player = players[key]; | |
| const wasPlaying = masterPlaying; | |
| if (player) { | |
| // Plyr can emit pause while destroy() is tearing down the removed | |
| // player. Suppress that transient event so remaining players do not | |
| // get paused as a side effect of removing one tile. | |
| _suppressMasterEvents = true; | |
| try { player.destroy(); } catch (e) {} finally { | |
| _suppressMasterEvents = false; | |
| } | |
| delete players[key]; | |
| } | |
| if (image && source.kind === 'hand' && typeof unregisterHandTile === 'function') { | |
| unregisterHandTile(image); | |
| } | |
| currentImageTiles = currentImageTiles.filter(entry => | |
| groupedSourceKey(entry.source) !== key); | |
| currentDepthPreviewTiles = currentDepthPreviewTiles.filter(entry => | |
| groupedSourceKey(entry.source) !== key); | |
| currentSlamTiles = currentSlamTiles.filter(entry => | |
| groupedSourceKey(entry.source) !== key); | |
| currentTactileTiles = currentTactileTiles.filter(entry => | |
| groupedSourceKey(entry.source) !== key); | |
| workspaceSources = workspaceSources.filter(item => groupedSourceKey(item) !== key); | |
| renderSourceBar(); | |
| renderPreviewVideoSources(); | |
| if (!workspaceGrid || !workspaceMainRow) { | |
| renderGroupedWorkspace(); | |
| if (window.__episodeOpen === true) { | |
| const previewOptions = document.getElementById('preview-options'); | |
| if (previewOptions) previewOptions.classList.remove('hidden'); | |
| } | |
| return; | |
| } | |
| if (tile) tile.remove(); | |
| _refreshWorkspaceLayout(); | |
| const activeAfterRemove = getActivePlayer(); | |
| if (!activeAfterRemove) { | |
| if (typeof stopHeatmapSync === 'function') stopHeatmapSync(); | |
| masterPlaying = false; | |
| refreshPlayButton(false); | |
| } else { | |
| masterPlaying = wasPlaying; | |
| _syncWorkspacePlayers(); | |
| } | |
| // 删除深度/3D/手套 tile 只影响工作区素材,不应把批次详情的 | |
| // Preview Options 一并隐藏。 | |
| if (window.__episodeOpen === true) { | |
| const previewOptions = document.getElementById('preview-options'); | |
| if (previewOptions) previewOptions.classList.remove('hidden'); | |
| } | |
| } | |
| /* ── Source button bar ─────────────────────────────────────── | |
| Every asset (stereo group / single cameras / aux streams / | |
| glove heatmaps / depth / hand-pressure tiles) appears as a | |
| button above the video canvas. Click or drag to add it to the | |
| canvas; the button disappears once placed and returns when the | |
| tile is removed. No palette drawer, no tabs. */ | |
| function _sourceBarButton(label, icon, key, source, accent, parent) { | |
| const bar = parent || document.getElementById('source-bar'); | |
| if (!bar || _sourceIsPlaced(source)) return; | |
| const btn = document.createElement('button'); | |
| btn.type = 'button'; | |
| btn.draggable = true; | |
| btn.dataset.sourceKey = key; | |
| btn.title = label; | |
| btn.className = 'text-xs px-2 py-1 rounded border cursor-grab flex-shrink-0 flex items-center gap-1 ' + | |
| (accent | |
| ? 'border-cyan-700/60 bg-gray-800 text-gray-300 hover:border-cyan-500 hover:text-white' | |
| : 'border-gray-700 bg-gray-800 text-gray-300 hover:border-blue-500 hover:text-white'); | |
| btn.innerHTML = `<iconify-icon icon="${icon}" class="icon-sm"></iconify-icon> ${label}`; | |
| let dragging = false; | |
| btn.addEventListener('click', () => { | |
| if (!dragging) addGroupedSource(source); | |
| }); | |
| btn.addEventListener('dragstart', event => { | |
| dragging = true; | |
| event.dataTransfer.setData('application/x-egodata-source', JSON.stringify(_serializeSource(source))); | |
| event.dataTransfer.effectAllowed = 'copy'; | |
| btn.classList.add('opacity-50'); | |
| }); | |
| btn.addEventListener('dragend', () => { | |
| btn.classList.remove('opacity-50'); | |
| setTimeout(() => { dragging = false; }, 0); | |
| }); | |
| bar.appendChild(btn); | |
| } | |
| function _videoSource(item) { | |
| return { | |
| kind: 'video', source_key: item.source_key, label: item.label, | |
| stream_url: item.stream_url, frame_count: item.frame_count, fps: item.fps, | |
| }; | |
| } | |
| function addSource(source, key) { | |
| addGroupedSource(source); | |
| } | |
| function addHandTile(hand) { | |
| addSource({ | |
| kind: 'hand', hand, source_key: `hand_${hand}`, | |
| label: t(hand === 'left' ? 'left_hand' : 'right_hand'), | |
| }, `hand:hand_${hand}`); | |
| } | |
| /* Swap two video tiles in place — DOM-level reorder only, so the Plyr | |
| players are NOT rebuilt and playback continues uninterrupted. */ | |
| function reorderTiles(fromKey, toKey, container) { | |
| const fromIdx = workspaceSources.findIndex(s => groupedSourceKey(s) === fromKey); | |
| const toIdx = workspaceSources.findIndex(s => groupedSourceKey(s) === toKey); | |
| if (fromIdx < 0 || toIdx < 0 || fromIdx === toIdx) return; | |
| const [item] = workspaceSources.splice(fromIdx, 1); | |
| workspaceSources.splice(toIdx, 0, item); | |
| const fromTile = container.querySelector(`[data-source-key="${CSS.escape(fromKey)}"]`); | |
| const toTile = container.querySelector(`[data-source-key="${CSS.escape(toKey)}"]`); | |
| if (fromTile && toTile) { | |
| container.insertBefore(fromTile, fromIdx < toIdx ? toTile.nextSibling : toTile); | |
| } | |
| } | |
| function renderSourceBar() { | |
| const bar = document.getElementById('source-bar'); | |
| if (!bar) return; | |
| bar.innerHTML = ''; | |
| // All preview sources are now controlled by Preview Options. Keep this | |
| // function as a compatibility no-op because older code still calls it. | |
| bar.classList.add('hidden'); | |
| } | |
| function _previewVideoEntries() { | |
| const entries = []; | |
| const add = item => { | |
| if (!item || !item.source_key || entries.some(e => e.source_key === item.source_key)) return; | |
| entries.push(_videoSource(item)); | |
| }; | |
| if (currentMediaGroups) { | |
| (currentMediaGroups.groups || []).forEach(group => { | |
| (group.members || []).forEach(add); | |
| (group.aux || []).forEach(add); | |
| }); | |
| (currentMediaGroups.singles || []).forEach(add); | |
| } | |
| if (!entries.length) { | |
| currentCameras.forEach(camera => add({ | |
| source_key: camera, label: camera, | |
| stream_url: `/api/v1/video/${currentEpisodeId}/${camera}/preview-stream`, | |
| })); | |
| } | |
| return entries; | |
| } | |
| function _previewDeviceLabel(source) { | |
| const raw = String(source.device_name || source.source_key || source.label || '').trim(); | |
| // D405_depth_rgb / D435_depth_rgb are RGB stream slots; retain the | |
| // physical device token (D405_depth / D435_depth) shown to the reviewer. | |
| const device = raw.replace(/_rgb$/i, '').replace(/_color$/i, ''); | |
| const role = source.label && source.label !== 'Primary' && source.label !== device | |
| ? ` · ${source.label}` : ''; | |
| return `${device || raw}${role}`; | |
| } | |
| function renderPreviewVideoSources() { | |
| const row = document.getElementById('po-video-row'); | |
| const list = document.getElementById('po-video-list'); | |
| const all = document.getElementById('po-video-all'); | |
| if (!row || !list) return; | |
| const entries = _previewVideoEntries(); | |
| list.innerHTML = ''; | |
| row.classList.toggle('hidden', entries.length === 0); | |
| row.classList.toggle('block', entries.length > 0); | |
| if (!entries.length) return; | |
| const isPlaced = source => workspaceSources.some(item => | |
| groupedSourceKey(item) === groupedSourceKey(source)); | |
| entries.forEach(source => { | |
| const label = document.createElement('label'); | |
| label.className = 'flex items-center gap-2 px-1 py-1 rounded hover:bg-gray-700/70 cursor-pointer'; | |
| const input = document.createElement('input'); | |
| input.type = 'checkbox'; | |
| input.className = 'w-3 h-3 accent-cyan-500'; | |
| input.checked = isPlaced(source); | |
| input.dataset.sourceKey = source.source_key; | |
| const text = document.createElement('span'); | |
| text.className = 'truncate text-[11px] text-gray-300'; | |
| text.title = source.source_key; | |
| text.textContent = _previewDeviceLabel(source); | |
| input.addEventListener('change', () => { | |
| if (input.checked) addGroupedSource(source); | |
| else removeGroupedSource(source); | |
| }); | |
| label.append(input, text); | |
| list.appendChild(label); | |
| }); | |
| if (all) { | |
| const selected = entries.filter(isPlaced).length; | |
| all.textContent = selected === entries.length ? 'None' : 'All'; | |
| all.onclick = () => { | |
| const shouldAdd = selected !== entries.length; | |
| entries.forEach(source => { | |
| if (shouldAdd && !isPlaced(source)) addGroupedSource(source); | |
| if (!shouldAdd && isPlaced(source)) removeGroupedSource(source); | |
| }); | |
| renderPreviewVideoSources(); | |
| }; | |
| } | |
| } | |
| /* ── Source bar refresh ── */ | |
| function resetSourceBar() { | |
| renderSourceBar(); | |
| } | |
| function _serializeSource(source) { | |
| if (source.kind === 'stereo_group') { | |
| return { kind: 'stereo_group', id: source.id, label: source.label, members: source.members }; | |
| } | |
| return { | |
| kind: source.kind, source_key: source.source_key, label: source.label, | |
| hand: source.hand, | |
| stream_url: source.stream_url, heatmap_url: source.heatmap_url, | |
| depth_url: source.depth_url, | |
| depth_preview_url: source.depth_preview_url, | |
| depth_codes_url: source.depth_codes_url, | |
| raw_depth_url: source.raw_depth_url, | |
| depth_cache_key: source.depth_cache_key, | |
| video_cache_key: source.video_cache_key, | |
| // Optional grayscale clock stream; never used as depth pixels. | |
| depth_video_url: source.depth_video_url, | |
| missing_frames: source.missing_frames, | |
| depth_scale: source.depth_scale, | |
| frame_count: source.frame_count, fps: source.fps, | |
| // 并排拼接标记(3D 拼接等宽画面)→ 画布 tile 跨两列 | |
| stereo_side_by_side: source.stereo_side_by_side, | |
| }; | |
| } | |
| /* 画布渲染:video/skeleton → Plyr 播放器,glove/depth → 图像 tile */ | |
| function _frameUrl(urlTemplate, frameIndex) { | |
| return urlTemplate.replace('{frame}', Math.max(0, frameIndex)); | |
| } | |
| function _tactileForceAt(frame, side) { | |
| const map = tactileMatrixData?._forceByFrame; | |
| const value = map && map[String(Math.max(0, Number(frame) || 0))]?.[side]; | |
| if (!Array.isArray(value) || value.length < 3 | |
| || !value.slice(0, 3).every(item => Number.isFinite(Number(item)))) { | |
| return null; | |
| } | |
| const fx = Number(value[0]); | |
| const fy = Number(value[1]); | |
| const fz = Number(value[2]); | |
| return { fx, fy, fz, magnitude: Math.hypot(fx, fy, fz) }; | |
| } | |
| function _updateTactileForceTile(entry, frame) { | |
| if (!entry) return; | |
| const force = _tactileForceAt(frame, entry.source.side); | |
| if (entry.forceValue) { | |
| if (!force) { | |
| entry.forceValue.textContent = 'Fx — · Fy — · Fz —'; | |
| entry.forceValue.title = 'Force data unavailable'; | |
| } else { | |
| entry.forceValue.textContent = | |
| `Fx ${Math.round(force.fx).toLocaleString()} · ` | |
| + `Fy ${Math.round(force.fy).toLocaleString()} · ` | |
| + `Fz ${Math.round(force.fz).toLocaleString()}`; | |
| entry.forceValue.title = | |
| `fx ${force.fx.toFixed(1)} · fy ${force.fy.toFixed(1)} · ` | |
| + `fz ${force.fz.toFixed(1)} mN`; | |
| } | |
| } | |
| _renderTactileForceCurve(entry, frame); | |
| } | |
| // Fx/Fy/Fz history for one side across the whole episode, with a playhead at | |
| // the current frame. Colours follow the acquisition-side ForceCurveWidget. | |
| const TACTILE_FORCE_COLORS = ['#4fc3f7', '#aed581', '#ff8a65']; | |
| function _forceSeriesIndex(frame) { | |
| const series = tactileMatrixData?.force_series; | |
| if (!series || !Array.isArray(series.frames)) return null; | |
| if (!tactileMatrixData._frameIndex) { | |
| const map = new Map(); | |
| series.frames.forEach((value, index) => map.set(Number(value), index)); | |
| tactileMatrixData._frameIndex = map; | |
| } | |
| const index = tactileMatrixData._frameIndex.get(Math.round(Number(frame))); | |
| return index === undefined ? null : index; | |
| } | |
| function _renderTactileForceCurve(entry, frame) { | |
| const canvas = entry?.curveCanvas; | |
| if (!canvas) return; | |
| const rect = canvas.getBoundingClientRect(); | |
| const width = Math.max(1, rect.width), height = Math.max(1, rect.height); | |
| if (rect.width < 2 || rect.height < 2) { | |
| // The tile is built before it is attached to the workspace, so the | |
| // first call can run before layout. Retry once the layout exists. | |
| if (!entry._curveRetry) { | |
| entry._curveRetry = true; | |
| requestAnimationFrame(() => { | |
| entry._curveRetry = false; | |
| _renderTactileForceCurve(entry, frame); | |
| }); | |
| } | |
| return; | |
| } | |
| const dpr = window.devicePixelRatio || 1; | |
| if (canvas.width !== Math.round(width * dpr) | |
| || canvas.height !== Math.round(height * dpr)) { | |
| canvas.width = Math.round(width * dpr); | |
| canvas.height = Math.round(height * dpr); | |
| } | |
| const ctx = canvas.getContext('2d'); | |
| ctx.setTransform(dpr, 0, 0, dpr, 0, 0); | |
| ctx.clearRect(0, 0, width, height); | |
| const series = tactileMatrixData?.force_series?.[entry.source.side]; | |
| if (!Array.isArray(series) || !series.length) return; | |
| const pad = { l: 4, r: 4, t: 8, b: 4 }; | |
| const innerW = width - pad.l - pad.r; | |
| const innerH = height - pad.t - pad.b; | |
| // Symmetric auto range: p99.5 of |value| rounded up to 500 mN, so the | |
| // fixed +/-3000 mN range of the acquisition widget does not clip our data. | |
| if (!entry._curveRange) { | |
| const samples = []; | |
| series.forEach(value => { | |
| if (!value) return; | |
| for (let axis = 0; axis < 3; axis++) { | |
| const item = Math.abs(Number(value[axis])); | |
| if (Number.isFinite(item)) samples.push(item); | |
| } | |
| }); | |
| samples.sort((a, b) => a - b); | |
| const p995 = samples.length | |
| ? samples[Math.min(samples.length - 1, Math.floor(samples.length * 0.995))] | |
| : 0; | |
| entry._curveRange = Math.max(500, Math.ceil(p995 / 500) * 500); | |
| } | |
| const range = entry._curveRange; | |
| const mid = pad.t + innerH / 2; | |
| const Y = value => mid - (value / range) * (innerH / 2) * 0.92; | |
| const X = index => pad.l + (series.length <= 1 | |
| ? 0 : (index / (series.length - 1)) * innerW); | |
| ctx.lineWidth = 1; | |
| ctx.strokeStyle = 'rgba(148,163,184,0.16)'; | |
| [0.5, -0.5].forEach(frac => { | |
| const y = Y(frac * range); | |
| ctx.beginPath(); ctx.moveTo(pad.l, y); ctx.lineTo(width - pad.r, y); ctx.stroke(); | |
| }); | |
| ctx.strokeStyle = 'rgba(148,163,184,0.38)'; | |
| ctx.beginPath(); | |
| ctx.moveTo(pad.l, mid); ctx.lineTo(width - pad.r, mid); | |
| ctx.stroke(); | |
| for (let axis = 0; axis < 3; axis++) { | |
| ctx.strokeStyle = TACTILE_FORCE_COLORS[axis]; | |
| ctx.lineWidth = 1.2; | |
| ctx.beginPath(); | |
| let started = false; | |
| series.forEach((value, index) => { | |
| const item = value ? Number(value[axis]) : NaN; | |
| if (!Number.isFinite(item)) { started = false; return; } | |
| const x = X(index), y = Y(item); | |
| if (!started) { ctx.moveTo(x, y); started = true; } | |
| else { ctx.lineTo(x, y); } | |
| }); | |
| ctx.stroke(); | |
| } | |
| const headIndex = _forceSeriesIndex(frame); | |
| if (headIndex !== null) { | |
| const head = X(headIndex); | |
| ctx.strokeStyle = 'rgba(248,250,252,0.85)'; | |
| ctx.lineWidth = 1; | |
| ctx.beginPath(); | |
| ctx.moveTo(head, pad.t - 3); ctx.lineTo(head, height - pad.b); | |
| ctx.stroke(); | |
| } | |
| } | |
| function _createTactileWebGLRenderer(canvas) { | |
| const gl = canvas.getContext('webgl2', { | |
| alpha: false, antialias: false, premultipliedAlpha: false, | |
| }); | |
| if (!gl) return null; | |
| const vertexSource = `#version 300 es | |
| in vec2 a_position; | |
| out vec2 v_uv; | |
| void main() { | |
| v_uv = a_position * 0.5 + 0.5; | |
| gl_Position = vec4(a_position, 0.0, 1.0); | |
| }`; | |
| const fragmentSource = `#version 300 es | |
| precision highp float; | |
| uniform sampler2D u_force_field; | |
| uniform float u_zoom; | |
| uniform vec2 u_pan; | |
| in vec2 v_uv; | |
| out vec4 out_color; | |
| vec3 jet(float x) { | |
| return clamp(vec3( | |
| 1.5 - abs(4.0 * x - 3.0), | |
| 1.5 - abs(4.0 * x - 2.0), | |
| 1.5 - abs(4.0 * x - 1.0) | |
| ), 0.0, 1.0); | |
| } | |
| void main() { | |
| // Zoom/pan happen in screen-uv space; the field itself is 250x250. | |
| vec2 uv = (v_uv - 0.5) / u_zoom + 0.5 + u_pan; | |
| if (uv.x < 0.0 || uv.x > 1.0 || uv.y < 0.0 || uv.y > 1.0) { | |
| out_color = vec4(0.0039, 0.0039, 0.502, 1.0); | |
| return; | |
| } | |
| // Matrix row 0 is the top of the sensor; WebGL texture v=0 is | |
| // the bottom, so flip only the sampling coordinate. The field is | |
| // uploaded already normalised to 0..1 (R8), so no rescaling here. | |
| float value = texture(u_force_field, vec2(uv.x, 1.0 - uv.y)).r; | |
| out_color = vec4(jet(clamp(value, 0.0, 1.0)), 1.0); | |
| }`; | |
| const compile = (type, source) => { | |
| const shader = gl.createShader(type); | |
| gl.shaderSource(shader, source); | |
| gl.compileShader(shader); | |
| if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { | |
| console.warn('[player] tactile WebGL shader failed', | |
| gl.getShaderInfoLog(shader)); | |
| gl.deleteShader(shader); | |
| return null; | |
| } | |
| return shader; | |
| }; | |
| const vertex = compile(gl.VERTEX_SHADER, vertexSource); | |
| const fragment = compile(gl.FRAGMENT_SHADER, fragmentSource); | |
| if (!vertex || !fragment) return null; | |
| const program = gl.createProgram(); | |
| gl.attachShader(program, vertex); | |
| gl.attachShader(program, fragment); | |
| gl.linkProgram(program); | |
| gl.deleteShader(vertex); | |
| gl.deleteShader(fragment); | |
| if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { | |
| console.warn('[player] tactile WebGL program failed', | |
| gl.getProgramInfoLog(program)); | |
| gl.deleteProgram(program); | |
| return null; | |
| } | |
| const position = gl.createBuffer(); | |
| gl.bindBuffer(gl.ARRAY_BUFFER, position); | |
| gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([ | |
| -1, -1, 1, -1, -1, 1, 1, 1, | |
| ]), gl.STATIC_DRAW); | |
| const texture = gl.createTexture(); | |
| gl.bindTexture(gl.TEXTURE_2D, texture); | |
| gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); | |
| gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); | |
| gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); | |
| gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); | |
| return { | |
| gl, program, position, texture, | |
| positionLocation: gl.getAttribLocation(program, 'a_position'), | |
| fieldLocation: gl.getUniformLocation(program, 'u_force_field'), | |
| zoomLocation: gl.getUniformLocation(program, 'u_zoom'), | |
| panLocation: gl.getUniformLocation(program, 'u_pan'), | |
| field: null, | |
| zoom: 1, pan: [0, 0], filter: null, | |
| render(field = null, min = 0, max = 1) { | |
| if (field) { | |
| // R32F float textures are not filterable without | |
| // OES_texture_float_linear; without it every sample returns 0 | |
| // and the whole tile renders as JET(0) - a flat dark blue that | |
| // is indistinguishable from the background. Normalising to | |
| // 8-bit here keeps the texture in the always-filterable R8 | |
| // format, so LINEAR/NEAREST both work on every driver. | |
| if (field instanceof Float32Array) { | |
| const lo = Number.isFinite(Number(min)) ? Number(min) : 0; | |
| const hi = Number.isFinite(Number(max)) ? Number(max) : 1; | |
| const span = Math.max(hi - lo, 1e-6); | |
| const bytes = new Uint8Array(field.length); | |
| for (let i = 0; i < field.length; i++) { | |
| const value = (field[i] - lo) / span; | |
| bytes[i] = value <= 0 ? 0 | |
| : value >= 1 ? 255 : Math.round(value * 255); | |
| } | |
| this.field = bytes; | |
| } else { | |
| this.field = field; | |
| } | |
| } | |
| const rect = canvas.getBoundingClientRect(); | |
| const dpr = window.devicePixelRatio || 1; | |
| const width = Math.max(1, Math.round(rect.width * dpr)); | |
| const height = Math.max(1, Math.round(rect.height * dpr)); | |
| if (canvas.width !== width || canvas.height !== height) { | |
| canvas.width = width; | |
| canvas.height = height; | |
| } | |
| // 传感器是 250x250 的方格阵列,必须等比显示 —— 铺满 canvas 会按 | |
| // tile 的宽高比横向拉伸(实测宽窗约 2.9:1),格子就不再是正方形, | |
| // 接触位置的形状判断会失真。取短边做居中正方形 viewport,多出来的 | |
| // 两侧留背景色(gl.clear 不受 viewport 限制,会整块清成底色)。 | |
| const side = Math.min(width, height); | |
| gl.viewport(Math.round((width - side) / 2), | |
| Math.round((height - side) / 2), side, side); | |
| gl.clearColor(0.0039, 0.0039, 0.502, 1.0); | |
| gl.clear(gl.COLOR_BUFFER_BIT); | |
| if (!this.field) return; | |
| gl.useProgram(program); | |
| gl.bindBuffer(gl.ARRAY_BUFFER, position); | |
| gl.enableVertexAttribArray(this.positionLocation); | |
| gl.vertexAttribPointer(this.positionLocation, 2, gl.FLOAT, false, 0, 0); | |
| gl.activeTexture(gl.TEXTURE0); | |
| gl.bindTexture(gl.TEXTURE_2D, texture); | |
| // One texel is one sensor cell. Sample NEAREST as soon as a cell | |
| // covers at least one screen pixel (crisp cells, like the | |
| // acquisition demo), and LINEAR only when shrinking - a NEAREST | |
| // minification would drop isolated contact cells entirely. | |
| const texelsPerPixel = (width / 250) * (this.zoom || 1); | |
| const filter = texelsPerPixel >= 1 ? gl.NEAREST : gl.LINEAR; | |
| if (this.filter !== filter) { | |
| gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, filter); | |
| gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, filter); | |
| this.filter = filter; | |
| } | |
| gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false); | |
| gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1); // 250-wide R8 rows | |
| gl.texImage2D(gl.TEXTURE_2D, 0, gl.R8, 250, 250, 0, | |
| gl.RED, gl.UNSIGNED_BYTE, this.field); | |
| gl.uniform1i(this.fieldLocation, 0); | |
| gl.uniform1f(this.zoomLocation, this.zoom || 1); | |
| gl.uniform2f(this.panLocation, | |
| this.pan ? this.pan[0] : 0, this.pan ? this.pan[1] : 0); | |
| gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4); | |
| }, | |
| }; | |
| } | |
| // The tactile force matrix tile is display-only. Zoom/pan were removed on | |
| // purpose: the review page is read-only evidence, and a stray wheel or drag | |
| // could leave the matrix zoomed into a few cells and look like a sensor | |
| // defect. The whole 250x250 field stays fixed, one texture texel per cell. | |
| // The renderer keeps supporting zoom/pan (uniforms u_zoom/u_pan) so the | |
| // acquisition-side demo viewer and any future opt-in can reuse it; here the | |
| // view is pinned to identity. | |
| function _resetTactileView(renderer) { | |
| if (!renderer) return; | |
| renderer.zoom = 1; | |
| renderer.pan = [0, 0]; | |
| if (renderer.canvas) renderer.canvas.style.cursor = 'default'; | |
| renderer.render(); | |
| } | |
| // One chunk = 96 frames of 250x250 R8 = 6 MB per request. The old code issued | |
| // one 250 KB request per frame per side (15 MB/s during playback), which | |
| // saturated the browser's socket buffers (net::ERR_NO_BUFFER_SPACE). Chunks | |
| // are cached in memory, so playback and scrubbing need no network at all. | |
| const TACTILE_CHUNK_FRAMES = 96; | |
| const TACTILE_FIELD_STRIDE = 250 * 250; | |
| function _tactileChunkKey(frame) { | |
| return Math.floor(Math.max(0, frame) / TACTILE_CHUNK_FRAMES) | |
| * TACTILE_CHUNK_FRAMES; | |
| } | |
| function _tactileFrameField(entry, frame) { | |
| const chunk = entry.chunks.get(_tactileChunkKey(frame)); | |
| if (!chunk) return null; | |
| const index = Math.round(Number(frame)) - chunk.first; | |
| if (index < 0 || index >= chunk.count) return null; | |
| return chunk.bytes.subarray( | |
| index * TACTILE_FIELD_STRIDE, (index + 1) * TACTILE_FIELD_STRIDE); | |
| } | |
| function _fetchTactileChunk(entry, chunkStart) { | |
| if (!entry?.renderer || entry.chunks.has(chunkStart) | |
| || entry.chunkInflight.has(chunkStart)) return; | |
| entry.chunkInflight.add(chunkStart); | |
| const episodeAtRequest = currentEpisodeId; | |
| const tokenAtRequest = _playbackSessionToken; | |
| const signal = getMediaLoadSignal(); | |
| const url = `/api/v1/video/${currentEpisodeId}/tactile-matrix-range` | |
| + `?side=${entry.source.side}&start=${chunkStart}` | |
| + `&limit=${TACTILE_CHUNK_FRAMES}&v=20260911-tactile-episode-color-scale`; | |
| const options = { credentials: 'same-origin', cache: 'force-cache' }; | |
| if (signal) options.signal = signal; | |
| fetch(url, options).then(response => { | |
| if (!response.ok) throw new Error(`tactile range failed: ${response.status}`); | |
| const first = Number(response.headers.get('X-Force-First')); | |
| const count = Number(response.headers.get('X-Force-Frames')); | |
| return response.arrayBuffer().then(buffer => ({ buffer, first, count })); | |
| }).then(({ buffer, first, count }) => { | |
| entry.chunkInflight.delete(chunkStart); | |
| if (!isCurrentPlaybackSession(episodeAtRequest, tokenAtRequest) | |
| || !entry.canvas.isConnected) return; | |
| if (!Number.isFinite(first) || !Number.isFinite(count) || count <= 0 | |
| || buffer.byteLength < count * TACTILE_FIELD_STRIDE) return; | |
| entry.chunks.set(chunkStart, { | |
| first, count, | |
| bytes: new Uint8Array(buffer, 0, count * TACTILE_FIELD_STRIDE), | |
| }); | |
| // Paint the frame the player is showing if it just arrived. | |
| if (entry.pendingFrame !== null && entry.pendingFrame !== undefined) { | |
| const field = _tactileFrameField(entry, entry.pendingFrame); | |
| if (field) { | |
| entry.renderer.render(field); | |
| entry.loadedFrame = entry.pendingFrame; | |
| } | |
| } | |
| // Keep one chunk of look-ahead warm so playback never waits on the | |
| // network; the server caches each chunk, so repeats are cheap. | |
| const next = chunkStart + TACTILE_CHUNK_FRAMES; | |
| if (!entry.frameCount || next < entry.frameCount) { | |
| _fetchTactileChunk(entry, next); | |
| } | |
| }).catch(error => { | |
| entry.chunkInflight.delete(chunkStart); | |
| if (error?.name !== 'AbortError' | |
| && isCurrentPlaybackSession(episodeAtRequest, tokenAtRequest)) { | |
| console.warn('[player] tactile chunk unavailable', error); | |
| } | |
| }); | |
| } | |
| function _loadTactileWebGLFrame(entry, frame) { | |
| if (!entry?.canvas || !entry.renderer || !entry.source) return; | |
| const targetFrame = Math.max(0, Math.round(Number(frame) || 0)); | |
| entry.pendingFrame = targetFrame; | |
| const field = _tactileFrameField(entry, targetFrame); | |
| if (field) { | |
| entry.renderer.render(field); | |
| entry.loadedFrame = targetFrame; | |
| const next = _tactileChunkKey(targetFrame) + TACTILE_CHUNK_FRAMES; | |
| if (!entry.frameCount || next < entry.frameCount) { | |
| _fetchTactileChunk(entry, next); | |
| } | |
| return; | |
| } | |
| _fetchTactileChunk(entry, _tactileChunkKey(targetFrame)); | |
| } | |
| async function mountGroupedSource(source, tile) { | |
| const episodeAtMount = currentEpisodeId; | |
| const sessionTokenAtMount = _playbackSessionToken; | |
| if (!tile || currentEpisodeId !== episodeAtMount) return; | |
| if (source.kind === 'slam_trajectory') { | |
| const canvas = document.createElement('canvas'); | |
| canvas.className = 'w-full h-full block bg-black'; | |
| tile.appendChild(canvas); | |
| const entry = { | |
| canvas, source, rotY: 0, rotX: 0.35, zoom: 1, | |
| pan: [0, 0, 0], dragging: false, dragMode: '', | |
| lastX: 0, lastY: 0, fit: null, _lastCam: null, | |
| }; | |
| currentSlamTiles.push(entry); | |
| initSlamDrag(entry); | |
| if (!isCurrentPlaybackSession(episodeAtMount, sessionTokenAtMount) | |
| || !tile.isConnected) return; | |
| renderSlamTile(entry, 0); | |
| return; | |
| } | |
| if (source.kind === 'hand3d_world') { | |
| // 3D 手部世界坐标交互窗口:canvas 渲染 /hand-3d 逐帧数据 | |
| // (worker 只产 parquet,前端实时投影;拖拽旋转 + 滚轮缩放) | |
| const canvas = document.createElement('canvas'); | |
| canvas.className = 'w-full h-full block bg-black'; | |
| tile.appendChild(canvas); | |
| const entry = { canvas, source, rotY: 0, rotX: 0, zoom: 1, | |
| pan: [0, 0, 0], dragging: false, dragMode: '', | |
| lastX: 0, lastY: 0, _lastCam: null, | |
| _hasCommittedFrame: false }; | |
| currentHand3dTiles.push(entry); | |
| initHand3dDrag(entry); | |
| // Do not reveal the spatial canvas before its initial frame window is | |
| // available. RGB/2D and 3D then enter the workspace together. | |
| await _ensureHand3DInitialWindow(source.source_key || 'default'); | |
| if (!isCurrentPlaybackSession(episodeAtMount, sessionTokenAtMount) | |
| || !tile.isConnected) return; | |
| renderHand3DTile(entry, 0); | |
| return; | |
| } | |
| if (source.kind === 'tactile_force_left' | |
| || source.kind === 'tactile_force_right') { | |
| // One tile per side: pressure field on top, its Fx/Fy/Fz history | |
| // underneath. The acquisition-side GripperSideWidget stacks the same | |
| // way (heatmap above, force curve below). | |
| const holder = document.createElement('div'); | |
| holder.className = 'w-full h-full min-h-0 flex flex-col'; | |
| const matrixBox = document.createElement('div'); | |
| matrixBox.className = 'relative flex-1 min-h-0'; | |
| matrixBox.style.backgroundColor = 'rgb(1, 1, 128)'; | |
| const canvas = document.createElement('canvas'); | |
| canvas.className = 'w-full h-full block'; | |
| canvas.setAttribute('aria-label', | |
| source.side === 'right' ? 'Right force matrix' : 'Left force matrix'); | |
| matrixBox.appendChild(canvas); | |
| const forceValue = document.createElement('div'); | |
| forceValue.className = 'absolute z-10 rounded text-white pointer-events-none'; | |
| forceValue.style.cssText = 'top:6px;left:6px;padding:2px 6px;font-size:11px;' | |
| + 'background:rgba(0,0,0,.6);font-variant-numeric:tabular-nums;'; | |
| forceValue.textContent = 'Fx — · Fy — · Fz —'; | |
| matrixBox.appendChild(forceValue); | |
| const curveBox = document.createElement('div'); | |
| curveBox.className = 'relative shrink-0'; | |
| curveBox.style.cssText = 'height:92px;background:#0b1220;' | |
| + 'border-top:1px solid rgba(51,65,85,.5);'; | |
| const curveCanvas = document.createElement('canvas'); | |
| curveCanvas.className = 'w-full h-full block'; | |
| curveBox.appendChild(curveCanvas); | |
| const curveCaption = document.createElement('div'); | |
| curveCaption.className = 'absolute pointer-events-none'; | |
| curveCaption.style.cssText = 'top:2px;left:6px;font-size:10px;color:#64748b;'; | |
| curveCaption.textContent = t('force_unit_mn'); | |
| curveBox.appendChild(curveCaption); | |
| holder.appendChild(matrixBox); | |
| holder.appendChild(curveBox); | |
| tile.appendChild(holder); | |
| const renderer = _createTactileWebGLRenderer(canvas); | |
| const entry = { canvas, renderer, source, lastFrame: -1, forceValue, | |
| curveCanvas, loadedFrame: -1, pendingFrame: null, | |
| chunks: new Map(), chunkInflight: new Set(), | |
| frameCount: Number(tactileMatrixData?.frame_count) || 0 }; | |
| _resetTactileView(renderer); | |
| canvas.title = t('force_field_title'); | |
| currentTactileTiles.push(entry); | |
| const frame = typeof currentFrameTarget === 'number' ? currentFrameTarget : 0; | |
| _updateTactileForceTile(entry, frame); | |
| if (renderer) { | |
| renderer.render(); | |
| _loadTactileWebGLFrame(entry, frame); | |
| } else { | |
| forceValue.textContent = 'WebGL2 unavailable'; | |
| } | |
| return; | |
| } | |
| if (source.kind === 'depth' && source.depth_video_url) { | |
| // Depth previews are browser-decodable 8-bit Turbo MP4s generated | |
| // once per source from the canonical 12-bit stream (identical | |
| // clamp/scale/colormap math as the former canvas renderer). Play | |
| // them as native videos: instant open, native seek, and frame sync | |
| // through the same master-drift loop as every other camera. | |
| const playbackUrl = await _preloadBrowserVideoAsset( | |
| source, source.depth_video_url, sessionTokenAtMount); | |
| if (!isCurrentPlaybackSession(episodeAtMount, sessionTokenAtMount) | |
| || !tile.isConnected) return; | |
| const holder = document.createElement('div'); | |
| holder.id = `player-container-${groupedSourceDomId(source)}`; | |
| holder.className = 'bg-black rounded overflow-hidden w-full h-full min-h-0 relative'; | |
| tile.appendChild(holder); | |
| const player = initPlayer( | |
| holder.id, playbackUrl || _browserVideoAssetUrl( | |
| source, source.depth_video_url)); | |
| if (player) players[groupedSourceKey(source)] = player; | |
| return; | |
| } | |
| if (source.kind === 'depth' || source.kind === 'hand') { | |
| const img = source.kind === 'hand' | |
| ? document.createElement('canvas') : document.createElement('img'); | |
| // Hand-pressure panels are generated on a wide 800x220 canvas. Keep | |
| // that native ratio instead of stretching the heatmap vertically. | |
| img.className = source.kind === 'hand' | |
| ? 'w-full block bg-gray-900 hand-heatmap-image' | |
| : 'w-full h-full object-contain bg-black'; | |
| if (source.kind === 'hand') { | |
| img.style.objectFit = 'contain'; | |
| img.style.width = '100%'; | |
| img.style.height = 'auto'; | |
| img.style.maxHeight = '100%'; | |
| img.style.aspectRatio = String(HAND_PREVIEW_ASPECT); | |
| tile.style.alignItems = 'center'; | |
| tile.style.justifyContent = 'center'; | |
| } | |
| if (source.kind !== 'hand') img.alt = source.label || 'Sensor / Depth'; | |
| tile.appendChild(img); | |
| const entry = { img, source }; | |
| currentImageTiles.push(entry); | |
| const frame = typeof currentFrameTarget === 'number' ? currentFrameTarget : 0; | |
| if (source.kind === 'hand') { | |
| // Hand-pressure tile: use the full frames-data payload and draw | |
| // locally; this avoids one PNG request per video frame. | |
| if (typeof registerHandTile === 'function') registerHandTile(img, source.hand); | |
| } else if (source.kind === 'depth') { | |
| // Depth streams are mounted by the Canvas/code path above. | |
| } else { | |
| img.src = _frameUrl( | |
| source.heatmap_url || source.depth_preview_url || source.depth_url, | |
| frame, | |
| ); | |
| } | |
| return; | |
| } | |
| const holder = document.createElement('div'); | |
| holder.id = `player-container-${groupedSourceDomId(source)}`; | |
| holder.className = 'bg-black rounded overflow-hidden w-full h-full min-h-0'; | |
| tile.appendChild(holder); | |
| const sourceUrl = source.stream_url || | |
| await pickVideoUrl(episodeAtMount, source.source_key); | |
| const playbackUrl = await _preloadBrowserVideoAsset( | |
| source, sourceUrl, sessionTokenAtMount); | |
| if (!isCurrentPlaybackSession(episodeAtMount, sessionTokenAtMount) | |
| || !tile.isConnected) return; | |
| const player = initPlayer(holder.id, playbackUrl || sourceUrl); | |
| if (player) players[groupedSourceKey(source)] = player; | |
| // 手部骨骼 SVG 叠加层(只加在原始视频上;skeleton/渲染 tile 不加) | |
| // The first keypoint window is part of the shared initial-frame barrier. | |
| if (typeof initHandOverlay === 'function') { | |
| await initHandOverlay(source, tile, holder); | |
| } | |
| } | |
| /* 图像素材(手套热力图/深度图)跟随画布内任一 player 的播放进度 */ | |
| function bindGloveSync(player) { | |
| if (!player || _gloveSyncBound.has(player)) return; | |
| _gloveSyncBound.add(player); | |
| player.on('timeupdate', () => { | |
| if (isFrameScrubbing()) return; | |
| // 暂停态下的 seek(逐帧步进)也走这里:timeupdate 会在 seek 后触发一次 | |
| _refreshImageTilesAt(Math.floor((player.currentTime || 0) * getEpisodeFps() + 0.002)); | |
| }); | |
| } | |
| let _lastImageTileFrame = -1; | |
| function _refreshImageTilesAt(frame) { | |
| if (!currentImageTiles.length && !currentDepthPreviewTiles.length | |
| && !currentTactileTiles.length) return; | |
| _refreshDepthTilesAt(frame); | |
| if (currentImageTiles.length && frame !== _lastImageTileFrame) { | |
| _lastImageTileFrame = frame; | |
| currentImageTiles.forEach((entry) => { | |
| const { img, source } = entry; | |
| // Hand/glove tiles are driven by heatmap.js from the preloaded | |
| // frames-data payload; only template-based non-depth tiles are | |
| // refreshed here. | |
| let url; | |
| if (source.kind === 'depth' || source.kind === 'hand' | |
| || source.kind === 'glove') { | |
| return; | |
| } else { | |
| url = source.heatmap_url || source.depth_preview_url || source.depth_url; | |
| } | |
| if (url) img.src = _frameUrl(url, frame); | |
| }); | |
| } | |
| currentTactileTiles.forEach(entry => { | |
| if (!entry.canvas || entry.lastFrame === frame) return; | |
| entry.lastFrame = frame; | |
| _updateTactileForceTile(entry, frame); | |
| _loadTactileWebGLFrame(entry, frame); | |
| }); | |
| } | |
| /* 播放期逐帧刷新图像素材。 | |
| 不能用 rAF 作为视频时钟: rAF 跟显示器刷新(通常 60Hz),而视频可能是 | |
| 30FPS/变速/掉帧,于是深度画布会在同一个视频帧上重复上传,或先于视频 | |
| 画面切到下一帧。优先使用 HTMLVideoElement.requestVideoFrameCallback, | |
| 让 RGB 视频真正送入 compositor 的那一帧驱动所有派生图层。回调内部 | |
| 只发布一个整数帧号,深度 renderer 采用 latest-only,不创建伪彩色文件。 | |
| 旧浏览器才走低频 rAF fallback。 */ | |
| let _imageRafStarted = false; | |
| let _centralLastFrame = -1; | |
| let _presentationClockToken = 0; | |
| let _presentationClockVideo = null; | |
| let _presentationClockHandle = null; | |
| function _rawVideoElement(player) { | |
| if (!player) return null; | |
| return player.media || player.elements?.media || | |
| player.elements?.container?.querySelector('video') || null; | |
| } | |
| function _publishPresentedFrame(video, metadata) { | |
| if (!video || video.paused || video.ended) return; | |
| const mediaTime = Number.isFinite(metadata?.mediaTime) | |
| ? metadata.mediaTime : (video.currentTime || 0); | |
| const frame = _clampFrameIndex(Math.floor( | |
| mediaTime * getEpisodeFps() + 0.002)); | |
| // currentTime assignment is a seek and can make a healthy slave stutter. | |
| // Only correct a real multi-frame drift; initial play/seek alignment is | |
| // handled by playAll/seekToFrame. | |
| const maxDrift = Math.max(0.08, 2.5 / getEpisodeFps()); | |
| Object.values(players).forEach(player => { | |
| const slaveVideo = _rawVideoElement(player); | |
| if (!slaveVideo || slaveVideo === video) return; | |
| if (Math.abs((slaveVideo.currentTime || 0) - mediaTime) > maxDrift) { | |
| try { slaveVideo.currentTime = mediaTime; } catch (_) {} | |
| } | |
| }); | |
| // 深度帧未到时暂停整组播放器，避免 RGB 继续前进而深度画面停在旧帧。 | |
| if (currentDepthPreviewTiles.length && !_depthFrameReady(frame)) { | |
| _stallPlaybackForDepth(frame); | |
| _refreshDepthTilesAt(frame); | |
| return; | |
| } | |
| if (frame !== _centralLastFrame) { | |
| _centralLastFrame = frame; | |
| updateFrameDisplay(frame); | |
| } | |
| _refreshImageTilesAt(frame); | |
| } | |
| function _schedulePresentationFrame(video, token) { | |
| if (token !== _presentationClockToken || | |
| video !== _presentationClockVideo || video.paused || video.ended) return; | |
| if (typeof video.requestVideoFrameCallback === 'function') { | |
| _presentationClockHandle = video.requestVideoFrameCallback((_, metadata) => { | |
| _presentationClockHandle = null; | |
| if (token !== _presentationClockToken || | |
| video !== _presentationClockVideo) return; | |
| _publishPresentedFrame(video, metadata); | |
| _schedulePresentationFrame(video, token); | |
| }); | |
| return; | |
| } | |
| // Compatibility fallback: one rAF still publishes at most one integer | |
| // video frame, so a 30FPS source is never rendered twice per frame. | |
| _presentationClockHandle = requestAnimationFrame(() => { | |
| _presentationClockHandle = null; | |
| if (token !== _presentationClockToken || | |
| video !== _presentationClockVideo) return; | |
| _publishPresentedFrame(video, null); | |
| _schedulePresentationFrame(video, token); | |
| }); | |
| } | |
| function _ensurePresentationClock() { | |
| const video = _rawVideoElement(getActivePlayer()); | |
| if (!video) return; | |
| if (video !== _presentationClockVideo) { | |
| _presentationClockToken += 1; | |
| _presentationClockVideo = video; | |
| _presentationClockHandle = null; | |
| } | |
| if (!video.paused && !video.ended && !_presentationClockHandle) { | |
| _schedulePresentationFrame(video, _presentationClockToken); | |
| } | |
| } | |
| function _startImageTileRaf() { | |
| if (_imageRafStarted) return; | |
| _imageRafStarted = true; | |
| window.__egodataCentralFrameClock = true; | |
| // This is only a cheap watchdog for play/pause and player replacement. | |
| // Actual frame work is scheduled by requestVideoFrameCallback above. | |
| const watchdog = () => { | |
| _ensurePresentationClock(); | |
| setTimeout(watchdog, 100); | |
| }; | |
| watchdog(); | |
| } | |
| function _setPlayerToCurrentFrame(player) { | |
| if (!player) return; | |
| const apply = () => { | |
| const time = Math.max(0, currentFrameTarget / (episodeFps || 30)); | |
| try { player.currentTime = time; } catch (e) {} | |
| }; | |
| // Set immediately when metadata is already available, and set again on | |
| // ready for newly-added videos whose media element is not seekable yet. | |
| apply(); | |
| if (typeof player.on === 'function' && typeof player.off === 'function') { | |
| const onReady = () => { | |
| player.off('ready', onReady); | |
| apply(); | |
| }; | |
| player.on('ready', onReady); | |
| } | |
| } | |
| function _restorePlayersToCurrentFrame() { | |
| Object.values(players).forEach(_setPlayerToCurrentFrame); | |
| } | |
| /* 并排拼接类源(3D 拼接等):内容是左右两路视频拼成的宽画面 | |
| (2560×800),tile 应横跨两列显示,而不是挤在单格里。 | |
| hand3d 是当前的 3D 拼接成品;未来任何带 stereo_side_by_side | |
| 标记的双目骨骼拼接视频自动同样处理。 */ | |
| function _isWideSource(source) { | |
| return Boolean(source) && ( | |
| source.kind === 'hand3d' || | |
| source.stereo_side_by_side === true | |
| ); | |
| } | |
| function _createWorkspaceTile(source, movable, reorderContainer) { | |
| const tile = document.createElement('div'); | |
| tile.className = 'relative bg-black rounded overflow-hidden min-w-0 min-h-0 flex flex-col'; | |
| tile.dataset.sourceKey = groupedSourceKey(source); | |
| tile.style.minWidth = '0'; | |
| tile.style.width = '100%'; | |
| tile.style.height = '100%'; | |
| if (_isWideSource(source)) { | |
| // 横跨两列:与"两路原图各占一格"的总宽度一致,宽画面不再被压成 | |
| // 单格(3D 拼接在 2 列布局下原本只占左侧格,右侧空置) | |
| tile.style.gridColumn = 'span 2'; | |
| } | |
| if (source.kind === 'hand') { | |
| // Match the generated 800x220 hand canvas so the card itself does not | |
| // introduce letterbox space after the image returns to contain mode. | |
| tile.classList.remove('bg-black'); | |
| tile.classList.add('bg-gray-900'); | |
| tile.style.height = 'auto'; | |
| tile.style.maxHeight = '100%'; | |
| tile.style.aspectRatio = String(HAND_PREVIEW_ASPECT); | |
| tile.style.alignSelf = 'center'; | |
| } | |
| if (!source.kind.startsWith('tactile_force_')) { | |
| const label = document.createElement('span'); | |
| label.className = 'absolute z-10 top-1 left-1 bg-black/70 text-gray-200 text-[10px] px-1.5 py-0.5 rounded'; | |
| label.textContent = groupedSourceLabel(source); | |
| tile.appendChild(label); | |
| } | |
| // \u5361\u7247\u53f3\u4e0a\u89d2\u7684 \u00d7 \u5df2\u79fb\u9664:\u5f53\u5361\u7247\u6bd4\u753b\u9762\u5bbd(\u5982 D435_depth \u4e24\u4fa7\u7559\u9ed1\u8fb9) | |
| // \u65f6\u5b83\u6302\u5728\u5361\u7247\u53f3\u4e0a\u89d2\u800c\u975e\u753b\u9762\u53f3\u4e0a\u89d2,\u79bb\u5185\u5bb9\u5f88\u8fdc\u3001\u770b\u7740\u50cf\u98d8\u5728\u5916\u9762\u3002 | |
| // \u589e\u5220\u4e00\u8def\u753b\u9762\u7684\u5165\u53e3\u4ecd\u5728\u6e90\u5217\u8868\u7684\u590d\u9009\u6846\u91cc(\u89c1 renderPreviewVideoSources | |
| // \u7684 input[type=checkbox] \u4e0e All/None \u6309\u94ae),\u529f\u80fd\u6ca1\u6709\u88ab\u5835\u6b7b\u3002 | |
| if (!movable) return tile; | |
| tile.draggable = true; | |
| tile.addEventListener('dragstart', event => { | |
| event.dataTransfer.setData('application/x-egodata-reorder', groupedSourceKey(source)); | |
| event.dataTransfer.effectAllowed = 'move'; | |
| tile.classList.add('opacity-50'); | |
| }); | |
| tile.addEventListener('dragend', () => | |
| tile.classList.remove('opacity-50', 'ring-2', 'ring-blue-500')); | |
| tile.addEventListener('dragover', event => { | |
| const types = event.dataTransfer.types || []; | |
| if (types.includes('application/x-egodata-reorder') || | |
| types.includes('application/x-egodata-source')) { | |
| event.preventDefault(); | |
| event.stopPropagation(); | |
| event.dataTransfer.dropEffect = types.includes('application/x-egodata-reorder') | |
| ? 'move' : 'copy'; | |
| tile.classList.add('ring-2', 'ring-blue-500'); | |
| } | |
| }); | |
| tile.addEventListener('dragleave', () => | |
| tile.classList.remove('ring-2', 'ring-blue-500')); | |
| tile.addEventListener('drop', event => { | |
| event.preventDefault(); | |
| event.stopPropagation(); | |
| tile.classList.remove('ring-2', 'ring-blue-500'); | |
| const fromKey = event.dataTransfer.getData('application/x-egodata-reorder'); | |
| if (fromKey && fromKey !== groupedSourceKey(source)) { | |
| reorderTiles(fromKey, groupedSourceKey(source), reorderContainer); | |
| return; | |
| } | |
| const raw = event.dataTransfer.getData('application/x-egodata-source'); | |
| if (raw) { | |
| try { addGroupedSource(JSON.parse(raw)); } catch (e) {} | |
| } | |
| }); | |
| return tile; | |
| } | |
| function _createHandFooter() { | |
| const footer = document.createElement('div'); | |
| footer.className = 'min-w-0 min-h-0 overflow-hidden grid grid-cols-2 gap-3'; | |
| footer.dataset.role = 'hand-footer'; | |
| const extra = document.createElement('div'); | |
| extra.className = 'min-w-0 min-h-0 overflow-hidden grid gap-3'; | |
| extra.dataset.role = 'bottom-extra'; | |
| extra.style.gridColumn = '1 / -1'; | |
| workspaceBottomExtra = extra; | |
| footer.appendChild(extra); | |
| ['left', 'right'].forEach(hand => { | |
| const slot = document.createElement('div'); | |
| slot.className = 'min-w-0 min-h-0 overflow-hidden bg-gray-900 rounded'; | |
| slot.dataset.handSlot = hand; | |
| footer.appendChild(slot); | |
| workspaceHandSlots[hand] = slot; | |
| }); | |
| return footer; | |
| } | |
| function _refreshWorkspaceLayout() { | |
| if (!workspaceGrid || !workspaceMainRow) return; | |
| const mainSources = workspaceSources.filter(source => !_isBottomSource(source)); | |
| const bottomSources = workspaceSources.filter(_isBottomSource); | |
| const handSources = workspaceSources.filter(source => source.kind === 'hand'); | |
| const columns = mainSources.length <= 1 ? 1 : 2; | |
| // 行数按"占用的格子数"算:并排拼接类(3D 拼接等)跨两列,占 2 格; | |
| // 单列布局下(只有一个源)跨列无意义,按 1 格计 | |
| const slots = mainSources.reduce( | |
| (n, source) => n + (columns >= 2 && _isWideSource(source) ? 2 : 1), 0); | |
| const rows = Math.max(1, Math.ceil(slots / columns)); | |
| const hasBottomExtras = bottomSources.some(source => source.kind !== 'hand'); | |
| const bottomHeight = handSources.length | |
| ? (hasBottomExtras ? HAND_FOOTER_WITH_EXTRA_HEIGHT : HAND_FOOTER_HEIGHT) | |
| : 260; | |
| workspaceGrid.style.gridTemplateRows = bottomSources.length | |
| ? `minmax(0, 1fr) ${bottomHeight}px` | |
| : 'minmax(0, 1fr)'; | |
| if (!bottomSources.length && workspaceHandFooter) { | |
| workspaceHandFooter.remove(); | |
| workspaceHandFooter = null; | |
| workspaceHandSlots = { left: null, right: null }; | |
| workspaceBottomExtra = null; | |
| } | |
| workspaceMainRow.style.gridTemplateColumns = | |
| `repeat(${columns}, minmax(0, 1fr))`; | |
| workspaceMainRow.style.gridTemplateRows = | |
| `repeat(${rows}, minmax(0, 1fr))`; | |
| if (!mainSources.length) { | |
| if (!workspaceMainRow.querySelector('[data-role="empty-workspace"]')) { | |
| workspaceMainRow.innerHTML = | |
| `<div data-role="empty-workspace" class="col-span-full row-span-full flex items-center justify-center text-gray-600">${t('drag_here')}</div>`; | |
| } | |
| } else { | |
| const empty = workspaceMainRow.querySelector('[data-role="empty-workspace"]'); | |
| if (empty) empty.remove(); | |
| mainSources.forEach(source => { | |
| const tile = workspaceMainRow.querySelector( | |
| `[data-source-key="${CSS.escape(groupedSourceKey(source))}"]`); | |
| if (tile) { | |
| // 跨列状态统一在这里兜底修正:只有多列布局下并排拼接类 | |
| // 才跨两列(单列时 span 2 会创建隐式列导致布局错位) | |
| tile.style.gridColumn = | |
| columns >= 2 && _isWideSource(source) ? 'span 2' : ''; | |
| workspaceMainRow.appendChild(tile); | |
| } | |
| }); | |
| } | |
| ['left', 'right'].forEach(hand => { | |
| const slot = workspaceHandSlots[hand]; | |
| if (!slot) return; | |
| const source = handSources.find(item => item.hand === hand); | |
| const tile = source | |
| ? workspaceGrid.querySelector(`[data-source-key="${CSS.escape(groupedSourceKey(source))}"]`) | |
| : null; | |
| if (tile && tile.parentNode !== slot) slot.appendChild(tile); | |
| slot.style.display = source ? '' : 'none'; | |
| }); | |
| if (workspaceBottomExtra) { | |
| const extras = bottomSources.filter(source => source.kind !== 'hand'); | |
| workspaceBottomExtra.style.display = extras.length ? 'grid' : 'none'; | |
| workspaceBottomExtra.style.gridTemplateColumns = | |
| `repeat(${Math.min(2, Math.max(1, extras.length))}, minmax(0, 1fr))`; | |
| extras.forEach(source => { | |
| const tile = workspaceGrid.querySelector( | |
| `[data-source-key="${CSS.escape(groupedSourceKey(source))}"]`); | |
| if (tile && tile.parentNode !== workspaceBottomExtra) { | |
| workspaceBottomExtra.appendChild(tile); | |
| } | |
| }); | |
| } | |
| if (workspaceHandFooter) { | |
| const hasExtras = bottomSources.some(source => source.kind !== 'hand'); | |
| const hasHands = handSources.length > 0; | |
| workspaceHandFooter.style.gridTemplateRows = hasExtras && hasHands | |
| ? 'minmax(0, 1fr) minmax(0, 1fr)' | |
| : 'minmax(0, 1fr)'; | |
| } | |
| } | |
| async function _appendWorkspaceSource(source) { | |
| if (!workspaceGrid || !workspaceMainRow) return; | |
| const movable = !_isBottomSource(source); | |
| const tile = _createWorkspaceTile(source, movable, workspaceMainRow); | |
| if (_isBottomSource(source)) { | |
| if (!workspaceHandFooter) { | |
| workspaceHandFooter = _createHandFooter(); | |
| workspaceGrid.appendChild(workspaceHandFooter); | |
| } | |
| if (source.kind === 'hand') { | |
| const slot = workspaceHandSlots[source.hand]; | |
| if (slot) slot.appendChild(tile); | |
| } else if (workspaceBottomExtra) { | |
| workspaceBottomExtra.appendChild(tile); | |
| } | |
| } else { | |
| const empty = workspaceMainRow.querySelector('[data-role="empty-workspace"]'); | |
| if (empty) empty.remove(); | |
| workspaceMainRow.appendChild(tile); | |
| } | |
| _refreshWorkspaceLayout(); | |
| await mountGroupedSource(source, tile); | |
| const player = players[groupedSourceKey(source)]; | |
| _setPlayerToCurrentFrame(player); | |
| _syncWorkspacePlayers(); | |
| } | |
| function renderGroupedWorkspace() { | |
| const grid = document.getElementById('video-grid'); | |
| if (!grid) return; | |
| const renderEpoch = currentEpisodeId; // 防快速切换批次:过期渲染不得启用控件/揭遮罩 | |
| const renderToken = _playbackSessionToken; | |
| masterPlaying = false; | |
| refreshPlayButton(false); | |
| if (typeof stopHeatmapSync === 'function') stopHeatmapSync(); | |
| Object.values(players).forEach(player => { try { player.destroy(); } catch (e) {} }); | |
| Object.keys(players).forEach(key => delete players[key]); | |
| currentImageTiles = []; | |
| currentDepthPreviewTiles = []; | |
| currentHand3dTiles = []; | |
| if (typeof clearHandTiles === 'function') clearHandTiles(); | |
| workspaceGrid = grid; | |
| workspaceMainRow = null; | |
| workspaceHandFooter = null; | |
| workspaceHandSlots = { left: null, right: null }; | |
| workspaceBottomExtra = null; | |
| grid.innerHTML = ''; | |
| grid.style.position = 'relative'; | |
| grid.className = 'flex-1 min-w-0 min-h-0 p-3 overflow-hidden grid gap-3'; | |
| grid.ondragover = event => { | |
| event.preventDefault(); | |
| grid.classList.add('ring-1', 'ring-blue-500/50'); | |
| }; | |
| grid.ondragleave = () => grid.classList.remove('ring-1', 'ring-blue-500/50'); | |
| grid.ondrop = event => { | |
| event.preventDefault(); | |
| grid.classList.remove('ring-1', 'ring-blue-500/50'); | |
| const raw = event.dataTransfer.getData('application/x-egodata-source'); | |
| if (raw) { | |
| try { addGroupedSource(JSON.parse(raw)); } catch (e) {} | |
| } else { | |
| const hand = event.dataTransfer.getData('application/x-egodata-hand'); | |
| if (hand) addHandTile(hand); | |
| } | |
| }; | |
| const mainSources = workspaceSources.filter(source => !_isBottomSource(source)); | |
| const bottomSources = workspaceSources.filter(_isBottomSource); | |
| const handSources = workspaceSources.filter(source => source.kind === 'hand'); | |
| const hasBottomExtras = bottomSources.some(source => source.kind !== 'hand'); | |
| grid.style.gridTemplateRows = bottomSources.length | |
| ? `minmax(0, 1fr) ${handSources.length | |
| ? (hasBottomExtras ? HAND_FOOTER_WITH_EXTRA_HEIGHT : HAND_FOOTER_HEIGHT) | |
| : 260}px` | |
| : 'minmax(0, 1fr)'; | |
| const mainRow = document.createElement('div'); | |
| mainRow.className = 'min-w-0 min-h-0 overflow-hidden grid gap-3'; | |
| mainRow.dataset.role = 'main-workspace'; | |
| workspaceMainRow = mainRow; | |
| grid.appendChild(mainRow); | |
| if (!mainSources.length) { | |
| mainRow.innerHTML = | |
| `<div data-role="empty-workspace" class="col-span-full row-span-full flex items-center justify-center text-gray-600">${t('drag_here')}</div>`; | |
| } | |
| if (bottomSources.length) { | |
| const footer = _createHandFooter(); | |
| workspaceHandFooter = footer; | |
| grid.appendChild(footer); | |
| } | |
| _refreshWorkspaceLayout(); | |
| renderSourceBar(); | |
| const mounted = []; | |
| mainSources.forEach(source => { | |
| const tile = _createWorkspaceTile(source, true, mainRow); | |
| mainRow.appendChild(tile); | |
| mounted.push(mountGroupedSource(source, tile)); | |
| }); | |
| bottomSources.forEach(source => { | |
| const tile = _createWorkspaceTile(source, false, null); | |
| if (source.kind === 'hand') { | |
| const slot = workspaceHandSlots[source.hand]; | |
| if (slot) slot.appendChild(tile); | |
| } else if (workspaceBottomExtra) { | |
| workspaceBottomExtra.appendChild(tile); | |
| } | |
| mounted.push(mountGroupedSource(source, tile)); | |
| }); | |
| Promise.all(mounted).then(async () => { | |
| if (!isCurrentPlaybackSession(renderEpoch, renderToken)) return; // 本次渲染作废 | |
| _restorePlayersToCurrentFrame(); | |
| const active = getActivePlayer(); | |
| if (!active) { hideVideoLoading(); return; } | |
| // A freshly mounted episode always starts paused. Clear the state and | |
| // sync before wiring the players: _syncWorkspacePlayers() re-issues | |
| // play() for a workspace still marked as playing, and the pauseAll() | |
| // further down then aborts it — a play/pause race that fires an | |
| // uncaught AbortError and leaves the transport button flapping. | |
| masterPlaying = false; | |
| _syncWorkspacePlayers(); | |
| refreshPlayButton(false); | |
| try { | |
| await loadFrameData(renderEpoch, renderToken); | |
| } catch (e) { | |
| if (isCurrentPlaybackSession(renderEpoch, renderToken)) hideVideoLoading(); | |
| return; | |
| } | |
| if (!isCurrentPlaybackSession(renderEpoch, renderToken)) return; | |
| if (!_defaultLayoutApplied && typeof hasHandSensor === 'function') { | |
| _defaultLayoutApplied = true; | |
| ['left', 'right'].forEach(hand => { | |
| if (hasHandSensor(hand) && | |
| !workspaceSources.some(source => source.kind === 'hand' && source.hand === hand)) { | |
| addHandTile(hand); | |
| } | |
| }); | |
| } | |
| // A newly opened episode always starts at its own frame zero. Do this | |
| // after the real FPS/frame count arrive so all overlays and media use | |
| // the same exact time origin, regardless of the previous episode. | |
| pauseAll(); | |
| seekToFrame(0); | |
| updateFrameDisplay(0); | |
| _refreshDepthTilesAt(0); | |
| if (typeof startHeatmapSync === 'function') startHeatmapSync(getActivePlayer()); | |
| if (typeof loadAnnotations === 'function') { | |
| await loadAnnotations(renderEpoch).catch(() => {}); | |
| } | |
| if (!isCurrentPlaybackSession(renderEpoch, renderToken)) return; | |
| // 深度首帧必须先落到 Canvas，再放开播放控件；否则用户会先 | |
| // 看到黑色深度窗，RGB 已经开始跑，随后每到深度边界就暂停。 | |
| // 短批次在这里等待整段 raw code preload，长批次等待首个 120 | |
| // 帧窗口，后续窗口由播放时钟提前预取。 | |
| // 全量加载契约:深度帧全部驻留才放开播放控件(手套/仿生手掌 | |
| // 数据随 loadFrameData 已在上面 await 完成并本地渲染,播放中 | |
| // 两类 tile 均无逐帧网络请求)。 | |
| const depthReady = await _awaitDepthInitialReady(renderEpoch); | |
| if (!isCurrentPlaybackSession(renderEpoch, renderToken)) return; | |
| if (!depthReady) { | |
| console.warn('[player] depth data not ready; keep playback gated'); | |
| return; | |
| } | |
| // 全量流解码时已顺带算出真实 q01/q99(服务端缓存),重拉一次 | |
| // 让 Turbo 色带从兜底全范围切到自适应范围。 | |
| _refreshDepthRanges(); | |
| // 全部就绪才放开控件/揭遮罩:消除「第二个视频晚 ready → 开头不同步」 | |
| // 与「帧数据返回前用旧 fps 交互 → 帧不对齐」两个竞态窗口 | |
| await _awaitPlayersReady(); | |
| if (!isCurrentPlaybackSession(renderEpoch, renderToken)) return; | |
| enableFrameControls(); | |
| hideVideoLoading(); | |
| }).catch(err => { | |
| // 兜底:任何未预期异常都不能让遮罩卡死/控件锁死(仅当前批次生效) | |
| console.error('[player] workspace load failed:', err); | |
| if (isCurrentPlaybackSession(renderEpoch, renderToken)) { | |
| enableFrameControls(); | |
| hideVideoLoading(); | |
| } | |
| }); | |
| } | |
| /* 默认画布:双目组 → 左右目并排;多物理设备单目流 → 全部显示 */ | |
| function _defaultWorkspaceFromGroups() { | |
| // 保留已自动挂载的 3D 世界窗口:loadHand3D 可能先于本函数完成 | |
| // 并已把 hand3d_world 加入 workspaceSources,整表重建不能冲掉 | |
| // (否则开关显示开启、窗口实际不存在 —— 竞态脱节 bug) | |
| const keep = workspaceSources.filter(s => | |
| s.kind === 'hand3d_world' || s.kind === 'slam_trajectory' | |
| || s.kind === 'depth' || s.kind === 'tactile_force_left' | |
| || s.kind === 'tactile_force_right'); | |
| // Depth is a first-class default preview source. It must be present in | |
| // workspaceSources before renderGroupedWorkspace takes its mount Promise | |
| // snapshot; adding it immediately after render creates a race where the | |
| // RGB tiles are ready but the depth canvas remains black with no request. | |
| const depthSources = (currentMediaGroups.sources || []) | |
| .filter(s => s.kind === 'depth') | |
| .filter(() => typeof window.isDepthOn !== 'function' || window.isDepthOn()) | |
| .map(_serializeSource); | |
| const retained = [...keep, ...depthSources.filter(depth => | |
| !keep.some(item => groupedSourceKey(item) === groupedSourceKey(depth)))]; | |
| workspaceSources = []; | |
| const grp = (currentMediaGroups.groups || [])[0]; | |
| if (grp && (grp.members || []).length) { | |
| workspaceSources = grp.members.map(m => ({ | |
| kind: 'video', source_key: m.source_key, label: m.label, | |
| stream_url: m.stream_url, video_cache_key: m.video_cache_key, | |
| frame_count: m.frame_count, fps: m.fps, | |
| })); | |
| workspaceSources.push(...retained); | |
| return; | |
| } | |
| const singles = currentMediaGroups.singles || []; | |
| if (singles.length) { | |
| workspaceSources = singles.map(single => ({ | |
| kind: 'video', source_key: single.source_key, label: single.label, | |
| stream_url: single.stream_url, video_cache_key: single.video_cache_key, | |
| frame_count: single.frame_count, fps: single.fps, | |
| })); | |
| } | |
| workspaceSources.push(...retained); | |
| } | |
| async function loadGroupedEpisodeVideo(episodeId, cameras, options = {}) { | |
| _cancelEpisodeMediaLoad(true); | |
| _episodeLoadInFlight = true; | |
| const loadToken = _playbackSessionToken; | |
| _episodeMediaLoadController = new AbortController(); | |
| const mediaSignal = _episodeMediaLoadController.signal; | |
| currentEpisodeId = episodeId; | |
| currentCameras = cameras || []; | |
| currentHasSkeleton = Boolean(options.hasSkeleton); | |
| _suppressMasterEvents = true; | |
| pauseAll(); | |
| _suppressMasterEvents = false; | |
| masterPlaying = false; | |
| refreshPlayButton(false); | |
| currentMediaGroups = null; | |
| workspaceSources = []; | |
| workspaceCameras = currentCameras.slice(); | |
| currentImageTiles = []; | |
| currentDepthPreviewTiles = []; | |
| currentHand3dTiles = []; | |
| currentSlamTiles = []; | |
| currentTactileTiles = []; | |
| hand3dData = null; | |
| hand3dDataBySource = {}; | |
| hand3dFrameCache = { frame: -1, data: null, inflight: -1 }; | |
| hand3dFrameCacheBySource = {}; | |
| hand3dWindow = { start: -1, end: -1, frames: {}, inflight: false }; | |
| hand3dWindowBySource = {}; | |
| slamData = null; | |
| slamFramesByIndex = {}; | |
| slamWindowInflight = null; | |
| tactileMatrixData = null; | |
| _depthPlaybackStall = null; | |
| _hand3dViewAnchor = null; | |
| _h3dBaseDist = null; | |
| _h3dGridY = null; | |
| _h3dCentroid.filters = [null, null]; | |
| _h3dCentroid.prevLabels = [null, null]; | |
| _h3dCentroid.prevC = [null, null]; | |
| _h3dCentroid.t0 = null; | |
| currentFrameTarget = 0; | |
| _centralLastFrame = -1; | |
| _lastDisplayedFrame = -1; | |
| _pendingSeekFrame = null; | |
| // Seed the UI with the selected episode's canonical metadata immediately. | |
| // frameDataReady remains false until frames-data has been validated. | |
| episodeTotalFrames = Math.max(0, Number(options.frameCount) || 0); | |
| episodeFps = Math.max(0, Number(options.fps) || 0); | |
| frameDataReady = false; | |
| _resetFrameUiForEpisode(0, episodeTotalFrames); | |
| _watchdogRetried = false; // 每个批次允许一次自愈重渲染 | |
| cancelSegmentPlayback(); // 切批次即终止段播放会话(旧会话的播放器即将被销毁) | |
| _defaultLayoutApplied = false; | |
| if (typeof resetHeatmapFrameState === 'function') resetHeatmapFrameState(episodeId); | |
| if (typeof clearAnnotationsNow === 'function') clearAnnotationsNow(); // 立即清掉上一批次残留的切片/overlay | |
| if (typeof stopHeatmapSync === 'function') stopHeatmapSync(); | |
| if (typeof clearHandTiles === 'function') clearHandTiles(); | |
| if (typeof destroyAllHandOverlays === 'function') destroyAllHandOverlays(); | |
| resetSourceBar(); | |
| disableFrameControls(); | |
| showVideoLoading(); | |
| // 自愈保险:加载链任何一环卡住,10 秒后强制启用控件并揭开遮罩 —— | |
| // 控件宁可可用,不可锁死("点了没反应"必须被兜底修掉)。 | |
| // 若此时还没有任何活跃播放器(挂载失败/被冲掉),重渲染工作区一次。 | |
| // 加载请求仍在进行中(如 hand-3d 元数据冷读 20s+)时不判定为故障, | |
| // 重新武装 watchdog 下一轮再查,避免误报"无活跃播放器"。 | |
| clearTimeout(_loadWatchdog); | |
| const watchdogTick = () => { | |
| if (!isCurrentPlaybackSession(episodeId, loadToken)) return; | |
| if (_episodeLoadInFlight) { | |
| _loadWatchdog = setTimeout(watchdogTick, 10000); | |
| return; | |
| } | |
| const depthStillBuffering = currentDepthPreviewTiles.some(entry => | |
| (entry.fullInflight && !entry.fullReady) || | |
| (entry.allPreloadInflight && !entry.allPreloaded)); | |
| if (!_controlsEnabled && !depthStillBuffering) { | |
| console.warn('[player] 加载自愈:强制启用控件'); | |
| enableFrameControls(); | |
| hideVideoLoading(); | |
| } | |
| if (!getActivePlayer() && !_watchdogRetried) { | |
| _watchdogRetried = true; | |
| console.warn('[player] 加载自愈:无活跃播放器 → 重渲染工作区'); | |
| try { renderGroupedWorkspace(); } catch (e) { | |
| console.error('[player] 自愈重渲染失败:', e); | |
| } | |
| } | |
| }; | |
| _loadWatchdog = setTimeout(watchdogTick, 10000); | |
| // Fetch 3D metadata in parallel with media-groups, but await both before | |
| // building the workspace. The 3D tile itself then awaits its first data | |
| // window together with each RGB overlay's first keypoint window. | |
| const hand3dReady = loadHand3D(episodeId, loadToken); | |
| const slamReady = loadSlamTrajectory(episodeId, loadToken); | |
| const tactileReady = loadTactileMatrixMeta(episodeId, loadToken); | |
| const grid = document.getElementById('video-grid'); | |
| if (!grid) { hideVideoLoading(); return; } | |
| // 媒体组 API:双目分组 + 素材清单(新/旧数据都兼容) | |
| try { | |
| // Media groups include transport URLs. They must not be served from | |
| // the browser's GET cache after a backend/frontend rollout. | |
| const res = await fetch( | |
| `/api/v1/episode/${episodeId}/media-groups?_v=202609022200`, | |
| { signal: mediaSignal, cache: 'no-store' }); | |
| if (!isCurrentPlaybackSession(episodeId, loadToken)) return; // 丢弃过期响应 | |
| if (res.ok) { | |
| currentMediaGroups = await res.json(); | |
| await Promise.all([hand3dReady, slamReady, tactileReady]); | |
| if (!isCurrentPlaybackSession(episodeId, loadToken)) return; | |
| renderSourceBar(); | |
| _defaultWorkspaceFromGroups(); | |
| // SLAM is a first-class spatial source. Add it before the initial | |
| // workspace render so it enters the same mount barrier as RGB. | |
| if (typeof window.ensureSlamTrajectoryTile === 'function') { | |
| window.ensureSlamTrajectoryTile(); | |
| } | |
| if (typeof window.ensureTactileMatrixTile === 'function') { | |
| window.ensureTactileMatrixTile(); | |
| } | |
| // Add the selected spatial source before renderGroupedWorkspace | |
| // takes its Promise.all(mounted) snapshot. | |
| if (typeof ensureHand3dWorldTile === 'function' | |
| && hand3dData && hand3dData.hasPreview | |
| && (typeof window.isHand3dWorldOn !== 'function' | |
| || window.isHand3dWorldOn())) { | |
| ensureHand3dWorldTile(); | |
| } | |
| renderGroupedWorkspace(); | |
| _episodeLoadInFlight = false; | |
| renderPreviewVideoSources(); | |
| // 3D 世界窗口兜底:开关开且有产物 → 确保已挂载 | |
| if (typeof ensureHand3dWorldTile === 'function' | |
| && hand3dData && hand3dData.hasPreview | |
| && (typeof window.isHand3dWorldOn !== 'function' | |
| || window.isHand3dWorldOn())) { | |
| ensureHand3dWorldTile(); | |
| } | |
| // 深度窗口兜底:☷ 深度开关开且有深度素材 → 自动挂载 | |
| // (素材栏已无深度按钮,挂载入口统一走 ☷) | |
| if (typeof ensureDepthTile === 'function' | |
| && (typeof window.isDepthOn !== 'function' | |
| || window.isDepthOn())) { | |
| ensureDepthTile(); | |
| } | |
| updateInfoBar(); | |
| if (typeof updatePreviewMenuData === 'function') updatePreviewMenuData(); | |
| return; | |
| } | |
| } catch (e) { | |
| if (e?.name === 'AbortError' || !isCurrentPlaybackSession(episodeId, loadToken)) return; | |
| /* fall through → 旧逻辑 */ | |
| } | |
| if (!isCurrentPlaybackSession(episodeId, loadToken)) return; | |
| // 兜底:API 不可用时按 camera_names 平铺到素材按钮条 | |
| renderSourceBar(); | |
| if (!currentCameras.length) { | |
| grid.innerHTML = '<div class="flex items-center justify-center h-64 text-gray-600">' + t('no_video') + '</div>'; | |
| hideVideoLoading(); | |
| _episodeLoadInFlight = false; | |
| return; | |
| } | |
| workspaceSources = currentCameras.slice(0, 2).map(camera => groupedSource('video', camera, { label: camera })); | |
| renderGroupedWorkspace(); | |
| _episodeLoadInFlight = false; | |
| renderPreviewVideoSources(); | |
| updateInfoBar(); | |
| if (typeof updatePreviewMenuData === 'function') updatePreviewMenuData(); | |
| } | |
| function loadEpisodeVideo(episodeId, cameras, options = {}) { | |
| _cancelEpisodeMediaLoad(true); | |
| const loadToken = _playbackSessionToken; | |
| _episodeMediaLoadController = new AbortController(); | |
| currentEpisodeId = episodeId; | |
| currentCameras = cameras || []; | |
| _suppressMasterEvents = true; | |
| pauseAll(); | |
| _suppressMasterEvents = false; | |
| masterPlaying = false; | |
| refreshPlayButton(false); | |
| currentFrameTarget = 0; | |
| _centralLastFrame = -1; | |
| _lastDisplayedFrame = -1; | |
| _pendingSeekFrame = null; // reset — prevent stale seek target from previous episode | |
| episodeTotalFrames = Math.max(0, Number(options.frameCount) || 0); | |
| episodeFps = Math.max(0, Number(options.fps) || 0); | |
| frameDataReady = false; | |
| _resetFrameUiForEpisode(0, episodeTotalFrames); | |
| cancelSegmentPlayback(); // 切批次即终止段播放会话(旧会话的播放器即将被销毁) | |
| if (typeof resetHeatmapFrameState === 'function') resetHeatmapFrameState(episodeId); | |
| if (typeof clearAnnotationsNow === 'function') clearAnnotationsNow(); // 立即清掉上一批次残留的切片/overlay | |
| _defaultLayoutApplied = false; | |
| showVideoLoading(); | |
| stopHeatmapSync(); | |
| if (typeof clearHandTiles === 'function') clearHandTiles(); | |
| // The legacy single-video fallback uses the same hand-overlay state as | |
| // the grouped workspace. Clear it here too, otherwise a no-data episode | |
| // can leave the global skeleton switch disabled for the next episode. | |
| if (typeof destroyAllHandOverlays === 'function') destroyAllHandOverlays(); | |
| currentHand3dTiles = []; | |
| currentSlamTiles = []; | |
| hand3dData = null; | |
| hand3dDataBySource = {}; | |
| hand3dFrameCache = { frame: -1, data: null, inflight: -1 }; | |
| hand3dFrameCacheBySource = {}; | |
| hand3dWindow = { start: -1, end: -1, frames: {}, inflight: false }; | |
| hand3dWindowBySource = {}; | |
| slamData = null; | |
| slamFramesByIndex = {}; | |
| slamWindowInflight = null; | |
| _depthPlaybackStall = null; | |
| _hand3dViewAnchor = null; | |
| _h3dBaseDist = null; | |
| _h3dGridY = null; | |
| _h3dCentroid.filters = [null, null]; | |
| _h3dCentroid.prevLabels = [null, null]; | |
| _h3dCentroid.prevC = [null, null]; | |
| _h3dCentroid.t0 = null; | |
| loadHand3D(episodeId, loadToken); | |
| resetSourceBar(); | |
| disableFrameControls(); // disabled until player fires 'ready' | |
| const grid = document.getElementById('video-grid'); | |
| const tabs = document.getElementById('camera-tabs'); | |
| if (!grid || !tabs) { hideVideoLoading(); return; } | |
| // Clear existing | |
| grid.innerHTML = ''; | |
| tabs.innerHTML = '<span class="text-gray-500 text-sm py-2">Cameras:</span>'; | |
| // Destroy old players | |
| Object.values(players).forEach(p => p.destroy()); | |
| Object.keys(players).forEach(k => delete players[k]); | |
| if (!cameras || cameras.length === 0) { | |
| grid.innerHTML = '<div class="flex items-center justify-center h-64 text-gray-600">' + t('no_video') + '</div>'; | |
| hideVideoLoading(); | |
| return; | |
| } | |
| // Create tabs | |
| cameras.forEach((cam, i) => { | |
| const btn = document.createElement('button'); | |
| btn.textContent = cam; | |
| btn.className = i === 0 | |
| ? 'px-3 py-1 text-sm rounded-t bg-blue-600 text-white' | |
| : 'px-3 py-1 text-sm rounded-t bg-gray-800 text-gray-400 hover:bg-gray-700'; | |
| btn.onclick = () => switchTab(cam); | |
| tabs.appendChild(btn); | |
| }); | |
| // Create video container(s) | |
| const fallbackColumns = cameras.length <= 1 ? 1 : 2; | |
| const fallbackRows = Math.max(1, Math.ceil(cameras.length / fallbackColumns)); | |
| grid.className = 'flex-1 min-w-0 min-h-0 p-3 overflow-hidden grid gap-3'; | |
| grid.style.gridTemplateColumns = `repeat(${fallbackColumns}, minmax(0, 1fr))`; | |
| grid.style.gridTemplateRows = `repeat(${fallbackRows}, minmax(0, 1fr))`; | |
| grid.style.position = 'relative'; // anchor for skeleton canvas overlay | |
| cameras.forEach(async cam => { | |
| const div = document.createElement('div'); | |
| div.id = `player-container-${cam}`; | |
| div.className = 'bg-black rounded overflow-hidden min-w-0 min-h-0 w-full h-full'; | |
| grid.appendChild(div); | |
| const url = await pickVideoUrl(episodeId, cam); | |
| if (!isCurrentPlaybackSession(episodeId, loadToken)) return; | |
| players[cam] = initPlayer(`player-container-${cam}`, url); | |
| if (players[cam] && typeof initHandOverlay === 'function') { | |
| const source = { kind: 'video', source_key: cam, label: cam }; | |
| initHandOverlay(source, div, div); | |
| } | |
| }); | |
| // Start heatmap sync with first camera's player | |
| const firstCam = cameras[0]; | |
| const loadEpoch = episodeId; // capture for guard — discard if episode changed | |
| if (firstCam && players[firstCam]) { | |
| Object.values(players).forEach(p => bindMasterSync(p)); | |
| bindFrameDrift(players[firstCam]); | |
| loadFrameData(episodeId, loadToken).then(async () => { | |
| if (!isCurrentPlaybackSession(loadEpoch, loadToken)) return; // stale | |
| startHeatmapSync(players[firstCam]); | |
| // Now FPS + frameCount are correct — safe to render annotation overlay | |
| if (typeof loadAnnotations === 'function') { | |
| await loadAnnotations(episodeId).catch(() => {}); | |
| } | |
| if (!isCurrentPlaybackSession(loadEpoch, loadToken)) return; | |
| // 全部就绪才放开控件/揭遮罩(与 grouped 路径同一门控) | |
| await _awaitPlayersReady(); | |
| if (!isCurrentPlaybackSession(loadEpoch, loadToken)) return; | |
| enableFrameControls(); | |
| hideVideoLoading(); | |
| }); | |
| } | |
| // Update info bar | |
| updateInfoBar(); | |
| } | |
| function switchTab(camera) { | |
| // Highlight the active tab | |
| const tabs = document.getElementById('camera-tabs'); | |
| if (tabs) { | |
| tabs.querySelectorAll('button').forEach(btn => { | |
| if (btn.textContent === camera) { | |
| btn.className = 'px-3 py-1 text-sm rounded-t bg-blue-600 text-white'; | |
| } else { | |
| btn.className = 'px-3 py-1 text-sm rounded-t bg-gray-800 text-gray-400 hover:bg-gray-700'; | |
| } | |
| }); | |
| } | |
| // Show only selected camera | |
| const grid = document.getElementById('video-grid'); | |
| if (grid) { | |
| grid.className = 'flex-1 min-w-0 min-h-0 p-3 overflow-hidden grid grid-cols-1 gap-3'; | |
| if (typeof destroyAllHandOverlays === 'function') destroyAllHandOverlays(); | |
| grid.innerHTML = ''; | |
| const oldContainer = document.getElementById(`player-container-${camera}`); | |
| const div = document.createElement('div'); | |
| div.id = `player-container-${camera}`; | |
| div.className = 'bg-black rounded overflow-hidden min-w-0 min-h-0 w-full h-full'; | |
| grid.appendChild(div); | |
| // Re-init player if needed | |
| if (players[camera]) { | |
| players[camera].destroy(); | |
| } | |
| pickVideoUrl(currentEpisodeId, camera).then(url => { | |
| players[camera] = initPlayer(`player-container-${camera}`, url); | |
| if (players[camera] && typeof initHandOverlay === 'function') { | |
| const source = { kind: 'video', source_key: camera, label: camera }; | |
| initHandOverlay(source, div, div); | |
| } | |
| // Re-bind heatmap/frame sync to the new player instance | |
| Object.values(players).forEach(p => bindMasterSync(p)); | |
| bindFrameDrift(players[camera]); | |
| startHeatmapSync(players[camera]); | |
| // Re-render annotation overlay on the rebuilt player | |
| if (typeof annotations !== 'undefined' && annotations.length > 0) { | |
| renderAnnotationOverlay(annotations, episodeTotalFrames); | |
| } | |
| }); | |
| } | |
| } | |
| function playAll() { | |
| // 已播完(ended)后再次点击:HTML5 在 ended 状态 play() 是 no-op, | |
| // 必须先把所有播放器 seek 回第 0 帧才能从头重播。 | |
| const master = getActivePlayer(); | |
| if (master) { | |
| const dur = master.duration || 0; | |
| if (master.ended || (dur > 0 && (master.currentTime || 0) >= dur - 0.1)) { | |
| seekToFrame(0); | |
| } | |
| } | |
| // HTMLMediaElement.play() on several videos is not a barrier: each | |
| // element can start on a different refresh tick. Snap the complete set | |
| // to the master's integer frame before starting, so RGB, overlays and | |
| // the display-only 3D canvas all share one time origin. This does not | |
| // touch the stored parquet/video data. | |
| const syncMaster = getActivePlayer(); | |
| if (syncMaster && !syncMaster.ended) { | |
| const frame = _clampFrameIndex(Math.floor( | |
| (syncMaster.currentTime || 0) * getEpisodeFps() + 0.002)); | |
| const time = frame / getEpisodeFps(); | |
| currentFrameTarget = frame; | |
| Object.values(players).forEach(p => { | |
| try { | |
| if (Math.abs((p.currentTime || 0) - time) > 0.001) { | |
| p.currentTime = time; | |
| } | |
| } catch (e) {} | |
| }); | |
| if (_centralLastFrame !== frame) { | |
| _centralLastFrame = frame; | |
| updateFrameDisplay(frame); | |
| } | |
| } | |
| Object.values(players).forEach(p => { | |
| // play() 是异步 promise:中断(如被 pause 打断)产生的 AbortError | |
| // 拒绝必须吞掉,否则控制台报红且可能破坏后续播放状态。 | |
| try { | |
| const result = p.play(); | |
| if (result && typeof result.catch === 'function') { | |
| result.catch(() => {}); | |
| } | |
| } catch (e) {} | |
| }); | |
| // play() enters the native media state asynchronously. Arm the | |
| // presentation clock on the next task so the first displayed RGB frame | |
| // also becomes the first frame for depth/3D/sensor layers. | |
| setTimeout(_ensurePresentationClock, 0); | |
| } | |
| function pauseAll() { | |
| // Keep the transport state consistent with reality: callers that pause | |
| // without flipping the flag left the button reading "Pause" while every | |
| // player was stopped, so the next click replayed instead of resuming. | |
| masterPlaying = false; | |
| Object.values(players).forEach(p => { | |
| try { p.pause(); } catch(e) {} | |
| }); | |
| } | |
| /* ── Frame-aligned master control ──────────────────────── | |
| One play/pause button drives every video on the canvas. | |
| The master player's timeupdate forces slaves to the same | |
| time (±50ms drift correction); the master seek bar seeks | |
| all videos to the exact same frame. */ | |
| let masterPlaying = false; | |
| const _syncedPlayers = new WeakSet(); // play/pause broadcast bound | |
| const _driftBound = new WeakSet(); // frame-drift correction bound (master only) | |
| function refreshPlayButton(playing) { | |
| const btn = document.getElementById('btn-play-all'); | |
| if (!btn) return; | |
| const icon = btn.querySelector('[data-play-icon]'); | |
| const label = btn.querySelector('.play-label'); | |
| if (icon) { | |
| icon.setAttribute('icon', playing | |
| ? 'ant-design:pause-circle-filled' | |
| : 'ant-design:caret-right-filled'); | |
| } | |
| if (label) label.textContent = playing ? t('pause_btn') : t('play_btn'); | |
| btn.setAttribute('aria-pressed', String(Boolean(playing))); | |
| } | |
| function _lastValidFrame() { | |
| return Math.max(0, (episodeTotalFrames || 0) - 1); | |
| } | |
| function _clampFrameIndex(frameIndex) { | |
| const numeric = Number(frameIndex); | |
| const safeFrame = Number.isFinite(numeric) ? Math.floor(numeric) : 0; | |
| return episodeTotalFrames > 0 | |
| ? Math.max(0, Math.min(_lastValidFrame(), safeFrame)) | |
| : Math.max(0, safeFrame); | |
| } | |
| function stopPlaybackAtLastFrame() { | |
| cancelSegmentPlayback(); | |
| const lastFrame = _lastValidFrame(); | |
| masterPlaying = false; | |
| // Prevent a pause event from one view from starting another pause cascade | |
| // while all synchronized views are being snapped together. | |
| _suppressMasterEvents = true; | |
| pauseAll(); | |
| _suppressMasterEvents = false; | |
| seekToFrame(lastFrame); | |
| updateFrameDisplay(lastFrame); | |
| refreshPlayButton(false); | |
| } | |
| function togglePlayAll() { | |
| const master = getActivePlayer(); | |
| if (!master) { | |
| // 诊断:打印具体原因到控制台,不再"静默没反应" | |
| console.warn('[player] 播放不可用:无活跃播放器', | |
| { players: Object.keys(players), workspaceSources: | |
| workspaceSources.map(s => groupedSourceKey(s)) }); | |
| return; | |
| } | |
| // 只挡"播放器未就绪"。帧数据缺失(如 frames-data 接口瞬时失败)不 | |
| // 挡播放:按时间同步不依赖 fps,图像 tile 短暂用默认 30fps 可接受; | |
| // 若用 frameDataReady 一起挡,接口抖动会导致播放按钮"点了没反应"。 | |
| if (!playerReady) { | |
| console.warn('[player] 播放不可用:播放器未就绪(ready 事件未触发)'); | |
| return; | |
| } | |
| cancelSegmentPlayback(); | |
| if (masterPlaying) { | |
| pauseAll(); | |
| } else { | |
| // Set the broadcast lock before calling play(). Otherwise the first | |
| // native play event sees masterPlaying=false and recursively calls | |
| // playAll(), which can seek the video backwards by one frame/time tick. | |
| masterPlaying = true; | |
| playAll(); | |
| } | |
| // button state follows the players' play/pause events | |
| } | |
| /* Bind play/pause broadcast to ANY player — clicking the play button on | |
| any view (left, right, aux) plays/pauses the whole frame-aligned set. */ | |
| function bindMasterSync(player) { | |
| if (!player || _syncedPlayers.has(player)) return; | |
| _syncedPlayers.add(player); | |
| player.on('play', () => { | |
| if (_suppressMasterEvents) return; | |
| if (!masterPlaying) { | |
| masterPlaying = true; // broadcast lock (prevents re-entry) | |
| playAll(); | |
| } | |
| refreshPlayButton(true); | |
| }); | |
| player.on('pause', () => { | |
| if (_suppressMasterEvents) return; | |
| if (masterPlaying) { | |
| masterPlaying = false; // broadcast lock | |
| pauseAll(); | |
| } | |
| refreshPlayButton(false); | |
| }); | |
| player.on('ended', () => { | |
| if (_suppressMasterEvents) return; | |
| stopPlaybackAtLastFrame(); | |
| }); | |
| } | |
| /* Frame-drift correction on the master player: while playing, every slave | |
| is snapped back within half a frame; on pause all slaves are snapped to | |
| the exact master time so every view shows the identical frame. */ | |
| function bindFrameDrift(master) { | |
| if (!master || _driftBound.has(master)) return; | |
| _driftBound.add(master); | |
| master.on('timeupdate', () => { | |
| if (!masterPlaying || _frameScrubbing) return; | |
| const t = master.currentTime || 0; | |
| const maxDrift = Math.max(0.08, 2.5 / getEpisodeFps()); | |
| Object.values(players).forEach(p => { | |
| if (p === master) return; | |
| try { | |
| if (Math.abs(p.currentTime - t) > maxDrift) p.currentTime = t; | |
| } catch (e) {} | |
| }); | |
| }); | |
| master.on('pause', () => { | |
| const t = master.currentTime || 0; | |
| Object.values(players).forEach(p => { | |
| if (p === master) return; | |
| try { | |
| if (Math.abs(p.currentTime - t) > 0.005) p.currentTime = t; | |
| } catch (e) {} | |
| }); | |
| }); | |
| } | |
| function updateInfoBar() { | |
| const bar = document.getElementById('episode-info'); | |
| if (!bar) return; | |
| // 顶部信息条:组名 / fps / 帧数 / 同步状态 | |
| let groupText = ''; | |
| if (currentMediaGroups) { | |
| groupText = (currentMediaGroups.groups || []).map(g => g.label).join(' / '); | |
| if (!groupText && (currentMediaGroups.singles || []).length) groupText = t('single_mono'); | |
| } | |
| groupText = groupText || (currentCameras.length > 0 ? currentCameras.length + ' Cameras' : ''); | |
| const depthMissing = currentMediaGroups | |
| ? (currentMediaGroups.sources || []) | |
| .filter(source => source.kind === 'depth') | |
| .reduce((total, source) => total + (source.missing_frames || []).length, 0) | |
| : 0; | |
| const syncMarkup = depthMissing > 0 | |
| ? `<span class="text-yellow-400" title="${depthMissing} depth frames are missing"><iconify-icon icon="ant-design:warning-outlined" class="icon-sm"></iconify-icon> Depth missing ${depthMissing}</span>` | |
| : `<span class="text-green-400"><iconify-icon icon="ant-design:sync-outlined" class="icon-sm"></iconify-icon> ${t('synced_ok')}</span>`; | |
| bar.innerHTML = ` | |
| <span class="font-medium text-gray-200">${groupText}</span> | |
| <span class="text-gray-600">|</span> | |
| <span class="font-mono">${getEpisodeFps()} FPS · ${episodeTotalFrames || 0} ${t('frame')}</span> | |
| <span class="text-gray-600">|</span> | |
| ${syncMarkup} | |
| `; | |
| bar.classList.remove('hidden'); | |
| bar.classList.add('flex'); | |
| updateEpisodeDetailSyncStatus(); | |
| } | |
| function updateEpisodeDetailSyncStatus() { | |
| const el = document.getElementById('episode-detail-sync-status'); | |
| if (!el) return; | |
| const depthSources = (currentMediaGroups?.sources || []) | |
| .filter(source => source.kind === 'depth'); | |
| if (!depthSources.length) { | |
| el.classList.add('hidden'); | |
| el.innerHTML = ''; | |
| return; | |
| } | |
| const missingFrames = [...new Set(depthSources.flatMap(source => source.missing_frames || []))] | |
| .sort((a, b) => a - b); | |
| if (!missingFrames.length) { | |
| el.className = 'flex items-center gap-1 text-green-400'; | |
| el.innerHTML = '<iconify-icon icon="ant-design:check-circle-outlined" class="icon-sm"></iconify-icon> Depth data complete'; | |
| return; | |
| } | |
| const preview = missingFrames.slice(0, 4).join(', '); | |
| const suffix = missingFrames.length > 4 ? ', …' : ''; | |
| el.className = 'flex items-start gap-1 text-yellow-400'; | |
| el.innerHTML = `<iconify-icon icon="ant-design:warning-outlined" class="icon-sm mt-0.5"></iconify-icon> | |
| <span>Depth data missing ${missingFrames.length} frame${missingFrames.length === 1 ? '' : 's'} | |
| <span class="text-yellow-500/80">(${preview}${suffix})</span></span>`; | |
| } | |
| // ═══════════════════════════════════════════════════════════ | |
| // Frame-level controls | |
| // ═══════════════════════════════════════════════════════════ | |
| const _frameChangeCallbacks = new Set(); | |
| let episodeFps = 30; | |
| let episodeTotalFrames = 0; | |
| function _syncFrameLabels(frameIndex) { | |
| const frame = Math.max(0, Math.floor(Number(frameIndex) || 0)); | |
| const detailEl = document.getElementById('detail-current-frame'); | |
| if (detailEl) detailEl.textContent = frame; | |
| const heatmapEl = document.getElementById('heatmap-frame'); | |
| if (heatmapEl) heatmapEl.textContent = frame; | |
| const timelineEl = document.getElementById('annotation-timeline-frame'); | |
| if (timelineEl) { | |
| timelineEl.textContent = episodeTotalFrames > 0 | |
| ? frame + ' / ' + episodeTotalFrames : '0 / 0'; | |
| } | |
| } | |
| function _resetFrameUiForEpisode(frameIndex = 0, totalFrames = 0) { | |
| currentFrameTarget = Math.max(0, Math.floor(Number(frameIndex) || 0)); | |
| _lastDisplayedFrame = -1; | |
| const safeTotal = Math.max(0, Math.floor(Number(totalFrames) || 0)); | |
| const detailTotal = document.getElementById('detail-total-frames'); | |
| if (detailTotal && safeTotal > 0) detailTotal.textContent = safeTotal; | |
| if (typeof resetAnnotationTimelineForEpisode === 'function') { | |
| resetAnnotationTimelineForEpisode(safeTotal); | |
| } | |
| _syncFrameLabels(currentFrameTarget); | |
| } | |
| function setEpisodeFrameInfo(fps, totalFrames) { | |
| const prevFps = episodeFps; | |
| episodeFps = fps || 30; | |
| episodeTotalFrames = Math.max(0, Number(totalFrames) || 0); | |
| frameDataReady = true; | |
| if (episodeTotalFrames > 0) { | |
| currentFrameTarget = _clampFrameIndex(currentFrameTarget); | |
| const detailTotal = document.getElementById('detail-total-frames'); | |
| if (detailTotal) detailTotal.textContent = episodeTotalFrames; | |
| } | |
| _syncFrameLabels(currentFrameTarget); | |
| updateInfoBar(); | |
| // fps 变化且已有非零目标帧 → 重对齐:此前用旧 fps 换算的 seek 停在 | |
| // 错误时间(loadFrameData 返回前点过 step/skip 的兜底修复) | |
| if (currentFrameTarget > 0 && episodeFps !== prevFps && !_frameScrubbing) { | |
| Object.values(players).forEach(_setPlayerToCurrentFrame); | |
| } | |
| } | |
| function setOnFrameChange(fn) { | |
| /* 多回调注册(标注高亮/手部骨骼叠加层各自挂一个,互不覆盖) */ | |
| if (typeof fn === 'function') _frameChangeCallbacks.add(fn); | |
| } | |
| let currentFrameTarget = 0; // integer frame index, avoids float error | |
| let _pendingSeekFrame = null; // target frame for in-flight seek, consumed by seeked handler | |
| let _lastDisplayedFrame = -1; // guard against redundant DOM updates on unchanged frame | |
| let _frameScrubbing = false; | |
| let _frameScrubWasPlaying = false; | |
| function isFrameScrubbing() { | |
| return _frameScrubbing; | |
| } | |
| function beginFrameScrub() { | |
| if (_frameScrubbing) return _frameScrubWasPlaying; | |
| _frameScrubWasPlaying = masterPlaying; | |
| _frameScrubbing = true; | |
| // Pause the synchronized set while the pointer previews frames. This | |
| // prevents drift correction from competing with the user's drag. | |
| if (_frameScrubWasPlaying) { | |
| masterPlaying = false; | |
| _suppressMasterEvents = true; | |
| pauseAll(); | |
| _suppressMasterEvents = false; | |
| refreshPlayButton(false); | |
| } | |
| return _frameScrubWasPlaying; | |
| } | |
| function previewFrame(frameIndex) { | |
| currentFrameTarget = _clampFrameIndex(frameIndex); | |
| updateFrameDisplay(currentFrameTarget); | |
| _refreshDepthTilesAt(currentFrameTarget); | |
| } | |
| function commitFrameScrub(frameIndex) { | |
| const resume = _frameScrubWasPlaying; | |
| _frameScrubbing = false; | |
| currentFrameTarget = _clampFrameIndex(frameIndex); | |
| const master = getActivePlayer(); | |
| if (!master) { | |
| seekToFrame(currentFrameTarget); | |
| return; | |
| } | |
| if (!resume) { | |
| seekToFrame(currentFrameTarget); | |
| return; | |
| } | |
| const resumeAfterSeek = () => { | |
| master.off('seeked', resumeAfterSeek); | |
| masterPlaying = true; | |
| playAll(); | |
| }; | |
| master.on('seeked', resumeAfterSeek); | |
| seekToFrame(currentFrameTarget); | |
| // Some media backends do not emit `seeked` when the target time is | |
| // already buffered/current. Keep the resumed-play behavior deterministic. | |
| setTimeout(() => { | |
| if (!_frameScrubbing && _frameScrubWasPlaying && !masterPlaying) { | |
| master.off('seeked', resumeAfterSeek); | |
| masterPlaying = true; | |
| playAll(); | |
| } | |
| }, 120); | |
| } | |
| function getActivePlayer() { | |
| for (const source of workspaceSources) { | |
| if (players[groupedSourceKey(source)]) return players[groupedSourceKey(source)]; | |
| } | |
| for (const cam of currentCameras) { | |
| if (players[cam]) return players[cam]; | |
| } | |
| return null; | |
| } | |
| function getCurrentFrame() { | |
| // During stepping, use the tracked integer target to avoid float precision loss. | |
| // During playback, timeupdate will sync currentFrameTarget from actual time. | |
| return currentFrameTarget; | |
| } | |
| function getCurrentTime() { | |
| const p = getActivePlayer(); | |
| if (!p) return 0; | |
| return p.currentTime || 0; | |
| } | |
| function getEpisodeFps() { | |
| /* Single source of truth for FPS — use this everywhere instead of a local copy. */ | |
| return episodeFps || 30; | |
| } | |
| function consumeSeekTarget() { | |
| /* Return the exact target frame from the most recent seekToFrame call, then clear it. | |
| This avoids the round-trip error where seeked recalculates a different frame | |
| from currentTime due to browser seek imprecision / millisecond truncation. */ | |
| if (_pendingSeekFrame != null) { | |
| const f = _pendingSeekFrame; | |
| _pendingSeekFrame = null; | |
| return f; | |
| } | |
| return null; | |
| } | |
| function seekToFrame(frameIndex) { | |
| frameIndex = _clampFrameIndex(frameIndex); | |
| currentFrameTarget = frameIndex; | |
| // 不提前返回:播放器尚未挂载时目标帧已记录在 currentFrameTarget, | |
| // 新播放器挂载(_setPlayerToCurrentFrame)或就绪后会自动补 seek, | |
| // 避免"刚切换视频时点切片静默失败"。 | |
| const time = frameIndex / (episodeFps || 30); | |
| // Only remember a pending exact-frame target when some player will | |
| // actually seek. A mount-time seekToFrame(0) on a fresh video never | |
| // fires 'seeked', leaving a stale 0 that the pause handler consumed and | |
| // reset the frame display to 0 after the first play. | |
| let willSeek = false; | |
| Object.values(players).forEach(player => { | |
| try { | |
| if (Math.abs((player.currentTime || 0) - time) > 0.001) { | |
| willSeek = true; | |
| } | |
| } catch (e) { /* ignore */ } | |
| }); | |
| if (willSeek) _pendingSeekFrame = frameIndex; | |
| Object.values(players).forEach(player => { | |
| try { player.currentTime = Math.max(0, time); } catch (e) {} | |
| }); | |
| } | |
| // ── Annotation segment auto-play ────────────────────── | |
| let _segmentPlayback = null; | |
| let _segAutoStopHandler = null; // compatibility for stale callers | |
| function cancelSegmentPlayback() { | |
| const session = _segmentPlayback; | |
| if (!session) return; | |
| _segmentPlayback = null; // 先清引用,off 失败也不影响后续段播放 | |
| if (session.player && session.handler) { | |
| // 会话的播放器可能已在切批次时被 destroy(Plyr 销毁后其内部 | |
| // container 为 null,off() 会抛 TypeError)—— 兜底吞掉。 | |
| try { session.player.off('timeupdate', session.handler); } catch (e) {} | |
| } | |
| if (session.player && session.startPlayback) { | |
| try { session.player.off('seeked', session.startPlayback); } catch (e) {} | |
| } | |
| } | |
| function legacyPlaySegment(startFrame, endFrame) { | |
| /* Legacy implementation kept only for compatibility with old callers. */ | |
| const p = getActivePlayer(); | |
| if (!p) return; | |
| // Remove any previous auto-stop handler(播放器可能已被销毁,兜底吞掉) | |
| if (_segAutoStopHandler) { | |
| try { p.off('timeupdate', _segAutoStopHandler); } catch (e) {} | |
| _segAutoStopHandler = null; | |
| } | |
| // Seek to start and play | |
| seekToFrame(startFrame); | |
| p.play().catch(() => {}); | |
| // Register one-shot auto-pause after endFrame | |
| _segAutoStopHandler = () => { | |
| const cf = Math.floor(p.currentTime * getEpisodeFps() + 0.002); | |
| if (cf >= endFrame) { | |
| p.pause(); | |
| seekToFrame(endFrame); // snap to exact endFrame — timeupdate may overshoot | |
| p.off('timeupdate', _segAutoStopHandler); | |
| _segAutoStopHandler = null; | |
| } | |
| }; | |
| p.on('timeupdate', _segAutoStopHandler); | |
| } | |
| // Final segment-playback implementation. It intentionally sits after the | |
| // legacy handler above so older cached markup cannot bypass synchronized | |
| // start/end handling. | |
| function playSegment(startFrame, endFrame) { | |
| const p = getActivePlayer(); | |
| if (!p) return; | |
| const start = _clampFrameIndex(startFrame); | |
| const end = Math.max(start, Math.min(_lastValidFrame(), Math.floor(Number(endFrame) || start))); | |
| cancelSegmentPlayback(); | |
| const session = { | |
| player: p, | |
| endFrame: end, | |
| started: false, | |
| handler: null, | |
| startPlayback: null, | |
| }; | |
| _segmentPlayback = session; | |
| const finish = () => { | |
| if (_segmentPlayback !== session) return; | |
| cancelSegmentPlayback(); | |
| masterPlaying = false; | |
| _suppressMasterEvents = true; | |
| pauseAll(); | |
| _suppressMasterEvents = false; | |
| seekToFrame(end); | |
| updateFrameDisplay(end); | |
| refreshPlayButton(false); | |
| }; | |
| session.handler = () => { | |
| if (_segmentPlayback !== session || !session.started) return; | |
| const frame = Math.floor((p.currentTime || 0) * getEpisodeFps() + 0.002); | |
| if (frame >= end) finish(); | |
| }; | |
| session.startPlayback = () => { | |
| if (_segmentPlayback !== session || session.started) return; | |
| session.started = true; | |
| p.off('seeked', session.startPlayback); | |
| masterPlaying = true; | |
| playAll(); | |
| }; | |
| p.on('timeupdate', session.handler); | |
| p.on('seeked', session.startPlayback); | |
| masterPlaying = false; | |
| _suppressMasterEvents = true; | |
| pauseAll(); | |
| _suppressMasterEvents = false; | |
| seekToFrame(start); | |
| // Buffered media can seek before emitting `seeked`. | |
| setTimeout(session.startPlayback, 120); | |
| } | |
| function stepFrame(delta) { | |
| const p = getActivePlayer(); | |
| if (!p) return; | |
| p.pause(); | |
| // Integer arithmetic — no float precision loss | |
| const newFrame = Math.max(0, Math.min(episodeTotalFrames - 1, currentFrameTarget + delta)); | |
| seekToFrame(newFrame); | |
| } | |
| function skipFrames(delta) { | |
| stepFrame(delta); | |
| } | |
| // Update frame display in UI | |
| function updateFrameDisplay(frameIndex) { | |
| // A media element may report its duration boundary as one frame past the | |
| // last actual image. Keep every UI surface on the valid frame range. | |
| frameIndex = _clampFrameIndex(frameIndex); | |
| // Skip if frame hasn't changed — prevents redundant DOM writes | |
| // that cause visual jitter on every timeupdate tick (~4 Hz) | |
| // but still refresh the timeline cursor. The timeline can be rebuilt after | |
| // the frame clock has already reached 0 (episode switch / annotation load), | |
| // so an early return must not leave the green cursor at the old position. | |
| if (frameIndex === _lastDisplayedFrame) { | |
| _syncFrameLabels(frameIndex); | |
| if (typeof updateAnnotationTimelineCursor === 'function') { | |
| updateAnnotationTimelineCursor(frameIndex); | |
| } | |
| return; | |
| } | |
| _lastDisplayedFrame = frameIndex; | |
| currentFrameTarget = frameIndex; | |
| _syncFrameLabels(frameIndex); | |
| const displayEl = document.getElementById('current-frame-display'); | |
| if (displayEl) displayEl.textContent = 'Frame ' + frameIndex; | |
| _frameChangeCallbacks.forEach(fn => { | |
| try { fn(frameIndex); } catch (e) { /* 单个回调异常不影响其他 */ } | |
| }); | |
| } | |
| // ═══════════════════════════════════════════════════════════ | |
| // Annotation overlay on Plyr progress bar | |
| // ═══════════════════════════════════════════════════════════ | |
| function renderAnnotationOverlay(annotations, totalFrames) { | |
| // Remove existing overlays from ALL players (not just first) | |
| document.querySelectorAll('.anno-progress-overlay').forEach(el => el.remove()); | |
| if (!annotations || annotations.length === 0) return; | |
| if (!totalFrames || totalFrames <= 0) return; // guard: frame data not yet loaded | |
| const maxFrame = totalFrames || (annotations.length > 0 | |
| ? Math.max(...annotations.map(a => a.end_frame_index)) | |
| : 1); | |
| if (maxFrame <= 0) return; | |
| // Build segment elements once, clone into each player's progress bar | |
| const buildSegments = () => annotations.map(seg => { | |
| const leftPct = (seg.start_frame_index / maxFrame) * 100; | |
| const widthPct = ((seg.end_frame_index - seg.start_frame_index + 1) / maxFrame) * 100; | |
| if (widthPct <= 0) return null; | |
| const segEl = document.createElement('div'); | |
| segEl.style.cssText = ` | |
| position: absolute; left: ${leftPct}%; width: ${widthPct}%; | |
| top: 0; height: 100%; background: ${seg.color || '#3B82F6'}; | |
| opacity: 0.55; pointer-events: auto; cursor: pointer; | |
| transition: opacity 0.15s; | |
| `; | |
| segEl.title = seg.label + ' (frame ' + seg.start_frame_index + '–' + seg.end_frame_index + ')'; | |
| segEl.onmouseenter = () => { segEl.style.opacity = '0.85'; }; | |
| segEl.onmouseleave = () => { segEl.style.opacity = '0.55'; }; | |
| segEl.onclick = (e) => { | |
| e.stopPropagation(); | |
| seekToFrame(seg.start_frame_index); | |
| }; | |
| return segEl; | |
| }).filter(Boolean); | |
| // Apply overlay to every player's progress bar (multi-camera support) | |
| document.querySelectorAll('.plyr__progress').forEach(progressContainer => { | |
| const overlay = document.createElement('div'); | |
| overlay.className = 'anno-progress-overlay'; | |
| overlay.style.cssText = ` | |
| position: absolute; top: 0; left: 0; width: 100%; height: 100%; | |
| pointer-events: none; z-index: 1; display: flex; border-radius: inherit; | |
| overflow: hidden; | |
| `; | |
| buildSegments().forEach(segEl => overlay.appendChild(segEl)); | |
| const parentStyle = window.getComputedStyle(progressContainer); | |
| if (parentStyle.position === 'static') { | |
| progressContainer.style.position = 'relative'; | |
| } | |
| progressContainer.appendChild(overlay); | |
| }); | |
| } | |
| function clearAnnotationOverlay() { | |
| document.querySelectorAll('.anno-progress-overlay').forEach(el => el.remove()); | |
| } | |
| // ═══════════════════════════════════════════════════════════ | |
| // Player ready state — controls enabled only after video loaded | |
| // ═══════════════════════════════════════════════════════════ | |
| let playerReady = false; | |
| let frameDataReady = false; // fps/总帧数已就绪;未就绪时禁止 seek/播放(旧 fps 是脏值) | |
| let _controlsEnabled = false; // 控件当前是否已启用(自愈看门狗用) | |
| let _loadWatchdog = null; | |
| let _episodeLoadInFlight = false; // 加载链(媒体组+hand3d)进行中;watchdog 不误判 | |
| let _watchdogRetried = false; // 自愈重渲染只试一次,防循环 | |
| function onPlayerReady() { | |
| playerReady = true; | |
| // 帧数据就绪前不放行控件:此时 fps 可能还是旧批次的脏值, | |
| // 提前放行会让 step/skip/播放用错误 fps 换算 → 帧不对齐。 | |
| // 控件最终由渲染完成链(renderGroupedWorkspace/loadEpisodeVideo)启用。 | |
| if (frameDataReady) enableFrameControls(); | |
| } | |
| function disableFrameControls() { | |
| playerReady = false; | |
| _controlsEnabled = false; | |
| const controls = document.getElementById('frame-controls'); | |
| if (controls) controls.classList.add('hidden'); | |
| const btns = ['btn-skip-back-10', 'btn-step-back', 'btn-play-all', 'btn-step-forward', 'btn-skip-forward-10']; | |
| btns.forEach(id => { | |
| const el = document.getElementById(id); | |
| if (el) { el.disabled = true; el.classList.add('opacity-40', 'cursor-not-allowed'); } | |
| }); | |
| } | |
| function enableFrameControls() { | |
| playerReady = true; | |
| _controlsEnabled = true; | |
| const controls = document.getElementById('frame-controls'); | |
| if (controls) controls.classList.remove('hidden'); | |
| const track = document.getElementById('annotation-timeline-track'); | |
| if (track && typeof bindFrameTimelineSeek === 'function') { | |
| bindFrameTimelineSeek(track, episodeTotalFrames || 1); | |
| } | |
| const btns = ['btn-skip-back-10', 'btn-step-back', 'btn-play-all', 'btn-step-forward', 'btn-skip-forward-10']; | |
| btns.forEach(id => { | |
| const el = document.getElementById(id); | |
| if (el) { el.disabled = false; el.classList.remove('opacity-40', 'cursor-not-allowed'); } | |
| }); | |
| // frame-controls 刚显示,其高度已参与工具条偏移 —— 重新定位播放工具条, | |
| // 否则工具条会按 controlsH=0 下沉,压到标注时间轴/切片块上 | |
| if (typeof _positionPlaybackToolbar === 'function') { | |
| const markers = document.getElementById('annotation-timeline-markers'); | |
| _positionPlaybackToolbar(markers ? markers.offsetHeight : 0); | |
| } | |
| } | |
| // Override stepFrame / seekToFrame to guard against unready state | |
| const _origStepFrame = stepFrame; | |
| stepFrame = function(delta) { | |
| if (!playerReady || !frameDataReady) return; | |
| _origStepFrame(delta); | |
| }; | |
| const _origSeekToFrame = seekToFrame; | |
| seekToFrame = function(frameIndex) { | |
| if (!playerReady || !frameDataReady) return; | |
| _origSeekToFrame(frameIndex); | |
| }; | |
| /* ── 加载遮罩 + 就绪门控 ───────────────────────────── | |
| 批次打开到「媒体组 + 全部播放器 + 帧数据 + 标注」就绪前, | |
| 画布盖遮罩、控件禁用,防止用旧批次 fps 提前交互。 */ | |
| function showVideoLoading() { | |
| const ov = document.getElementById('video-loading-overlay'); | |
| if (ov) ov.classList.remove('hidden'); | |
| } | |
| function hideVideoLoading() { | |
| const ov = document.getElementById('video-loading-overlay'); | |
| if (ov) ov.classList.add('hidden'); | |
| } | |
| function _awaitPlayersReady(timeoutMs = 45000) { | |
| /* 等待当前全部播放器 ready(元数据可 seek)且 canplay(首帧可画)。 | |
| 只等 ready 会让深度预览在控制解锁时还没画出第一帧(黑窗)—— | |
| canplay 保证点播放时每个 tile 立即出画面。单播放器与整体都有 | |
| 超时兜底,防止遮罩卡死。深度预览首次生成需要服务端转码 | |
| (约 4-10s),所以超时给足;纯 RGB 批次仍会在 1-2 秒内放行。 */ | |
| const list = Object.values(players); | |
| if (!list.length) return Promise.resolve(); | |
| const checks = list.map(player => new Promise(resolve => { | |
| let done = false; | |
| const finish = () => { if (!done) { done = true; resolve(); } }; | |
| const tryFinish = () => { | |
| if (!player.ready) return false; | |
| try { | |
| const media = player.media; | |
| if (media && media.readyState >= 3) { finish(); return true; } | |
| } catch (e) { /* fall through to event wait */ } | |
| return false; | |
| }; | |
| if (tryFinish()) return; | |
| try { | |
| const on = typeof player.once === 'function' | |
| ? player.once.bind(player) : player.on.bind(player); | |
| on('ready', () => tryFinish()); | |
| on('canplay', () => tryFinish()); | |
| } catch (e) { finish(); } | |
| setTimeout(finish, 30000); | |
| })); | |
| return Promise.race([ | |
| Promise.all(checks), | |
| new Promise(resolve => setTimeout(resolve, timeoutMs)), | |
| ]); | |
| } | |
| function _refreshDepthRanges() { | |
| // Fetch the adaptive q01/q99 range for every mounted depth tile. | |
| // Called at mount (fallback range arrives instantly) and again after | |
| // the all-loaded barrier (real bounds cached by the full-stream pass). | |
| if (!currentEpisodeId) return; | |
| const episodeAtRequest = currentEpisodeId; | |
| currentDepthPreviewTiles.forEach(entry => { | |
| if (!entry.source || !entry.canvas) return; | |
| fetch(`/api/v1/video/${encodeURIComponent(currentEpisodeId)}` + | |
| `/depth-stats/${encodeURIComponent(entry.source.source_key || '')}`) | |
| .then(response => (response.ok ? response.json() : null)) | |
| .then(stats => { | |
| if (stats && currentEpisodeId === episodeAtRequest && | |
| Number.isFinite(Number(stats.q01)) && | |
| Number.isFinite(Number(stats.q99)) && | |
| Number(stats.q99) > Number(stats.q01)) { | |
| entry.canvas._depthRange = { | |
| q01: Number(stats.q01), | |
| q99: Number(stats.q99), | |
| }; | |
| } | |
| }) | |
| .catch(() => { /* 统计失败 → 渲染器回退固定 0-4095 */ }); | |
| }); | |
| } | |
| async function _awaitDepthInitialReady(episodeId, timeoutMs = 60000) { | |
| const entries = currentDepthPreviewTiles.slice(); | |
| if (!entries.length) return true; | |
| // Strict barrier: playback may start only after EVERY depth frame is | |
| // resident (full stream buffered, or every aligned window preloaded). | |
| // The progressive early-release branches were removed on purpose — | |
| // releasing early caused boundary stalls that read as stutter. | |
| const isReady = (entry) => | |
| (entry.fullReady && entry.frameCount > 0) || | |
| (entry.allPreloaded && entry.frameCount > 0) || | |
| (Number(entry.source?.frame_count) <= 0 && entry.frames.has(0)); | |
| const deadline = performance.now() + timeoutMs; | |
| while (currentEpisodeId === episodeId && performance.now() < deadline) { | |
| if (entries.every(isReady)) return true; | |
| await new Promise(resolve => setTimeout(resolve, 50)); | |
| } | |
| return currentEpisodeId === episodeId && entries.every(isReady); | |
| } | |
| /* ══ 手部骨骼 3D 视图 ══════════════════════════════════ | |
| 数据来自 /api/v1/video/{episode}/hand-3d(hand_3d 产物, | |
| 可能是相对 3D 或相机米制坐标)。Canvas 透视投影,可拖拽旋转, | |
| 随播放帧同步;手 0 = 青色,手 1 = 橙色,label 标记左右。 */ | |
| // MediaPipe 21 点连接拓扑(官方 HAND_CONNECTIONS) | |
| const HAND3D_CONNECTIONS = [ | |
| [0, 1], [1, 2], [2, 3], [3, 4], // 拇指 | |
| [0, 5], [5, 6], [6, 7], [7, 8], // 食指 | |
| [5, 9], [9, 10], [10, 11], [11, 12], // 中指 | |
| [9, 13], [13, 14], [14, 15], [15, 16], // 无名指 | |
| [13, 17], [17, 18], [18, 19], [19, 20], // 小指 | |
| [0, 17], // 掌根 | |
| ]; | |
| async function loadSlamTrajectory(episodeId, sessionToken = null) { | |
| try { | |
| const signal = getMediaLoadSignal(); | |
| const res = await fetch( | |
| `/api/v1/video/${episodeId}/slam-trajectory?max_points=10000`, | |
| signal ? { signal, cache: 'no-store' } : { cache: 'no-store' }); | |
| if (!isCurrentPlaybackSession(episodeId, sessionToken)) return null; | |
| if (!res.ok) return null; | |
| const data = await res.json(); | |
| if (!isCurrentPlaybackSession(episodeId, sessionToken)) return null; | |
| if (!data || data.available !== true) return null; | |
| slamData = data; | |
| slamFramesByIndex = {}; | |
| (data.frames || data.trajectory || []).forEach(row => { | |
| const frame = Number(row && row.f); | |
| const pose = row && row.p; | |
| const valid = Array.isArray(pose) && pose.length >= 7 | |
| && pose.slice(0, 7).every(value => Number.isFinite(Number(value))) | |
| && Math.hypot(...pose.slice(3, 7).map(Number)) >= 0.5; | |
| if (Number.isFinite(frame) && valid) slamFramesByIndex[frame] = row; | |
| }); | |
| return data; | |
| } catch (_) { | |
| return null; | |
| } | |
| } | |
| async function loadTactileMatrixMeta(episodeId, sessionToken = null) { | |
| try { | |
| const signal = getMediaLoadSignal(); | |
| const res = await fetch( | |
| `/api/v1/video/${episodeId}/tactile-matrix-meta?v=202609091720-tactile-fetch-slam-axis`, | |
| signal ? { signal, cache: 'no-store' } : { cache: 'no-store' }); | |
| if (!isCurrentPlaybackSession(episodeId, sessionToken)) return null; | |
| if (!res.ok) return null; | |
| const data = await res.json(); | |
| if (!isCurrentPlaybackSession(episodeId, sessionToken)) return null; | |
| if (!data || data.available !== true) return null; | |
| // Scalar fx/fy/fz values are small enough to preload once per | |
| // episode. Keep them indexed by frame so scrubbing does not issue a | |
| // second parquet request for every tactile image. | |
| const series = data.force_series; | |
| const forceByFrame = Object.create(null); | |
| if (series && Array.isArray(series.frames)) { | |
| series.frames.forEach((frame, index) => { | |
| const key = String(Number(frame)); | |
| forceByFrame[key] = { | |
| left: Array.isArray(series.left) ? series.left[index] : null, | |
| right: Array.isArray(series.right) ? series.right[index] : null, | |
| }; | |
| }); | |
| } | |
| data._forceByFrame = forceByFrame; | |
| tactileMatrixData = data; | |
| return data; | |
| } catch (_) { | |
| return null; | |
| } | |
| } | |
| window.ensureSlamTrajectoryTile = function () { | |
| if (!slamData || slamData.available !== true) return; | |
| const source = { | |
| kind: 'slam_trajectory', | |
| source_key: 'slam_trajectory', | |
| label: 'SLAM', | |
| }; | |
| const key = groupedSourceKey(source); | |
| if (workspaceSources.some(item => groupedSourceKey(item) === key)) return; | |
| workspaceSources.push(source); | |
| }; | |
| window.ensureTactileMatrixTile = function () { | |
| if (!tactileMatrixData || tactileMatrixData.available !== true) return; | |
| const sources = [ | |
| { kind: 'tactile_force_left', source_key: 'tactile_force_left', | |
| side: 'left', label: 'Left' }, | |
| { kind: 'tactile_force_right', source_key: 'tactile_force_right', | |
| side: 'right', label: 'Right' }, | |
| ]; | |
| sources.forEach(source => { | |
| const key = groupedSourceKey(source); | |
| if (!workspaceSources.some(item => groupedSourceKey(item) === key)) { | |
| workspaceSources.push(source); | |
| } | |
| }); | |
| }; | |
| function _slamRows() { | |
| if (!slamData) return []; | |
| const rows = Array.isArray(slamData.trajectory) && slamData.trajectory.length | |
| ? slamData.trajectory : slamData.frames; | |
| return (rows || []).filter(row => row && Array.isArray(row.p) | |
| && row.p.length >= 7 | |
| && row.p.slice(0, 3).every(value => Number.isFinite(Number(value)))); | |
| } | |
| function _slamSmoothRows() { | |
| if (!slamData) return []; | |
| if (Array.isArray(slamData._displayRows)) return slamData._displayRows; | |
| const rows = _slamRows(); | |
| let previous = null; | |
| let previousFrame = null; | |
| const smoothed = rows.map(row => { | |
| const raw = row.p.slice(0, 3).map(Number); | |
| const frame = Number(row.f); | |
| const frameGap = previousFrame === null ? Infinity : frame - previousFrame; | |
| const canSmooth = previous && frameGap <= SLAM_DISPLAY_JOIN_GAP_FRAMES; | |
| const point = canSmooth | |
| ? raw.map((value, axis) => previous[axis] | |
| + SLAM_DISPLAY_SMOOTH_ALPHA * (value - previous[axis])) | |
| : raw; | |
| row._displayPosition = point; | |
| previous = point; | |
| previousFrame = frame; | |
| return row; | |
| }); | |
| slamData._displayRows = smoothed; | |
| return smoothed; | |
| } | |
| // The acquisition side records the S80M camera pose in a gravity-aligned | |
| // world frame: +Z is up (measured on real data - a physical turn of the | |
| // gripper rotates about world +Z, and the gripper's own up axis sits within | |
| // ~12 deg of it for the whole episode). The browser camera uses a right-handed | |
| // frame with Y up, so the display mapping puts world up on screen up. | |
| // | |
| // 源 +X 指向设备的**左侧**(S80M R_corr 之后源系是"左、后、上"), | |
| // 直接透传会让整个空间视图相对真实动作左右镜像:设备往左绕圈, | |
| // 画面里往右走。取反 X 修正左右;Y/Z 不动,前后与上下仍与实测一致。 | |
| function _slamDisplayPoint(sourcePoint) { | |
| return [-Number(sourcePoint[0]), Number(sourcePoint[2]), -Number(sourcePoint[1])]; | |
| } | |
| function _slamSourcePoint(row) { | |
| const point = Array.isArray(row && row._displayPosition) | |
| ? row._displayPosition : row && row.p; | |
| return point ? point.slice(0, 3).map(Number) : null; | |
| } | |
| function _slamCanJoinRows(previousRow, row) { | |
| if (!previousRow || !row) return false; | |
| const gap = Number(row.f) - Number(previousRow.f); | |
| return Number.isFinite(gap) && gap <= SLAM_DISPLAY_JOIN_GAP_FRAMES; | |
| } | |
| function _slamNearestRow(frame) { | |
| const target = Math.max(0, Math.round(Number(frame) || 0)); | |
| const rows = _slamSmoothRows(); | |
| if (!rows.length) return null; | |
| // If the current video frame has no valid pose, keep the last pose that | |
| // has already happened. Never select a future pose for the live marker. | |
| let best = null; | |
| rows.forEach(row => { | |
| const rowFrame = Number(row.f); | |
| if (rowFrame <= target && (!best || rowFrame > Number(best.f))) { | |
| best = row; | |
| } | |
| }); | |
| return best; | |
| } | |
| function _slamFit(rows) { | |
| const points = rows.map(row => _slamDisplayPoint(_slamSourcePoint(row))); | |
| const min = [Infinity, Infinity, Infinity]; | |
| const max = [-Infinity, -Infinity, -Infinity]; | |
| points.forEach(point => point.forEach((value, axis) => { | |
| min[axis] = Math.min(min[axis], value); | |
| max[axis] = Math.max(max[axis], value); | |
| })); | |
| const span = Math.max(max[0] - min[0], max[1] - min[1], max[2] - min[2], 0.4); | |
| const center = min.map((value, axis) => (value + max[axis]) / 2); | |
| const gridStep = 0.2; | |
| const gridY = Math.floor((min[1] - 0.08) / gridStep) * gridStep; | |
| const half = Math.max(0.6, Math.ceil((span * 0.7) / gridStep) * gridStep); | |
| const fov = 45 * Math.PI / 180; | |
| const baseDist = Math.max( | |
| 1.2, | |
| Math.min( | |
| 6, | |
| span * 1.25 * SLAM_DISPLAY_CAMERA_DISTANCE_SCALE | |
| / Math.tan(fov / 2), | |
| ), | |
| ); | |
| return { points, min, max, center, gridY, half, baseDist }; | |
| } | |
| function _slamRotateVector(pose, vector) { | |
| let qx = Number(pose[3]), qy = Number(pose[4]); | |
| let qz = Number(pose[5]), qw = Number(pose[6]); | |
| const norm = Math.hypot(qx, qy, qz, qw); | |
| if (![qx, qy, qz, qw].every(Number.isFinite) || norm < 1e-6) return vector; | |
| qx /= norm; | |
| qy /= norm; | |
| qz /= norm; | |
| qw /= norm; | |
| const tx = 2 * (qy * vector[2] - qz * vector[1]); | |
| const ty = 2 * (qz * vector[0] - qx * vector[2]); | |
| const tz = 2 * (qx * vector[1] - qy * vector[0]); | |
| return [ | |
| vector[0] + qw * tx + qy * tz - qz * ty, | |
| vector[1] + qw * ty + qz * tx - qx * tz, | |
| vector[2] + qw * tz + qx * ty - qy * tx, | |
| ]; | |
| } | |
| function _slamGripperState(row) { | |
| const values = row && Array.isArray(row.g) ? row.g : null; | |
| if (!values || values.length < 1) return null; | |
| const percent = Number(values[0]); | |
| if (!Number.isFinite(percent)) return null; | |
| return { | |
| percent: Math.max(0, Math.min(100, percent)), | |
| gripped: Number(values[1]) >= 0.5, | |
| }; | |
| } | |
| function renderSlamTile(entry, frameOverride) { | |
| if (!entry || !entry.canvas) return; | |
| const canvas = entry.canvas; | |
| const rect = canvas.getBoundingClientRect(); | |
| const dpr = window.devicePixelRatio || 1; | |
| const w = Math.max(1, rect.width), h = Math.max(1, rect.height); | |
| if (canvas.width !== Math.round(w * dpr) || canvas.height !== Math.round(h * dpr)) { | |
| canvas.width = Math.round(w * dpr); | |
| canvas.height = Math.round(h * dpr); | |
| } | |
| const ctx = canvas.getContext('2d'); | |
| ctx.setTransform(dpr, 0, 0, dpr, 0, 0); | |
| ctx.clearRect(0, 0, w, h); | |
| ctx.fillStyle = 'rgba(3,7,18,0.95)'; | |
| ctx.fillRect(0, 0, w, h); | |
| const rows = _slamSmoothRows(); | |
| if (!rows.length) return; | |
| if (!entry.fit) entry.fit = _slamFit(rows); | |
| const fit = entry.fit; | |
| const frame = typeof frameOverride === 'number' | |
| ? frameOverride : (typeof currentFrameTarget === 'number' ? currentFrameTarget : 0); | |
| const cam = _hand3dCam( | |
| fit.points, w, h, entry.rotY || 0, entry.rotX || 0.35, | |
| entry.zoom || 1, fit.baseDist, entry.pan || [0, 0, 0]); | |
| entry._lastCam = cam; | |
| const project = point => _projectHand3D(point, cam); | |
| const line = (a, b, color, width = 1) => { | |
| const pa = project(a), pb = project(b); | |
| if (!Number.isFinite(pa[0]) || !Number.isFinite(pa[1]) | |
| || !Number.isFinite(pb[0]) || !Number.isFinite(pb[1])) return; | |
| ctx.strokeStyle = color; | |
| ctx.lineWidth = width; | |
| ctx.beginPath(); | |
| ctx.moveTo(pa[0], pa[1]); | |
| ctx.lineTo(pb[0], pb[1]); | |
| ctx.stroke(); | |
| }; | |
| // Ground grid: every square is exactly 0.2 m. Labels are intentionally | |
| // kept to one small unit marker so the spatial view stays uncluttered. | |
| const minX = fit.center[0] - fit.half; | |
| const maxX = fit.center[0] + fit.half; | |
| const minZ = fit.center[2] - fit.half; | |
| const maxZ = fit.center[2] + fit.half; | |
| for (let x = minX; x <= maxX + 0.001; x += 0.2) { | |
| line([x, fit.gridY, minZ], [x, fit.gridY, maxZ], 'rgba(75,85,99,0.55)'); | |
| } | |
| for (let z = minZ; z <= maxZ + 0.001; z += 0.2) { | |
| line([minX, fit.gridY, z], [maxX, fit.gridY, z], 'rgba(75,85,99,0.55)'); | |
| } | |
| // Small axis marker at the fitted area: the recorded world axes, drawn in | |
| // the display frame that _slamDisplayPoint produces (world +Z is up). | |
| // The X tip follows the same negation as _slamDisplayPoint: display +X is | |
| // source -X, so the recorded +X axis points toward display -X. | |
| const axisOrigin = [fit.center[0], fit.gridY, fit.center[2]]; | |
| const axisTips = [ | |
| // World axes: X and Y horizontal, Z up. | |
| [[axisOrigin[0] - 0.2, axisOrigin[1], axisOrigin[2]], 'rgb(248,90,90)', 'X'], | |
| [[axisOrigin[0], axisOrigin[1], axisOrigin[2] - 0.2], 'rgb(80,220,130)', 'Y'], | |
| [[axisOrigin[0], axisOrigin[1] + 0.2, axisOrigin[2]], 'rgb(90,150,255)', 'Z'], | |
| ]; | |
| axisTips.forEach(([tip, color, label]) => { | |
| line(axisOrigin, tip, color, 1.5); | |
| const projected = project(tip); | |
| if (!Number.isFinite(projected[0]) || !Number.isFinite(projected[1])) return; | |
| ctx.font = '10px sans-serif'; | |
| ctx.fillStyle = color; | |
| ctx.fillText(label, projected[0] + 3, projected[1] - 3); | |
| }); | |
| const axis = project(axisOrigin); | |
| if (Number.isFinite(axis[0]) && Number.isFinite(axis[1])) { | |
| ctx.font = '9px sans-serif'; | |
| ctx.fillStyle = 'rgba(148,163,184,0.8)'; | |
| ctx.fillText('0.2m', axis[0] + 5, axis[1] + 12); | |
| } | |
| const current = Number(frame); | |
| let previous = null; | |
| let previousRow = null; | |
| rows.forEach(row => { | |
| const rowFrame = Number(row.f); | |
| if (rowFrame > current) return; | |
| const sourcePoint = _slamSourcePoint(row); | |
| const point = _slamDisplayPoint(sourcePoint); | |
| if (previous && _slamCanJoinRows(previousRow, row)) { | |
| line(previous, point, 'rgba(34,211,238,0.95)', 2); | |
| } | |
| previous = point; | |
| previousRow = row; | |
| }); | |
| const currentRow = _slamNearestRow(current); | |
| if (currentRow) { | |
| const pose = currentRow.p.map(Number); | |
| const sourcePoint = _slamSourcePoint(currentRow); | |
| const point = _slamDisplayPoint(sourcePoint); | |
| const projected = project(point); | |
| if (Number.isFinite(projected[0]) && Number.isFinite(projected[1])) { | |
| ctx.fillStyle = '#f8fafc'; | |
| ctx.beginPath(); | |
| ctx.arc(projected[0], projected[1], 4, 0, Math.PI * 2); | |
| ctx.fill(); | |
| ctx.strokeStyle = '#06b6d4'; | |
| ctx.lineWidth = 2; | |
| ctx.stroke(); | |
| } | |
| const axisLength = 0.08; | |
| const colors = ['rgb(248,90,90)', 'rgb(80,220,130)', 'rgb(90,150,255)']; | |
| [[1, 0, 0], [0, 1, 0], [0, 0, 1]].forEach((vector, index) => { | |
| const rotated = _slamRotateVector(pose, vector); | |
| const displayVector = _slamDisplayPoint(rotated); | |
| line(point, [ | |
| point[0] + displayVector[0] * axisLength, | |
| point[1] + displayVector[1] * axisLength, | |
| point[2] + displayVector[2] * axisLength, | |
| ], colors[index], 2); | |
| }); | |
| // Simplified UMI gripper model. The recorded pose frame is the S80M | |
| // camera frame; measured on the uploaded UMIGripper_AI episodes (every | |
| // frame of every batch) the device axes sit at | |
| // +X = gripper right (jaw separation, horizontal), | |
| // +Y = gripper forward (the optical axis, horizontal) and | |
| // +Z = gripper up (vertical, within ~12 deg of world +Z — the world | |
| // frame is gravity-aligned, so this is a fixed sensor-frame fact, not | |
| // a per-episode pose). So the body extends along local +Y, the two | |
| // plates separate along local X, and each plate is 0.20m long | |
| // (forward) by 0.05m tall (up) with a 0.12m maximum opening. | |
| const grip = _slamGripperState(currentRow); | |
| if (grip) { | |
| const toDisplay = local => { | |
| const rotated = _slamRotateVector(pose, local); | |
| return _slamDisplayPoint([ | |
| sourcePoint[0] + rotated[0], | |
| sourcePoint[1] + rotated[1], | |
| sourcePoint[2] + rotated[2], | |
| ]); | |
| }; | |
| const plateLength = 0.20; | |
| const plateHeight = 0.05; | |
| const maxHalfGap = 0.06; | |
| const baseOffset = 0.10; | |
| const halfGap = maxHalfGap * (1 - grip.percent / 100); | |
| const jawColor = grip.gripped | |
| ? 'rgba(248,90,90,0.95)' : 'rgba(88,168,232,0.95)'; | |
| const jawFill = grip.gripped | |
| ? 'rgba(248,90,90,0.16)' : 'rgba(88,168,232,0.12)'; | |
| const relativeCorners = [ | |
| [0, -plateLength / 2, -plateHeight / 2], | |
| [0, plateLength / 2, -plateHeight / 2], | |
| [0, plateLength / 2, plateHeight / 2], | |
| [0, -plateLength / 2, plateHeight / 2], | |
| ]; | |
| for (const sign of [1, -1]) { | |
| const centerX = sign * halfGap; | |
| const corners = relativeCorners.map(corner => project(toDisplay([ | |
| centerX + corner[0], | |
| baseOffset + corner[1], | |
| corner[2], | |
| ]))); | |
| if (corners.every(item => Number.isFinite(item[0]) && Number.isFinite(item[1]))) { | |
| ctx.fillStyle = jawFill; | |
| ctx.strokeStyle = jawColor; | |
| ctx.lineWidth = 2; | |
| ctx.beginPath(); | |
| ctx.moveTo(corners[0][0], corners[0][1]); | |
| corners.slice(1).forEach(item => ctx.lineTo(item[0], item[1])); | |
| ctx.closePath(); | |
| ctx.fill(); | |
| ctx.stroke(); | |
| } | |
| line( | |
| toDisplay([centerX, baseOffset - plateLength / 2, 0]), | |
| toDisplay([centerX, baseOffset + plateLength / 2, 0]), | |
| jawColor, 1); | |
| } | |
| // Rear crossbar plus the two supports that tie each plate back to | |
| // the body, so the opening stays readable when it is small. | |
| line(toDisplay([-maxHalfGap, baseOffset, 0]), | |
| toDisplay([maxHalfGap, baseOffset, 0]), | |
| 'rgba(148,163,184,0.8)', 1.5); | |
| for (const sign of [1, -1]) { | |
| for (const cornerY of [-plateLength / 2, plateLength / 2]) { | |
| line( | |
| toDisplay([sign * halfGap, baseOffset + cornerY, 0]), | |
| toDisplay([0, baseOffset + cornerY, 0]), | |
| 'rgba(148,163,184,0.8)', 1); | |
| } | |
| } | |
| } | |
| } | |
| } | |
| function initSlamDrag(entry) { | |
| if (!entry || !entry.canvas) return; | |
| const canvas = entry.canvas; | |
| canvas.style.cursor = 'grab'; | |
| canvas.addEventListener('mousedown', event => { | |
| if (event.button !== 0 && event.button !== 1) return; | |
| entry.dragging = true; | |
| entry.dragMode = event.button === 1 ? 'pan' : 'rotate'; | |
| entry.lastX = event.clientX; | |
| entry.lastY = event.clientY; | |
| canvas.style.cursor = entry.dragMode === 'pan' ? 'move' : 'grabbing'; | |
| event.preventDefault(); | |
| }); | |
| window.addEventListener('mousemove', event => { | |
| if (!entry.dragging) return; | |
| const dx = event.clientX - entry.lastX; | |
| const dy = event.clientY - entry.lastY; | |
| if (entry.dragMode === 'pan' && entry._lastCam) { | |
| const distance = Math.max(0.2, Math.hypot( | |
| entry._lastCam.eye[0] - entry._lastCam.centroid[0], | |
| entry._lastCam.eye[1] - entry._lastCam.centroid[1], | |
| entry._lastCam.eye[2] - entry._lastCam.centroid[2])); | |
| const worldPerPixel = distance / Math.max(1, entry._lastCam.f); | |
| const pan = entry.pan || (entry.pan = [0, 0, 0]); | |
| for (let i = 0; i < 3; i++) { | |
| pan[i] += (-dx * entry._lastCam.right[i] | |
| + dy * entry._lastCam.up[i]) * worldPerPixel; | |
| } | |
| } else { | |
| entry.rotY += dx * 0.01; | |
| entry.rotX = Math.max(-1.2, Math.min(1.2, entry.rotX + dy * 0.01)); | |
| } | |
| entry.lastX = event.clientX; | |
| entry.lastY = event.clientY; | |
| renderSlamTile(entry); | |
| }); | |
| window.addEventListener('mouseup', () => { | |
| if (!entry.dragging) return; | |
| entry.dragging = false; | |
| entry.dragMode = ''; | |
| canvas.style.cursor = 'grab'; | |
| }); | |
| canvas.addEventListener('contextmenu', event => event.preventDefault()); | |
| canvas.addEventListener('wheel', event => { | |
| event.preventDefault(); | |
| entry.zoom = Math.min(6, Math.max(0.3, | |
| (entry.zoom || 1) * Math.exp(-event.deltaY * 0.0015))); | |
| renderSlamTile(entry); | |
| }, { passive: false }); | |
| } | |
| async function loadHand3D(episodeId, sessionToken = null) { | |
| try { | |
| const signal = getMediaLoadSignal(); | |
| const res = await fetch( | |
| `/api/v1/video/${episodeId}/hand-3d`, | |
| signal ? { signal, cache: 'no-store' } : { cache: 'no-store' }); | |
| if (!isCurrentPlaybackSession(episodeId, sessionToken)) return; // 丢弃过期响应 | |
| if (!res.ok) return; | |
| const data = await res.json(); | |
| if (!isCurrentPlaybackSession(episodeId, sessionToken)) return; | |
| if (!data) return; | |
| const availableOptions = data.available_sources && data.available_sources.length | |
| ? data.available_sources | |
| : [{ ...data, source_key: data.source_key || 'hand3d' }]; | |
| // Stereo produces one 3D sequence per RGB view. The server scores | |
| // valid finite landmarks/frames and exposes the winning source. The | |
| // spatial preview intentionally mounts only that source; hand_0 and | |
| // hand_1 inside it remain available, so both physical hands are kept. | |
| const selectedKey = data.selected_source_key || data.source_key; | |
| const selectedOption = availableOptions.find(option => | |
| option.source_key === selectedKey); | |
| const options = selectedOption ? [selectedOption] : [ | |
| [...availableOptions].sort((a, b) => | |
| (Number(b.valid_landmark_points || 0) | |
| - Number(a.valid_landmark_points || 0)) | |
| || (Number(b.valid_hand_frames || 0) | |
| - Number(a.valid_hand_frames || 0)) | |
| || (a.source_key === 'stereo_left' ? -1 : 1) | |
| )[0], | |
| ]; | |
| hand3dDataBySource = {}; | |
| options.forEach(option => { | |
| const item = { ...data, ...option }; | |
| const relativeMode = item.unit === 'mediapipe_world_relative' | |
| || String(item.mode || '').toLowerCase().includes('relative'); | |
| item.relativeMode = relativeMode; | |
| item.worldMode = item.world_preview === true | |
| && item.depth_available === true | |
| && item.source === 'depth_camera_meters' | |
| && item.unit === 'camera_meters'; | |
| item.rgbEstimatedMode = item.rgb_estimated_preview === true | |
| || item.unit === 'rgb_estimated_meters' | |
| || String(item.mode || '').toLowerCase() === 'rgb_estimated_3d'; | |
| item.spaceMode = item.worldMode || item.rgbEstimatedMode; | |
| // The depth lift stores OpenCV camera coordinates (Y down), while | |
| // RGBWorldTracker stores display coordinates (Y up). Normalize | |
| // only the depth path before projection; do not flip RGB points. | |
| const frameName = String(item.coordinate_frame || '').toLowerCase(); | |
| item.cameraYDown = item.source === 'depth_camera_meters' | |
| && !frameName.includes('y_up'); | |
| item.hasPreview = item.spaceMode; | |
| hand3dDataBySource[item.source_key] = item; | |
| }); | |
| const firstSource = options[0].source_key || data.source_key || 'hand3d'; | |
| hand3dData = hand3dDataBySource[firstSource] || data; | |
| // Mounting is intentionally done by loadGroupedEpisodeVideo after | |
| // media-groups is ready. That keeps this metadata request from | |
| // creating a late tile outside the initial loading barrier. | |
| if (typeof updatePreviewMenuData === 'function') updatePreviewMenuData(); | |
| return hand3dData; | |
| } catch (e) { /* 无 3D 产物 → 不显示 3D 开关 */ | |
| return null; | |
| } | |
| } | |
| /* ☷ "3D Hand World" 开关联动:开启且工作区未挂载时自动加 3D canvas 窗口。 | |
| 幂等性校验必须落到 DOM:源在 workspaceSources 但 tile 被重建冲掉时 | |
| (加载竞态),列表存在 ≠ 窗口存在 —— 会造成"开关显示开实际关"。 */ | |
| window.ensureHand3dWorldTile = function () { | |
| const sources = Object.values(hand3dDataBySource || {}) | |
| .filter(item => item && item.hasPreview); | |
| if (!sources.length && hand3dData && hand3dData.hasPreview) sources.push(hand3dData); | |
| sources.forEach(item => { | |
| const sourceKey = item.source_key || 'hand3d'; | |
| const src = { | |
| kind: 'hand3d_world', | |
| source_key: sourceKey, | |
| label: `${item.device_name || sourceKey} · ${item.rgbEstimatedMode | |
| ? 'RGB Estimated 3D' : t('hand3d_world')}`, | |
| }; | |
| const key = groupedSourceKey(src); | |
| const present = workspaceSources.some(s => groupedSourceKey(s) === key); | |
| if (present) { | |
| const tile = workspaceGrid | |
| ? workspaceGrid.querySelector(`[data-source-key="${CSS.escape(key)}"]`) | |
| : null; | |
| if (tile) return; | |
| workspaceSources = workspaceSources.filter(s => groupedSourceKey(s) !== key); | |
| } | |
| // During the first episode load the metadata can finish before the | |
| // workspace DOM exists. Queue the source in the source list; mounting | |
| // it here would recursively render an incomplete workspace. | |
| if (!workspaceGrid || !workspaceMainRow) { | |
| workspaceSources.push(src); | |
| return; | |
| } | |
| addGroupedSource(src); | |
| }); | |
| }; | |
| /* ☷ 开关关闭 → 从 workspaceSources 卸除 source(复用 removeGroupedSource | |
| 的完整清理:销毁 Plyr、删 tile、布局回收)。display:none 只藏 tile, | |
| _refreshWorkspaceLayout 仍按源列表预留 260px 底部条 = "位置被占用", | |
| 且再开时无可见变化 —— 必须卸源。再开由 ensure*Tile 重新挂载。 */ | |
| window.removeHand3dWorldTile = function () { | |
| workspaceSources.filter(s => s.kind === 'hand3d_world').forEach(src => { | |
| removeGroupedSource(src); | |
| }); | |
| currentHand3dTiles = currentHand3dTiles.filter( | |
| e => e.source.kind !== 'hand3d_world'); | |
| }; | |
| window.removeDepthTile = function () { | |
| workspaceSources.filter(s => s.kind === 'depth').forEach(src => { | |
| removeGroupedSource(src); | |
| }); | |
| }; | |
| /* ☷ 开关统一入口(hand-overlay.js _applyTileVisibility 调用): | |
| on → 自动挂载(幂等);off → 卸除 source(布局回收)。 */ | |
| window.setHand3dWorldVisible = function (on) { | |
| if (on) window.ensureHand3dWorldTile(); | |
| else window.removeHand3dWorldTile(); | |
| }; | |
| window.setDepthTileVisible = function (on) { | |
| if (on) window.ensureDepthTile(); | |
| else window.removeDepthTile(); | |
| }; | |
| /* ☷ 深度开关联动:开启且批次有深度素材 → 自动挂载深度视频 tile | |
| (与 3D Hand World 同机制:素材栏不再提供深度按钮,唯一入口是 ☷)。 | |
| 同一幂等校验:源在列表但 DOM 无 tile 时重建。 */ | |
| window.ensureDepthTile = function () { | |
| const sources = (typeof currentMediaGroups !== 'undefined' && currentMediaGroups) | |
| ? (currentMediaGroups.sources || []) : []; | |
| const depths = sources.filter(s => s.kind === 'depth'); | |
| depths.forEach(depth => { | |
| const src = _serializeSource(depth); | |
| const key = groupedSourceKey(src); | |
| const present = workspaceSources.some(s => groupedSourceKey(s) === key); | |
| if (present) { | |
| const tile = workspaceGrid | |
| ? workspaceGrid.querySelector(`[data-source-key="${CSS.escape(key)}"]`) | |
| : null; | |
| if (tile) return; | |
| workspaceSources = workspaceSources.filter(s => groupedSourceKey(s) !== key); | |
| } | |
| addGroupedSource(src); | |
| }); | |
| }; | |
| /* 按帧懒加载 3D 手部点:播放/拖动到哪一帧就取哪一帧(全量返回在 | |
| 60 分钟级批次上达 226MB/35s,会把页面拖死) */ | |
| async function fetchHand3DFrame(frame, sourceKey) { | |
| const source = sourceKey || 'default'; | |
| const cache = _hand3dFrameCacheFor(source); | |
| if (!currentEpisodeId || cache.inflight === frame) return; | |
| const episodeAtRequest = currentEpisodeId; | |
| const sessionToken = _playbackSessionToken; | |
| const signal = getMediaLoadSignal(); | |
| cache.inflight = frame; | |
| try { | |
| const res = await fetch( | |
| `/api/v1/video/${episodeAtRequest}/hand-3d?source_key=${encodeURIComponent(source)}&frame=${frame}`, | |
| signal ? { signal } : {}); | |
| if (!isCurrentPlaybackSession(episodeAtRequest, sessionToken)) return; | |
| if (!res.ok) { cache.inflight = -1; return; } | |
| const data = await res.json(); | |
| if (!isCurrentPlaybackSession(episodeAtRequest, sessionToken)) return; | |
| if (cache.inflight !== frame) return; // 帧已跳走,丢弃过期响应 | |
| hand3dFrameCacheBySource[source] = { frame, data, inflight: -1 }; | |
| currentHand3dTiles | |
| .filter(entry => (entry.source.source_key || 'default') === source) | |
| .forEach(entry => renderHand3DTile(entry)); | |
| } catch (e) { | |
| cache.inflight = -1; | |
| } | |
| } | |
| /* 米制 3D 点 → 绕 Y/X 轴旋转 → 屏幕坐标(等距投影) */ | |
| // ── 3D 世界坐标渲染(d435 demo RotatingSkeletonRenderer 的同款数学)── | |
| // 五指分色 + 掌心灰线 + 腕部白点 + 地面网格 + 相机系坐标轴 + 腕部 | |
| // 深度标注;透视投影 f=(h/2)/tan(fov/2),静态视角 θ=π(正面)+ 仰角 25°, | |
| // 世界坐标相机使用固定安全视距,不随当前可见手数量变化。用户拖拽 = | |
| // 相对静态视角的偏转;滚轮缩放 = 相机距离。 | |
| const HAND3D_FINGERS = [ | |
| // Keep the 3D tile identical to hand-overlay.js (the visible RGB video | |
| // overlay): thumb blue and pinky orange. | |
| { name: 'Thumb', ids: [1, 2, 3, 4], color: [0, 128, 255] }, | |
| { name: 'Index', ids: [5, 6, 7, 8], color: [0, 255, 0] }, | |
| { name: 'Middle', ids: [9, 10, 11, 12], color: [0, 255, 255] }, | |
| { name: 'Ring', ids: [13, 14, 15, 16], color: [255, 0, 255] }, | |
| { name: 'Pinky', ids: [17, 18, 19, 20], color: [255, 128, 0] }, | |
| ]; | |
| const HAND3D_PALM = [[0, 1], [0, 5], [5, 9], [9, 13], [13, 17], [0, 17]]; | |
| let _hand3dViewAnchor = null; // 世界坐标模式:统一的固定相机目标/世界原点 | |
| let _h3dBaseDist = null; // 世界坐标模式:首帧锁定的相机距离(仅滚轮缩放改变) | |
| let _h3dGridY = null; // 世界坐标模式:首帧锁定的地面网格高度(视角固定) | |
| // M1 质心锚定(demo _CentroidAnchor 展示路径的 JS 移植):每槽质心 | |
| // 强 One-Euro(freq 3.0/beta 0.3/dcutoff 0.3)+ 共模平移校正 —— | |
| // 静态视角下手稳,"跟随乱动"的抖动主要来自逐帧质心漂移,这里吸收。 | |
| const _h3dCentroid = { filters: [null, null], prevLabels: [null, null], | |
| prevC: [null, null], t0: null }; | |
| function _h3dMakeOneEuro(freqMin, beta, dcutoff) { | |
| let prev = null, prevDx = [0, 0, 0], prevTs = null; | |
| const alpha = (cutoff, dt) => { | |
| const tau = 1 / (2 * Math.PI * cutoff); | |
| return dt / (dt + tau); | |
| }; | |
| return (x, ts) => { | |
| if (prev === null || prevTs === null) { | |
| prev = x.slice(); prevDx = [0, 0, 0]; prevTs = ts; | |
| return x.slice(); | |
| } | |
| const dt = (ts - prevTs) / 1000; | |
| if (dt <= 1e-9) return prev.slice(); | |
| const dx = x.map((v, i) => (v - prev[i]) / dt); | |
| const ad = alpha(dcutoff, dt); | |
| prevDx = dx.map((v, i) => ad * v + (1 - ad) * prevDx[i]); | |
| const fc = freqMin + beta * Math.hypot(prevDx[0], prevDx[1], prevDx[2]); | |
| const a = alpha(fc, dt); | |
| const out = x.map((v, i) => a * v + (1 - a) * prev[i]); | |
| prev = out.slice(); prevTs = ts; | |
| return out.slice(); | |
| }; | |
| } | |
| function _h3dAnchorApply(ptsBySlot, labels, state = _h3dCentroid) { | |
| if (state.t0 === null) state.t0 = performance.now(); | |
| const ts = performance.now() - state.t0; | |
| const out = ptsBySlot.map(arr => arr.map(p => (p ? p.slice() : null))); | |
| for (let s = 0; s < 2; s++) { | |
| const finIdx = []; | |
| out[s].forEach((p, i) => { if (p) finIdx.push(i); }); | |
| if (finIdx.length < 4) { | |
| state.filters[s] = null; | |
| state.prevC[s] = null; | |
| state.prevLabels[s] = labels[s]; | |
| continue; | |
| } | |
| let c = [0, 0, 0]; | |
| finIdx.forEach(i => { c[0] += out[s][i][0]; c[1] += out[s][i][1]; c[2] += out[s][i][2]; }); | |
| c = c.map(v => v / finIdx.length); | |
| if (labels[s] !== state.prevLabels[s]) { | |
| // 重建帧且几何近(<0.1m)软衔接,否则硬重置(防跨手污染) | |
| const pc = state.prevC[s]; | |
| if (state.filters[s] && pc && pc.every(Number.isFinite) | |
| && Math.hypot(pc[0] - c[0], pc[1] - c[1], pc[2] - c[2]) < 0.10) { | |
| c = c.map((v, k) => 0.5 * pc[k] + 0.5 * v); | |
| } | |
| state.filters[s] = _h3dMakeOneEuro(3.0, 0.3, 0.3); | |
| } | |
| if (!state.filters[s]) state.filters[s] = _h3dMakeOneEuro(3.0, 0.3, 0.3); | |
| const ch = state.filters[s](c, ts); | |
| const delta = [ch[0] - c[0], ch[1] - c[1], ch[2] - c[2]]; | |
| finIdx.forEach(i => { | |
| out[s][i][0] += delta[0]; | |
| out[s][i][1] += delta[1]; | |
| out[s][i][2] += delta[2]; | |
| }); | |
| state.prevC[s] = ch; | |
| state.prevLabels[s] = labels[s]; | |
| } | |
| return out; | |
| } | |
| function _hand3dCam(valid, w, h, rotY, rotX, zoom, | |
| baseDistOverride = null, targetOffset = null) { | |
| // demo RotatingSkeletonRenderer.render 同款相机布置:质心为目标、 | |
| // fov 45°、静态正面视角 θ=π + 默认仰角 0°。世界坐标模式传入统一的 | |
| // baseDist,播放期不随手部跨度自适应;默认视角静止,滚轮可调 zoom。 | |
| const n = valid.length; | |
| const baseCentroid = valid.reduce( | |
| (a, p) => [a[0] + p[0], a[1] + p[1], a[2] + p[2]], [0, 0, 0]) | |
| .map(v => v / n); | |
| const centroid = baseCentroid.map((value, index) => value | |
| + (targetOffset && Number.isFinite(targetOffset[index]) | |
| ? targetOffset[index] : 0)); | |
| const fov = 45 * Math.PI / 180; | |
| if (_h3dBaseDist === null && !Number.isFinite(baseDistOverride)) { | |
| const span = Math.max( | |
| Math.max(...valid.map(p => p[0])) - Math.min(...valid.map(p => p[0])), | |
| Math.max(...valid.map(p => p[1])) - Math.min(...valid.map(p => p[1])), | |
| Math.max(...valid.map(p => p[2])) - Math.min(...valid.map(p => p[2]))); | |
| const half = span > 1e-6 ? span / 2 : 0.3; | |
| // 5.0 倍系数:手约占画面高度 1/5(默认视角按需求再拉远); | |
| // 距离下限 0.5m:手离相机远(跨度小)时默认视角也不会怼脸, | |
| // 需要更近/更远仍可用滚轮缩放 | |
| _h3dBaseDist = Math.min(2.0, Math.max(0.5, | |
| 6.0 * half / Math.tan(fov / 2))); | |
| } | |
| const baseDist = Number.isFinite(baseDistOverride) | |
| ? baseDistOverride : _h3dBaseDist; | |
| // Keep the default at 2m, but allow the user to zoom farther out when | |
| // inspecting a handless/world-only frame. | |
| const dist = Math.min(HAND3D_MAX_CAMERA_DISTANCE, | |
| Math.max(0.2, baseDist / (zoom || 1))); | |
| const theta = Math.PI + (rotY || 0); // 静态正面 + 拖拽偏转 | |
| // 仰角钳制 ±80°(标准轨道相机 pitch clamp):越过 ±90° 时 | |
| // right = upW×fwd 与 up 基同时反向,画面绕屏幕中心 180° 翻转 | |
| // —— 即"拖拽到达中心点后手部位置突变"的来源 | |
| const elev = Math.max(-80 * Math.PI / 180, Math.min(80 * Math.PI / 180, | |
| HAND3D_DEFAULT_ELEVATION * Math.PI / 180 + (rotX || 0))); | |
| const ce = Math.cos(elev), se = Math.sin(elev); | |
| const eye = [ | |
| centroid[0] + dist * Math.sin(theta) * ce, | |
| // 世界 Y 已统一为向上，因此正仰角时相机应位于目标上方。 | |
| centroid[1] + dist * se, | |
| centroid[2] + dist * Math.cos(theta) * ce, | |
| ]; | |
| // look-at 基:处理链路输出的世界坐标是 X 右、Y 上、Z 向前。 | |
| // Canvas 的屏幕 Y 由投影公式负责翻转，因此这里不能再把世界上方向 | |
| // 写成 -Y，否则 X/Y 轴都会被镜像，右侧空间视图会和原图相反。 | |
| let fwd = [centroid[0] - eye[0], centroid[1] - eye[1], centroid[2] - eye[2]]; | |
| const fn = Math.hypot(fwd[0], fwd[1], fwd[2]) + 1e-9; | |
| fwd = fwd.map(v => v / fn); | |
| const upW = [0, 1, 0]; | |
| let right = [ | |
| upW[1] * fwd[2] - upW[2] * fwd[1], | |
| upW[2] * fwd[0] - upW[0] * fwd[2], | |
| upW[0] * fwd[1] - upW[1] * fwd[0], | |
| ]; | |
| const rn = Math.hypot(right[0], right[1], right[2]) + 1e-9; | |
| right = right.map(v => v / rn); | |
| const up = [ | |
| fwd[1] * right[2] - fwd[2] * right[1], | |
| fwd[2] * right[0] - fwd[0] * right[2], | |
| fwd[0] * right[1] - fwd[1] * right[0], | |
| ]; | |
| return { centroid, eye, fwd, right, up, f: (h / 2) / Math.tan(fov / 2), w, h }; | |
| } | |
| function _projectHand3D(p, cam) { | |
| // 透视投影(demo _project 同款):u = cx + f·(d·right)/z,v = cy − f·(d·up)/z | |
| if (!p || !Number.isFinite(p[0]) || !Number.isFinite(p[1]) || !Number.isFinite(p[2])) { | |
| return [NaN, NaN]; | |
| } | |
| const d = [p[0] - cam.eye[0], p[1] - cam.eye[1], p[2] - cam.eye[2]]; | |
| const z = d[0] * cam.fwd[0] + d[1] * cam.fwd[1] + d[2] * cam.fwd[2]; | |
| if (z <= 1e-4) return [NaN, NaN]; | |
| const u = cam.w / 2 + cam.f * (d[0] * cam.right[0] + d[1] * cam.right[1] + d[2] * cam.right[2]) / z; | |
| const v = cam.h / 2 - cam.f * (d[0] * cam.up[0] + d[1] * cam.up[1] + d[2] * cam.up[2]) / z; | |
| return [u, v]; | |
| } | |
| // 透视视图中的骨骼长度已经包含了距离缩放。使用它计算画笔尺寸， | |
| // 避免手移动到远处后，固定像素半径的关节点互相盖住而看起来像圆球。 | |
| function _hand3dProjectedLength(a, b) { | |
| if (!a || !b || !Number.isFinite(a[0]) || !Number.isFinite(a[1]) | |
| || !Number.isFinite(b[0]) || !Number.isFinite(b[1])) return 0; | |
| return Math.hypot(a[0] - b[0], a[1] - b[1]); | |
| } | |
| function _hand3dStrokeWidth(a, b, maxWidth) { | |
| const length = _hand3dProjectedLength(a, b); | |
| // 远处骨骼最小保留约 0.5px，近处不超过原来的视觉宽度。 | |
| return Math.max(0.5, Math.min(maxWidth, length * 0.14)); | |
| } | |
| function _hand3dStyleScale(proj) { | |
| const valid = (proj || []).filter(p => p && Number.isFinite(p[0]) | |
| && Number.isFinite(p[1])); | |
| if (valid.length < 2) return 1; | |
| const xs = valid.map(p => p[0]), ys = valid.map(p => p[1]); | |
| const span = Math.max(Math.max(...xs) - Math.min(...xs), | |
| Math.max(...ys) - Math.min(...ys)); | |
| // Same reference size as the left SVG/hand_render style. | |
| return Math.max(0.45, Math.min(1, span / 160)); | |
| } | |
| function _normalizeRgbEstimatedDisplayScale(slots, entry) { | |
| const handSpans = []; | |
| const all = []; | |
| for (const slot of slots) { | |
| const valid = slot.valid || []; | |
| if (valid.length < 4) continue; | |
| const xs = valid.map(p => p[0]), ys = valid.map(p => p[1]); | |
| handSpans.push(Math.max(Math.max(...xs) - Math.min(...xs), | |
| Math.max(...ys) - Math.min(...ys))); | |
| all.push(...valid); | |
| } | |
| if (!handSpans.length || !all.length) return 1; | |
| const measured = handSpans.reduce((sum, value) => sum + value, 0) | |
| / handSpans.length; | |
| if (!(measured > 1e-6)) return 1; | |
| const targetScale = Math.max(RGB_ESTIMATED_DISPLAY_SCALE_MIN, | |
| Math.min(RGB_ESTIMATED_DISPLAY_SCALE_MAX, | |
| RGB_ESTIMATED_DISPLAY_HAND_SPAN / measured)); | |
| // Do not resize the whole RGB skeleton independently on every frame. | |
| // Small 2D/PnP scale changes otherwise become visible as breathing and | |
| // jitter after the wrist root is anchored in the world tile. | |
| let scale = targetScale; | |
| if (entry) { | |
| const previous = Number(entry._rgbDisplayScale); | |
| scale = Number.isFinite(previous) | |
| ? previous * 0.92 + targetScale * 0.08 | |
| : targetScale; | |
| entry._rgbDisplayScale = scale; | |
| } | |
| if (Math.abs(scale - 1) < 1e-3) return 1; | |
| const center = all.reduce((sum, p) => [ | |
| sum[0] + p[0], sum[1] + p[1], sum[2] + p[2], | |
| ], [0, 0, 0]).map(value => value / all.length); | |
| for (const slot of slots) { | |
| slot.pts = slot.pts.map(p => p ? [ | |
| center[0] + (p[0] - center[0]) * scale, | |
| center[1] + (p[1] - center[1]) * scale, | |
| center[2] + (p[2] - center[2]) * scale, | |
| ] : null); | |
| slot.fin = slot.pts.map(Boolean); | |
| slot.valid = slot.pts.filter(Boolean); | |
| } | |
| return scale; | |
| } | |
| function _normalizeDepthWorldDisplayScale(slots, entry, w, h) { | |
| // Calculate once from the first usable frame. Recomputing this per frame | |
| // would make the skeleton visibly breathe while playing. | |
| let scale = Number(entry?._worldDisplayScale); | |
| if (!Number.isFinite(scale)) { | |
| const spans = []; | |
| for (const slot of slots) { | |
| const valid = slot.valid || []; | |
| if (valid.length < 4) continue; | |
| // Ignore isolated depth outliers when choosing the fixed preview | |
| // scale. A single bad landmark in #3's first frame was metres | |
| // away from the hand and permanently made later frames tiny. | |
| const robustSpan = values => { | |
| const sorted = values.filter(Number.isFinite).sort((a, b) => a - b); | |
| if (!sorted.length) return 0; | |
| const quantile = fraction => sorted[Math.floor( | |
| (sorted.length - 1) * fraction)]; | |
| return quantile(0.95) - quantile(0.05); | |
| }; | |
| spans.push(Math.max( | |
| robustSpan(valid.map(p => p[0])), | |
| robustSpan(valid.map(p => p[1])), | |
| robustSpan(valid.map(p => p[2])), | |
| )); | |
| } | |
| const measured = spans.length | |
| ? spans.reduce((sum, value) => sum + value, 0) / spans.length : 0; | |
| const fov = 45 * Math.PI / 180; | |
| const focal = (Math.max(1, h) / 2) / Math.tan(fov / 2); | |
| const targetPx = Math.max(HAND3D_DISPLAY_SPAN_MIN_PX, | |
| Math.min(HAND3D_DISPLAY_SPAN_MAX_PX, | |
| Math.min(w, h) * HAND3D_DISPLAY_SPAN_RATIO)); | |
| // At the fixed world camera distance, convert the target screen span | |
| // into a metric display span. This is only a preview transform. | |
| const targetWorldSpan = targetPx * HAND3D_WORLD_DEFAULT_DISTANCE / focal; | |
| scale = measured > 1e-6 | |
| ? Math.max(HAND3D_DISPLAY_SCALE_MIN, | |
| Math.min(HAND3D_DISPLAY_SCALE_MAX, targetWorldSpan / measured)) : 1; | |
| if (entry) entry._worldDisplayScale = scale; | |
| } | |
| if (Math.abs(scale - 1) < 1e-3) return scale; | |
| slots.forEach(slot => { | |
| const wrist = slot.pts[0] || slot.valid[0]; | |
| if (!wrist) return; | |
| slot.pts = slot.pts.map(point => point ? [ | |
| wrist[0] + (point[0] - wrist[0]) * scale, | |
| wrist[1] + (point[1] - wrist[1]) * scale, | |
| wrist[2] + (point[2] - wrist[2]) * scale, | |
| ] : null); | |
| slot.fin = slot.pts.map(Boolean); | |
| slot.valid = slot.pts.filter(Boolean); | |
| }); | |
| return scale; | |
| } | |
| // A few D435 frames contain landmark coordinates that jump metres away from | |
| // the wrist. Keep those points out of the 3D preview only. The parquet/API | |
| // values remain untouched, and a real hand landmark cannot be this far from | |
| // its wrist in the camera coordinate system. | |
| const HAND3D_WORLD_DISPLAY_MAX_HAND_RADIUS = 0.35; | |
| function _filterDepthWorldDisplayOutliers(slots) { | |
| slots.forEach(slot => { | |
| const points = slot.pts || []; | |
| const wrist = points[0] || points.find(Boolean); | |
| if (!wrist) return; | |
| let validCount = 0; | |
| let compactCount = 0; | |
| const maxRadius = HAND3D_WORLD_DISPLAY_MAX_HAND_RADIUS; | |
| const keep = points.map(point => { | |
| if (!point) return false; | |
| validCount += 1; | |
| const dx = point[0] - wrist[0]; | |
| const dy = point[1] - wrist[1]; | |
| const dz = point[2] - wrist[2]; | |
| const isCompact = Math.hypot(dx, dy, dz) <= maxRadius; | |
| if (isCompact) compactCount += 1; | |
| return isCompact; | |
| }); | |
| // Do not blank a sparse hand just because it has one questionable | |
| // point. Only apply the filter when a usable compact skeleton remains. | |
| if (compactCount < 4 || compactCount === validCount) return; | |
| slot.pts = points.map((point, index) => point && keep[index] ? point : null); | |
| slot.fin = slot.pts.map(Boolean); | |
| slot.valid = slot.pts.filter(Boolean); | |
| }); | |
| } | |
| function _hand3dJointRadius(proj, index, tip = false, styleScale = 1) { | |
| if (!proj[index] || !Number.isFinite(proj[index][0]) | |
| || !Number.isFinite(proj[index][1])) return 0; | |
| const lengths = []; | |
| for (const [a, b] of HAND3D_CONNECTIONS) { | |
| if (a !== index && b !== index) continue; | |
| const length = _hand3dProjectedLength(proj[a], proj[b]); | |
| if (length > 0) lengths.push(length); | |
| } | |
| const preferred = (tip ? 7 : 5) * styleScale; | |
| if (!lengths.length) return Math.max(0.65, preferred); | |
| const mean = lengths.reduce((sum, value) => sum + value, 0) / lengths.length; | |
| // Prefer the canonical overlay size, but cap it by the projected local | |
| // bone size so distant joints never cover one another. | |
| return Math.max(0.65, Math.min(preferred, mean * (tip ? 0.18 : 0.16))); | |
| } | |
| function _drawHand3DWorldReference(ctx, w, h, state) { | |
| if (!state || !state.cam || !state.anchor) return; | |
| const { cam, anchor } = state; | |
| const px = p => _projectHand3D(p, cam); | |
| const line = (a, b, color, width) => { | |
| if (!Number.isFinite(a[0]) || !Number.isFinite(a[1]) | |
| || !Number.isFinite(b[0]) || !Number.isFinite(b[1])) return; | |
| ctx.strokeStyle = color; | |
| ctx.lineWidth = _hand3dStrokeWidth(a, b, width); | |
| ctx.beginPath(); | |
| ctx.moveTo(a[0], a[1]); | |
| ctx.lineTo(b[0], b[1]); | |
| ctx.stroke(); | |
| }; | |
| const yGrid = Number.isFinite(state.gridY) ? state.gridY : -0.25; | |
| // 网格是世界坐标的一部分，平移视角时不能跟着相机目标重新居中。 | |
| const zc = anchor[2], xc = anchor[0]; | |
| for (let x = xc - 0.4; x <= xc + 0.401; x += 0.05) { | |
| line(px([x, yGrid, zc - 0.45]), px([x, yGrid, zc + 0.45]), | |
| 'rgba(58,58,58,0.9)', 1); | |
| } | |
| for (let z = zc - 0.45; z <= zc + 0.451; z += 0.05) { | |
| line(px([xc - 0.4, yGrid, z]), px([xc + 0.4, yGrid, z]), | |
| 'rgba(58,58,58,0.9)', 1); | |
| } | |
| const axes = [[[0.1, 0, 0], 'rgb(60,60,255)', 'X'], | |
| [[0, 0.1, 0], 'rgb(60,200,60)', 'Y'], | |
| [[0, 0, 0.1], 'rgb(255,120,60)', 'Z']]; | |
| for (const [v, color, name] of axes) { | |
| const o = px(anchor); | |
| const e = px([anchor[0] + v[0], anchor[1] + v[1], anchor[2] + v[2]]); | |
| line(o, e, color, 1); | |
| if (Number.isFinite(e[0])) { | |
| ctx.fillStyle = color; | |
| ctx.font = '9px sans-serif'; | |
| ctx.fillText(name, e[0] + 3, e[1] - 3); | |
| } | |
| } | |
| } | |
| function _worldReferenceCam(state, w, h, entry) { | |
| if (!state) return null; | |
| if (state.center && Number.isFinite(state.baseDist)) { | |
| return _hand3dCam( | |
| [state.center], w, h, | |
| entry.rotY || 0, entry.rotX || 0, entry.zoom || 1, | |
| state.baseDist, entry.pan || [0, 0, 0]); | |
| } | |
| return state.cam || null; | |
| } | |
| function _defaultHand3DWorldReference(w, h, entry) { | |
| const anchor = HAND3D_WORLD_DEFAULT_CENTER.slice(); | |
| const gridY = HAND3D_WORLD_DEFAULT_GRID_Y; | |
| return { | |
| center: anchor.slice(), | |
| baseDist: HAND3D_WORLD_DEFAULT_DISTANCE, | |
| // 视距固定在安全默认值。不能用首个有效帧拟合:首帧可能只有 | |
| // 一只手或只有部分关键点,会让网格和后续双手画面突然缩放。 | |
| fitted: true, | |
| anchor, | |
| gridY, | |
| rootAnchors: { | |
| left: [anchor[0] - HAND3D_ROOT_ANCHOR_X, | |
| gridY + HAND3D_ROOT_ANCHOR_LIFT, anchor[2]], | |
| right: [anchor[0] + HAND3D_ROOT_ANCHOR_X, | |
| gridY + HAND3D_ROOT_ANCHOR_LIFT, anchor[2]], | |
| }, | |
| }; | |
| } | |
| function _h3dRootAnchorKey(label, slotIndex) { | |
| const value = String(label || '').toLowerCase(); | |
| if (value === 'left') return 'left'; | |
| if (value === 'right') return 'right'; | |
| // Older artifacts may not carry handedness. Keep a deterministic | |
| // fallback without changing the anchor when the other hand appears. | |
| return slotIndex === 0 ? 'left' : 'right'; | |
| } | |
| function _h3dStableAnchorSide(entry, label, slotIndex) { | |
| if (!entry._anchorSides) entry._anchorSides = [null, null]; | |
| const value = String(label || '').toLowerCase(); | |
| const reported = value === 'left' || value === 'right' ? value : null; | |
| if (!entry._anchorSides[slotIndex] && reported) { | |
| const other = entry._anchorSides[slotIndex === 0 ? 1 : 0]; | |
| // If two unstable labels claim the same side, preserve two distinct | |
| // display anchors rather than letting the hands overlap. | |
| entry._anchorSides[slotIndex] = (other && other === reported) | |
| ? (reported === 'left' ? 'right' : 'left') : reported; | |
| } | |
| return entry._anchorSides[slotIndex] | |
| || (slotIndex === 0 ? 'left' : 'right'); | |
| } | |
| function _h3dRootAnchorApply(ptsBySlot, labels, state) { | |
| const rootAnchors = state && state.rootAnchors; | |
| if (!rootAnchors) return ptsBySlot; | |
| if (!state.lastRawRoots) state.lastRawRoots = [null, null]; | |
| return ptsBySlot.map((points, slotIndex) => { | |
| const root = points && points[0] | |
| ? points[0] | |
| : state.lastRawRoots[slotIndex]; | |
| if (!root || !root.every(Number.isFinite)) { | |
| return points.map(point => point ? point.slice() : null); | |
| } | |
| if (points && points[0]) state.lastRawRoots[slotIndex] = points[0].slice(); | |
| const anchor = rootAnchors[ | |
| _h3dRootAnchorKey(labels[slotIndex], slotIndex) | |
| ]; | |
| return points.map(point => point ? [ | |
| point[0] - root[0] + anchor[0], | |
| point[1] - root[1] + anchor[1], | |
| point[2] - root[2] + anchor[2], | |
| ] : null); | |
| }); | |
| } | |
| function _h3dApplyFixedDisplayAnchors(slots, spaceMode, entry) { | |
| // The preview is intentionally a fixed layout. RGB-only coordinates are | |
| // camera-relative estimates, and MediaPipe-relative coordinates are | |
| // hand-local, so neither source can reliably define the two-hand layout. | |
| // Apply the same explicit wrist targets on every render path. | |
| const centerZ = spaceMode ? HAND3D_WORLD_DEFAULT_CENTER[2] : 0; | |
| const centerY = spaceMode | |
| ? HAND3D_WORLD_DEFAULT_GRID_Y + HAND3D_ROOT_ANCHOR_LIFT : 0; | |
| // Old RGB artifacts may contain duplicate handedness labels (commonly | |
| // both hands reported as Right). The original 2D overlay is slot-based, | |
| // so it still looks correct, but label-only anchoring mirrors the two | |
| // hands in the 3D tile. For that legacy case, initialize the semantic | |
| // sides from the raw camera-space wrist X order and then keep them fixed. | |
| const reported = slots.map(slot => String(slot.label || '').toLowerCase()); | |
| const duplicateLabels = reported.length === 2 | |
| && reported[0] && reported[0] === reported[1]; | |
| if (duplicateLabels && !entry._duplicateAnchorSides) { | |
| const ordered = slots | |
| .map((slot, index) => ({ index, wrist: slot.pts[0] || slot.valid[0] })) | |
| .filter(item => item.wrist && Number.isFinite(item.wrist[0])) | |
| .sort((a, b) => a.wrist[0] - b.wrist[0]); | |
| if (ordered.length === 2) { | |
| entry._duplicateAnchorSides = []; | |
| entry._duplicateAnchorSides[ordered[0].index] = 'left'; | |
| entry._duplicateAnchorSides[ordered[1].index] = 'right'; | |
| } | |
| } | |
| slots.forEach((slot, slotIndex) => { | |
| const wrist = slot.pts[0] || slot.valid[0]; | |
| if (!wrist) return; | |
| // Lock the semantic side once per tile. A transient per-frame | |
| // Left/Right classification must never move a whole hand to the | |
| // other side of the canvas, while the initial semantic orientation | |
| // remains compatible with both black-glove and bare-hand outputs. | |
| const side = duplicateLabels && entry._duplicateAnchorSides | |
| ? entry._duplicateAnchorSides[slotIndex] | |
| : _h3dStableAnchorSide(entry, slot.label, slotIndex); | |
| const target = [ | |
| side === 'left' ? -HAND3D_ROOT_ANCHOR_X : HAND3D_ROOT_ANCHOR_X, | |
| centerY, | |
| centerZ, | |
| ]; | |
| slot.pts = slot.pts.map(point => point ? [ | |
| point[0] - wrist[0] + target[0], | |
| point[1] - wrist[1] + target[1], | |
| point[2] - wrist[2] + target[2], | |
| ] : null); | |
| slot.fin = slot.pts.map(Boolean); | |
| slot.valid = slot.pts.filter(Boolean); | |
| }); | |
| } | |
| function _ensureHand3DWorldReference(w, h, entry) { | |
| const sourceData = hand3dDataBySource[entry.source.source_key] || hand3dData; | |
| if (!sourceData || !sourceData.spaceMode) return null; | |
| if (!entry._worldReference) entry._worldReference = _defaultHand3DWorldReference(w, h, entry); | |
| // Use the same world-space anchor for every episode. Hand data is shifted | |
| // relative to this stable reference, so its arrival cannot move the view. | |
| if (_hand3dViewAnchor === null) _hand3dViewAnchor = entry._worldReference.anchor.slice(); | |
| if (_h3dGridY === null) _h3dGridY = HAND3D_WORLD_DEFAULT_GRID_Y; | |
| entry._worldReference.cam = _worldReferenceCam(entry._worldReference, w, h, entry); | |
| return entry._worldReference; | |
| } | |
| function renderHand3DTile(entry, frameOverride) { | |
| if (!entry || !entry.canvas) return; | |
| const canvas = entry.canvas; | |
| const rect = canvas.getBoundingClientRect(); | |
| const dpr = window.devicePixelRatio || 1; | |
| const w = Math.max(1, rect.width), h = Math.max(1, rect.height); | |
| if (canvas.width !== Math.round(w * dpr) || canvas.height !== Math.round(h * dpr)) { | |
| canvas.width = Math.round(w * dpr); | |
| canvas.height = Math.round(h * dpr); | |
| } | |
| const ctx = canvas.getContext('2d'); | |
| ctx.setTransform(dpr, 0, 0, dpr, 0, 0); | |
| const frame = typeof frameOverride === 'number' | |
| ? frameOverride | |
| : (typeof currentFrameTarget === 'number' ? currentFrameTarget : 0); | |
| _hand3dTileEntry = entry; | |
| const sourceKey = entry.source.source_key || 'default'; | |
| const sourceData = hand3dDataBySource[sourceKey] || hand3dData; | |
| const worldMode = !!(sourceData && sourceData.worldMode); | |
| const spaceMode = !!(sourceData && sourceData.spaceMode); | |
| const cameraYDown = !!(sourceData && sourceData.cameraYDown); | |
| let fd = _hand3dCached(frame, sourceKey); | |
| const frameCache = _hand3dFrameCacheFor(sourceKey); | |
| if (!spaceMode && frameCache.frame === frame && frameCache.data) { | |
| fd = { | |
| h0: frameCache.data.h0 || null, | |
| h1: frameCache.data.h1 || null, | |
| }; | |
| } | |
| if (sourceData && (sourceData.hasPreview || spaceMode)) { | |
| if (spaceMode) fetchHand3DWindow(frame, sourceKey); | |
| else fetchHand3DFrame(frame, sourceKey); | |
| } | |
| // 缓存窗口切换期间保留上一张已提交的 3D 画面，不清空画布。 | |
| // 数据返回后本函数再次调用，并在下面一次性提交新帧，避免闪烁。 | |
| // 若某帧确实没有手部数据，fd 是非 null 的空对象，仍会正常清空并 | |
| // 绘制参考系；因此这里只处理“尚未拿到当前帧”的网络等待状态。 | |
| if (fd === null && entry._hasCommittedFrame) return; | |
| ctx.clearRect(0, 0, w, h); | |
| ctx.fillStyle = 'rgba(3,7,18,0.95)'; | |
| ctx.fillRect(0, 0, w, h); | |
| if (!fd || (!fd.h0 && !fd.h1)) { | |
| if (spaceMode) { | |
| _ensureHand3DWorldReference(w, h, entry); | |
| _drawHand3DWorldReference(ctx, w, h, entry._worldReference); | |
| return; | |
| } | |
| return; | |
| } | |
| entry._hasCommittedFrame = true; | |
| // 收集有效点(世界坐标模式:d435 相机系米制) | |
| const slots = [ | |
| { key: 'h0', label: fd.h0 && fd.h0.label ? String(fd.h0.label) : '' }, | |
| { key: 'h1', label: fd.h1 && fd.h1.label ? String(fd.h1.label) : '' }, | |
| ]; | |
| const all = []; | |
| for (const s of slots) { | |
| const hand = fd[s.key]; | |
| s.pts = hand && hand.lm ? hand.lm.map(p => ( | |
| p && Number.isFinite(p[0]) && Number.isFinite(p[1]) && Number.isFinite(p[2]) | |
| ? [p[0], cameraYDown ? -p[1] : p[1], p[2]] : null)) : []; | |
| s.fin = s.pts.map(Boolean); | |
| s.valid = s.pts.filter(Boolean); | |
| all.push(...s.valid); | |
| } | |
| if (all.length < 4) { | |
| if (spaceMode) { | |
| _ensureHand3DWorldReference(w, h, entry); | |
| } | |
| if (spaceMode && entry._worldReference) { | |
| entry._worldReference.cam = _worldReferenceCam(entry._worldReference, w, h, entry); | |
| _drawHand3DWorldReference(ctx, w, h, entry._worldReference); | |
| return; | |
| } | |
| return; | |
| } | |
| // Keep the stored RGB estimate untouched. This is a presentation-only | |
| // normalization so RGB and depth tiles occupy a comparable visual scale. | |
| if (sourceData && sourceData.rgbEstimatedMode) { | |
| _normalizeRgbEstimatedDisplayScale(slots, entry); | |
| all.length = 0; | |
| slots.forEach(s => all.push(...s.valid)); | |
| } | |
| if (sourceData && sourceData.worldMode) { | |
| _filterDepthWorldDisplayOutliers(slots); | |
| all.length = 0; | |
| slots.forEach(s => all.push(...s.valid)); | |
| _normalizeDepthWorldDisplayScale(slots, entry, w, h); | |
| all.length = 0; | |
| slots.forEach(s => all.push(...s.valid)); | |
| } | |
| if (spaceMode) _ensureHand3DWorldReference(w, h, entry); | |
| if (spaceMode && entry._worldReference | |
| && entry._worldReference.fitted !== true) { | |
| // 兼容旧的 tile 状态,但不再依据当前帧的关键点尺度拟合。 | |
| // 尤其不能让首个单手帧把相机拉近,否则第二只手出现时视角会跳。 | |
| entry._worldReference.baseDist = HAND3D_WORLD_DEFAULT_DISTANCE; | |
| entry._worldReference.fitted = true; | |
| } | |
| // All 3D preview modes use the same fixed wrist layout. This is a | |
| // display-only transform; the original camera-relative coordinates in | |
| // parquet/API remain unchanged. | |
| _h3dApplyFixedDisplayAnchors(slots, spaceMode, entry); | |
| all.length = 0; | |
| slots.forEach(s => all.push(...s.valid)); | |
| // 视图锚点:世界坐标模式使用真实的固定相机坐标,不按当前可见手的 | |
| // 质心平移。否则双手变单手时 c0 会改变,剩余手会被人为挪动。 | |
| // 相对 3D 仍按腕点独立归一化。 | |
| if (!spaceMode && _hand3dViewAnchor === null) { | |
| const c0 = all.reduce((a, p) => [a[0] + p[0], a[1] + p[1], a[2] + p[2]], [0, 0, 0]) | |
| .map(v => v / all.length); | |
| _hand3dViewAnchor = c0.slice(); | |
| } | |
| // 世界坐标保留处理链路输出的绝对相机坐标。anchor 只用于 | |
| // 相机/网格参考,不再把当前帧整体搬到 anchor。相对 3D 已在 | |
| // 上面的腕点归一化阶段完成布局。 | |
| const shifted = all.map(p => p.slice()); | |
| let cam; | |
| if (spaceMode) { | |
| // The current hand points move and render, but never redefine the | |
| // shared initial world-coordinate viewpoint. | |
| entry._worldReference.cam = _worldReferenceCam(entry._worldReference, w, h, entry); | |
| cam = entry._worldReference.cam; | |
| } else { | |
| cam = _hand3dCam(shifted, w, h, entry.rotY || 0, entry.rotX || 0, | |
| entry.zoom || 1, null, entry.pan || [0, 0, 0]); | |
| } | |
| entry._lastCam = cam; | |
| const px = p => _projectHand3D(p, cam); | |
| const line = (a, b, color, width, styleScale = 1) => { | |
| if (!Number.isFinite(a[0]) || !Number.isFinite(a[1]) | |
| || !Number.isFinite(b[0]) || !Number.isFinite(b[1])) return; | |
| ctx.strokeStyle = color; | |
| ctx.lineWidth = _hand3dStrokeWidth(a, b, width * styleScale); | |
| ctx.beginPath(); | |
| ctx.moveTo(a[0], a[1]); | |
| ctx.lineTo(b[0], b[1]); | |
| ctx.stroke(); | |
| }; | |
| if (spaceMode) { | |
| // 世界坐标参考系独立于当前帧关键点,关键点缺失时仍保留。 | |
| _drawHand3DWorldReference(ctx, w, h, entry._worldReference); | |
| } | |
| // painter 算法:远手先画(demo order.sort(key=-depth)) | |
| const order = slots | |
| .filter(s => s.valid.length) | |
| .map(s => ({ | |
| s, | |
| depth: s.valid.reduce((a, p) => a | |
| + ((p[0] - cam.eye[0]) * cam.fwd[0] | |
| + (p[1] - cam.eye[1]) * cam.fwd[1] | |
| + (p[2] - cam.eye[2]) * cam.fwd[2]), 0) / s.valid.length, | |
| })) | |
| .sort((a, b) => b.depth - a.depth); | |
| for (const { s } of order) { | |
| const proj = s.pts.map(p => px(p)); | |
| const styleScale = _hand3dStyleScale(proj); | |
| // 掌心灰线 | |
| for (const [a, b] of HAND3D_PALM) { | |
| if (s.fin[a] && s.fin[b]) { | |
| line(proj[a], proj[b], 'rgb(200,200,200)', 2, styleScale); | |
| } | |
| } | |
| // 五指分色链(拇指 1→2→3→4;其余 0→MCP→…→指尖) | |
| for (const f of HAND3D_FINGERS) { | |
| const chain = f.name === 'Thumb' ? f.ids : [0, ...f.ids]; | |
| for (let i = 0; i < chain.length - 1; i++) { | |
| if (s.fin[chain[i]] && s.fin[chain[i + 1]]) { | |
| line(proj[chain[i]], proj[chain[i + 1]], | |
| `rgb(${f.color[0]},${f.color[1]},${f.color[2]})`, 3, | |
| styleScale); | |
| } | |
| } | |
| for (const idx of f.ids) { | |
| if (!s.fin[idx]) continue; | |
| const r = _hand3dJointRadius( | |
| proj, idx, idx === f.ids[f.ids.length - 1], styleScale); | |
| if (r <= 0) continue; | |
| ctx.fillStyle = `rgb(${f.color[0]},${f.color[1]},${f.color[2]})`; | |
| ctx.beginPath(); | |
| ctx.arc(proj[idx][0], proj[idx][1], r, 0, Math.PI * 2); | |
| ctx.fill(); | |
| ctx.strokeStyle = 'rgb(30,30,30)'; | |
| ctx.lineWidth = Math.max(0.5, Math.min(1, r * 0.22)); | |
| ctx.stroke(); | |
| } | |
| } | |
| // 腕部白点 | |
| if (s.fin[0]) { | |
| const wristRadius = Math.max(0.65, Math.min( | |
| 9 * styleScale, | |
| _hand3dJointRadius(proj, 0, false, styleScale) * 1.45)); | |
| ctx.fillStyle = '#fff'; | |
| ctx.beginPath(); | |
| ctx.arc(proj[0][0], proj[0][1], Math.max(1.2, Math.min(7, wristRadius)), 0, Math.PI * 2); | |
| ctx.fill(); | |
| ctx.strokeStyle = 'rgb(40,40,40)'; | |
| ctx.lineWidth = Math.max(0.5, Math.min(1.5, wristRadius * 0.22)); | |
| ctx.stroke(); | |
| } | |
| // 腕部深度标注只在相机米制世界坐标中显示。 | |
| if (worldMode && s.fin[0] && Number.isFinite(proj[0][0])) { | |
| ctx.fillStyle = s.label === 'Right' ? 'rgb(0,255,0)' : 'rgb(255,200,0)'; | |
| ctx.font = '11px sans-serif'; | |
| ctx.fillText(`${s.label} z=${s.pts[0][2].toFixed(2)}m`, | |
| proj[0][0] + 12, proj[0][1] - 12); | |
| } | |
| } | |
| // HUD | |
| ctx.fillStyle = 'rgba(235,235,235,0.9)'; | |
| ctx.font = '11px sans-serif'; | |
| ctx.textAlign = 'left'; | |
| ctx.fillText(`frame ${frame}`, 12, 22); | |
| // The interaction/mode hint was only a debug HUD and is intentionally | |
| // omitted from the spatial preview. | |
| } | |
| function initHand3dDrag(entry) { | |
| if (!entry || !entry.canvas) return; | |
| const canvas = entry.canvas; | |
| canvas.style.cursor = 'grab'; | |
| canvas.addEventListener('mousedown', e => { | |
| if (e.button !== 0 && e.button !== 1) return; | |
| entry.dragging = true; | |
| entry.dragMode = e.button === 1 ? 'pan' : 'rotate'; | |
| entry.lastX = e.clientX; | |
| entry.lastY = e.clientY; | |
| canvas.style.cursor = entry.dragMode === 'pan' ? 'move' : 'grabbing'; | |
| e.preventDefault(); | |
| }); | |
| window.addEventListener('mousemove', e => { | |
| if (!entry.dragging) return; | |
| // 左键是轨道旋转:水平向右看右侧,垂直向上看上方。 | |
| const dx = e.clientX - entry.lastX; | |
| const dy = e.clientY - entry.lastY; | |
| if (entry.dragMode === 'pan') { | |
| // 让空间像被鼠标抓住一样移动:拖右→内容右移,拖下→内容下移。 | |
| // 使用当前相机 right/up 基向量,旋转后平移仍符合屏幕方向。 | |
| const cam = entry._lastCam; | |
| if (cam) { | |
| const distance = Math.max(0.2, Math.hypot( | |
| cam.eye[0] - cam.centroid[0], | |
| cam.eye[1] - cam.centroid[1], | |
| cam.eye[2] - cam.centroid[2])); | |
| const worldPerPixel = distance / Math.max(1, cam.f); | |
| const pan = entry.pan || (entry.pan = [0, 0, 0]); | |
| for (let i = 0; i < 3; i++) { | |
| pan[i] += (-dx * cam.right[i] + dy * cam.up[i]) | |
| * worldPerPixel; | |
| } | |
| } | |
| } else { | |
| // 水平向右拖动查看右侧；向上拖动查看上方。 | |
| entry.rotY += dx * 0.01; | |
| entry.rotX += dy * 0.01; | |
| } | |
| entry.lastX = e.clientX; | |
| entry.lastY = e.clientY; | |
| renderHand3DTile(entry); | |
| }); | |
| window.addEventListener('mouseup', () => { | |
| if (entry.dragging) { | |
| entry.dragging = false; | |
| entry.dragMode = ''; | |
| canvas.style.cursor = 'grab'; | |
| } | |
| }); | |
| canvas.addEventListener('contextmenu', e => e.preventDefault()); | |
| // 滚轮缩放(世界坐标模式:0.3x – 6x) | |
| canvas.addEventListener('wheel', e => { | |
| e.preventDefault(); | |
| entry.zoom = Math.min(6, Math.max(0.3, | |
| (entry.zoom || 1) * Math.exp(-e.deltaY * 0.0015))); | |
| renderHand3DTile(entry); | |
| }, { passive: false }); | |
| } | |
| /* 逐帧步进回调:3D 世界窗口随统一帧号重绘(多回调机制,与骨骼叠加层/ | |
| 标注高亮共用;闭包在调用时遍历,脚本加载时 tile 列表为空无影响) */ | |
| setOnFrameChange((f) => { | |
| for (const entry of currentHand3dTiles) renderHand3DTile(entry, f); | |
| for (const entry of currentSlamTiles) renderSlamTile(entry, f); | |
| }); | |
| /* 图像素材(深度图/手套热力图)的逐帧 rAF 刷新循环(见 _startImageTileRaf) */ | |
| if (document.readyState !== 'loading') _startImageTileRaf(); | |
| else document.addEventListener('DOMContentLoaded', _startImageTileRaf); | |
| /* 漂浮预览控件的显隐:点开批次详情后才显示(☷ 与 Slice Preview | |
| 在标注/审核/审核通过页面可用);返回批次列表时隐藏。 */ | |
| function setPreviewPanelsVisible(open) { | |
| window.__episodeOpen = !!open; | |
| const po = document.getElementById('preview-options'); | |
| if (po) po.classList.toggle('hidden', !open); | |
| const menu = document.getElementById('preview-options-menu'); | |
| if (!open && menu) menu.classList.add('hidden'); | |
| if (open) { | |
| renderPreviewVideoSources(); | |
| if (typeof updatePreviewMenuData === 'function') updatePreviewMenuData(); | |
| } | |
| const pt = document.getElementById('playback-toolbar'); | |
| if (pt) pt.classList.toggle('hidden', !open); | |
| if (!open && typeof resetSlicePreview === 'function') resetSlicePreview(''); | |
| if (typeof refreshSlicePreview === 'function') refreshSlicePreview(); | |
| } | |
| window.ensurePreviewOptionsVisible = function () { | |
| if (window.__episodeOpen !== true) return; | |
| const box = document.getElementById('preview-options'); | |
| if (box) box.classList.remove('hidden'); | |
| }; | |
| /* 诊断入口:播放按钮"点了没反应"时,在浏览器控制台(F12 → Console)输入 | |
| __playerDebug() 回车,把输出发过来 —— 一次定位卡在哪个状态。 */ | |
| window.__playerDebug = function () { | |
| return { | |
| episode: currentEpisodeId, | |
| playerReady: playerReady, | |
| frameDataReady: frameDataReady, | |
| controlsEnabled: _controlsEnabled, | |
| players: Object.keys(players), | |
| activePlayer: Boolean(getActivePlayer()), | |
| workspaceSources: workspaceSources.map(s => groupedSourceKey(s)), | |
| overlayHidden: (document.getElementById('video-loading-overlay') || {}) | |
| .classList.contains('hidden'), | |
| }; | |
| }; | |