import { Component, onMount, onCleanup, createSignal } from 'solid-js'; import type { AudioEngine } from '../lib/audio/types'; import type { MelWorkerClient } from '../lib/audio/MelWorkerClient'; import { normalizeMelForDisplay } from '../lib/audio/mel-display'; import { appStore } from '../stores/appStore'; interface LayeredBufferVisualizerProps { audioEngine?: AudioEngine; melClient?: MelWorkerClient; height?: number; // Total height windowDuration?: number; // default 8.0s } const MEL_BINS = 128; // Standard for this app // dB scaling is in mel-display.ts (shared with bar visualizer) // Pre-computed 256-entry RGB lookup table for mel heatmap (black to red). // Built once at module load; indexed by Math.round(intensity * 255). // Colormap: black -> blue -> purple -> green -> yellow -> orange -> red. const COLORMAP_LUT = (() => { const stops: [number, number, number, number][] = [ [0, 0, 0, 0], // black [0.12, 0, 0, 180], // blue [0.30, 120, 0, 160], // purple [0.48, 0, 180, 80], // green [0.65, 220, 220, 0], // yellow [0.82, 255, 140, 0], // orange [1, 255, 0, 0], // red ]; // 256 entries * 3 channels (R, G, B) packed into a Uint8Array const lut = new Uint8Array(256 * 3); for (let i = 0; i < 256; i++) { const intensity = i / 255; let r = 0, g = 0, b = 0; for (let s = 0; s < stops.length - 1; s++) { const [t0, r0, g0, b0] = stops[s]; const [t1, r1, g1, b1] = stops[s + 1]; if (intensity >= t0 && intensity <= t1) { const t = (intensity - t0) / (t1 - t0); r = Math.round(r0 + t * (r1 - r0)); g = Math.round(g0 + t * (g1 - g0)); b = Math.round(b0 + t * (b1 - b0)); break; } } if (intensity >= stops[stops.length - 1][0]) { const last = stops[stops.length - 1]; r = last[1]; g = last[2]; b = last[3]; } const base = i * 3; lut[base] = r; lut[base + 1] = g; lut[base + 2] = b; } return lut; })(); export const LayeredBufferVisualizer: Component = (props) => { let canvasRef: HTMLCanvasElement | undefined; let ctx: CanvasRenderingContext2D | null = null; let animationFrameId: number; const getWindowDuration = () => props.windowDuration || 8.0; // Offscreen canvas for spectrogram caching (scrolling) let specCanvas: HTMLCanvasElement | undefined; let specCtx: CanvasRenderingContext2D | null = null; // State for last fetch to throttle spectrogram updates let lastSpecFetchTime = 0; const SPEC_FETCH_INTERVAL = 100; // Update spectrogram every 100ms (10fps) const DRAW_INTERVAL_MS = 33; // Throttle full redraw to ~30fps let lastDrawTime = 0; // --- Cached layout dimensions (updated via ResizeObserver, NOT per-frame) --- // Avoids getBoundingClientRect() every animation frame which forces synchronous // layout reflow and was the #1 perf bottleneck (1.5s layout-shift clusters). let cachedPhysicalWidth = 0; let cachedPhysicalHeight = 0; let cachedDpr = window.devicePixelRatio || 1; let resizeObserver: ResizeObserver | null = null; let dprMediaQuery: MediaQueryList | null = null; /** Recompute physical canvas dimensions from cached logical size + DPR. */ const updateCanvasDimensions = (logicalW: number, logicalH: number) => { cachedDpr = window.devicePixelRatio || 1; cachedPhysicalWidth = Math.floor(logicalW * cachedDpr); cachedPhysicalHeight = Math.floor(logicalH * cachedDpr); // Resize canvases immediately so next frame uses correct size if (canvasRef && (canvasRef.width !== cachedPhysicalWidth || canvasRef.height !== cachedPhysicalHeight)) { canvasRef.width = cachedPhysicalWidth; canvasRef.height = cachedPhysicalHeight; } if (specCanvas && (specCanvas.width !== cachedPhysicalWidth || specCanvas.height !== cachedPhysicalHeight)) { specCanvas.width = cachedPhysicalWidth; specCanvas.height = cachedPhysicalHeight; } }; // --- Pre-allocated ImageData for spectrogram rendering --- // Avoids creating a new ImageData object every spectrogram draw (~10fps), // which caused GC pressure from large short-lived allocations. let cachedSpecImgData: ImageData | null = null; let cachedSpecImgWidth = 0; let cachedSpecImgHeight = 0; // --- Pre-allocated waveform read buffer --- // Avoids allocating a new Float32Array(~128000) every animation frame. // Grows only when the required size exceeds current capacity. let waveformReadBuf: Float32Array | null = null; // Store spectrogram data with its time alignment let cachedSpecData: { features: Float32Array; melBins: number; timeSteps: number; startTime: number; endTime: number; } | null = null; onMount(() => { if (canvasRef) { ctx = canvasRef.getContext('2d', { alpha: false }); // Use ResizeObserver to cache dimensions instead of per-frame getBoundingClientRect resizeObserver = new ResizeObserver((entries) => { for (const entry of entries) { // contentRect gives CSS-pixel (logical) dimensions without forcing layout const cr = entry.contentRect; updateCanvasDimensions(cr.width, cr.height); } }); resizeObserver.observe(canvasRef); // Watch for DPR changes (browser zoom, display change) const setupDprWatch = () => { dprMediaQuery = window.matchMedia(`(resolution: ${window.devicePixelRatio}dppx)`); const onDprChange = () => { if (canvasRef) { const rect = canvasRef.getBoundingClientRect(); // one-time on zoom change only updateCanvasDimensions(rect.width, rect.height); } // Re-register for the next change at the new DPR setupDprWatch(); }; dprMediaQuery.addEventListener('change', onDprChange, { once: true }); }; setupDprWatch(); // Initial dimensions (one-time) const rect = canvasRef.getBoundingClientRect(); updateCanvasDimensions(rect.width, rect.height); } // Create offscreen canvas specCanvas = document.createElement('canvas'); specCtx = specCanvas.getContext('2d', { alpha: false }); loop(); }); onCleanup(() => { cancelAnimationFrame(animationFrameId); if (resizeObserver) { resizeObserver.disconnect(); resizeObserver = null; } }); const loop = (now: number = performance.now()) => { if (!ctx || !canvasRef || !props.audioEngine) { animationFrameId = requestAnimationFrame(loop); return; } if (now - lastDrawTime < DRAW_INTERVAL_MS) { animationFrameId = requestAnimationFrame(loop); return; } lastDrawTime = now; // Use cached dimensions (updated by ResizeObserver / DPR watcher) const dpr = cachedDpr; const width = cachedPhysicalWidth; const height = cachedPhysicalHeight; if (width === 0 || height === 0) { animationFrameId = requestAnimationFrame(loop); return; } // Colors const bgColor = '#0f172a'; ctx.fillStyle = bgColor; ctx.fillRect(0, 0, width, height); const ringBuffer = props.audioEngine.getRingBuffer(); const currentTime = ringBuffer.getCurrentTime(); const duration = getWindowDuration(); const startTime = currentTime - duration; const sampleRate = ringBuffer.sampleRate; // Layout: // Top 55%: Spectrogram // Middle 35%: Waveform // Bottom 10%: VAD signal const specHeight = Math.floor(height * 0.55); const waveHeight = Math.floor(height * 0.35); const vadHeight = height - specHeight - waveHeight; const waveY = specHeight; const vadY = specHeight + waveHeight; // 1. Spectrogram (async fetch with stored alignment) if (props.melClient && specCtx && specCanvas) { if (now - lastSpecFetchTime > SPEC_FETCH_INTERVAL) { lastSpecFetchTime = now; const fetchStartSample = Math.round(startTime * sampleRate); const fetchEndSample = Math.round(currentTime * sampleRate); // Request RAW (unnormalized) features for fixed dB scaling. // ASR transcription still uses normalized features (default). props.melClient.getFeatures(fetchStartSample, fetchEndSample, false).then(features => { if (features && specCtx && specCanvas) { // Store with time alignment info cachedSpecData = { features: features.features, melBins: features.melBins, timeSteps: features.T, startTime: startTime, endTime: currentTime }; drawSpectrogramToCanvas(specCtx, features.features, features.melBins, features.T, width, specHeight); } }).catch(() => { }); } // Draw cached spectrogram aligned to current view if (cachedSpecData && cachedSpecData.timeSteps > 0) { // Calculate offset to align cached data with current time window const cachedDuration = cachedSpecData.endTime - cachedSpecData.startTime; const timeOffset = startTime - cachedSpecData.startTime; const offsetX = Math.floor((timeOffset / cachedDuration) * width); // Draw the portion of cached spectrogram that's still visible ctx.drawImage(specCanvas, offsetX, 0, width - offsetX, specHeight, 0, 0, width - offsetX, specHeight); } } // 2. Waveform (sync with current time window, zero-allocation read) try { const startSample = Math.floor(startTime * sampleRate); const endSample = Math.floor(currentTime * sampleRate); const neededLen = endSample - startSample; const baseFrame = ringBuffer.getBaseFrameOffset(); if (startSample >= baseFrame && neededLen > 0) { // Use readInto if available (zero-alloc), fall back to read() if (ringBuffer.readInto) { // Grow the pre-allocated buffer only when capacity is insufficient if (!waveformReadBuf || waveformReadBuf.length < neededLen) { waveformReadBuf = new Float32Array(neededLen); } const written = ringBuffer.readInto(startSample, endSample, waveformReadBuf); // Pass a subarray view (no copy) of the exact length drawWaveform(ctx, waveformReadBuf.subarray(0, written), width, waveHeight, waveY); } else { const audioData = ringBuffer.read(startSample, endSample); drawWaveform(ctx, audioData, width, waveHeight, waveY); } } } catch (e) { // Data likely overwritten or not available } // 3. VAD Signal Layer drawVadLayer(ctx, width, vadHeight, vadY, startTime, duration, dpr); // 4. Overlay (time labels, trigger line) drawOverlay(ctx, width, height, startTime, duration, dpr); animationFrameId = requestAnimationFrame(loop); }; const drawSpectrogramToCanvas = ( ctx: CanvasRenderingContext2D, features: Float32Array, melBins: number, timeSteps: number, width: number, height: number ) => { // features layout: [melBins, T] (mel-major, flattened from [mel, time]) // So features[m * timeSteps + t]. if (timeSteps === 0) return; // Reuse cached ImageData if dimensions match; allocate only on size change if (!cachedSpecImgData || cachedSpecImgWidth !== width || cachedSpecImgHeight !== height) { cachedSpecImgData = ctx.createImageData(width, height); cachedSpecImgWidth = width; cachedSpecImgHeight = height; } const imgData = cachedSpecImgData; const data = imgData.data; // Scaling factors const timeScale = timeSteps / width; const freqScale = melBins / height; for (let x = 0; x < width; x++) { const t = Math.floor(x * timeScale); if (t >= timeSteps) break; for (let y = 0; y < height; y++) { // y=0 is top (high freq), y=height is bottom (low freq). const m = Math.floor((height - 1 - y) * freqScale); if (m >= melBins) continue; const val = features[m * timeSteps + t]; const clamped = normalizeMelForDisplay(val); const lutIdx = (clamped * 255) | 0; const lutBase = lutIdx * 3; const idx = (y * width + x) * 4; data[idx] = COLORMAP_LUT[lutBase]; data[idx + 1] = COLORMAP_LUT[lutBase + 1]; data[idx + 2] = COLORMAP_LUT[lutBase + 2]; data[idx + 3] = 255; } } ctx.putImageData(imgData, 0, 0); }; // Use gain 1 so waveform shows true amplitude (float32 in [-1,1] fills half-height). // No display amplification; ASR pipeline is unchanged. const WAVEFORM_GAIN = 1; const drawWaveform = (ctx: CanvasRenderingContext2D, data: Float32Array, width: number, height: number, offsetY: number) => { if (data.length === 0) return; const step = Math.ceil(data.length / width); const amp = (height / 2) * WAVEFORM_GAIN; const centerY = offsetY + height / 2; ctx.strokeStyle = '#4ade80'; // Green ctx.lineWidth = 1; ctx.beginPath(); for (let x = 0; x < width; x++) { const startIdx = x * step; const endIdx = Math.min((x + 1) * step, data.length); let min = 1; let max = -1; let hasData = false; for (let i = startIdx; i < endIdx; i += Math.max(1, Math.floor((endIdx - startIdx) / 10))) { const s = data[i]; if (s < min) min = s; if (s > max) max = s; hasData = true; } if (hasData) { const yMin = centerY - min * amp; const yMax = centerY - max * amp; ctx.moveTo(x, Math.max(offsetY, Math.min(offsetY + height, yMin))); ctx.lineTo(x, Math.max(offsetY, Math.min(offsetY + height, yMax))); } } ctx.stroke(); }; const drawVadLayer = (ctx: CanvasRenderingContext2D, width: number, height: number, offsetY: number, startTime: number, duration: number, dpr: number) => { // Draw VAD state as a colored bar // For now, just show current VAD state as a solid bar (could be enhanced with historical data) const vadState = appStore.vadState(); const isSpeech = vadState.isSpeech; // Background ctx.fillStyle = isSpeech ? 'rgba(249, 115, 22, 0.4)' : 'rgba(100, 116, 139, 0.2)'; // Orange when speech, slate when silence ctx.fillRect(0, offsetY, width, height); // If energy-based detection is active, show energy level as a bar const energyLevel = appStore.audioLevel(); const energyThreshold = appStore.energyThreshold(); if (energyLevel > 0) { const barWidth = Math.min(width, width * (energyLevel / 0.3)); // Scale to max 30% energy ctx.fillStyle = energyLevel > energyThreshold ? 'rgba(249, 115, 22, 0.8)' : 'rgba(74, 222, 128, 0.6)'; ctx.fillRect(width - barWidth, offsetY, barWidth, height); } // Draw a thin separator line at top ctx.strokeStyle = 'rgba(148, 163, 184, 0.3)'; ctx.lineWidth = 1 * dpr; ctx.beginPath(); ctx.moveTo(0, offsetY); ctx.lineTo(width, offsetY); ctx.stroke(); // Label ctx.fillStyle = isSpeech ? '#fb923c' : '#64748b'; ctx.font = `${8 * dpr}px monospace`; ctx.fillText(isSpeech ? 'SPEECH' : 'SILENCE', 4 * dpr, offsetY + height - 2 * dpr); }; const drawOverlay = (ctx: CanvasRenderingContext2D, width: number, height: number, startTime: number, duration: number, dpr: number) => { // Draw Trigger line (1.5s from right) if in V3 mode const triggerX = width - (1.5 / duration) * width; ctx.strokeStyle = 'rgba(255, 255, 0, 0.5)'; ctx.lineWidth = 1 * dpr; ctx.beginPath(); ctx.moveTo(triggerX, 0); ctx.lineTo(triggerX, height); ctx.stroke(); // Time labels ctx.fillStyle = '#94a3b8'; ctx.font = `${10 * dpr}px monospace`; for (let i = 0; i <= 8; i += 2) { const t = i; const x = width - (t / duration) * width; ctx.fillText(`-${t}s`, x + 3 * dpr, height - 6 * dpr); } }; return (
SPECTROGRAM + WAVEFORM ({getWindowDuration()}s)
); };