Timeline prettify

decoder_worker.js

@@ -170,6 +170,12 @@ let tokenizer = null;
 let accumulatedFeatures = null;
 let currentSegmentId = null;
 
+// Live caption throttling to prevent pipeline backup
+let isDecoding = false;
+let lastDecodeTime = 0;
+let pendingDecode = false;
+const MIN_DECODE_INTERVAL_MS = 500;  // Don't decode more often than every 500ms for live captions
+
 class MoonshineTokenizer {
     constructor() {
         this.decoder = null;
@@ -320,7 +326,11 @@ async function decodeAccumulated() {
     }
 }
 
-self.onmessage = async function(e) {
+// Message queue for sequential processing
+const messageQueue = [];
+let isProcessingQueue = false;
+
+async function processMessage(e) {
     const { type, data } = e.data;
 
     switch (type) {
@@ -376,6 +386,9 @@ self.onmessage = async function(e) {
         case 'segment_start': {
             accumulatedFeatures = null;
             currentSegmentId = data.segmentId;
+            isDecoding = false;
+            lastDecodeTime = 0;
+            pendingDecode = false;
             self.postMessage({ type: 'live_caption', text: '' });
             break;
         }
@@ -383,7 +396,15 @@ self.onmessage = async function(e) {
         case 'segment_end': {
             if (data.segmentId !== currentSegmentId) break;
 
+            // Wait for any in-progress decode to finish before final decode
+            while (isDecoding) {
+                await new Promise(resolve => setTimeout(resolve, 50));
+            }
+
+            isDecoding = true;
             const text = await decodeAccumulated();
+            isDecoding = false;
+
             self.postMessage({
                 type: 'transcript',
                 segmentId: data.segmentId,
@@ -435,10 +456,64 @@ self.onmessage = async function(e) {
                 }
             }
 
-            // Live caption
-            const partialText = await decodeAccumulated();
-            self.postMessage({ type: 'live_caption', text: partialText });
+            // Live caption with throttling to prevent pipeline backup
+            const now = Date.now();
+            const timeSinceLastDecode = now - lastDecodeTime;
+
+            if (isDecoding) {
+                // Already decoding, mark that we need another decode when done
+                pendingDecode = true;
+            } else if (timeSinceLastDecode >= MIN_DECODE_INTERVAL_MS) {
+                // Enough time has passed, decode now
+                isDecoding = true;
+                lastDecodeTime = now;
+
+                try {
+                    const partialText = await decodeAccumulated();
+                    self.postMessage({ type: 'live_caption', text: partialText });
+                } finally {
+                    isDecoding = false;
+
+                    // If there was a pending decode request, schedule it
+                    if (pendingDecode) {
+                        pendingDecode = false;
+                        // Use setTimeout to avoid blocking - decode will happen on next message or timeout
+                        setTimeout(async () => {
+                            if (!isDecoding && currentSegmentId !== null) {
+                                isDecoding = true;
+                                lastDecodeTime = Date.now();
+                                try {
+                                    const text = await decodeAccumulated();
+                                    self.postMessage({ type: 'live_caption', text: text });
+                                } finally {
+                                    isDecoding = false;
+                                }
+                            }
+                        }, MIN_DECODE_INTERVAL_MS);
+                    }
+                }
+            } else {
+                // Too soon since last decode, mark pending
+                pendingDecode = true;
+            }
             break;
         }
     }
+}
+
+async function processQueue() {
+    if (isProcessingQueue) return;
+    isProcessingQueue = true;
+
+    while (messageQueue.length > 0) {
+        const msg = messageQueue.shift();
+        await processMessage(msg);
+    }
+
+    isProcessingQueue = false;
+}
+
+self.onmessage = function(e) {
+    messageQueue.push(e);
+    processQueue();
 };

encoder_worker.js

@@ -228,7 +228,11 @@ async function processEncoder(melData, melDims, flush = true) {
     return { data: resultData, dims: [1, newOutputCount, encDim] };
 }
 
-self.onmessage = async function(e) {
+// Message queue for sequential processing
+const messageQueue = [];
+let isProcessing = false;
+
+async function processMessage(e) {
     const { type, data } = e.data;
 
     switch (type) {
@@ -320,4 +324,21 @@ self.onmessage = async function(e) {
             break;
         }
     }
+}
+
+async function processQueue() {
+    if (isProcessing) return;
+    isProcessing = true;
+
+    while (messageQueue.length > 0) {
+        const msg = messageQueue.shift();
+        await processMessage(msg);
+    }
+
+    isProcessing = false;
+}
+
+self.onmessage = function(e) {
+    messageQueue.push(e);
+    processQueue();
 };

index.html

@@ -115,7 +115,7 @@
             background: #0f0f23;
             border-radius: 5px;
             padding: 10px;
-            height: 120px;
+            height: 140px;
             position: relative;
             overflow: hidden;
         }
@@ -531,7 +531,7 @@
             }
 
             .vad-graph {
-                height: 80px;
+                height: 100px;
             }
 
             .pipeline-status {

streaming_asr.js

@@ -19,6 +19,8 @@ const ENCODER_BATCH_SAMPLES = 5120;  // 320ms - batch size for encoder
 const PRE_BUFFER_CHUNKS = 25;  // ~500ms at 20ms chunks - capture more audio before onset
 const POST_BUFFER_CHUNKS = 5;  // ~100ms at 20ms chunks
 const MIN_SEGMENT_DURATION_MS = 2000;  // Minimum 2 seconds before allowing segment end
+const OFFSET_RAMP_START_MS = 6000;    // Start ramping offset threshold at 6 seconds
+const OFFSET_RAMP_END_MS = 8000;      // Reach max offset threshold (1.0) at 8 seconds
 const OFFSET_CHUNKS_REQUIRED = 10;  // ~100ms of silence needed to end segment
 
 const MODEL_CONFIGS = {
@@ -225,6 +227,8 @@ class PipelinedStreamingASR {
         this.vadHistory = [];
         this.vadUpdateCounter = 0;
         this.vadUpdateInterval = 5;  // Update display every 5 VAD chunks (50ms)
+        this.segmentEvents = [];  // Track segment start/end for visualization
+        this.vadHistoryStartTime = 0;  // When the current history window started
 
         // Callbacks
         this.onVadUpdate = null;
@@ -504,9 +508,26 @@ class PipelinedStreamingASR {
                 this.vadUpdateCounter++;
                 if (this.vadUpdateCounter >= this.vadUpdateInterval) {
                     this.vadUpdateCounter = 0;
+                    const now = Date.now();
+
+                    // Initialize history start time
+                    if (this.vadHistory.length === 0) {
+                        this.vadHistoryStartTime = now;
+                    }
+
                     this.vadHistory.push(this.emaProb);
-                    if (this.vadHistory.length > 100) this.vadHistory.shift();
-                    this.onVadUpdate?.(this.emaProb, this.vadHistory);
+                    if (this.vadHistory.length > 100) {
+                        this.vadHistory.shift();
+                        // Shift the start time by 50ms (one history entry)
+                        this.vadHistoryStartTime += 50;
+                    }
+
+                    // Remove old segment events that are outside the history window
+                    const historyDuration = this.vadHistory.length * 50;  // ms
+                    const historyStart = now - historyDuration;
+                    this.segmentEvents = this.segmentEvents.filter(e => e.time >= historyStart);
+
+                    this.onVadUpdate?.(this.emaProb, this.vadHistory, this.segmentEvents, this.vadHistoryStartTime);
                 }
 
                 this.updateSegmentState();
@@ -572,11 +593,25 @@ class PipelinedStreamingASR {
                 this.onsetCounter = 0;
             }
         } else if (this.state === 'speech') {
-            // Check if minimum segment duration has passed
             const segmentDuration = Date.now() - this.segmentStartTime;
+
+            // Calculate effective offset threshold with gradual ramp
+            // Before OFFSET_RAMP_START_MS: use normal offsetThreshold
+            // Between OFFSET_RAMP_START_MS and OFFSET_RAMP_END_MS: linearly ramp to 1.0
+            // After OFFSET_RAMP_END_MS: use 1.0 (any VAD level will trigger offset)
+            let effectiveOffsetThreshold = this.offsetThreshold;
+            if (segmentDuration >= OFFSET_RAMP_START_MS) {
+                const rampProgress = Math.min(1.0,
+                    (segmentDuration - OFFSET_RAMP_START_MS) / (OFFSET_RAMP_END_MS - OFFSET_RAMP_START_MS)
+                );
+                // Lerp from offsetThreshold to 1.0
+                effectiveOffsetThreshold = this.offsetThreshold + rampProgress * (1.0 - this.offsetThreshold);
+            }
+
+            // Check if minimum segment duration has passed
             const minDurationMet = segmentDuration >= MIN_SEGMENT_DURATION_MS;
 
-            if (this.emaProb < this.offsetThreshold) {
+            if (this.emaProb < effectiveOffsetThreshold) {
                 this.offsetCounter++;
                 // Only end segment if minimum duration met AND enough silence chunks
                 if (minDurationMet && this.offsetCounter >= OFFSET_CHUNKS_REQUIRED) {
@@ -589,6 +624,12 @@ class PipelinedStreamingASR {
     }
 
     startSegment() {
+        // If previous segment wasn't fully finalized, finalize it now
+        if (this.postBufferRemaining > 0) {
+            this.finalizeSegmentEnd();
+            this.postBufferRemaining = 0;
+        }
+
         this.currentSegmentId++;
         this.state = 'speech';
         this.onsetCounter = 0;
@@ -596,6 +637,9 @@ class PipelinedStreamingASR {
         this.segmentStartTime = Date.now();
         this.encoderBatchBuffer = [];  // Reset batch buffer for new segment
 
+        // Record segment start for visualization
+        this.segmentEvents.push({ type: 'start', time: this.segmentStartTime });
+
         // Tell encoder to start new segment
         this.encoderWorker?.postMessage({
             type: 'segment_start',
@@ -616,6 +660,9 @@ class PipelinedStreamingASR {
         this.offsetCounter = 0;
         this.postBufferRemaining = POST_BUFFER_CHUNKS;
 
+        // Record segment end for visualization
+        this.segmentEvents.push({ type: 'end', time: Date.now() });
+
         if (this.postBufferRemaining === 0) {
             this.finalizeSegmentEnd();
         }
@@ -741,7 +788,7 @@ class ASRDemoUI {
 
             this.asr = new PipelinedStreamingASR(config);
 
-            this.asr.onVadUpdate = (prob, history) => this.updateVadDisplay(prob, history);
+            this.asr.onVadUpdate = (prob, history, segmentEvents, historyStartTime) => this.updateVadDisplay(prob, history, segmentEvents, historyStartTime);
             this.asr.onTranscript = (text, segmentId) => this.addTranscript(text, segmentId);
             this.asr.onLiveCaption = (text) => this.updateLiveCaption(text);
             this.asr.onStatusUpdate = (status, text) => this.updateStatus(status, text);
@@ -781,7 +828,7 @@ class ASRDemoUI {
         this.updateStatus('idle', 'Ready');
     }
 
-    updateVadDisplay(prob, history) {
+    updateVadDisplay(prob, history, segmentEvents = [], historyStartTime = 0) {
         this.vadBarFill.style.width = `${prob * 100}%`;
         this.vadValue.textContent = `${Math.round(prob * 100)}%`;
 
@@ -790,15 +837,79 @@ class ASRDemoUI {
         const width = rect.width;
         const height = rect.height;
 
+        // Leave space for x-axis labels
+        const graphHeight = height - 20;
+        const graphTop = 0;
+
         ctx.fillStyle = '#0f0f23';
         ctx.fillRect(0, 0, width, height);
 
         if (history.length < 2) return;
 
-        const onsetY = height * (1 - parseFloat(this.onsetThreshold.value));
-        const offsetY = height * (1 - parseFloat(this.offsetThreshold.value));
+        const historyDuration = history.length * 50;  // ms (each entry is 50ms)
+        const now = Date.now();
+
+        // Draw x-axis ticks (every 0.1 seconds = 100ms)
+        ctx.strokeStyle = '#333';
+        ctx.fillStyle = '#666';
+        ctx.font = '10px monospace';
+        ctx.textAlign = 'center';
+        ctx.lineWidth = 1;
+
+        for (let t = 0; t <= historyDuration; t += 100) {
+            const x = (t / historyDuration) * width;
+
+            // Draw tick mark
+            ctx.beginPath();
+            ctx.moveTo(x, graphHeight);
+            ctx.lineTo(x, graphHeight + 5);
+            ctx.stroke();
+
+            // Draw vertical grid line (lighter for minor ticks)
+            if (t % 500 === 0) {
+                ctx.strokeStyle = '#444';
+            } else {
+                ctx.strokeStyle = '#222';
+            }
+            ctx.beginPath();
+            ctx.moveTo(x, graphTop);
+            ctx.lineTo(x, graphHeight);
+            ctx.stroke();
+            ctx.strokeStyle = '#333';
+
+            // Draw label every 0.5 seconds
+            if (t % 500 === 0) {
+                const seconds = (t / 1000).toFixed(1);
+                ctx.fillText(seconds + 's', x, height - 2);
+            }
+        }
+
+        // Draw segment events (start = green line, end = red line)
+        for (const event of segmentEvents) {
+            const eventAge = now - event.time;  // ms ago
+            const eventPos = historyDuration - eventAge;  // position in history
+            if (eventPos < 0 || eventPos > historyDuration) continue;
+
+            const x = (eventPos / historyDuration) * width;
+
+            ctx.lineWidth = 2;
+            if (event.type === 'start') {
+                ctx.strokeStyle = '#00ff88';  // Green for start
+            } else {
+                ctx.strokeStyle = '#ff4444';  // Red for end
+            }
+            ctx.beginPath();
+            ctx.moveTo(x, graphTop);
+            ctx.lineTo(x, graphHeight);
+            ctx.stroke();
+        }
+
+        // Draw threshold lines
+        const onsetY = graphHeight * (1 - parseFloat(this.onsetThreshold.value));
+        const offsetY = graphHeight * (1 - parseFloat(this.offsetThreshold.value));
 
         ctx.strokeStyle = '#ff444466';
+        ctx.lineWidth = 1;
         ctx.beginPath();
         ctx.moveTo(0, onsetY);
         ctx.lineTo(width, onsetY);
@@ -810,13 +921,14 @@ class ASRDemoUI {
         ctx.lineTo(width, offsetY);
         ctx.stroke();
 
+        // Draw VAD probability line
         ctx.strokeStyle = '#00d4ff';
         ctx.lineWidth = 2;
         ctx.beginPath();
 
         for (let i = 0; i < history.length; i++) {
             const x = (i / (history.length - 1)) * width;
-            const y = height * (1 - history[i]);
+            const y = graphHeight * (1 - history[i]);
             if (i === 0) {
                 ctx.moveTo(x, y);
             } else {