Mike0021 commited on
Commit
9036e71
·
verified ·
1 Parent(s): 83e70f3

Report LLM start immediately

Browse files
Files changed (3) hide show
  1. README.md +1 -1
  2. app.js +3 -0
  3. workers/llm-worker.js +1 -1
README.md CHANGED
@@ -75,7 +75,7 @@ Use **Run benchmark suite** after loading models to execute the current stack's
75
 
76
  Microphone startup is capped at 15 seconds. If browser permissions, fake media devices, or audio worklet startup hang, the active mic benchmark is cancelled and the event log records the failure instead of leaving a silent pending run. Loopback audio is also followed by an explicit ASR flush after the synthesized prompt and trailing silence have been fed, so sticky VAD/STT candidates fail or finish as rows instead of waiting for the global benchmark timeout.
77
 
78
- Exported result rows include runtime metadata under `stack.environment`: page URL, secure-context status, browser user agent, CPU concurrency hint, device memory hint when exposed, WebGPU availability, WebGPU adapter info/features/software-adapter classification when exposed by the browser, microphone or loopback sample rate, and sanitized microphone track settings such as channel count, sample rate, latency, and audio-processing flags when the browser reports them. The benchmark stack also records voice, TTS steps, VAD silence, partial-ASR state, full-stack model load time as `modelLoadMs`, first-response latency, and scheduled full-audio completion as `audioEndMs` / `speechEndToAudioEndMs`. Keep that metadata with WebGPU results so latency numbers can be compared across machines.
79
 
80
  The latency panel counts **speech end** from the detected acoustic end of speech, not from the later worker event that closes the turn after trailing silence. The separate **VAD close delay** metric shows how much silence was observed before the turn was finalized.
81
 
 
75
 
76
  Microphone startup is capped at 15 seconds. If browser permissions, fake media devices, or audio worklet startup hang, the active mic benchmark is cancelled and the event log records the failure instead of leaving a silent pending run. Loopback audio is also followed by an explicit ASR flush after the synthesized prompt and trailing silence have been fed, so sticky VAD/STT candidates fail or finish as rows instead of waiting for the global benchmark timeout.
77
 
78
+ Exported result rows include runtime metadata under `stack.environment`: page URL, secure-context status, browser user agent, CPU concurrency hint, device memory hint when exposed, WebGPU availability, WebGPU adapter info/features/software-adapter classification when exposed by the browser, microphone or loopback sample rate, and sanitized microphone track settings such as channel count, sample rate, latency, and audio-processing flags when the browser reports them. The benchmark stack also records voice, TTS steps, VAD silence, partial-ASR state, full-stack model load time as `modelLoadMs`, LLM worker acknowledgement latency as `llmStartMs`, first-response latency, and scheduled full-audio completion as `audioEndMs` / `speechEndToAudioEndMs`. Keep that metadata with WebGPU results so latency numbers can be compared across machines.
79
 
80
  The latency panel counts **speech end** from the detected acoustic end of speech, not from the later worker event that closes the turn after trailing silence. The separate **VAD close delay** metric shows how much silence was observed before the turn was finalized.
81
 
app.js CHANGED
@@ -1203,6 +1203,7 @@ function onLlmMessage(event) {
1203
  }
1204
  if (message.type === "start") {
1205
  if (message.turnId !== state.activeTurnId) return;
 
1206
  state.currentAssistant = "";
1207
  state.ttsBuffer = "";
1208
  state.awaitingFirstToken = true;
@@ -1215,6 +1216,7 @@ function onLlmMessage(event) {
1215
  elements.firstAudioLatency.textContent = "-";
1216
  elements.speechToAudioLatency.textContent = "-";
1217
  elements.decodeRate.textContent = "-";
 
1218
  setTile("llm", "Generating", "active");
1219
  return;
1220
  }
@@ -1532,6 +1534,7 @@ function startBenchmark(kind, prompt, { referenceText = "", timeoutMs = 120000 }
1532
  sttWer: null,
1533
  sttCer: null,
1534
  vadCloseDelayMs: null,
 
1535
  firstTokenMs: null,
1536
  firstTtsQueuedMs: null,
1537
  firstTtsSynthesisMs: null,
 
1203
  }
1204
  if (message.type === "start") {
1205
  if (message.turnId !== state.activeTurnId) return;
1206
+ const llmStartMs = Number.isFinite(state.lastTranscriptAt) ? performance.now() - state.lastTranscriptAt : null;
1207
  state.currentAssistant = "";
1208
  state.ttsBuffer = "";
1209
  state.awaitingFirstToken = true;
 
1216
  elements.firstAudioLatency.textContent = "-";
1217
  elements.speechToAudioLatency.textContent = "-";
1218
  elements.decodeRate.textContent = "-";
1219
+ if (state.activeBenchmark) state.activeBenchmark.llmStartMs = llmStartMs;
1220
  setTile("llm", "Generating", "active");
1221
  return;
1222
  }
 
1534
  sttWer: null,
1535
  sttCer: null,
1536
  vadCloseDelayMs: null,
1537
+ llmStartMs: null,
1538
  firstTokenMs: null,
1539
  firstTtsQueuedMs: null,
1540
  firstTtsSynthesisMs: null,
workers/llm-worker.js CHANGED
@@ -61,6 +61,7 @@ function reportProgress(label) {
61
 
62
  async function generate(messages, turnId, { sentenceLimit = 1 } = {}) {
63
  stoppingCriteria = new InterruptableStoppingCriteria();
 
64
  const promptMessages = normalizeMessages(messages);
65
  const inputs = tokenizer.apply_chat_template(promptMessages, {
66
  add_generation_prompt: true,
@@ -102,7 +103,6 @@ async function generate(messages, turnId, { sentenceLimit = 1 } = {}) {
102
  },
103
  });
104
 
105
- self.postMessage({ type: "start", turnId });
106
  await model.generate({
107
  ...inputs,
108
  max_new_tokens: maxNewTokens(),
 
61
 
62
  async function generate(messages, turnId, { sentenceLimit = 1 } = {}) {
63
  stoppingCriteria = new InterruptableStoppingCriteria();
64
+ self.postMessage({ type: "start", turnId });
65
  const promptMessages = normalizeMessages(messages);
66
  const inputs = tokenizer.apply_chat_template(promptMessages, {
67
  add_generation_prompt: true,
 
103
  },
104
  });
105
 
 
106
  await model.generate({
107
  ...inputs,
108
  max_new_tokens: maxNewTokens(),