menoone Claude Opus 5 commited on
Commit
b8422e1
·
1 Parent(s): 53a9410

Read the timing keys the harness actually emits

Browse files

The page showed "926.2 ms · retrieval + guardrails + reader · against a
200 ms budget" in red. 877.6 ms of that was speech synthesis.

Both pages read t.tts_ms and t.asr_ms. The harness names every timing
after its stage, so the keys are speak_ms and transcribe_ms and the two
the pages wanted have never existed. A missing key reads as absent rather
than as an error, so `spoken` was always false: the branch that appends
"+ speech synthesis" to the label, and the line that breaks synthesis out
of the total, were both unreachable. The label was honest about a value it
was never given.

Fixed in web/index.html and in serve.py's PAGE. The split line now also
leads with the graded path on its own -- 48.5 ms here against 926.2 end to
end -- because one number dominated by synthesis tells a reader nothing
about retrieval.

test_pages_read_timing_keys_the_harness_emits pins the contract: every
t.<x>_ms a page reads must be a key the harness emits, with the stage
names parsed from harness.py rather than duplicated. Verified non-vacuous
and verified to reject tts_ms and asr_ms.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Files changed (3) hide show
  1. src/serve.py +2 -2
  2. tests/test_deploy.py +22 -0
  3. web/index.html +8 -3
src/serve.py CHANGED
@@ -496,10 +496,10 @@ function render(d){
496
  '</div>' : '')+
497
  '<div class="lbl">answer</div><div class="ans">'+(d.answer||"—")+'</div>'+
498
  '<div class="t">'+
499
- (t.asr_ms?'<span>ASR <b>'+t.asr_ms+' ms</b></span>':'')+
500
  '<span>retrieve <b>'+ms(t.retrieve_ms)+'</b></span>'+
501
  '<span>read <b>'+ms(t.read_ms)+'</b></span>'+
502
- (t.tts_ms?'<span>TTS <b>'+t.tts_ms+' ms</b></span>':'')+
503
  '<span>total <b>'+ms(t.total_ms)+'</b></span>'+
504
  '<span>conf <b>'+d.confidence+'</b></span>'+
505
  '</div>'+
 
496
  '</div>' : '')+
497
  '<div class="lbl">answer</div><div class="ans">'+(d.answer||"—")+'</div>'+
498
  '<div class="t">'+
499
+ (t.transcribe_ms?'<span>ASR <b>'+t.transcribe_ms+' ms</b></span>':'')+
500
  '<span>retrieve <b>'+ms(t.retrieve_ms)+'</b></span>'+
501
  '<span>read <b>'+ms(t.read_ms)+'</b></span>'+
502
+ (t.speak_ms?'<span>TTS <b>'+t.speak_ms+' ms</b></span>':'')+
503
  '<span>total <b>'+ms(t.total_ms)+'</b></span>'+
504
  '<span>conf <b>'+d.confidence+'</b></span>'+
505
  '</div>'+
tests/test_deploy.py CHANGED
@@ -365,6 +365,28 @@ def test_gradio_ssr_is_off_before_the_import():
365
  assert "ssr_mode=False" in src, "launch/mount must also pass ssr_mode"
366
 
367
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
368
  if __name__ == "__main__":
369
  tests = [v for k, v in sorted(globals().items()) if k.startswith("test_")]
370
  for t in tests:
 
365
  assert "ssr_mode=False" in src, "launch/mount must also pass ssr_mode"
366
 
367
 
368
+ def test_pages_read_timing_keys_the_harness_emits():
369
+ """The harness names every timing after its stage: speak_ms, transcribe_ms.
370
+ Both pages read tts_ms and asr_ms, which never exist -- so the budget line
371
+ said "retrieval + guardrails + reader" over a number carrying 877 ms of
372
+ speech synthesis. A missing key reads as absent, not as an error, which is
373
+ why it survived. Any t.<x>_ms a page reads must be a key harness emits."""
374
+ import re
375
+
376
+ # Stage names are declared inline in _stages; read them from the source so
377
+ # this test needs no built index to run.
378
+ hsrc = (REPO / "src" / "harness.py").read_text(encoding="utf-8")
379
+ stages = re.findall(r'Stage\("([a-z_]+)"', hsrc)
380
+ assert len(stages) >= 7, f"expected 7 stages, found {stages}"
381
+ allowed = {n + "_ms" for n in stages} | {"total_ms", "pipeline_ms", "budget_ms"}
382
+
383
+ for rel in ("web/index.html", "src/serve.py"):
384
+ text = (REPO / rel).read_text(encoding="utf-8")
385
+ used = set(re.findall(r"t\.([a-z_]+_ms)", text))
386
+ unknown = used - allowed
387
+ assert not unknown, f"{rel} reads timing keys the harness never emits: {sorted(unknown)}"
388
+
389
+
390
  if __name__ == "__main__":
391
  tests = [v for k, v in sorted(globals().items()) if k.startswith("test_")]
392
  for t in tests:
web/index.html CHANGED
@@ -884,14 +884,19 @@ function renderBudget(t){
884
  transcribe_ms. Speech synthesis is therefore INSIDE the budget, and a voice
885
  request legitimately blows it where a text request does not. Saying
886
  otherwise on the page would be the pleasant version, not the true one. */
887
- const spoken = (t.tts_ms ?? 0) > 0;
888
  $("#budgetLbl").innerHTML = spoken
889
  ? "retrieval + guardrails + reader <b>+ speech synthesis</b>,<br>against a 200 ms budget"
890
  : "retrieval + guardrails + reader,<br>against a 200 ms budget";
891
 
892
  const parts = [];
893
- if(t.asr_ms) parts.push(`speech-to-text, excluded <b>${t.asr_ms.toFixed(0)} ms</b>`);
894
- if(spoken) parts.push(`of which synthesis <b>${t.tts_ms.toFixed(0)} ms</b>`);
 
 
 
 
 
895
  parts.push(`end to end <b>${(t.total_ms ?? ms).toFixed(0)} ms</b>`);
896
  $("#voiceSplit").innerHTML = parts.map(p => `<span>${p}</span>`).join("");
897
  }
 
884
  transcribe_ms. Speech synthesis is therefore INSIDE the budget, and a voice
885
  request legitimately blows it where a text request does not. Saying
886
  otherwise on the page would be the pleasant version, not the true one. */
887
+ const spoken = (t.speak_ms ?? 0) > 0;
888
  $("#budgetLbl").innerHTML = spoken
889
  ? "retrieval + guardrails + reader <b>+ speech synthesis</b>,<br>against a 200 ms budget"
890
  : "retrieval + guardrails + reader,<br>against a 200 ms budget";
891
 
892
  const parts = [];
893
+ // The graded path on its own. Without it the page shows one number, dominated
894
+ // by synthesis, and a reader cannot tell retrieval from text-to-speech.
895
+ if(spoken) parts.push(
896
+ `retrieval + guardrails + reader <b>${(ms - t.speak_ms).toFixed(1)} ms</b>`);
897
+ if(t.transcribe_ms) parts.push(
898
+ `speech-to-text, excluded <b>${t.transcribe_ms.toFixed(0)} ms</b>`);
899
+ if(spoken) parts.push(`of which synthesis <b>${t.speak_ms.toFixed(0)} ms</b>`);
900
  parts.push(`end to end <b>${(t.total_ms ?? ms).toFixed(0)} ms</b>`);
901
  $("#voiceSplit").innerHTML = parts.map(p => `<span>${p}</span>`).join("");
902
  }