Read the timing keys the harness actually emits
Browse filesThe page showed "926.2 ms · retrieval + guardrails + reader · against a
200 ms budget" in red. 877.6 ms of that was speech synthesis.
Both pages read t.tts_ms and t.asr_ms. The harness names every timing
after its stage, so the keys are speak_ms and transcribe_ms and the two
the pages wanted have never existed. A missing key reads as absent rather
than as an error, so `spoken` was always false: the branch that appends
"+ speech synthesis" to the label, and the line that breaks synthesis out
of the total, were both unreachable. The label was honest about a value it
was never given.
Fixed in web/index.html and in serve.py's PAGE. The split line now also
leads with the graded path on its own -- 48.5 ms here against 926.2 end to
end -- because one number dominated by synthesis tells a reader nothing
about retrieval.
test_pages_read_timing_keys_the_harness_emits pins the contract: every
t.<x>_ms a page reads must be a key the harness emits, with the stage
names parsed from harness.py rather than duplicated. Verified non-vacuous
and verified to reject tts_ms and asr_ms.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
- src/serve.py +2 -2
- tests/test_deploy.py +22 -0
- web/index.html +8 -3
|
@@ -496,10 +496,10 @@ function render(d){
|
|
| 496 |
'</div>' : '')+
|
| 497 |
'<div class="lbl">answer</div><div class="ans">'+(d.answer||"—")+'</div>'+
|
| 498 |
'<div class="t">'+
|
| 499 |
-
(t.
|
| 500 |
'<span>retrieve <b>'+ms(t.retrieve_ms)+'</b></span>'+
|
| 501 |
'<span>read <b>'+ms(t.read_ms)+'</b></span>'+
|
| 502 |
-
(t.
|
| 503 |
'<span>total <b>'+ms(t.total_ms)+'</b></span>'+
|
| 504 |
'<span>conf <b>'+d.confidence+'</b></span>'+
|
| 505 |
'</div>'+
|
|
|
|
| 496 |
'</div>' : '')+
|
| 497 |
'<div class="lbl">answer</div><div class="ans">'+(d.answer||"—")+'</div>'+
|
| 498 |
'<div class="t">'+
|
| 499 |
+
(t.transcribe_ms?'<span>ASR <b>'+t.transcribe_ms+' ms</b></span>':'')+
|
| 500 |
'<span>retrieve <b>'+ms(t.retrieve_ms)+'</b></span>'+
|
| 501 |
'<span>read <b>'+ms(t.read_ms)+'</b></span>'+
|
| 502 |
+
(t.speak_ms?'<span>TTS <b>'+t.speak_ms+' ms</b></span>':'')+
|
| 503 |
'<span>total <b>'+ms(t.total_ms)+'</b></span>'+
|
| 504 |
'<span>conf <b>'+d.confidence+'</b></span>'+
|
| 505 |
'</div>'+
|
|
@@ -365,6 +365,28 @@ def test_gradio_ssr_is_off_before_the_import():
|
|
| 365 |
assert "ssr_mode=False" in src, "launch/mount must also pass ssr_mode"
|
| 366 |
|
| 367 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 368 |
if __name__ == "__main__":
|
| 369 |
tests = [v for k, v in sorted(globals().items()) if k.startswith("test_")]
|
| 370 |
for t in tests:
|
|
|
|
| 365 |
assert "ssr_mode=False" in src, "launch/mount must also pass ssr_mode"
|
| 366 |
|
| 367 |
|
| 368 |
+
def test_pages_read_timing_keys_the_harness_emits():
|
| 369 |
+
"""The harness names every timing after its stage: speak_ms, transcribe_ms.
|
| 370 |
+
Both pages read tts_ms and asr_ms, which never exist -- so the budget line
|
| 371 |
+
said "retrieval + guardrails + reader" over a number carrying 877 ms of
|
| 372 |
+
speech synthesis. A missing key reads as absent, not as an error, which is
|
| 373 |
+
why it survived. Any t.<x>_ms a page reads must be a key harness emits."""
|
| 374 |
+
import re
|
| 375 |
+
|
| 376 |
+
# Stage names are declared inline in _stages; read them from the source so
|
| 377 |
+
# this test needs no built index to run.
|
| 378 |
+
hsrc = (REPO / "src" / "harness.py").read_text(encoding="utf-8")
|
| 379 |
+
stages = re.findall(r'Stage\("([a-z_]+)"', hsrc)
|
| 380 |
+
assert len(stages) >= 7, f"expected 7 stages, found {stages}"
|
| 381 |
+
allowed = {n + "_ms" for n in stages} | {"total_ms", "pipeline_ms", "budget_ms"}
|
| 382 |
+
|
| 383 |
+
for rel in ("web/index.html", "src/serve.py"):
|
| 384 |
+
text = (REPO / rel).read_text(encoding="utf-8")
|
| 385 |
+
used = set(re.findall(r"t\.([a-z_]+_ms)", text))
|
| 386 |
+
unknown = used - allowed
|
| 387 |
+
assert not unknown, f"{rel} reads timing keys the harness never emits: {sorted(unknown)}"
|
| 388 |
+
|
| 389 |
+
|
| 390 |
if __name__ == "__main__":
|
| 391 |
tests = [v for k, v in sorted(globals().items()) if k.startswith("test_")]
|
| 392 |
for t in tests:
|
|
@@ -884,14 +884,19 @@ function renderBudget(t){
|
|
| 884 |
transcribe_ms. Speech synthesis is therefore INSIDE the budget, and a voice
|
| 885 |
request legitimately blows it where a text request does not. Saying
|
| 886 |
otherwise on the page would be the pleasant version, not the true one. */
|
| 887 |
-
const spoken = (t.
|
| 888 |
$("#budgetLbl").innerHTML = spoken
|
| 889 |
? "retrieval + guardrails + reader <b>+ speech synthesis</b>,<br>against a 200 ms budget"
|
| 890 |
: "retrieval + guardrails + reader,<br>against a 200 ms budget";
|
| 891 |
|
| 892 |
const parts = [];
|
| 893 |
-
|
| 894 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 895 |
parts.push(`end to end <b>${(t.total_ms ?? ms).toFixed(0)} ms</b>`);
|
| 896 |
$("#voiceSplit").innerHTML = parts.map(p => `<span>${p}</span>`).join("");
|
| 897 |
}
|
|
|
|
| 884 |
transcribe_ms. Speech synthesis is therefore INSIDE the budget, and a voice
|
| 885 |
request legitimately blows it where a text request does not. Saying
|
| 886 |
otherwise on the page would be the pleasant version, not the true one. */
|
| 887 |
+
const spoken = (t.speak_ms ?? 0) > 0;
|
| 888 |
$("#budgetLbl").innerHTML = spoken
|
| 889 |
? "retrieval + guardrails + reader <b>+ speech synthesis</b>,<br>against a 200 ms budget"
|
| 890 |
: "retrieval + guardrails + reader,<br>against a 200 ms budget";
|
| 891 |
|
| 892 |
const parts = [];
|
| 893 |
+
// The graded path on its own. Without it the page shows one number, dominated
|
| 894 |
+
// by synthesis, and a reader cannot tell retrieval from text-to-speech.
|
| 895 |
+
if(spoken) parts.push(
|
| 896 |
+
`retrieval + guardrails + reader <b>${(ms - t.speak_ms).toFixed(1)} ms</b>`);
|
| 897 |
+
if(t.transcribe_ms) parts.push(
|
| 898 |
+
`speech-to-text, excluded <b>${t.transcribe_ms.toFixed(0)} ms</b>`);
|
| 899 |
+
if(spoken) parts.push(`of which synthesis <b>${t.speak_ms.toFixed(0)} ms</b>`);
|
| 900 |
parts.push(`end to end <b>${(t.total_ms ?? ms).toFixed(0)} ms</b>`);
|
| 901 |
$("#voiceSplit").innerHTML = parts.map(p => `<span>${p}</span>`).join("");
|
| 902 |
}
|