Spaces:
Configuration error
Configuration error
Add worker network telemetry to browser evidence
Browse files- README.md +2 -2
- app.js +122 -10
- tools/audit-browser-evidence.mjs +386 -0
- tools/audit-validation.mjs +439 -0
- tools/run-client-side-smoke.mjs +645 -0
- tools/run-evidence-export-smoke.mjs +414 -0
- tools/run-fake-mic-benchmark.mjs +485 -0
- tools/run-final-validation.mjs +307 -0
- tools/run-hosted-evidence-capture.mjs +245 -0
- tools/run-hosted-smoke.mjs +270 -0
- tools/run-local-candidate-benchmark.mjs +617 -0
- tools/run-loopback-series.mjs +142 -0
- tools/run-real-mic-series.mjs +517 -0
- tools/run-tts-sweep-benchmark.mjs +453 -0
- tools/run-ui-smoke.mjs +482 -0
- tools/run-webgpu-benchmark.mjs +491 -0
- tools/source-fingerprint.mjs +44 -0
- tools/summarize-evidence.mjs +311 -0
- workers/asr-worker.js +44 -0
- workers/llm-worker.js +44 -0
- workers/tts-worker.js +44 -0
README.md
CHANGED
|
@@ -104,9 +104,9 @@ Candidate comparisons to run:
|
|
| 104 |
|
| 105 |
For model comparisons, choose the candidate model before **Load models**, run **Run benchmark suite**, add at least three real microphone rows with **Run 3 real-mic series**, then click **Unload models** and choose the next candidate. The benchmark table is preserved across unloads, shows prompt token/build metrics for LLM-backed rows, and **Copy JSON** can export the summary and all rows after the comparison set. For TTS voice/step comparisons, change the voice or step slider after loading and rerun **Run TTS benchmark** plus the text/loopback benchmarks; those values are captured per row.
|
| 106 |
|
| 107 |
-
Use **Download JSON** after benchmark or real-mic runs on the hosted Space to save a timestamped evidence file with the same payload as **Copy JSON** and `window.browserSpeakBench.exportResults()`. The export includes `schemaVersion`, a unique `exportId`, `generatedAt`, `hostMetadata` with the hosted Space commit when the static host exposes it, the current `runtime`, an `evidence` summary for real-mic/hardware-WebGPU/hosted-build readiness, an embedded `evidenceGuide` with the audit command and row requirements, aggregate `summary`, and raw `results`. The **Run evidence capture** button starts a timestamped `browser-speak-evidence-*.json` download when the capture sequence finishes; if the browser blocks the automatic download, **Download JSON** saves the same payload manually. Benchmark rows are also autosaved in browser storage for the current host and Space commit, so a reload during manual evidence capture can restore rows from the same deployed build; **Clear** removes both the visible rows and the saved copy.
|
| 108 |
|
| 109 |
-
Use `node tools/audit-browser-evidence.mjs path/to/browser-speak-benchmarks.json` to audit a JSON file downloaded from the hosted browser UI. This checks that the export came from the static Space revision, reports the page-generated `evidence` summary, verifies that at least three real-mic rows meet the WER/identity/latency gates from raw rows, and verifies that at least one completed row reports hardware WebGPU rather than a software adapter. Browser-downloaded JSON cannot prove no-server network behavior or Chrome launch flags by itself, so pair it with `node tools/run-hosted-smoke.mjs` for network evidence or `node tools/run-real-mic-series.mjs` for harness-collected real-mic provenance.
|
| 110 |
|
| 111 |
When served from localhost or the hosted Hugging Face Space, the page installs `window.browserSpeakBench` for browser automation. It exposes `loadStack(options)`, `setRuntimeOptions(options)`, `preloadVoice(options)`, `runSuite()`, `runMicSeries()`, `runEvidenceCapture()`, one-off runners such as `runTts()`, `runLoopback()`, and `runWebGpuEvidence()`, `exportResults()`, `downloadResults(options)`, `clearResults()`, `webgpuInfo()`, and `state()`. This lets a Playwright, Puppeteer, or DevTools Protocol harness drive the same UI-backed benchmark paths and collect the same JSON as **Copy JSON**. `setRuntimeOptions()` and `loadStack()` also accept `ttsChunking` so first-response chunk thresholds can be benchmarked without editing source; the exported stack records the active chunk profile. The helper `node tools/run-fake-mic-benchmark.mjs` synthesizes the scripted prompt with local Supertonic, relaunches Chrome with that WAV as `--use-file-for-fake-audio-capture`, and runs three scripted mic rows through getUserMedia, the audio worklet, VAD, STT, LLM, and TTS. Set `BROWSER_SPEAK_REUSE_FAKE_MIC_WAV=true` to reuse an existing fixture WAV and skip the synthesis/load pass during regression checks; row timeouts can be adjusted with `BROWSER_SPEAK_FAKE_MIC_TURN_TIMEOUT_MS` and `BROWSER_SPEAK_FAKE_MIC_ROW_TIMEOUT_MS`. The CDP harnesses use a 60-second protocol watchdog by default, configurable with `BROWSER_SPEAK_CDP_TIMEOUT_MS`, short 5-second status-poll watchdogs, configurable with `BROWSER_SPEAK_CDP_POLL_TIMEOUT_MS`, and a page-unresponsive watchdog, configurable with `BROWSER_SPEAK_PAGE_UNRESPONSIVE_TIMEOUT_MS`, so a slow or wedged browser target produces a controlled failure and cleanup instead of blocking indefinitely. A fake capture file is useful for regression testing the mic pipeline, but real microphone rows are still required before judging conversational quality.
|
| 112 |
|
|
|
|
| 104 |
|
| 105 |
For model comparisons, choose the candidate model before **Load models**, run **Run benchmark suite**, add at least three real microphone rows with **Run 3 real-mic series**, then click **Unload models** and choose the next candidate. The benchmark table is preserved across unloads, shows prompt token/build metrics for LLM-backed rows, and **Copy JSON** can export the summary and all rows after the comparison set. For TTS voice/step comparisons, change the voice or step slider after loading and rerun **Run TTS benchmark** plus the text/loopback benchmarks; those values are captured per row.
|
| 106 |
|
| 107 |
+
Use **Download JSON** after benchmark or real-mic runs on the hosted Space to save a timestamped evidence file with the same payload as **Copy JSON** and `window.browserSpeakBench.exportResults()`. The export includes `schemaVersion`, a unique `exportId`, `generatedAt`, `hostMetadata` with the hosted Space commit when the static host exposes it, the current `runtime`, a bounded `network` trace of model-worker fetches with benchmark-phase server-inference suspect counts, an `evidence` summary for real-mic/hardware-WebGPU/hosted-build readiness, an embedded `evidenceGuide` with the audit command and row requirements, aggregate `summary`, and raw `results`. Each benchmark row also records worker-network request counts and the recent per-row worker requests when any fetch occurs while the row is active. The **Run evidence capture** button starts a timestamped `browser-speak-evidence-*.json` download when the capture sequence finishes; if the browser blocks the automatic download, **Download JSON** saves the same payload manually. Benchmark rows are also autosaved in browser storage for the current host and Space commit, so a reload during manual evidence capture can restore rows from the same deployed build; **Clear** removes both the visible rows and the saved copy.
|
| 108 |
|
| 109 |
+
Use `node tools/audit-browser-evidence.mjs path/to/browser-speak-benchmarks.json` to audit a JSON file downloaded from the hosted browser UI. This checks that the export came from the static Space revision, reports the page-generated `evidence` summary, verifies that at least three real-mic rows meet the WER/identity/latency gates from raw rows, and verifies that at least one completed row reports hardware WebGPU rather than a software adapter. Browser-downloaded JSON includes worker-side fetch telemetry, but it cannot prove whole-page no-server network behavior or Chrome launch flags by itself, so pair it with `node tools/run-hosted-smoke.mjs` for CDP network evidence or `node tools/run-real-mic-series.mjs` for harness-collected real-mic provenance.
|
| 110 |
|
| 111 |
When served from localhost or the hosted Hugging Face Space, the page installs `window.browserSpeakBench` for browser automation. It exposes `loadStack(options)`, `setRuntimeOptions(options)`, `preloadVoice(options)`, `runSuite()`, `runMicSeries()`, `runEvidenceCapture()`, one-off runners such as `runTts()`, `runLoopback()`, and `runWebGpuEvidence()`, `exportResults()`, `downloadResults(options)`, `clearResults()`, `webgpuInfo()`, and `state()`. This lets a Playwright, Puppeteer, or DevTools Protocol harness drive the same UI-backed benchmark paths and collect the same JSON as **Copy JSON**. `setRuntimeOptions()` and `loadStack()` also accept `ttsChunking` so first-response chunk thresholds can be benchmarked without editing source; the exported stack records the active chunk profile. The helper `node tools/run-fake-mic-benchmark.mjs` synthesizes the scripted prompt with local Supertonic, relaunches Chrome with that WAV as `--use-file-for-fake-audio-capture`, and runs three scripted mic rows through getUserMedia, the audio worklet, VAD, STT, LLM, and TTS. Set `BROWSER_SPEAK_REUSE_FAKE_MIC_WAV=true` to reuse an existing fixture WAV and skip the synthesis/load pass during regression checks; row timeouts can be adjusted with `BROWSER_SPEAK_FAKE_MIC_TURN_TIMEOUT_MS` and `BROWSER_SPEAK_FAKE_MIC_ROW_TIMEOUT_MS`. The CDP harnesses use a 60-second protocol watchdog by default, configurable with `BROWSER_SPEAK_CDP_TIMEOUT_MS`, short 5-second status-poll watchdogs, configurable with `BROWSER_SPEAK_CDP_POLL_TIMEOUT_MS`, and a page-unresponsive watchdog, configurable with `BROWSER_SPEAK_PAGE_UNRESPONSIVE_TIMEOUT_MS`, so a slow or wedged browser target produces a controlled failure and cleanup instead of blocking indefinitely. A fake capture file is useful for regression testing the mic pipeline, but real microphone rows are still required before judging conversational quality.
|
| 112 |
|
app.js
CHANGED
|
@@ -152,6 +152,8 @@ const HARDWARE_WEBGPU_EVIDENCE_STACK = Object.freeze({
|
|
| 152 |
const MIC_START_TIMEOUT_MS = 15000;
|
| 153 |
const WEBGPU_ADAPTER_TIMEOUT_MS = 5000;
|
| 154 |
const MIC_LEVEL_UI_INTERVAL_MS = 80;
|
|
|
|
|
|
|
| 155 |
const DEFAULT_TTS_CHUNKING = Object.freeze({
|
| 156 |
firstSentenceMinChars: 5,
|
| 157 |
sentenceMinChars: 8,
|
|
@@ -229,6 +231,8 @@ const state = {
|
|
| 229 |
completed: 0,
|
| 230 |
timer: 0,
|
| 231 |
},
|
|
|
|
|
|
|
| 232 |
ttsChunking: { ...DEFAULT_TTS_CHUNKING },
|
| 233 |
benchmarkResults: [],
|
| 234 |
};
|
|
@@ -812,7 +816,13 @@ async function resolveDevice() {
|
|
| 812 |
|
| 813 |
function createWorker(path, kind, handler) {
|
| 814 |
const worker = new Worker(new URL(path, import.meta.url), { type: "module" });
|
| 815 |
-
worker.addEventListener("message",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 816 |
worker.addEventListener("error", (event) => {
|
| 817 |
const message = `${path} error: ${event.message || "worker failed to load"}`;
|
| 818 |
logEvent(message);
|
|
@@ -1448,6 +1458,91 @@ function handleStatus(kind, message) {
|
|
| 1448 |
if (message.detail) logEvent(message.detail);
|
| 1449 |
}
|
| 1450 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1451 |
function currentBenchmarkStack() {
|
| 1452 |
return {
|
| 1453 |
...state.loadedStack,
|
|
@@ -1604,12 +1699,15 @@ function startBenchmark(kind, prompt, { referenceText = "", timeoutMs = 120000 }
|
|
| 1604 |
micInputRms: null,
|
| 1605 |
loopbackSampleRate: null,
|
| 1606 |
bargeInMs: null,
|
| 1607 |
-
|
| 1608 |
-
|
| 1609 |
-
|
| 1610 |
-
|
| 1611 |
-
|
| 1612 |
-
|
|
|
|
|
|
|
|
|
|
| 1613 |
llmQualityScore: null,
|
| 1614 |
llmQualityTotal: APP_IDENTITY_QUALITY_RULES.length,
|
| 1615 |
llmQualityHits: [],
|
|
@@ -1931,6 +2029,7 @@ function benchmarkExportPayload() {
|
|
| 1931 |
generatedAt,
|
| 1932 |
hostMetadata: { ...state.hostMetadata },
|
| 1933 |
runtime: runtimeEnvironment(),
|
|
|
|
| 1934 |
evidence,
|
| 1935 |
evidenceGuide: browserEvidenceGuide(evidence),
|
| 1936 |
summary: exportBenchmarkSummary(),
|
|
@@ -1982,12 +2081,24 @@ function browserEvidenceGuide(evidence) {
|
|
| 1982 |
nextActions: browserEvidenceNextActions(evidence),
|
| 1983 |
limitations: [
|
| 1984 |
"Downloaded browser JSON proves page-reported rows and adapter metadata.",
|
|
|
|
| 1985 |
"Pair with hosted smoke for no-server network evidence.",
|
| 1986 |
"Use the real-mic harness for Chrome launch provenance when fake-capture flags must be ruled out.",
|
| 1987 |
],
|
| 1988 |
};
|
| 1989 |
}
|
| 1990 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1991 |
function browserEvidenceNextActions(evidence) {
|
| 1992 |
const actions = [];
|
| 1993 |
if (!evidence.hosted.passed) {
|
|
@@ -2900,9 +3011,10 @@ function automationSnapshot() {
|
|
| 2900 |
suiteRunning: state.suiteRunning,
|
| 2901 |
evidenceCaptureRunning: state.evidenceCaptureRunning,
|
| 2902 |
micSeries: { ...state.micSeries, timer: Boolean(state.micSeries.timer) },
|
| 2903 |
-
|
| 2904 |
-
|
| 2905 |
-
|
|
|
|
| 2906 |
results: [...state.benchmarkResults],
|
| 2907 |
events: [...elements.eventLog.children].map((item) => item.textContent),
|
| 2908 |
};
|
|
|
|
| 152 |
const MIC_START_TIMEOUT_MS = 15000;
|
| 153 |
const WEBGPU_ADAPTER_TIMEOUT_MS = 5000;
|
| 154 |
const MIC_LEVEL_UI_INTERVAL_MS = 80;
|
| 155 |
+
const MAX_WORKER_NETWORK_EVENTS = 300;
|
| 156 |
+
const MAX_ROW_WORKER_NETWORK_EVENTS = 40;
|
| 157 |
const DEFAULT_TTS_CHUNKING = Object.freeze({
|
| 158 |
firstSentenceMinChars: 5,
|
| 159 |
sentenceMinChars: 8,
|
|
|
|
| 231 |
completed: 0,
|
| 232 |
timer: 0,
|
| 233 |
},
|
| 234 |
+
workerNetworkSeq: 0,
|
| 235 |
+
workerNetworkEvents: [],
|
| 236 |
ttsChunking: { ...DEFAULT_TTS_CHUNKING },
|
| 237 |
benchmarkResults: [],
|
| 238 |
};
|
|
|
|
| 816 |
|
| 817 |
function createWorker(path, kind, handler) {
|
| 818 |
const worker = new Worker(new URL(path, import.meta.url), { type: "module" });
|
| 819 |
+
worker.addEventListener("message", (event) => {
|
| 820 |
+
if (event.data?.type === "network") {
|
| 821 |
+
handleWorkerNetworkEvent(kind, event.data);
|
| 822 |
+
return;
|
| 823 |
+
}
|
| 824 |
+
handler(event);
|
| 825 |
+
});
|
| 826 |
worker.addEventListener("error", (event) => {
|
| 827 |
const message = `${path} error: ${event.message || "worker failed to load"}`;
|
| 828 |
logEvent(message);
|
|
|
|
| 1458 |
if (message.detail) logEvent(message.detail);
|
| 1459 |
}
|
| 1460 |
|
| 1461 |
+
function handleWorkerNetworkEvent(worker, message) {
|
| 1462 |
+
const event = {
|
| 1463 |
+
id: ++state.workerNetworkSeq,
|
| 1464 |
+
capturedAt: new Date().toISOString(),
|
| 1465 |
+
elapsedMs: Math.round(performance.now()),
|
| 1466 |
+
phase: workerNetworkPhase(),
|
| 1467 |
+
worker,
|
| 1468 |
+
method: String(message.method || "GET").toUpperCase(),
|
| 1469 |
+
url: sanitizeNetworkUrl(message.responseUrl || message.url),
|
| 1470 |
+
requestUrl: sanitizeNetworkUrl(message.url),
|
| 1471 |
+
status: Number.isFinite(message.status) ? message.status : null,
|
| 1472 |
+
ok: typeof message.ok === "boolean" ? message.ok : null,
|
| 1473 |
+
durationMs: Number.isFinite(message.durationMs) ? message.durationMs : null,
|
| 1474 |
+
error: message.error ? String(message.error) : "",
|
| 1475 |
+
};
|
| 1476 |
+
event.serverInferenceSuspect = isWorkerServerInferenceSuspect(event);
|
| 1477 |
+
state.workerNetworkEvents.push(event);
|
| 1478 |
+
if (state.workerNetworkEvents.length > MAX_WORKER_NETWORK_EVENTS) {
|
| 1479 |
+
state.workerNetworkEvents.splice(0, state.workerNetworkEvents.length - MAX_WORKER_NETWORK_EVENTS);
|
| 1480 |
+
}
|
| 1481 |
+
|
| 1482 |
+
if (!state.activeBenchmark) return;
|
| 1483 |
+
state.activeBenchmark.workerNetworkRequestCount += 1;
|
| 1484 |
+
if (event.serverInferenceSuspect) state.activeBenchmark.workerNetworkServerInferenceSuspects += 1;
|
| 1485 |
+
state.activeBenchmark.workerNetworkRequests.push(event);
|
| 1486 |
+
if (state.activeBenchmark.workerNetworkRequests.length > MAX_ROW_WORKER_NETWORK_EVENTS) {
|
| 1487 |
+
state.activeBenchmark.workerNetworkRequests.splice(
|
| 1488 |
+
0,
|
| 1489 |
+
state.activeBenchmark.workerNetworkRequests.length - MAX_ROW_WORKER_NETWORK_EVENTS,
|
| 1490 |
+
);
|
| 1491 |
+
}
|
| 1492 |
+
}
|
| 1493 |
+
|
| 1494 |
+
function workerNetworkPhase() {
|
| 1495 |
+
if (state.modelsLoading) return "load";
|
| 1496 |
+
if (state.activeBenchmark) return "benchmark";
|
| 1497 |
+
if (state.modelsLoaded) return "ready";
|
| 1498 |
+
return "idle";
|
| 1499 |
+
}
|
| 1500 |
+
|
| 1501 |
+
function sanitizeNetworkUrl(value = "") {
|
| 1502 |
+
const raw = String(value || "");
|
| 1503 |
+
if (!raw) return "";
|
| 1504 |
+
try {
|
| 1505 |
+
const url = new URL(raw, location.href);
|
| 1506 |
+
url.hash = "";
|
| 1507 |
+
url.search = "";
|
| 1508 |
+
return url.href;
|
| 1509 |
+
} catch {
|
| 1510 |
+
return raw.slice(0, 500);
|
| 1511 |
+
}
|
| 1512 |
+
}
|
| 1513 |
+
|
| 1514 |
+
function isWorkerServerInferenceSuspect(event) {
|
| 1515 |
+
if (!event.url) return false;
|
| 1516 |
+
const lowerUrl = event.url.toLowerCase();
|
| 1517 |
+
if (isKnownModelAssetUrl(lowerUrl)) return false;
|
| 1518 |
+
if (
|
| 1519 |
+
lowerUrl.includes("/v1/chat/completions") ||
|
| 1520 |
+
lowerUrl.includes("/v1/completions") ||
|
| 1521 |
+
lowerUrl.includes("/v1/audio/") ||
|
| 1522 |
+
lowerUrl.includes("/api/inference") ||
|
| 1523 |
+
lowerUrl.includes("/api/pipeline")
|
| 1524 |
+
) {
|
| 1525 |
+
return true;
|
| 1526 |
+
}
|
| 1527 |
+
return event.phase === "benchmark";
|
| 1528 |
+
}
|
| 1529 |
+
|
| 1530 |
+
function isKnownModelAssetUrl(value = "") {
|
| 1531 |
+
try {
|
| 1532 |
+
const url = new URL(value);
|
| 1533 |
+
const host = url.hostname.toLowerCase();
|
| 1534 |
+
return (
|
| 1535 |
+
host === "cdn.jsdelivr.net" ||
|
| 1536 |
+
host === "huggingface.co" ||
|
| 1537 |
+
host.endsWith(".huggingface.co") ||
|
| 1538 |
+
host.endsWith(".hf.co") ||
|
| 1539 |
+
host.endsWith(".xethub.hf.co")
|
| 1540 |
+
);
|
| 1541 |
+
} catch {
|
| 1542 |
+
return false;
|
| 1543 |
+
}
|
| 1544 |
+
}
|
| 1545 |
+
|
| 1546 |
function currentBenchmarkStack() {
|
| 1547 |
return {
|
| 1548 |
...state.loadedStack,
|
|
|
|
| 1699 |
micInputRms: null,
|
| 1700 |
loopbackSampleRate: null,
|
| 1701 |
bargeInMs: null,
|
| 1702 |
+
bargeInPass: null,
|
| 1703 |
+
firstAudioMs: null,
|
| 1704 |
+
speechEndToFirstAudioMs: null,
|
| 1705 |
+
workerNetworkRequestCount: 0,
|
| 1706 |
+
workerNetworkServerInferenceSuspects: 0,
|
| 1707 |
+
workerNetworkRequests: [],
|
| 1708 |
+
decodeRate: "-",
|
| 1709 |
+
llmCompleteMs: null,
|
| 1710 |
+
llmQualityPass: null,
|
| 1711 |
llmQualityScore: null,
|
| 1712 |
llmQualityTotal: APP_IDENTITY_QUALITY_RULES.length,
|
| 1713 |
llmQualityHits: [],
|
|
|
|
| 2029 |
generatedAt,
|
| 2030 |
hostMetadata: { ...state.hostMetadata },
|
| 2031 |
runtime: runtimeEnvironment(),
|
| 2032 |
+
network: workerNetworkSummary(),
|
| 2033 |
evidence,
|
| 2034 |
evidenceGuide: browserEvidenceGuide(evidence),
|
| 2035 |
summary: exportBenchmarkSummary(),
|
|
|
|
| 2081 |
nextActions: browserEvidenceNextActions(evidence),
|
| 2082 |
limitations: [
|
| 2083 |
"Downloaded browser JSON proves page-reported rows and adapter metadata.",
|
| 2084 |
+
"Worker fetch telemetry is bounded and page-reported; hosted CDP smoke is the authoritative whole-page network check.",
|
| 2085 |
"Pair with hosted smoke for no-server network evidence.",
|
| 2086 |
"Use the real-mic harness for Chrome launch provenance when fake-capture flags must be ruled out.",
|
| 2087 |
],
|
| 2088 |
};
|
| 2089 |
}
|
| 2090 |
|
| 2091 |
+
function workerNetworkSummary() {
|
| 2092 |
+
const events = [...state.workerNetworkEvents];
|
| 2093 |
+
const benchmarkEvents = events.filter((event) => event.phase === "benchmark");
|
| 2094 |
+
return {
|
| 2095 |
+
capturedEvents: events.length,
|
| 2096 |
+
benchmarkRequests: benchmarkEvents.length,
|
| 2097 |
+
benchmarkServerInferenceSuspects: benchmarkEvents.filter((event) => event.serverInferenceSuspect).length,
|
| 2098 |
+
recent: events.slice(-MAX_WORKER_NETWORK_EVENTS),
|
| 2099 |
+
};
|
| 2100 |
+
}
|
| 2101 |
+
|
| 2102 |
function browserEvidenceNextActions(evidence) {
|
| 2103 |
const actions = [];
|
| 2104 |
if (!evidence.hosted.passed) {
|
|
|
|
| 3011 |
suiteRunning: state.suiteRunning,
|
| 3012 |
evidenceCaptureRunning: state.evidenceCaptureRunning,
|
| 3013 |
micSeries: { ...state.micSeries, timer: Boolean(state.micSeries.timer) },
|
| 3014 |
+
ttsChunking: { ...state.ttsChunking },
|
| 3015 |
+
network: workerNetworkSummary(),
|
| 3016 |
+
stack: state.loadedStack ? currentBenchmarkStack() : null,
|
| 3017 |
+
summary: exportBenchmarkSummary(),
|
| 3018 |
results: [...state.benchmarkResults],
|
| 3019 |
events: [...elements.eventLog.children].map((item) => item.textContent),
|
| 3020 |
};
|
tools/audit-browser-evidence.mjs
ADDED
|
@@ -0,0 +1,386 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env node
|
| 2 |
+
import { readFile, writeFile, mkdir } from "node:fs/promises";
|
| 3 |
+
import { tmpdir } from "node:os";
|
| 4 |
+
import { dirname, resolve } from "node:path";
|
| 5 |
+
import { sourceFingerprint } from "./source-fingerprint.mjs";
|
| 6 |
+
|
| 7 |
+
const rawInputPath = process.argv[2] ?? process.env.BROWSER_SPEAK_BROWSER_EXPORT_JSON ?? "";
|
| 8 |
+
const inputPath = rawInputPath ? resolve(rawInputPath) : "";
|
| 9 |
+
const resultPath = resolve(
|
| 10 |
+
process.env.BROWSER_SPEAK_BROWSER_AUDIT_JSON ?? `${tmpdir()}/browser-speak-browser-evidence-audit.json`,
|
| 11 |
+
);
|
| 12 |
+
const requireRealMic = process.env.BROWSER_SPEAK_BROWSER_AUDIT_REQUIRE_REAL_MIC !== "false";
|
| 13 |
+
const requireHardwareWebgpu = process.env.BROWSER_SPEAK_BROWSER_AUDIT_REQUIRE_HARDWARE_WEBGPU !== "false";
|
| 14 |
+
const minRealMicRows = Math.max(1, Number(process.env.BROWSER_SPEAK_BROWSER_AUDIT_REAL_MIC_ROWS ?? 3));
|
| 15 |
+
const realMicMaxWer = Number(process.env.BROWSER_SPEAK_BROWSER_AUDIT_REAL_MIC_MAX_WER ?? 0.25);
|
| 16 |
+
const expectedHostSuffix = process.env.BROWSER_SPEAK_BROWSER_AUDIT_HOST_SUFFIX ?? ".hf.space";
|
| 17 |
+
|
| 18 |
+
async function main() {
|
| 19 |
+
if (!inputPath) {
|
| 20 |
+
throw new Error(
|
| 21 |
+
"Pass a downloaded benchmark JSON path, or set BROWSER_SPEAK_BROWSER_EXPORT_JSON.",
|
| 22 |
+
);
|
| 23 |
+
}
|
| 24 |
+
const exportPayload = JSON.parse(await readFile(inputPath, "utf8"));
|
| 25 |
+
const browserEvidence = evidenceSummaryFromPayload(exportPayload);
|
| 26 |
+
const checks = [
|
| 27 |
+
checkExportShape(exportPayload),
|
| 28 |
+
checkSelfDescription(exportPayload),
|
| 29 |
+
checkBrowserEvidenceSummary(exportPayload),
|
| 30 |
+
checkHostedMetadata(exportPayload),
|
| 31 |
+
checkRealMic(exportPayload),
|
| 32 |
+
checkHardwareWebgpu(exportPayload),
|
| 33 |
+
];
|
| 34 |
+
const requiredChecks = checks.filter((check) => check.required);
|
| 35 |
+
const passed = requiredChecks.every((check) => check.status === "pass");
|
| 36 |
+
const payload = {
|
| 37 |
+
generatedAt: new Date().toISOString(),
|
| 38 |
+
sourceFingerprint: await sourceFingerprint(),
|
| 39 |
+
passed,
|
| 40 |
+
inputPath,
|
| 41 |
+
config: {
|
| 42 |
+
requireRealMic,
|
| 43 |
+
requireHardwareWebgpu,
|
| 44 |
+
minRealMicRows,
|
| 45 |
+
realMicMaxWer,
|
| 46 |
+
expectedHostSuffix,
|
| 47 |
+
},
|
| 48 |
+
browserEvidence,
|
| 49 |
+
checks,
|
| 50 |
+
limitations: [
|
| 51 |
+
"Downloaded browser benchmark JSON can prove the page-reported runtime, real-mic rows, and WebGPU adapter metadata.",
|
| 52 |
+
"It cannot prove the absence of server inference by itself; pair it with tools/run-hosted-smoke.mjs or tools/run-client-side-smoke.mjs network evidence.",
|
| 53 |
+
"It cannot prove Chrome was not launched with fake microphone capture flags; use tools/run-real-mic-series.mjs when that provenance matters.",
|
| 54 |
+
],
|
| 55 |
+
};
|
| 56 |
+
await mkdir(dirname(resultPath), { recursive: true });
|
| 57 |
+
await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`);
|
| 58 |
+
console.log(`Wrote browser evidence audit JSON: ${resultPath}`);
|
| 59 |
+
for (const check of checks) {
|
| 60 |
+
console.log(`${check.required ? "required" : "supporting"} ${check.status}: ${check.name} - ${check.message}`);
|
| 61 |
+
}
|
| 62 |
+
if (!passed) process.exitCode = 1;
|
| 63 |
+
}
|
| 64 |
+
|
| 65 |
+
function checkExportShape(payload) {
|
| 66 |
+
const rows = rowsFromPayload(payload);
|
| 67 |
+
const hostMetadata = hostMetadataFromPayload(payload);
|
| 68 |
+
const passed =
|
| 69 |
+
typeof generatedAtFromPayload(payload) === "string" &&
|
| 70 |
+
summaryFromPayload(payload) &&
|
| 71 |
+
typeof summaryFromPayload(payload) === "object" &&
|
| 72 |
+
Array.isArray(rows);
|
| 73 |
+
return {
|
| 74 |
+
name: "browser export shape",
|
| 75 |
+
required: true,
|
| 76 |
+
status: passed ? "pass" : "fail",
|
| 77 |
+
message: passed
|
| 78 |
+
? `Export has ${rows.length} benchmark row(s).`
|
| 79 |
+
: "Export is missing generatedAt, summary, or benchmark rows.",
|
| 80 |
+
evidence: {
|
| 81 |
+
generatedAt: generatedAtFromPayload(payload),
|
| 82 |
+
schemaVersion: schemaVersionFromPayload(payload),
|
| 83 |
+
exportId: exportIdFromPayload(payload),
|
| 84 |
+
rowCount: rows.length,
|
| 85 |
+
summaryScopes: summaryFromPayload(payload) ? Object.keys(summaryFromPayload(payload)) : [],
|
| 86 |
+
hostMetadata,
|
| 87 |
+
runtimePresent: Boolean(runtimeFromPayload(payload)),
|
| 88 |
+
evidenceSummaryPresent: Boolean(evidenceSummaryFromPayload(payload)),
|
| 89 |
+
},
|
| 90 |
+
};
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
+
function checkSelfDescription(payload) {
|
| 94 |
+
const schemaVersion = schemaVersionFromPayload(payload);
|
| 95 |
+
const exportId = exportIdFromPayload(payload);
|
| 96 |
+
const guide = evidenceGuideFromPayload(payload);
|
| 97 |
+
const realMic = guide?.requirements?.realMic ?? {};
|
| 98 |
+
const hardwareWebgpu = guide?.requirements?.hardwareWebgpu ?? {};
|
| 99 |
+
const passed =
|
| 100 |
+
typeof schemaVersion === "string" &&
|
| 101 |
+
schemaVersion.startsWith("browser-speak-benchmarks/") &&
|
| 102 |
+
typeof exportId === "string" &&
|
| 103 |
+
exportId.length > 12 &&
|
| 104 |
+
typeof guide?.auditCommand === "string" &&
|
| 105 |
+
guide.auditCommand.includes("audit-browser-evidence.mjs") &&
|
| 106 |
+
realMic.requiredRows >= minRealMicRows &&
|
| 107 |
+
hardwareWebgpu.requiredRows >= 1;
|
| 108 |
+
return {
|
| 109 |
+
name: "browser export self-description",
|
| 110 |
+
required: true,
|
| 111 |
+
status: passed ? "pass" : "fail",
|
| 112 |
+
message: passed
|
| 113 |
+
? `Export ${exportId} declares ${schemaVersion} and embeds audit requirements.`
|
| 114 |
+
: "Export is missing schemaVersion, exportId, or embedded audit requirements.",
|
| 115 |
+
evidence: {
|
| 116 |
+
schemaVersion,
|
| 117 |
+
exportId,
|
| 118 |
+
auditCommand: guide?.auditCommand ?? null,
|
| 119 |
+
realMic,
|
| 120 |
+
hardwareWebgpu,
|
| 121 |
+
nextActions: guide?.nextActions ?? [],
|
| 122 |
+
},
|
| 123 |
+
};
|
| 124 |
+
}
|
| 125 |
+
|
| 126 |
+
function checkBrowserEvidenceSummary(payload) {
|
| 127 |
+
const evidence = evidenceSummaryFromPayload(payload);
|
| 128 |
+
if (!evidence) {
|
| 129 |
+
return {
|
| 130 |
+
name: "browser evidence summary",
|
| 131 |
+
required: false,
|
| 132 |
+
status: "missing",
|
| 133 |
+
message: "Export does not include the page-generated evidence summary.",
|
| 134 |
+
evidence: {},
|
| 135 |
+
};
|
| 136 |
+
}
|
| 137 |
+
return {
|
| 138 |
+
name: "browser evidence summary",
|
| 139 |
+
required: false,
|
| 140 |
+
status: "pass",
|
| 141 |
+
message: summaryEvidenceMessage(evidence),
|
| 142 |
+
evidence,
|
| 143 |
+
};
|
| 144 |
+
}
|
| 145 |
+
|
| 146 |
+
function checkHostedMetadata(payload) {
|
| 147 |
+
const hostMetadata = hostMetadataFromPayload(payload);
|
| 148 |
+
const evidence = evidenceSummaryFromPayload(payload)?.hosted ?? null;
|
| 149 |
+
const host = hostMetadata.host ?? "";
|
| 150 |
+
const hosted = expectedHostSuffix ? host.endsWith(expectedHostSuffix) : Boolean(host);
|
| 151 |
+
const commit = hostMetadata.hfSpaceCommit ?? "";
|
| 152 |
+
const passed = hosted && commit.length >= 7;
|
| 153 |
+
return {
|
| 154 |
+
name: "hosted Space metadata",
|
| 155 |
+
required: true,
|
| 156 |
+
status: passed ? "pass" : "fail",
|
| 157 |
+
message: passed
|
| 158 |
+
? `Export came from ${host} at Space commit ${commit}.`
|
| 159 |
+
: "Export is missing hosted Space host or commit metadata.",
|
| 160 |
+
evidence: { hostMetadata, expectedHostSuffix, browserEvidence: evidence },
|
| 161 |
+
};
|
| 162 |
+
}
|
| 163 |
+
|
| 164 |
+
function checkRealMic(payload) {
|
| 165 |
+
if (!requireRealMic) return skipped("real microphone rows", "Real microphone requirement disabled.");
|
| 166 |
+
const evidence = evidenceSummaryFromPayload(payload)?.realMic ?? null;
|
| 167 |
+
const rows = completedRows(payload).filter((row) => row.kind === "mic");
|
| 168 |
+
const exactRows = rows.filter((row) => row.sttWer === 0).length;
|
| 169 |
+
const identityPasses = rows.filter((row) => row.llmQualityPass === true).length;
|
| 170 |
+
const inputRows = rows.filter(hasRealMicInputStats).length;
|
| 171 |
+
const trackSettingsRows = rows.filter(hasInputTrackSettings).length;
|
| 172 |
+
const medianWer = median(rows.map((row) => row.sttWer));
|
| 173 |
+
const medianSpeechEndToFirstAudioMs = median(rows.map((row) => row.speechEndToFirstAudioMs));
|
| 174 |
+
const passed =
|
| 175 |
+
rows.length >= minRealMicRows &&
|
| 176 |
+
inputRows >= minRealMicRows &&
|
| 177 |
+
identityPasses >= minRealMicRows &&
|
| 178 |
+
Number.isFinite(medianWer) &&
|
| 179 |
+
medianWer <= realMicMaxWer &&
|
| 180 |
+
Number.isFinite(medianSpeechEndToFirstAudioMs);
|
| 181 |
+
return {
|
| 182 |
+
name: "real microphone rows",
|
| 183 |
+
required: true,
|
| 184 |
+
status: passed ? "pass" : "missing",
|
| 185 |
+
message: passed
|
| 186 |
+
? `${rows.length} completed real-mic row(s), median WER ${formatPercent(
|
| 187 |
+
medianWer,
|
| 188 |
+
)}, speech end to first audio ${formatMs(medianSpeechEndToFirstAudioMs)}.`
|
| 189 |
+
: `Need ${minRealMicRows} completed real-mic row(s) with mic input stats, identity pass, median WER <= ${formatPercent(
|
| 190 |
+
realMicMaxWer,
|
| 191 |
+
)}, and finite speech-end-to-audio latency.`,
|
| 192 |
+
evidence: {
|
| 193 |
+
rows: rows.length,
|
| 194 |
+
exactRows,
|
| 195 |
+
identityPasses,
|
| 196 |
+
inputRows,
|
| 197 |
+
trackSettingsRows,
|
| 198 |
+
medianWer,
|
| 199 |
+
medianSpeechEndToFirstAudioMs,
|
| 200 |
+
rowSummaries: rows.slice(0, 8).map((row) => ({
|
| 201 |
+
id: row.id,
|
| 202 |
+
sttWer: row.sttWer,
|
| 203 |
+
transcript: row.transcript,
|
| 204 |
+
llmQualityPass: row.llmQualityPass,
|
| 205 |
+
speechEndToFirstAudioMs: row.speechEndToFirstAudioMs,
|
| 206 |
+
micInputChunks: row.micInputChunks,
|
| 207 |
+
micInputPeak: row.micInputPeak,
|
| 208 |
+
inputSampleRate: row.inputSampleRate ?? row.stack?.environment?.inputSampleRate ?? null,
|
| 209 |
+
inputTrackSettings: row.inputTrackSettings ?? row.stack?.environment?.inputTrackSettings ?? null,
|
| 210 |
+
})),
|
| 211 |
+
browserEvidence: evidence,
|
| 212 |
+
},
|
| 213 |
+
};
|
| 214 |
+
}
|
| 215 |
+
|
| 216 |
+
function checkHardwareWebgpu(payload) {
|
| 217 |
+
if (!requireHardwareWebgpu) return skipped("hardware WebGPU rows", "Hardware WebGPU requirement disabled.");
|
| 218 |
+
const evidence = evidenceSummaryFromPayload(payload)?.hardwareWebgpu ?? null;
|
| 219 |
+
const rows = completedRows(payload).filter((row) => row.stack?.device === "webgpu");
|
| 220 |
+
const hardwareRows = rows.filter((row) => {
|
| 221 |
+
const environment = row.stack?.environment ?? {};
|
| 222 |
+
return (
|
| 223 |
+
environment.webgpuAvailable === true &&
|
| 224 |
+
environment.webgpuSoftwareAdapter !== true &&
|
| 225 |
+
!isSoftwareWebGpuAdapter(environment.webgpuAdapter)
|
| 226 |
+
);
|
| 227 |
+
});
|
| 228 |
+
const passed = hardwareRows.length > 0;
|
| 229 |
+
return {
|
| 230 |
+
name: "hardware WebGPU rows",
|
| 231 |
+
required: true,
|
| 232 |
+
status: passed ? "pass" : "missing",
|
| 233 |
+
message: passed
|
| 234 |
+
? `${hardwareRows.length} completed row(s) used hardware WebGPU.`
|
| 235 |
+
: "No completed benchmark rows prove hardware WebGPU execution.",
|
| 236 |
+
evidence: {
|
| 237 |
+
webgpuRows: rows.length,
|
| 238 |
+
hardwareRows: hardwareRows.length,
|
| 239 |
+
adapters: uniqueAdapters(rows),
|
| 240 |
+
rowKinds: rows.map((row) => row.kind),
|
| 241 |
+
browserEvidence: evidence,
|
| 242 |
+
},
|
| 243 |
+
};
|
| 244 |
+
}
|
| 245 |
+
|
| 246 |
+
function generatedAtFromPayload(payload) {
|
| 247 |
+
return payload?.generatedAt ?? payload?.browserExport?.generatedAt ?? null;
|
| 248 |
+
}
|
| 249 |
+
|
| 250 |
+
function schemaVersionFromPayload(payload) {
|
| 251 |
+
return payload?.schemaVersion ?? payload?.browserExport?.schemaVersion ?? null;
|
| 252 |
+
}
|
| 253 |
+
|
| 254 |
+
function exportIdFromPayload(payload) {
|
| 255 |
+
return payload?.exportId ?? payload?.browserExport?.exportId ?? null;
|
| 256 |
+
}
|
| 257 |
+
|
| 258 |
+
function evidenceGuideFromPayload(payload) {
|
| 259 |
+
return payload?.evidenceGuide ?? payload?.browserExport?.evidenceGuide ?? null;
|
| 260 |
+
}
|
| 261 |
+
|
| 262 |
+
function summaryFromPayload(payload) {
|
| 263 |
+
return payload?.summary ?? payload?.browserExport?.summary ?? payload?.benchmarkSummary ?? null;
|
| 264 |
+
}
|
| 265 |
+
|
| 266 |
+
function hostMetadataFromPayload(payload) {
|
| 267 |
+
return (
|
| 268 |
+
payload?.hostMetadata ??
|
| 269 |
+
payload?.browserExport?.hostMetadata ??
|
| 270 |
+
payload?.runtime?.hostMetadata ??
|
| 271 |
+
payload?.browserExport?.runtime?.hostMetadata ??
|
| 272 |
+
{}
|
| 273 |
+
);
|
| 274 |
+
}
|
| 275 |
+
|
| 276 |
+
function runtimeFromPayload(payload) {
|
| 277 |
+
return payload?.runtime ?? payload?.browserExport?.runtime ?? null;
|
| 278 |
+
}
|
| 279 |
+
|
| 280 |
+
function evidenceSummaryFromPayload(payload) {
|
| 281 |
+
return payload?.evidence ?? payload?.browserExport?.evidence ?? null;
|
| 282 |
+
}
|
| 283 |
+
|
| 284 |
+
function rowsFromPayload(payload) {
|
| 285 |
+
if (Array.isArray(payload?.results)) return payload.results;
|
| 286 |
+
if (Array.isArray(payload?.benchmarkResults)) return payload.benchmarkResults;
|
| 287 |
+
if (Array.isArray(payload?.browserExport?.results)) return payload.browserExport.results;
|
| 288 |
+
return [];
|
| 289 |
+
}
|
| 290 |
+
|
| 291 |
+
function completedRows(payload) {
|
| 292 |
+
return rowsFromPayload(payload).filter((row) => !row.error);
|
| 293 |
+
}
|
| 294 |
+
|
| 295 |
+
function hasRealMicInputStats(row) {
|
| 296 |
+
return (
|
| 297 |
+
Number.isFinite(row.micInputChunks) &&
|
| 298 |
+
row.micInputChunks > 0 &&
|
| 299 |
+
Number.isFinite(row.micInputPeak) &&
|
| 300 |
+
row.micInputPeak > 0 &&
|
| 301 |
+
Number.isFinite(row.inputSampleRate ?? row.stack?.environment?.inputSampleRate)
|
| 302 |
+
);
|
| 303 |
+
}
|
| 304 |
+
|
| 305 |
+
function hasInputTrackSettings(row) {
|
| 306 |
+
const settings = row.inputTrackSettings ?? row.stack?.environment?.inputTrackSettings;
|
| 307 |
+
return Boolean(settings && typeof settings === "object" && Object.keys(settings).length > 0);
|
| 308 |
+
}
|
| 309 |
+
|
| 310 |
+
function uniqueAdapters(rows) {
|
| 311 |
+
const seen = new Set();
|
| 312 |
+
const adapters = [];
|
| 313 |
+
for (const row of rows) {
|
| 314 |
+
const adapter = row.stack?.environment?.webgpuAdapter ?? null;
|
| 315 |
+
const label = formatAdapter(adapter);
|
| 316 |
+
if (seen.has(label)) continue;
|
| 317 |
+
seen.add(label);
|
| 318 |
+
adapters.push({
|
| 319 |
+
label,
|
| 320 |
+
softwareAdapter: row.stack?.environment?.webgpuSoftwareAdapter ?? null,
|
| 321 |
+
adapter,
|
| 322 |
+
});
|
| 323 |
+
}
|
| 324 |
+
return adapters;
|
| 325 |
+
}
|
| 326 |
+
|
| 327 |
+
function isSoftwareWebGpuAdapter(adapter) {
|
| 328 |
+
const text = [adapter?.vendor, adapter?.architecture, adapter?.device, adapter?.description]
|
| 329 |
+
.filter(Boolean)
|
| 330 |
+
.join(" ")
|
| 331 |
+
.toLowerCase();
|
| 332 |
+
return /\b(swiftshader|llvmpipe|software rasterizer|software adapter|warp)\b/.test(text);
|
| 333 |
+
}
|
| 334 |
+
|
| 335 |
+
function formatAdapter(adapter) {
|
| 336 |
+
if (!adapter) return "unknown adapter";
|
| 337 |
+
return (
|
| 338 |
+
[adapter.vendor, adapter.architecture, adapter.device, adapter.description]
|
| 339 |
+
.filter(Boolean)
|
| 340 |
+
.join(" / ") || "unknown adapter"
|
| 341 |
+
);
|
| 342 |
+
}
|
| 343 |
+
|
| 344 |
+
function skipped(name, message) {
|
| 345 |
+
return { name, required: false, status: "skip", message, evidence: {} };
|
| 346 |
+
}
|
| 347 |
+
|
| 348 |
+
function summaryEvidenceMessage(evidence) {
|
| 349 |
+
const parts = [];
|
| 350 |
+
if (evidence.realMic) {
|
| 351 |
+
parts.push(
|
| 352 |
+
`real mic ${evidence.realMic.rows ?? 0}/${evidence.realMic.requiredRows ?? minRealMicRows}`,
|
| 353 |
+
);
|
| 354 |
+
}
|
| 355 |
+
if (evidence.hardwareWebgpu) {
|
| 356 |
+
parts.push(`hardware WebGPU rows ${evidence.hardwareWebgpu.hardwareRows ?? 0}`);
|
| 357 |
+
}
|
| 358 |
+
if (evidence.hosted) {
|
| 359 |
+
parts.push(evidence.hosted.passed ? `hosted ${evidence.hosted.hfSpaceCommit ?? ""}` : "hosted missing");
|
| 360 |
+
}
|
| 361 |
+
return parts.length > 0 ? `Browser summary: ${parts.join(", ")}.` : "Browser evidence summary is present.";
|
| 362 |
+
}
|
| 363 |
+
|
| 364 |
+
function median(values) {
|
| 365 |
+
const finite = values.filter(Number.isFinite).sort((a, b) => a - b);
|
| 366 |
+
if (finite.length === 0) return null;
|
| 367 |
+
const middle = Math.floor(finite.length / 2);
|
| 368 |
+
if (finite.length % 2 === 1) return finite[middle];
|
| 369 |
+
return (finite[middle - 1] + finite[middle]) / 2;
|
| 370 |
+
}
|
| 371 |
+
|
| 372 |
+
function formatMs(value) {
|
| 373 |
+
if (!Number.isFinite(value)) return "-";
|
| 374 |
+
if (value < 1000) return `${Math.round(value)} ms`;
|
| 375 |
+
return `${(value / 1000).toFixed(2)} s`;
|
| 376 |
+
}
|
| 377 |
+
|
| 378 |
+
function formatPercent(value) {
|
| 379 |
+
if (!Number.isFinite(value)) return "-";
|
| 380 |
+
return `${Math.round(value * 100)}%`;
|
| 381 |
+
}
|
| 382 |
+
|
| 383 |
+
main().catch((error) => {
|
| 384 |
+
console.error(error.stack ?? error.message);
|
| 385 |
+
process.exitCode = 1;
|
| 386 |
+
});
|
tools/audit-validation.mjs
ADDED
|
@@ -0,0 +1,439 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env node
|
| 2 |
+
import { readFile, writeFile, access } from "node:fs/promises";
|
| 3 |
+
import { constants } from "node:fs";
|
| 4 |
+
import { tmpdir } from "node:os";
|
| 5 |
+
import { dirname, resolve } from "node:path";
|
| 6 |
+
import { mkdir } from "node:fs/promises";
|
| 7 |
+
import { sourceFingerprint } from "./source-fingerprint.mjs";
|
| 8 |
+
|
| 9 |
+
const paths = {
|
| 10 |
+
clientSide: resolve(process.env.BROWSER_SPEAK_CLIENT_SIDE_JSON ?? `${tmpdir()}/browser-speak-client-side-smoke.json`),
|
| 11 |
+
loopback: resolve(process.env.BROWSER_SPEAK_LOOPBACK_JSON ?? `${tmpdir()}/browser-speak-loopback-series.json`),
|
| 12 |
+
fakeMic: resolve(process.env.BROWSER_SPEAK_FAKE_MIC_JSON ?? `${tmpdir()}/browser-speak-fake-mic-results.json`),
|
| 13 |
+
realMic: resolve(process.env.BROWSER_SPEAK_REAL_MIC_JSON ?? `${tmpdir()}/browser-speak-real-mic-series.json`),
|
| 14 |
+
webgpu: resolve(process.env.BROWSER_SPEAK_WEBGPU_JSON ?? `${tmpdir()}/browser-speak-webgpu-results.json`),
|
| 15 |
+
ui: resolve(process.env.BROWSER_SPEAK_UI_JSON ?? `${tmpdir()}/browser-speak-ui-smoke.json`),
|
| 16 |
+
evidenceExport: resolve(
|
| 17 |
+
process.env.BROWSER_SPEAK_EVIDENCE_EXPORT_JSON ?? `${tmpdir()}/browser-speak-evidence-export-smoke.json`,
|
| 18 |
+
),
|
| 19 |
+
};
|
| 20 |
+
const resultPath = resolve(process.env.BROWSER_SPEAK_AUDIT_JSON ?? `${tmpdir()}/browser-speak-validation-audit.json`);
|
| 21 |
+
const soft = process.env.BROWSER_SPEAK_AUDIT_SOFT === "true";
|
| 22 |
+
const requireRealMic = process.env.BROWSER_SPEAK_AUDIT_REQUIRE_REAL_MIC !== "false";
|
| 23 |
+
const requireHardwareWebgpu = process.env.BROWSER_SPEAK_AUDIT_REQUIRE_HARDWARE_WEBGPU !== "false";
|
| 24 |
+
const realMicMaxWer = Number(process.env.BROWSER_SPEAK_AUDIT_REAL_MIC_MAX_WER ?? 0.25);
|
| 25 |
+
const loopbackMaxMedianWer = Number(process.env.BROWSER_SPEAK_AUDIT_LOOPBACK_MAX_MEDIAN_WER ?? 0);
|
| 26 |
+
let currentSourceFingerprint = null;
|
| 27 |
+
|
| 28 |
+
async function main() {
|
| 29 |
+
currentSourceFingerprint = await sourceFingerprint();
|
| 30 |
+
const artifacts = Object.fromEntries(
|
| 31 |
+
await Promise.all(Object.entries(paths).map(async ([key, path]) => [key, await readJson(path)])),
|
| 32 |
+
);
|
| 33 |
+
const checks = [
|
| 34 |
+
await checkStaticFiles(),
|
| 35 |
+
checkUi(artifacts.ui),
|
| 36 |
+
checkEvidenceExport(artifacts.evidenceExport),
|
| 37 |
+
checkClientSide(artifacts.clientSide),
|
| 38 |
+
checkLoopback(artifacts.loopback),
|
| 39 |
+
checkFirstTtsChunkSafety(artifacts),
|
| 40 |
+
checkFakeMic(artifacts.fakeMic),
|
| 41 |
+
checkRealMic(artifacts.realMic),
|
| 42 |
+
checkHardwareWebgpu(artifacts.webgpu),
|
| 43 |
+
];
|
| 44 |
+
const requiredChecks = checks.filter((check) => check.required);
|
| 45 |
+
const passed = requiredChecks.every((check) => check.status === "pass");
|
| 46 |
+
const payload = {
|
| 47 |
+
generatedAt: new Date().toISOString(),
|
| 48 |
+
sourceFingerprint: currentSourceFingerprint,
|
| 49 |
+
passed,
|
| 50 |
+
paths,
|
| 51 |
+
checks,
|
| 52 |
+
nextActions: nextActions(checks),
|
| 53 |
+
};
|
| 54 |
+
await mkdir(dirname(resultPath), { recursive: true });
|
| 55 |
+
await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`);
|
| 56 |
+
console.log(`Wrote validation audit JSON: ${resultPath}`);
|
| 57 |
+
for (const check of checks) {
|
| 58 |
+
console.log(`${check.required ? "required" : "supporting"} ${check.status}: ${check.name} - ${check.message}`);
|
| 59 |
+
}
|
| 60 |
+
if (!passed && !soft) process.exitCode = 1;
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
async function checkStaticFiles() {
|
| 64 |
+
const files = [
|
| 65 |
+
"index.html",
|
| 66 |
+
"styles.css",
|
| 67 |
+
"app.js",
|
| 68 |
+
"workers/asr-worker.js",
|
| 69 |
+
"workers/llm-worker.js",
|
| 70 |
+
"workers/tts-worker.js",
|
| 71 |
+
];
|
| 72 |
+
const missing = [];
|
| 73 |
+
for (const file of files) {
|
| 74 |
+
if (!(await exists(file))) missing.push(file);
|
| 75 |
+
}
|
| 76 |
+
return {
|
| 77 |
+
name: "demo files",
|
| 78 |
+
required: true,
|
| 79 |
+
status: missing.length === 0 ? "pass" : "fail",
|
| 80 |
+
message: missing.length === 0 ? "Static demo files are present." : `Missing files: ${missing.join(", ")}.`,
|
| 81 |
+
evidence: { files, missing },
|
| 82 |
+
};
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
function checkUi(artifact) {
|
| 86 |
+
if (!artifact.ok) return missing("UI smoke", true, artifact, "UI smoke JSON is missing or unreadable.");
|
| 87 |
+
const freshness = sourceFreshness(artifact);
|
| 88 |
+
if (!freshness.ok) return stale("UI smoke", true, artifact, freshness.message);
|
| 89 |
+
const passed = artifact.value?.passed === true;
|
| 90 |
+
return {
|
| 91 |
+
name: "UI smoke",
|
| 92 |
+
required: true,
|
| 93 |
+
status: passed ? "pass" : "fail",
|
| 94 |
+
message: passed ? "Desktop/mobile UI smoke passed." : "Desktop/mobile UI smoke has not passed.",
|
| 95 |
+
evidence: pick(artifact.value, ["passed", "config", "errors"]),
|
| 96 |
+
};
|
| 97 |
+
}
|
| 98 |
+
|
| 99 |
+
function checkEvidenceExport(artifact) {
|
| 100 |
+
if (!artifact.ok) return missing("evidence export smoke", true, artifact, "Evidence export smoke JSON is missing.");
|
| 101 |
+
const freshness = sourceFreshness(artifact);
|
| 102 |
+
if (!freshness.ok) return stale("evidence export smoke", true, artifact, freshness.message);
|
| 103 |
+
const summary = artifact.value?.summary ?? {};
|
| 104 |
+
const passed =
|
| 105 |
+
artifact.value?.passed === true &&
|
| 106 |
+
summary.restoredRows >= 1 &&
|
| 107 |
+
summary.exportedRows >= 1 &&
|
| 108 |
+
/^browser-speak-evidence-.*\.json$/.test(summary.downloadName ?? "") &&
|
| 109 |
+
summary.downloadHrefScheme === "blob" &&
|
| 110 |
+
summary.clearedRows === 0 &&
|
| 111 |
+
summary.persistedAfterClear === null;
|
| 112 |
+
return {
|
| 113 |
+
name: "evidence export smoke",
|
| 114 |
+
required: true,
|
| 115 |
+
status: passed ? "pass" : "fail",
|
| 116 |
+
message: passed
|
| 117 |
+
? "Autosaved rows restore, evidence JSON download is requested, and Clear removes saved rows."
|
| 118 |
+
: "Evidence export, autosave restore, or Clear behavior did not pass.",
|
| 119 |
+
evidence: pick(artifact.value, ["passed", "summary", "error", "config"]),
|
| 120 |
+
};
|
| 121 |
+
}
|
| 122 |
+
|
| 123 |
+
function checkClientSide(artifact) {
|
| 124 |
+
if (!artifact.ok) return missing("client-side/no-server smoke", true, artifact, "Client-side smoke JSON is missing.");
|
| 125 |
+
const freshness = sourceFreshness(artifact);
|
| 126 |
+
if (!freshness.ok) return stale("client-side/no-server smoke", true, artifact, freshness.message);
|
| 127 |
+
const summary = artifact.value?.summary ?? {};
|
| 128 |
+
const passed =
|
| 129 |
+
artifact.value?.passed === true &&
|
| 130 |
+
summary.benchmarkRequests === 0 &&
|
| 131 |
+
summary.serverInferenceSuspects === 0 &&
|
| 132 |
+
summary.benchmarkErrors === 0 &&
|
| 133 |
+
Array.isArray(summary.missingTasks) &&
|
| 134 |
+
summary.missingTasks.length === 0;
|
| 135 |
+
return {
|
| 136 |
+
name: "client-side/no-server smoke",
|
| 137 |
+
required: true,
|
| 138 |
+
status: passed ? "pass" : "fail",
|
| 139 |
+
message: passed
|
| 140 |
+
? "Benchmark phase had no network requests, server-inference suspects, row errors, or missing tasks."
|
| 141 |
+
: "Client-side smoke did not prove the no-server benchmark phase.",
|
| 142 |
+
evidence: pick(artifact.value, ["passed", "summary", "error", "config"]),
|
| 143 |
+
};
|
| 144 |
+
}
|
| 145 |
+
|
| 146 |
+
function checkLoopback(artifact) {
|
| 147 |
+
if (!artifact.ok) return missing("loopback stability", true, artifact, "Loopback series JSON is missing.");
|
| 148 |
+
const freshness = sourceFreshness(artifact);
|
| 149 |
+
if (!freshness.ok) return stale("loopback stability", true, artifact, freshness.message);
|
| 150 |
+
const summary = artifact.value?.summary ?? {};
|
| 151 |
+
const passed =
|
| 152 |
+
artifact.value?.passed === true &&
|
| 153 |
+
summary.completedRuns >= 3 &&
|
| 154 |
+
summary.errorRuns === 0 &&
|
| 155 |
+
summary.exactTranscriptRuns >= Math.ceil(summary.completedRuns / 2) &&
|
| 156 |
+
summary.identityPasses >= summary.exactTranscriptRuns &&
|
| 157 |
+
Number.isFinite(summary.medianWer) &&
|
| 158 |
+
summary.medianWer <= loopbackMaxMedianWer &&
|
| 159 |
+
Number.isFinite(summary.medianSpeechEndToFirstAudioMs);
|
| 160 |
+
return {
|
| 161 |
+
name: "loopback stability",
|
| 162 |
+
required: true,
|
| 163 |
+
status: passed ? "pass" : "fail",
|
| 164 |
+
message: passed
|
| 165 |
+
? `Synthetic loopback completed 3 rows with median WER ${formatPercent(
|
| 166 |
+
summary.medianWer,
|
| 167 |
+
)} and ${summary.exactTranscriptRuns}/${summary.completedRuns} exact transcripts.`
|
| 168 |
+
: "Loopback stability evidence is missing or below threshold.",
|
| 169 |
+
evidence: pick(artifact.value, ["passed", "summary", "config", "error"]),
|
| 170 |
+
};
|
| 171 |
+
}
|
| 172 |
+
|
| 173 |
+
function checkFirstTtsChunkSafety(artifacts) {
|
| 174 |
+
const rows = [
|
| 175 |
+
...rowsFromArtifact(artifacts.clientSide, "client-side smoke"),
|
| 176 |
+
...rowsFromArtifact(artifacts.loopback, "loopback series"),
|
| 177 |
+
...rowsFromArtifact(artifacts.fakeMic, "fake microphone"),
|
| 178 |
+
...rowsFromArtifact(artifacts.realMic, "real microphone"),
|
| 179 |
+
].filter(({ row }) => !row.error && row.firstTtsText);
|
| 180 |
+
if (rows.length === 0) {
|
| 181 |
+
return {
|
| 182 |
+
name: "first TTS chunk safety",
|
| 183 |
+
required: true,
|
| 184 |
+
status: "fail",
|
| 185 |
+
message: "No completed benchmark rows with first TTS chunk text were available.",
|
| 186 |
+
evidence: {},
|
| 187 |
+
};
|
| 188 |
+
}
|
| 189 |
+
const unsafe = rows.filter(({ row }) => unsafeFirstTtsChunk(row));
|
| 190 |
+
return {
|
| 191 |
+
name: "first TTS chunk safety",
|
| 192 |
+
required: true,
|
| 193 |
+
status: unsafe.length === 0 ? "pass" : "fail",
|
| 194 |
+
message:
|
| 195 |
+
unsafe.length === 0
|
| 196 |
+
? `${rows.length} first TTS chunk(s) ended at a safe boundary.`
|
| 197 |
+
: `${unsafe.length}/${rows.length} first TTS chunk(s) appear to split a word.`,
|
| 198 |
+
evidence: {
|
| 199 |
+
checkedRows: rows.length,
|
| 200 |
+
unsafe: unsafe.slice(0, 8).map(({ source, row }) => ({
|
| 201 |
+
source,
|
| 202 |
+
kind: row.kind,
|
| 203 |
+
firstTtsText: row.firstTtsText,
|
| 204 |
+
output: row.output,
|
| 205 |
+
firstTtsBoundaryKind: row.firstTtsBoundaryKind ?? null,
|
| 206 |
+
firstTtsWordBoundarySafe: row.firstTtsWordBoundarySafe ?? null,
|
| 207 |
+
})),
|
| 208 |
+
},
|
| 209 |
+
};
|
| 210 |
+
}
|
| 211 |
+
|
| 212 |
+
function checkFakeMic(artifact) {
|
| 213 |
+
if (!artifact.ok) return missing("fake microphone regression", false, artifact, "Fake mic JSON is missing.");
|
| 214 |
+
const freshness = sourceFreshness(artifact);
|
| 215 |
+
const summary = artifact.value?.summary?.current ?? artifact.value?.summary?.all ?? {};
|
| 216 |
+
const passed =
|
| 217 |
+
freshness.ok &&
|
| 218 |
+
summary.micRuns >= 3 &&
|
| 219 |
+
summary.micMedianWer === 0 &&
|
| 220 |
+
Number.isFinite(summary.micMedianSpeechEndToFirstAudioMs);
|
| 221 |
+
return {
|
| 222 |
+
name: "fake microphone regression",
|
| 223 |
+
required: false,
|
| 224 |
+
status: passed ? "pass" : "warn",
|
| 225 |
+
message: passed
|
| 226 |
+
? "Fake mic browser-capture regression has 3 rows and 0% median WER."
|
| 227 |
+
: freshness.ok
|
| 228 |
+
? "Fake mic regression is unavailable or below the expected threshold."
|
| 229 |
+
: freshness.message,
|
| 230 |
+
evidence: { summary, freshness },
|
| 231 |
+
};
|
| 232 |
+
}
|
| 233 |
+
|
| 234 |
+
function checkRealMic(artifact) {
|
| 235 |
+
if (!requireRealMic) return skipped("real microphone validation", "Real mic requirement disabled by environment.");
|
| 236 |
+
if (!artifact.ok) return missing("real microphone validation", true, artifact, "Real human microphone JSON is missing.");
|
| 237 |
+
if (artifact.value?.dryRun) {
|
| 238 |
+
return {
|
| 239 |
+
name: "real microphone validation",
|
| 240 |
+
required: true,
|
| 241 |
+
status: "missing",
|
| 242 |
+
message: "Only a real-mic dry-run artifact exists; no human speech rows were collected.",
|
| 243 |
+
evidence: pick(artifact.value, ["dryRun", "config", "preflight"]),
|
| 244 |
+
};
|
| 245 |
+
}
|
| 246 |
+
const freshness = sourceFreshness(artifact);
|
| 247 |
+
if (!freshness.ok) return stale("real microphone validation", true, artifact, freshness.message);
|
| 248 |
+
const summary = artifact.value?.summary ?? {};
|
| 249 |
+
const config = artifact.value?.config ?? {};
|
| 250 |
+
const expected = artifact.value?.config?.count ?? 3;
|
| 251 |
+
const extraChromeArgs = Array.isArray(config.extraChromeArgs) ? config.extraChromeArgs : [];
|
| 252 |
+
const fakeCaptureArgs = extraChromeArgs.filter(isFakeCaptureArg);
|
| 253 |
+
const fakeCaptureAllowed = config.allowFakeCapture === true;
|
| 254 |
+
const passed =
|
| 255 |
+
artifact.value?.passed === true &&
|
| 256 |
+
fakeCaptureArgs.length === 0 &&
|
| 257 |
+
!fakeCaptureAllowed &&
|
| 258 |
+
summary.completedRows >= expected &&
|
| 259 |
+
summary.errorRows === 0 &&
|
| 260 |
+
summary.identityPasses >= expected &&
|
| 261 |
+
Number.isFinite(summary.medianSpeechEndToFirstAudioMs) &&
|
| 262 |
+
Number.isFinite(summary.medianWer) &&
|
| 263 |
+
summary.medianWer <= realMicMaxWer;
|
| 264 |
+
return {
|
| 265 |
+
name: "real microphone validation",
|
| 266 |
+
required: true,
|
| 267 |
+
status: passed ? "pass" : "fail",
|
| 268 |
+
message: realMicMessage(passed, fakeCaptureArgs, fakeCaptureAllowed),
|
| 269 |
+
evidence: { ...pick(artifact.value, ["passed", "summary", "config", "error"]), fakeCaptureArgs },
|
| 270 |
+
};
|
| 271 |
+
}
|
| 272 |
+
|
| 273 |
+
function checkHardwareWebgpu(artifact) {
|
| 274 |
+
if (!requireHardwareWebgpu) return skipped("hardware WebGPU benchmark", "Hardware WebGPU requirement disabled by environment.");
|
| 275 |
+
if (!artifact.ok) return missing("hardware WebGPU benchmark", true, artifact, "WebGPU benchmark JSON is missing.");
|
| 276 |
+
const freshness = sourceFreshness(artifact);
|
| 277 |
+
if (!freshness.ok) return stale("hardware WebGPU benchmark", true, artifact, freshness.message);
|
| 278 |
+
const webgpu = artifact.value?.webgpu ?? {};
|
| 279 |
+
const candidates = artifact.value?.candidates ?? [];
|
| 280 |
+
const completeCandidates = candidates.filter((candidate) => candidate.status === "complete");
|
| 281 |
+
const passed =
|
| 282 |
+
artifact.value?.skipped !== true &&
|
| 283 |
+
webgpu.available === true &&
|
| 284 |
+
webgpu.softwareAdapter !== true &&
|
| 285 |
+
completeCandidates.length > 0;
|
| 286 |
+
return {
|
| 287 |
+
name: "hardware WebGPU benchmark",
|
| 288 |
+
required: true,
|
| 289 |
+
status: passed ? "pass" : "missing",
|
| 290 |
+
message: passed
|
| 291 |
+
? "Hardware WebGPU benchmark completed at least one candidate."
|
| 292 |
+
: artifact.value?.reason ?? "Hardware WebGPU benchmark has not completed on a real adapter.",
|
| 293 |
+
evidence: pick(artifact.value, ["skipped", "reason", "webgpu", "candidates", "summary"]),
|
| 294 |
+
};
|
| 295 |
+
}
|
| 296 |
+
|
| 297 |
+
function skipped(name, message) {
|
| 298 |
+
return { name, required: false, status: "skip", message, evidence: {} };
|
| 299 |
+
}
|
| 300 |
+
|
| 301 |
+
function missing(name, required, artifact, message) {
|
| 302 |
+
return {
|
| 303 |
+
name,
|
| 304 |
+
required,
|
| 305 |
+
status: "missing",
|
| 306 |
+
message,
|
| 307 |
+
evidence: { path: artifact.path, error: artifact.error },
|
| 308 |
+
};
|
| 309 |
+
}
|
| 310 |
+
|
| 311 |
+
function stale(name, required, artifact, message) {
|
| 312 |
+
return {
|
| 313 |
+
name,
|
| 314 |
+
required,
|
| 315 |
+
status: "stale",
|
| 316 |
+
message,
|
| 317 |
+
evidence: {
|
| 318 |
+
path: artifact.path,
|
| 319 |
+
artifactHash: artifact.value?.sourceFingerprint?.hash ?? null,
|
| 320 |
+
currentHash: currentSourceFingerprint?.hash ?? null,
|
| 321 |
+
},
|
| 322 |
+
};
|
| 323 |
+
}
|
| 324 |
+
|
| 325 |
+
function nextActions(checks) {
|
| 326 |
+
const missingNames = checks
|
| 327 |
+
.filter((check) => check.required && check.status !== "pass")
|
| 328 |
+
.map((check) => check.name);
|
| 329 |
+
const actions = [];
|
| 330 |
+
if (missingNames.includes("real microphone validation") || missingNames.includes("hardware WebGPU benchmark")) {
|
| 331 |
+
actions.push(
|
| 332 |
+
"Run node tools/run-hosted-evidence-capture.mjs from a desktop Chrome session with a real microphone and hardware WebGPU to collect hosted browser evidence.",
|
| 333 |
+
);
|
| 334 |
+
}
|
| 335 |
+
return checks
|
| 336 |
+
.filter((check) => check.required && check.status !== "pass")
|
| 337 |
+
.map((check) => {
|
| 338 |
+
if (check.name === "real microphone validation") return "Run node tools/run-real-mic-series.mjs on a machine with a real microphone.";
|
| 339 |
+
if (check.name === "hardware WebGPU benchmark") return "Run node tools/run-webgpu-benchmark.mjs in a Chrome session exposing a hardware WebGPU adapter.";
|
| 340 |
+
if (check.name === "client-side/no-server smoke") return "Run node tools/run-client-side-smoke.mjs and inspect /tmp/browser-speak-client-side-smoke.json.";
|
| 341 |
+
if (check.name === "evidence export smoke") return "Run node tools/run-evidence-export-smoke.mjs and inspect /tmp/browser-speak-evidence-export-smoke.json.";
|
| 342 |
+
if (check.name === "loopback stability") return "Run node tools/run-loopback-series.mjs and inspect /tmp/browser-speak-loopback-series.json.";
|
| 343 |
+
if (check.name === "first TTS chunk safety") return "Run the client-side smoke and inspect firstTtsText / firstTtsWordBoundarySafe in benchmark rows.";
|
| 344 |
+
if (check.name === "UI smoke") return "Run node tools/run-ui-smoke.mjs.";
|
| 345 |
+
return `Resolve failed check: ${check.name}.`;
|
| 346 |
+
})
|
| 347 |
+
.reduce((unique, action) => {
|
| 348 |
+
if (!unique.includes(action)) unique.push(action);
|
| 349 |
+
return unique;
|
| 350 |
+
}, actions);
|
| 351 |
+
}
|
| 352 |
+
|
| 353 |
+
async function readJson(path) {
|
| 354 |
+
try {
|
| 355 |
+
return { ok: true, path, value: JSON.parse(await readFile(path, "utf8")) };
|
| 356 |
+
} catch (error) {
|
| 357 |
+
return { ok: false, path, error: error.message };
|
| 358 |
+
}
|
| 359 |
+
}
|
| 360 |
+
|
| 361 |
+
async function exists(path) {
|
| 362 |
+
try {
|
| 363 |
+
await access(path, constants.F_OK);
|
| 364 |
+
return true;
|
| 365 |
+
} catch {
|
| 366 |
+
return false;
|
| 367 |
+
}
|
| 368 |
+
}
|
| 369 |
+
|
| 370 |
+
function pick(value, keys) {
|
| 371 |
+
if (!value || typeof value !== "object") return {};
|
| 372 |
+
return Object.fromEntries(keys.filter((key) => key in value).map((key) => [key, value[key]]));
|
| 373 |
+
}
|
| 374 |
+
|
| 375 |
+
function realMicMessage(passed, fakeCaptureArgs, fakeCaptureAllowed) {
|
| 376 |
+
if (passed) return "Real human microphone rows meet count, WER, latency, and identity gates.";
|
| 377 |
+
if (fakeCaptureArgs.length > 0) {
|
| 378 |
+
return `Real microphone evidence used fake-capture Chrome args: ${fakeCaptureArgs.join(", ")}.`;
|
| 379 |
+
}
|
| 380 |
+
if (fakeCaptureAllowed) return "Real microphone evidence was collected with fake capture explicitly allowed.";
|
| 381 |
+
return "Real human microphone rows are missing or below threshold.";
|
| 382 |
+
}
|
| 383 |
+
|
| 384 |
+
function sourceFreshness(artifact) {
|
| 385 |
+
const artifactHash = artifact.value?.sourceFingerprint?.hash;
|
| 386 |
+
if (!artifactHash) {
|
| 387 |
+
return { ok: false, message: "Artifact is missing source fingerprint metadata; rerun the harness." };
|
| 388 |
+
}
|
| 389 |
+
if (artifactHash !== currentSourceFingerprint?.hash) {
|
| 390 |
+
return { ok: false, message: "Artifact source fingerprint is stale; rerun the harness." };
|
| 391 |
+
}
|
| 392 |
+
return { ok: true, message: "Artifact source fingerprint matches current files." };
|
| 393 |
+
}
|
| 394 |
+
|
| 395 |
+
function rowsFromArtifact(artifact, source) {
|
| 396 |
+
if (!artifact.ok || !artifact.value) return [];
|
| 397 |
+
const rows = [];
|
| 398 |
+
for (const key of ["benchmarkResults", "rows", "results"]) {
|
| 399 |
+
if (Array.isArray(artifact.value[key])) {
|
| 400 |
+
rows.push(...artifact.value[key].map((row) => ({ source, row })));
|
| 401 |
+
}
|
| 402 |
+
}
|
| 403 |
+
return rows;
|
| 404 |
+
}
|
| 405 |
+
|
| 406 |
+
function unsafeFirstTtsChunk(row) {
|
| 407 |
+
return row.firstTtsWordBoundarySafe === false || firstTtsLooksMidWord(row);
|
| 408 |
+
}
|
| 409 |
+
|
| 410 |
+
function firstTtsLooksMidWord(row) {
|
| 411 |
+
const chunk = String(row.firstTtsText ?? "").trim();
|
| 412 |
+
const output = String(row.output ?? "").trim();
|
| 413 |
+
if (!chunk || !output.startsWith(chunk)) return false;
|
| 414 |
+
const before = chunk.at(-1) ?? "";
|
| 415 |
+
const after = output[chunk.length] ?? "";
|
| 416 |
+
return isWordLikeChar(before) && isWordLikeChar(after);
|
| 417 |
+
}
|
| 418 |
+
|
| 419 |
+
function isFakeCaptureArg(arg) {
|
| 420 |
+
return (
|
| 421 |
+
arg === "--use-fake-device-for-media-stream" ||
|
| 422 |
+
arg.startsWith("--use-file-for-fake-audio-capture=") ||
|
| 423 |
+
arg.startsWith("--use-file-for-fake-video-capture=")
|
| 424 |
+
);
|
| 425 |
+
}
|
| 426 |
+
|
| 427 |
+
function isWordLikeChar(char) {
|
| 428 |
+
return /[A-Za-z0-9']/.test(char);
|
| 429 |
+
}
|
| 430 |
+
|
| 431 |
+
function formatPercent(value) {
|
| 432 |
+
if (!Number.isFinite(value)) return "-";
|
| 433 |
+
return `${Math.round(value * 100)}%`;
|
| 434 |
+
}
|
| 435 |
+
|
| 436 |
+
main().catch((error) => {
|
| 437 |
+
console.error(error.stack ?? error.message);
|
| 438 |
+
process.exitCode = 1;
|
| 439 |
+
});
|
tools/run-client-side-smoke.mjs
ADDED
|
@@ -0,0 +1,645 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env node
|
| 2 |
+
import { spawn } from "node:child_process";
|
| 3 |
+
import { existsSync } from "node:fs";
|
| 4 |
+
import { mkdir, rm, writeFile } from "node:fs/promises";
|
| 5 |
+
import { tmpdir } from "node:os";
|
| 6 |
+
import { dirname, resolve } from "node:path";
|
| 7 |
+
import { sourceFingerprint } from "./source-fingerprint.mjs";
|
| 8 |
+
|
| 9 |
+
const url = process.env.BROWSER_SPEAK_URL ?? "http://127.0.0.1:5174/";
|
| 10 |
+
const chrome =
|
| 11 |
+
process.env.CHROME_BIN ??
|
| 12 |
+
(existsSync("/opt/google/chrome/chrome")
|
| 13 |
+
? "/opt/google/chrome/chrome"
|
| 14 |
+
: existsSync("/usr/bin/google-chrome")
|
| 15 |
+
? "/usr/bin/google-chrome"
|
| 16 |
+
: "chromium");
|
| 17 |
+
const resultPath = resolve(
|
| 18 |
+
process.env.BROWSER_SPEAK_CLIENT_SIDE_JSON ?? `${tmpdir()}/browser-speak-client-side-smoke.json`,
|
| 19 |
+
);
|
| 20 |
+
const profileDir = resolve(
|
| 21 |
+
process.env.BROWSER_SPEAK_CLIENT_SIDE_PROFILE_DIR ?? `${tmpdir()}/browser-speak-client-side-profile`,
|
| 22 |
+
);
|
| 23 |
+
const protocolTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_TIMEOUT_MS ?? 60000);
|
| 24 |
+
const pollTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_POLL_TIMEOUT_MS ?? 5000);
|
| 25 |
+
const pageUnresponsiveTimeoutMs = Number(process.env.BROWSER_SPEAK_PAGE_UNRESPONSIVE_TIMEOUT_MS ?? 120000);
|
| 26 |
+
const loadTimeoutMs = Number(process.env.BROWSER_SPEAK_LOAD_TIMEOUT_MS ?? 600000);
|
| 27 |
+
const taskTimeoutMs = Number(process.env.BROWSER_SPEAK_TASK_TIMEOUT_MS ?? 180000);
|
| 28 |
+
const voicePreloadTimeoutMs = Number(
|
| 29 |
+
process.env.BROWSER_SPEAK_VOICE_PRELOAD_TIMEOUT_MS ?? Math.max(180000, taskTimeoutMs),
|
| 30 |
+
);
|
| 31 |
+
const reuseProfile = process.env.BROWSER_SPEAK_CLIENT_SIDE_REUSE_PROFILE === "true";
|
| 32 |
+
const ttsWarmup = process.env.BROWSER_SPEAK_TTS_WARMUP === "true";
|
| 33 |
+
const headless = process.env.BROWSER_SPEAK_HEADLESS !== "false";
|
| 34 |
+
const tasks = parseList(process.env.BROWSER_SPEAK_CLIENT_SIDE_TASKS ?? "tts,identity,loopback");
|
| 35 |
+
const stack = {
|
| 36 |
+
device: process.env.BROWSER_SPEAK_DEVICE ?? "wasm",
|
| 37 |
+
llm: process.env.BROWSER_SPEAK_LLM ?? "HuggingFaceTB/SmolLM2-135M-Instruct",
|
| 38 |
+
asr: process.env.BROWSER_SPEAK_ASR ?? "onnx-community/moonshine-base-ONNX",
|
| 39 |
+
voice: process.env.BROWSER_SPEAK_VOICE ?? "F2",
|
| 40 |
+
ttsSteps: Number(process.env.BROWSER_SPEAK_TTS_STEPS ?? 2),
|
| 41 |
+
vadSilenceMs: Number(process.env.BROWSER_SPEAK_VAD_SILENCE_MS ?? 480),
|
| 42 |
+
partialAsr: process.env.BROWSER_SPEAK_PARTIAL_ASR !== "false",
|
| 43 |
+
ttsWarmup,
|
| 44 |
+
};
|
| 45 |
+
|
| 46 |
+
async function main() {
|
| 47 |
+
await ensureServer();
|
| 48 |
+
await mkdir(dirname(resultPath), { recursive: true });
|
| 49 |
+
if (!reuseProfile) await rm(profileDir, { recursive: true, force: true });
|
| 50 |
+
|
| 51 |
+
const browser = launchBrowser(9346, profileDir);
|
| 52 |
+
const recorder = new NetworkRecorder(url);
|
| 53 |
+
let client = null;
|
| 54 |
+
try {
|
| 55 |
+
client = await connectToPage(9346, browser);
|
| 56 |
+
client.on("Network.requestWillBeSent", (event) => recorder.requestWillBeSent(event));
|
| 57 |
+
client.on("Network.responseReceived", (event) => recorder.responseReceived(event));
|
| 58 |
+
client.on("Network.loadingFailed", (event) => recorder.loadingFailed(event));
|
| 59 |
+
await client.call("Network.enable");
|
| 60 |
+
await waitForBenchApi(client);
|
| 61 |
+
|
| 62 |
+
recorder.markStage("load");
|
| 63 |
+
await runPageTask(client, `window.browserSpeakBench.loadStack(${JSON.stringify(stack)})`, {
|
| 64 |
+
label: "model load",
|
| 65 |
+
timeoutMs: loadTimeoutMs,
|
| 66 |
+
});
|
| 67 |
+
await runPageTask(
|
| 68 |
+
client,
|
| 69 |
+
`Promise.all(${JSON.stringify(["F1", "F2", "M1", "M2"])}.map((voice) => window.browserSpeakBench.preloadVoice({ voice, timeoutMs: ${JSON.stringify(voicePreloadTimeoutMs)} })))`,
|
| 70 |
+
{ label: "voice preload", timeoutMs: voicePreloadTimeoutMs },
|
| 71 |
+
);
|
| 72 |
+
await runPageTask(client, "window.browserSpeakBench.clearResults()", {
|
| 73 |
+
label: "clear restored benchmark rows",
|
| 74 |
+
timeoutMs: 30000,
|
| 75 |
+
});
|
| 76 |
+
await sleep(1000);
|
| 77 |
+
|
| 78 |
+
recorder.markStage("benchmark");
|
| 79 |
+
for (const task of tasks) {
|
| 80 |
+
await runBenchmarkTask(client, task);
|
| 81 |
+
}
|
| 82 |
+
await sleep(1500);
|
| 83 |
+
const snapshot = await runPageTask(client, "window.browserSpeakBench.exportResults()", {
|
| 84 |
+
label: "export results",
|
| 85 |
+
timeoutMs: 30000,
|
| 86 |
+
});
|
| 87 |
+
|
| 88 |
+
const requests = recorder.requests();
|
| 89 |
+
const benchmarkRequests = requests.filter((request) => request.stage === "benchmark");
|
| 90 |
+
const suspects = benchmarkRequests.flatMap((request) =>
|
| 91 |
+
classifyRequest(request).map((reason) => ({ reason, request })),
|
| 92 |
+
);
|
| 93 |
+
const benchmarkErrors = snapshot.results.filter((result) => result.error);
|
| 94 |
+
const missingTasks = missingTaskRows(snapshot.results, tasks);
|
| 95 |
+
const payload = {
|
| 96 |
+
generatedAt: new Date().toISOString(),
|
| 97 |
+
sourceFingerprint: await sourceFingerprint(),
|
| 98 |
+
url,
|
| 99 |
+
passed: suspects.length === 0 && benchmarkErrors.length === 0 && missingTasks.length === 0,
|
| 100 |
+
config: {
|
| 101 |
+
stack,
|
| 102 |
+
tasks,
|
| 103 |
+
loadTimeoutMs,
|
| 104 |
+
taskTimeoutMs,
|
| 105 |
+
voicePreloadTimeoutMs,
|
| 106 |
+
protocolTimeoutMs,
|
| 107 |
+
pollTimeoutMs,
|
| 108 |
+
pageUnresponsiveTimeoutMs,
|
| 109 |
+
reuseProfile,
|
| 110 |
+
headless,
|
| 111 |
+
chrome,
|
| 112 |
+
extraChromeArgs: parseChromeArgs(),
|
| 113 |
+
},
|
| 114 |
+
summary: {
|
| 115 |
+
totalRequests: requests.length,
|
| 116 |
+
loadRequests: requests.filter((request) => request.stage === "load").length,
|
| 117 |
+
benchmarkRequests: benchmarkRequests.length,
|
| 118 |
+
benchmarkHosts: summarizeHosts(benchmarkRequests),
|
| 119 |
+
benchmarkMethods: summarizeMethods(benchmarkRequests),
|
| 120 |
+
serverInferenceSuspects: suspects.length,
|
| 121 |
+
benchmarkErrors: benchmarkErrors.length,
|
| 122 |
+
missingTasks,
|
| 123 |
+
},
|
| 124 |
+
suspects,
|
| 125 |
+
benchmarkErrors,
|
| 126 |
+
requests,
|
| 127 |
+
browserExport: snapshot,
|
| 128 |
+
browserExportMetadata: exportMetadata(snapshot),
|
| 129 |
+
hostMetadata: snapshot.hostMetadata ?? null,
|
| 130 |
+
benchmarkResults: snapshot.results,
|
| 131 |
+
benchmarkSummary: snapshot.summary,
|
| 132 |
+
};
|
| 133 |
+
await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`);
|
| 134 |
+
console.log(`Wrote client-side smoke JSON: ${resultPath}`);
|
| 135 |
+
console.log(
|
| 136 |
+
`Benchmark network requests: ${benchmarkRequests.length}, server-inference suspects: ${suspects.length}.`,
|
| 137 |
+
);
|
| 138 |
+
for (const suspect of suspects.slice(0, 8)) {
|
| 139 |
+
console.log(`${suspect.reason}: ${suspect.request.method} ${suspect.request.url}`);
|
| 140 |
+
}
|
| 141 |
+
for (const errorRow of benchmarkErrors.slice(0, 8)) {
|
| 142 |
+
console.log(`${errorRow.kind} benchmark error: ${errorRow.error}`);
|
| 143 |
+
}
|
| 144 |
+
for (const missingTask of missingTasks) {
|
| 145 |
+
console.log(`Missing benchmark row for task: ${missingTask}`);
|
| 146 |
+
}
|
| 147 |
+
if (!payload.passed) process.exitCode = 1;
|
| 148 |
+
await client.closeBrowser();
|
| 149 |
+
} catch (error) {
|
| 150 |
+
await writeFailurePayload(error, recorder, browser).catch((writeError) => {
|
| 151 |
+
console.error(`Could not write client-side smoke failure JSON: ${writeError.message}`);
|
| 152 |
+
});
|
| 153 |
+
if (client) await client.closeBrowser().catch(() => {});
|
| 154 |
+
throw error;
|
| 155 |
+
} finally {
|
| 156 |
+
await stopBrowser(browser, reuseProfile ? null : profileDir);
|
| 157 |
+
}
|
| 158 |
+
}
|
| 159 |
+
|
| 160 |
+
async function writeFailurePayload(error, recorder, browser = null) {
|
| 161 |
+
const requests = recorder.requests();
|
| 162 |
+
const benchmarkRequests = requests.filter((request) => request.stage === "benchmark");
|
| 163 |
+
const suspects = benchmarkRequests.flatMap((request) =>
|
| 164 |
+
classifyRequest(request).map((reason) => ({ reason, request })),
|
| 165 |
+
);
|
| 166 |
+
const payload = {
|
| 167 |
+
generatedAt: new Date().toISOString(),
|
| 168 |
+
sourceFingerprint: await sourceFingerprint(),
|
| 169 |
+
url,
|
| 170 |
+
passed: false,
|
| 171 |
+
error: error.stack ?? error.message ?? String(error),
|
| 172 |
+
browserLog: browser?.browserLog ?? "",
|
| 173 |
+
config: {
|
| 174 |
+
stack,
|
| 175 |
+
tasks,
|
| 176 |
+
loadTimeoutMs,
|
| 177 |
+
taskTimeoutMs,
|
| 178 |
+
voicePreloadTimeoutMs,
|
| 179 |
+
protocolTimeoutMs,
|
| 180 |
+
pollTimeoutMs,
|
| 181 |
+
pageUnresponsiveTimeoutMs,
|
| 182 |
+
reuseProfile,
|
| 183 |
+
headless,
|
| 184 |
+
chrome,
|
| 185 |
+
extraChromeArgs: parseChromeArgs(),
|
| 186 |
+
},
|
| 187 |
+
summary: {
|
| 188 |
+
totalRequests: requests.length,
|
| 189 |
+
loadRequests: requests.filter((request) => request.stage === "load").length,
|
| 190 |
+
benchmarkRequests: benchmarkRequests.length,
|
| 191 |
+
benchmarkHosts: summarizeHosts(benchmarkRequests),
|
| 192 |
+
benchmarkMethods: summarizeMethods(benchmarkRequests),
|
| 193 |
+
serverInferenceSuspects: suspects.length,
|
| 194 |
+
benchmarkErrors: null,
|
| 195 |
+
missingTasks: [],
|
| 196 |
+
},
|
| 197 |
+
suspects,
|
| 198 |
+
requests,
|
| 199 |
+
};
|
| 200 |
+
await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`);
|
| 201 |
+
console.log(`Wrote client-side smoke failure JSON: ${resultPath}`);
|
| 202 |
+
}
|
| 203 |
+
|
| 204 |
+
async function runBenchmarkTask(client, task) {
|
| 205 |
+
const normalized = task.toLowerCase();
|
| 206 |
+
const loopbackOptions = {
|
| 207 |
+
timeoutMs: 180000,
|
| 208 |
+
...(process.env.BROWSER_SPEAK_LOOPBACK_TEXT ? { text: process.env.BROWSER_SPEAK_LOOPBACK_TEXT } : {}),
|
| 209 |
+
...(Number.isFinite(Number(process.env.BROWSER_SPEAK_LOOPBACK_SPEED))
|
| 210 |
+
? { speed: Number(process.env.BROWSER_SPEAK_LOOPBACK_SPEED) }
|
| 211 |
+
: {}),
|
| 212 |
+
};
|
| 213 |
+
const expression = {
|
| 214 |
+
tts: "window.browserSpeakBench.runTts({ timeoutMs: 90000 })",
|
| 215 |
+
identity: "window.browserSpeakBench.runIdentity({ timeoutMs: 180000 })",
|
| 216 |
+
chat: "window.browserSpeakBench.runChat({ timeoutMs: 180000 })",
|
| 217 |
+
loopback: `window.browserSpeakBench.runLoopback(${JSON.stringify(loopbackOptions)})`,
|
| 218 |
+
barge: "window.browserSpeakBench.runBargeIn({ timeoutMs: 90000 })",
|
| 219 |
+
"barge-in": "window.browserSpeakBench.runBargeIn({ timeoutMs: 90000 })",
|
| 220 |
+
}[normalized];
|
| 221 |
+
if (!expression) throw new Error(`Unknown client-side smoke task: ${task}`);
|
| 222 |
+
await runPageTask(client, expression, {
|
| 223 |
+
label: `${normalized} benchmark`,
|
| 224 |
+
timeoutMs: normalized === "loopback" ? Math.max(taskTimeoutMs, 240000) : taskTimeoutMs,
|
| 225 |
+
});
|
| 226 |
+
}
|
| 227 |
+
|
| 228 |
+
function exportMetadata(snapshot) {
|
| 229 |
+
if (!snapshot || typeof snapshot !== "object") return null;
|
| 230 |
+
return {
|
| 231 |
+
schemaVersion: snapshot.schemaVersion ?? null,
|
| 232 |
+
exportId: snapshot.exportId ?? null,
|
| 233 |
+
generatedAt: snapshot.generatedAt ?? null,
|
| 234 |
+
hostMetadata: snapshot.hostMetadata ?? null,
|
| 235 |
+
runtime: snapshot.runtime ?? null,
|
| 236 |
+
evidence: snapshot.evidence ?? null,
|
| 237 |
+
evidenceGuide: snapshot.evidenceGuide ?? null,
|
| 238 |
+
summary: snapshot.summary ?? null,
|
| 239 |
+
resultCount: Array.isArray(snapshot.results) ? snapshot.results.length : null,
|
| 240 |
+
};
|
| 241 |
+
}
|
| 242 |
+
|
| 243 |
+
function missingTaskRows(results, requestedTasks) {
|
| 244 |
+
const aliases = new Map([
|
| 245 |
+
["barge", "barge-in"],
|
| 246 |
+
["barge-in", "barge-in"],
|
| 247 |
+
]);
|
| 248 |
+
const kinds = new Set(results.map((result) => result.kind));
|
| 249 |
+
return requestedTasks
|
| 250 |
+
.map((task) => task.toLowerCase())
|
| 251 |
+
.map((task) => aliases.get(task) ?? task)
|
| 252 |
+
.filter((kind, index, requestedKinds) => requestedKinds.indexOf(kind) === index)
|
| 253 |
+
.filter((kind) => !kinds.has(kind));
|
| 254 |
+
}
|
| 255 |
+
|
| 256 |
+
function classifyRequest(request) {
|
| 257 |
+
const reasons = [];
|
| 258 |
+
const parsed = new URL(request.url);
|
| 259 |
+
const host = parsed.hostname.toLowerCase();
|
| 260 |
+
const path = parsed.pathname.toLowerCase();
|
| 261 |
+
const method = request.method.toUpperCase();
|
| 262 |
+
if (!["GET", "HEAD"].includes(method)) reasons.push(`non-asset HTTP method ${method}`);
|
| 263 |
+
if (isForbiddenHost(host)) reasons.push(`forbidden inference host ${host}`);
|
| 264 |
+
if (isInferencePath(path)) reasons.push(`inference-like path ${parsed.pathname}`);
|
| 265 |
+
if (!isAllowedHost(host)) reasons.push(`unexpected host ${host}`);
|
| 266 |
+
return reasons;
|
| 267 |
+
}
|
| 268 |
+
|
| 269 |
+
function isAllowedHost(host) {
|
| 270 |
+
if (["localhost", "127.0.0.1", "::1"].includes(host)) return true;
|
| 271 |
+
if (host === "cdn.jsdelivr.net") return true;
|
| 272 |
+
if (host === "huggingface.co" || host.endsWith(".huggingface.co")) return true;
|
| 273 |
+
if (host === "hf.co" || host.endsWith(".hf.co")) return true;
|
| 274 |
+
if (host === "xethub.hf.co" || host.endsWith(".xethub.hf.co")) return true;
|
| 275 |
+
return false;
|
| 276 |
+
}
|
| 277 |
+
|
| 278 |
+
function isForbiddenHost(host) {
|
| 279 |
+
return /(^|\.)api-inference\.huggingface\.co$/.test(host) || /(^|\.)router\.hf\.co$/.test(host);
|
| 280 |
+
}
|
| 281 |
+
|
| 282 |
+
function isInferencePath(path) {
|
| 283 |
+
return /\/(chat\/completions|completions|generate|predictions|queue\/(join|data)|run\/predict|api\/predict|v1\/)/.test(
|
| 284 |
+
path,
|
| 285 |
+
);
|
| 286 |
+
}
|
| 287 |
+
|
| 288 |
+
function summarizeHosts(requests) {
|
| 289 |
+
return summarize(requests.map((request) => new URL(request.url).hostname.toLowerCase()));
|
| 290 |
+
}
|
| 291 |
+
|
| 292 |
+
function summarizeMethods(requests) {
|
| 293 |
+
return summarize(requests.map((request) => request.method.toUpperCase()));
|
| 294 |
+
}
|
| 295 |
+
|
| 296 |
+
function summarize(values) {
|
| 297 |
+
const counts = new Map();
|
| 298 |
+
for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1);
|
| 299 |
+
return Object.fromEntries([...counts.entries()].sort((a, b) => a[0].localeCompare(b[0])));
|
| 300 |
+
}
|
| 301 |
+
|
| 302 |
+
class NetworkRecorder {
|
| 303 |
+
constructor(pageUrl) {
|
| 304 |
+
this.pageOrigin = new URL(pageUrl).origin;
|
| 305 |
+
this.stage = "startup";
|
| 306 |
+
this.byId = new Map();
|
| 307 |
+
}
|
| 308 |
+
|
| 309 |
+
markStage(stage) {
|
| 310 |
+
this.stage = stage;
|
| 311 |
+
}
|
| 312 |
+
|
| 313 |
+
requestWillBeSent(event) {
|
| 314 |
+
const request = event.request;
|
| 315 |
+
if (!request?.url || shouldIgnoreUrl(request.url)) return;
|
| 316 |
+
this.byId.set(event.requestId, {
|
| 317 |
+
stage: this.stage,
|
| 318 |
+
requestId: event.requestId,
|
| 319 |
+
url: request.url,
|
| 320 |
+
method: request.method ?? "GET",
|
| 321 |
+
resourceType: event.type ?? "",
|
| 322 |
+
initiatorType: event.initiator?.type ?? "",
|
| 323 |
+
hasPostData: Boolean(request.hasPostData),
|
| 324 |
+
status: null,
|
| 325 |
+
failed: false,
|
| 326 |
+
failureText: "",
|
| 327 |
+
});
|
| 328 |
+
}
|
| 329 |
+
|
| 330 |
+
responseReceived(event) {
|
| 331 |
+
const item = this.byId.get(event.requestId);
|
| 332 |
+
if (!item) return;
|
| 333 |
+
item.status = event.response?.status ?? null;
|
| 334 |
+
item.mimeType = event.response?.mimeType ?? "";
|
| 335 |
+
item.fromDiskCache = Boolean(event.response?.fromDiskCache);
|
| 336 |
+
item.fromServiceWorker = Boolean(event.response?.fromServiceWorker);
|
| 337 |
+
}
|
| 338 |
+
|
| 339 |
+
loadingFailed(event) {
|
| 340 |
+
const item = this.byId.get(event.requestId);
|
| 341 |
+
if (!item) return;
|
| 342 |
+
item.failed = true;
|
| 343 |
+
item.failureText = event.errorText ?? "";
|
| 344 |
+
}
|
| 345 |
+
|
| 346 |
+
requests() {
|
| 347 |
+
return [...this.byId.values()].sort((a, b) => a.requestId.localeCompare(b.requestId));
|
| 348 |
+
}
|
| 349 |
+
}
|
| 350 |
+
|
| 351 |
+
function shouldIgnoreUrl(rawUrl) {
|
| 352 |
+
return /^(data|blob|devtools):/i.test(rawUrl);
|
| 353 |
+
}
|
| 354 |
+
|
| 355 |
+
async function ensureServer() {
|
| 356 |
+
const response = await fetch(url).catch((error) => {
|
| 357 |
+
throw new Error(`Could not reach ${url}: ${error.message}`);
|
| 358 |
+
});
|
| 359 |
+
if (!response.ok) throw new Error(`${url} returned HTTP ${response.status}`);
|
| 360 |
+
}
|
| 361 |
+
|
| 362 |
+
async function waitForBenchApi(client) {
|
| 363 |
+
const deadline = Date.now() + 15000;
|
| 364 |
+
while (Date.now() < deadline) {
|
| 365 |
+
try {
|
| 366 |
+
if (await client.evaluate("Boolean(window.browserSpeakBench)")) return;
|
| 367 |
+
} catch {
|
| 368 |
+
// The target may still be navigating.
|
| 369 |
+
}
|
| 370 |
+
await sleep(100);
|
| 371 |
+
}
|
| 372 |
+
throw new Error("window.browserSpeakBench was not installed.");
|
| 373 |
+
}
|
| 374 |
+
|
| 375 |
+
async function runPageTask(client, expression, { label = "page task", timeoutMs = 30000 } = {}) {
|
| 376 |
+
const taskId = `task_${Date.now()}_${Math.random().toString(16).slice(2)}`;
|
| 377 |
+
await client.evaluate(`(() => {
|
| 378 |
+
const taskId = ${JSON.stringify(taskId)};
|
| 379 |
+
window.__browserSpeakHarnessTasks ||= {};
|
| 380 |
+
window.__browserSpeakHarnessTasks[taskId] = { done: false, label: ${JSON.stringify(label)} };
|
| 381 |
+
Promise.resolve(${expression})
|
| 382 |
+
.then((value) => {
|
| 383 |
+
window.__browserSpeakHarnessTasks[taskId] = { done: true, value };
|
| 384 |
+
})
|
| 385 |
+
.catch((error) => {
|
| 386 |
+
window.__browserSpeakHarnessTasks[taskId] = {
|
| 387 |
+
done: true,
|
| 388 |
+
error: error?.stack || error?.message || String(error),
|
| 389 |
+
};
|
| 390 |
+
});
|
| 391 |
+
return true;
|
| 392 |
+
})()`);
|
| 393 |
+
const deadline = Date.now() + timeoutMs;
|
| 394 |
+
let lastEvents = [];
|
| 395 |
+
let lastPollError = "";
|
| 396 |
+
let pollErrorSince = 0;
|
| 397 |
+
while (Date.now() < deadline) {
|
| 398 |
+
let task = null;
|
| 399 |
+
try {
|
| 400 |
+
task = await client.evaluate(
|
| 401 |
+
`window.__browserSpeakHarnessTasks?.[${JSON.stringify(taskId)}] ?? null`,
|
| 402 |
+
pollTimeoutMs,
|
| 403 |
+
);
|
| 404 |
+
lastPollError = "";
|
| 405 |
+
pollErrorSince = 0;
|
| 406 |
+
} catch (error) {
|
| 407 |
+
const message = error.message ?? String(error);
|
| 408 |
+
pollErrorSince ||= Date.now();
|
| 409 |
+
if (message !== lastPollError) {
|
| 410 |
+
console.log(`${label}: waiting for page response (${message})`);
|
| 411 |
+
lastPollError = message;
|
| 412 |
+
}
|
| 413 |
+
if (Date.now() - pollErrorSince > pageUnresponsiveTimeoutMs) {
|
| 414 |
+
throw new Error(
|
| 415 |
+
`${label} page stayed unresponsive for ${(pageUnresponsiveTimeoutMs / 1000).toFixed(0)} seconds; last poll error: ${message}`,
|
| 416 |
+
);
|
| 417 |
+
}
|
| 418 |
+
await sleep(500);
|
| 419 |
+
continue;
|
| 420 |
+
}
|
| 421 |
+
if (task?.done) {
|
| 422 |
+
await client
|
| 423 |
+
.evaluate(`delete window.__browserSpeakHarnessTasks?.[${JSON.stringify(taskId)}]`, pollTimeoutMs)
|
| 424 |
+
.catch(() => {});
|
| 425 |
+
if (task.error) throw new Error(`${label} failed: ${task.error}`);
|
| 426 |
+
return task.value;
|
| 427 |
+
}
|
| 428 |
+
const snapshot = await client
|
| 429 |
+
.evaluate(`(() => {
|
| 430 |
+
const state = window.browserSpeakBench?.state?.();
|
| 431 |
+
return state ? {
|
| 432 |
+
modelsLoaded: state.modelsLoaded,
|
| 433 |
+
modelsLoading: state.modelsLoading,
|
| 434 |
+
activeBenchmark: state.activeBenchmark?.kind ?? null,
|
| 435 |
+
events: state.events?.slice(0, 3) ?? [],
|
| 436 |
+
} : null;
|
| 437 |
+
})()`, pollTimeoutMs)
|
| 438 |
+
.catch(() => null);
|
| 439 |
+
const events = snapshot?.events ?? [];
|
| 440 |
+
if (events.join("\\n") !== lastEvents.join("\\n")) {
|
| 441 |
+
lastEvents = events;
|
| 442 |
+
if (events[0]) console.log(`${label}: ${events[0]}`);
|
| 443 |
+
}
|
| 444 |
+
await sleep(500);
|
| 445 |
+
}
|
| 446 |
+
throw new Error(`${label} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`);
|
| 447 |
+
}
|
| 448 |
+
|
| 449 |
+
function launchBrowser(port, profileDir) {
|
| 450 |
+
const child = spawn(
|
| 451 |
+
chrome,
|
| 452 |
+
[
|
| 453 |
+
...(headless ? ["--headless=new"] : []),
|
| 454 |
+
"--no-sandbox",
|
| 455 |
+
"--disable-gpu",
|
| 456 |
+
"--disable-dev-shm-usage",
|
| 457 |
+
"--disable-background-networking",
|
| 458 |
+
"--disable-extensions",
|
| 459 |
+
"--no-default-browser-check",
|
| 460 |
+
"--no-first-run",
|
| 461 |
+
"--autoplay-policy=no-user-gesture-required",
|
| 462 |
+
`--remote-debugging-port=${port}`,
|
| 463 |
+
`--user-data-dir=${profileDir}`,
|
| 464 |
+
...parseChromeArgs(),
|
| 465 |
+
],
|
| 466 |
+
{ stdio: ["ignore", "pipe", "pipe"] },
|
| 467 |
+
);
|
| 468 |
+
child.browserLog = "";
|
| 469 |
+
const appendLog = (chunk) => {
|
| 470 |
+
child.browserLog = `${child.browserLog}${chunk}`;
|
| 471 |
+
if (child.browserLog.length > 8000) child.browserLog = child.browserLog.slice(-8000);
|
| 472 |
+
};
|
| 473 |
+
child.stdout.on("data", appendLog);
|
| 474 |
+
child.stderr.on("data", appendLog);
|
| 475 |
+
return child;
|
| 476 |
+
}
|
| 477 |
+
|
| 478 |
+
async function stopBrowser(child, profileDir) {
|
| 479 |
+
if (child.exitCode == null) child.kill("SIGTERM");
|
| 480 |
+
await new Promise((resolve) => {
|
| 481 |
+
child.once("exit", resolve);
|
| 482 |
+
setTimeout(resolve, 3000);
|
| 483 |
+
});
|
| 484 |
+
if (child.exitCode == null) child.kill("SIGKILL");
|
| 485 |
+
if (!profileDir) return;
|
| 486 |
+
for (let attempt = 0; attempt < 5; attempt += 1) {
|
| 487 |
+
try {
|
| 488 |
+
await rm(profileDir, { recursive: true, force: true });
|
| 489 |
+
return;
|
| 490 |
+
} catch (error) {
|
| 491 |
+
if (attempt === 4) {
|
| 492 |
+
console.warn(`Could not remove ${profileDir}: ${error.message}`);
|
| 493 |
+
return;
|
| 494 |
+
}
|
| 495 |
+
await sleep(500);
|
| 496 |
+
}
|
| 497 |
+
}
|
| 498 |
+
}
|
| 499 |
+
|
| 500 |
+
async function connectToPage(port, child) {
|
| 501 |
+
const deadline = Date.now() + 60000;
|
| 502 |
+
let lastError = null;
|
| 503 |
+
while (Date.now() < deadline) {
|
| 504 |
+
if (child.exitCode != null) {
|
| 505 |
+
throw new Error(`Chrome exited before DevTools became available.\n${child.browserLog}`);
|
| 506 |
+
}
|
| 507 |
+
try {
|
| 508 |
+
const version = await fetch(`http://127.0.0.1:${port}/json/version`).then((response) => response.json());
|
| 509 |
+
if (version.webSocketDebuggerUrl) {
|
| 510 |
+
const page = await createPageTarget(port);
|
| 511 |
+
return new CdpClient(page.webSocketDebuggerUrl);
|
| 512 |
+
}
|
| 513 |
+
} catch (error) {
|
| 514 |
+
lastError = error;
|
| 515 |
+
}
|
| 516 |
+
await sleep(250);
|
| 517 |
+
}
|
| 518 |
+
throw new Error(
|
| 519 |
+
`Could not connect to Chrome DevTools on port ${port}: ${lastError?.message ?? "unknown error"}\n${child.browserLog}`,
|
| 520 |
+
);
|
| 521 |
+
}
|
| 522 |
+
|
| 523 |
+
async function createPageTarget(port) {
|
| 524 |
+
for (const method of ["PUT", "GET"]) {
|
| 525 |
+
const response = await fetch(`http://127.0.0.1:${port}/json/new?${encodeURIComponent(url)}`, {
|
| 526 |
+
method,
|
| 527 |
+
}).catch(() => null);
|
| 528 |
+
if (response?.ok) {
|
| 529 |
+
const target = await response.json();
|
| 530 |
+
if (target.webSocketDebuggerUrl) return target;
|
| 531 |
+
}
|
| 532 |
+
}
|
| 533 |
+
const targets = await fetch(`http://127.0.0.1:${port}/json`).then((response) => response.json());
|
| 534 |
+
const page = targets.find((target) => target.type === "page" && target.url === url);
|
| 535 |
+
if (page?.webSocketDebuggerUrl) return page;
|
| 536 |
+
throw new Error("Could not create or find a page target.");
|
| 537 |
+
}
|
| 538 |
+
|
| 539 |
+
class CdpClient {
|
| 540 |
+
constructor(webSocketUrl) {
|
| 541 |
+
this.nextId = 1;
|
| 542 |
+
this.pending = new Map();
|
| 543 |
+
this.listeners = new Map();
|
| 544 |
+
this.socket = new WebSocket(webSocketUrl);
|
| 545 |
+
this.opened = new Promise((resolve, reject) => {
|
| 546 |
+
this.socket.onopen = resolve;
|
| 547 |
+
this.socket.onerror = reject;
|
| 548 |
+
this.socket.onmessage = (event) => this.onMessage(event);
|
| 549 |
+
});
|
| 550 |
+
}
|
| 551 |
+
|
| 552 |
+
on(method, handler) {
|
| 553 |
+
if (!this.listeners.has(method)) this.listeners.set(method, new Set());
|
| 554 |
+
this.listeners.get(method).add(handler);
|
| 555 |
+
}
|
| 556 |
+
|
| 557 |
+
onMessage(event) {
|
| 558 |
+
const message = JSON.parse(event.data);
|
| 559 |
+
if (message.id && this.pending.has(message.id)) {
|
| 560 |
+
const { resolve: onResolve, reject } = this.pending.get(message.id);
|
| 561 |
+
this.pending.delete(message.id);
|
| 562 |
+
if (message.error) reject(new Error(message.error.message));
|
| 563 |
+
else onResolve(message.result);
|
| 564 |
+
return;
|
| 565 |
+
}
|
| 566 |
+
if (message.method && this.listeners.has(message.method)) {
|
| 567 |
+
for (const handler of this.listeners.get(message.method)) handler(message.params ?? {});
|
| 568 |
+
}
|
| 569 |
+
}
|
| 570 |
+
|
| 571 |
+
async call(method, params = {}, timeoutMs = protocolTimeoutMs) {
|
| 572 |
+
await this.opened;
|
| 573 |
+
const id = this.nextId++;
|
| 574 |
+
this.socket.send(JSON.stringify({ id, method, params }));
|
| 575 |
+
return new Promise((resolvePromise, reject) => {
|
| 576 |
+
const timer = setTimeout(() => {
|
| 577 |
+
this.pending.delete(id);
|
| 578 |
+
reject(new Error(`${method} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`));
|
| 579 |
+
}, timeoutMs);
|
| 580 |
+
this.pending.set(id, {
|
| 581 |
+
resolve: (value) => {
|
| 582 |
+
clearTimeout(timer);
|
| 583 |
+
resolvePromise(value);
|
| 584 |
+
},
|
| 585 |
+
reject: (error) => {
|
| 586 |
+
clearTimeout(timer);
|
| 587 |
+
reject(error);
|
| 588 |
+
},
|
| 589 |
+
});
|
| 590 |
+
});
|
| 591 |
+
}
|
| 592 |
+
|
| 593 |
+
async evaluate(expression, timeoutMs = protocolTimeoutMs) {
|
| 594 |
+
const result = await this.call(
|
| 595 |
+
"Runtime.evaluate",
|
| 596 |
+
{
|
| 597 |
+
expression,
|
| 598 |
+
awaitPromise: true,
|
| 599 |
+
returnByValue: true,
|
| 600 |
+
},
|
| 601 |
+
timeoutMs,
|
| 602 |
+
);
|
| 603 |
+
if (result.exceptionDetails) throw new Error(formatException(result.exceptionDetails));
|
| 604 |
+
return result.result.value;
|
| 605 |
+
}
|
| 606 |
+
|
| 607 |
+
async closeBrowser() {
|
| 608 |
+
await this.call("Browser.close", {}, 5000).catch(() => {});
|
| 609 |
+
try {
|
| 610 |
+
this.socket.close();
|
| 611 |
+
} catch {
|
| 612 |
+
// Ignore close races.
|
| 613 |
+
}
|
| 614 |
+
}
|
| 615 |
+
}
|
| 616 |
+
|
| 617 |
+
function formatException(exceptionDetails) {
|
| 618 |
+
const exception = exceptionDetails.exception;
|
| 619 |
+
return exception?.description ?? exception?.value ?? exceptionDetails.text ?? "Evaluation failed.";
|
| 620 |
+
}
|
| 621 |
+
|
| 622 |
+
function parseList(value) {
|
| 623 |
+
return String(value ?? "")
|
| 624 |
+
.split(",")
|
| 625 |
+
.map((item) => item.trim())
|
| 626 |
+
.filter(Boolean);
|
| 627 |
+
}
|
| 628 |
+
|
| 629 |
+
function parseChromeArgs() {
|
| 630 |
+
const raw = process.env.BROWSER_SPEAK_CHROME_ARGS ?? "";
|
| 631 |
+
return (
|
| 632 |
+
raw
|
| 633 |
+
.match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g)
|
| 634 |
+
?.map((arg) => arg.replace(/^["']|["']$/g, "")) ?? []
|
| 635 |
+
);
|
| 636 |
+
}
|
| 637 |
+
|
| 638 |
+
function sleep(ms) {
|
| 639 |
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
| 640 |
+
}
|
| 641 |
+
|
| 642 |
+
main().catch((error) => {
|
| 643 |
+
console.error(error.stack ?? error.message);
|
| 644 |
+
process.exitCode = 1;
|
| 645 |
+
});
|
tools/run-evidence-export-smoke.mjs
ADDED
|
@@ -0,0 +1,414 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env node
|
| 2 |
+
import { spawn } from "node:child_process";
|
| 3 |
+
import { existsSync } from "node:fs";
|
| 4 |
+
import { mkdir, rm, writeFile } from "node:fs/promises";
|
| 5 |
+
import { tmpdir } from "node:os";
|
| 6 |
+
import { dirname, resolve } from "node:path";
|
| 7 |
+
import { sourceFingerprint } from "./source-fingerprint.mjs";
|
| 8 |
+
|
| 9 |
+
const url = process.env.BROWSER_SPEAK_URL ?? "http://127.0.0.1:5174/";
|
| 10 |
+
const chrome =
|
| 11 |
+
process.env.CHROME_BIN ??
|
| 12 |
+
(existsSync("/opt/google/chrome/chrome")
|
| 13 |
+
? "/opt/google/chrome/chrome"
|
| 14 |
+
: existsSync("/usr/bin/google-chrome")
|
| 15 |
+
? "/usr/bin/google-chrome"
|
| 16 |
+
: "chromium");
|
| 17 |
+
const resultPath = resolve(
|
| 18 |
+
process.env.BROWSER_SPEAK_EVIDENCE_EXPORT_JSON ?? `${tmpdir()}/browser-speak-evidence-export-smoke.json`,
|
| 19 |
+
);
|
| 20 |
+
const profileDir = resolve(
|
| 21 |
+
process.env.BROWSER_SPEAK_EVIDENCE_EXPORT_PROFILE_DIR ??
|
| 22 |
+
`${tmpdir()}/browser-speak-evidence-export-profile`,
|
| 23 |
+
);
|
| 24 |
+
const protocolTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_TIMEOUT_MS ?? 60000);
|
| 25 |
+
const pollTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_POLL_TIMEOUT_MS ?? 5000);
|
| 26 |
+
const headless = process.env.BROWSER_SPEAK_HEADLESS !== "false";
|
| 27 |
+
|
| 28 |
+
async function main() {
|
| 29 |
+
await ensureServer();
|
| 30 |
+
await mkdir(dirname(resultPath), { recursive: true });
|
| 31 |
+
await rm(profileDir, { recursive: true, force: true });
|
| 32 |
+
|
| 33 |
+
const browser = launchBrowser(9349, profileDir);
|
| 34 |
+
let client = null;
|
| 35 |
+
try {
|
| 36 |
+
client = await connectToPage(9349, browser);
|
| 37 |
+
await client.call("Page.enable");
|
| 38 |
+
await client.call("Runtime.enable");
|
| 39 |
+
await waitForBenchApi(client);
|
| 40 |
+
|
| 41 |
+
const initialExport = await waitForHostMetadata(client);
|
| 42 |
+
const pageScope = await client.evaluate("location.origin + location.pathname", pollTimeoutMs);
|
| 43 |
+
const scope = `${pageScope}#${initialExport.hostMetadata?.hfSpaceCommit || "local"}`;
|
| 44 |
+
const seed = seedStoragePayload(scope, initialExport.hostMetadata);
|
| 45 |
+
await client.evaluate(
|
| 46 |
+
`localStorage.setItem("browser-speak:benchmark-results:v1", ${JSON.stringify(
|
| 47 |
+
JSON.stringify(seed),
|
| 48 |
+
)})`,
|
| 49 |
+
pollTimeoutMs,
|
| 50 |
+
);
|
| 51 |
+
await client.call("Page.reload", { ignoreCache: true });
|
| 52 |
+
await waitForBenchApi(client);
|
| 53 |
+
const restoredRows = await waitForValue(
|
| 54 |
+
client,
|
| 55 |
+
"window.browserSpeakBench.state().results.length",
|
| 56 |
+
(value) => value === seed.results.length,
|
| 57 |
+
10000,
|
| 58 |
+
);
|
| 59 |
+
|
| 60 |
+
await client.evaluate(
|
| 61 |
+
`(() => {
|
| 62 |
+
window.__browserSpeakDownloadClicks = [];
|
| 63 |
+
HTMLAnchorElement.prototype.__browserSpeakOriginalClick ??= HTMLAnchorElement.prototype.click;
|
| 64 |
+
HTMLAnchorElement.prototype.click = function patchedClick() {
|
| 65 |
+
window.__browserSpeakDownloadClicks.push({
|
| 66 |
+
download: this.download,
|
| 67 |
+
href: this.href,
|
| 68 |
+
});
|
| 69 |
+
return true;
|
| 70 |
+
};
|
| 71 |
+
return true;
|
| 72 |
+
})()`,
|
| 73 |
+
pollTimeoutMs,
|
| 74 |
+
);
|
| 75 |
+
await client.evaluate(
|
| 76 |
+
`window.browserSpeakBench.downloadResults({
|
| 77 |
+
prefix: "browser-speak-evidence",
|
| 78 |
+
message: "Evidence JSON download smoke."
|
| 79 |
+
})`,
|
| 80 |
+
pollTimeoutMs,
|
| 81 |
+
);
|
| 82 |
+
const clicks = await client.evaluate("window.__browserSpeakDownloadClicks", pollTimeoutMs);
|
| 83 |
+
const exportPayload = await client.evaluate("window.browserSpeakBench.exportResults()", pollTimeoutMs);
|
| 84 |
+
const exportSelfDescription = validateExportSelfDescription(exportPayload);
|
| 85 |
+
|
| 86 |
+
await client.evaluate("window.browserSpeakBench.clearResults()", pollTimeoutMs);
|
| 87 |
+
const clearedRows = await client.evaluate("window.browserSpeakBench.state().results.length", pollTimeoutMs);
|
| 88 |
+
const persistedAfterClear = await client.evaluate(
|
| 89 |
+
'localStorage.getItem("browser-speak:benchmark-results:v1")',
|
| 90 |
+
pollTimeoutMs,
|
| 91 |
+
);
|
| 92 |
+
|
| 93 |
+
const download = Array.isArray(clicks) ? clicks.find((click) => /^browser-speak-evidence-.*\.json$/.test(click.download)) : null;
|
| 94 |
+
const passed =
|
| 95 |
+
restoredRows === seed.results.length &&
|
| 96 |
+
exportPayload?.results?.length === seed.results.length &&
|
| 97 |
+
exportSelfDescription.passed &&
|
| 98 |
+
Boolean(download) &&
|
| 99 |
+
String(download?.href ?? "").startsWith("blob:") &&
|
| 100 |
+
clearedRows === 0 &&
|
| 101 |
+
persistedAfterClear === null;
|
| 102 |
+
const payload = {
|
| 103 |
+
generatedAt: new Date().toISOString(),
|
| 104 |
+
sourceFingerprint: await sourceFingerprint(),
|
| 105 |
+
url,
|
| 106 |
+
passed,
|
| 107 |
+
config: {
|
| 108 |
+
chrome,
|
| 109 |
+
headless,
|
| 110 |
+
protocolTimeoutMs,
|
| 111 |
+
pollTimeoutMs,
|
| 112 |
+
profileDir,
|
| 113 |
+
extraChromeArgs: parseChromeArgs(),
|
| 114 |
+
},
|
| 115 |
+
summary: {
|
| 116 |
+
restoredRows,
|
| 117 |
+
exportedRows: exportPayload?.results?.length ?? null,
|
| 118 |
+
downloadName: download?.download ?? null,
|
| 119 |
+
downloadHrefScheme: String(download?.href ?? "").split(":")[0] || null,
|
| 120 |
+
schemaVersion: exportPayload?.schemaVersion ?? null,
|
| 121 |
+
exportId: exportPayload?.exportId ?? null,
|
| 122 |
+
evidenceGuidePresent: Boolean(exportPayload?.evidenceGuide),
|
| 123 |
+
networkSummaryPresent: Boolean(exportPayload?.network),
|
| 124 |
+
workerNetworkCapturedEvents: exportPayload?.network?.capturedEvents ?? null,
|
| 125 |
+
benchmarkWorkerNetworkRequests: exportPayload?.network?.benchmarkRequests ?? null,
|
| 126 |
+
benchmarkWorkerNetworkSuspects: exportPayload?.network?.benchmarkServerInferenceSuspects ?? null,
|
| 127 |
+
selfDescription: exportSelfDescription,
|
| 128 |
+
clearedRows,
|
| 129 |
+
persistedAfterClear,
|
| 130 |
+
},
|
| 131 |
+
clicks,
|
| 132 |
+
exportMetadata: {
|
| 133 |
+
generatedAt: exportPayload?.generatedAt ?? null,
|
| 134 |
+
schemaVersion: exportPayload?.schemaVersion ?? null,
|
| 135 |
+
exportId: exportPayload?.exportId ?? null,
|
| 136 |
+
evidenceGuide: exportPayload?.evidenceGuide ?? null,
|
| 137 |
+
hostMetadata: exportPayload?.hostMetadata ?? null,
|
| 138 |
+
evidence: exportPayload?.evidence ?? null,
|
| 139 |
+
resultCount: exportPayload?.results?.length ?? null,
|
| 140 |
+
},
|
| 141 |
+
};
|
| 142 |
+
await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`);
|
| 143 |
+
console.log(`Wrote evidence export smoke JSON: ${resultPath}`);
|
| 144 |
+
console.log(
|
| 145 |
+
passed
|
| 146 |
+
? `Evidence export smoke passed: restored ${restoredRows} row(s), download ${download.download}, clear removed saved rows.`
|
| 147 |
+
: "Evidence export smoke failed.",
|
| 148 |
+
);
|
| 149 |
+
if (!passed) process.exitCode = 1;
|
| 150 |
+
await client.closeBrowser();
|
| 151 |
+
} catch (error) {
|
| 152 |
+
await writeFailurePayload(error, browser).catch(() => {});
|
| 153 |
+
if (client) await client.closeBrowser().catch(() => {});
|
| 154 |
+
throw error;
|
| 155 |
+
} finally {
|
| 156 |
+
await stopBrowser(browser, profileDir);
|
| 157 |
+
}
|
| 158 |
+
}
|
| 159 |
+
|
| 160 |
+
function seedStoragePayload(scope, hostMetadata = {}) {
|
| 161 |
+
return {
|
| 162 |
+
version: 1,
|
| 163 |
+
scope,
|
| 164 |
+
savedAt: new Date().toISOString(),
|
| 165 |
+
hostMetadata: {
|
| 166 |
+
page: hostMetadata.page ?? url,
|
| 167 |
+
host: hostMetadata.host ?? new URL(url).host,
|
| 168 |
+
hfSpaceCommit: hostMetadata.hfSpaceCommit ?? "",
|
| 169 |
+
etag: hostMetadata.etag ?? "",
|
| 170 |
+
fetchedAt: hostMetadata.fetchedAt ?? new Date().toISOString(),
|
| 171 |
+
},
|
| 172 |
+
results: [
|
| 173 |
+
{
|
| 174 |
+
id: 1,
|
| 175 |
+
kind: "identity",
|
| 176 |
+
startedAt: new Date().toISOString(),
|
| 177 |
+
stack: {
|
| 178 |
+
device: "wasm",
|
| 179 |
+
llm: "HuggingFaceTB/SmolLM2-135M-Instruct",
|
| 180 |
+
voice: "F2",
|
| 181 |
+
ttsSteps: 2,
|
| 182 |
+
environment: {},
|
| 183 |
+
},
|
| 184 |
+
prompt: "What app is this?",
|
| 185 |
+
output: "This browser demo runs speech recognition, a local LLM, and Supertonic TTS client-side.",
|
| 186 |
+
llmQualityPass: true,
|
| 187 |
+
llmQualityScore: 4,
|
| 188 |
+
llmQualityTotal: 4,
|
| 189 |
+
error: "",
|
| 190 |
+
},
|
| 191 |
+
],
|
| 192 |
+
};
|
| 193 |
+
}
|
| 194 |
+
|
| 195 |
+
function validateExportSelfDescription(exportPayload) {
|
| 196 |
+
const guide = exportPayload?.evidenceGuide ?? {};
|
| 197 |
+
const realMic = guide.requirements?.realMic ?? {};
|
| 198 |
+
const hardwareWebgpu = guide.requirements?.hardwareWebgpu ?? {};
|
| 199 |
+
const errors = [];
|
| 200 |
+
if (!String(exportPayload?.schemaVersion ?? "").startsWith("browser-speak-benchmarks/")) {
|
| 201 |
+
errors.push("missing schemaVersion");
|
| 202 |
+
}
|
| 203 |
+
if (!String(exportPayload?.exportId ?? "").startsWith("browser-speak-")) {
|
| 204 |
+
errors.push("missing exportId");
|
| 205 |
+
}
|
| 206 |
+
if (!String(guide.auditCommand ?? "").includes("audit-browser-evidence.mjs")) {
|
| 207 |
+
errors.push("missing audit command");
|
| 208 |
+
}
|
| 209 |
+
if (realMic.requiredRows !== 3 || realMic.prompt !== "What app is this?") {
|
| 210 |
+
errors.push("missing real-mic requirements");
|
| 211 |
+
}
|
| 212 |
+
if (hardwareWebgpu.requiredRows !== 1) {
|
| 213 |
+
errors.push("missing hardware-WebGPU requirements");
|
| 214 |
+
}
|
| 215 |
+
if (!exportPayload?.network || typeof exportPayload.network.benchmarkServerInferenceSuspects !== "number") {
|
| 216 |
+
errors.push("missing worker-network summary");
|
| 217 |
+
}
|
| 218 |
+
return {
|
| 219 |
+
passed: errors.length === 0,
|
| 220 |
+
errors,
|
| 221 |
+
};
|
| 222 |
+
}
|
| 223 |
+
|
| 224 |
+
async function writeFailurePayload(error, browser = null) {
|
| 225 |
+
const payload = {
|
| 226 |
+
generatedAt: new Date().toISOString(),
|
| 227 |
+
sourceFingerprint: await sourceFingerprint().catch(() => null),
|
| 228 |
+
url,
|
| 229 |
+
passed: false,
|
| 230 |
+
error: error.stack ?? error.message ?? String(error),
|
| 231 |
+
browserLog: browser?.browserLog ?? "",
|
| 232 |
+
config: {
|
| 233 |
+
chrome,
|
| 234 |
+
headless,
|
| 235 |
+
protocolTimeoutMs,
|
| 236 |
+
pollTimeoutMs,
|
| 237 |
+
profileDir,
|
| 238 |
+
extraChromeArgs: parseChromeArgs(),
|
| 239 |
+
},
|
| 240 |
+
};
|
| 241 |
+
await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`);
|
| 242 |
+
console.log(`Wrote evidence export smoke failure JSON: ${resultPath}`);
|
| 243 |
+
}
|
| 244 |
+
|
| 245 |
+
async function ensureServer() {
|
| 246 |
+
const response = await fetch(url).catch((error) => {
|
| 247 |
+
throw new Error(`Could not reach ${url}: ${error.message}`);
|
| 248 |
+
});
|
| 249 |
+
if (!response.ok) throw new Error(`${url} returned HTTP ${response.status}`);
|
| 250 |
+
}
|
| 251 |
+
|
| 252 |
+
function launchBrowser(port, profileDir) {
|
| 253 |
+
const child = spawn(
|
| 254 |
+
chrome,
|
| 255 |
+
[
|
| 256 |
+
...(headless ? ["--headless=new"] : []),
|
| 257 |
+
"--no-sandbox",
|
| 258 |
+
"--disable-dev-shm-usage",
|
| 259 |
+
"--no-default-browser-check",
|
| 260 |
+
"--no-first-run",
|
| 261 |
+
`--remote-debugging-port=${port}`,
|
| 262 |
+
`--user-data-dir=${profileDir}`,
|
| 263 |
+
...parseChromeArgs(),
|
| 264 |
+
url,
|
| 265 |
+
],
|
| 266 |
+
{ stdio: ["ignore", "pipe", "pipe"] },
|
| 267 |
+
);
|
| 268 |
+
child.browserLog = "";
|
| 269 |
+
const appendLog = (chunk) => {
|
| 270 |
+
child.browserLog = `${child.browserLog}${chunk}`;
|
| 271 |
+
if (child.browserLog.length > 8000) child.browserLog = child.browserLog.slice(-8000);
|
| 272 |
+
};
|
| 273 |
+
child.stdout.on("data", appendLog);
|
| 274 |
+
child.stderr.on("data", appendLog);
|
| 275 |
+
return child;
|
| 276 |
+
}
|
| 277 |
+
|
| 278 |
+
async function connectToPage(port, browser) {
|
| 279 |
+
const deadline = Date.now() + 15000;
|
| 280 |
+
let lastError = "";
|
| 281 |
+
while (Date.now() < deadline) {
|
| 282 |
+
try {
|
| 283 |
+
const targets = await fetch(`http://127.0.0.1:${port}/json`).then((response) => response.json());
|
| 284 |
+
const page = targets.find((target) => target.type === "page" && target.webSocketDebuggerUrl);
|
| 285 |
+
if (page) return new CdpClient(page.webSocketDebuggerUrl, browser);
|
| 286 |
+
} catch (error) {
|
| 287 |
+
lastError = error.message;
|
| 288 |
+
}
|
| 289 |
+
await sleep(100);
|
| 290 |
+
}
|
| 291 |
+
throw new Error(`Could not connect to Chrome DevTools. ${lastError}`.trim());
|
| 292 |
+
}
|
| 293 |
+
|
| 294 |
+
class CdpClient {
|
| 295 |
+
constructor(wsUrl, browser) {
|
| 296 |
+
this.ws = new WebSocket(wsUrl);
|
| 297 |
+
this.browser = browser;
|
| 298 |
+
this.nextId = 0;
|
| 299 |
+
this.pending = new Map();
|
| 300 |
+
this.ready = new Promise((resolve, reject) => {
|
| 301 |
+
this.ws.addEventListener("open", resolve, { once: true });
|
| 302 |
+
this.ws.addEventListener("error", () => reject(new Error("WebSocket connection failed")), { once: true });
|
| 303 |
+
});
|
| 304 |
+
this.ws.addEventListener("message", (event) => {
|
| 305 |
+
const message = JSON.parse(event.data);
|
| 306 |
+
if (!message.id || !this.pending.has(message.id)) return;
|
| 307 |
+
const pending = this.pending.get(message.id);
|
| 308 |
+
this.pending.delete(message.id);
|
| 309 |
+
clearTimeout(pending.timeout);
|
| 310 |
+
if (message.error) pending.reject(new Error(message.error.message));
|
| 311 |
+
else pending.resolve(message.result);
|
| 312 |
+
});
|
| 313 |
+
}
|
| 314 |
+
|
| 315 |
+
async call(method, params = {}, timeoutMs = protocolTimeoutMs) {
|
| 316 |
+
await this.ready;
|
| 317 |
+
const id = ++this.nextId;
|
| 318 |
+
this.ws.send(JSON.stringify({ id, method, params }));
|
| 319 |
+
return new Promise((resolve, reject) => {
|
| 320 |
+
const timeout = setTimeout(() => {
|
| 321 |
+
this.pending.delete(id);
|
| 322 |
+
reject(new Error(`${method} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`));
|
| 323 |
+
}, timeoutMs);
|
| 324 |
+
this.pending.set(id, { resolve, reject, timeout });
|
| 325 |
+
});
|
| 326 |
+
}
|
| 327 |
+
|
| 328 |
+
async evaluate(expression, timeoutMs = protocolTimeoutMs) {
|
| 329 |
+
const result = await this.call(
|
| 330 |
+
"Runtime.evaluate",
|
| 331 |
+
{
|
| 332 |
+
expression,
|
| 333 |
+
awaitPromise: true,
|
| 334 |
+
returnByValue: true,
|
| 335 |
+
},
|
| 336 |
+
timeoutMs,
|
| 337 |
+
);
|
| 338 |
+
if (result.exceptionDetails) {
|
| 339 |
+
throw new Error(result.exceptionDetails.exception?.description ?? result.exceptionDetails.text);
|
| 340 |
+
}
|
| 341 |
+
return result.result.value;
|
| 342 |
+
}
|
| 343 |
+
|
| 344 |
+
async closeBrowser() {
|
| 345 |
+
await this.call("Browser.close", {}, 5000).catch(() => {});
|
| 346 |
+
this.ws.close();
|
| 347 |
+
}
|
| 348 |
+
}
|
| 349 |
+
|
| 350 |
+
async function waitForBenchApi(client) {
|
| 351 |
+
await waitForValue(
|
| 352 |
+
client,
|
| 353 |
+
"Boolean(window.browserSpeakBench?.downloadResults && window.browserSpeakBench?.clearResults)",
|
| 354 |
+
Boolean,
|
| 355 |
+
15000,
|
| 356 |
+
);
|
| 357 |
+
}
|
| 358 |
+
|
| 359 |
+
async function waitForHostMetadata(client) {
|
| 360 |
+
const needsHostedCommit = await client.evaluate('location.hostname.endsWith(".hf.space")', pollTimeoutMs);
|
| 361 |
+
return waitForValue(
|
| 362 |
+
client,
|
| 363 |
+
"window.browserSpeakBench.exportResults()",
|
| 364 |
+
(payload) => {
|
| 365 |
+
const metadata = payload?.hostMetadata ?? {};
|
| 366 |
+
return Boolean(metadata.fetchedAt) && (!needsHostedCommit || String(metadata.hfSpaceCommit ?? "").length >= 7);
|
| 367 |
+
},
|
| 368 |
+
15000,
|
| 369 |
+
);
|
| 370 |
+
}
|
| 371 |
+
|
| 372 |
+
async function waitForValue(client, expression, predicate, timeoutMs) {
|
| 373 |
+
const deadline = Date.now() + timeoutMs;
|
| 374 |
+
while (Date.now() < deadline) {
|
| 375 |
+
const value = await client.evaluate(expression, pollTimeoutMs).catch(() => undefined);
|
| 376 |
+
if (predicate(value)) return value;
|
| 377 |
+
await sleep(100);
|
| 378 |
+
}
|
| 379 |
+
throw new Error(`Timed out waiting for ${expression}`);
|
| 380 |
+
}
|
| 381 |
+
|
| 382 |
+
async function stopBrowser(child, profileDir) {
|
| 383 |
+
if (child.exitCode == null) child.kill("SIGTERM");
|
| 384 |
+
await new Promise((resolve) => {
|
| 385 |
+
child.once("exit", resolve);
|
| 386 |
+
setTimeout(resolve, 3000);
|
| 387 |
+
});
|
| 388 |
+
if (child.exitCode == null) child.kill("SIGKILL");
|
| 389 |
+
for (let attempt = 0; attempt < 10; attempt += 1) {
|
| 390 |
+
try {
|
| 391 |
+
await rm(profileDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
|
| 392 |
+
return;
|
| 393 |
+
} catch (error) {
|
| 394 |
+
if (attempt === 9) throw error;
|
| 395 |
+
await sleep(200);
|
| 396 |
+
}
|
| 397 |
+
}
|
| 398 |
+
}
|
| 399 |
+
|
| 400 |
+
function parseChromeArgs() {
|
| 401 |
+
return (process.env.BROWSER_SPEAK_CHROME_ARGS ?? "")
|
| 402 |
+
.split(/\s+/)
|
| 403 |
+
.map((arg) => arg.trim())
|
| 404 |
+
.filter(Boolean);
|
| 405 |
+
}
|
| 406 |
+
|
| 407 |
+
function sleep(ms) {
|
| 408 |
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
| 409 |
+
}
|
| 410 |
+
|
| 411 |
+
main().catch((error) => {
|
| 412 |
+
console.error(error.stack ?? error.message);
|
| 413 |
+
process.exitCode = 1;
|
| 414 |
+
});
|
tools/run-fake-mic-benchmark.mjs
ADDED
|
@@ -0,0 +1,485 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env node
|
| 2 |
+
import { spawn } from "node:child_process";
|
| 3 |
+
import { existsSync } from "node:fs";
|
| 4 |
+
import { mkdir, rm, writeFile } from "node:fs/promises";
|
| 5 |
+
import { tmpdir } from "node:os";
|
| 6 |
+
import { dirname, resolve } from "node:path";
|
| 7 |
+
import { sourceFingerprint } from "./source-fingerprint.mjs";
|
| 8 |
+
|
| 9 |
+
const url = process.env.BROWSER_SPEAK_URL ?? "http://127.0.0.1:5174/";
|
| 10 |
+
const chrome =
|
| 11 |
+
process.env.CHROME_BIN ??
|
| 12 |
+
(existsSync("/opt/google/chrome/chrome")
|
| 13 |
+
? "/opt/google/chrome/chrome"
|
| 14 |
+
: existsSync("/usr/bin/google-chrome")
|
| 15 |
+
? "/usr/bin/google-chrome"
|
| 16 |
+
: "chromium");
|
| 17 |
+
const prompt = process.env.BROWSER_SPEAK_PROMPT ?? "What app is this?";
|
| 18 |
+
const fixturePath = resolve(process.env.BROWSER_SPEAK_FAKE_MIC_WAV ?? `${tmpdir()}/browser-speak-what-app-is-this.wav`);
|
| 19 |
+
const resultPath = resolve(process.env.BROWSER_SPEAK_FAKE_MIC_JSON ?? `${tmpdir()}/browser-speak-fake-mic-results.json`);
|
| 20 |
+
const profileDir = resolve(process.env.BROWSER_SPEAK_PROFILE_DIR ?? `${tmpdir()}/browser-speak-benchmark-profile`);
|
| 21 |
+
const reuseFixture = process.env.BROWSER_SPEAK_REUSE_FAKE_MIC_WAV === "true";
|
| 22 |
+
const protocolTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_TIMEOUT_MS ?? 60000);
|
| 23 |
+
const pollTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_POLL_TIMEOUT_MS ?? 5000);
|
| 24 |
+
const fakeMicTurnTimeoutMs = Number(process.env.BROWSER_SPEAK_FAKE_MIC_TURN_TIMEOUT_MS ?? 80000);
|
| 25 |
+
const fakeMicRowTimeoutMs = Number(
|
| 26 |
+
process.env.BROWSER_SPEAK_FAKE_MIC_ROW_TIMEOUT_MS ?? fakeMicTurnTimeoutMs + 30000,
|
| 27 |
+
);
|
| 28 |
+
const stack = {
|
| 29 |
+
device: process.env.BROWSER_SPEAK_DEVICE ?? "wasm",
|
| 30 |
+
llm: process.env.BROWSER_SPEAK_LLM ?? "HuggingFaceTB/SmolLM2-135M-Instruct",
|
| 31 |
+
asr: process.env.BROWSER_SPEAK_ASR ?? "onnx-community/moonshine-base-ONNX",
|
| 32 |
+
voice: process.env.BROWSER_SPEAK_VOICE ?? "F2",
|
| 33 |
+
ttsSteps: Number(process.env.BROWSER_SPEAK_TTS_STEPS ?? 2),
|
| 34 |
+
vadSilenceMs: Number(process.env.BROWSER_SPEAK_VAD_SILENCE_MS ?? 480),
|
| 35 |
+
partialAsr: process.env.BROWSER_SPEAK_PARTIAL_ASR !== "false",
|
| 36 |
+
};
|
| 37 |
+
|
| 38 |
+
async function main() {
|
| 39 |
+
await ensureServer();
|
| 40 |
+
await mkdir(dirname(fixturePath), { recursive: true });
|
| 41 |
+
await mkdir(dirname(resultPath), { recursive: true });
|
| 42 |
+
|
| 43 |
+
await rm(profileDir, { recursive: true, force: true });
|
| 44 |
+
if (reuseFixture && existsSync(fixturePath)) {
|
| 45 |
+
console.log(`Reusing fake microphone fixture: ${fixturePath}`);
|
| 46 |
+
} else {
|
| 47 |
+
const fixtureBrowser = launchBrowser(9341, profileDir);
|
| 48 |
+
try {
|
| 49 |
+
const client = await connectToPage(9341, fixtureBrowser);
|
| 50 |
+
await waitForBenchApi(client);
|
| 51 |
+
await runPageTask(client, `window.browserSpeakBench.loadStack(${JSON.stringify(stack)})`, {
|
| 52 |
+
label: "fixture model load",
|
| 53 |
+
timeoutMs: 240000,
|
| 54 |
+
});
|
| 55 |
+
const fixture = await runPageTask(
|
| 56 |
+
client,
|
| 57 |
+
`window.browserSpeakBench.synthesizeAudio(${JSON.stringify({ text: prompt, voice: stack.voice, steps: 2, speed: 1.05 })})`,
|
| 58 |
+
{
|
| 59 |
+
label: "fixture synthesis",
|
| 60 |
+
timeoutMs: 90000,
|
| 61 |
+
},
|
| 62 |
+
);
|
| 63 |
+
await writeFile(fixturePath, wavBuffer(fixture.audio, fixture.sampleRate));
|
| 64 |
+
await client.closeBrowser();
|
| 65 |
+
console.log(`Wrote fake microphone fixture: ${fixturePath}`);
|
| 66 |
+
} finally {
|
| 67 |
+
await stopBrowser(fixtureBrowser, profileDir, { cleanupProfile: false });
|
| 68 |
+
}
|
| 69 |
+
}
|
| 70 |
+
|
| 71 |
+
const fakeMicBrowser = launchBrowser(9342, profileDir, [
|
| 72 |
+
"--use-fake-ui-for-media-stream",
|
| 73 |
+
"--use-fake-device-for-media-stream",
|
| 74 |
+
`--use-file-for-fake-audio-capture=${fixturePath}`,
|
| 75 |
+
]);
|
| 76 |
+
|
| 77 |
+
let client = null;
|
| 78 |
+
try {
|
| 79 |
+
client = await connectToPage(9342, fakeMicBrowser);
|
| 80 |
+
await waitForBenchApi(client);
|
| 81 |
+
await runPageTask(client, `window.browserSpeakBench.loadStack(${JSON.stringify(stack)})`, {
|
| 82 |
+
label: "fake-mic model load",
|
| 83 |
+
timeoutMs: 240000,
|
| 84 |
+
});
|
| 85 |
+
await runPageTask(client, "window.browserSpeakBench.clearResults()", { label: "clear results" });
|
| 86 |
+
for (let i = 1; i <= 3; i += 1) {
|
| 87 |
+
console.log(`Running fake microphone scripted row ${i}/3`);
|
| 88 |
+
const snapshot = await runPageTask(
|
| 89 |
+
client,
|
| 90 |
+
`window.browserSpeakBench.runMic(${JSON.stringify({
|
| 91 |
+
timeoutMs: fakeMicTurnTimeoutMs,
|
| 92 |
+
stopMicAfterTranscript: true,
|
| 93 |
+
requireExactTranscript: true,
|
| 94 |
+
})})`,
|
| 95 |
+
{
|
| 96 |
+
label: `fake microphone row ${i}`,
|
| 97 |
+
timeoutMs: fakeMicRowTimeoutMs,
|
| 98 |
+
},
|
| 99 |
+
);
|
| 100 |
+
const latest = snapshot.results[0];
|
| 101 |
+
if (!latest || latest.kind !== "mic" || latest.error) {
|
| 102 |
+
await writeFailurePayload(client, latest?.error || `Missing fake microphone row ${i}.`);
|
| 103 |
+
throw new Error(latest?.error || `Missing fake microphone row ${i}.`);
|
| 104 |
+
}
|
| 105 |
+
await runPageTask(client, "window.browserSpeakBench.stopMic()", { label: "stop microphone" });
|
| 106 |
+
}
|
| 107 |
+
const exportPayload = await runPageTask(client, "window.browserSpeakBench.exportResults()", {
|
| 108 |
+
label: "export results",
|
| 109 |
+
});
|
| 110 |
+
exportPayload.sourceFingerprint = await sourceFingerprint();
|
| 111 |
+
await writeFile(resultPath, `${JSON.stringify(exportPayload, null, 2)}\n`);
|
| 112 |
+
const current = exportPayload.summary.current ?? exportPayload.summary.all;
|
| 113 |
+
const micRows = exportPayload.results.filter((result) => result.kind === "mic" && !result.error);
|
| 114 |
+
console.log(`Wrote fake microphone benchmark JSON: ${resultPath}`);
|
| 115 |
+
console.log(`Scripted mic rows: ${current.micRuns}/${current.micTargetRuns}`);
|
| 116 |
+
console.log(`Median mic ASR: ${formatMs(median(micRows.map((result) => result.asrMs)))}`);
|
| 117 |
+
console.log(`Median mic WER: ${formatPercent(current.micMedianWer)}`);
|
| 118 |
+
console.log(`Median mic first token: ${formatMs(median(micRows.map((result) => result.firstTokenMs)))}`);
|
| 119 |
+
console.log(`Median mic speech-end-to-audio: ${formatMs(current.micMedianSpeechEndToFirstAudioMs)}`);
|
| 120 |
+
console.log(`Median mic speech-end-to-audio-done: ${formatMs(current.micMedianSpeechEndToAudioEndMs)}`);
|
| 121 |
+
await client.closeBrowser();
|
| 122 |
+
} catch (error) {
|
| 123 |
+
if (client) await writeFailurePayload(client, error.message).catch(() => {});
|
| 124 |
+
throw error;
|
| 125 |
+
} finally {
|
| 126 |
+
await stopBrowser(fakeMicBrowser, profileDir);
|
| 127 |
+
}
|
| 128 |
+
}
|
| 129 |
+
|
| 130 |
+
async function writeFailurePayload(client, error) {
|
| 131 |
+
const snapshot = await client
|
| 132 |
+
.evaluate("window.browserSpeakBench?.state?.() ?? null", pollTimeoutMs)
|
| 133 |
+
.catch(() => null);
|
| 134 |
+
const fingerprint = await sourceFingerprint().catch(() => null);
|
| 135 |
+
await writeFile(
|
| 136 |
+
resultPath,
|
| 137 |
+
`${JSON.stringify(
|
| 138 |
+
snapshot ? { error, sourceFingerprint: fingerprint, snapshot } : { error, sourceFingerprint: fingerprint },
|
| 139 |
+
null,
|
| 140 |
+
2,
|
| 141 |
+
)}\n`,
|
| 142 |
+
);
|
| 143 |
+
console.log(`Wrote fake microphone failure JSON: ${resultPath}`);
|
| 144 |
+
}
|
| 145 |
+
|
| 146 |
+
async function ensureServer() {
|
| 147 |
+
const response = await fetch(url).catch((error) => {
|
| 148 |
+
throw new Error(`Could not reach ${url}: ${error.message}`);
|
| 149 |
+
});
|
| 150 |
+
if (!response.ok) throw new Error(`${url} returned HTTP ${response.status}`);
|
| 151 |
+
}
|
| 152 |
+
|
| 153 |
+
async function waitForBenchApi(client) {
|
| 154 |
+
const deadline = Date.now() + 15000;
|
| 155 |
+
while (Date.now() < deadline) {
|
| 156 |
+
try {
|
| 157 |
+
if (await client.evaluate("Boolean(window.browserSpeakBench)")) return;
|
| 158 |
+
} catch {
|
| 159 |
+
// The target may still be navigating and can destroy the execution context.
|
| 160 |
+
}
|
| 161 |
+
await sleep(100);
|
| 162 |
+
}
|
| 163 |
+
throw new Error("window.browserSpeakBench was not installed.");
|
| 164 |
+
}
|
| 165 |
+
|
| 166 |
+
async function runPageTask(client, expression, { label = "page task", timeoutMs = 30000 } = {}) {
|
| 167 |
+
const taskId = `task_${Date.now()}_${Math.random().toString(16).slice(2)}`;
|
| 168 |
+
await client.evaluate(`(() => {
|
| 169 |
+
const taskId = ${JSON.stringify(taskId)};
|
| 170 |
+
window.__browserSpeakHarnessTasks ||= {};
|
| 171 |
+
window.__browserSpeakHarnessTasks[taskId] = { done: false, label: ${JSON.stringify(label)} };
|
| 172 |
+
Promise.resolve(${expression})
|
| 173 |
+
.then((value) => {
|
| 174 |
+
window.__browserSpeakHarnessTasks[taskId] = { done: true, value };
|
| 175 |
+
})
|
| 176 |
+
.catch((error) => {
|
| 177 |
+
window.__browserSpeakHarnessTasks[taskId] = {
|
| 178 |
+
done: true,
|
| 179 |
+
error: error?.stack || error?.message || String(error),
|
| 180 |
+
};
|
| 181 |
+
});
|
| 182 |
+
return true;
|
| 183 |
+
})()`);
|
| 184 |
+
const deadline = Date.now() + timeoutMs;
|
| 185 |
+
let lastEvents = [];
|
| 186 |
+
let lastPollError = "";
|
| 187 |
+
while (Date.now() < deadline) {
|
| 188 |
+
let task = null;
|
| 189 |
+
try {
|
| 190 |
+
task = await client.evaluate(
|
| 191 |
+
`window.__browserSpeakHarnessTasks?.[${JSON.stringify(taskId)}] ?? null`,
|
| 192 |
+
pollTimeoutMs,
|
| 193 |
+
);
|
| 194 |
+
lastPollError = "";
|
| 195 |
+
} catch (error) {
|
| 196 |
+
const message = error.message ?? String(error);
|
| 197 |
+
if (message !== lastPollError) {
|
| 198 |
+
console.log(`${label}: waiting for page response (${message})`);
|
| 199 |
+
lastPollError = message;
|
| 200 |
+
}
|
| 201 |
+
await sleep(500);
|
| 202 |
+
continue;
|
| 203 |
+
}
|
| 204 |
+
if (task?.done) {
|
| 205 |
+
await client
|
| 206 |
+
.evaluate(`delete window.__browserSpeakHarnessTasks?.[${JSON.stringify(taskId)}]`, pollTimeoutMs)
|
| 207 |
+
.catch(() => {});
|
| 208 |
+
if (task.error) throw new Error(`${label} failed: ${task.error}`);
|
| 209 |
+
return task.value;
|
| 210 |
+
}
|
| 211 |
+
const snapshot = await client
|
| 212 |
+
.evaluate(`(() => {
|
| 213 |
+
const state = window.browserSpeakBench?.state?.();
|
| 214 |
+
return state ? {
|
| 215 |
+
modelsLoaded: state.modelsLoaded,
|
| 216 |
+
modelsLoading: state.modelsLoading,
|
| 217 |
+
activeBenchmark: state.activeBenchmark?.kind ?? null,
|
| 218 |
+
events: state.events?.slice(0, 3) ?? [],
|
| 219 |
+
} : null;
|
| 220 |
+
})()`, pollTimeoutMs)
|
| 221 |
+
.catch(() => null);
|
| 222 |
+
const events = snapshot?.events ?? [];
|
| 223 |
+
if (events.join("\\n") !== lastEvents.join("\\n")) {
|
| 224 |
+
lastEvents = events;
|
| 225 |
+
if (events[0]) console.log(`${label}: ${events[0]}`);
|
| 226 |
+
}
|
| 227 |
+
await sleep(500);
|
| 228 |
+
}
|
| 229 |
+
throw new Error(`${label} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`);
|
| 230 |
+
}
|
| 231 |
+
|
| 232 |
+
function launchBrowser(port, profileDir, extraArgs = []) {
|
| 233 |
+
const child = spawn(
|
| 234 |
+
chrome,
|
| 235 |
+
[
|
| 236 |
+
"--headless=new",
|
| 237 |
+
"--no-sandbox",
|
| 238 |
+
"--disable-gpu",
|
| 239 |
+
"--disable-dev-shm-usage",
|
| 240 |
+
"--disable-background-networking",
|
| 241 |
+
"--disable-extensions",
|
| 242 |
+
"--no-default-browser-check",
|
| 243 |
+
"--no-first-run",
|
| 244 |
+
"--autoplay-policy=no-user-gesture-required",
|
| 245 |
+
`--remote-debugging-port=${port}`,
|
| 246 |
+
`--user-data-dir=${profileDir}`,
|
| 247 |
+
...extraArgs,
|
| 248 |
+
],
|
| 249 |
+
{ stdio: ["ignore", "pipe", "pipe"] },
|
| 250 |
+
);
|
| 251 |
+
child.browserLog = "";
|
| 252 |
+
const appendLog = (chunk) => {
|
| 253 |
+
child.browserLog = `${child.browserLog}${chunk}`;
|
| 254 |
+
if (child.browserLog.length > 8000) child.browserLog = child.browserLog.slice(-8000);
|
| 255 |
+
};
|
| 256 |
+
child.stdout.on("data", appendLog);
|
| 257 |
+
child.stderr.on("data", appendLog);
|
| 258 |
+
return child;
|
| 259 |
+
}
|
| 260 |
+
|
| 261 |
+
async function stopBrowser(child, profileDir, { cleanupProfile = true } = {}) {
|
| 262 |
+
if (child.exitCode == null) child.kill("SIGTERM");
|
| 263 |
+
await new Promise((resolve) => {
|
| 264 |
+
child.once("exit", resolve);
|
| 265 |
+
setTimeout(resolve, 3000);
|
| 266 |
+
});
|
| 267 |
+
if (child.exitCode == null) child.kill("SIGKILL");
|
| 268 |
+
if (!cleanupProfile) return;
|
| 269 |
+
for (let attempt = 0; attempt < 5; attempt += 1) {
|
| 270 |
+
try {
|
| 271 |
+
await rm(profileDir, { recursive: true, force: true });
|
| 272 |
+
return;
|
| 273 |
+
} catch (error) {
|
| 274 |
+
if (attempt === 4) {
|
| 275 |
+
console.warn(`Could not remove ${profileDir}: ${error.message}`);
|
| 276 |
+
return;
|
| 277 |
+
}
|
| 278 |
+
await sleep(500);
|
| 279 |
+
}
|
| 280 |
+
}
|
| 281 |
+
}
|
| 282 |
+
|
| 283 |
+
async function connectToPage(port, child) {
|
| 284 |
+
const deadline = Date.now() + 60000;
|
| 285 |
+
let lastError = null;
|
| 286 |
+
while (Date.now() < deadline) {
|
| 287 |
+
if (child.exitCode != null) {
|
| 288 |
+
throw new Error(`Chrome exited before DevTools became available.\n${child.browserLog}`);
|
| 289 |
+
}
|
| 290 |
+
try {
|
| 291 |
+
const version = await fetch(`http://127.0.0.1:${port}/json/version`).then((response) => response.json());
|
| 292 |
+
if (version.webSocketDebuggerUrl) {
|
| 293 |
+
const page = await createPageTarget(port);
|
| 294 |
+
return new CdpClient(page.webSocketDebuggerUrl);
|
| 295 |
+
}
|
| 296 |
+
} catch (error) {
|
| 297 |
+
lastError = error;
|
| 298 |
+
}
|
| 299 |
+
await sleep(250);
|
| 300 |
+
}
|
| 301 |
+
throw new Error(
|
| 302 |
+
`Could not connect to Chrome DevTools on port ${port}: ${lastError?.message ?? "unknown error"}\n${child.browserLog}`,
|
| 303 |
+
);
|
| 304 |
+
}
|
| 305 |
+
|
| 306 |
+
async function createPageTarget(port) {
|
| 307 |
+
for (const method of ["PUT", "GET"]) {
|
| 308 |
+
const response = await fetch(`http://127.0.0.1:${port}/json/new?${encodeURIComponent(url)}`, {
|
| 309 |
+
method,
|
| 310 |
+
}).catch(() => null);
|
| 311 |
+
if (response?.ok) {
|
| 312 |
+
const target = await response.json();
|
| 313 |
+
if (target.webSocketDebuggerUrl) return target;
|
| 314 |
+
}
|
| 315 |
+
}
|
| 316 |
+
const targets = await fetch(`http://127.0.0.1:${port}/json`).then((response) => response.json());
|
| 317 |
+
const page = targets.find((target) => target.type === "page" && target.url === url);
|
| 318 |
+
if (page?.webSocketDebuggerUrl) return page;
|
| 319 |
+
throw new Error("Could not create or find a page target.");
|
| 320 |
+
}
|
| 321 |
+
|
| 322 |
+
class CdpClient {
|
| 323 |
+
constructor(webSocketUrl) {
|
| 324 |
+
this.nextId = 1;
|
| 325 |
+
this.pending = new Map();
|
| 326 |
+
this.socket = new WebSocket(webSocketUrl);
|
| 327 |
+
this.opened = new Promise((resolve, reject) => {
|
| 328 |
+
this.socket.onopen = resolve;
|
| 329 |
+
this.socket.onerror = reject;
|
| 330 |
+
this.socket.onmessage = (event) => this.onMessage(event);
|
| 331 |
+
});
|
| 332 |
+
}
|
| 333 |
+
|
| 334 |
+
onMessage(event) {
|
| 335 |
+
const message = JSON.parse(event.data);
|
| 336 |
+
if (!message.id || !this.pending.has(message.id)) return;
|
| 337 |
+
const { resolve: onResolve, reject } = this.pending.get(message.id);
|
| 338 |
+
this.pending.delete(message.id);
|
| 339 |
+
if (message.error) reject(new Error(message.error.message));
|
| 340 |
+
else onResolve(message.result);
|
| 341 |
+
}
|
| 342 |
+
|
| 343 |
+
async call(method, params = {}, timeoutMs = protocolTimeoutMs) {
|
| 344 |
+
await this.opened;
|
| 345 |
+
const id = this.nextId++;
|
| 346 |
+
this.socket.send(JSON.stringify({ id, method, params }));
|
| 347 |
+
return new Promise((resolvePromise, reject) => {
|
| 348 |
+
const timer = setTimeout(() => {
|
| 349 |
+
this.pending.delete(id);
|
| 350 |
+
reject(new Error(`${method} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`));
|
| 351 |
+
}, timeoutMs);
|
| 352 |
+
this.pending.set(id, {
|
| 353 |
+
resolve: (value) => {
|
| 354 |
+
clearTimeout(timer);
|
| 355 |
+
resolvePromise(value);
|
| 356 |
+
},
|
| 357 |
+
reject: (error) => {
|
| 358 |
+
clearTimeout(timer);
|
| 359 |
+
reject(error);
|
| 360 |
+
},
|
| 361 |
+
});
|
| 362 |
+
});
|
| 363 |
+
}
|
| 364 |
+
|
| 365 |
+
async evaluate(expression, timeoutMs = protocolTimeoutMs) {
|
| 366 |
+
const result = await this.call("Runtime.evaluate", {
|
| 367 |
+
expression,
|
| 368 |
+
returnByValue: true,
|
| 369 |
+
}, timeoutMs);
|
| 370 |
+
if (result.exceptionDetails) throw new Error(formatException(result.exceptionDetails));
|
| 371 |
+
return result.result.value;
|
| 372 |
+
}
|
| 373 |
+
|
| 374 |
+
async evaluateAsync(expression, timeoutMs = protocolTimeoutMs) {
|
| 375 |
+
const result = await this.call("Runtime.evaluate", {
|
| 376 |
+
expression,
|
| 377 |
+
awaitPromise: true,
|
| 378 |
+
returnByValue: true,
|
| 379 |
+
}, timeoutMs);
|
| 380 |
+
if (result.exceptionDetails) throw new Error(formatException(result.exceptionDetails));
|
| 381 |
+
return result.result.value;
|
| 382 |
+
}
|
| 383 |
+
|
| 384 |
+
closeBrowser() {
|
| 385 |
+
try {
|
| 386 |
+
this.socket.send(JSON.stringify({ id: this.nextId++, method: "Browser.close", params: {} }));
|
| 387 |
+
} catch {
|
| 388 |
+
// The surrounding process cleanup handles already-closed targets.
|
| 389 |
+
}
|
| 390 |
+
try {
|
| 391 |
+
this.socket.close();
|
| 392 |
+
} catch {
|
| 393 |
+
// Ignore close races.
|
| 394 |
+
}
|
| 395 |
+
}
|
| 396 |
+
}
|
| 397 |
+
|
| 398 |
+
function formatException(exceptionDetails) {
|
| 399 |
+
const exception = exceptionDetails.exception;
|
| 400 |
+
return exception?.description ?? exception?.value ?? exceptionDetails.text ?? "Evaluation failed.";
|
| 401 |
+
}
|
| 402 |
+
|
| 403 |
+
function wavBuffer(samples, sourceSampleRate) {
|
| 404 |
+
const targetRate = 48000;
|
| 405 |
+
const lead = new Float32Array(Math.round(targetRate * 2.0));
|
| 406 |
+
const speech = normalizeSpeech(resample(samples, sourceSampleRate, targetRate));
|
| 407 |
+
const trail = new Float32Array(Math.round(targetRate * 2.0));
|
| 408 |
+
const combined = new Float32Array(lead.length + speech.length + trail.length);
|
| 409 |
+
combined.set(lead, 0);
|
| 410 |
+
combined.set(speech, lead.length);
|
| 411 |
+
combined.set(trail, lead.length + speech.length);
|
| 412 |
+
|
| 413 |
+
const dataBytes = combined.length * 2;
|
| 414 |
+
const buffer = Buffer.alloc(44 + dataBytes);
|
| 415 |
+
buffer.write("RIFF", 0);
|
| 416 |
+
buffer.writeUInt32LE(36 + dataBytes, 4);
|
| 417 |
+
buffer.write("WAVE", 8);
|
| 418 |
+
buffer.write("fmt ", 12);
|
| 419 |
+
buffer.writeUInt32LE(16, 16);
|
| 420 |
+
buffer.writeUInt16LE(1, 20);
|
| 421 |
+
buffer.writeUInt16LE(1, 22);
|
| 422 |
+
buffer.writeUInt32LE(targetRate, 24);
|
| 423 |
+
buffer.writeUInt32LE(targetRate * 2, 28);
|
| 424 |
+
buffer.writeUInt16LE(2, 32);
|
| 425 |
+
buffer.writeUInt16LE(16, 34);
|
| 426 |
+
buffer.write("data", 36);
|
| 427 |
+
buffer.writeUInt32LE(dataBytes, 40);
|
| 428 |
+
for (let i = 0; i < combined.length; i += 1) {
|
| 429 |
+
const value = Math.max(-1, Math.min(1, combined[i]));
|
| 430 |
+
buffer.writeInt16LE(Math.round(value * 32767), 44 + i * 2);
|
| 431 |
+
}
|
| 432 |
+
return buffer;
|
| 433 |
+
}
|
| 434 |
+
|
| 435 |
+
function normalizeSpeech(samples) {
|
| 436 |
+
let peak = 0;
|
| 437 |
+
for (const sample of samples) {
|
| 438 |
+
peak = Math.max(peak, Math.abs(sample));
|
| 439 |
+
}
|
| 440 |
+
if (peak < 0.01) return samples;
|
| 441 |
+
const gain = Math.min(3, 0.85 / peak);
|
| 442 |
+
const output = new Float32Array(samples.length);
|
| 443 |
+
for (let i = 0; i < samples.length; i += 1) {
|
| 444 |
+
output[i] = samples[i] * gain;
|
| 445 |
+
}
|
| 446 |
+
return output;
|
| 447 |
+
}
|
| 448 |
+
|
| 449 |
+
function resample(samples, sourceRate, targetRate) {
|
| 450 |
+
if (sourceRate === targetRate) return Float32Array.from(samples);
|
| 451 |
+
const output = new Float32Array(Math.ceil((samples.length * targetRate) / sourceRate));
|
| 452 |
+
for (let i = 0; i < output.length; i += 1) {
|
| 453 |
+
const position = (i * sourceRate) / targetRate;
|
| 454 |
+
const left = Math.floor(position);
|
| 455 |
+
const right = Math.min(samples.length - 1, left + 1);
|
| 456 |
+
const weight = position - left;
|
| 457 |
+
output[i] = samples[left] * (1 - weight) + samples[right] * weight;
|
| 458 |
+
}
|
| 459 |
+
return output;
|
| 460 |
+
}
|
| 461 |
+
|
| 462 |
+
function formatMs(value) {
|
| 463 |
+
if (!Number.isFinite(value)) return "-";
|
| 464 |
+
if (value < 1000) return `${Math.round(value)} ms`;
|
| 465 |
+
return `${(value / 1000).toFixed(2)} s`;
|
| 466 |
+
}
|
| 467 |
+
|
| 468 |
+
function formatPercent(value) {
|
| 469 |
+
if (!Number.isFinite(value)) return "-";
|
| 470 |
+
return `${Math.round(value * 100)}%`;
|
| 471 |
+
}
|
| 472 |
+
|
| 473 |
+
function median(values) {
|
| 474 |
+
const finite = values.filter(Number.isFinite).sort((a, b) => a - b);
|
| 475 |
+
if (finite.length === 0) return null;
|
| 476 |
+
const middle = Math.floor(finite.length / 2);
|
| 477 |
+
if (finite.length % 2 === 1) return finite[middle];
|
| 478 |
+
return (finite[middle - 1] + finite[middle]) / 2;
|
| 479 |
+
}
|
| 480 |
+
|
| 481 |
+
function sleep(ms) {
|
| 482 |
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
| 483 |
+
}
|
| 484 |
+
|
| 485 |
+
await main();
|
tools/run-final-validation.mjs
ADDED
|
@@ -0,0 +1,307 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env node
|
| 2 |
+
import { spawn } from "node:child_process";
|
| 3 |
+
import { readFile, writeFile, mkdir } from "node:fs/promises";
|
| 4 |
+
import { tmpdir } from "node:os";
|
| 5 |
+
import { dirname, resolve } from "node:path";
|
| 6 |
+
import { fileURLToPath } from "node:url";
|
| 7 |
+
import { sourceFingerprint } from "./source-fingerprint.mjs";
|
| 8 |
+
|
| 9 |
+
const scriptDir = dirname(fileURLToPath(import.meta.url));
|
| 10 |
+
const resultPath = resolve(process.env.BROWSER_SPEAK_FINAL_JSON ?? `${tmpdir()}/browser-speak-final-validation.json`);
|
| 11 |
+
const auditPath = resolve(process.env.BROWSER_SPEAK_AUDIT_JSON ?? `${tmpdir()}/browser-speak-validation-audit.json`);
|
| 12 |
+
const soft = process.env.BROWSER_SPEAK_FINAL_SOFT === "true";
|
| 13 |
+
const skipLocal = process.env.BROWSER_SPEAK_FINAL_SKIP_LOCAL === "true";
|
| 14 |
+
const skipUi = skipLocal || process.env.BROWSER_SPEAK_FINAL_SKIP_UI === "true";
|
| 15 |
+
const skipEvidenceExport = skipLocal || process.env.BROWSER_SPEAK_FINAL_SKIP_EVIDENCE_EXPORT === "true";
|
| 16 |
+
const skipClientSide = skipLocal || process.env.BROWSER_SPEAK_FINAL_SKIP_CLIENT_SIDE === "true";
|
| 17 |
+
const skipLoopback = skipLocal || process.env.BROWSER_SPEAK_FINAL_SKIP_LOOPBACK === "true";
|
| 18 |
+
const skipWebgpu = process.env.BROWSER_SPEAK_FINAL_SKIP_WEBGPU === "true";
|
| 19 |
+
const realMicMode = parseRealMicMode(process.env.BROWSER_SPEAK_FINAL_REAL_MIC ?? "skip");
|
| 20 |
+
|
| 21 |
+
const envDefaults = {
|
| 22 |
+
BROWSER_SPEAK_CLIENT_SIDE_REUSE_PROFILE: "true",
|
| 23 |
+
BROWSER_SPEAK_LOCAL_REUSE_PROFILE: "true",
|
| 24 |
+
BROWSER_SPEAK_LOAD_TIMEOUT_MS: "900000",
|
| 25 |
+
BROWSER_SPEAK_TASK_TIMEOUT_MS: "240000",
|
| 26 |
+
BROWSER_SPEAK_VOICE_PRELOAD_TIMEOUT_MS: "240000",
|
| 27 |
+
};
|
| 28 |
+
|
| 29 |
+
const steps = [
|
| 30 |
+
{
|
| 31 |
+
name: "UI smoke",
|
| 32 |
+
script: "run-ui-smoke.mjs",
|
| 33 |
+
artifactEnv: "BROWSER_SPEAK_UI_JSON",
|
| 34 |
+
defaultArtifact: `${tmpdir()}/browser-speak-ui-smoke.json`,
|
| 35 |
+
skipped: skipUi,
|
| 36 |
+
skipReason: "disabled by BROWSER_SPEAK_FINAL_SKIP_UI or BROWSER_SPEAK_FINAL_SKIP_LOCAL",
|
| 37 |
+
},
|
| 38 |
+
{
|
| 39 |
+
name: "evidence export smoke",
|
| 40 |
+
script: "run-evidence-export-smoke.mjs",
|
| 41 |
+
artifactEnv: "BROWSER_SPEAK_EVIDENCE_EXPORT_JSON",
|
| 42 |
+
defaultArtifact: `${tmpdir()}/browser-speak-evidence-export-smoke.json`,
|
| 43 |
+
skipped: skipEvidenceExport,
|
| 44 |
+
skipReason: "disabled by BROWSER_SPEAK_FINAL_SKIP_EVIDENCE_EXPORT or BROWSER_SPEAK_FINAL_SKIP_LOCAL",
|
| 45 |
+
},
|
| 46 |
+
{
|
| 47 |
+
name: "client-side/no-server smoke",
|
| 48 |
+
script: "run-client-side-smoke.mjs",
|
| 49 |
+
artifactEnv: "BROWSER_SPEAK_CLIENT_SIDE_JSON",
|
| 50 |
+
defaultArtifact: `${tmpdir()}/browser-speak-client-side-smoke.json`,
|
| 51 |
+
skipped: skipClientSide,
|
| 52 |
+
skipReason: "disabled by BROWSER_SPEAK_FINAL_SKIP_CLIENT_SIDE or BROWSER_SPEAK_FINAL_SKIP_LOCAL",
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
name: "loopback stability",
|
| 56 |
+
script: "run-loopback-series.mjs",
|
| 57 |
+
artifactEnv: "BROWSER_SPEAK_LOOPBACK_JSON",
|
| 58 |
+
defaultArtifact: `${tmpdir()}/browser-speak-loopback-series.json`,
|
| 59 |
+
skipped: skipLoopback,
|
| 60 |
+
skipReason: "disabled by BROWSER_SPEAK_FINAL_SKIP_LOOPBACK or BROWSER_SPEAK_FINAL_SKIP_LOCAL",
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
name: "hardware WebGPU benchmark",
|
| 64 |
+
script: "run-webgpu-benchmark.mjs",
|
| 65 |
+
artifactEnv: "BROWSER_SPEAK_WEBGPU_JSON",
|
| 66 |
+
defaultArtifact: `${tmpdir()}/browser-speak-webgpu-results.json`,
|
| 67 |
+
skipped: skipWebgpu,
|
| 68 |
+
skipReason: "disabled by BROWSER_SPEAK_FINAL_SKIP_WEBGPU",
|
| 69 |
+
},
|
| 70 |
+
{
|
| 71 |
+
name: realMicMode === "dry-run" ? "real microphone preflight" : "real microphone validation",
|
| 72 |
+
script: "run-real-mic-series.mjs",
|
| 73 |
+
artifactEnv: "BROWSER_SPEAK_REAL_MIC_JSON",
|
| 74 |
+
defaultArtifact:
|
| 75 |
+
realMicMode === "dry-run"
|
| 76 |
+
? `${tmpdir()}/browser-speak-real-mic-series-dry-run.json`
|
| 77 |
+
: `${tmpdir()}/browser-speak-real-mic-series.json`,
|
| 78 |
+
skipped: realMicMode === "skip",
|
| 79 |
+
skipReason: "set BROWSER_SPEAK_FINAL_REAL_MIC=true to collect rows, or dry-run for preflight only",
|
| 80 |
+
env: realMicMode === "dry-run" ? { BROWSER_SPEAK_REAL_MIC_DRY_RUN: "true" } : {},
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
name: "validation audit",
|
| 84 |
+
script: "audit-validation.mjs",
|
| 85 |
+
artifactEnv: "BROWSER_SPEAK_AUDIT_JSON",
|
| 86 |
+
defaultArtifact: `${tmpdir()}/browser-speak-validation-audit.json`,
|
| 87 |
+
env: soft ? { BROWSER_SPEAK_AUDIT_SOFT: "true" } : {},
|
| 88 |
+
},
|
| 89 |
+
];
|
| 90 |
+
|
| 91 |
+
const commandResults = [];
|
| 92 |
+
|
| 93 |
+
async function main() {
|
| 94 |
+
await mkdir(dirname(resultPath), { recursive: true });
|
| 95 |
+
await writeSummary({ status: "running" });
|
| 96 |
+
|
| 97 |
+
for (const step of steps) {
|
| 98 |
+
const result = await runStep(step);
|
| 99 |
+
commandResults.push(result);
|
| 100 |
+
await writeSummary({ status: "running" });
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
+
const audit = await readJson(auditPath);
|
| 104 |
+
const commandFailures = commandResults.filter((result) => result.commandStatus === "fail");
|
| 105 |
+
const auditPassed = audit?.passed === true;
|
| 106 |
+
await writeSummary({
|
| 107 |
+
status: "complete",
|
| 108 |
+
audit,
|
| 109 |
+
commandFailures,
|
| 110 |
+
auditPassed,
|
| 111 |
+
});
|
| 112 |
+
|
| 113 |
+
console.log(`Wrote final validation JSON: ${resultPath}`);
|
| 114 |
+
printFinalSummary(audit, commandFailures);
|
| 115 |
+
if (commandFailures.length > 0 || (!soft && !auditPassed)) process.exitCode = 1;
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
async function runStep(step) {
|
| 119 |
+
const artifactPath = resolve(process.env[step.artifactEnv] ?? step.defaultArtifact);
|
| 120 |
+
const command = `${process.execPath} tools/${step.script}`;
|
| 121 |
+
if (step.skipped) {
|
| 122 |
+
console.log(`Skipping ${step.name}: ${step.skipReason}.`);
|
| 123 |
+
return {
|
| 124 |
+
name: step.name,
|
| 125 |
+
status: "skip",
|
| 126 |
+
commandStatus: "skip",
|
| 127 |
+
evidenceStatus: "skip",
|
| 128 |
+
command,
|
| 129 |
+
artifactPath,
|
| 130 |
+
reason: step.skipReason,
|
| 131 |
+
durationMs: 0,
|
| 132 |
+
};
|
| 133 |
+
}
|
| 134 |
+
|
| 135 |
+
console.log(`Running ${step.name}: ${command}`);
|
| 136 |
+
const startedAt = Date.now();
|
| 137 |
+
const exitCode = await runChild(process.execPath, [resolve(scriptDir, step.script)], envWithDefaults(step.env));
|
| 138 |
+
const durationMs = Date.now() - startedAt;
|
| 139 |
+
const commandStatus = exitCode === 0 ? "pass" : "fail";
|
| 140 |
+
const evidence = await evidenceForStep(step, artifactPath);
|
| 141 |
+
console.log(formatStepResult(step.name, commandStatus, evidence?.status, durationMs));
|
| 142 |
+
return {
|
| 143 |
+
name: step.name,
|
| 144 |
+
status: evidence?.status ?? commandStatus,
|
| 145 |
+
commandStatus,
|
| 146 |
+
evidenceStatus: evidence?.status ?? null,
|
| 147 |
+
command,
|
| 148 |
+
artifactPath,
|
| 149 |
+
exitCode,
|
| 150 |
+
durationMs,
|
| 151 |
+
evidence,
|
| 152 |
+
};
|
| 153 |
+
}
|
| 154 |
+
|
| 155 |
+
function runChild(command, args, env) {
|
| 156 |
+
return new Promise((resolvePromise) => {
|
| 157 |
+
const child = spawn(command, args, { env, stdio: "inherit" });
|
| 158 |
+
child.on("error", (error) => {
|
| 159 |
+
console.error(`${command} failed to start: ${error.message}`);
|
| 160 |
+
resolvePromise(1);
|
| 161 |
+
});
|
| 162 |
+
child.on("exit", (code) => resolvePromise(code ?? 1));
|
| 163 |
+
});
|
| 164 |
+
}
|
| 165 |
+
|
| 166 |
+
function envWithDefaults(overrides = {}) {
|
| 167 |
+
const env = { ...process.env };
|
| 168 |
+
for (const [key, value] of Object.entries(envDefaults)) {
|
| 169 |
+
if (!(key in env)) env[key] = value;
|
| 170 |
+
}
|
| 171 |
+
return { ...env, ...overrides };
|
| 172 |
+
}
|
| 173 |
+
|
| 174 |
+
async function writeSummary({ status, audit = null, commandFailures = [], auditPassed = null } = {}) {
|
| 175 |
+
const currentAudit = audit ?? (await readJson(auditPath));
|
| 176 |
+
const payload = {
|
| 177 |
+
generatedAt: new Date().toISOString(),
|
| 178 |
+
sourceFingerprint: await sourceFingerprint(),
|
| 179 |
+
status,
|
| 180 |
+
passed: currentAudit?.passed === true,
|
| 181 |
+
completed: status === "complete" && commandFailures.length === 0,
|
| 182 |
+
soft,
|
| 183 |
+
config: {
|
| 184 |
+
resultPath,
|
| 185 |
+
auditPath,
|
| 186 |
+
skipLocal,
|
| 187 |
+
skipUi,
|
| 188 |
+
skipEvidenceExport,
|
| 189 |
+
skipClientSide,
|
| 190 |
+
skipLoopback,
|
| 191 |
+
skipWebgpu,
|
| 192 |
+
realMicMode,
|
| 193 |
+
envDefaults,
|
| 194 |
+
},
|
| 195 |
+
commands: commandResults,
|
| 196 |
+
audit: summarizeAudit(currentAudit),
|
| 197 |
+
};
|
| 198 |
+
if (auditPassed !== null) payload.auditPassed = auditPassed;
|
| 199 |
+
if (commandFailures.length > 0) payload.commandFailures = commandFailures;
|
| 200 |
+
await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`);
|
| 201 |
+
}
|
| 202 |
+
|
| 203 |
+
function summarizeAudit(audit) {
|
| 204 |
+
if (!audit || typeof audit !== "object") return null;
|
| 205 |
+
const checks = Array.isArray(audit.checks) ? audit.checks : [];
|
| 206 |
+
return {
|
| 207 |
+
generatedAt: audit.generatedAt ?? null,
|
| 208 |
+
passed: audit.passed === true,
|
| 209 |
+
required: checks.filter((check) => check.required).map(summarizeCheck),
|
| 210 |
+
supporting: checks.filter((check) => !check.required).map(summarizeCheck),
|
| 211 |
+
nextActions: Array.isArray(audit.nextActions) ? audit.nextActions : [],
|
| 212 |
+
};
|
| 213 |
+
}
|
| 214 |
+
|
| 215 |
+
function summarizeCheck(check) {
|
| 216 |
+
return {
|
| 217 |
+
name: check.name,
|
| 218 |
+
status: check.status,
|
| 219 |
+
message: check.message,
|
| 220 |
+
};
|
| 221 |
+
}
|
| 222 |
+
|
| 223 |
+
async function readJson(path) {
|
| 224 |
+
try {
|
| 225 |
+
return JSON.parse(await readFile(path, "utf8"));
|
| 226 |
+
} catch {
|
| 227 |
+
return null;
|
| 228 |
+
}
|
| 229 |
+
}
|
| 230 |
+
|
| 231 |
+
async function evidenceForStep(step, artifactPath) {
|
| 232 |
+
const artifact = await readJson(artifactPath);
|
| 233 |
+
if (!artifact) return { status: "missing", message: "Artifact JSON was not readable after the step." };
|
| 234 |
+
if (step.script === "audit-validation.mjs") {
|
| 235 |
+
return {
|
| 236 |
+
status: artifact.passed === true ? "pass" : "fail",
|
| 237 |
+
message: artifact.passed === true ? "Final audit passed." : "Final audit did not pass all required evidence gates.",
|
| 238 |
+
};
|
| 239 |
+
}
|
| 240 |
+
if (step.script === "run-webgpu-benchmark.mjs") {
|
| 241 |
+
if (artifact.skipped) {
|
| 242 |
+
return {
|
| 243 |
+
status: "missing",
|
| 244 |
+
message: artifact.reason ?? "WebGPU benchmark was skipped.",
|
| 245 |
+
};
|
| 246 |
+
}
|
| 247 |
+
const completed = Array.isArray(artifact.candidates)
|
| 248 |
+
? artifact.candidates.filter((candidate) => candidate.status === "complete").length
|
| 249 |
+
: 0;
|
| 250 |
+
return {
|
| 251 |
+
status: artifact.webgpu?.available === true && artifact.webgpu?.softwareAdapter !== true && completed > 0 ? "pass" : "missing",
|
| 252 |
+
message: completed > 0 ? `${completed} hardware WebGPU candidate(s) completed.` : "No hardware WebGPU candidate completed.",
|
| 253 |
+
};
|
| 254 |
+
}
|
| 255 |
+
if (step.script === "run-real-mic-series.mjs") {
|
| 256 |
+
if (artifact.dryRun) return { status: "preflight", message: "Real-mic dry run completed; no human speech rows collected." };
|
| 257 |
+
return {
|
| 258 |
+
status: artifact.passed === true ? "pass" : "fail",
|
| 259 |
+
message: artifact.passed === true ? "Real microphone rows passed." : "Real microphone rows are missing or below threshold.",
|
| 260 |
+
};
|
| 261 |
+
}
|
| 262 |
+
if ("passed" in artifact) {
|
| 263 |
+
return {
|
| 264 |
+
status: artifact.passed === true ? "pass" : "fail",
|
| 265 |
+
message: artifact.passed === true ? "Step artifact reports passed." : "Step artifact reports failure.",
|
| 266 |
+
};
|
| 267 |
+
}
|
| 268 |
+
return { status: "unknown", message: "Step artifact did not expose a passed flag." };
|
| 269 |
+
}
|
| 270 |
+
|
| 271 |
+
function parseRealMicMode(value) {
|
| 272 |
+
const normalized = String(value).trim().toLowerCase();
|
| 273 |
+
if (["1", "true", "yes", "real", "run"].includes(normalized)) return "real";
|
| 274 |
+
if (["dry-run", "dryrun", "preflight"].includes(normalized)) return "dry-run";
|
| 275 |
+
return "skip";
|
| 276 |
+
}
|
| 277 |
+
|
| 278 |
+
function printFinalSummary(audit, commandFailures) {
|
| 279 |
+
if (commandFailures.length > 0) {
|
| 280 |
+
console.log(`Command failures: ${commandFailures.map((result) => result.name).join(", ")}`);
|
| 281 |
+
}
|
| 282 |
+
if (!audit) {
|
| 283 |
+
console.log("Validation audit JSON was not available.");
|
| 284 |
+
return;
|
| 285 |
+
}
|
| 286 |
+
const required = Array.isArray(audit.checks) ? audit.checks.filter((check) => check.required) : [];
|
| 287 |
+
console.log(`Final audit: ${audit.passed ? "pass" : "fail"}`);
|
| 288 |
+
for (const check of required) {
|
| 289 |
+
console.log(`${check.status}: ${check.name} - ${check.message}`);
|
| 290 |
+
}
|
| 291 |
+
if (!audit.passed && soft) console.log("Soft mode is enabled, so missing external gates do not fail this runner.");
|
| 292 |
+
}
|
| 293 |
+
|
| 294 |
+
function formatStepResult(name, commandStatus, evidenceStatus, durationMs) {
|
| 295 |
+
const evidenceSuffix = evidenceStatus && evidenceStatus !== commandStatus ? `, evidence ${evidenceStatus}` : "";
|
| 296 |
+
return `${name}: command ${commandStatus}${evidenceSuffix} (${formatDuration(durationMs)})`;
|
| 297 |
+
}
|
| 298 |
+
|
| 299 |
+
function formatDuration(ms) {
|
| 300 |
+
if (ms < 1000) return `${ms} ms`;
|
| 301 |
+
return `${(ms / 1000).toFixed(1)} s`;
|
| 302 |
+
}
|
| 303 |
+
|
| 304 |
+
main().catch((error) => {
|
| 305 |
+
console.error(error.stack ?? error.message);
|
| 306 |
+
process.exitCode = 1;
|
| 307 |
+
});
|
tools/run-hosted-evidence-capture.mjs
ADDED
|
@@ -0,0 +1,245 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env node
|
| 2 |
+
import { spawn } from "node:child_process";
|
| 3 |
+
import { readFile, writeFile, mkdir } from "node:fs/promises";
|
| 4 |
+
import { existsSync } from "node:fs";
|
| 5 |
+
import { tmpdir } from "node:os";
|
| 6 |
+
import { dirname, resolve } from "node:path";
|
| 7 |
+
import { sourceFingerprint } from "./source-fingerprint.mjs";
|
| 8 |
+
|
| 9 |
+
const hostedUrl = normalizeUrl(
|
| 10 |
+
process.env.BROWSER_SPEAK_HOSTED_URL ??
|
| 11 |
+
process.env.BROWSER_SPEAK_URL ??
|
| 12 |
+
"https://mike0021-browser-speak.static.hf.space/",
|
| 13 |
+
);
|
| 14 |
+
const rawRealMicPath = resolve(
|
| 15 |
+
process.env.BROWSER_SPEAK_HOSTED_REAL_MIC_JSON ??
|
| 16 |
+
process.env.BROWSER_SPEAK_REAL_MIC_JSON ??
|
| 17 |
+
`${tmpdir()}/browser-speak-hosted-real-mic-series.json`,
|
| 18 |
+
);
|
| 19 |
+
const evidencePath = resolve(
|
| 20 |
+
process.env.BROWSER_SPEAK_HOSTED_EVIDENCE_JSON ??
|
| 21 |
+
`${tmpdir()}/browser-speak-hosted-browser-evidence.json`,
|
| 22 |
+
);
|
| 23 |
+
const auditPath = resolve(
|
| 24 |
+
process.env.BROWSER_SPEAK_HOSTED_EVIDENCE_AUDIT_JSON ??
|
| 25 |
+
`${tmpdir()}/browser-speak-hosted-browser-evidence-audit.json`,
|
| 26 |
+
);
|
| 27 |
+
const summaryPath = resolve(
|
| 28 |
+
process.env.BROWSER_SPEAK_HOSTED_EVIDENCE_SUMMARY_JSON ??
|
| 29 |
+
`${tmpdir()}/browser-speak-evidence-summary.json`,
|
| 30 |
+
);
|
| 31 |
+
const reportPath = resolve(
|
| 32 |
+
process.env.BROWSER_SPEAK_HOSTED_EVIDENCE_REPORT_JSON ??
|
| 33 |
+
`${tmpdir()}/browser-speak-hosted-evidence-capture.json`,
|
| 34 |
+
);
|
| 35 |
+
const dryRun =
|
| 36 |
+
process.env.BROWSER_SPEAK_HOSTED_EVIDENCE_DRY_RUN === "true" ||
|
| 37 |
+
process.env.BROWSER_SPEAK_REAL_MIC_DRY_RUN === "true";
|
| 38 |
+
const skipAudit = process.env.BROWSER_SPEAK_HOSTED_EVIDENCE_SKIP_AUDIT === "true" || dryRun;
|
| 39 |
+
const skipSummary = process.env.BROWSER_SPEAK_HOSTED_EVIDENCE_SKIP_SUMMARY === "true" || dryRun;
|
| 40 |
+
|
| 41 |
+
async function main() {
|
| 42 |
+
await mkdir(dirname(reportPath), { recursive: true });
|
| 43 |
+
await mkdir(dirname(rawRealMicPath), { recursive: true });
|
| 44 |
+
await mkdir(dirname(evidencePath), { recursive: true });
|
| 45 |
+
await mkdir(dirname(auditPath), { recursive: true });
|
| 46 |
+
await mkdir(dirname(summaryPath), { recursive: true });
|
| 47 |
+
|
| 48 |
+
const steps = [];
|
| 49 |
+
const realMicEnv = {
|
| 50 |
+
...process.env,
|
| 51 |
+
BROWSER_SPEAK_URL: hostedUrl,
|
| 52 |
+
BROWSER_SPEAK_DEVICE: process.env.BROWSER_SPEAK_DEVICE ?? "auto",
|
| 53 |
+
BROWSER_SPEAK_REAL_MIC_JSON: rawRealMicPath,
|
| 54 |
+
BROWSER_SPEAK_HEADLESS: process.env.BROWSER_SPEAK_HEADLESS ?? "false",
|
| 55 |
+
};
|
| 56 |
+
if (dryRun) realMicEnv.BROWSER_SPEAK_REAL_MIC_DRY_RUN = "true";
|
| 57 |
+
|
| 58 |
+
console.log(`Hosted evidence URL: ${hostedUrl}`);
|
| 59 |
+
console.log(`Real-mic artifact: ${rawRealMicPath}`);
|
| 60 |
+
console.log(`Enriched browser evidence: ${evidencePath}`);
|
| 61 |
+
if (dryRun) console.log("Dry run enabled; audit and summary steps will be skipped.");
|
| 62 |
+
|
| 63 |
+
steps.push(
|
| 64 |
+
await runStep("hosted real-mic series", process.execPath, ["tools/run-real-mic-series.mjs"], {
|
| 65 |
+
env: realMicEnv,
|
| 66 |
+
inherit: true,
|
| 67 |
+
}),
|
| 68 |
+
);
|
| 69 |
+
|
| 70 |
+
const enrich = await enrichRealMicArtifact(rawRealMicPath, evidencePath).catch((error) => ({
|
| 71 |
+
label: "enrich browser evidence",
|
| 72 |
+
status: "fail",
|
| 73 |
+
error: error.message,
|
| 74 |
+
}));
|
| 75 |
+
steps.push(enrich);
|
| 76 |
+
|
| 77 |
+
if (!skipAudit && enrich.status === "pass") {
|
| 78 |
+
steps.push(
|
| 79 |
+
await runStep("browser evidence audit", process.execPath, ["tools/audit-browser-evidence.mjs", evidencePath], {
|
| 80 |
+
env: {
|
| 81 |
+
...process.env,
|
| 82 |
+
BROWSER_SPEAK_BROWSER_AUDIT_JSON: auditPath,
|
| 83 |
+
},
|
| 84 |
+
inherit: true,
|
| 85 |
+
}),
|
| 86 |
+
);
|
| 87 |
+
}
|
| 88 |
+
|
| 89 |
+
if (!skipSummary) {
|
| 90 |
+
steps.push(
|
| 91 |
+
await runStep("evidence summary", process.execPath, ["tools/summarize-evidence.mjs"], {
|
| 92 |
+
env: {
|
| 93 |
+
...process.env,
|
| 94 |
+
BROWSER_SPEAK_REAL_MIC_JSON: rawRealMicPath,
|
| 95 |
+
BROWSER_SPEAK_BROWSER_AUDIT_JSON: auditPath,
|
| 96 |
+
BROWSER_SPEAK_EVIDENCE_SUMMARY_JSON: summaryPath,
|
| 97 |
+
},
|
| 98 |
+
inherit: true,
|
| 99 |
+
}),
|
| 100 |
+
);
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
+
const stepsPassed = steps.every((step) => step.status === "pass");
|
| 104 |
+
const passed = !dryRun && stepsPassed;
|
| 105 |
+
const payload = {
|
| 106 |
+
generatedAt: new Date().toISOString(),
|
| 107 |
+
sourceFingerprint: await sourceFingerprint(),
|
| 108 |
+
passed,
|
| 109 |
+
stepsPassed,
|
| 110 |
+
dryRun,
|
| 111 |
+
hostedUrl,
|
| 112 |
+
artifacts: {
|
| 113 |
+
rawRealMicJson: rawRealMicPath,
|
| 114 |
+
browserEvidenceJson: evidencePath,
|
| 115 |
+
browserEvidenceAuditJson: skipAudit ? null : auditPath,
|
| 116 |
+
evidenceSummaryJson: skipSummary ? null : summaryPath,
|
| 117 |
+
},
|
| 118 |
+
steps,
|
| 119 |
+
nextActions: nextActions(steps, dryRun),
|
| 120 |
+
};
|
| 121 |
+
await writeFile(reportPath, `${JSON.stringify(payload, null, 2)}\n`);
|
| 122 |
+
console.log(`Wrote hosted evidence capture report: ${reportPath}`);
|
| 123 |
+
if (!stepsPassed || (!dryRun && !passed)) process.exitCode = 1;
|
| 124 |
+
}
|
| 125 |
+
|
| 126 |
+
async function enrichRealMicArtifact(inputPath, outputPath) {
|
| 127 |
+
const startedAt = Date.now();
|
| 128 |
+
const payload = JSON.parse(await readFile(inputPath, "utf8"));
|
| 129 |
+
const rows = Array.isArray(payload.results) ? payload.results : [];
|
| 130 |
+
const firstEnvironment = rows.map((row) => row.stack?.environment).find((environment) => environment && typeof environment === "object") ?? null;
|
| 131 |
+
const hostMetadata =
|
| 132 |
+
payload.hostMetadata ??
|
| 133 |
+
payload.browserExport?.hostMetadata ??
|
| 134 |
+
payload.runtime?.hostMetadata ??
|
| 135 |
+
firstEnvironment?.hostMetadata ??
|
| 136 |
+
{};
|
| 137 |
+
const runtime = payload.runtime ?? payload.browserExport?.runtime ?? firstEnvironment ?? null;
|
| 138 |
+
const benchmarkSummary = payload.exportSummary ?? payload.browserExport?.summary ?? payload.summary ?? null;
|
| 139 |
+
const enriched = {
|
| 140 |
+
...payload,
|
| 141 |
+
hostMetadata,
|
| 142 |
+
runtime,
|
| 143 |
+
benchmarkSummary,
|
| 144 |
+
browserExport: payload.browserExport ?? {
|
| 145 |
+
generatedAt: payload.generatedAt ?? new Date().toISOString(),
|
| 146 |
+
hostMetadata,
|
| 147 |
+
runtime,
|
| 148 |
+
summary: benchmarkSummary,
|
| 149 |
+
results: rows,
|
| 150 |
+
},
|
| 151 |
+
};
|
| 152 |
+
await writeFile(outputPath, `${JSON.stringify(enriched, null, 2)}\n`);
|
| 153 |
+
return {
|
| 154 |
+
label: "enrich browser evidence",
|
| 155 |
+
status: "pass",
|
| 156 |
+
durationMs: Date.now() - startedAt,
|
| 157 |
+
inputPath,
|
| 158 |
+
outputPath,
|
| 159 |
+
rowCount: rows.length,
|
| 160 |
+
host: hostMetadata.host ?? "",
|
| 161 |
+
hfSpaceCommit: hostMetadata.hfSpaceCommit ?? "",
|
| 162 |
+
};
|
| 163 |
+
}
|
| 164 |
+
|
| 165 |
+
function runStep(label, command, args, { env = process.env, inherit = false } = {}) {
|
| 166 |
+
console.log(`Running ${label}.`);
|
| 167 |
+
const startedAt = Date.now();
|
| 168 |
+
return new Promise((resolvePromise) => {
|
| 169 |
+
const child = spawn(command, args, {
|
| 170 |
+
env,
|
| 171 |
+
stdio: inherit ? "inherit" : ["ignore", "pipe", "pipe"],
|
| 172 |
+
});
|
| 173 |
+
let stdout = "";
|
| 174 |
+
let stderr = "";
|
| 175 |
+
if (!inherit) {
|
| 176 |
+
child.stdout.on("data", (chunk) => {
|
| 177 |
+
stdout += chunk;
|
| 178 |
+
});
|
| 179 |
+
child.stderr.on("data", (chunk) => {
|
| 180 |
+
stderr += chunk;
|
| 181 |
+
});
|
| 182 |
+
}
|
| 183 |
+
child.on("error", (error) => {
|
| 184 |
+
resolvePromise({
|
| 185 |
+
label,
|
| 186 |
+
command: [command, ...args],
|
| 187 |
+
status: "fail",
|
| 188 |
+
error: error.message,
|
| 189 |
+
durationMs: Date.now() - startedAt,
|
| 190 |
+
});
|
| 191 |
+
});
|
| 192 |
+
child.on("exit", (exitCode) => {
|
| 193 |
+
resolvePromise({
|
| 194 |
+
label,
|
| 195 |
+
command: [command, ...args],
|
| 196 |
+
status: exitCode === 0 ? "pass" : "fail",
|
| 197 |
+
exitCode,
|
| 198 |
+
durationMs: Date.now() - startedAt,
|
| 199 |
+
stdout: inherit ? undefined : stdout,
|
| 200 |
+
stderr: inherit ? undefined : stderr,
|
| 201 |
+
});
|
| 202 |
+
});
|
| 203 |
+
});
|
| 204 |
+
}
|
| 205 |
+
|
| 206 |
+
function nextActions(steps, isDryRun) {
|
| 207 |
+
if (isDryRun) {
|
| 208 |
+
return [
|
| 209 |
+
"Run node tools/run-hosted-evidence-capture.mjs from a desktop browser session with a real microphone.",
|
| 210 |
+
"Keep BROWSER_SPEAK_DEVICE=auto on a machine with hardware WebGPU so the mic rows also prove hardware execution.",
|
| 211 |
+
];
|
| 212 |
+
}
|
| 213 |
+
const failed = steps.filter((step) => step.status !== "pass").map((step) => step.label);
|
| 214 |
+
if (failed.length === 0) return ["Use the browser evidence audit JSON in the final validation review."];
|
| 215 |
+
const actions = [];
|
| 216 |
+
if (failed.includes("hosted real-mic series")) {
|
| 217 |
+
actions.push("Rerun from a desktop session with microphone permission granted and no fake-capture Chrome flags.");
|
| 218 |
+
}
|
| 219 |
+
if (failed.includes("browser evidence audit")) {
|
| 220 |
+
actions.push("Check whether the captured rows include three real-mic passes and at least one hardware WebGPU row.");
|
| 221 |
+
}
|
| 222 |
+
if (failed.includes("enrich browser evidence")) {
|
| 223 |
+
actions.push("Inspect the raw real-mic artifact; enrichment requires a JSON file with benchmark rows.");
|
| 224 |
+
}
|
| 225 |
+
return actions;
|
| 226 |
+
}
|
| 227 |
+
|
| 228 |
+
function normalizeUrl(value) {
|
| 229 |
+
return value.endsWith("/") ? value : `${value}/`;
|
| 230 |
+
}
|
| 231 |
+
|
| 232 |
+
main().catch(async (error) => {
|
| 233 |
+
await mkdir(dirname(reportPath), { recursive: true }).catch(() => {});
|
| 234 |
+
const payload = {
|
| 235 |
+
generatedAt: new Date().toISOString(),
|
| 236 |
+
sourceFingerprint: await sourceFingerprint().catch(() => null),
|
| 237 |
+
passed: false,
|
| 238 |
+
dryRun,
|
| 239 |
+
hostedUrl,
|
| 240 |
+
error: error.stack ?? error.message ?? String(error),
|
| 241 |
+
};
|
| 242 |
+
if (!existsSync(reportPath)) await writeFile(reportPath, `${JSON.stringify(payload, null, 2)}\n`).catch(() => {});
|
| 243 |
+
console.error(error.stack ?? error.message);
|
| 244 |
+
process.exitCode = 1;
|
| 245 |
+
});
|
tools/run-hosted-smoke.mjs
ADDED
|
@@ -0,0 +1,270 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env node
|
| 2 |
+
import { spawn } from "node:child_process";
|
| 3 |
+
import { readFile, writeFile, mkdir } from "node:fs/promises";
|
| 4 |
+
import { tmpdir } from "node:os";
|
| 5 |
+
import { dirname, resolve } from "node:path";
|
| 6 |
+
import { sourceFingerprint } from "./source-fingerprint.mjs";
|
| 7 |
+
|
| 8 |
+
const spaceId = process.env.BROWSER_SPEAK_SPACE_ID ?? "Mike0021/browser-speak";
|
| 9 |
+
const hostedUrl = normalizeUrl(
|
| 10 |
+
process.env.BROWSER_SPEAK_HOSTED_URL ?? "https://mike0021-browser-speak.static.hf.space/",
|
| 11 |
+
);
|
| 12 |
+
const resultPath = resolve(process.env.BROWSER_SPEAK_HOSTED_SMOKE_JSON ?? `${tmpdir()}/browser-speak-hosted-smoke.json`);
|
| 13 |
+
const uiJson = resolve(process.env.BROWSER_SPEAK_HOSTED_UI_JSON ?? `${tmpdir()}/browser-speak-hosted-ui-smoke.json`);
|
| 14 |
+
const uiScreenshotDir = resolve(
|
| 15 |
+
process.env.BROWSER_SPEAK_HOSTED_UI_SCREENSHOT_DIR ?? `${tmpdir()}/browser-speak-hosted-ui-smoke`,
|
| 16 |
+
);
|
| 17 |
+
const clientSideJson = resolve(
|
| 18 |
+
process.env.BROWSER_SPEAK_HOSTED_CLIENT_SIDE_JSON ?? `${tmpdir()}/browser-speak-hosted-client-side-smoke.json`,
|
| 19 |
+
);
|
| 20 |
+
const evidenceExportJson = resolve(
|
| 21 |
+
process.env.BROWSER_SPEAK_HOSTED_EVIDENCE_EXPORT_JSON ??
|
| 22 |
+
`${tmpdir()}/browser-speak-hosted-evidence-export-smoke.json`,
|
| 23 |
+
);
|
| 24 |
+
const clientSideProfileDir = resolve(
|
| 25 |
+
process.env.BROWSER_SPEAK_HOSTED_CLIENT_SIDE_PROFILE_DIR ??
|
| 26 |
+
`${tmpdir()}/browser-speak-hosted-client-side-profile`,
|
| 27 |
+
);
|
| 28 |
+
const skipUi = process.env.BROWSER_SPEAK_HOSTED_SKIP_UI === "true";
|
| 29 |
+
const skipEvidenceExport = process.env.BROWSER_SPEAK_HOSTED_SKIP_EVIDENCE_EXPORT === "true";
|
| 30 |
+
const skipClientSide = process.env.BROWSER_SPEAK_HOSTED_SKIP_CLIENT_SIDE === "true";
|
| 31 |
+
const skipHfInfo = process.env.BROWSER_SPEAK_HOSTED_SKIP_HF_INFO === "true";
|
| 32 |
+
|
| 33 |
+
async function main() {
|
| 34 |
+
await mkdir(dirname(resultPath), { recursive: true });
|
| 35 |
+
|
| 36 |
+
const steps = [];
|
| 37 |
+
const hfInfo = skipHfInfo
|
| 38 |
+
? null
|
| 39 |
+
: await runJsonCommand("hf", ["spaces", "info", spaceId], {
|
| 40 |
+
label: "HF Space info",
|
| 41 |
+
}).catch((error) => ({ error: error.message }));
|
| 42 |
+
const hubApiInfo = skipHfInfo ? null : await fetchHubApiInfo().catch((error) => ({ error: error.message }));
|
| 43 |
+
const hostHead = await fetchHostHead().catch((error) => ({ ok: false, error: error.message }));
|
| 44 |
+
|
| 45 |
+
if (!skipUi) {
|
| 46 |
+
steps.push(
|
| 47 |
+
await runStep("hosted UI smoke", process.execPath, ["tools/run-ui-smoke.mjs"], {
|
| 48 |
+
BROWSER_SPEAK_URL: hostedUrl,
|
| 49 |
+
BROWSER_SPEAK_UI_JSON: uiJson,
|
| 50 |
+
BROWSER_SPEAK_UI_SCREENSHOT_DIR: uiScreenshotDir,
|
| 51 |
+
}),
|
| 52 |
+
);
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
if (!skipEvidenceExport) {
|
| 56 |
+
steps.push(
|
| 57 |
+
await runStep("hosted evidence export smoke", process.execPath, ["tools/run-evidence-export-smoke.mjs"], {
|
| 58 |
+
BROWSER_SPEAK_URL: hostedUrl,
|
| 59 |
+
BROWSER_SPEAK_EVIDENCE_EXPORT_JSON: evidenceExportJson,
|
| 60 |
+
}),
|
| 61 |
+
);
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
if (!skipClientSide) {
|
| 65 |
+
steps.push(
|
| 66 |
+
await runStep("hosted client-side/no-server smoke", process.execPath, ["tools/run-client-side-smoke.mjs"], {
|
| 67 |
+
BROWSER_SPEAK_URL: hostedUrl,
|
| 68 |
+
BROWSER_SPEAK_CLIENT_SIDE_JSON: clientSideJson,
|
| 69 |
+
BROWSER_SPEAK_CLIENT_SIDE_PROFILE_DIR: clientSideProfileDir,
|
| 70 |
+
}),
|
| 71 |
+
);
|
| 72 |
+
}
|
| 73 |
+
|
| 74 |
+
const ui = skipUi ? null : await readJsonIfExists(uiJson);
|
| 75 |
+
const evidenceExport = skipEvidenceExport ? null : await readJsonIfExists(evidenceExportJson);
|
| 76 |
+
const clientSide = skipClientSide ? null : await readJsonIfExists(clientSideJson);
|
| 77 |
+
const spaceShas = [hfInfo?.sha, hubApiInfo?.sha].filter(Boolean);
|
| 78 |
+
const commitMatch =
|
| 79 |
+
spaceShas.length > 0 && hostHead?.hfSpaceCommit
|
| 80 |
+
? spaceShas.every((sha) => sha === hostHead.hfSpaceCommit)
|
| 81 |
+
: hfInfo?.error && hubApiInfo?.error
|
| 82 |
+
? false
|
| 83 |
+
: null;
|
| 84 |
+
const cardData = hubApiInfo?.cardData ?? hfInfo?.card_data ?? null;
|
| 85 |
+
const spaceConfigParsed =
|
| 86 |
+
hubApiInfo?.error || skipHfInfo ? null : cardData?.sdk === "static" && Boolean(cardData?.app_file);
|
| 87 |
+
const noServerSmoke =
|
| 88 |
+
clientSide == null
|
| 89 |
+
? null
|
| 90 |
+
: Boolean(
|
| 91 |
+
clientSide.passed &&
|
| 92 |
+
Array.isArray(clientSide.suspects) &&
|
| 93 |
+
clientSide.suspects.length === 0 &&
|
| 94 |
+
Array.isArray(clientSide.benchmarkErrors) &&
|
| 95 |
+
clientSide.benchmarkErrors.length === 0,
|
| 96 |
+
);
|
| 97 |
+
const passed =
|
| 98 |
+
hostHead?.ok !== false &&
|
| 99 |
+
commitMatch !== false &&
|
| 100 |
+
steps.every((step) => step.status === "pass") &&
|
| 101 |
+
(ui == null || ui.passed !== false) &&
|
| 102 |
+
(clientSide == null || noServerSmoke === true);
|
| 103 |
+
|
| 104 |
+
const payload = {
|
| 105 |
+
generatedAt: new Date().toISOString(),
|
| 106 |
+
sourceFingerprint: await sourceFingerprint(),
|
| 107 |
+
passed,
|
| 108 |
+
spaceId,
|
| 109 |
+
hostedUrl,
|
| 110 |
+
hfInfo,
|
| 111 |
+
hubApiInfo,
|
| 112 |
+
hostHead,
|
| 113 |
+
commitMatch,
|
| 114 |
+
spaceConfigParsed,
|
| 115 |
+
noServerSmoke,
|
| 116 |
+
artifacts: {
|
| 117 |
+
uiJson: skipUi ? null : uiJson,
|
| 118 |
+
uiScreenshotDir: skipUi ? null : uiScreenshotDir,
|
| 119 |
+
evidenceExportJson: skipEvidenceExport ? null : evidenceExportJson,
|
| 120 |
+
clientSideJson: skipClientSide ? null : clientSideJson,
|
| 121 |
+
},
|
| 122 |
+
steps,
|
| 123 |
+
uiSummary: ui ? summarizeUiSmoke(ui) : null,
|
| 124 |
+
evidenceExportSummary: evidenceExport ? summarizeEvidenceExportSmoke(evidenceExport) : null,
|
| 125 |
+
clientSideSummary: clientSide ? summarizeClientSideSmoke(clientSide) : null,
|
| 126 |
+
};
|
| 127 |
+
|
| 128 |
+
await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`);
|
| 129 |
+
console.log(`Wrote hosted smoke JSON: ${resultPath}`);
|
| 130 |
+
if (hfInfo?.sha) console.log(`HF Space SHA: ${hfInfo.sha}`);
|
| 131 |
+
if (hubApiInfo?.sha) console.log(`Hub API SHA: ${hubApiInfo.sha}`);
|
| 132 |
+
if (spaceConfigParsed != null) console.log(`Hub card config: ${spaceConfigParsed ? "parsed" : "missing"}`);
|
| 133 |
+
if (hostHead?.hfSpaceCommit) console.log(`Static host commit: ${hostHead.hfSpaceCommit}`);
|
| 134 |
+
if (commitMatch != null) console.log(`Commit match: ${commitMatch ? "yes" : "no"}`);
|
| 135 |
+
if (noServerSmoke != null) console.log(`No-server smoke: ${noServerSmoke ? "pass" : "fail"}`);
|
| 136 |
+
if (!passed) process.exitCode = 1;
|
| 137 |
+
}
|
| 138 |
+
|
| 139 |
+
async function fetchHubApiInfo() {
|
| 140 |
+
const response = await fetch(`https://huggingface.co/api/spaces/${spaceId}`, { cache: "no-store" });
|
| 141 |
+
if (!response.ok) throw new Error(`Hub API returned HTTP ${response.status}`);
|
| 142 |
+
const payload = await response.json();
|
| 143 |
+
return {
|
| 144 |
+
id: payload.id ?? spaceId,
|
| 145 |
+
sha: payload.sha ?? "",
|
| 146 |
+
sdk: payload.sdk ?? "",
|
| 147 |
+
cardData: payload.cardData ?? null,
|
| 148 |
+
runtime: payload.runtime ?? null,
|
| 149 |
+
host: payload.host ?? "",
|
| 150 |
+
lastModified: payload.lastModified ?? null,
|
| 151 |
+
};
|
| 152 |
+
}
|
| 153 |
+
|
| 154 |
+
async function fetchHostHead() {
|
| 155 |
+
const response = await fetch(hostedUrl, { method: "HEAD", cache: "no-store" });
|
| 156 |
+
return {
|
| 157 |
+
ok: response.ok,
|
| 158 |
+
status: response.status,
|
| 159 |
+
statusText: response.statusText,
|
| 160 |
+
url: response.url,
|
| 161 |
+
hfSpaceCommit: response.headers.get("x-repo-commit") ?? "",
|
| 162 |
+
etag: response.headers.get("etag") ?? "",
|
| 163 |
+
contentType: response.headers.get("content-type") ?? "",
|
| 164 |
+
cacheControl: response.headers.get("cache-control") ?? "",
|
| 165 |
+
};
|
| 166 |
+
}
|
| 167 |
+
|
| 168 |
+
async function runStep(label, command, args, env = {}) {
|
| 169 |
+
console.log(`Running ${label}.`);
|
| 170 |
+
const startedAt = Date.now();
|
| 171 |
+
const result = await runCommand(command, args, {
|
| 172 |
+
env: {
|
| 173 |
+
...process.env,
|
| 174 |
+
...env,
|
| 175 |
+
},
|
| 176 |
+
inherit: true,
|
| 177 |
+
});
|
| 178 |
+
return {
|
| 179 |
+
label,
|
| 180 |
+
command: [command, ...args],
|
| 181 |
+
status: result.exitCode === 0 ? "pass" : "fail",
|
| 182 |
+
exitCode: result.exitCode,
|
| 183 |
+
durationMs: Date.now() - startedAt,
|
| 184 |
+
};
|
| 185 |
+
}
|
| 186 |
+
|
| 187 |
+
async function runJsonCommand(command, args, { label }) {
|
| 188 |
+
const result = await runCommand(command, args, { inherit: false });
|
| 189 |
+
if (result.exitCode !== 0) {
|
| 190 |
+
throw new Error(`${label} failed with exit ${result.exitCode}: ${result.stderr || result.stdout}`);
|
| 191 |
+
}
|
| 192 |
+
try {
|
| 193 |
+
return JSON.parse(result.stdout);
|
| 194 |
+
} catch (error) {
|
| 195 |
+
throw new Error(`${label} did not return JSON: ${error.message}`);
|
| 196 |
+
}
|
| 197 |
+
}
|
| 198 |
+
|
| 199 |
+
function runCommand(command, args, { env = process.env, inherit = false } = {}) {
|
| 200 |
+
return new Promise((resolvePromise, reject) => {
|
| 201 |
+
const child = spawn(command, args, {
|
| 202 |
+
env,
|
| 203 |
+
stdio: inherit ? "inherit" : ["ignore", "pipe", "pipe"],
|
| 204 |
+
});
|
| 205 |
+
let stdout = "";
|
| 206 |
+
let stderr = "";
|
| 207 |
+
if (!inherit) {
|
| 208 |
+
child.stdout.on("data", (chunk) => {
|
| 209 |
+
stdout += chunk;
|
| 210 |
+
});
|
| 211 |
+
child.stderr.on("data", (chunk) => {
|
| 212 |
+
stderr += chunk;
|
| 213 |
+
});
|
| 214 |
+
}
|
| 215 |
+
child.on("error", reject);
|
| 216 |
+
child.on("exit", (exitCode) => resolvePromise({ exitCode, stdout, stderr }));
|
| 217 |
+
});
|
| 218 |
+
}
|
| 219 |
+
|
| 220 |
+
async function readJsonIfExists(path) {
|
| 221 |
+
try {
|
| 222 |
+
return JSON.parse(await readFile(path, "utf8"));
|
| 223 |
+
} catch {
|
| 224 |
+
return null;
|
| 225 |
+
}
|
| 226 |
+
}
|
| 227 |
+
|
| 228 |
+
function summarizeUiSmoke(ui) {
|
| 229 |
+
return {
|
| 230 |
+
passed: ui.passed,
|
| 231 |
+
url: ui.url,
|
| 232 |
+
viewports: ui.viewports?.map((viewport) => ({
|
| 233 |
+
name: viewport.name,
|
| 234 |
+
passed: viewport.passed,
|
| 235 |
+
overflowCount: viewport.overflow?.length ?? 0,
|
| 236 |
+
})),
|
| 237 |
+
};
|
| 238 |
+
}
|
| 239 |
+
|
| 240 |
+
function summarizeClientSideSmoke(clientSide) {
|
| 241 |
+
return {
|
| 242 |
+
passed: clientSide.passed,
|
| 243 |
+
url: clientSide.url,
|
| 244 |
+
totalRequestCount: clientSide.summary?.totalRequests ?? clientSide.requests?.length ?? 0,
|
| 245 |
+
benchmarkRequestCount: clientSide.summary?.benchmarkRequests ?? null,
|
| 246 |
+
serverInferenceSuspectCount: clientSide.summary?.serverInferenceSuspects ?? clientSide.suspects?.length ?? 0,
|
| 247 |
+
suspectCount: clientSide.suspects?.length ?? 0,
|
| 248 |
+
benchmarkErrorCount: clientSide.benchmarkErrors?.length ?? 0,
|
| 249 |
+
resultKinds: clientSide.benchmarkResults?.map((result) => result.kind) ?? [],
|
| 250 |
+
browserExport: clientSide.browserExport ?? null,
|
| 251 |
+
hostMetadata: clientSide.hostMetadata ?? clientSide.browserExport?.hostMetadata ?? null,
|
| 252 |
+
};
|
| 253 |
+
}
|
| 254 |
+
|
| 255 |
+
function summarizeEvidenceExportSmoke(evidenceExport) {
|
| 256 |
+
return {
|
| 257 |
+
passed: evidenceExport.passed,
|
| 258 |
+
url: evidenceExport.url,
|
| 259 |
+
summary: evidenceExport.summary ?? null,
|
| 260 |
+
};
|
| 261 |
+
}
|
| 262 |
+
|
| 263 |
+
function normalizeUrl(value) {
|
| 264 |
+
return value.endsWith("/") ? value : `${value}/`;
|
| 265 |
+
}
|
| 266 |
+
|
| 267 |
+
main().catch((error) => {
|
| 268 |
+
console.error(error.stack ?? error.message);
|
| 269 |
+
process.exitCode = 1;
|
| 270 |
+
});
|
tools/run-local-candidate-benchmark.mjs
ADDED
|
@@ -0,0 +1,617 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env node
|
| 2 |
+
import { spawn } from "node:child_process";
|
| 3 |
+
import { existsSync } from "node:fs";
|
| 4 |
+
import { mkdir, rm, writeFile } from "node:fs/promises";
|
| 5 |
+
import { tmpdir } from "node:os";
|
| 6 |
+
import { dirname, resolve } from "node:path";
|
| 7 |
+
|
| 8 |
+
const url = process.env.BROWSER_SPEAK_URL ?? "http://127.0.0.1:5174/";
|
| 9 |
+
const chrome =
|
| 10 |
+
process.env.CHROME_BIN ??
|
| 11 |
+
(existsSync("/opt/google/chrome/chrome")
|
| 12 |
+
? "/opt/google/chrome/chrome"
|
| 13 |
+
: existsSync("/usr/bin/google-chrome")
|
| 14 |
+
? "/usr/bin/google-chrome"
|
| 15 |
+
: "chromium");
|
| 16 |
+
const resultPath = resolve(process.env.BROWSER_SPEAK_LOCAL_JSON ?? `${tmpdir()}/browser-speak-local-candidates.json`);
|
| 17 |
+
const profileDir = resolve(process.env.BROWSER_SPEAK_LOCAL_PROFILE_DIR ?? `${tmpdir()}/browser-speak-local-profile`);
|
| 18 |
+
const protocolTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_TIMEOUT_MS ?? 60000);
|
| 19 |
+
const pollTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_POLL_TIMEOUT_MS ?? 5000);
|
| 20 |
+
const pageUnresponsiveTimeoutMs = Number(process.env.BROWSER_SPEAK_PAGE_UNRESPONSIVE_TIMEOUT_MS ?? 120000);
|
| 21 |
+
const loadTimeoutMs = Number(process.env.BROWSER_SPEAK_LOAD_TIMEOUT_MS ?? 360000);
|
| 22 |
+
const taskTimeoutMs = Number(process.env.BROWSER_SPEAK_TASK_TIMEOUT_MS ?? 180000);
|
| 23 |
+
const ttsWarmup = process.env.BROWSER_SPEAK_TTS_WARMUP === "true";
|
| 24 |
+
const reuseProfile = process.env.BROWSER_SPEAK_LOCAL_REUSE_PROFILE === "true";
|
| 25 |
+
const headless = process.env.BROWSER_SPEAK_HEADLESS !== "false";
|
| 26 |
+
const dryRun = process.env.BROWSER_SPEAK_LOCAL_DRY_RUN === "true";
|
| 27 |
+
const localTasks = parseList(process.env.BROWSER_SPEAK_LOCAL_TASKS ?? "tts,loopback");
|
| 28 |
+
const stacks = buildStacks().slice(0, maxStacks());
|
| 29 |
+
|
| 30 |
+
async function main() {
|
| 31 |
+
const config = benchmarkConfig();
|
| 32 |
+
if (dryRun) {
|
| 33 |
+
const payload = {
|
| 34 |
+
generatedAt: new Date().toISOString(),
|
| 35 |
+
dryRun: true,
|
| 36 |
+
url,
|
| 37 |
+
config,
|
| 38 |
+
candidates: stacks.map((stack) => ({ stack, status: "planned" })),
|
| 39 |
+
};
|
| 40 |
+
await mkdir(dirname(resultPath), { recursive: true });
|
| 41 |
+
await writeJson(payload);
|
| 42 |
+
console.log(`Wrote local candidate dry-run JSON: ${resultPath}`);
|
| 43 |
+
console.log(`Planned ${stacks.length} stack(s): ${stacks.map(stackLabel).join("; ")}`);
|
| 44 |
+
return;
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
await ensureServer();
|
| 48 |
+
await mkdir(dirname(resultPath), { recursive: true });
|
| 49 |
+
if (!reuseProfile) await rm(profileDir, { recursive: true, force: true });
|
| 50 |
+
|
| 51 |
+
const browser = launchBrowser(9344, profileDir);
|
| 52 |
+
const candidates = stacks.map((stack) => ({ stack, status: "pending" }));
|
| 53 |
+
try {
|
| 54 |
+
const client = await connectToPage(9344, browser);
|
| 55 |
+
await waitForBenchApi(client);
|
| 56 |
+
await runPageTask(client, "window.browserSpeakBench.clearResults()", { label: "clear results" });
|
| 57 |
+
|
| 58 |
+
for (const candidate of candidates) {
|
| 59 |
+
const label = stackLabel(candidate.stack);
|
| 60 |
+
const startedAt = Date.now();
|
| 61 |
+
let pageUsable = true;
|
| 62 |
+
console.log(`Running local candidate: ${label}`);
|
| 63 |
+
try {
|
| 64 |
+
const before = await runPageTask(client, "window.browserSpeakBench.state()", {
|
| 65 |
+
label: `${label} preflight`,
|
| 66 |
+
});
|
| 67 |
+
await runPageTask(client, `window.browserSpeakBench.loadStack(${JSON.stringify(candidate.stack)})`, {
|
| 68 |
+
label: `${label} model load`,
|
| 69 |
+
timeoutMs: loadTimeoutMs,
|
| 70 |
+
});
|
| 71 |
+
for (const task of localTasks) {
|
| 72 |
+
await runTask(client, task, label);
|
| 73 |
+
}
|
| 74 |
+
const snapshot = await runPageTask(client, "window.browserSpeakBench.state()", {
|
| 75 |
+
label: `${label} state`,
|
| 76 |
+
});
|
| 77 |
+
const addedCount = Math.max(0, snapshot.results.length - before.results.length);
|
| 78 |
+
const rows = snapshot.results.slice(0, addedCount);
|
| 79 |
+
const errors = rows.filter((row) => row.error);
|
| 80 |
+
candidate.status = errors.length === 0 ? "complete" : "partial";
|
| 81 |
+
candidate.durationMs = Date.now() - startedAt;
|
| 82 |
+
candidate.rowCount = rows.length;
|
| 83 |
+
candidate.errorCount = errors.length;
|
| 84 |
+
candidate.metrics = metricsForRows(rows);
|
| 85 |
+
candidate.summary = snapshot.summary.current ?? snapshot.summary.all;
|
| 86 |
+
if (errors.length > 0) candidate.errors = errors.map((row) => `${row.kind}: ${row.error}`);
|
| 87 |
+
} catch (error) {
|
| 88 |
+
candidate.status = "failed";
|
| 89 |
+
candidate.durationMs = Date.now() - startedAt;
|
| 90 |
+
candidate.error = error.message;
|
| 91 |
+
pageUsable = !isPageUnresponsiveError(error);
|
| 92 |
+
console.error(`${label} failed: ${error.message}`);
|
| 93 |
+
} finally {
|
| 94 |
+
if (pageUsable) {
|
| 95 |
+
await runPageTask(client, "window.browserSpeakBench.stop()", {
|
| 96 |
+
label: `${label} stop`,
|
| 97 |
+
timeoutMs: 20000,
|
| 98 |
+
}).catch(() => {});
|
| 99 |
+
const state = await runPageTask(client, "window.browserSpeakBench.state()", {
|
| 100 |
+
label: `${label} loaded state`,
|
| 101 |
+
timeoutMs: 20000,
|
| 102 |
+
}).catch(() => null);
|
| 103 |
+
if (state?.modelsLoaded || state?.modelsLoading) {
|
| 104 |
+
await runPageTask(client, "window.browserSpeakBench.unload()", {
|
| 105 |
+
label: `${label} unload`,
|
| 106 |
+
timeoutMs: 60000,
|
| 107 |
+
}).catch((error) => {
|
| 108 |
+
candidate.unloadError = error.message;
|
| 109 |
+
});
|
| 110 |
+
}
|
| 111 |
+
} else {
|
| 112 |
+
candidate.unloadError = "Skipped page cleanup because the renderer was unresponsive.";
|
| 113 |
+
}
|
| 114 |
+
await writeJson(await exportPayload(client, candidates));
|
| 115 |
+
}
|
| 116 |
+
if (!pageUsable) break;
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
const payload = await exportPayload(client, candidates);
|
| 120 |
+
await writeJson(payload);
|
| 121 |
+
console.log(`Wrote local candidate benchmark JSON: ${resultPath}`);
|
| 122 |
+
summarizeCandidates(payload.candidates);
|
| 123 |
+
await client.closeBrowser();
|
| 124 |
+
} finally {
|
| 125 |
+
await stopBrowser(browser, reuseProfile ? null : profileDir);
|
| 126 |
+
}
|
| 127 |
+
}
|
| 128 |
+
|
| 129 |
+
async function runTask(client, task, label) {
|
| 130 |
+
const normalized = task.toLowerCase();
|
| 131 |
+
const loopbackOptions = {
|
| 132 |
+
timeoutMs: 180000,
|
| 133 |
+
...(process.env.BROWSER_SPEAK_LOOPBACK_TEXT ? { text: process.env.BROWSER_SPEAK_LOOPBACK_TEXT } : {}),
|
| 134 |
+
...(Number.isFinite(Number(process.env.BROWSER_SPEAK_LOOPBACK_SPEED))
|
| 135 |
+
? { speed: Number(process.env.BROWSER_SPEAK_LOOPBACK_SPEED) }
|
| 136 |
+
: {}),
|
| 137 |
+
};
|
| 138 |
+
const expression = {
|
| 139 |
+
tts: "window.browserSpeakBench.runTts({ timeoutMs: 90000 })",
|
| 140 |
+
identity: "window.browserSpeakBench.runIdentity({ timeoutMs: 180000 })",
|
| 141 |
+
chat: "window.browserSpeakBench.runChat({ timeoutMs: 180000 })",
|
| 142 |
+
loopback: `window.browserSpeakBench.runLoopback(${JSON.stringify(loopbackOptions)})`,
|
| 143 |
+
barge: "window.browserSpeakBench.runBargeIn({ timeoutMs: 90000 })",
|
| 144 |
+
"barge-in": "window.browserSpeakBench.runBargeIn({ timeoutMs: 90000 })",
|
| 145 |
+
suite: "window.browserSpeakBench.runSuite()",
|
| 146 |
+
}[normalized];
|
| 147 |
+
if (!expression) throw new Error(`Unknown local benchmark task: ${task}`);
|
| 148 |
+
await runPageTask(client, expression, {
|
| 149 |
+
label: `${label} ${normalized}`,
|
| 150 |
+
timeoutMs: normalized === "suite" ? Math.max(taskTimeoutMs, 420000) : taskTimeoutMs,
|
| 151 |
+
});
|
| 152 |
+
}
|
| 153 |
+
|
| 154 |
+
async function exportPayload(client, candidates) {
|
| 155 |
+
const exportResult = await runPageTask(client, "window.browserSpeakBench.exportResults()", {
|
| 156 |
+
label: "export results",
|
| 157 |
+
timeoutMs: 30000,
|
| 158 |
+
}).catch(() => ({ summary: null, results: [] }));
|
| 159 |
+
return {
|
| 160 |
+
generatedAt: new Date().toISOString(),
|
| 161 |
+
url,
|
| 162 |
+
config: benchmarkConfig(),
|
| 163 |
+
candidates,
|
| 164 |
+
summary: exportResult.summary,
|
| 165 |
+
results: exportResult.results,
|
| 166 |
+
};
|
| 167 |
+
}
|
| 168 |
+
|
| 169 |
+
function benchmarkConfig() {
|
| 170 |
+
return {
|
| 171 |
+
tasks: localTasks,
|
| 172 |
+
stackCount: stacks.length,
|
| 173 |
+
loadTimeoutMs,
|
| 174 |
+
taskTimeoutMs,
|
| 175 |
+
protocolTimeoutMs,
|
| 176 |
+
pollTimeoutMs,
|
| 177 |
+
pageUnresponsiveTimeoutMs,
|
| 178 |
+
ttsWarmup,
|
| 179 |
+
reuseProfile,
|
| 180 |
+
headless,
|
| 181 |
+
chrome,
|
| 182 |
+
profileDir,
|
| 183 |
+
extraChromeArgs: parseChromeArgs(),
|
| 184 |
+
};
|
| 185 |
+
}
|
| 186 |
+
|
| 187 |
+
function isPageUnresponsiveError(error) {
|
| 188 |
+
return /page stayed unresponsive|Runtime\.evaluate timed out/i.test(error?.message ?? String(error));
|
| 189 |
+
}
|
| 190 |
+
|
| 191 |
+
async function writeJson(payload) {
|
| 192 |
+
await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`);
|
| 193 |
+
}
|
| 194 |
+
|
| 195 |
+
function buildStacks() {
|
| 196 |
+
if (process.env.BROWSER_SPEAK_LOCAL_STACKS) {
|
| 197 |
+
const parsed = JSON.parse(process.env.BROWSER_SPEAK_LOCAL_STACKS);
|
| 198 |
+
if (!Array.isArray(parsed)) throw new Error("BROWSER_SPEAK_LOCAL_STACKS must be a JSON array.");
|
| 199 |
+
return parsed.map((stack, index) => normalizeLocalStack(stack, index));
|
| 200 |
+
}
|
| 201 |
+
const llms = parseList(process.env.BROWSER_SPEAK_LOCAL_LLMS ?? "HuggingFaceTB/SmolLM2-135M-Instruct");
|
| 202 |
+
const asrs = parseList(
|
| 203 |
+
process.env.BROWSER_SPEAK_LOCAL_ASRS ??
|
| 204 |
+
"onnx-community/moonshine-base-ONNX,onnx-community/moonshine-tiny-ONNX,onnx-community/whisper-tiny.en",
|
| 205 |
+
);
|
| 206 |
+
const voices = parseList(process.env.BROWSER_SPEAK_LOCAL_VOICES ?? "F2");
|
| 207 |
+
const steps = parseList(process.env.BROWSER_SPEAK_LOCAL_TTS_STEPS ?? "2").map((value) => Number(value));
|
| 208 |
+
const vadSilenceValues = parseList(process.env.BROWSER_SPEAK_LOCAL_VAD_SILENCE_MS ?? "480").map((value) =>
|
| 209 |
+
Number(value),
|
| 210 |
+
);
|
| 211 |
+
const partialAsr = process.env.BROWSER_SPEAK_PARTIAL_ASR !== "false";
|
| 212 |
+
const stacks = [];
|
| 213 |
+
for (const llm of llms) {
|
| 214 |
+
for (const asr of asrs) {
|
| 215 |
+
for (const voice of voices) {
|
| 216 |
+
for (const ttsSteps of steps) {
|
| 217 |
+
for (const vadSilenceMs of vadSilenceValues) {
|
| 218 |
+
stacks.push({
|
| 219 |
+
device: "wasm",
|
| 220 |
+
llm,
|
| 221 |
+
asr,
|
| 222 |
+
voice,
|
| 223 |
+
ttsSteps,
|
| 224 |
+
vadSilenceMs,
|
| 225 |
+
partialAsr,
|
| 226 |
+
ttsWarmup,
|
| 227 |
+
});
|
| 228 |
+
}
|
| 229 |
+
}
|
| 230 |
+
}
|
| 231 |
+
}
|
| 232 |
+
}
|
| 233 |
+
return stacks;
|
| 234 |
+
}
|
| 235 |
+
|
| 236 |
+
function normalizeLocalStack(stack, index) {
|
| 237 |
+
return {
|
| 238 |
+
device: "wasm",
|
| 239 |
+
name: stack.name ?? `stack-${index + 1}`,
|
| 240 |
+
llm: stack.llm ?? parseList(process.env.BROWSER_SPEAK_LOCAL_LLMS ?? "HuggingFaceTB/SmolLM2-135M-Instruct")[0],
|
| 241 |
+
asr: stack.asr ?? parseList(process.env.BROWSER_SPEAK_LOCAL_ASRS ?? "onnx-community/moonshine-base-ONNX")[0],
|
| 242 |
+
voice: stack.voice ?? parseList(process.env.BROWSER_SPEAK_LOCAL_VOICES ?? "F2")[0],
|
| 243 |
+
ttsSteps: Number(stack.ttsSteps ?? parseList(process.env.BROWSER_SPEAK_LOCAL_TTS_STEPS ?? "2")[0]),
|
| 244 |
+
vadSilenceMs: Number(stack.vadSilenceMs ?? parseList(process.env.BROWSER_SPEAK_LOCAL_VAD_SILENCE_MS ?? "480")[0]),
|
| 245 |
+
partialAsr: stack.partialAsr ?? (process.env.BROWSER_SPEAK_PARTIAL_ASR !== "false"),
|
| 246 |
+
ttsWarmup: stack.ttsWarmup ?? ttsWarmup,
|
| 247 |
+
...(stack.ttsChunking ? { ttsChunking: stack.ttsChunking } : {}),
|
| 248 |
+
};
|
| 249 |
+
}
|
| 250 |
+
|
| 251 |
+
function maxStacks() {
|
| 252 |
+
if (!process.env.BROWSER_SPEAK_LOCAL_MAX_STACKS) return Number.POSITIVE_INFINITY;
|
| 253 |
+
const value = Number(process.env.BROWSER_SPEAK_LOCAL_MAX_STACKS);
|
| 254 |
+
return Number.isFinite(value) ? Math.max(0, value) : Number.POSITIVE_INFINITY;
|
| 255 |
+
}
|
| 256 |
+
|
| 257 |
+
async function ensureServer() {
|
| 258 |
+
const response = await fetch(url).catch((error) => {
|
| 259 |
+
throw new Error(`Could not reach ${url}: ${error.message}`);
|
| 260 |
+
});
|
| 261 |
+
if (!response.ok) throw new Error(`${url} returned HTTP ${response.status}`);
|
| 262 |
+
}
|
| 263 |
+
|
| 264 |
+
async function waitForBenchApi(client) {
|
| 265 |
+
const deadline = Date.now() + 15000;
|
| 266 |
+
while (Date.now() < deadline) {
|
| 267 |
+
try {
|
| 268 |
+
if (await client.evaluate("Boolean(window.browserSpeakBench)")) return;
|
| 269 |
+
} catch {
|
| 270 |
+
// The target may still be navigating and can destroy the execution context.
|
| 271 |
+
}
|
| 272 |
+
await sleep(100);
|
| 273 |
+
}
|
| 274 |
+
throw new Error("window.browserSpeakBench was not installed.");
|
| 275 |
+
}
|
| 276 |
+
|
| 277 |
+
async function runPageTask(client, expression, { label = "page task", timeoutMs = 30000 } = {}) {
|
| 278 |
+
const taskId = `task_${Date.now()}_${Math.random().toString(16).slice(2)}`;
|
| 279 |
+
await client.evaluate(`(() => {
|
| 280 |
+
const taskId = ${JSON.stringify(taskId)};
|
| 281 |
+
window.__browserSpeakHarnessTasks ||= {};
|
| 282 |
+
window.__browserSpeakHarnessTasks[taskId] = { done: false, label: ${JSON.stringify(label)} };
|
| 283 |
+
Promise.resolve(${expression})
|
| 284 |
+
.then((value) => {
|
| 285 |
+
window.__browserSpeakHarnessTasks[taskId] = { done: true, value };
|
| 286 |
+
})
|
| 287 |
+
.catch((error) => {
|
| 288 |
+
window.__browserSpeakHarnessTasks[taskId] = {
|
| 289 |
+
done: true,
|
| 290 |
+
error: error?.stack || error?.message || String(error),
|
| 291 |
+
};
|
| 292 |
+
});
|
| 293 |
+
return true;
|
| 294 |
+
})()`);
|
| 295 |
+
const deadline = Date.now() + timeoutMs;
|
| 296 |
+
let lastEvents = [];
|
| 297 |
+
let lastPollError = "";
|
| 298 |
+
let pollErrorSince = 0;
|
| 299 |
+
while (Date.now() < deadline) {
|
| 300 |
+
let task = null;
|
| 301 |
+
try {
|
| 302 |
+
task = await client.evaluate(
|
| 303 |
+
`window.__browserSpeakHarnessTasks?.[${JSON.stringify(taskId)}] ?? null`,
|
| 304 |
+
pollTimeoutMs,
|
| 305 |
+
);
|
| 306 |
+
lastPollError = "";
|
| 307 |
+
pollErrorSince = 0;
|
| 308 |
+
} catch (error) {
|
| 309 |
+
const message = error.message ?? String(error);
|
| 310 |
+
pollErrorSince ||= Date.now();
|
| 311 |
+
if (message !== lastPollError) {
|
| 312 |
+
console.log(`${label}: waiting for page response (${message})`);
|
| 313 |
+
lastPollError = message;
|
| 314 |
+
}
|
| 315 |
+
if (Date.now() - pollErrorSince > pageUnresponsiveTimeoutMs) {
|
| 316 |
+
throw new Error(
|
| 317 |
+
`${label} page stayed unresponsive for ${(pageUnresponsiveTimeoutMs / 1000).toFixed(0)} seconds; last poll error: ${message}`,
|
| 318 |
+
);
|
| 319 |
+
}
|
| 320 |
+
await sleep(500);
|
| 321 |
+
continue;
|
| 322 |
+
}
|
| 323 |
+
if (task?.done) {
|
| 324 |
+
await client
|
| 325 |
+
.evaluate(`delete window.__browserSpeakHarnessTasks?.[${JSON.stringify(taskId)}]`, pollTimeoutMs)
|
| 326 |
+
.catch(() => {});
|
| 327 |
+
if (task.error) throw new Error(`${label} failed: ${task.error}`);
|
| 328 |
+
return task.value;
|
| 329 |
+
}
|
| 330 |
+
const snapshot = await client
|
| 331 |
+
.evaluate(`(() => {
|
| 332 |
+
const state = window.browserSpeakBench?.state?.();
|
| 333 |
+
return state ? {
|
| 334 |
+
modelsLoaded: state.modelsLoaded,
|
| 335 |
+
modelsLoading: state.modelsLoading,
|
| 336 |
+
activeBenchmark: state.activeBenchmark?.kind ?? null,
|
| 337 |
+
suiteRunning: state.suiteRunning,
|
| 338 |
+
events: state.events?.slice(0, 3) ?? [],
|
| 339 |
+
} : null;
|
| 340 |
+
})()`, pollTimeoutMs)
|
| 341 |
+
.catch(() => null);
|
| 342 |
+
const events = snapshot?.events ?? [];
|
| 343 |
+
if (events.join("\\n") !== lastEvents.join("\\n")) {
|
| 344 |
+
lastEvents = events;
|
| 345 |
+
if (events[0]) console.log(`${label}: ${events[0]}`);
|
| 346 |
+
}
|
| 347 |
+
await sleep(500);
|
| 348 |
+
}
|
| 349 |
+
throw new Error(`${label} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`);
|
| 350 |
+
}
|
| 351 |
+
|
| 352 |
+
function launchBrowser(port, profileDir) {
|
| 353 |
+
const child = spawn(
|
| 354 |
+
chrome,
|
| 355 |
+
[
|
| 356 |
+
...(headless ? ["--headless=new"] : []),
|
| 357 |
+
"--no-sandbox",
|
| 358 |
+
"--disable-gpu",
|
| 359 |
+
"--disable-dev-shm-usage",
|
| 360 |
+
"--disable-background-networking",
|
| 361 |
+
"--disable-extensions",
|
| 362 |
+
"--no-default-browser-check",
|
| 363 |
+
"--no-first-run",
|
| 364 |
+
"--autoplay-policy=no-user-gesture-required",
|
| 365 |
+
`--remote-debugging-port=${port}`,
|
| 366 |
+
`--user-data-dir=${profileDir}`,
|
| 367 |
+
...parseChromeArgs(),
|
| 368 |
+
],
|
| 369 |
+
{ stdio: ["ignore", "pipe", "pipe"] },
|
| 370 |
+
);
|
| 371 |
+
child.browserLog = "";
|
| 372 |
+
const appendLog = (chunk) => {
|
| 373 |
+
child.browserLog = `${child.browserLog}${chunk}`;
|
| 374 |
+
if (child.browserLog.length > 8000) child.browserLog = child.browserLog.slice(-8000);
|
| 375 |
+
};
|
| 376 |
+
child.stdout.on("data", appendLog);
|
| 377 |
+
child.stderr.on("data", appendLog);
|
| 378 |
+
return child;
|
| 379 |
+
}
|
| 380 |
+
|
| 381 |
+
async function stopBrowser(child, profileDir) {
|
| 382 |
+
if (child.exitCode == null) child.kill("SIGTERM");
|
| 383 |
+
await new Promise((resolve) => {
|
| 384 |
+
child.once("exit", resolve);
|
| 385 |
+
setTimeout(resolve, 3000);
|
| 386 |
+
});
|
| 387 |
+
if (child.exitCode == null) child.kill("SIGKILL");
|
| 388 |
+
if (!profileDir) return;
|
| 389 |
+
for (let attempt = 0; attempt < 5; attempt += 1) {
|
| 390 |
+
try {
|
| 391 |
+
await rm(profileDir, { recursive: true, force: true });
|
| 392 |
+
return;
|
| 393 |
+
} catch (error) {
|
| 394 |
+
if (attempt === 4) {
|
| 395 |
+
console.warn(`Could not remove ${profileDir}: ${error.message}`);
|
| 396 |
+
return;
|
| 397 |
+
}
|
| 398 |
+
await sleep(500);
|
| 399 |
+
}
|
| 400 |
+
}
|
| 401 |
+
}
|
| 402 |
+
|
| 403 |
+
async function connectToPage(port, child) {
|
| 404 |
+
const deadline = Date.now() + 60000;
|
| 405 |
+
let lastError = null;
|
| 406 |
+
while (Date.now() < deadline) {
|
| 407 |
+
if (child.exitCode != null) {
|
| 408 |
+
throw new Error(`Chrome exited before DevTools became available.\n${child.browserLog}`);
|
| 409 |
+
}
|
| 410 |
+
try {
|
| 411 |
+
const version = await fetch(`http://127.0.0.1:${port}/json/version`).then((response) => response.json());
|
| 412 |
+
if (version.webSocketDebuggerUrl) {
|
| 413 |
+
const page = await createPageTarget(port);
|
| 414 |
+
return new CdpClient(page.webSocketDebuggerUrl);
|
| 415 |
+
}
|
| 416 |
+
} catch (error) {
|
| 417 |
+
lastError = error;
|
| 418 |
+
}
|
| 419 |
+
await sleep(250);
|
| 420 |
+
}
|
| 421 |
+
throw new Error(
|
| 422 |
+
`Could not connect to Chrome DevTools on port ${port}: ${lastError?.message ?? "unknown error"}\n${child.browserLog}`,
|
| 423 |
+
);
|
| 424 |
+
}
|
| 425 |
+
|
| 426 |
+
async function createPageTarget(port) {
|
| 427 |
+
for (const method of ["PUT", "GET"]) {
|
| 428 |
+
const response = await fetch(`http://127.0.0.1:${port}/json/new?${encodeURIComponent(url)}`, {
|
| 429 |
+
method,
|
| 430 |
+
}).catch(() => null);
|
| 431 |
+
if (response?.ok) {
|
| 432 |
+
const target = await response.json();
|
| 433 |
+
if (target.webSocketDebuggerUrl) return target;
|
| 434 |
+
}
|
| 435 |
+
}
|
| 436 |
+
const targets = await fetch(`http://127.0.0.1:${port}/json`).then((response) => response.json());
|
| 437 |
+
const page = targets.find((target) => target.type === "page" && target.url === url);
|
| 438 |
+
if (page?.webSocketDebuggerUrl) return page;
|
| 439 |
+
throw new Error("Could not create or find a page target.");
|
| 440 |
+
}
|
| 441 |
+
|
| 442 |
+
class CdpClient {
|
| 443 |
+
constructor(webSocketUrl) {
|
| 444 |
+
this.nextId = 1;
|
| 445 |
+
this.pending = new Map();
|
| 446 |
+
this.socket = new WebSocket(webSocketUrl);
|
| 447 |
+
this.opened = new Promise((resolve, reject) => {
|
| 448 |
+
this.socket.onopen = resolve;
|
| 449 |
+
this.socket.onerror = reject;
|
| 450 |
+
this.socket.onmessage = (event) => this.onMessage(event);
|
| 451 |
+
});
|
| 452 |
+
}
|
| 453 |
+
|
| 454 |
+
onMessage(event) {
|
| 455 |
+
const message = JSON.parse(event.data);
|
| 456 |
+
if (!message.id || !this.pending.has(message.id)) return;
|
| 457 |
+
const { resolve: onResolve, reject } = this.pending.get(message.id);
|
| 458 |
+
this.pending.delete(message.id);
|
| 459 |
+
if (message.error) reject(new Error(message.error.message));
|
| 460 |
+
else onResolve(message.result);
|
| 461 |
+
}
|
| 462 |
+
|
| 463 |
+
async call(method, params = {}, timeoutMs = protocolTimeoutMs) {
|
| 464 |
+
await this.opened;
|
| 465 |
+
const id = this.nextId++;
|
| 466 |
+
this.socket.send(JSON.stringify({ id, method, params }));
|
| 467 |
+
return new Promise((resolvePromise, reject) => {
|
| 468 |
+
const timer = setTimeout(() => {
|
| 469 |
+
this.pending.delete(id);
|
| 470 |
+
reject(new Error(`${method} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`));
|
| 471 |
+
}, timeoutMs);
|
| 472 |
+
this.pending.set(id, {
|
| 473 |
+
resolve: (value) => {
|
| 474 |
+
clearTimeout(timer);
|
| 475 |
+
resolvePromise(value);
|
| 476 |
+
},
|
| 477 |
+
reject: (error) => {
|
| 478 |
+
clearTimeout(timer);
|
| 479 |
+
reject(error);
|
| 480 |
+
},
|
| 481 |
+
});
|
| 482 |
+
});
|
| 483 |
+
}
|
| 484 |
+
|
| 485 |
+
async evaluate(expression, timeoutMs = protocolTimeoutMs) {
|
| 486 |
+
const result = await this.call("Runtime.evaluate", {
|
| 487 |
+
expression,
|
| 488 |
+
returnByValue: true,
|
| 489 |
+
}, timeoutMs);
|
| 490 |
+
if (result.exceptionDetails) throw new Error(formatException(result.exceptionDetails));
|
| 491 |
+
return result.result.value;
|
| 492 |
+
}
|
| 493 |
+
|
| 494 |
+
closeBrowser() {
|
| 495 |
+
try {
|
| 496 |
+
this.socket.send(JSON.stringify({ id: this.nextId++, method: "Browser.close", params: {} }));
|
| 497 |
+
} catch {
|
| 498 |
+
// The surrounding process cleanup handles already-closed targets.
|
| 499 |
+
}
|
| 500 |
+
try {
|
| 501 |
+
this.socket.close();
|
| 502 |
+
} catch {
|
| 503 |
+
// Ignore close races.
|
| 504 |
+
}
|
| 505 |
+
}
|
| 506 |
+
}
|
| 507 |
+
|
| 508 |
+
function formatException(exceptionDetails) {
|
| 509 |
+
const exception = exceptionDetails.exception;
|
| 510 |
+
return exception?.description ?? exception?.value ?? exceptionDetails.text ?? "Evaluation failed.";
|
| 511 |
+
}
|
| 512 |
+
|
| 513 |
+
function parseList(value) {
|
| 514 |
+
return String(value ?? "")
|
| 515 |
+
.split(",")
|
| 516 |
+
.map((item) => item.trim())
|
| 517 |
+
.filter(Boolean);
|
| 518 |
+
}
|
| 519 |
+
|
| 520 |
+
function parseChromeArgs() {
|
| 521 |
+
const raw = process.env.BROWSER_SPEAK_CHROME_ARGS ?? "";
|
| 522 |
+
return (
|
| 523 |
+
raw
|
| 524 |
+
.match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g)
|
| 525 |
+
?.map((arg) => arg.replace(/^["']|["']$/g, "")) ?? []
|
| 526 |
+
);
|
| 527 |
+
}
|
| 528 |
+
|
| 529 |
+
function stackLabel(stack) {
|
| 530 |
+
return [
|
| 531 |
+
shortModel(stack.llm),
|
| 532 |
+
shortModel(stack.asr),
|
| 533 |
+
stack.voice,
|
| 534 |
+
`${stack.ttsSteps} steps`,
|
| 535 |
+
`${stack.vadSilenceMs}ms`,
|
| 536 |
+
stack.ttsChunking ? `chunk ${stack.ttsChunking.firstTargetChars ?? "?"}/${stack.ttsChunking.targetChars ?? "?"}` : "",
|
| 537 |
+
]
|
| 538 |
+
.filter(Boolean)
|
| 539 |
+
.join(" / ");
|
| 540 |
+
}
|
| 541 |
+
|
| 542 |
+
function shortModel(modelId = "") {
|
| 543 |
+
return modelId.split("/").at(-1)?.replace(/-ONNX$/, "") ?? modelId;
|
| 544 |
+
}
|
| 545 |
+
|
| 546 |
+
function summarizeCandidates(candidates) {
|
| 547 |
+
for (const candidate of candidates) {
|
| 548 |
+
const metrics = candidate.metrics ?? {};
|
| 549 |
+
const parts = [];
|
| 550 |
+
if (Number.isFinite(metrics.ttsMedianFirstAudioMs)) {
|
| 551 |
+
parts.push(`tts first audio ${formatMs(metrics.ttsMedianFirstAudioMs)}`);
|
| 552 |
+
}
|
| 553 |
+
if (Number.isFinite(metrics.modelLoadMs)) {
|
| 554 |
+
parts.push(`load ${formatMs(metrics.modelLoadMs)}`);
|
| 555 |
+
}
|
| 556 |
+
if (Number.isFinite(metrics.loopbackMedianSpeechEndToFirstAudioMs)) {
|
| 557 |
+
parts.push(`loopback end-to-audio ${formatMs(metrics.loopbackMedianSpeechEndToFirstAudioMs)}`);
|
| 558 |
+
}
|
| 559 |
+
if (Number.isFinite(metrics.identityMedianFirstAudioMs)) {
|
| 560 |
+
parts.push(`identity first audio ${formatMs(metrics.identityMedianFirstAudioMs)}`);
|
| 561 |
+
}
|
| 562 |
+
const suffix = parts.length > 0 ? `, ${parts.join(", ")}` : "";
|
| 563 |
+
const error = candidate.error ? ` (${candidate.error})` : "";
|
| 564 |
+
console.log(`${stackLabel(candidate.stack)}: ${candidate.status}, ${candidate.rowCount ?? 0} rows${suffix}${error}`);
|
| 565 |
+
}
|
| 566 |
+
}
|
| 567 |
+
|
| 568 |
+
function metricsForRows(rows) {
|
| 569 |
+
const completed = rows.filter((row) => !row.error);
|
| 570 |
+
const byKind = (kind) => completed.filter((row) => row.kind === kind);
|
| 571 |
+
const ttsRows = byKind("tts");
|
| 572 |
+
const identityRows = byKind("identity");
|
| 573 |
+
const chatRows = byKind("chat");
|
| 574 |
+
const loopbackRows = byKind("loopback");
|
| 575 |
+
return {
|
| 576 |
+
modelLoadMs: rows.find((row) => Number.isFinite(row.stack?.modelLoadMs))?.stack.modelLoadMs ?? null,
|
| 577 |
+
ttsRuns: ttsRows.length,
|
| 578 |
+
ttsMedianFirstAudioMs: median(ttsRows.map((row) => row.firstAudioMs)),
|
| 579 |
+
ttsMedianSynthesisMs: median(ttsRows.map((row) => row.firstTtsSynthesisMs)),
|
| 580 |
+
ttsMedianAudioEndMs: median(ttsRows.map((row) => row.audioEndMs)),
|
| 581 |
+
identityRuns: identityRows.length,
|
| 582 |
+
identityMedianFirstTokenMs: median(identityRows.map((row) => row.firstTokenMs)),
|
| 583 |
+
identityMedianFirstAudioMs: median(identityRows.map((row) => row.firstAudioMs)),
|
| 584 |
+
identityPasses: identityRows.filter((row) => row.llmQualityPass).length,
|
| 585 |
+
chatRuns: chatRows.length,
|
| 586 |
+
chatMedianFirstTokenMs: median(chatRows.map((row) => row.firstTokenMs)),
|
| 587 |
+
chatMedianFirstAudioMs: median(chatRows.map((row) => row.firstAudioMs)),
|
| 588 |
+
loopbackRuns: loopbackRows.length,
|
| 589 |
+
loopbackMedianWer: median(loopbackRows.map((row) => row.sttWer)),
|
| 590 |
+
loopbackMedianSpeechEndToFirstAudioMs: median(
|
| 591 |
+
loopbackRows.map((row) => row.speechEndToFirstAudioMs),
|
| 592 |
+
),
|
| 593 |
+
loopbackMedianSpeechEndToAudioEndMs: median(
|
| 594 |
+
loopbackRows.map((row) => row.speechEndToAudioEndMs),
|
| 595 |
+
),
|
| 596 |
+
};
|
| 597 |
+
}
|
| 598 |
+
|
| 599 |
+
function median(values) {
|
| 600 |
+
const finite = values.filter(Number.isFinite).sort((a, b) => a - b);
|
| 601 |
+
if (finite.length === 0) return null;
|
| 602 |
+
const middle = Math.floor(finite.length / 2);
|
| 603 |
+
if (finite.length % 2 === 1) return finite[middle];
|
| 604 |
+
return (finite[middle - 1] + finite[middle]) / 2;
|
| 605 |
+
}
|
| 606 |
+
|
| 607 |
+
function formatMs(value) {
|
| 608 |
+
if (!Number.isFinite(value)) return "-";
|
| 609 |
+
if (value < 1000) return `${Math.round(value)} ms`;
|
| 610 |
+
return `${(value / 1000).toFixed(2)} s`;
|
| 611 |
+
}
|
| 612 |
+
|
| 613 |
+
function sleep(ms) {
|
| 614 |
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
| 615 |
+
}
|
| 616 |
+
|
| 617 |
+
await main();
|
tools/run-loopback-series.mjs
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env node
|
| 2 |
+
import { spawn } from "node:child_process";
|
| 3 |
+
import { readFile, writeFile, mkdir } from "node:fs/promises";
|
| 4 |
+
import { tmpdir } from "node:os";
|
| 5 |
+
import { dirname, resolve } from "node:path";
|
| 6 |
+
import { fileURLToPath } from "node:url";
|
| 7 |
+
import { sourceFingerprint } from "./source-fingerprint.mjs";
|
| 8 |
+
|
| 9 |
+
const scriptDir = dirname(fileURLToPath(import.meta.url));
|
| 10 |
+
const resultPath = resolve(process.env.BROWSER_SPEAK_LOOPBACK_JSON ?? `${tmpdir()}/browser-speak-loopback-series.json`);
|
| 11 |
+
const rawPath = resolve(
|
| 12 |
+
process.env.BROWSER_SPEAK_LOOPBACK_RAW_JSON ?? `${tmpdir()}/browser-speak-loopback-series-raw.json`,
|
| 13 |
+
);
|
| 14 |
+
const count = Math.max(1, Number(process.env.BROWSER_SPEAK_LOOPBACK_COUNT ?? 3));
|
| 15 |
+
const speed = Number(process.env.BROWSER_SPEAK_LOOPBACK_SPEED ?? 1.0);
|
| 16 |
+
const text = process.env.BROWSER_SPEAK_LOOPBACK_TEXT ?? "Identify this browser demo.";
|
| 17 |
+
const stack = {
|
| 18 |
+
device: process.env.BROWSER_SPEAK_DEVICE ?? "wasm",
|
| 19 |
+
llm: process.env.BROWSER_SPEAK_LLM ?? "HuggingFaceTB/SmolLM2-135M-Instruct",
|
| 20 |
+
asr: process.env.BROWSER_SPEAK_ASR ?? "onnx-community/moonshine-base-ONNX",
|
| 21 |
+
voice: process.env.BROWSER_SPEAK_VOICE ?? "F2",
|
| 22 |
+
ttsSteps: Number(process.env.BROWSER_SPEAK_TTS_STEPS ?? 2),
|
| 23 |
+
vadSilenceMs: Number(process.env.BROWSER_SPEAK_VAD_SILENCE_MS ?? 480),
|
| 24 |
+
partialAsr: process.env.BROWSER_SPEAK_PARTIAL_ASR !== "false",
|
| 25 |
+
};
|
| 26 |
+
|
| 27 |
+
async function main() {
|
| 28 |
+
await mkdir(dirname(resultPath), { recursive: true });
|
| 29 |
+
await mkdir(dirname(rawPath), { recursive: true });
|
| 30 |
+
const env = {
|
| 31 |
+
...process.env,
|
| 32 |
+
BROWSER_SPEAK_LOCAL_JSON: rawPath,
|
| 33 |
+
BROWSER_SPEAK_LOCAL_STACKS: JSON.stringify([stack]),
|
| 34 |
+
BROWSER_SPEAK_LOCAL_MAX_STACKS: "1",
|
| 35 |
+
BROWSER_SPEAK_LOCAL_PROFILE_DIR:
|
| 36 |
+
process.env.BROWSER_SPEAK_LOCAL_PROFILE_DIR ?? `${tmpdir()}/browser-speak-loopback-series-profile`,
|
| 37 |
+
BROWSER_SPEAK_LOCAL_TASKS: Array.from({ length: count }, () => "loopback").join(","),
|
| 38 |
+
BROWSER_SPEAK_LOAD_TIMEOUT_MS: process.env.BROWSER_SPEAK_LOAD_TIMEOUT_MS ?? "900000",
|
| 39 |
+
BROWSER_SPEAK_TASK_TIMEOUT_MS: process.env.BROWSER_SPEAK_TASK_TIMEOUT_MS ?? "240000",
|
| 40 |
+
BROWSER_SPEAK_LOOPBACK_SPEED: String(speed),
|
| 41 |
+
BROWSER_SPEAK_LOOPBACK_TEXT: text,
|
| 42 |
+
};
|
| 43 |
+
const exitCode = await runChild(process.execPath, [resolve(scriptDir, "run-local-candidate-benchmark.mjs")], env);
|
| 44 |
+
const raw = await readJson(rawPath).catch(() => null);
|
| 45 |
+
const payload = await summarize(raw, exitCode);
|
| 46 |
+
await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`);
|
| 47 |
+
console.log(`Wrote loopback series JSON: ${resultPath}`);
|
| 48 |
+
printSummary(payload);
|
| 49 |
+
if (!payload.passed) process.exitCode = exitCode || 1;
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
function runChild(command, args, env) {
|
| 53 |
+
return new Promise((resolvePromise) => {
|
| 54 |
+
const child = spawn(command, args, { env, stdio: "inherit" });
|
| 55 |
+
child.on("exit", (code) => resolvePromise(code ?? 1));
|
| 56 |
+
});
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
async function readJson(path) {
|
| 60 |
+
return JSON.parse(await readFile(path, "utf8"));
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
async function summarize(raw, exitCode) {
|
| 64 |
+
const rows = raw?.results?.filter((row) => row.kind === "loopback") ?? [];
|
| 65 |
+
const completed = rows.filter((row) => !row.error);
|
| 66 |
+
const errors = rows.filter((row) => row.error);
|
| 67 |
+
const transcripts = completed.map((row) => row.transcript ?? "");
|
| 68 |
+
return {
|
| 69 |
+
generatedAt: new Date().toISOString(),
|
| 70 |
+
sourceFingerprint: await sourceFingerprint(),
|
| 71 |
+
passed: exitCode === 0 && rows.length >= count && errors.length === 0,
|
| 72 |
+
rawPath,
|
| 73 |
+
config: {
|
| 74 |
+
count,
|
| 75 |
+
text,
|
| 76 |
+
speed,
|
| 77 |
+
stack,
|
| 78 |
+
taskTimeoutMs: Number(process.env.BROWSER_SPEAK_TASK_TIMEOUT_MS ?? 240000),
|
| 79 |
+
url: process.env.BROWSER_SPEAK_URL ?? "http://127.0.0.1:5174/",
|
| 80 |
+
},
|
| 81 |
+
summary: {
|
| 82 |
+
requestedRuns: count,
|
| 83 |
+
producedRuns: rows.length,
|
| 84 |
+
completedRuns: completed.length,
|
| 85 |
+
errorRuns: errors.length,
|
| 86 |
+
exactTranscriptRuns: completed.filter((row) => row.sttWer === 0).length,
|
| 87 |
+
medianWer: median(completed.map((row) => row.sttWer)),
|
| 88 |
+
medianCer: median(completed.map((row) => row.sttCer)),
|
| 89 |
+
medianAsrMs: median(completed.map((row) => row.asrMs)),
|
| 90 |
+
medianFirstTokenMs: median(completed.map((row) => row.firstTokenMs)),
|
| 91 |
+
medianFirstAudioMs: median(completed.map((row) => row.firstAudioMs)),
|
| 92 |
+
medianSpeechEndToFirstAudioMs: median(completed.map((row) => row.speechEndToFirstAudioMs)),
|
| 93 |
+
medianSpeechEndToAudioEndMs: median(completed.map((row) => row.speechEndToAudioEndMs)),
|
| 94 |
+
identityPasses: completed.filter((row) => row.llmQualityPass).length,
|
| 95 |
+
transcripts,
|
| 96 |
+
errors: errors.map((row) => `${row.id ?? "?"}: ${row.error}`),
|
| 97 |
+
},
|
| 98 |
+
rows,
|
| 99 |
+
rawSummary: raw?.summary ?? null,
|
| 100 |
+
rawCandidates: raw?.candidates ?? [],
|
| 101 |
+
};
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
function median(values) {
|
| 105 |
+
const finite = values.filter(Number.isFinite).sort((a, b) => a - b);
|
| 106 |
+
if (finite.length === 0) return null;
|
| 107 |
+
const middle = Math.floor(finite.length / 2);
|
| 108 |
+
if (finite.length % 2 === 1) return finite[middle];
|
| 109 |
+
return (finite[middle - 1] + finite[middle]) / 2;
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
function printSummary(payload) {
|
| 113 |
+
const summary = payload.summary;
|
| 114 |
+
console.log(
|
| 115 |
+
[
|
| 116 |
+
`${summary.completedRuns}/${summary.requestedRuns} completed`,
|
| 117 |
+
`${summary.exactTranscriptRuns}/${summary.completedRuns} exact transcripts`,
|
| 118 |
+
`median WER ${formatPercent(summary.medianWer)}`,
|
| 119 |
+
`median speech-end-to-audio ${formatMs(summary.medianSpeechEndToFirstAudioMs)}`,
|
| 120 |
+
].join(", "),
|
| 121 |
+
);
|
| 122 |
+
if (summary.transcripts.length > 0) {
|
| 123 |
+
console.log(`Transcripts: ${summary.transcripts.map((text) => JSON.stringify(text)).join(", ")}`);
|
| 124 |
+
}
|
| 125 |
+
for (const error of summary.errors) console.log(`Loopback error: ${error}`);
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
function formatMs(value) {
|
| 129 |
+
if (!Number.isFinite(value)) return "-";
|
| 130 |
+
if (value < 1000) return `${Math.round(value)} ms`;
|
| 131 |
+
return `${(value / 1000).toFixed(2)} s`;
|
| 132 |
+
}
|
| 133 |
+
|
| 134 |
+
function formatPercent(value) {
|
| 135 |
+
if (!Number.isFinite(value)) return "-";
|
| 136 |
+
return `${Math.round(value * 100)}%`;
|
| 137 |
+
}
|
| 138 |
+
|
| 139 |
+
main().catch((error) => {
|
| 140 |
+
console.error(error.stack ?? error.message);
|
| 141 |
+
process.exitCode = 1;
|
| 142 |
+
});
|
tools/run-real-mic-series.mjs
ADDED
|
@@ -0,0 +1,517 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env node
|
| 2 |
+
import { spawn } from "node:child_process";
|
| 3 |
+
import { existsSync } from "node:fs";
|
| 4 |
+
import { mkdir, rm, writeFile } from "node:fs/promises";
|
| 5 |
+
import { tmpdir } from "node:os";
|
| 6 |
+
import { dirname, resolve } from "node:path";
|
| 7 |
+
import { sourceFingerprint } from "./source-fingerprint.mjs";
|
| 8 |
+
|
| 9 |
+
const url = process.env.BROWSER_SPEAK_URL ?? "http://127.0.0.1:5174/";
|
| 10 |
+
const chrome =
|
| 11 |
+
process.env.CHROME_BIN ??
|
| 12 |
+
(existsSync("/opt/google/chrome/chrome")
|
| 13 |
+
? "/opt/google/chrome/chrome"
|
| 14 |
+
: existsSync("/usr/bin/google-chrome")
|
| 15 |
+
? "/usr/bin/google-chrome"
|
| 16 |
+
: "chromium");
|
| 17 |
+
const dryRun = process.env.BROWSER_SPEAK_REAL_MIC_DRY_RUN === "true";
|
| 18 |
+
const resultPath = resolve(
|
| 19 |
+
process.env.BROWSER_SPEAK_REAL_MIC_JSON ??
|
| 20 |
+
`${tmpdir()}/browser-speak-real-mic-series${dryRun ? "-dry-run" : ""}.json`,
|
| 21 |
+
);
|
| 22 |
+
const profileDir = resolve(process.env.BROWSER_SPEAK_REAL_MIC_PROFILE_DIR ?? `${tmpdir()}/browser-speak-real-mic-profile`);
|
| 23 |
+
const protocolTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_TIMEOUT_MS ?? 60000);
|
| 24 |
+
const pollTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_POLL_TIMEOUT_MS ?? 5000);
|
| 25 |
+
const loadTimeoutMs = Number(process.env.BROWSER_SPEAK_LOAD_TIMEOUT_MS ?? 900000);
|
| 26 |
+
const rowTimeoutMs = Number(process.env.BROWSER_SPEAK_REAL_MIC_ROW_TIMEOUT_MS ?? 150000);
|
| 27 |
+
const count = Math.max(1, Number(process.env.BROWSER_SPEAK_REAL_MIC_COUNT ?? 3));
|
| 28 |
+
const headless = process.env.BROWSER_SPEAK_HEADLESS === "true";
|
| 29 |
+
const reuseProfile = process.env.BROWSER_SPEAK_REAL_MIC_REUSE_PROFILE === "true";
|
| 30 |
+
const autoAcceptMic = process.env.BROWSER_SPEAK_REAL_MIC_AUTO_ACCEPT !== "false";
|
| 31 |
+
const allowFakeCapture = process.env.BROWSER_SPEAK_REAL_MIC_ALLOW_FAKE_CAPTURE === "true";
|
| 32 |
+
const keepBrowserOpen = process.env.BROWSER_SPEAK_KEEP_BROWSER_OPEN === "true";
|
| 33 |
+
const requireExactTranscript = process.env.BROWSER_SPEAK_REAL_MIC_REQUIRE_EXACT === "true";
|
| 34 |
+
const ttsWarmup = process.env.BROWSER_SPEAK_TTS_WARMUP !== "false";
|
| 35 |
+
const stack = {
|
| 36 |
+
device: process.env.BROWSER_SPEAK_DEVICE ?? "wasm",
|
| 37 |
+
llm: process.env.BROWSER_SPEAK_LLM ?? "HuggingFaceTB/SmolLM2-135M-Instruct",
|
| 38 |
+
asr: process.env.BROWSER_SPEAK_ASR ?? "onnx-community/moonshine-base-ONNX",
|
| 39 |
+
voice: process.env.BROWSER_SPEAK_VOICE ?? "F2",
|
| 40 |
+
ttsSteps: Number(process.env.BROWSER_SPEAK_TTS_STEPS ?? 2),
|
| 41 |
+
vadSilenceMs: Number(process.env.BROWSER_SPEAK_VAD_SILENCE_MS ?? 480),
|
| 42 |
+
partialAsr: process.env.BROWSER_SPEAK_PARTIAL_ASR !== "false",
|
| 43 |
+
ttsWarmup,
|
| 44 |
+
};
|
| 45 |
+
|
| 46 |
+
async function main() {
|
| 47 |
+
if (dryRun) {
|
| 48 |
+
await mkdir(dirname(resultPath), { recursive: true });
|
| 49 |
+
const payload = {
|
| 50 |
+
generatedAt: new Date().toISOString(),
|
| 51 |
+
sourceFingerprint: await sourceFingerprint(),
|
| 52 |
+
dryRun: true,
|
| 53 |
+
url,
|
| 54 |
+
config: runConfig(),
|
| 55 |
+
preflight: realMicPreflight(),
|
| 56 |
+
plannedRows: count,
|
| 57 |
+
prompt: "What app is this?",
|
| 58 |
+
};
|
| 59 |
+
await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`);
|
| 60 |
+
console.log(`Wrote real microphone dry-run JSON: ${resultPath}`);
|
| 61 |
+
console.log(`Planned ${count} real microphone row(s).`);
|
| 62 |
+
for (const warning of payload.preflight.warnings) console.log(`Preflight warning: ${warning}`);
|
| 63 |
+
return;
|
| 64 |
+
}
|
| 65 |
+
const preflight = realMicPreflight();
|
| 66 |
+
if (!preflight.ok) throw new Error(preflight.warnings.join(" "));
|
| 67 |
+
await ensureServer();
|
| 68 |
+
await mkdir(dirname(resultPath), { recursive: true });
|
| 69 |
+
if (!reuseProfile) await rm(profileDir, { recursive: true, force: true });
|
| 70 |
+
|
| 71 |
+
const browser = launchBrowser(9347, profileDir);
|
| 72 |
+
let client = null;
|
| 73 |
+
try {
|
| 74 |
+
client = await connectToPage(9347, browser);
|
| 75 |
+
await waitForBenchApi(client);
|
| 76 |
+
await runPageTask(client, `window.browserSpeakBench.loadStack(${JSON.stringify(stack)})`, {
|
| 77 |
+
label: "real-mic model load",
|
| 78 |
+
timeoutMs: loadTimeoutMs,
|
| 79 |
+
});
|
| 80 |
+
await runPageTask(client, "window.browserSpeakBench.clearResults()", { label: "clear results" });
|
| 81 |
+
|
| 82 |
+
console.log(`Real microphone validation: say "What app is this?" for each of ${count} row(s).`);
|
| 83 |
+
console.log("Keep the browser window focused and allow microphone access if Chrome asks.");
|
| 84 |
+
for (let i = 1; i <= count; i += 1) {
|
| 85 |
+
console.log(`Real microphone row ${i}/${count}: say "What app is this?"`);
|
| 86 |
+
await runPageTask(
|
| 87 |
+
client,
|
| 88 |
+
`window.browserSpeakBench.runMic(${JSON.stringify({
|
| 89 |
+
timeoutMs: Math.max(1000, rowTimeoutMs - 5000),
|
| 90 |
+
requireExactTranscript,
|
| 91 |
+
})})`,
|
| 92 |
+
{
|
| 93 |
+
label: `real microphone row ${i}`,
|
| 94 |
+
timeoutMs: rowTimeoutMs,
|
| 95 |
+
},
|
| 96 |
+
);
|
| 97 |
+
}
|
| 98 |
+
await runPageTask(client, "window.browserSpeakBench.stopMic()", { label: "stop microphone", timeoutMs: 30000 });
|
| 99 |
+
|
| 100 |
+
const exportPayload = await runPageTask(client, "window.browserSpeakBench.exportResults()", {
|
| 101 |
+
label: "export results",
|
| 102 |
+
timeoutMs: 30000,
|
| 103 |
+
});
|
| 104 |
+
const rows = exportPayload.results.filter((row) => row.kind === "mic").slice(0, count);
|
| 105 |
+
const errors = rows.filter((row) => row.error);
|
| 106 |
+
const exactRows = rows.filter((row) => row.sttWer === 0).length;
|
| 107 |
+
const payload = {
|
| 108 |
+
generatedAt: new Date().toISOString(),
|
| 109 |
+
sourceFingerprint: await sourceFingerprint(),
|
| 110 |
+
url,
|
| 111 |
+
passed: rows.length >= count && errors.length === 0 && (!requireExactTranscript || exactRows >= count),
|
| 112 |
+
config: {
|
| 113 |
+
...runConfig(),
|
| 114 |
+
},
|
| 115 |
+
hostMetadata: exportPayload.hostMetadata ?? null,
|
| 116 |
+
runtime: exportPayload.runtime ?? null,
|
| 117 |
+
evidence: exportPayload.evidence ?? null,
|
| 118 |
+
summary: summarizeRows(rows),
|
| 119 |
+
exportSummary: exportPayload.summary,
|
| 120 |
+
browserExport: exportPayload,
|
| 121 |
+
results: exportPayload.results,
|
| 122 |
+
};
|
| 123 |
+
await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`);
|
| 124 |
+
console.log(`Wrote real microphone benchmark JSON: ${resultPath}`);
|
| 125 |
+
console.log(
|
| 126 |
+
[
|
| 127 |
+
`${rows.length}/${count} rows`,
|
| 128 |
+
`${exactRows}/${rows.length} exact transcripts`,
|
| 129 |
+
`median WER ${formatPercent(payload.summary.medianWer)}`,
|
| 130 |
+
`median speech-end-to-audio ${formatMs(payload.summary.medianSpeechEndToFirstAudioMs)}`,
|
| 131 |
+
].join(", "),
|
| 132 |
+
);
|
| 133 |
+
if (!payload.passed) process.exitCode = 1;
|
| 134 |
+
if (!keepBrowserOpen) await client.closeBrowser();
|
| 135 |
+
} catch (error) {
|
| 136 |
+
await writeFailurePayload(client, browser, error).catch(() => {});
|
| 137 |
+
if (client && !keepBrowserOpen) await client.closeBrowser().catch(() => {});
|
| 138 |
+
throw error;
|
| 139 |
+
} finally {
|
| 140 |
+
if (!keepBrowserOpen) await stopBrowser(browser, reuseProfile ? null : profileDir);
|
| 141 |
+
}
|
| 142 |
+
}
|
| 143 |
+
|
| 144 |
+
async function writeFailurePayload(client, browser, error) {
|
| 145 |
+
const snapshot = client
|
| 146 |
+
? await client.evaluate("window.browserSpeakBench?.state?.() ?? null", pollTimeoutMs).catch(() => null)
|
| 147 |
+
: null;
|
| 148 |
+
await writeFile(
|
| 149 |
+
resultPath,
|
| 150 |
+
`${JSON.stringify(
|
| 151 |
+
{
|
| 152 |
+
generatedAt: new Date().toISOString(),
|
| 153 |
+
sourceFingerprint: await sourceFingerprint(),
|
| 154 |
+
passed: false,
|
| 155 |
+
error: error.stack ?? error.message ?? String(error),
|
| 156 |
+
browserLog: browser?.browserLog ?? "",
|
| 157 |
+
config: {
|
| 158 |
+
...runConfig(),
|
| 159 |
+
},
|
| 160 |
+
snapshot,
|
| 161 |
+
},
|
| 162 |
+
null,
|
| 163 |
+
2,
|
| 164 |
+
)}\n`,
|
| 165 |
+
);
|
| 166 |
+
console.log(`Wrote real microphone failure JSON: ${resultPath}`);
|
| 167 |
+
}
|
| 168 |
+
|
| 169 |
+
function runConfig() {
|
| 170 |
+
return {
|
| 171 |
+
stack,
|
| 172 |
+
count,
|
| 173 |
+
rowTimeoutMs,
|
| 174 |
+
requireExactTranscript,
|
| 175 |
+
headless,
|
| 176 |
+
dryRun,
|
| 177 |
+
reuseProfile,
|
| 178 |
+
autoAcceptMic,
|
| 179 |
+
allowFakeCapture,
|
| 180 |
+
keepBrowserOpen,
|
| 181 |
+
chrome,
|
| 182 |
+
profileDir,
|
| 183 |
+
extraChromeArgs: parseChromeArgs(),
|
| 184 |
+
};
|
| 185 |
+
}
|
| 186 |
+
|
| 187 |
+
function realMicPreflight() {
|
| 188 |
+
const warnings = [];
|
| 189 |
+
const fakeCaptureArgs = parseChromeArgs().filter(isFakeCaptureArg);
|
| 190 |
+
if (!headless && process.platform === "linux" && !process.env.DISPLAY && !process.env.WAYLAND_DISPLAY) {
|
| 191 |
+
warnings.push(
|
| 192 |
+
"Visible Chrome needs DISPLAY or WAYLAND_DISPLAY on Linux. Run from a desktop session, set BROWSER_SPEAK_HEADLESS=true, or provide a display.",
|
| 193 |
+
);
|
| 194 |
+
}
|
| 195 |
+
if (!existsSync(chrome) && chrome.includes("/")) {
|
| 196 |
+
warnings.push(`Chrome binary was not found: ${chrome}`);
|
| 197 |
+
}
|
| 198 |
+
if (!allowFakeCapture && fakeCaptureArgs.length > 0) {
|
| 199 |
+
warnings.push(
|
| 200 |
+
`Real microphone validation cannot use fake-capture Chrome args: ${fakeCaptureArgs.join(
|
| 201 |
+
", ",
|
| 202 |
+
)}. Set BROWSER_SPEAK_REAL_MIC_ALLOW_FAKE_CAPTURE=true only for harness debugging.`,
|
| 203 |
+
);
|
| 204 |
+
}
|
| 205 |
+
return {
|
| 206 |
+
ok: warnings.length === 0,
|
| 207 |
+
warnings,
|
| 208 |
+
fakeCaptureArgs,
|
| 209 |
+
platform: process.platform,
|
| 210 |
+
display: process.env.DISPLAY ?? "",
|
| 211 |
+
waylandDisplay: process.env.WAYLAND_DISPLAY ?? "",
|
| 212 |
+
};
|
| 213 |
+
}
|
| 214 |
+
|
| 215 |
+
function summarizeRows(rows) {
|
| 216 |
+
return {
|
| 217 |
+
rows: rows.length,
|
| 218 |
+
completedRows: rows.filter((row) => !row.error).length,
|
| 219 |
+
errorRows: rows.filter((row) => row.error).length,
|
| 220 |
+
exactTranscriptRows: rows.filter((row) => row.sttWer === 0).length,
|
| 221 |
+
medianWer: median(rows.map((row) => row.sttWer)),
|
| 222 |
+
medianCer: median(rows.map((row) => row.sttCer)),
|
| 223 |
+
medianAsrMs: median(rows.map((row) => row.asrMs)),
|
| 224 |
+
medianFirstTokenMs: median(rows.map((row) => row.firstTokenMs)),
|
| 225 |
+
medianFirstAudioMs: median(rows.map((row) => row.firstAudioMs)),
|
| 226 |
+
medianSpeechEndToFirstAudioMs: median(rows.map((row) => row.speechEndToFirstAudioMs)),
|
| 227 |
+
medianSpeechEndToAudioEndMs: median(rows.map((row) => row.speechEndToAudioEndMs)),
|
| 228 |
+
identityPasses: rows.filter((row) => row.llmQualityPass).length,
|
| 229 |
+
transcripts: rows.map((row) => row.transcript ?? ""),
|
| 230 |
+
errors: rows.filter((row) => row.error).map((row) => row.error),
|
| 231 |
+
};
|
| 232 |
+
}
|
| 233 |
+
|
| 234 |
+
async function ensureServer() {
|
| 235 |
+
const response = await fetch(url).catch((error) => {
|
| 236 |
+
throw new Error(`Could not reach ${url}: ${error.message}`);
|
| 237 |
+
});
|
| 238 |
+
if (!response.ok) throw new Error(`${url} returned HTTP ${response.status}`);
|
| 239 |
+
}
|
| 240 |
+
|
| 241 |
+
async function waitForBenchApi(client) {
|
| 242 |
+
const deadline = Date.now() + 15000;
|
| 243 |
+
while (Date.now() < deadline) {
|
| 244 |
+
try {
|
| 245 |
+
if (await client.evaluate("Boolean(window.browserSpeakBench)")) return;
|
| 246 |
+
} catch {
|
| 247 |
+
// The target may still be navigating.
|
| 248 |
+
}
|
| 249 |
+
await sleep(100);
|
| 250 |
+
}
|
| 251 |
+
throw new Error("window.browserSpeakBench was not installed.");
|
| 252 |
+
}
|
| 253 |
+
|
| 254 |
+
async function runPageTask(client, expression, { label = "page task", timeoutMs = 30000 } = {}) {
|
| 255 |
+
const taskId = `task_${Date.now()}_${Math.random().toString(16).slice(2)}`;
|
| 256 |
+
await client.evaluate(`(() => {
|
| 257 |
+
const taskId = ${JSON.stringify(taskId)};
|
| 258 |
+
window.__browserSpeakHarnessTasks ||= {};
|
| 259 |
+
window.__browserSpeakHarnessTasks[taskId] = { done: false, label: ${JSON.stringify(label)} };
|
| 260 |
+
Promise.resolve(${expression})
|
| 261 |
+
.then((value) => {
|
| 262 |
+
window.__browserSpeakHarnessTasks[taskId] = { done: true, value };
|
| 263 |
+
})
|
| 264 |
+
.catch((error) => {
|
| 265 |
+
window.__browserSpeakHarnessTasks[taskId] = {
|
| 266 |
+
done: true,
|
| 267 |
+
error: error?.stack || error?.message || String(error),
|
| 268 |
+
};
|
| 269 |
+
});
|
| 270 |
+
return true;
|
| 271 |
+
})()`);
|
| 272 |
+
const deadline = Date.now() + timeoutMs;
|
| 273 |
+
let lastEvents = [];
|
| 274 |
+
let lastPollError = "";
|
| 275 |
+
while (Date.now() < deadline) {
|
| 276 |
+
let task = null;
|
| 277 |
+
try {
|
| 278 |
+
task = await client.evaluate(
|
| 279 |
+
`window.__browserSpeakHarnessTasks?.[${JSON.stringify(taskId)}] ?? null`,
|
| 280 |
+
pollTimeoutMs,
|
| 281 |
+
);
|
| 282 |
+
lastPollError = "";
|
| 283 |
+
} catch (error) {
|
| 284 |
+
const message = error.message ?? String(error);
|
| 285 |
+
if (message !== lastPollError) {
|
| 286 |
+
console.log(`${label}: waiting for page response (${message})`);
|
| 287 |
+
lastPollError = message;
|
| 288 |
+
}
|
| 289 |
+
await sleep(500);
|
| 290 |
+
continue;
|
| 291 |
+
}
|
| 292 |
+
if (task?.done) {
|
| 293 |
+
await client
|
| 294 |
+
.evaluate(`delete window.__browserSpeakHarnessTasks?.[${JSON.stringify(taskId)}]`, pollTimeoutMs)
|
| 295 |
+
.catch(() => {});
|
| 296 |
+
if (task.error) throw new Error(`${label} failed: ${task.error}`);
|
| 297 |
+
return task.value;
|
| 298 |
+
}
|
| 299 |
+
const snapshot = await client
|
| 300 |
+
.evaluate(`(() => {
|
| 301 |
+
const state = window.browserSpeakBench?.state?.();
|
| 302 |
+
return state ? {
|
| 303 |
+
modelsLoaded: state.modelsLoaded,
|
| 304 |
+
modelsLoading: state.modelsLoading,
|
| 305 |
+
micActive: state.micActive,
|
| 306 |
+
activeBenchmark: state.activeBenchmark?.kind ?? null,
|
| 307 |
+
micSeries: state.micSeries,
|
| 308 |
+
events: state.events?.slice(0, 3) ?? [],
|
| 309 |
+
} : null;
|
| 310 |
+
})()`, pollTimeoutMs)
|
| 311 |
+
.catch(() => null);
|
| 312 |
+
const events = snapshot?.events ?? [];
|
| 313 |
+
if (events.join("\n") !== lastEvents.join("\n")) {
|
| 314 |
+
lastEvents = events;
|
| 315 |
+
if (events[0]) console.log(`${label}: ${events[0]}`);
|
| 316 |
+
}
|
| 317 |
+
await sleep(500);
|
| 318 |
+
}
|
| 319 |
+
throw new Error(`${label} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`);
|
| 320 |
+
}
|
| 321 |
+
|
| 322 |
+
function launchBrowser(port, profileDir) {
|
| 323 |
+
const child = spawn(
|
| 324 |
+
chrome,
|
| 325 |
+
[
|
| 326 |
+
...(headless ? ["--headless=new"] : []),
|
| 327 |
+
"--no-sandbox",
|
| 328 |
+
"--disable-dev-shm-usage",
|
| 329 |
+
"--no-default-browser-check",
|
| 330 |
+
"--no-first-run",
|
| 331 |
+
"--autoplay-policy=no-user-gesture-required",
|
| 332 |
+
...(autoAcceptMic ? ["--use-fake-ui-for-media-stream"] : []),
|
| 333 |
+
`--remote-debugging-port=${port}`,
|
| 334 |
+
`--user-data-dir=${profileDir}`,
|
| 335 |
+
...parseChromeArgs(),
|
| 336 |
+
],
|
| 337 |
+
{ stdio: ["ignore", "pipe", "pipe"] },
|
| 338 |
+
);
|
| 339 |
+
child.browserLog = "";
|
| 340 |
+
const appendLog = (chunk) => {
|
| 341 |
+
child.browserLog = `${child.browserLog}${chunk}`;
|
| 342 |
+
if (child.browserLog.length > 8000) child.browserLog = child.browserLog.slice(-8000);
|
| 343 |
+
};
|
| 344 |
+
child.stdout.on("data", appendLog);
|
| 345 |
+
child.stderr.on("data", appendLog);
|
| 346 |
+
return child;
|
| 347 |
+
}
|
| 348 |
+
|
| 349 |
+
async function stopBrowser(child, profileDir) {
|
| 350 |
+
if (child.exitCode == null) child.kill("SIGTERM");
|
| 351 |
+
await new Promise((resolve) => {
|
| 352 |
+
child.once("exit", resolve);
|
| 353 |
+
setTimeout(resolve, 3000);
|
| 354 |
+
});
|
| 355 |
+
if (child.exitCode == null) child.kill("SIGKILL");
|
| 356 |
+
if (!profileDir) return;
|
| 357 |
+
for (let attempt = 0; attempt < 5; attempt += 1) {
|
| 358 |
+
try {
|
| 359 |
+
await rm(profileDir, { recursive: true, force: true });
|
| 360 |
+
return;
|
| 361 |
+
} catch (error) {
|
| 362 |
+
if (attempt === 4) {
|
| 363 |
+
console.warn(`Could not remove ${profileDir}: ${error.message}`);
|
| 364 |
+
return;
|
| 365 |
+
}
|
| 366 |
+
await sleep(500);
|
| 367 |
+
}
|
| 368 |
+
}
|
| 369 |
+
}
|
| 370 |
+
|
| 371 |
+
async function connectToPage(port, child) {
|
| 372 |
+
const deadline = Date.now() + 60000;
|
| 373 |
+
let lastError = null;
|
| 374 |
+
while (Date.now() < deadline) {
|
| 375 |
+
if (child.exitCode != null) {
|
| 376 |
+
throw new Error(`Chrome exited before DevTools became available.\n${child.browserLog}`);
|
| 377 |
+
}
|
| 378 |
+
try {
|
| 379 |
+
const version = await fetch(`http://127.0.0.1:${port}/json/version`).then((response) => response.json());
|
| 380 |
+
if (version.webSocketDebuggerUrl) {
|
| 381 |
+
const page = await createPageTarget(port);
|
| 382 |
+
return new CdpClient(page.webSocketDebuggerUrl);
|
| 383 |
+
}
|
| 384 |
+
} catch (error) {
|
| 385 |
+
lastError = error;
|
| 386 |
+
}
|
| 387 |
+
await sleep(250);
|
| 388 |
+
}
|
| 389 |
+
throw new Error(
|
| 390 |
+
`Could not connect to Chrome DevTools on port ${port}: ${lastError?.message ?? "unknown error"}\n${child.browserLog}`,
|
| 391 |
+
);
|
| 392 |
+
}
|
| 393 |
+
|
| 394 |
+
async function createPageTarget(port) {
|
| 395 |
+
for (const method of ["PUT", "GET"]) {
|
| 396 |
+
const response = await fetch(`http://127.0.0.1:${port}/json/new?${encodeURIComponent(url)}`, {
|
| 397 |
+
method,
|
| 398 |
+
}).catch(() => null);
|
| 399 |
+
if (response?.ok) {
|
| 400 |
+
const target = await response.json();
|
| 401 |
+
if (target.webSocketDebuggerUrl) return target;
|
| 402 |
+
}
|
| 403 |
+
}
|
| 404 |
+
const targets = await fetch(`http://127.0.0.1:${port}/json`).then((response) => response.json());
|
| 405 |
+
const page = targets.find((target) => target.type === "page" && target.url === url);
|
| 406 |
+
if (page?.webSocketDebuggerUrl) return page;
|
| 407 |
+
throw new Error("Could not create or find a page target.");
|
| 408 |
+
}
|
| 409 |
+
|
| 410 |
+
class CdpClient {
|
| 411 |
+
constructor(webSocketUrl) {
|
| 412 |
+
this.nextId = 1;
|
| 413 |
+
this.pending = new Map();
|
| 414 |
+
this.socket = new WebSocket(webSocketUrl);
|
| 415 |
+
this.opened = new Promise((resolve, reject) => {
|
| 416 |
+
this.socket.onopen = resolve;
|
| 417 |
+
this.socket.onerror = reject;
|
| 418 |
+
this.socket.onmessage = (event) => this.onMessage(event);
|
| 419 |
+
});
|
| 420 |
+
}
|
| 421 |
+
|
| 422 |
+
onMessage(event) {
|
| 423 |
+
const message = JSON.parse(event.data);
|
| 424 |
+
if (!message.id || !this.pending.has(message.id)) return;
|
| 425 |
+
const { resolve: onResolve, reject } = this.pending.get(message.id);
|
| 426 |
+
this.pending.delete(message.id);
|
| 427 |
+
if (message.error) reject(new Error(message.error.message));
|
| 428 |
+
else onResolve(message.result);
|
| 429 |
+
}
|
| 430 |
+
|
| 431 |
+
async call(method, params = {}, timeoutMs = protocolTimeoutMs) {
|
| 432 |
+
await this.opened;
|
| 433 |
+
const id = this.nextId++;
|
| 434 |
+
this.socket.send(JSON.stringify({ id, method, params }));
|
| 435 |
+
return new Promise((resolvePromise, reject) => {
|
| 436 |
+
const timer = setTimeout(() => {
|
| 437 |
+
this.pending.delete(id);
|
| 438 |
+
reject(new Error(`${method} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`));
|
| 439 |
+
}, timeoutMs);
|
| 440 |
+
this.pending.set(id, {
|
| 441 |
+
resolve: (value) => {
|
| 442 |
+
clearTimeout(timer);
|
| 443 |
+
resolvePromise(value);
|
| 444 |
+
},
|
| 445 |
+
reject: (error) => {
|
| 446 |
+
clearTimeout(timer);
|
| 447 |
+
reject(error);
|
| 448 |
+
},
|
| 449 |
+
});
|
| 450 |
+
});
|
| 451 |
+
}
|
| 452 |
+
|
| 453 |
+
async evaluate(expression, timeoutMs = protocolTimeoutMs) {
|
| 454 |
+
const result = await this.call(
|
| 455 |
+
"Runtime.evaluate",
|
| 456 |
+
{
|
| 457 |
+
expression,
|
| 458 |
+
awaitPromise: true,
|
| 459 |
+
returnByValue: true,
|
| 460 |
+
},
|
| 461 |
+
timeoutMs,
|
| 462 |
+
);
|
| 463 |
+
if (result.exceptionDetails) throw new Error(formatException(result.exceptionDetails));
|
| 464 |
+
return result.result.value;
|
| 465 |
+
}
|
| 466 |
+
|
| 467 |
+
async closeBrowser() {
|
| 468 |
+
await this.call("Browser.close").catch(() => {});
|
| 469 |
+
}
|
| 470 |
+
}
|
| 471 |
+
|
| 472 |
+
function formatException(details) {
|
| 473 |
+
return details.exception?.description ?? details.text ?? "Evaluation failed.";
|
| 474 |
+
}
|
| 475 |
+
|
| 476 |
+
function parseChromeArgs() {
|
| 477 |
+
return (process.env.BROWSER_SPEAK_CHROME_ARGS ?? "")
|
| 478 |
+
.split(/\s+/)
|
| 479 |
+
.map((arg) => arg.trim())
|
| 480 |
+
.filter(Boolean);
|
| 481 |
+
}
|
| 482 |
+
|
| 483 |
+
function isFakeCaptureArg(arg) {
|
| 484 |
+
return (
|
| 485 |
+
arg === "--use-fake-device-for-media-stream" ||
|
| 486 |
+
arg.startsWith("--use-file-for-fake-audio-capture=") ||
|
| 487 |
+
arg.startsWith("--use-file-for-fake-video-capture=")
|
| 488 |
+
);
|
| 489 |
+
}
|
| 490 |
+
|
| 491 |
+
function median(values) {
|
| 492 |
+
const finite = values.filter(Number.isFinite).sort((a, b) => a - b);
|
| 493 |
+
if (finite.length === 0) return null;
|
| 494 |
+
const middle = Math.floor(finite.length / 2);
|
| 495 |
+
if (finite.length % 2 === 1) return finite[middle];
|
| 496 |
+
return (finite[middle - 1] + finite[middle]) / 2;
|
| 497 |
+
}
|
| 498 |
+
|
| 499 |
+
function formatMs(value) {
|
| 500 |
+
if (!Number.isFinite(value)) return "-";
|
| 501 |
+
if (value < 1000) return `${Math.round(value)} ms`;
|
| 502 |
+
return `${(value / 1000).toFixed(2)} s`;
|
| 503 |
+
}
|
| 504 |
+
|
| 505 |
+
function formatPercent(value) {
|
| 506 |
+
if (!Number.isFinite(value)) return "-";
|
| 507 |
+
return `${Math.round(value * 100)}%`;
|
| 508 |
+
}
|
| 509 |
+
|
| 510 |
+
function sleep(ms) {
|
| 511 |
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
| 512 |
+
}
|
| 513 |
+
|
| 514 |
+
main().catch((error) => {
|
| 515 |
+
console.error(error.stack ?? error.message);
|
| 516 |
+
process.exitCode = 1;
|
| 517 |
+
});
|
tools/run-tts-sweep-benchmark.mjs
ADDED
|
@@ -0,0 +1,453 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env node
|
| 2 |
+
import { spawn } from "node:child_process";
|
| 3 |
+
import { existsSync } from "node:fs";
|
| 4 |
+
import { mkdir, rm, writeFile } from "node:fs/promises";
|
| 5 |
+
import { tmpdir } from "node:os";
|
| 6 |
+
import { dirname, resolve } from "node:path";
|
| 7 |
+
|
| 8 |
+
const url = process.env.BROWSER_SPEAK_URL ?? "http://127.0.0.1:5174/";
|
| 9 |
+
const chrome =
|
| 10 |
+
process.env.CHROME_BIN ??
|
| 11 |
+
(existsSync("/opt/google/chrome/chrome")
|
| 12 |
+
? "/opt/google/chrome/chrome"
|
| 13 |
+
: existsSync("/usr/bin/google-chrome")
|
| 14 |
+
? "/usr/bin/google-chrome"
|
| 15 |
+
: "chromium");
|
| 16 |
+
const resultPath = resolve(process.env.BROWSER_SPEAK_TTS_SWEEP_JSON ?? `${tmpdir()}/browser-speak-tts-sweep.json`);
|
| 17 |
+
const profileDir = resolve(process.env.BROWSER_SPEAK_TTS_SWEEP_PROFILE_DIR ?? `${tmpdir()}/browser-speak-tts-sweep-profile`);
|
| 18 |
+
const protocolTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_TIMEOUT_MS ?? 60000);
|
| 19 |
+
const pollTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_POLL_TIMEOUT_MS ?? 5000);
|
| 20 |
+
const loadTimeoutMs = Number(process.env.BROWSER_SPEAK_LOAD_TIMEOUT_MS ?? 360000);
|
| 21 |
+
const taskTimeoutMs = Number(process.env.BROWSER_SPEAK_TASK_TIMEOUT_MS ?? 120000);
|
| 22 |
+
const headless = process.env.BROWSER_SPEAK_HEADLESS !== "false";
|
| 23 |
+
const dryRun = process.env.BROWSER_SPEAK_TTS_SWEEP_DRY_RUN === "true";
|
| 24 |
+
const voices = parseList(process.env.BROWSER_SPEAK_TTS_SWEEP_VOICES ?? "F1,F2,M1,M2");
|
| 25 |
+
const steps = parseList(process.env.BROWSER_SPEAK_TTS_SWEEP_STEPS ?? "2,4,8").map((value) => Number(value));
|
| 26 |
+
const stack = {
|
| 27 |
+
device: process.env.BROWSER_SPEAK_DEVICE ?? "wasm",
|
| 28 |
+
llm: process.env.BROWSER_SPEAK_LLM ?? "HuggingFaceTB/SmolLM2-135M-Instruct",
|
| 29 |
+
asr: process.env.BROWSER_SPEAK_ASR ?? "onnx-community/moonshine-base-ONNX",
|
| 30 |
+
voice: voices[0] ?? "F2",
|
| 31 |
+
ttsSteps: steps[0] ?? 2,
|
| 32 |
+
vadSilenceMs: Number(process.env.BROWSER_SPEAK_VAD_SILENCE_MS ?? 480),
|
| 33 |
+
partialAsr: process.env.BROWSER_SPEAK_PARTIAL_ASR !== "false",
|
| 34 |
+
};
|
| 35 |
+
const candidates = buildCandidates();
|
| 36 |
+
|
| 37 |
+
async function main() {
|
| 38 |
+
if (dryRun) {
|
| 39 |
+
await mkdir(dirname(resultPath), { recursive: true });
|
| 40 |
+
await writeJson({
|
| 41 |
+
generatedAt: new Date().toISOString(),
|
| 42 |
+
dryRun: true,
|
| 43 |
+
url,
|
| 44 |
+
config: benchmarkConfig(),
|
| 45 |
+
candidates: candidates.map((candidate) => ({ ...candidate, status: "planned" })),
|
| 46 |
+
});
|
| 47 |
+
console.log(`Wrote TTS sweep dry-run JSON: ${resultPath}`);
|
| 48 |
+
console.log(`Planned ${candidates.length} TTS candidate(s): ${candidates.map(candidateLabel).join("; ")}`);
|
| 49 |
+
return;
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
await ensureServer();
|
| 53 |
+
await mkdir(dirname(resultPath), { recursive: true });
|
| 54 |
+
await rm(profileDir, { recursive: true, force: true });
|
| 55 |
+
|
| 56 |
+
const browser = launchBrowser(9345, profileDir);
|
| 57 |
+
try {
|
| 58 |
+
const client = await connectToPage(9345, browser);
|
| 59 |
+
await waitForBenchApi(client);
|
| 60 |
+
await runPageTask(client, `window.browserSpeakBench.loadStack(${JSON.stringify(stack)})`, {
|
| 61 |
+
label: "model load",
|
| 62 |
+
timeoutMs: loadTimeoutMs,
|
| 63 |
+
});
|
| 64 |
+
await runPageTask(client, "window.browserSpeakBench.clearResults()", { label: "clear results" });
|
| 65 |
+
|
| 66 |
+
for (const candidate of candidates) {
|
| 67 |
+
console.log(`Running TTS sweep candidate: ${candidateLabel(candidate)}`);
|
| 68 |
+
try {
|
| 69 |
+
await runPageTask(client, `window.browserSpeakBench.setRuntimeOptions(${JSON.stringify(candidate)})`, {
|
| 70 |
+
label: `${candidateLabel(candidate)} options`,
|
| 71 |
+
timeoutMs: 20000,
|
| 72 |
+
});
|
| 73 |
+
await runPageTask(
|
| 74 |
+
client,
|
| 75 |
+
`window.browserSpeakBench.preloadVoice(${JSON.stringify({ voice: candidate.voice })})`,
|
| 76 |
+
{
|
| 77 |
+
label: `${candidateLabel(candidate)} voice preload`,
|
| 78 |
+
timeoutMs: 45000,
|
| 79 |
+
},
|
| 80 |
+
);
|
| 81 |
+
const before = await runPageTask(client, "window.browserSpeakBench.state()", {
|
| 82 |
+
label: `${candidateLabel(candidate)} preflight`,
|
| 83 |
+
});
|
| 84 |
+
const snapshot = await runPageTask(client, "window.browserSpeakBench.runTts({ timeoutMs: 90000 })", {
|
| 85 |
+
label: `${candidateLabel(candidate)} tts`,
|
| 86 |
+
timeoutMs: taskTimeoutMs,
|
| 87 |
+
});
|
| 88 |
+
const added = snapshot.results.slice(0, Math.max(0, snapshot.results.length - before.results.length));
|
| 89 |
+
const row = added.find((result) => result.kind === "tts");
|
| 90 |
+
candidate.status = row && !row.error ? "complete" : "failed";
|
| 91 |
+
candidate.rowId = row?.id ?? null;
|
| 92 |
+
candidate.error = row?.error || "";
|
| 93 |
+
candidate.metrics = row
|
| 94 |
+
? {
|
| 95 |
+
firstAudioMs: row.firstAudioMs,
|
| 96 |
+
synthesisMs: row.firstTtsSynthesisMs,
|
| 97 |
+
audioEndMs: row.audioEndMs,
|
| 98 |
+
firstTtsRoundTripMs: row.firstTtsRoundTripMs,
|
| 99 |
+
firstTtsPlaybackDelayMs: row.firstTtsPlaybackDelayMs,
|
| 100 |
+
modelLoadMs: row.stack?.modelLoadMs ?? null,
|
| 101 |
+
}
|
| 102 |
+
: null;
|
| 103 |
+
} catch (error) {
|
| 104 |
+
candidate.status = "failed";
|
| 105 |
+
candidate.error = error.message;
|
| 106 |
+
console.error(`${candidateLabel(candidate)} failed: ${error.message}`);
|
| 107 |
+
} finally {
|
| 108 |
+
await writeJson(await exportPayload(client));
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
const payload = await exportPayload(client);
|
| 113 |
+
await writeJson(payload);
|
| 114 |
+
console.log(`Wrote TTS sweep benchmark JSON: ${resultPath}`);
|
| 115 |
+
summarizeCandidates(candidates);
|
| 116 |
+
client.closeBrowser();
|
| 117 |
+
} finally {
|
| 118 |
+
await stopBrowser(browser, profileDir);
|
| 119 |
+
}
|
| 120 |
+
}
|
| 121 |
+
|
| 122 |
+
async function exportPayload(client) {
|
| 123 |
+
const exportResult = await runPageTask(client, "window.browserSpeakBench.exportResults()", {
|
| 124 |
+
label: "export results",
|
| 125 |
+
timeoutMs: 30000,
|
| 126 |
+
}).catch(() => ({ summary: null, results: [] }));
|
| 127 |
+
return {
|
| 128 |
+
generatedAt: new Date().toISOString(),
|
| 129 |
+
url,
|
| 130 |
+
config: benchmarkConfig(),
|
| 131 |
+
candidates,
|
| 132 |
+
summary: exportResult.summary,
|
| 133 |
+
results: exportResult.results,
|
| 134 |
+
};
|
| 135 |
+
}
|
| 136 |
+
|
| 137 |
+
function benchmarkConfig() {
|
| 138 |
+
return {
|
| 139 |
+
stack,
|
| 140 |
+
voices,
|
| 141 |
+
steps,
|
| 142 |
+
candidateCount: candidates.length,
|
| 143 |
+
loadTimeoutMs,
|
| 144 |
+
taskTimeoutMs,
|
| 145 |
+
protocolTimeoutMs,
|
| 146 |
+
pollTimeoutMs,
|
| 147 |
+
headless,
|
| 148 |
+
chrome,
|
| 149 |
+
profileDir,
|
| 150 |
+
extraChromeArgs: parseChromeArgs(),
|
| 151 |
+
};
|
| 152 |
+
}
|
| 153 |
+
|
| 154 |
+
function buildCandidates() {
|
| 155 |
+
const list = [];
|
| 156 |
+
for (const voice of voices) {
|
| 157 |
+
for (const ttsSteps of steps) {
|
| 158 |
+
list.push({ voice, ttsSteps });
|
| 159 |
+
}
|
| 160 |
+
}
|
| 161 |
+
return list;
|
| 162 |
+
}
|
| 163 |
+
|
| 164 |
+
async function writeJson(payload) {
|
| 165 |
+
await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`);
|
| 166 |
+
}
|
| 167 |
+
|
| 168 |
+
async function ensureServer() {
|
| 169 |
+
const response = await fetch(url).catch((error) => {
|
| 170 |
+
throw new Error(`Could not reach ${url}: ${error.message}`);
|
| 171 |
+
});
|
| 172 |
+
if (!response.ok) throw new Error(`${url} returned HTTP ${response.status}`);
|
| 173 |
+
}
|
| 174 |
+
|
| 175 |
+
async function waitForBenchApi(client) {
|
| 176 |
+
const deadline = Date.now() + 15000;
|
| 177 |
+
while (Date.now() < deadline) {
|
| 178 |
+
try {
|
| 179 |
+
if (await client.evaluate("Boolean(window.browserSpeakBench?.setRuntimeOptions)")) return;
|
| 180 |
+
} catch {
|
| 181 |
+
// The target may still be navigating and can destroy the execution context.
|
| 182 |
+
}
|
| 183 |
+
await sleep(100);
|
| 184 |
+
}
|
| 185 |
+
throw new Error("window.browserSpeakBench.setRuntimeOptions was not installed.");
|
| 186 |
+
}
|
| 187 |
+
|
| 188 |
+
async function runPageTask(client, expression, { label = "page task", timeoutMs = 30000 } = {}) {
|
| 189 |
+
const taskId = `task_${Date.now()}_${Math.random().toString(16).slice(2)}`;
|
| 190 |
+
await client.evaluate(`(() => {
|
| 191 |
+
const taskId = ${JSON.stringify(taskId)};
|
| 192 |
+
window.__browserSpeakHarnessTasks ||= {};
|
| 193 |
+
window.__browserSpeakHarnessTasks[taskId] = { done: false, label: ${JSON.stringify(label)} };
|
| 194 |
+
Promise.resolve(${expression})
|
| 195 |
+
.then((value) => {
|
| 196 |
+
window.__browserSpeakHarnessTasks[taskId] = { done: true, value };
|
| 197 |
+
})
|
| 198 |
+
.catch((error) => {
|
| 199 |
+
window.__browserSpeakHarnessTasks[taskId] = {
|
| 200 |
+
done: true,
|
| 201 |
+
error: error?.stack || error?.message || String(error),
|
| 202 |
+
};
|
| 203 |
+
});
|
| 204 |
+
return true;
|
| 205 |
+
})()`);
|
| 206 |
+
const deadline = Date.now() + timeoutMs;
|
| 207 |
+
let lastEvents = [];
|
| 208 |
+
let lastPollError = "";
|
| 209 |
+
while (Date.now() < deadline) {
|
| 210 |
+
let task = null;
|
| 211 |
+
try {
|
| 212 |
+
task = await client.evaluate(
|
| 213 |
+
`window.__browserSpeakHarnessTasks?.[${JSON.stringify(taskId)}] ?? null`,
|
| 214 |
+
pollTimeoutMs,
|
| 215 |
+
);
|
| 216 |
+
lastPollError = "";
|
| 217 |
+
} catch (error) {
|
| 218 |
+
const message = error.message ?? String(error);
|
| 219 |
+
if (message !== lastPollError) {
|
| 220 |
+
console.log(`${label}: waiting for page response (${message})`);
|
| 221 |
+
lastPollError = message;
|
| 222 |
+
}
|
| 223 |
+
await sleep(500);
|
| 224 |
+
continue;
|
| 225 |
+
}
|
| 226 |
+
if (task?.done) {
|
| 227 |
+
await client
|
| 228 |
+
.evaluate(`delete window.__browserSpeakHarnessTasks?.[${JSON.stringify(taskId)}]`, pollTimeoutMs)
|
| 229 |
+
.catch(() => {});
|
| 230 |
+
if (task.error) throw new Error(`${label} failed: ${task.error}`);
|
| 231 |
+
return task.value;
|
| 232 |
+
}
|
| 233 |
+
const snapshot = await client
|
| 234 |
+
.evaluate(`(() => {
|
| 235 |
+
const state = window.browserSpeakBench?.state?.();
|
| 236 |
+
return state ? {
|
| 237 |
+
modelsLoaded: state.modelsLoaded,
|
| 238 |
+
modelsLoading: state.modelsLoading,
|
| 239 |
+
activeBenchmark: state.activeBenchmark?.kind ?? null,
|
| 240 |
+
events: state.events?.slice(0, 3) ?? [],
|
| 241 |
+
} : null;
|
| 242 |
+
})()`, pollTimeoutMs)
|
| 243 |
+
.catch(() => null);
|
| 244 |
+
const events = snapshot?.events ?? [];
|
| 245 |
+
if (events.join("\\n") !== lastEvents.join("\\n")) {
|
| 246 |
+
lastEvents = events;
|
| 247 |
+
if (events[0]) console.log(`${label}: ${events[0]}`);
|
| 248 |
+
}
|
| 249 |
+
await sleep(500);
|
| 250 |
+
}
|
| 251 |
+
throw new Error(`${label} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`);
|
| 252 |
+
}
|
| 253 |
+
|
| 254 |
+
function launchBrowser(port, profileDir) {
|
| 255 |
+
const child = spawn(
|
| 256 |
+
chrome,
|
| 257 |
+
[
|
| 258 |
+
...(headless ? ["--headless=new"] : []),
|
| 259 |
+
"--no-sandbox",
|
| 260 |
+
"--disable-gpu",
|
| 261 |
+
"--disable-dev-shm-usage",
|
| 262 |
+
"--disable-background-networking",
|
| 263 |
+
"--disable-extensions",
|
| 264 |
+
"--no-default-browser-check",
|
| 265 |
+
"--no-first-run",
|
| 266 |
+
"--autoplay-policy=no-user-gesture-required",
|
| 267 |
+
`--remote-debugging-port=${port}`,
|
| 268 |
+
`--user-data-dir=${profileDir}`,
|
| 269 |
+
...parseChromeArgs(),
|
| 270 |
+
],
|
| 271 |
+
{ stdio: ["ignore", "pipe", "pipe"] },
|
| 272 |
+
);
|
| 273 |
+
child.browserLog = "";
|
| 274 |
+
const appendLog = (chunk) => {
|
| 275 |
+
child.browserLog = `${child.browserLog}${chunk}`;
|
| 276 |
+
if (child.browserLog.length > 8000) child.browserLog = child.browserLog.slice(-8000);
|
| 277 |
+
};
|
| 278 |
+
child.stdout.on("data", appendLog);
|
| 279 |
+
child.stderr.on("data", appendLog);
|
| 280 |
+
return child;
|
| 281 |
+
}
|
| 282 |
+
|
| 283 |
+
async function stopBrowser(child, profileDir) {
|
| 284 |
+
if (child.exitCode == null) child.kill("SIGTERM");
|
| 285 |
+
await new Promise((resolve) => {
|
| 286 |
+
child.once("exit", resolve);
|
| 287 |
+
setTimeout(resolve, 3000);
|
| 288 |
+
});
|
| 289 |
+
if (child.exitCode == null) child.kill("SIGKILL");
|
| 290 |
+
for (let attempt = 0; attempt < 5; attempt += 1) {
|
| 291 |
+
try {
|
| 292 |
+
await rm(profileDir, { recursive: true, force: true });
|
| 293 |
+
return;
|
| 294 |
+
} catch (error) {
|
| 295 |
+
if (attempt === 4) {
|
| 296 |
+
console.warn(`Could not remove ${profileDir}: ${error.message}`);
|
| 297 |
+
return;
|
| 298 |
+
}
|
| 299 |
+
await sleep(500);
|
| 300 |
+
}
|
| 301 |
+
}
|
| 302 |
+
}
|
| 303 |
+
|
| 304 |
+
async function connectToPage(port, child) {
|
| 305 |
+
const deadline = Date.now() + 60000;
|
| 306 |
+
let lastError = null;
|
| 307 |
+
while (Date.now() < deadline) {
|
| 308 |
+
if (child.exitCode != null) {
|
| 309 |
+
throw new Error(`Chrome exited before DevTools became available.\n${child.browserLog}`);
|
| 310 |
+
}
|
| 311 |
+
try {
|
| 312 |
+
const version = await fetch(`http://127.0.0.1:${port}/json/version`).then((response) => response.json());
|
| 313 |
+
if (version.webSocketDebuggerUrl) {
|
| 314 |
+
const page = await createPageTarget(port);
|
| 315 |
+
return new CdpClient(page.webSocketDebuggerUrl);
|
| 316 |
+
}
|
| 317 |
+
} catch (error) {
|
| 318 |
+
lastError = error;
|
| 319 |
+
}
|
| 320 |
+
await sleep(250);
|
| 321 |
+
}
|
| 322 |
+
throw new Error(
|
| 323 |
+
`Could not connect to Chrome DevTools on port ${port}: ${lastError?.message ?? "unknown error"}\n${child.browserLog}`,
|
| 324 |
+
);
|
| 325 |
+
}
|
| 326 |
+
|
| 327 |
+
async function createPageTarget(port) {
|
| 328 |
+
for (const method of ["PUT", "GET"]) {
|
| 329 |
+
const response = await fetch(`http://127.0.0.1:${port}/json/new?${encodeURIComponent(url)}`, {
|
| 330 |
+
method,
|
| 331 |
+
}).catch(() => null);
|
| 332 |
+
if (response?.ok) {
|
| 333 |
+
const target = await response.json();
|
| 334 |
+
if (target.webSocketDebuggerUrl) return target;
|
| 335 |
+
}
|
| 336 |
+
}
|
| 337 |
+
const targets = await fetch(`http://127.0.0.1:${port}/json`).then((response) => response.json());
|
| 338 |
+
const page = targets.find((target) => target.type === "page" && target.url === url);
|
| 339 |
+
if (page?.webSocketDebuggerUrl) return page;
|
| 340 |
+
throw new Error("Could not create or find a page target.");
|
| 341 |
+
}
|
| 342 |
+
|
| 343 |
+
class CdpClient {
|
| 344 |
+
constructor(webSocketUrl) {
|
| 345 |
+
this.nextId = 1;
|
| 346 |
+
this.pending = new Map();
|
| 347 |
+
this.socket = new WebSocket(webSocketUrl);
|
| 348 |
+
this.opened = new Promise((resolve, reject) => {
|
| 349 |
+
this.socket.onopen = resolve;
|
| 350 |
+
this.socket.onerror = reject;
|
| 351 |
+
this.socket.onmessage = (event) => this.onMessage(event);
|
| 352 |
+
});
|
| 353 |
+
}
|
| 354 |
+
|
| 355 |
+
onMessage(event) {
|
| 356 |
+
const message = JSON.parse(event.data);
|
| 357 |
+
if (!message.id || !this.pending.has(message.id)) return;
|
| 358 |
+
const { resolve: onResolve, reject } = this.pending.get(message.id);
|
| 359 |
+
this.pending.delete(message.id);
|
| 360 |
+
if (message.error) reject(new Error(message.error.message));
|
| 361 |
+
else onResolve(message.result);
|
| 362 |
+
}
|
| 363 |
+
|
| 364 |
+
async call(method, params = {}, timeoutMs = protocolTimeoutMs) {
|
| 365 |
+
await this.opened;
|
| 366 |
+
const id = this.nextId++;
|
| 367 |
+
this.socket.send(JSON.stringify({ id, method, params }));
|
| 368 |
+
return new Promise((resolvePromise, reject) => {
|
| 369 |
+
const timer = setTimeout(() => {
|
| 370 |
+
this.pending.delete(id);
|
| 371 |
+
reject(new Error(`${method} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`));
|
| 372 |
+
}, timeoutMs);
|
| 373 |
+
this.pending.set(id, {
|
| 374 |
+
resolve: (value) => {
|
| 375 |
+
clearTimeout(timer);
|
| 376 |
+
resolvePromise(value);
|
| 377 |
+
},
|
| 378 |
+
reject: (error) => {
|
| 379 |
+
clearTimeout(timer);
|
| 380 |
+
reject(error);
|
| 381 |
+
},
|
| 382 |
+
});
|
| 383 |
+
});
|
| 384 |
+
}
|
| 385 |
+
|
| 386 |
+
async evaluate(expression, timeoutMs = protocolTimeoutMs) {
|
| 387 |
+
const result = await this.call("Runtime.evaluate", {
|
| 388 |
+
expression,
|
| 389 |
+
returnByValue: true,
|
| 390 |
+
}, timeoutMs);
|
| 391 |
+
if (result.exceptionDetails) throw new Error(formatException(result.exceptionDetails));
|
| 392 |
+
return result.result.value;
|
| 393 |
+
}
|
| 394 |
+
|
| 395 |
+
closeBrowser() {
|
| 396 |
+
try {
|
| 397 |
+
this.socket.send(JSON.stringify({ id: this.nextId++, method: "Browser.close", params: {} }));
|
| 398 |
+
} catch {
|
| 399 |
+
// The surrounding process cleanup handles already-closed targets.
|
| 400 |
+
}
|
| 401 |
+
try {
|
| 402 |
+
this.socket.close();
|
| 403 |
+
} catch {
|
| 404 |
+
// Ignore close races.
|
| 405 |
+
}
|
| 406 |
+
}
|
| 407 |
+
}
|
| 408 |
+
|
| 409 |
+
function formatException(exceptionDetails) {
|
| 410 |
+
const exception = exceptionDetails.exception;
|
| 411 |
+
return exception?.description ?? exception?.value ?? exceptionDetails.text ?? "Evaluation failed.";
|
| 412 |
+
}
|
| 413 |
+
|
| 414 |
+
function parseList(value) {
|
| 415 |
+
return String(value ?? "")
|
| 416 |
+
.split(",")
|
| 417 |
+
.map((item) => item.trim())
|
| 418 |
+
.filter(Boolean);
|
| 419 |
+
}
|
| 420 |
+
|
| 421 |
+
function parseChromeArgs() {
|
| 422 |
+
const raw = process.env.BROWSER_SPEAK_CHROME_ARGS ?? "";
|
| 423 |
+
return (
|
| 424 |
+
raw
|
| 425 |
+
.match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g)
|
| 426 |
+
?.map((arg) => arg.replace(/^["']|["']$/g, "")) ?? []
|
| 427 |
+
);
|
| 428 |
+
}
|
| 429 |
+
|
| 430 |
+
function candidateLabel(candidate) {
|
| 431 |
+
return `${candidate.voice} / ${candidate.ttsSteps} steps`;
|
| 432 |
+
}
|
| 433 |
+
|
| 434 |
+
function summarizeCandidates(items) {
|
| 435 |
+
for (const candidate of items) {
|
| 436 |
+
const metrics = candidate.metrics ?? {};
|
| 437 |
+
const suffix = Number.isFinite(metrics.firstAudioMs) ? `, first audio ${formatMs(metrics.firstAudioMs)}` : "";
|
| 438 |
+
const error = candidate.error ? ` (${candidate.error})` : "";
|
| 439 |
+
console.log(`${candidateLabel(candidate)}: ${candidate.status ?? "pending"}${suffix}${error}`);
|
| 440 |
+
}
|
| 441 |
+
}
|
| 442 |
+
|
| 443 |
+
function formatMs(value) {
|
| 444 |
+
if (!Number.isFinite(value)) return "-";
|
| 445 |
+
if (value < 1000) return `${Math.round(value)} ms`;
|
| 446 |
+
return `${(value / 1000).toFixed(2)} s`;
|
| 447 |
+
}
|
| 448 |
+
|
| 449 |
+
function sleep(ms) {
|
| 450 |
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
| 451 |
+
}
|
| 452 |
+
|
| 453 |
+
await main();
|
tools/run-ui-smoke.mjs
ADDED
|
@@ -0,0 +1,482 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env node
|
| 2 |
+
import { spawn } from "node:child_process";
|
| 3 |
+
import { existsSync } from "node:fs";
|
| 4 |
+
import { mkdir, rm, writeFile } from "node:fs/promises";
|
| 5 |
+
import { tmpdir } from "node:os";
|
| 6 |
+
import { dirname, resolve } from "node:path";
|
| 7 |
+
import { sourceFingerprint } from "./source-fingerprint.mjs";
|
| 8 |
+
|
| 9 |
+
const url = process.env.BROWSER_SPEAK_URL ?? "http://127.0.0.1:5174/";
|
| 10 |
+
const chrome =
|
| 11 |
+
process.env.CHROME_BIN ??
|
| 12 |
+
(existsSync("/opt/google/chrome/chrome")
|
| 13 |
+
? "/opt/google/chrome/chrome"
|
| 14 |
+
: existsSync("/usr/bin/google-chrome")
|
| 15 |
+
? "/usr/bin/google-chrome"
|
| 16 |
+
: "chromium");
|
| 17 |
+
const resultPath = resolve(process.env.BROWSER_SPEAK_UI_JSON ?? `${tmpdir()}/browser-speak-ui-smoke.json`);
|
| 18 |
+
const screenshotDir = resolve(process.env.BROWSER_SPEAK_UI_SCREENSHOT_DIR ?? `${tmpdir()}/browser-speak-ui-smoke`);
|
| 19 |
+
const profileDir = resolve(process.env.BROWSER_SPEAK_UI_PROFILE_DIR ?? `${tmpdir()}/browser-speak-ui-profile`);
|
| 20 |
+
const protocolTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_TIMEOUT_MS ?? 60000);
|
| 21 |
+
const pollTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_POLL_TIMEOUT_MS ?? 5000);
|
| 22 |
+
const headless = process.env.BROWSER_SPEAK_HEADLESS !== "false";
|
| 23 |
+
const viewports = parseViewports(
|
| 24 |
+
process.env.BROWSER_SPEAK_UI_VIEWPORTS ?? "desktop:1440x1200,mobile:390x1100",
|
| 25 |
+
);
|
| 26 |
+
|
| 27 |
+
async function main() {
|
| 28 |
+
await ensureServer();
|
| 29 |
+
await mkdir(dirname(resultPath), { recursive: true });
|
| 30 |
+
await mkdir(screenshotDir, { recursive: true });
|
| 31 |
+
await rm(profileDir, { recursive: true, force: true });
|
| 32 |
+
|
| 33 |
+
const browser = launchBrowser(9345, profileDir);
|
| 34 |
+
const results = [];
|
| 35 |
+
try {
|
| 36 |
+
const client = await connectToPage(9345, browser);
|
| 37 |
+
await client.call("Page.enable");
|
| 38 |
+
await client.call("Runtime.enable");
|
| 39 |
+
|
| 40 |
+
for (const viewport of viewports) {
|
| 41 |
+
console.log(`Checking UI viewport: ${viewport.name} ${viewport.width}x${viewport.height}`);
|
| 42 |
+
await client.call("Emulation.setDeviceMetricsOverride", {
|
| 43 |
+
width: viewport.width,
|
| 44 |
+
height: viewport.height,
|
| 45 |
+
deviceScaleFactor: viewport.deviceScaleFactor,
|
| 46 |
+
mobile: viewport.mobile,
|
| 47 |
+
});
|
| 48 |
+
await client.call("Page.navigate", { url });
|
| 49 |
+
await waitForUi(client);
|
| 50 |
+
const audit = await client.evaluate(`(${auditUi.toString()})()`, pollTimeoutMs);
|
| 51 |
+
const screenshotPath = resolve(screenshotDir, `${viewport.name}.png`);
|
| 52 |
+
const screenshot = await client.call("Page.captureScreenshot", {
|
| 53 |
+
format: "png",
|
| 54 |
+
captureBeyondViewport: false,
|
| 55 |
+
});
|
| 56 |
+
await writeFile(screenshotPath, Buffer.from(screenshot.data, "base64"));
|
| 57 |
+
const errors = auditErrors(audit);
|
| 58 |
+
results.push({
|
| 59 |
+
...viewport,
|
| 60 |
+
screenshotPath,
|
| 61 |
+
...audit,
|
| 62 |
+
status: errors.length === 0 ? "pass" : "fail",
|
| 63 |
+
errors,
|
| 64 |
+
});
|
| 65 |
+
}
|
| 66 |
+
|
| 67 |
+
const payload = {
|
| 68 |
+
generatedAt: new Date().toISOString(),
|
| 69 |
+
sourceFingerprint: await sourceFingerprint(),
|
| 70 |
+
url,
|
| 71 |
+
config: {
|
| 72 |
+
chrome,
|
| 73 |
+
headless,
|
| 74 |
+
protocolTimeoutMs,
|
| 75 |
+
pollTimeoutMs,
|
| 76 |
+
screenshotDir,
|
| 77 |
+
viewports,
|
| 78 |
+
extraChromeArgs: parseChromeArgs(),
|
| 79 |
+
},
|
| 80 |
+
results,
|
| 81 |
+
passed: results.every((result) => result.status === "pass"),
|
| 82 |
+
};
|
| 83 |
+
await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`);
|
| 84 |
+
console.log(`Wrote UI smoke JSON: ${resultPath}`);
|
| 85 |
+
console.log(`Wrote screenshots: ${screenshotDir}`);
|
| 86 |
+
for (const result of results) {
|
| 87 |
+
const suffix = result.errors.length > 0 ? ` (${result.errors.join("; ")})` : "";
|
| 88 |
+
console.log(`${result.name}: ${result.status}${suffix}`);
|
| 89 |
+
}
|
| 90 |
+
if (!payload.passed) process.exitCode = 1;
|
| 91 |
+
await client.closeBrowser();
|
| 92 |
+
} finally {
|
| 93 |
+
await stopBrowser(browser, profileDir);
|
| 94 |
+
}
|
| 95 |
+
}
|
| 96 |
+
|
| 97 |
+
function auditUi() {
|
| 98 |
+
const byId = (id) => document.getElementById(id);
|
| 99 |
+
const required = [
|
| 100 |
+
"loadButton",
|
| 101 |
+
"micButton",
|
| 102 |
+
"stopButton",
|
| 103 |
+
"deviceSelect",
|
| 104 |
+
"runtimeStatus",
|
| 105 |
+
"runtimeDeviceStatus",
|
| 106 |
+
"runtimeDeviceDetail",
|
| 107 |
+
"runtimeBuildStatus",
|
| 108 |
+
"micValidationCard",
|
| 109 |
+
"micValidationStatus",
|
| 110 |
+
"micValidationDetail",
|
| 111 |
+
"micValidationProgressBar",
|
| 112 |
+
"gpuValidationCard",
|
| 113 |
+
"gpuValidationStatus",
|
| 114 |
+
"gpuValidationDetail",
|
| 115 |
+
"gpuValidationProgressBar",
|
| 116 |
+
"gpuBenchmarkButton",
|
| 117 |
+
"evidenceCaptureButton",
|
| 118 |
+
"partialTranscript",
|
| 119 |
+
"finalTranscript",
|
| 120 |
+
"llmOutput",
|
| 121 |
+
"benchmarkSummary",
|
| 122 |
+
"resultsBody",
|
| 123 |
+
];
|
| 124 |
+
const missing = required.filter((id) => !byId(id));
|
| 125 |
+
const runtime = {
|
| 126 |
+
state: byId("runtimeStatus")?.dataset.state ?? "",
|
| 127 |
+
label: byId("runtimeDeviceStatus")?.textContent?.trim() ?? "",
|
| 128 |
+
detail: byId("runtimeDeviceDetail")?.textContent?.trim() ?? "",
|
| 129 |
+
build: byId("runtimeBuildStatus")?.textContent?.trim() ?? "",
|
| 130 |
+
};
|
| 131 |
+
const micValidation = {
|
| 132 |
+
state: byId("micValidationCard")?.dataset.state ?? "",
|
| 133 |
+
status: byId("micValidationStatus")?.textContent?.trim() ?? "",
|
| 134 |
+
detail: byId("micValidationDetail")?.textContent?.trim() ?? "",
|
| 135 |
+
progress: byId("micValidationProgressBar")?.style.width ?? "",
|
| 136 |
+
};
|
| 137 |
+
const gpuValidation = {
|
| 138 |
+
state: byId("gpuValidationCard")?.dataset.state ?? "",
|
| 139 |
+
status: byId("gpuValidationStatus")?.textContent?.trim() ?? "",
|
| 140 |
+
detail: byId("gpuValidationDetail")?.textContent?.trim() ?? "",
|
| 141 |
+
progress: byId("gpuValidationProgressBar")?.style.width ?? "",
|
| 142 |
+
};
|
| 143 |
+
const selectors = [
|
| 144 |
+
"button",
|
| 145 |
+
"select",
|
| 146 |
+
".status-tile",
|
| 147 |
+
".runtime-status",
|
| 148 |
+
".build-status",
|
| 149 |
+
".mic-validation",
|
| 150 |
+
".badge",
|
| 151 |
+
".summary-item",
|
| 152 |
+
".settings label",
|
| 153 |
+
".metrics dl div",
|
| 154 |
+
];
|
| 155 |
+
const inspected = Array.from(document.querySelectorAll(selectors.join(","))).filter(
|
| 156 |
+
(element) => !element.closest(".table-wrap"),
|
| 157 |
+
);
|
| 158 |
+
const overflows = inspected
|
| 159 |
+
.map((element) => {
|
| 160 |
+
const rect = element.getBoundingClientRect();
|
| 161 |
+
const style = getComputedStyle(element);
|
| 162 |
+
const text = element.textContent?.replace(/\s+/g, " ").trim() ?? "";
|
| 163 |
+
const allowsHorizontalScroll = ["auto", "scroll", "hidden", "clip"].includes(style.overflowX);
|
| 164 |
+
return {
|
| 165 |
+
selector: element.id ? `#${element.id}` : element.className ? `.${String(element.className).split(/\s+/)[0]}` : element.tagName.toLowerCase(),
|
| 166 |
+
text: text.slice(0, 80),
|
| 167 |
+
left: Math.round(rect.left),
|
| 168 |
+
right: Math.round(rect.right),
|
| 169 |
+
width: Math.round(rect.width),
|
| 170 |
+
scrollWidth: element.scrollWidth,
|
| 171 |
+
clientWidth: element.clientWidth,
|
| 172 |
+
offscreen: rect.left < -1 || rect.right > window.innerWidth + 1,
|
| 173 |
+
clippedText: element.scrollWidth > element.clientWidth + 3 && !allowsHorizontalScroll,
|
| 174 |
+
};
|
| 175 |
+
})
|
| 176 |
+
.filter((item) => item.offscreen || item.clippedText);
|
| 177 |
+
const tiles = Array.from(document.querySelectorAll(".status-tile")).map((tile) => ({
|
| 178 |
+
id: tile.id,
|
| 179 |
+
state: tile.dataset.state,
|
| 180 |
+
text: tile.textContent?.replace(/\s+/g, " ").trim() ?? "",
|
| 181 |
+
}));
|
| 182 |
+
const requiredApiMethods = [
|
| 183 |
+
"state",
|
| 184 |
+
"exportResults",
|
| 185 |
+
"downloadResults",
|
| 186 |
+
"clearResults",
|
| 187 |
+
"loadStack",
|
| 188 |
+
"unload",
|
| 189 |
+
"runSuite",
|
| 190 |
+
"runEvidenceCapture",
|
| 191 |
+
"runMic",
|
| 192 |
+
"runMicSeries",
|
| 193 |
+
"runWebGpuEvidence",
|
| 194 |
+
"webgpuInfo",
|
| 195 |
+
];
|
| 196 |
+
return {
|
| 197 |
+
title: document.title,
|
| 198 |
+
viewport: {
|
| 199 |
+
width: window.innerWidth,
|
| 200 |
+
height: window.innerHeight,
|
| 201 |
+
documentWidth: document.documentElement.scrollWidth,
|
| 202 |
+
documentHeight: document.documentElement.scrollHeight,
|
| 203 |
+
},
|
| 204 |
+
automationApi: {
|
| 205 |
+
installed: Boolean(window.browserSpeakBench),
|
| 206 |
+
version: window.browserSpeakBench?.version ?? null,
|
| 207 |
+
missingMethods: requiredApiMethods.filter((method) => typeof window.browserSpeakBench?.[method] !== "function"),
|
| 208 |
+
},
|
| 209 |
+
controls: {
|
| 210 |
+
loadButtonDisabled: byId("loadButton")?.disabled ?? null,
|
| 211 |
+
micButtonDisabled: byId("micButton")?.disabled ?? null,
|
| 212 |
+
stopButtonDisabled: byId("stopButton")?.disabled ?? null,
|
| 213 |
+
gpuBenchmarkButtonDisabled: byId("gpuBenchmarkButton")?.disabled ?? null,
|
| 214 |
+
evidenceCaptureButtonDisabled: byId("evidenceCaptureButton")?.disabled ?? null,
|
| 215 |
+
deviceValue: byId("deviceSelect")?.value ?? "",
|
| 216 |
+
},
|
| 217 |
+
runtime,
|
| 218 |
+
micValidation,
|
| 219 |
+
gpuValidation,
|
| 220 |
+
tiles,
|
| 221 |
+
missing,
|
| 222 |
+
bodyHorizontalOverflow: document.documentElement.scrollWidth > window.innerWidth + 1,
|
| 223 |
+
visibleOverflow: overflows,
|
| 224 |
+
benchmarkEmptyText: byId("benchmarkSummary")?.textContent?.replace(/\s+/g, " ").trim() ?? "",
|
| 225 |
+
};
|
| 226 |
+
}
|
| 227 |
+
|
| 228 |
+
function auditErrors(audit) {
|
| 229 |
+
const errors = [];
|
| 230 |
+
if (audit.title !== "Local Voice LLM") errors.push(`unexpected title: ${audit.title}`);
|
| 231 |
+
if (!audit.automationApi.installed || audit.automationApi.version !== 1) errors.push("automation API missing");
|
| 232 |
+
if (audit.automationApi.missingMethods?.length > 0) {
|
| 233 |
+
errors.push(`automation API missing methods: ${audit.automationApi.missingMethods.join(", ")}`);
|
| 234 |
+
}
|
| 235 |
+
if (audit.missing.length > 0) errors.push(`missing elements: ${audit.missing.join(", ")}`);
|
| 236 |
+
if (audit.tiles.length !== 4) errors.push(`expected 4 status tiles, found ${audit.tiles.length}`);
|
| 237 |
+
if (audit.runtime.state === "checking" || !audit.runtime.label) errors.push("runtime status did not settle");
|
| 238 |
+
if (!audit.runtime.build || /pending/i.test(audit.runtime.build)) errors.push("runtime build status did not settle");
|
| 239 |
+
if (!/real-mic rows|models/i.test(audit.micValidation.status)) errors.push("real-mic validation status missing");
|
| 240 |
+
if (!/What app is this|Load a stack/i.test(audit.micValidation.detail)) errors.push("real-mic prompt detail missing");
|
| 241 |
+
if (!/WebGPU|adapter/i.test(audit.gpuValidation.status)) errors.push("hardware WebGPU validation status missing");
|
| 242 |
+
if (!/adapter|WebGPU|Hardware/i.test(audit.gpuValidation.detail)) errors.push("hardware WebGPU detail missing");
|
| 243 |
+
if (/Software WebGPU|No WebGPU/i.test(audit.gpuValidation.status) && audit.controls.gpuBenchmarkButtonDisabled !== true) {
|
| 244 |
+
errors.push("WebGPU evidence button should be disabled without hardware WebGPU");
|
| 245 |
+
}
|
| 246 |
+
if (audit.bodyHorizontalOverflow) errors.push("body has horizontal overflow");
|
| 247 |
+
if (audit.visibleOverflow.length > 0) {
|
| 248 |
+
errors.push(
|
| 249 |
+
`visible element overflow: ${audit.visibleOverflow
|
| 250 |
+
.slice(0, 4)
|
| 251 |
+
.map((item) => `${item.selector} ${item.text}`)
|
| 252 |
+
.join(" | ")}`,
|
| 253 |
+
);
|
| 254 |
+
}
|
| 255 |
+
if (!/No benchmark runs yet/.test(audit.benchmarkEmptyText)) errors.push("benchmark empty state missing");
|
| 256 |
+
return errors;
|
| 257 |
+
}
|
| 258 |
+
|
| 259 |
+
async function waitForUi(client) {
|
| 260 |
+
const deadline = Date.now() + 15000;
|
| 261 |
+
while (Date.now() < deadline) {
|
| 262 |
+
const ready = await client
|
| 263 |
+
.evaluate(`(() => {
|
| 264 |
+
const runtime = document.getElementById("runtimeStatus");
|
| 265 |
+
const build = document.getElementById("runtimeBuildStatus");
|
| 266 |
+
return document.readyState === "complete" &&
|
| 267 |
+
Boolean(window.browserSpeakBench?.version) &&
|
| 268 |
+
runtime &&
|
| 269 |
+
runtime.dataset.state !== "checking" &&
|
| 270 |
+
build &&
|
| 271 |
+
!/pending/i.test(build.textContent || "");
|
| 272 |
+
})()`, pollTimeoutMs)
|
| 273 |
+
.catch(() => false);
|
| 274 |
+
if (ready) return;
|
| 275 |
+
await sleep(100);
|
| 276 |
+
}
|
| 277 |
+
throw new Error("UI did not become ready.");
|
| 278 |
+
}
|
| 279 |
+
|
| 280 |
+
async function ensureServer() {
|
| 281 |
+
const response = await fetch(url).catch((error) => {
|
| 282 |
+
throw new Error(`Could not reach ${url}: ${error.message}`);
|
| 283 |
+
});
|
| 284 |
+
if (!response.ok) throw new Error(`${url} returned HTTP ${response.status}`);
|
| 285 |
+
}
|
| 286 |
+
|
| 287 |
+
function launchBrowser(port, profileDir) {
|
| 288 |
+
const child = spawn(
|
| 289 |
+
chrome,
|
| 290 |
+
[
|
| 291 |
+
...(headless ? ["--headless=new"] : []),
|
| 292 |
+
"--no-sandbox",
|
| 293 |
+
"--disable-dev-shm-usage",
|
| 294 |
+
"--disable-background-networking",
|
| 295 |
+
"--disable-extensions",
|
| 296 |
+
"--no-default-browser-check",
|
| 297 |
+
"--no-first-run",
|
| 298 |
+
"--autoplay-policy=no-user-gesture-required",
|
| 299 |
+
"--enable-unsafe-webgpu",
|
| 300 |
+
"--ignore-gpu-blocklist",
|
| 301 |
+
`--remote-debugging-port=${port}`,
|
| 302 |
+
`--user-data-dir=${profileDir}`,
|
| 303 |
+
...parseChromeArgs(),
|
| 304 |
+
],
|
| 305 |
+
{ stdio: ["ignore", "pipe", "pipe"] },
|
| 306 |
+
);
|
| 307 |
+
child.browserLog = "";
|
| 308 |
+
const appendLog = (chunk) => {
|
| 309 |
+
child.browserLog = `${child.browserLog}${chunk}`;
|
| 310 |
+
if (child.browserLog.length > 8000) child.browserLog = child.browserLog.slice(-8000);
|
| 311 |
+
};
|
| 312 |
+
child.stdout.on("data", appendLog);
|
| 313 |
+
child.stderr.on("data", appendLog);
|
| 314 |
+
return child;
|
| 315 |
+
}
|
| 316 |
+
|
| 317 |
+
async function stopBrowser(child, profileDir) {
|
| 318 |
+
if (child.exitCode == null) child.kill("SIGTERM");
|
| 319 |
+
await new Promise((resolve) => {
|
| 320 |
+
child.once("exit", resolve);
|
| 321 |
+
setTimeout(resolve, 3000);
|
| 322 |
+
});
|
| 323 |
+
if (child.exitCode == null) child.kill("SIGKILL");
|
| 324 |
+
for (let attempt = 0; attempt < 5; attempt += 1) {
|
| 325 |
+
try {
|
| 326 |
+
await rm(profileDir, { recursive: true, force: true });
|
| 327 |
+
return;
|
| 328 |
+
} catch (error) {
|
| 329 |
+
if (attempt === 4) {
|
| 330 |
+
console.warn(`Could not remove ${profileDir}: ${error.message}`);
|
| 331 |
+
return;
|
| 332 |
+
}
|
| 333 |
+
await sleep(500);
|
| 334 |
+
}
|
| 335 |
+
}
|
| 336 |
+
}
|
| 337 |
+
|
| 338 |
+
async function connectToPage(port, child) {
|
| 339 |
+
const deadline = Date.now() + 60000;
|
| 340 |
+
let lastError = null;
|
| 341 |
+
while (Date.now() < deadline) {
|
| 342 |
+
if (child.exitCode != null) {
|
| 343 |
+
throw new Error(`Chrome exited before DevTools became available.\n${child.browserLog}`);
|
| 344 |
+
}
|
| 345 |
+
try {
|
| 346 |
+
const version = await fetch(`http://127.0.0.1:${port}/json/version`).then((response) => response.json());
|
| 347 |
+
if (version.webSocketDebuggerUrl) {
|
| 348 |
+
const page = await createPageTarget(port);
|
| 349 |
+
return new CdpClient(page.webSocketDebuggerUrl);
|
| 350 |
+
}
|
| 351 |
+
} catch (error) {
|
| 352 |
+
lastError = error;
|
| 353 |
+
}
|
| 354 |
+
await sleep(250);
|
| 355 |
+
}
|
| 356 |
+
throw new Error(
|
| 357 |
+
`Could not connect to Chrome DevTools on port ${port}: ${lastError?.message ?? "unknown error"}\n${child.browserLog}`,
|
| 358 |
+
);
|
| 359 |
+
}
|
| 360 |
+
|
| 361 |
+
async function createPageTarget(port) {
|
| 362 |
+
for (const method of ["PUT", "GET"]) {
|
| 363 |
+
const response = await fetch(`http://127.0.0.1:${port}/json/new?about%3Ablank`, { method }).catch(() => null);
|
| 364 |
+
if (response?.ok) {
|
| 365 |
+
const target = await response.json();
|
| 366 |
+
if (target.webSocketDebuggerUrl) return target;
|
| 367 |
+
}
|
| 368 |
+
}
|
| 369 |
+
const targets = await fetch(`http://127.0.0.1:${port}/json`).then((response) => response.json());
|
| 370 |
+
const page = targets.find((target) => target.type === "page");
|
| 371 |
+
if (page?.webSocketDebuggerUrl) return page;
|
| 372 |
+
throw new Error("Could not create or find a page target.");
|
| 373 |
+
}
|
| 374 |
+
|
| 375 |
+
class CdpClient {
|
| 376 |
+
constructor(webSocketUrl) {
|
| 377 |
+
this.nextId = 1;
|
| 378 |
+
this.pending = new Map();
|
| 379 |
+
this.socket = new WebSocket(webSocketUrl);
|
| 380 |
+
this.opened = new Promise((resolve, reject) => {
|
| 381 |
+
this.socket.onopen = resolve;
|
| 382 |
+
this.socket.onerror = reject;
|
| 383 |
+
this.socket.onmessage = (event) => this.onMessage(event);
|
| 384 |
+
});
|
| 385 |
+
}
|
| 386 |
+
|
| 387 |
+
onMessage(event) {
|
| 388 |
+
const message = JSON.parse(event.data);
|
| 389 |
+
if (!message.id || !this.pending.has(message.id)) return;
|
| 390 |
+
const { resolve: onResolve, reject } = this.pending.get(message.id);
|
| 391 |
+
this.pending.delete(message.id);
|
| 392 |
+
if (message.error) reject(new Error(message.error.message));
|
| 393 |
+
else onResolve(message.result);
|
| 394 |
+
}
|
| 395 |
+
|
| 396 |
+
async call(method, params = {}, timeoutMs = protocolTimeoutMs) {
|
| 397 |
+
await this.opened;
|
| 398 |
+
const id = this.nextId++;
|
| 399 |
+
this.socket.send(JSON.stringify({ id, method, params }));
|
| 400 |
+
return new Promise((resolvePromise, reject) => {
|
| 401 |
+
const timer = setTimeout(() => {
|
| 402 |
+
this.pending.delete(id);
|
| 403 |
+
reject(new Error(`${method} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`));
|
| 404 |
+
}, timeoutMs);
|
| 405 |
+
this.pending.set(id, {
|
| 406 |
+
resolve: (value) => {
|
| 407 |
+
clearTimeout(timer);
|
| 408 |
+
resolvePromise(value);
|
| 409 |
+
},
|
| 410 |
+
reject: (error) => {
|
| 411 |
+
clearTimeout(timer);
|
| 412 |
+
reject(error);
|
| 413 |
+
},
|
| 414 |
+
});
|
| 415 |
+
});
|
| 416 |
+
}
|
| 417 |
+
|
| 418 |
+
async evaluate(expression, timeoutMs = protocolTimeoutMs) {
|
| 419 |
+
const result = await this.call(
|
| 420 |
+
"Runtime.evaluate",
|
| 421 |
+
{
|
| 422 |
+
expression,
|
| 423 |
+
awaitPromise: true,
|
| 424 |
+
returnByValue: true,
|
| 425 |
+
},
|
| 426 |
+
timeoutMs,
|
| 427 |
+
);
|
| 428 |
+
if (result.exceptionDetails) throw new Error(formatException(result.exceptionDetails));
|
| 429 |
+
return result.result.value;
|
| 430 |
+
}
|
| 431 |
+
|
| 432 |
+
async closeBrowser() {
|
| 433 |
+
await this.call("Browser.close", {}, 5000).catch(() => {});
|
| 434 |
+
try {
|
| 435 |
+
this.socket.close();
|
| 436 |
+
} catch {
|
| 437 |
+
// Ignore close races.
|
| 438 |
+
}
|
| 439 |
+
}
|
| 440 |
+
}
|
| 441 |
+
|
| 442 |
+
function formatException(exceptionDetails) {
|
| 443 |
+
const exception = exceptionDetails.exception;
|
| 444 |
+
return exception?.description ?? exception?.value ?? exceptionDetails.text ?? "Evaluation failed.";
|
| 445 |
+
}
|
| 446 |
+
|
| 447 |
+
function parseViewports(value) {
|
| 448 |
+
return String(value ?? "")
|
| 449 |
+
.split(",")
|
| 450 |
+
.map((entry) => {
|
| 451 |
+
const [namePart, sizePart] = entry.split(":");
|
| 452 |
+
const [width, height] = String(sizePart ?? "").split("x").map((part) => Number(part));
|
| 453 |
+
if (!namePart || !Number.isFinite(width) || !Number.isFinite(height)) {
|
| 454 |
+
throw new Error(`Invalid viewport entry: ${entry}`);
|
| 455 |
+
}
|
| 456 |
+
return {
|
| 457 |
+
name: namePart.trim(),
|
| 458 |
+
width,
|
| 459 |
+
height,
|
| 460 |
+
mobile: width < 600,
|
| 461 |
+
deviceScaleFactor: width < 600 ? 2 : 1,
|
| 462 |
+
};
|
| 463 |
+
});
|
| 464 |
+
}
|
| 465 |
+
|
| 466 |
+
function parseChromeArgs() {
|
| 467 |
+
const raw = process.env.BROWSER_SPEAK_CHROME_ARGS ?? "";
|
| 468 |
+
return (
|
| 469 |
+
raw
|
| 470 |
+
.match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g)
|
| 471 |
+
?.map((arg) => arg.replace(/^["']|["']$/g, "")) ?? []
|
| 472 |
+
);
|
| 473 |
+
}
|
| 474 |
+
|
| 475 |
+
function sleep(ms) {
|
| 476 |
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
| 477 |
+
}
|
| 478 |
+
|
| 479 |
+
main().catch((error) => {
|
| 480 |
+
console.error(error.stack ?? error.message);
|
| 481 |
+
process.exitCode = 1;
|
| 482 |
+
});
|
tools/run-webgpu-benchmark.mjs
ADDED
|
@@ -0,0 +1,491 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env node
|
| 2 |
+
import { spawn } from "node:child_process";
|
| 3 |
+
import { existsSync } from "node:fs";
|
| 4 |
+
import { mkdir, rm, writeFile } from "node:fs/promises";
|
| 5 |
+
import { tmpdir } from "node:os";
|
| 6 |
+
import { dirname, resolve } from "node:path";
|
| 7 |
+
import { sourceFingerprint } from "./source-fingerprint.mjs";
|
| 8 |
+
|
| 9 |
+
const url = process.env.BROWSER_SPEAK_URL ?? "http://127.0.0.1:5174/";
|
| 10 |
+
const chrome =
|
| 11 |
+
process.env.CHROME_BIN ??
|
| 12 |
+
(existsSync("/opt/google/chrome/chrome")
|
| 13 |
+
? "/opt/google/chrome/chrome"
|
| 14 |
+
: existsSync("/usr/bin/google-chrome")
|
| 15 |
+
? "/usr/bin/google-chrome"
|
| 16 |
+
: "chromium");
|
| 17 |
+
const resultPath = resolve(process.env.BROWSER_SPEAK_WEBGPU_JSON ?? `${tmpdir()}/browser-speak-webgpu-results.json`);
|
| 18 |
+
const profileDir = resolve(process.env.BROWSER_SPEAK_WEBGPU_PROFILE_DIR ?? `${tmpdir()}/browser-speak-webgpu-profile`);
|
| 19 |
+
const llmCandidates = parseList(
|
| 20 |
+
process.env.BROWSER_SPEAK_WEBGPU_LLMS ??
|
| 21 |
+
"HuggingFaceTB/SmolLM2-135M-Instruct,onnx-community/granite-4.0-350m-ONNX-web,onnx-community/Qwen3-0.6B-ONNX",
|
| 22 |
+
);
|
| 23 |
+
const stackBase = {
|
| 24 |
+
device: "webgpu",
|
| 25 |
+
asr: process.env.BROWSER_SPEAK_ASR ?? "onnx-community/moonshine-base-ONNX",
|
| 26 |
+
voice: process.env.BROWSER_SPEAK_VOICE ?? "F2",
|
| 27 |
+
ttsSteps: Number(process.env.BROWSER_SPEAK_TTS_STEPS ?? 2),
|
| 28 |
+
vadSilenceMs: Number(process.env.BROWSER_SPEAK_VAD_SILENCE_MS ?? 480),
|
| 29 |
+
partialAsr: process.env.BROWSER_SPEAK_PARTIAL_ASR !== "false",
|
| 30 |
+
};
|
| 31 |
+
const loadTimeoutMs = Number(process.env.BROWSER_SPEAK_LOAD_TIMEOUT_MS ?? 600000);
|
| 32 |
+
const suiteTimeoutMs = Number(process.env.BROWSER_SPEAK_SUITE_TIMEOUT_MS ?? 900000);
|
| 33 |
+
const headless = process.env.BROWSER_SPEAK_HEADLESS !== "false";
|
| 34 |
+
const allowSoftwareWebgpu = process.env.BROWSER_SPEAK_ALLOW_SOFTWARE_WEBGPU === "true";
|
| 35 |
+
const protocolTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_TIMEOUT_MS ?? 60000);
|
| 36 |
+
const pollTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_POLL_TIMEOUT_MS ?? 5000);
|
| 37 |
+
|
| 38 |
+
async function main() {
|
| 39 |
+
await ensureServer();
|
| 40 |
+
await mkdir(dirname(resultPath), { recursive: true });
|
| 41 |
+
await rm(profileDir, { recursive: true, force: true });
|
| 42 |
+
|
| 43 |
+
const browser = launchBrowser(9343, profileDir);
|
| 44 |
+
const candidates = llmCandidates.map((llm) => ({ llm, status: "pending" }));
|
| 45 |
+
try {
|
| 46 |
+
const client = await connectToPage(9343, browser);
|
| 47 |
+
await waitForBenchApi(client);
|
| 48 |
+
const webgpu = await runPageTask(client, "window.browserSpeakBench.webgpuInfo()", {
|
| 49 |
+
label: "WebGPU probe",
|
| 50 |
+
timeoutMs: 30000,
|
| 51 |
+
});
|
| 52 |
+
|
| 53 |
+
const skipReason = webgpuSkipReason(webgpu);
|
| 54 |
+
if (skipReason) {
|
| 55 |
+
for (const candidate of candidates) {
|
| 56 |
+
candidate.status = "skipped";
|
| 57 |
+
candidate.reason = skipReason;
|
| 58 |
+
}
|
| 59 |
+
await writePayload({
|
| 60 |
+
skipped: true,
|
| 61 |
+
reason: skipReason,
|
| 62 |
+
webgpu,
|
| 63 |
+
config: benchmarkConfig(),
|
| 64 |
+
candidates,
|
| 65 |
+
results: [],
|
| 66 |
+
summary: null,
|
| 67 |
+
});
|
| 68 |
+
console.log(`Wrote WebGPU benchmark JSON: ${resultPath}`);
|
| 69 |
+
console.log(`Skipped WebGPU benchmark: ${skipReason}.`);
|
| 70 |
+
await client.closeBrowser();
|
| 71 |
+
return;
|
| 72 |
+
}
|
| 73 |
+
|
| 74 |
+
await runPageTask(client, "window.browserSpeakBench.clearResults()", { label: "clear results" });
|
| 75 |
+
console.log(`WebGPU adapter: ${formatAdapter(webgpu.adapter)}`);
|
| 76 |
+
|
| 77 |
+
for (const candidate of candidates) {
|
| 78 |
+
const stack = { ...stackBase, llm: candidate.llm };
|
| 79 |
+
const label = shortModelName(candidate.llm);
|
| 80 |
+
console.log(`Running WebGPU benchmark candidate: ${candidate.llm}`);
|
| 81 |
+
const startedAt = Date.now();
|
| 82 |
+
try {
|
| 83 |
+
const before = await runPageTask(client, "window.browserSpeakBench.state()", {
|
| 84 |
+
label: `${label} preflight`,
|
| 85 |
+
});
|
| 86 |
+
await runPageTask(client, `window.browserSpeakBench.loadStack(${JSON.stringify(stack)})`, {
|
| 87 |
+
label: `${label} model load`,
|
| 88 |
+
timeoutMs: loadTimeoutMs,
|
| 89 |
+
});
|
| 90 |
+
const snapshot = await runPageTask(client, "window.browserSpeakBench.runSuite()", {
|
| 91 |
+
label: `${label} suite`,
|
| 92 |
+
timeoutMs: suiteTimeoutMs,
|
| 93 |
+
});
|
| 94 |
+
const addedCount = Math.max(0, snapshot.results.length - before.results.length);
|
| 95 |
+
const rows = snapshot.results.slice(0, addedCount);
|
| 96 |
+
const errors = rows.filter((row) => row.error);
|
| 97 |
+
candidate.status = rows.length >= 5 && errors.length === 0 ? "complete" : "partial";
|
| 98 |
+
candidate.durationMs = Date.now() - startedAt;
|
| 99 |
+
candidate.rowCount = rows.length;
|
| 100 |
+
candidate.errorCount = errors.length;
|
| 101 |
+
candidate.summary = snapshot.summary.current ?? snapshot.summary.all;
|
| 102 |
+
candidate.stack = snapshot.stack;
|
| 103 |
+
if (errors.length > 0) candidate.errors = errors.map((row) => `${row.kind}: ${row.error}`);
|
| 104 |
+
} catch (error) {
|
| 105 |
+
candidate.status = "failed";
|
| 106 |
+
candidate.durationMs = Date.now() - startedAt;
|
| 107 |
+
candidate.error = error.message;
|
| 108 |
+
console.error(`${candidate.llm} failed: ${error.message}`);
|
| 109 |
+
} finally {
|
| 110 |
+
await runPageTask(client, "window.browserSpeakBench.stop()", {
|
| 111 |
+
label: `${label} stop`,
|
| 112 |
+
timeoutMs: 15000,
|
| 113 |
+
}).catch(() => {});
|
| 114 |
+
const state = await runPageTask(client, "window.browserSpeakBench.state()", {
|
| 115 |
+
label: `${label} state`,
|
| 116 |
+
timeoutMs: 15000,
|
| 117 |
+
}).catch(() => null);
|
| 118 |
+
if (state?.modelsLoaded || state?.modelsLoading) {
|
| 119 |
+
await runPageTask(client, "window.browserSpeakBench.unload()", {
|
| 120 |
+
label: `${label} unload`,
|
| 121 |
+
timeoutMs: 60000,
|
| 122 |
+
}).catch((error) => {
|
| 123 |
+
candidate.unloadError = error.message;
|
| 124 |
+
});
|
| 125 |
+
}
|
| 126 |
+
await writePayload(await exportPayload(client, webgpu, candidates));
|
| 127 |
+
}
|
| 128 |
+
}
|
| 129 |
+
|
| 130 |
+
const payload = await exportPayload(client, webgpu, candidates);
|
| 131 |
+
await writePayload(payload);
|
| 132 |
+
console.log(`Wrote WebGPU benchmark JSON: ${resultPath}`);
|
| 133 |
+
summarizeResults(payload);
|
| 134 |
+
await client.closeBrowser();
|
| 135 |
+
} finally {
|
| 136 |
+
await stopBrowser(browser, profileDir);
|
| 137 |
+
}
|
| 138 |
+
}
|
| 139 |
+
|
| 140 |
+
async function exportPayload(client, webgpu, candidates) {
|
| 141 |
+
const exportResult = await runPageTask(client, "window.browserSpeakBench.exportResults()", {
|
| 142 |
+
label: "export results",
|
| 143 |
+
timeoutMs: 30000,
|
| 144 |
+
}).catch(() => ({ summary: null, results: [] }));
|
| 145 |
+
return {
|
| 146 |
+
generatedAt: new Date().toISOString(),
|
| 147 |
+
sourceFingerprint: await sourceFingerprint(),
|
| 148 |
+
url,
|
| 149 |
+
webgpu,
|
| 150 |
+
config: benchmarkConfig(),
|
| 151 |
+
candidates,
|
| 152 |
+
summary: exportResult.summary,
|
| 153 |
+
results: exportResult.results,
|
| 154 |
+
};
|
| 155 |
+
}
|
| 156 |
+
|
| 157 |
+
function benchmarkConfig() {
|
| 158 |
+
return {
|
| 159 |
+
stackBase,
|
| 160 |
+
loadTimeoutMs,
|
| 161 |
+
suiteTimeoutMs,
|
| 162 |
+
headless,
|
| 163 |
+
allowSoftwareWebgpu,
|
| 164 |
+
protocolTimeoutMs,
|
| 165 |
+
pollTimeoutMs,
|
| 166 |
+
chrome,
|
| 167 |
+
extraChromeArgs: parseChromeArgs(),
|
| 168 |
+
};
|
| 169 |
+
}
|
| 170 |
+
|
| 171 |
+
async function writePayload(payload) {
|
| 172 |
+
await writeFile(
|
| 173 |
+
resultPath,
|
| 174 |
+
`${JSON.stringify(
|
| 175 |
+
{
|
| 176 |
+
generatedAt: new Date().toISOString(),
|
| 177 |
+
sourceFingerprint: await sourceFingerprint(),
|
| 178 |
+
url,
|
| 179 |
+
...payload,
|
| 180 |
+
},
|
| 181 |
+
null,
|
| 182 |
+
2,
|
| 183 |
+
)}\n`,
|
| 184 |
+
);
|
| 185 |
+
}
|
| 186 |
+
|
| 187 |
+
async function ensureServer() {
|
| 188 |
+
const response = await fetch(url).catch((error) => {
|
| 189 |
+
throw new Error(`Could not reach ${url}: ${error.message}`);
|
| 190 |
+
});
|
| 191 |
+
if (!response.ok) throw new Error(`${url} returned HTTP ${response.status}`);
|
| 192 |
+
}
|
| 193 |
+
|
| 194 |
+
async function waitForBenchApi(client) {
|
| 195 |
+
const deadline = Date.now() + 15000;
|
| 196 |
+
while (Date.now() < deadline) {
|
| 197 |
+
try {
|
| 198 |
+
if (await client.evaluate("Boolean(window.browserSpeakBench?.webgpuInfo)")) return;
|
| 199 |
+
} catch {
|
| 200 |
+
// The target may still be navigating and can destroy the execution context.
|
| 201 |
+
}
|
| 202 |
+
await sleep(100);
|
| 203 |
+
}
|
| 204 |
+
throw new Error("window.browserSpeakBench.webgpuInfo was not installed.");
|
| 205 |
+
}
|
| 206 |
+
|
| 207 |
+
async function runPageTask(client, expression, { label = "page task", timeoutMs = 30000 } = {}) {
|
| 208 |
+
const taskId = `task_${Date.now()}_${Math.random().toString(16).slice(2)}`;
|
| 209 |
+
await client.evaluate(`(() => {
|
| 210 |
+
const taskId = ${JSON.stringify(taskId)};
|
| 211 |
+
window.__browserSpeakHarnessTasks ||= {};
|
| 212 |
+
window.__browserSpeakHarnessTasks[taskId] = { done: false, label: ${JSON.stringify(label)} };
|
| 213 |
+
Promise.resolve(${expression})
|
| 214 |
+
.then((value) => {
|
| 215 |
+
window.__browserSpeakHarnessTasks[taskId] = { done: true, value };
|
| 216 |
+
})
|
| 217 |
+
.catch((error) => {
|
| 218 |
+
window.__browserSpeakHarnessTasks[taskId] = {
|
| 219 |
+
done: true,
|
| 220 |
+
error: error?.stack || error?.message || String(error),
|
| 221 |
+
};
|
| 222 |
+
});
|
| 223 |
+
return true;
|
| 224 |
+
})()`);
|
| 225 |
+
const deadline = Date.now() + timeoutMs;
|
| 226 |
+
let lastEvents = [];
|
| 227 |
+
let lastPollError = "";
|
| 228 |
+
while (Date.now() < deadline) {
|
| 229 |
+
let task = null;
|
| 230 |
+
try {
|
| 231 |
+
task = await client.evaluate(
|
| 232 |
+
`window.__browserSpeakHarnessTasks?.[${JSON.stringify(taskId)}] ?? null`,
|
| 233 |
+
pollTimeoutMs,
|
| 234 |
+
);
|
| 235 |
+
lastPollError = "";
|
| 236 |
+
} catch (error) {
|
| 237 |
+
const message = error.message ?? String(error);
|
| 238 |
+
if (message !== lastPollError) {
|
| 239 |
+
console.log(`${label}: waiting for page response (${message})`);
|
| 240 |
+
lastPollError = message;
|
| 241 |
+
}
|
| 242 |
+
await sleep(500);
|
| 243 |
+
continue;
|
| 244 |
+
}
|
| 245 |
+
if (task?.done) {
|
| 246 |
+
await client
|
| 247 |
+
.evaluate(`delete window.__browserSpeakHarnessTasks?.[${JSON.stringify(taskId)}]`, pollTimeoutMs)
|
| 248 |
+
.catch(() => {});
|
| 249 |
+
if (task.error) throw new Error(`${label} failed: ${task.error}`);
|
| 250 |
+
return task.value;
|
| 251 |
+
}
|
| 252 |
+
const snapshot = await client
|
| 253 |
+
.evaluate(`(() => {
|
| 254 |
+
const state = window.browserSpeakBench?.state?.();
|
| 255 |
+
return state ? {
|
| 256 |
+
modelsLoaded: state.modelsLoaded,
|
| 257 |
+
modelsLoading: state.modelsLoading,
|
| 258 |
+
activeBenchmark: state.activeBenchmark?.kind ?? null,
|
| 259 |
+
suiteRunning: state.suiteRunning,
|
| 260 |
+
events: state.events?.slice(0, 3) ?? [],
|
| 261 |
+
} : null;
|
| 262 |
+
})()`, pollTimeoutMs)
|
| 263 |
+
.catch(() => null);
|
| 264 |
+
const events = snapshot?.events ?? [];
|
| 265 |
+
if (events.join("\\n") !== lastEvents.join("\\n")) {
|
| 266 |
+
lastEvents = events;
|
| 267 |
+
if (events[0]) console.log(`${label}: ${events[0]}`);
|
| 268 |
+
}
|
| 269 |
+
await sleep(500);
|
| 270 |
+
}
|
| 271 |
+
throw new Error(`${label} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`);
|
| 272 |
+
}
|
| 273 |
+
|
| 274 |
+
function launchBrowser(port, profileDir) {
|
| 275 |
+
const child = spawn(
|
| 276 |
+
chrome,
|
| 277 |
+
[
|
| 278 |
+
...(headless ? ["--headless=new"] : []),
|
| 279 |
+
"--no-sandbox",
|
| 280 |
+
"--disable-dev-shm-usage",
|
| 281 |
+
"--disable-background-networking",
|
| 282 |
+
"--disable-extensions",
|
| 283 |
+
"--no-default-browser-check",
|
| 284 |
+
"--no-first-run",
|
| 285 |
+
"--autoplay-policy=no-user-gesture-required",
|
| 286 |
+
"--enable-unsafe-webgpu",
|
| 287 |
+
"--ignore-gpu-blocklist",
|
| 288 |
+
`--remote-debugging-port=${port}`,
|
| 289 |
+
`--user-data-dir=${profileDir}`,
|
| 290 |
+
...parseChromeArgs(),
|
| 291 |
+
],
|
| 292 |
+
{ stdio: ["ignore", "pipe", "pipe"] },
|
| 293 |
+
);
|
| 294 |
+
child.browserLog = "";
|
| 295 |
+
const appendLog = (chunk) => {
|
| 296 |
+
child.browserLog = `${child.browserLog}${chunk}`;
|
| 297 |
+
if (child.browserLog.length > 8000) child.browserLog = child.browserLog.slice(-8000);
|
| 298 |
+
};
|
| 299 |
+
child.stdout.on("data", appendLog);
|
| 300 |
+
child.stderr.on("data", appendLog);
|
| 301 |
+
return child;
|
| 302 |
+
}
|
| 303 |
+
|
| 304 |
+
async function stopBrowser(child, profileDir) {
|
| 305 |
+
if (child.exitCode == null) child.kill("SIGTERM");
|
| 306 |
+
await new Promise((resolve) => {
|
| 307 |
+
child.once("exit", resolve);
|
| 308 |
+
setTimeout(resolve, 3000);
|
| 309 |
+
});
|
| 310 |
+
if (child.exitCode == null) child.kill("SIGKILL");
|
| 311 |
+
for (let attempt = 0; attempt < 5; attempt += 1) {
|
| 312 |
+
try {
|
| 313 |
+
await rm(profileDir, { recursive: true, force: true });
|
| 314 |
+
return;
|
| 315 |
+
} catch (error) {
|
| 316 |
+
if (attempt === 4) {
|
| 317 |
+
console.warn(`Could not remove ${profileDir}: ${error.message}`);
|
| 318 |
+
return;
|
| 319 |
+
}
|
| 320 |
+
await sleep(500);
|
| 321 |
+
}
|
| 322 |
+
}
|
| 323 |
+
}
|
| 324 |
+
|
| 325 |
+
async function connectToPage(port, child) {
|
| 326 |
+
const deadline = Date.now() + 60000;
|
| 327 |
+
let lastError = null;
|
| 328 |
+
while (Date.now() < deadline) {
|
| 329 |
+
if (child.exitCode != null) {
|
| 330 |
+
throw new Error(`Chrome exited before DevTools became available.\n${child.browserLog}`);
|
| 331 |
+
}
|
| 332 |
+
try {
|
| 333 |
+
const version = await fetch(`http://127.0.0.1:${port}/json/version`).then((response) => response.json());
|
| 334 |
+
if (version.webSocketDebuggerUrl) {
|
| 335 |
+
const page = await createPageTarget(port);
|
| 336 |
+
return new CdpClient(page.webSocketDebuggerUrl);
|
| 337 |
+
}
|
| 338 |
+
} catch (error) {
|
| 339 |
+
lastError = error;
|
| 340 |
+
}
|
| 341 |
+
await sleep(250);
|
| 342 |
+
}
|
| 343 |
+
throw new Error(
|
| 344 |
+
`Could not connect to Chrome DevTools on port ${port}: ${lastError?.message ?? "unknown error"}\n${child.browserLog}`,
|
| 345 |
+
);
|
| 346 |
+
}
|
| 347 |
+
|
| 348 |
+
async function createPageTarget(port) {
|
| 349 |
+
for (const method of ["PUT", "GET"]) {
|
| 350 |
+
const response = await fetch(`http://127.0.0.1:${port}/json/new?${encodeURIComponent(url)}`, {
|
| 351 |
+
method,
|
| 352 |
+
}).catch(() => null);
|
| 353 |
+
if (response?.ok) {
|
| 354 |
+
const target = await response.json();
|
| 355 |
+
if (target.webSocketDebuggerUrl) return target;
|
| 356 |
+
}
|
| 357 |
+
}
|
| 358 |
+
const targets = await fetch(`http://127.0.0.1:${port}/json`).then((response) => response.json());
|
| 359 |
+
const page = targets.find((target) => target.type === "page" && target.url === url);
|
| 360 |
+
if (page?.webSocketDebuggerUrl) return page;
|
| 361 |
+
throw new Error("Could not create or find a page target.");
|
| 362 |
+
}
|
| 363 |
+
|
| 364 |
+
class CdpClient {
|
| 365 |
+
constructor(webSocketUrl) {
|
| 366 |
+
this.nextId = 1;
|
| 367 |
+
this.pending = new Map();
|
| 368 |
+
this.socket = new WebSocket(webSocketUrl);
|
| 369 |
+
this.opened = new Promise((resolve, reject) => {
|
| 370 |
+
this.socket.onopen = resolve;
|
| 371 |
+
this.socket.onerror = reject;
|
| 372 |
+
this.socket.onmessage = (event) => this.onMessage(event);
|
| 373 |
+
});
|
| 374 |
+
}
|
| 375 |
+
|
| 376 |
+
onMessage(event) {
|
| 377 |
+
const message = JSON.parse(event.data);
|
| 378 |
+
if (!message.id || !this.pending.has(message.id)) return;
|
| 379 |
+
const { resolve: onResolve, reject } = this.pending.get(message.id);
|
| 380 |
+
this.pending.delete(message.id);
|
| 381 |
+
if (message.error) reject(new Error(message.error.message));
|
| 382 |
+
else onResolve(message.result);
|
| 383 |
+
}
|
| 384 |
+
|
| 385 |
+
async call(method, params = {}, timeoutMs = protocolTimeoutMs) {
|
| 386 |
+
await this.opened;
|
| 387 |
+
const id = this.nextId++;
|
| 388 |
+
this.socket.send(JSON.stringify({ id, method, params }));
|
| 389 |
+
return new Promise((resolvePromise, reject) => {
|
| 390 |
+
const timer = setTimeout(() => {
|
| 391 |
+
this.pending.delete(id);
|
| 392 |
+
reject(new Error(`${method} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`));
|
| 393 |
+
}, timeoutMs);
|
| 394 |
+
this.pending.set(id, {
|
| 395 |
+
resolve: (value) => {
|
| 396 |
+
clearTimeout(timer);
|
| 397 |
+
resolvePromise(value);
|
| 398 |
+
},
|
| 399 |
+
reject: (error) => {
|
| 400 |
+
clearTimeout(timer);
|
| 401 |
+
reject(error);
|
| 402 |
+
},
|
| 403 |
+
});
|
| 404 |
+
});
|
| 405 |
+
}
|
| 406 |
+
|
| 407 |
+
async evaluate(expression, timeoutMs = protocolTimeoutMs) {
|
| 408 |
+
const result = await this.call("Runtime.evaluate", {
|
| 409 |
+
expression,
|
| 410 |
+
returnByValue: true,
|
| 411 |
+
}, timeoutMs);
|
| 412 |
+
if (result.exceptionDetails) throw new Error(formatException(result.exceptionDetails));
|
| 413 |
+
return result.result.value;
|
| 414 |
+
}
|
| 415 |
+
|
| 416 |
+
closeBrowser() {
|
| 417 |
+
try {
|
| 418 |
+
this.socket.send(JSON.stringify({ id: this.nextId++, method: "Browser.close", params: {} }));
|
| 419 |
+
} catch {
|
| 420 |
+
// The surrounding process cleanup handles already-closed targets.
|
| 421 |
+
}
|
| 422 |
+
try {
|
| 423 |
+
this.socket.close();
|
| 424 |
+
} catch {
|
| 425 |
+
// Ignore close races.
|
| 426 |
+
}
|
| 427 |
+
}
|
| 428 |
+
}
|
| 429 |
+
|
| 430 |
+
function formatException(exceptionDetails) {
|
| 431 |
+
const exception = exceptionDetails.exception;
|
| 432 |
+
return exception?.description ?? exception?.value ?? exceptionDetails.text ?? "Evaluation failed.";
|
| 433 |
+
}
|
| 434 |
+
|
| 435 |
+
function parseList(value) {
|
| 436 |
+
return value
|
| 437 |
+
.split(",")
|
| 438 |
+
.map((item) => item.trim())
|
| 439 |
+
.filter(Boolean);
|
| 440 |
+
}
|
| 441 |
+
|
| 442 |
+
function parseChromeArgs() {
|
| 443 |
+
const raw = process.env.BROWSER_SPEAK_CHROME_ARGS ?? "";
|
| 444 |
+
return (
|
| 445 |
+
raw
|
| 446 |
+
.match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g)
|
| 447 |
+
?.map((arg) => arg.replace(/^["']|["']$/g, "")) ?? []
|
| 448 |
+
);
|
| 449 |
+
}
|
| 450 |
+
|
| 451 |
+
function shortModelName(model) {
|
| 452 |
+
return model.split("/").at(-1) ?? model;
|
| 453 |
+
}
|
| 454 |
+
|
| 455 |
+
function formatAdapter(adapter) {
|
| 456 |
+
if (!adapter) return "unknown";
|
| 457 |
+
return [adapter.vendor, adapter.architecture, adapter.device, adapter.description].filter(Boolean).join(" / ") || "unknown";
|
| 458 |
+
}
|
| 459 |
+
|
| 460 |
+
function webgpuSkipReason(webgpu) {
|
| 461 |
+
if (!webgpu?.available) return "WebGPU unavailable";
|
| 462 |
+
if (allowSoftwareWebgpu) return "";
|
| 463 |
+
if (webgpu.softwareAdapter || isSoftwareAdapter(webgpu.adapter)) {
|
| 464 |
+
return `Software WebGPU adapter exposed (${formatAdapter(webgpu.adapter)})`;
|
| 465 |
+
}
|
| 466 |
+
return "";
|
| 467 |
+
}
|
| 468 |
+
|
| 469 |
+
function isSoftwareAdapter(adapter) {
|
| 470 |
+
const text = [adapter?.vendor, adapter?.architecture, adapter?.device, adapter?.description]
|
| 471 |
+
.filter(Boolean)
|
| 472 |
+
.join(" ")
|
| 473 |
+
.toLowerCase();
|
| 474 |
+
return /\b(swiftshader|llvmpipe|software rasterizer|software adapter|warp)\b/.test(text);
|
| 475 |
+
}
|
| 476 |
+
|
| 477 |
+
function summarizeResults(payload) {
|
| 478 |
+
for (const candidate of payload.candidates) {
|
| 479 |
+
const suffix = candidate.error ? ` (${candidate.error})` : "";
|
| 480 |
+
console.log(`${candidate.llm}: ${candidate.status}, ${candidate.rowCount ?? 0} rows${suffix}`);
|
| 481 |
+
}
|
| 482 |
+
const byStack = payload.summary?.byStack ?? {};
|
| 483 |
+
const stackCount = Object.keys(byStack).length;
|
| 484 |
+
console.log(`Exported ${payload.results.length} rows across ${stackCount} stack summaries.`);
|
| 485 |
+
}
|
| 486 |
+
|
| 487 |
+
function sleep(ms) {
|
| 488 |
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
| 489 |
+
}
|
| 490 |
+
|
| 491 |
+
await main();
|
tools/source-fingerprint.mjs
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { createHash } from "node:crypto";
|
| 2 |
+
import { readFile } from "node:fs/promises";
|
| 3 |
+
import { dirname, resolve } from "node:path";
|
| 4 |
+
import { fileURLToPath } from "node:url";
|
| 5 |
+
|
| 6 |
+
const projectRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
| 7 |
+
const SOURCE_FILES = [
|
| 8 |
+
"index.html",
|
| 9 |
+
"styles.css",
|
| 10 |
+
"app.js",
|
| 11 |
+
"workers/asr-worker.js",
|
| 12 |
+
"workers/llm-worker.js",
|
| 13 |
+
"workers/tts-worker.js",
|
| 14 |
+
"tools/run-ui-smoke.mjs",
|
| 15 |
+
"tools/run-evidence-export-smoke.mjs",
|
| 16 |
+
"tools/run-client-side-smoke.mjs",
|
| 17 |
+
"tools/run-hosted-smoke.mjs",
|
| 18 |
+
"tools/run-loopback-series.mjs",
|
| 19 |
+
"tools/run-fake-mic-benchmark.mjs",
|
| 20 |
+
"tools/run-local-candidate-benchmark.mjs",
|
| 21 |
+
"tools/run-real-mic-series.mjs",
|
| 22 |
+
"tools/run-hosted-evidence-capture.mjs",
|
| 23 |
+
"tools/run-webgpu-benchmark.mjs",
|
| 24 |
+
"tools/audit-browser-evidence.mjs",
|
| 25 |
+
"tools/audit-validation.mjs",
|
| 26 |
+
"tools/summarize-evidence.mjs",
|
| 27 |
+
"tools/source-fingerprint.mjs",
|
| 28 |
+
];
|
| 29 |
+
|
| 30 |
+
export async function sourceFingerprint() {
|
| 31 |
+
const fileFingerprints = [];
|
| 32 |
+
const aggregate = createHash("sha256");
|
| 33 |
+
for (const path of SOURCE_FILES) {
|
| 34 |
+
const bytes = await readFile(resolve(projectRoot, path));
|
| 35 |
+
const hash = createHash("sha256").update(bytes).digest("hex");
|
| 36 |
+
fileFingerprints.push({ path, sha256: hash, bytes: bytes.length });
|
| 37 |
+
aggregate.update(`${path}\0${hash}\0`);
|
| 38 |
+
}
|
| 39 |
+
return {
|
| 40 |
+
algorithm: "sha256",
|
| 41 |
+
hash: aggregate.digest("hex"),
|
| 42 |
+
files: fileFingerprints,
|
| 43 |
+
};
|
| 44 |
+
}
|
tools/summarize-evidence.mjs
ADDED
|
@@ -0,0 +1,311 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env node
|
| 2 |
+
import { readFile, writeFile, mkdir } from "node:fs/promises";
|
| 3 |
+
import { existsSync } from "node:fs";
|
| 4 |
+
import { tmpdir } from "node:os";
|
| 5 |
+
import { dirname, resolve } from "node:path";
|
| 6 |
+
import { sourceFingerprint } from "./source-fingerprint.mjs";
|
| 7 |
+
|
| 8 |
+
const paths = {
|
| 9 |
+
finalValidation: resolve(
|
| 10 |
+
process.env.BROWSER_SPEAK_FINAL_JSON ?? `${tmpdir()}/browser-speak-final-validation.json`,
|
| 11 |
+
),
|
| 12 |
+
validationAudit: resolve(
|
| 13 |
+
process.env.BROWSER_SPEAK_AUDIT_JSON ?? `${tmpdir()}/browser-speak-validation-audit.json`,
|
| 14 |
+
),
|
| 15 |
+
hostedSmoke: resolve(
|
| 16 |
+
process.env.BROWSER_SPEAK_HOSTED_SMOKE_JSON ?? `${tmpdir()}/browser-speak-hosted-smoke.json`,
|
| 17 |
+
),
|
| 18 |
+
browserEvidenceAudit: resolve(
|
| 19 |
+
process.env.BROWSER_SPEAK_BROWSER_AUDIT_JSON ?? `${tmpdir()}/browser-speak-browser-evidence-audit.json`,
|
| 20 |
+
),
|
| 21 |
+
realMic: resolve(process.env.BROWSER_SPEAK_REAL_MIC_JSON ?? `${tmpdir()}/browser-speak-real-mic-series.json`),
|
| 22 |
+
webgpu: resolve(process.env.BROWSER_SPEAK_WEBGPU_JSON ?? `${tmpdir()}/browser-speak-webgpu-results.json`),
|
| 23 |
+
};
|
| 24 |
+
const resultPath = resolve(
|
| 25 |
+
process.env.BROWSER_SPEAK_EVIDENCE_SUMMARY_JSON ?? `${tmpdir()}/browser-speak-evidence-summary.json`,
|
| 26 |
+
);
|
| 27 |
+
|
| 28 |
+
async function main() {
|
| 29 |
+
const currentSourceFingerprint = await sourceFingerprint();
|
| 30 |
+
const artifacts = Object.fromEntries(
|
| 31 |
+
await Promise.all(Object.entries(paths).map(async ([key, path]) => [key, await readArtifact(path, currentSourceFingerprint)])),
|
| 32 |
+
);
|
| 33 |
+
const checks = [
|
| 34 |
+
...validationChecks(artifacts.validationAudit.value ?? artifacts.finalValidation.value?.audit),
|
| 35 |
+
hostedSmokeCheck(artifacts.hostedSmoke.value),
|
| 36 |
+
browserEvidenceCheck(artifacts.browserEvidenceAudit.value),
|
| 37 |
+
realMicArtifactCheck(artifacts.realMic.value),
|
| 38 |
+
webgpuArtifactCheck(artifacts.webgpu.value),
|
| 39 |
+
freshnessCheck(artifacts),
|
| 40 |
+
];
|
| 41 |
+
const requiredChecks = dedupeChecks(checks).filter((check) => check.required);
|
| 42 |
+
const passed = requiredChecks.every((check) => check.status === "pass");
|
| 43 |
+
const payload = {
|
| 44 |
+
generatedAt: new Date().toISOString(),
|
| 45 |
+
sourceFingerprint: currentSourceFingerprint,
|
| 46 |
+
passed,
|
| 47 |
+
artifacts: Object.fromEntries(Object.entries(artifacts).map(([key, artifact]) => [key, summarizeArtifact(artifact)])),
|
| 48 |
+
checks: requiredChecks,
|
| 49 |
+
nextActions: nextActions(requiredChecks),
|
| 50 |
+
};
|
| 51 |
+
await mkdir(dirname(resultPath), { recursive: true });
|
| 52 |
+
await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`);
|
| 53 |
+
console.log(`Wrote evidence summary JSON: ${resultPath}`);
|
| 54 |
+
console.log(`Source fingerprint: ${currentSourceFingerprint.hash}`);
|
| 55 |
+
for (const check of requiredChecks) {
|
| 56 |
+
console.log(`${check.status}: ${check.name} - ${check.message}`);
|
| 57 |
+
}
|
| 58 |
+
if (!passed) {
|
| 59 |
+
console.log("Next actions:");
|
| 60 |
+
for (const action of payload.nextActions) console.log(`- ${action}`);
|
| 61 |
+
}
|
| 62 |
+
if (!passed) process.exitCode = 1;
|
| 63 |
+
}
|
| 64 |
+
|
| 65 |
+
function validationChecks(audit) {
|
| 66 |
+
const checks = Array.isArray(audit?.checks)
|
| 67 |
+
? audit.checks
|
| 68 |
+
: Array.isArray(audit?.required)
|
| 69 |
+
? audit.required.map((check) => ({ ...check, required: true }))
|
| 70 |
+
: [];
|
| 71 |
+
return checks
|
| 72 |
+
.filter((check) => check.required !== false)
|
| 73 |
+
.map((check) => ({
|
| 74 |
+
name: check.name,
|
| 75 |
+
required: true,
|
| 76 |
+
status: check.status ?? "unknown",
|
| 77 |
+
message: check.message ?? "",
|
| 78 |
+
source: "validation audit",
|
| 79 |
+
}));
|
| 80 |
+
}
|
| 81 |
+
|
| 82 |
+
function hostedSmokeCheck(hostedSmoke) {
|
| 83 |
+
if (!hostedSmoke) {
|
| 84 |
+
return {
|
| 85 |
+
name: "hosted Space no-server smoke",
|
| 86 |
+
required: true,
|
| 87 |
+
status: "missing",
|
| 88 |
+
message: "Hosted smoke JSON is missing.",
|
| 89 |
+
source: "hosted smoke",
|
| 90 |
+
};
|
| 91 |
+
}
|
| 92 |
+
const passed =
|
| 93 |
+
hostedSmoke.passed === true &&
|
| 94 |
+
hostedSmoke.commitMatch === true &&
|
| 95 |
+
hostedSmoke.noServerSmoke === true &&
|
| 96 |
+
hostedSmoke.clientSideSummary?.benchmarkRequestCount === 0 &&
|
| 97 |
+
hostedSmoke.clientSideSummary?.serverInferenceSuspectCount === 0;
|
| 98 |
+
return {
|
| 99 |
+
name: "hosted Space no-server smoke",
|
| 100 |
+
required: true,
|
| 101 |
+
status: passed ? "pass" : "fail",
|
| 102 |
+
message: passed
|
| 103 |
+
? `Hosted Space commit ${
|
| 104 |
+
hostedSmoke.hfInfo?.sha ?? hostedSmoke.hubApiInfo?.sha ?? "unknown"
|
| 105 |
+
} passed UI and no-server smoke.`
|
| 106 |
+
: "Hosted Space smoke did not prove the deployed no-server path.",
|
| 107 |
+
source: "hosted smoke",
|
| 108 |
+
evidence: {
|
| 109 |
+
spaceSha: hostedSmoke.hfInfo?.sha ?? null,
|
| 110 |
+
hubApiSha: hostedSmoke.hubApiInfo?.sha ?? null,
|
| 111 |
+
hostCommit: hostedSmoke.hostHead?.hfSpaceCommit ?? null,
|
| 112 |
+
commitMatch: hostedSmoke.commitMatch ?? null,
|
| 113 |
+
spaceConfigParsed: hostedSmoke.spaceConfigParsed ?? null,
|
| 114 |
+
runtimeStage: hostedSmoke.hubApiInfo?.runtime?.stage ?? hostedSmoke.hfInfo?.runtime?.stage ?? null,
|
| 115 |
+
noServerSmoke: hostedSmoke.noServerSmoke ?? null,
|
| 116 |
+
benchmarkRequestCount: hostedSmoke.clientSideSummary?.benchmarkRequestCount ?? null,
|
| 117 |
+
serverInferenceSuspectCount: hostedSmoke.clientSideSummary?.serverInferenceSuspectCount ?? null,
|
| 118 |
+
hostMetadata: hostedSmoke.clientSideSummary?.hostMetadata ?? null,
|
| 119 |
+
},
|
| 120 |
+
};
|
| 121 |
+
}
|
| 122 |
+
|
| 123 |
+
function browserEvidenceCheck(audit) {
|
| 124 |
+
if (!audit) {
|
| 125 |
+
return {
|
| 126 |
+
name: "downloaded browser evidence audit",
|
| 127 |
+
required: true,
|
| 128 |
+
status: "missing",
|
| 129 |
+
message: "Browser-downloaded benchmark evidence has not been audited.",
|
| 130 |
+
source: "browser evidence audit",
|
| 131 |
+
};
|
| 132 |
+
}
|
| 133 |
+
return {
|
| 134 |
+
name: "downloaded browser evidence audit",
|
| 135 |
+
required: true,
|
| 136 |
+
status: audit.passed === true ? "pass" : "missing",
|
| 137 |
+
message:
|
| 138 |
+
audit.passed === true
|
| 139 |
+
? "Downloaded browser evidence meets real-mic and hardware-WebGPU gates."
|
| 140 |
+
: "Downloaded browser evidence is missing real-mic rows, hardware-WebGPU rows, or hosted metadata.",
|
| 141 |
+
source: "browser evidence audit",
|
| 142 |
+
evidence: {
|
| 143 |
+
inputPath: audit.inputPath ?? null,
|
| 144 |
+
checks: audit.checks?.map(({ name, status, message }) => ({ name, status, message })) ?? [],
|
| 145 |
+
},
|
| 146 |
+
};
|
| 147 |
+
}
|
| 148 |
+
|
| 149 |
+
function realMicArtifactCheck(realMic) {
|
| 150 |
+
if (!realMic) {
|
| 151 |
+
return {
|
| 152 |
+
name: "real microphone artifact",
|
| 153 |
+
required: true,
|
| 154 |
+
status: "missing",
|
| 155 |
+
message: "Real microphone artifact is missing.",
|
| 156 |
+
source: "real mic",
|
| 157 |
+
};
|
| 158 |
+
}
|
| 159 |
+
return {
|
| 160 |
+
name: "real microphone artifact",
|
| 161 |
+
required: true,
|
| 162 |
+
status: realMic.passed === true ? "pass" : "fail",
|
| 163 |
+
message: realMic.passed === true ? "Real microphone artifact passed." : "Real microphone artifact did not pass.",
|
| 164 |
+
source: "real mic",
|
| 165 |
+
evidence: {
|
| 166 |
+
generatedAt: realMic.generatedAt ?? null,
|
| 167 |
+
summary: realMic.summary ?? null,
|
| 168 |
+
config: realMic.config ?? null,
|
| 169 |
+
},
|
| 170 |
+
};
|
| 171 |
+
}
|
| 172 |
+
|
| 173 |
+
function webgpuArtifactCheck(webgpu) {
|
| 174 |
+
if (!webgpu) {
|
| 175 |
+
return {
|
| 176 |
+
name: "hardware WebGPU artifact",
|
| 177 |
+
required: true,
|
| 178 |
+
status: "missing",
|
| 179 |
+
message: "WebGPU benchmark artifact is missing.",
|
| 180 |
+
source: "webgpu",
|
| 181 |
+
};
|
| 182 |
+
}
|
| 183 |
+
const completeCandidates = Array.isArray(webgpu.candidates)
|
| 184 |
+
? webgpu.candidates.filter((candidate) => candidate.status === "complete")
|
| 185 |
+
: [];
|
| 186 |
+
const passed = webgpu.skipped !== true && webgpu.webgpu?.available === true && webgpu.webgpu?.softwareAdapter !== true && completeCandidates.length > 0;
|
| 187 |
+
return {
|
| 188 |
+
name: "hardware WebGPU artifact",
|
| 189 |
+
required: true,
|
| 190 |
+
status: passed ? "pass" : "missing",
|
| 191 |
+
message: passed
|
| 192 |
+
? `${completeCandidates.length} hardware WebGPU candidate(s) completed.`
|
| 193 |
+
: webgpu.reason ?? "No hardware WebGPU candidate completed.",
|
| 194 |
+
source: "webgpu",
|
| 195 |
+
evidence: {
|
| 196 |
+
generatedAt: webgpu.generatedAt ?? null,
|
| 197 |
+
webgpu: webgpu.webgpu ?? null,
|
| 198 |
+
candidates: webgpu.candidates ?? [],
|
| 199 |
+
},
|
| 200 |
+
};
|
| 201 |
+
}
|
| 202 |
+
|
| 203 |
+
function freshnessCheck(artifacts) {
|
| 204 |
+
const stale = Object.entries(artifacts)
|
| 205 |
+
.filter(([, artifact]) => artifact.exists && artifact.stale)
|
| 206 |
+
.map(([key, artifact]) => ({ key, path: artifact.path, artifactHash: artifact.sourceHash }));
|
| 207 |
+
return {
|
| 208 |
+
name: "artifact source freshness",
|
| 209 |
+
required: true,
|
| 210 |
+
status: stale.length === 0 ? "pass" : "stale",
|
| 211 |
+
message:
|
| 212 |
+
stale.length === 0
|
| 213 |
+
? "Readable artifacts with source fingerprints match current files."
|
| 214 |
+
: `${stale.length} artifact(s) were generated from an older source fingerprint.`,
|
| 215 |
+
source: "summary",
|
| 216 |
+
evidence: { stale },
|
| 217 |
+
};
|
| 218 |
+
}
|
| 219 |
+
|
| 220 |
+
async function readArtifact(path, currentSourceFingerprint) {
|
| 221 |
+
if (!existsSync(path)) {
|
| 222 |
+
return { exists: false, path, ok: false, error: "missing" };
|
| 223 |
+
}
|
| 224 |
+
try {
|
| 225 |
+
const value = JSON.parse(await readFile(path, "utf8"));
|
| 226 |
+
const sourceHash = value.sourceFingerprint?.hash ?? null;
|
| 227 |
+
return {
|
| 228 |
+
exists: true,
|
| 229 |
+
ok: true,
|
| 230 |
+
path,
|
| 231 |
+
value,
|
| 232 |
+
generatedAt: value.generatedAt ?? null,
|
| 233 |
+
sourceHash,
|
| 234 |
+
stale: Boolean(sourceHash && sourceHash !== currentSourceFingerprint.hash),
|
| 235 |
+
};
|
| 236 |
+
} catch (error) {
|
| 237 |
+
return { exists: true, ok: false, path, error: error.message };
|
| 238 |
+
}
|
| 239 |
+
}
|
| 240 |
+
|
| 241 |
+
function summarizeArtifact(artifact) {
|
| 242 |
+
return {
|
| 243 |
+
exists: artifact.exists,
|
| 244 |
+
ok: artifact.ok,
|
| 245 |
+
path: artifact.path,
|
| 246 |
+
generatedAt: artifact.generatedAt ?? null,
|
| 247 |
+
sourceHash: artifact.sourceHash ?? null,
|
| 248 |
+
stale: artifact.stale ?? false,
|
| 249 |
+
error: artifact.error ?? null,
|
| 250 |
+
};
|
| 251 |
+
}
|
| 252 |
+
|
| 253 |
+
function dedupeChecks(checks) {
|
| 254 |
+
const merged = new Map();
|
| 255 |
+
for (const check of checks) {
|
| 256 |
+
if (!check?.name) continue;
|
| 257 |
+
const previous = merged.get(check.name);
|
| 258 |
+
if (!previous || statusRank(check.status) < statusRank(previous.status)) {
|
| 259 |
+
merged.set(check.name, check);
|
| 260 |
+
}
|
| 261 |
+
}
|
| 262 |
+
return [...merged.values()];
|
| 263 |
+
}
|
| 264 |
+
|
| 265 |
+
function statusRank(status) {
|
| 266 |
+
return {
|
| 267 |
+
fail: 0,
|
| 268 |
+
stale: 1,
|
| 269 |
+
missing: 2,
|
| 270 |
+
unknown: 3,
|
| 271 |
+
warn: 4,
|
| 272 |
+
pass: 5,
|
| 273 |
+
skip: 6,
|
| 274 |
+
}[status] ?? 3;
|
| 275 |
+
}
|
| 276 |
+
|
| 277 |
+
function nextActions(checks) {
|
| 278 |
+
const actions = [];
|
| 279 |
+
const needsRealMic = checks.some((check) => check.status !== "pass" && check.name.includes("real microphone"));
|
| 280 |
+
const needsHardwareWebgpu = checks.some((check) => check.status !== "pass" && check.name.includes("WebGPU"));
|
| 281 |
+
const needsBrowserEvidence = checks.some(
|
| 282 |
+
(check) => check.status !== "pass" && check.name === "downloaded browser evidence audit",
|
| 283 |
+
);
|
| 284 |
+
if (needsRealMic || needsHardwareWebgpu || needsBrowserEvidence) {
|
| 285 |
+
actions.push(
|
| 286 |
+
"Run node tools/run-hosted-evidence-capture.mjs from a desktop Chrome session with a real microphone; use a machine exposing hardware WebGPU to satisfy both hosted browser evidence gates.",
|
| 287 |
+
);
|
| 288 |
+
}
|
| 289 |
+
for (const check of checks) {
|
| 290 |
+
if (check.status === "pass") continue;
|
| 291 |
+
if (check.name.includes("real microphone")) {
|
| 292 |
+
actions.push("Run node tools/run-real-mic-series.mjs on a machine with a real microphone.");
|
| 293 |
+
} else if (check.name.includes("WebGPU")) {
|
| 294 |
+
actions.push("Run node tools/run-webgpu-benchmark.mjs in Chrome on a machine exposing a hardware WebGPU adapter.");
|
| 295 |
+
} else if (check.name === "downloaded browser evidence audit") {
|
| 296 |
+
actions.push("Download JSON from the hosted Space after real-mic/WebGPU runs, then run node tools/audit-browser-evidence.mjs <file>.");
|
| 297 |
+
} else if (check.name === "hosted Space no-server smoke") {
|
| 298 |
+
actions.push("Run node tools/run-hosted-smoke.mjs.");
|
| 299 |
+
} else if (check.name === "artifact source freshness") {
|
| 300 |
+
actions.push("Rerun the stale harnesses so their source fingerprints match current files.");
|
| 301 |
+
} else {
|
| 302 |
+
actions.push(`Resolve evidence check: ${check.name}.`);
|
| 303 |
+
}
|
| 304 |
+
}
|
| 305 |
+
return [...new Set(actions)];
|
| 306 |
+
}
|
| 307 |
+
|
| 308 |
+
main().catch((error) => {
|
| 309 |
+
console.error(error.stack ?? error.message);
|
| 310 |
+
process.exitCode = 1;
|
| 311 |
+
});
|
workers/asr-worker.js
CHANGED
|
@@ -2,6 +2,7 @@ import { AutoModel, Tensor, env, pipeline } from "https://cdn.jsdelivr.net/npm/@
|
|
| 2 |
|
| 3 |
env.allowLocalModels = false;
|
| 4 |
env.useBrowserCache = true;
|
|
|
|
| 5 |
|
| 6 |
const SAMPLE_RATE = 16000;
|
| 7 |
const SPEECH_THRESHOLD = 0.3;
|
|
@@ -35,6 +36,49 @@ let lastPartialAt = 0;
|
|
| 35 |
let utteranceStartedAt = 0;
|
| 36 |
const recordingBuffer = new Float32Array(MAX_BUFFER_DURATION * SAMPLE_RATE);
|
| 37 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
self.onmessage = async (event) => {
|
| 39 |
const message = event.data;
|
| 40 |
try {
|
|
|
|
| 2 |
|
| 3 |
env.allowLocalModels = false;
|
| 4 |
env.useBrowserCache = true;
|
| 5 |
+
installFetchTelemetry("asr");
|
| 6 |
|
| 7 |
const SAMPLE_RATE = 16000;
|
| 8 |
const SPEECH_THRESHOLD = 0.3;
|
|
|
|
| 36 |
let utteranceStartedAt = 0;
|
| 37 |
const recordingBuffer = new Float32Array(MAX_BUFFER_DURATION * SAMPLE_RATE);
|
| 38 |
|
| 39 |
+
function installFetchTelemetry(scope) {
|
| 40 |
+
const originalFetch = globalThis.fetch?.bind(globalThis);
|
| 41 |
+
if (!originalFetch || globalThis.__browserSpeakFetchTelemetryInstalled) return;
|
| 42 |
+
globalThis.__browserSpeakFetchTelemetryInstalled = true;
|
| 43 |
+
globalThis.fetch = async (input, init) => {
|
| 44 |
+
const startedAt = performance.now();
|
| 45 |
+
const url = fetchUrl(input);
|
| 46 |
+
const method = String(init?.method || input?.method || "GET").toUpperCase();
|
| 47 |
+
try {
|
| 48 |
+
const response = await originalFetch(input, init);
|
| 49 |
+
self.postMessage({
|
| 50 |
+
type: "network",
|
| 51 |
+
scope,
|
| 52 |
+
method,
|
| 53 |
+
url,
|
| 54 |
+
responseUrl: response.url || url,
|
| 55 |
+
status: response.status,
|
| 56 |
+
ok: response.ok,
|
| 57 |
+
durationMs: performance.now() - startedAt,
|
| 58 |
+
});
|
| 59 |
+
return response;
|
| 60 |
+
} catch (error) {
|
| 61 |
+
self.postMessage({
|
| 62 |
+
type: "network",
|
| 63 |
+
scope,
|
| 64 |
+
method,
|
| 65 |
+
url,
|
| 66 |
+
status: null,
|
| 67 |
+
ok: false,
|
| 68 |
+
durationMs: performance.now() - startedAt,
|
| 69 |
+
error: error.message ?? String(error),
|
| 70 |
+
});
|
| 71 |
+
throw error;
|
| 72 |
+
}
|
| 73 |
+
};
|
| 74 |
+
}
|
| 75 |
+
|
| 76 |
+
function fetchUrl(input) {
|
| 77 |
+
if (typeof input === "string") return input;
|
| 78 |
+
if (input instanceof URL) return input.href;
|
| 79 |
+
return input?.url ?? "";
|
| 80 |
+
}
|
| 81 |
+
|
| 82 |
self.onmessage = async (event) => {
|
| 83 |
const message = event.data;
|
| 84 |
try {
|
workers/llm-worker.js
CHANGED
|
@@ -8,6 +8,7 @@ import {
|
|
| 8 |
|
| 9 |
env.allowLocalModels = false;
|
| 10 |
env.useBrowserCache = true;
|
|
|
|
| 11 |
|
| 12 |
let tokenizer = null;
|
| 13 |
let model = null;
|
|
@@ -15,6 +16,49 @@ let modelId = "";
|
|
| 15 |
let stoppingCriteria = new InterruptableStoppingCriteria();
|
| 16 |
let currentTurnId = 0;
|
| 17 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
self.onmessage = async (event) => {
|
| 19 |
const message = event.data;
|
| 20 |
try {
|
|
|
|
| 8 |
|
| 9 |
env.allowLocalModels = false;
|
| 10 |
env.useBrowserCache = true;
|
| 11 |
+
installFetchTelemetry("llm");
|
| 12 |
|
| 13 |
let tokenizer = null;
|
| 14 |
let model = null;
|
|
|
|
| 16 |
let stoppingCriteria = new InterruptableStoppingCriteria();
|
| 17 |
let currentTurnId = 0;
|
| 18 |
|
| 19 |
+
function installFetchTelemetry(scope) {
|
| 20 |
+
const originalFetch = globalThis.fetch?.bind(globalThis);
|
| 21 |
+
if (!originalFetch || globalThis.__browserSpeakFetchTelemetryInstalled) return;
|
| 22 |
+
globalThis.__browserSpeakFetchTelemetryInstalled = true;
|
| 23 |
+
globalThis.fetch = async (input, init) => {
|
| 24 |
+
const startedAt = performance.now();
|
| 25 |
+
const url = fetchUrl(input);
|
| 26 |
+
const method = String(init?.method || input?.method || "GET").toUpperCase();
|
| 27 |
+
try {
|
| 28 |
+
const response = await originalFetch(input, init);
|
| 29 |
+
self.postMessage({
|
| 30 |
+
type: "network",
|
| 31 |
+
scope,
|
| 32 |
+
method,
|
| 33 |
+
url,
|
| 34 |
+
responseUrl: response.url || url,
|
| 35 |
+
status: response.status,
|
| 36 |
+
ok: response.ok,
|
| 37 |
+
durationMs: performance.now() - startedAt,
|
| 38 |
+
});
|
| 39 |
+
return response;
|
| 40 |
+
} catch (error) {
|
| 41 |
+
self.postMessage({
|
| 42 |
+
type: "network",
|
| 43 |
+
scope,
|
| 44 |
+
method,
|
| 45 |
+
url,
|
| 46 |
+
status: null,
|
| 47 |
+
ok: false,
|
| 48 |
+
durationMs: performance.now() - startedAt,
|
| 49 |
+
error: error.message ?? String(error),
|
| 50 |
+
});
|
| 51 |
+
throw error;
|
| 52 |
+
}
|
| 53 |
+
};
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
function fetchUrl(input) {
|
| 57 |
+
if (typeof input === "string") return input;
|
| 58 |
+
if (input instanceof URL) return input.href;
|
| 59 |
+
return input?.url ?? "";
|
| 60 |
+
}
|
| 61 |
+
|
| 62 |
self.onmessage = async (event) => {
|
| 63 |
const message = event.data;
|
| 64 |
try {
|
workers/tts-worker.js
CHANGED
|
@@ -2,6 +2,7 @@ import { env, pipeline } from "https://cdn.jsdelivr.net/npm/@huggingface/transfo
|
|
| 2 |
|
| 3 |
env.allowLocalModels = false;
|
| 4 |
env.useBrowserCache = true;
|
|
|
|
| 5 |
|
| 6 |
let synthesizer = null;
|
| 7 |
let modelId = "";
|
|
@@ -11,6 +12,49 @@ const voiceEmbeddings = new Map();
|
|
| 11 |
const voicePromises = new Map();
|
| 12 |
const VOICES = ["F1", "F2", "M1", "M2"];
|
| 13 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
self.onmessage = (event) => {
|
| 15 |
const message = event.data;
|
| 16 |
if (message.type === "load") {
|
|
|
|
| 2 |
|
| 3 |
env.allowLocalModels = false;
|
| 4 |
env.useBrowserCache = true;
|
| 5 |
+
installFetchTelemetry("tts");
|
| 6 |
|
| 7 |
let synthesizer = null;
|
| 8 |
let modelId = "";
|
|
|
|
| 12 |
const voicePromises = new Map();
|
| 13 |
const VOICES = ["F1", "F2", "M1", "M2"];
|
| 14 |
|
| 15 |
+
function installFetchTelemetry(scope) {
|
| 16 |
+
const originalFetch = globalThis.fetch?.bind(globalThis);
|
| 17 |
+
if (!originalFetch || globalThis.__browserSpeakFetchTelemetryInstalled) return;
|
| 18 |
+
globalThis.__browserSpeakFetchTelemetryInstalled = true;
|
| 19 |
+
globalThis.fetch = async (input, init) => {
|
| 20 |
+
const startedAt = performance.now();
|
| 21 |
+
const url = fetchUrl(input);
|
| 22 |
+
const method = String(init?.method || input?.method || "GET").toUpperCase();
|
| 23 |
+
try {
|
| 24 |
+
const response = await originalFetch(input, init);
|
| 25 |
+
self.postMessage({
|
| 26 |
+
type: "network",
|
| 27 |
+
scope,
|
| 28 |
+
method,
|
| 29 |
+
url,
|
| 30 |
+
responseUrl: response.url || url,
|
| 31 |
+
status: response.status,
|
| 32 |
+
ok: response.ok,
|
| 33 |
+
durationMs: performance.now() - startedAt,
|
| 34 |
+
});
|
| 35 |
+
return response;
|
| 36 |
+
} catch (error) {
|
| 37 |
+
self.postMessage({
|
| 38 |
+
type: "network",
|
| 39 |
+
scope,
|
| 40 |
+
method,
|
| 41 |
+
url,
|
| 42 |
+
status: null,
|
| 43 |
+
ok: false,
|
| 44 |
+
durationMs: performance.now() - startedAt,
|
| 45 |
+
error: error.message ?? String(error),
|
| 46 |
+
});
|
| 47 |
+
throw error;
|
| 48 |
+
}
|
| 49 |
+
};
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
function fetchUrl(input) {
|
| 53 |
+
if (typeof input === "string") return input;
|
| 54 |
+
if (input instanceof URL) return input.href;
|
| 55 |
+
return input?.url ?? "";
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
self.onmessage = (event) => {
|
| 59 |
const message = event.data;
|
| 60 |
if (message.type === "load") {
|