browser-speak / tools /run-local-candidate-benchmark.mjs
Mike0021's picture
Add worker network telemetry to browser evidence
d2ae80e verified
Raw
History Blame Contribute Delete
22.9 kB
#!/usr/bin/env node
import { spawn } from "node:child_process";
import { existsSync } from "node:fs";
import { mkdir, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, resolve } from "node:path";
const url = process.env.BROWSER_SPEAK_URL ?? "http://127.0.0.1:5174/";
const chrome =
process.env.CHROME_BIN ??
(existsSync("/opt/google/chrome/chrome")
? "/opt/google/chrome/chrome"
: existsSync("/usr/bin/google-chrome")
? "/usr/bin/google-chrome"
: "chromium");
const resultPath = resolve(process.env.BROWSER_SPEAK_LOCAL_JSON ?? `${tmpdir()}/browser-speak-local-candidates.json`);
const profileDir = resolve(process.env.BROWSER_SPEAK_LOCAL_PROFILE_DIR ?? `${tmpdir()}/browser-speak-local-profile`);
const protocolTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_TIMEOUT_MS ?? 60000);
const pollTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_POLL_TIMEOUT_MS ?? 5000);
const pageUnresponsiveTimeoutMs = Number(process.env.BROWSER_SPEAK_PAGE_UNRESPONSIVE_TIMEOUT_MS ?? 120000);
const loadTimeoutMs = Number(process.env.BROWSER_SPEAK_LOAD_TIMEOUT_MS ?? 360000);
const taskTimeoutMs = Number(process.env.BROWSER_SPEAK_TASK_TIMEOUT_MS ?? 180000);
const ttsWarmup = process.env.BROWSER_SPEAK_TTS_WARMUP === "true";
const reuseProfile = process.env.BROWSER_SPEAK_LOCAL_REUSE_PROFILE === "true";
const headless = process.env.BROWSER_SPEAK_HEADLESS !== "false";
const dryRun = process.env.BROWSER_SPEAK_LOCAL_DRY_RUN === "true";
const localTasks = parseList(process.env.BROWSER_SPEAK_LOCAL_TASKS ?? "tts,loopback");
const stacks = buildStacks().slice(0, maxStacks());
async function main() {
const config = benchmarkConfig();
if (dryRun) {
const payload = {
generatedAt: new Date().toISOString(),
dryRun: true,
url,
config,
candidates: stacks.map((stack) => ({ stack, status: "planned" })),
};
await mkdir(dirname(resultPath), { recursive: true });
await writeJson(payload);
console.log(`Wrote local candidate dry-run JSON: ${resultPath}`);
console.log(`Planned ${stacks.length} stack(s): ${stacks.map(stackLabel).join("; ")}`);
return;
}
await ensureServer();
await mkdir(dirname(resultPath), { recursive: true });
if (!reuseProfile) await rm(profileDir, { recursive: true, force: true });
const browser = launchBrowser(9344, profileDir);
const candidates = stacks.map((stack) => ({ stack, status: "pending" }));
try {
const client = await connectToPage(9344, browser);
await waitForBenchApi(client);
await runPageTask(client, "window.browserSpeakBench.clearResults()", { label: "clear results" });
for (const candidate of candidates) {
const label = stackLabel(candidate.stack);
const startedAt = Date.now();
let pageUsable = true;
console.log(`Running local candidate: ${label}`);
try {
const before = await runPageTask(client, "window.browserSpeakBench.state()", {
label: `${label} preflight`,
});
await runPageTask(client, `window.browserSpeakBench.loadStack(${JSON.stringify(candidate.stack)})`, {
label: `${label} model load`,
timeoutMs: loadTimeoutMs,
});
for (const task of localTasks) {
await runTask(client, task, label);
}
const snapshot = await runPageTask(client, "window.browserSpeakBench.state()", {
label: `${label} state`,
});
const addedCount = Math.max(0, snapshot.results.length - before.results.length);
const rows = snapshot.results.slice(0, addedCount);
const errors = rows.filter((row) => row.error);
candidate.status = errors.length === 0 ? "complete" : "partial";
candidate.durationMs = Date.now() - startedAt;
candidate.rowCount = rows.length;
candidate.errorCount = errors.length;
candidate.metrics = metricsForRows(rows);
candidate.summary = snapshot.summary.current ?? snapshot.summary.all;
if (errors.length > 0) candidate.errors = errors.map((row) => `${row.kind}: ${row.error}`);
} catch (error) {
candidate.status = "failed";
candidate.durationMs = Date.now() - startedAt;
candidate.error = error.message;
pageUsable = !isPageUnresponsiveError(error);
console.error(`${label} failed: ${error.message}`);
} finally {
if (pageUsable) {
await runPageTask(client, "window.browserSpeakBench.stop()", {
label: `${label} stop`,
timeoutMs: 20000,
}).catch(() => {});
const state = await runPageTask(client, "window.browserSpeakBench.state()", {
label: `${label} loaded state`,
timeoutMs: 20000,
}).catch(() => null);
if (state?.modelsLoaded || state?.modelsLoading) {
await runPageTask(client, "window.browserSpeakBench.unload()", {
label: `${label} unload`,
timeoutMs: 60000,
}).catch((error) => {
candidate.unloadError = error.message;
});
}
} else {
candidate.unloadError = "Skipped page cleanup because the renderer was unresponsive.";
}
await writeJson(await exportPayload(client, candidates));
}
if (!pageUsable) break;
}
const payload = await exportPayload(client, candidates);
await writeJson(payload);
console.log(`Wrote local candidate benchmark JSON: ${resultPath}`);
summarizeCandidates(payload.candidates);
await client.closeBrowser();
} finally {
await stopBrowser(browser, reuseProfile ? null : profileDir);
}
}
async function runTask(client, task, label) {
const normalized = task.toLowerCase();
const loopbackOptions = {
timeoutMs: 180000,
...(process.env.BROWSER_SPEAK_LOOPBACK_TEXT ? { text: process.env.BROWSER_SPEAK_LOOPBACK_TEXT } : {}),
...(Number.isFinite(Number(process.env.BROWSER_SPEAK_LOOPBACK_SPEED))
? { speed: Number(process.env.BROWSER_SPEAK_LOOPBACK_SPEED) }
: {}),
};
const expression = {
tts: "window.browserSpeakBench.runTts({ timeoutMs: 90000 })",
identity: "window.browserSpeakBench.runIdentity({ timeoutMs: 180000 })",
chat: "window.browserSpeakBench.runChat({ timeoutMs: 180000 })",
loopback: `window.browserSpeakBench.runLoopback(${JSON.stringify(loopbackOptions)})`,
barge: "window.browserSpeakBench.runBargeIn({ timeoutMs: 90000 })",
"barge-in": "window.browserSpeakBench.runBargeIn({ timeoutMs: 90000 })",
suite: "window.browserSpeakBench.runSuite()",
}[normalized];
if (!expression) throw new Error(`Unknown local benchmark task: ${task}`);
await runPageTask(client, expression, {
label: `${label} ${normalized}`,
timeoutMs: normalized === "suite" ? Math.max(taskTimeoutMs, 420000) : taskTimeoutMs,
});
}
async function exportPayload(client, candidates) {
const exportResult = await runPageTask(client, "window.browserSpeakBench.exportResults()", {
label: "export results",
timeoutMs: 30000,
}).catch(() => ({ summary: null, results: [] }));
return {
generatedAt: new Date().toISOString(),
url,
config: benchmarkConfig(),
candidates,
summary: exportResult.summary,
results: exportResult.results,
};
}
function benchmarkConfig() {
return {
tasks: localTasks,
stackCount: stacks.length,
loadTimeoutMs,
taskTimeoutMs,
protocolTimeoutMs,
pollTimeoutMs,
pageUnresponsiveTimeoutMs,
ttsWarmup,
reuseProfile,
headless,
chrome,
profileDir,
extraChromeArgs: parseChromeArgs(),
};
}
function isPageUnresponsiveError(error) {
return /page stayed unresponsive|Runtime\.evaluate timed out/i.test(error?.message ?? String(error));
}
async function writeJson(payload) {
await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`);
}
function buildStacks() {
if (process.env.BROWSER_SPEAK_LOCAL_STACKS) {
const parsed = JSON.parse(process.env.BROWSER_SPEAK_LOCAL_STACKS);
if (!Array.isArray(parsed)) throw new Error("BROWSER_SPEAK_LOCAL_STACKS must be a JSON array.");
return parsed.map((stack, index) => normalizeLocalStack(stack, index));
}
const llms = parseList(process.env.BROWSER_SPEAK_LOCAL_LLMS ?? "HuggingFaceTB/SmolLM2-135M-Instruct");
const asrs = parseList(
process.env.BROWSER_SPEAK_LOCAL_ASRS ??
"onnx-community/moonshine-base-ONNX,onnx-community/moonshine-tiny-ONNX,onnx-community/whisper-tiny.en",
);
const voices = parseList(process.env.BROWSER_SPEAK_LOCAL_VOICES ?? "F2");
const steps = parseList(process.env.BROWSER_SPEAK_LOCAL_TTS_STEPS ?? "2").map((value) => Number(value));
const vadSilenceValues = parseList(process.env.BROWSER_SPEAK_LOCAL_VAD_SILENCE_MS ?? "480").map((value) =>
Number(value),
);
const partialAsr = process.env.BROWSER_SPEAK_PARTIAL_ASR !== "false";
const stacks = [];
for (const llm of llms) {
for (const asr of asrs) {
for (const voice of voices) {
for (const ttsSteps of steps) {
for (const vadSilenceMs of vadSilenceValues) {
stacks.push({
device: "wasm",
llm,
asr,
voice,
ttsSteps,
vadSilenceMs,
partialAsr,
ttsWarmup,
});
}
}
}
}
}
return stacks;
}
function normalizeLocalStack(stack, index) {
return {
device: "wasm",
name: stack.name ?? `stack-${index + 1}`,
llm: stack.llm ?? parseList(process.env.BROWSER_SPEAK_LOCAL_LLMS ?? "HuggingFaceTB/SmolLM2-135M-Instruct")[0],
asr: stack.asr ?? parseList(process.env.BROWSER_SPEAK_LOCAL_ASRS ?? "onnx-community/moonshine-base-ONNX")[0],
voice: stack.voice ?? parseList(process.env.BROWSER_SPEAK_LOCAL_VOICES ?? "F2")[0],
ttsSteps: Number(stack.ttsSteps ?? parseList(process.env.BROWSER_SPEAK_LOCAL_TTS_STEPS ?? "2")[0]),
vadSilenceMs: Number(stack.vadSilenceMs ?? parseList(process.env.BROWSER_SPEAK_LOCAL_VAD_SILENCE_MS ?? "480")[0]),
partialAsr: stack.partialAsr ?? (process.env.BROWSER_SPEAK_PARTIAL_ASR !== "false"),
ttsWarmup: stack.ttsWarmup ?? ttsWarmup,
...(stack.ttsChunking ? { ttsChunking: stack.ttsChunking } : {}),
};
}
function maxStacks() {
if (!process.env.BROWSER_SPEAK_LOCAL_MAX_STACKS) return Number.POSITIVE_INFINITY;
const value = Number(process.env.BROWSER_SPEAK_LOCAL_MAX_STACKS);
return Number.isFinite(value) ? Math.max(0, value) : Number.POSITIVE_INFINITY;
}
async function ensureServer() {
const response = await fetch(url).catch((error) => {
throw new Error(`Could not reach ${url}: ${error.message}`);
});
if (!response.ok) throw new Error(`${url} returned HTTP ${response.status}`);
}
async function waitForBenchApi(client) {
const deadline = Date.now() + 15000;
while (Date.now() < deadline) {
try {
if (await client.evaluate("Boolean(window.browserSpeakBench)")) return;
} catch {
// The target may still be navigating and can destroy the execution context.
}
await sleep(100);
}
throw new Error("window.browserSpeakBench was not installed.");
}
async function runPageTask(client, expression, { label = "page task", timeoutMs = 30000 } = {}) {
const taskId = `task_${Date.now()}_${Math.random().toString(16).slice(2)}`;
await client.evaluate(`(() => {
const taskId = ${JSON.stringify(taskId)};
window.__browserSpeakHarnessTasks ||= {};
window.__browserSpeakHarnessTasks[taskId] = { done: false, label: ${JSON.stringify(label)} };
Promise.resolve(${expression})
.then((value) => {
window.__browserSpeakHarnessTasks[taskId] = { done: true, value };
})
.catch((error) => {
window.__browserSpeakHarnessTasks[taskId] = {
done: true,
error: error?.stack || error?.message || String(error),
};
});
return true;
})()`);
const deadline = Date.now() + timeoutMs;
let lastEvents = [];
let lastPollError = "";
let pollErrorSince = 0;
while (Date.now() < deadline) {
let task = null;
try {
task = await client.evaluate(
`window.__browserSpeakHarnessTasks?.[${JSON.stringify(taskId)}] ?? null`,
pollTimeoutMs,
);
lastPollError = "";
pollErrorSince = 0;
} catch (error) {
const message = error.message ?? String(error);
pollErrorSince ||= Date.now();
if (message !== lastPollError) {
console.log(`${label}: waiting for page response (${message})`);
lastPollError = message;
}
if (Date.now() - pollErrorSince > pageUnresponsiveTimeoutMs) {
throw new Error(
`${label} page stayed unresponsive for ${(pageUnresponsiveTimeoutMs / 1000).toFixed(0)} seconds; last poll error: ${message}`,
);
}
await sleep(500);
continue;
}
if (task?.done) {
await client
.evaluate(`delete window.__browserSpeakHarnessTasks?.[${JSON.stringify(taskId)}]`, pollTimeoutMs)
.catch(() => {});
if (task.error) throw new Error(`${label} failed: ${task.error}`);
return task.value;
}
const snapshot = await client
.evaluate(`(() => {
const state = window.browserSpeakBench?.state?.();
return state ? {
modelsLoaded: state.modelsLoaded,
modelsLoading: state.modelsLoading,
activeBenchmark: state.activeBenchmark?.kind ?? null,
suiteRunning: state.suiteRunning,
events: state.events?.slice(0, 3) ?? [],
} : null;
})()`, pollTimeoutMs)
.catch(() => null);
const events = snapshot?.events ?? [];
if (events.join("\\n") !== lastEvents.join("\\n")) {
lastEvents = events;
if (events[0]) console.log(`${label}: ${events[0]}`);
}
await sleep(500);
}
throw new Error(`${label} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`);
}
function launchBrowser(port, profileDir) {
const child = spawn(
chrome,
[
...(headless ? ["--headless=new"] : []),
"--no-sandbox",
"--disable-gpu",
"--disable-dev-shm-usage",
"--disable-background-networking",
"--disable-extensions",
"--no-default-browser-check",
"--no-first-run",
"--autoplay-policy=no-user-gesture-required",
`--remote-debugging-port=${port}`,
`--user-data-dir=${profileDir}`,
...parseChromeArgs(),
],
{ stdio: ["ignore", "pipe", "pipe"] },
);
child.browserLog = "";
const appendLog = (chunk) => {
child.browserLog = `${child.browserLog}${chunk}`;
if (child.browserLog.length > 8000) child.browserLog = child.browserLog.slice(-8000);
};
child.stdout.on("data", appendLog);
child.stderr.on("data", appendLog);
return child;
}
async function stopBrowser(child, profileDir) {
if (child.exitCode == null) child.kill("SIGTERM");
await new Promise((resolve) => {
child.once("exit", resolve);
setTimeout(resolve, 3000);
});
if (child.exitCode == null) child.kill("SIGKILL");
if (!profileDir) return;
for (let attempt = 0; attempt < 5; attempt += 1) {
try {
await rm(profileDir, { recursive: true, force: true });
return;
} catch (error) {
if (attempt === 4) {
console.warn(`Could not remove ${profileDir}: ${error.message}`);
return;
}
await sleep(500);
}
}
}
async function connectToPage(port, child) {
const deadline = Date.now() + 60000;
let lastError = null;
while (Date.now() < deadline) {
if (child.exitCode != null) {
throw new Error(`Chrome exited before DevTools became available.\n${child.browserLog}`);
}
try {
const version = await fetch(`http://127.0.0.1:${port}/json/version`).then((response) => response.json());
if (version.webSocketDebuggerUrl) {
const page = await createPageTarget(port);
return new CdpClient(page.webSocketDebuggerUrl);
}
} catch (error) {
lastError = error;
}
await sleep(250);
}
throw new Error(
`Could not connect to Chrome DevTools on port ${port}: ${lastError?.message ?? "unknown error"}\n${child.browserLog}`,
);
}
async function createPageTarget(port) {
for (const method of ["PUT", "GET"]) {
const response = await fetch(`http://127.0.0.1:${port}/json/new?${encodeURIComponent(url)}`, {
method,
}).catch(() => null);
if (response?.ok) {
const target = await response.json();
if (target.webSocketDebuggerUrl) return target;
}
}
const targets = await fetch(`http://127.0.0.1:${port}/json`).then((response) => response.json());
const page = targets.find((target) => target.type === "page" && target.url === url);
if (page?.webSocketDebuggerUrl) return page;
throw new Error("Could not create or find a page target.");
}
class CdpClient {
constructor(webSocketUrl) {
this.nextId = 1;
this.pending = new Map();
this.socket = new WebSocket(webSocketUrl);
this.opened = new Promise((resolve, reject) => {
this.socket.onopen = resolve;
this.socket.onerror = reject;
this.socket.onmessage = (event) => this.onMessage(event);
});
}
onMessage(event) {
const message = JSON.parse(event.data);
if (!message.id || !this.pending.has(message.id)) return;
const { resolve: onResolve, reject } = this.pending.get(message.id);
this.pending.delete(message.id);
if (message.error) reject(new Error(message.error.message));
else onResolve(message.result);
}
async call(method, params = {}, timeoutMs = protocolTimeoutMs) {
await this.opened;
const id = this.nextId++;
this.socket.send(JSON.stringify({ id, method, params }));
return new Promise((resolvePromise, reject) => {
const timer = setTimeout(() => {
this.pending.delete(id);
reject(new Error(`${method} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`));
}, timeoutMs);
this.pending.set(id, {
resolve: (value) => {
clearTimeout(timer);
resolvePromise(value);
},
reject: (error) => {
clearTimeout(timer);
reject(error);
},
});
});
}
async evaluate(expression, timeoutMs = protocolTimeoutMs) {
const result = await this.call("Runtime.evaluate", {
expression,
returnByValue: true,
}, timeoutMs);
if (result.exceptionDetails) throw new Error(formatException(result.exceptionDetails));
return result.result.value;
}
closeBrowser() {
try {
this.socket.send(JSON.stringify({ id: this.nextId++, method: "Browser.close", params: {} }));
} catch {
// The surrounding process cleanup handles already-closed targets.
}
try {
this.socket.close();
} catch {
// Ignore close races.
}
}
}
function formatException(exceptionDetails) {
const exception = exceptionDetails.exception;
return exception?.description ?? exception?.value ?? exceptionDetails.text ?? "Evaluation failed.";
}
function parseList(value) {
return String(value ?? "")
.split(",")
.map((item) => item.trim())
.filter(Boolean);
}
function parseChromeArgs() {
const raw = process.env.BROWSER_SPEAK_CHROME_ARGS ?? "";
return (
raw
.match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g)
?.map((arg) => arg.replace(/^["']|["']$/g, "")) ?? []
);
}
function stackLabel(stack) {
return [
shortModel(stack.llm),
shortModel(stack.asr),
stack.voice,
`${stack.ttsSteps} steps`,
`${stack.vadSilenceMs}ms`,
stack.ttsChunking ? `chunk ${stack.ttsChunking.firstTargetChars ?? "?"}/${stack.ttsChunking.targetChars ?? "?"}` : "",
]
.filter(Boolean)
.join(" / ");
}
function shortModel(modelId = "") {
return modelId.split("/").at(-1)?.replace(/-ONNX$/, "") ?? modelId;
}
function summarizeCandidates(candidates) {
for (const candidate of candidates) {
const metrics = candidate.metrics ?? {};
const parts = [];
if (Number.isFinite(metrics.ttsMedianFirstAudioMs)) {
parts.push(`tts first audio ${formatMs(metrics.ttsMedianFirstAudioMs)}`);
}
if (Number.isFinite(metrics.modelLoadMs)) {
parts.push(`load ${formatMs(metrics.modelLoadMs)}`);
}
if (Number.isFinite(metrics.loopbackMedianSpeechEndToFirstAudioMs)) {
parts.push(`loopback end-to-audio ${formatMs(metrics.loopbackMedianSpeechEndToFirstAudioMs)}`);
}
if (Number.isFinite(metrics.identityMedianFirstAudioMs)) {
parts.push(`identity first audio ${formatMs(metrics.identityMedianFirstAudioMs)}`);
}
const suffix = parts.length > 0 ? `, ${parts.join(", ")}` : "";
const error = candidate.error ? ` (${candidate.error})` : "";
console.log(`${stackLabel(candidate.stack)}: ${candidate.status}, ${candidate.rowCount ?? 0} rows${suffix}${error}`);
}
}
function metricsForRows(rows) {
const completed = rows.filter((row) => !row.error);
const byKind = (kind) => completed.filter((row) => row.kind === kind);
const ttsRows = byKind("tts");
const identityRows = byKind("identity");
const chatRows = byKind("chat");
const loopbackRows = byKind("loopback");
return {
modelLoadMs: rows.find((row) => Number.isFinite(row.stack?.modelLoadMs))?.stack.modelLoadMs ?? null,
ttsRuns: ttsRows.length,
ttsMedianFirstAudioMs: median(ttsRows.map((row) => row.firstAudioMs)),
ttsMedianSynthesisMs: median(ttsRows.map((row) => row.firstTtsSynthesisMs)),
ttsMedianAudioEndMs: median(ttsRows.map((row) => row.audioEndMs)),
identityRuns: identityRows.length,
identityMedianFirstTokenMs: median(identityRows.map((row) => row.firstTokenMs)),
identityMedianFirstAudioMs: median(identityRows.map((row) => row.firstAudioMs)),
identityPasses: identityRows.filter((row) => row.llmQualityPass).length,
chatRuns: chatRows.length,
chatMedianFirstTokenMs: median(chatRows.map((row) => row.firstTokenMs)),
chatMedianFirstAudioMs: median(chatRows.map((row) => row.firstAudioMs)),
loopbackRuns: loopbackRows.length,
loopbackMedianWer: median(loopbackRows.map((row) => row.sttWer)),
loopbackMedianSpeechEndToFirstAudioMs: median(
loopbackRows.map((row) => row.speechEndToFirstAudioMs),
),
loopbackMedianSpeechEndToAudioEndMs: median(
loopbackRows.map((row) => row.speechEndToAudioEndMs),
),
};
}
function median(values) {
const finite = values.filter(Number.isFinite).sort((a, b) => a - b);
if (finite.length === 0) return null;
const middle = Math.floor(finite.length / 2);
if (finite.length % 2 === 1) return finite[middle];
return (finite[middle - 1] + finite[middle]) / 2;
}
function formatMs(value) {
if (!Number.isFinite(value)) return "-";
if (value < 1000) return `${Math.round(value)} ms`;
return `${(value / 1000).toFixed(2)} s`;
}
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
await main();