tools/run-client-side-smoke.mjs

#!/usr/bin/env node
import { spawn } from "node:child_process";
import { existsSync } from "node:fs";
import { mkdir, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, resolve } from "node:path";
import { sourceFingerprint } from "./source-fingerprint.mjs";

const url = process.env.BROWSER_SPEAK_URL ?? "http://127.0.0.1:5174/";
const chrome =
  process.env.CHROME_BIN ??
  (existsSync("/opt/google/chrome/chrome")
    ? "/opt/google/chrome/chrome"
    : existsSync("/usr/bin/google-chrome")
      ? "/usr/bin/google-chrome"
      : "chromium");
const resultPath = resolve(
  process.env.BROWSER_SPEAK_CLIENT_SIDE_JSON ?? `${tmpdir()}/browser-speak-client-side-smoke.json`,
);
const profileDir = resolve(
  process.env.BROWSER_SPEAK_CLIENT_SIDE_PROFILE_DIR ?? `${tmpdir()}/browser-speak-client-side-profile`,
);
const protocolTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_TIMEOUT_MS ?? 60000);
const pollTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_POLL_TIMEOUT_MS ?? 5000);
const pageUnresponsiveTimeoutMs = Number(process.env.BROWSER_SPEAK_PAGE_UNRESPONSIVE_TIMEOUT_MS ?? 120000);
const loadTimeoutMs = Number(process.env.BROWSER_SPEAK_LOAD_TIMEOUT_MS ?? 600000);
const taskTimeoutMs = Number(process.env.BROWSER_SPEAK_TASK_TIMEOUT_MS ?? 180000);
const voicePreloadTimeoutMs = Number(
  process.env.BROWSER_SPEAK_VOICE_PRELOAD_TIMEOUT_MS ?? Math.max(180000, taskTimeoutMs),
);
const reuseProfile = process.env.BROWSER_SPEAK_CLIENT_SIDE_REUSE_PROFILE === "true";
const ttsWarmup = process.env.BROWSER_SPEAK_TTS_WARMUP === "true";
const headless = process.env.BROWSER_SPEAK_HEADLESS !== "false";
const tasks = parseList(process.env.BROWSER_SPEAK_CLIENT_SIDE_TASKS ?? "tts,identity,loopback");
const stack = {
  device: process.env.BROWSER_SPEAK_DEVICE ?? "wasm",
  llm: process.env.BROWSER_SPEAK_LLM ?? "HuggingFaceTB/SmolLM2-135M-Instruct",
  asr: process.env.BROWSER_SPEAK_ASR ?? "onnx-community/moonshine-base-ONNX",
  voice: process.env.BROWSER_SPEAK_VOICE ?? "F2",
  ttsSteps: Number(process.env.BROWSER_SPEAK_TTS_STEPS ?? 2),
  vadSilenceMs: Number(process.env.BROWSER_SPEAK_VAD_SILENCE_MS ?? 480),
  partialAsr: process.env.BROWSER_SPEAK_PARTIAL_ASR !== "false",
  ttsWarmup,
};

async function main() {
  await ensureServer();
  await mkdir(dirname(resultPath), { recursive: true });
  if (!reuseProfile) await rm(profileDir, { recursive: true, force: true });

  const browser = launchBrowser(9346, profileDir);
  const recorder = new NetworkRecorder(url);
  let client = null;
  try {
    client = await connectToPage(9346, browser);
    client.on("Network.requestWillBeSent", (event) => recorder.requestWillBeSent(event));
    client.on("Network.responseReceived", (event) => recorder.responseReceived(event));
    client.on("Network.loadingFailed", (event) => recorder.loadingFailed(event));
    await client.call("Network.enable");
    await waitForBenchApi(client);

    recorder.markStage("load");
    await runPageTask(client, `window.browserSpeakBench.loadStack(${JSON.stringify(stack)})`, {
      label: "model load",
      timeoutMs: loadTimeoutMs,
    });
    await runPageTask(
      client,
      `Promise.all(${JSON.stringify(["F1", "F2", "M1", "M2"])}.map((voice) => window.browserSpeakBench.preloadVoice({ voice, timeoutMs: ${JSON.stringify(voicePreloadTimeoutMs)} })))`,
      { label: "voice preload", timeoutMs: voicePreloadTimeoutMs },
    );
    await runPageTask(client, "window.browserSpeakBench.clearResults()", {
      label: "clear restored benchmark rows",
      timeoutMs: 30000,
    });
    await sleep(1000);

    recorder.markStage("benchmark");
    for (const task of tasks) {
      await runBenchmarkTask(client, task);
    }
    await sleep(1500);
    const snapshot = await runPageTask(client, "window.browserSpeakBench.exportResults()", {
      label: "export results",
      timeoutMs: 30000,
    });

    const requests = recorder.requests();
    const benchmarkRequests = requests.filter((request) => request.stage === "benchmark");
    const suspects = benchmarkRequests.flatMap((request) =>
      classifyRequest(request).map((reason) => ({ reason, request })),
    );
    const benchmarkErrors = snapshot.results.filter((result) => result.error);
    const missingTasks = missingTaskRows(snapshot.results, tasks);
    const payload = {
      generatedAt: new Date().toISOString(),
      sourceFingerprint: await sourceFingerprint(),
      url,
      passed: suspects.length === 0 && benchmarkErrors.length === 0 && missingTasks.length === 0,
      config: {
        stack,
        tasks,
        loadTimeoutMs,
        taskTimeoutMs,
        voicePreloadTimeoutMs,
        protocolTimeoutMs,
        pollTimeoutMs,
        pageUnresponsiveTimeoutMs,
        reuseProfile,
        headless,
        chrome,
        extraChromeArgs: parseChromeArgs(),
      },
      summary: {
        totalRequests: requests.length,
        loadRequests: requests.filter((request) => request.stage === "load").length,
        benchmarkRequests: benchmarkRequests.length,
        benchmarkHosts: summarizeHosts(benchmarkRequests),
        benchmarkMethods: summarizeMethods(benchmarkRequests),
        serverInferenceSuspects: suspects.length,
        benchmarkErrors: benchmarkErrors.length,
        missingTasks,
      },
      suspects,
      benchmarkErrors,
      requests,
      browserExport: snapshot,
      browserExportMetadata: exportMetadata(snapshot),
      hostMetadata: snapshot.hostMetadata ?? null,
      benchmarkResults: snapshot.results,
      benchmarkSummary: snapshot.summary,
    };
    await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`);
    console.log(`Wrote client-side smoke JSON: ${resultPath}`);
    console.log(
      `Benchmark network requests: ${benchmarkRequests.length}, server-inference suspects: ${suspects.length}.`,
    );
    for (const suspect of suspects.slice(0, 8)) {
      console.log(`${suspect.reason}: ${suspect.request.method} ${suspect.request.url}`);
    }
    for (const errorRow of benchmarkErrors.slice(0, 8)) {
      console.log(`${errorRow.kind} benchmark error: ${errorRow.error}`);
    }
    for (const missingTask of missingTasks) {
      console.log(`Missing benchmark row for task: ${missingTask}`);
    }
    if (!payload.passed) process.exitCode = 1;
    await client.closeBrowser();
  } catch (error) {
    await writeFailurePayload(error, recorder, browser).catch((writeError) => {
      console.error(`Could not write client-side smoke failure JSON: ${writeError.message}`);
    });
    if (client) await client.closeBrowser().catch(() => {});
    throw error;
  } finally {
    await stopBrowser(browser, reuseProfile ? null : profileDir);
  }
}

async function writeFailurePayload(error, recorder, browser = null) {
  const requests = recorder.requests();
  const benchmarkRequests = requests.filter((request) => request.stage === "benchmark");
  const suspects = benchmarkRequests.flatMap((request) =>
    classifyRequest(request).map((reason) => ({ reason, request })),
  );
  const payload = {
    generatedAt: new Date().toISOString(),
    sourceFingerprint: await sourceFingerprint(),
    url,
    passed: false,
    error: error.stack ?? error.message ?? String(error),
    browserLog: browser?.browserLog ?? "",
    config: {
      stack,
      tasks,
      loadTimeoutMs,
      taskTimeoutMs,
      voicePreloadTimeoutMs,
      protocolTimeoutMs,
      pollTimeoutMs,
      pageUnresponsiveTimeoutMs,
      reuseProfile,
      headless,
      chrome,
      extraChromeArgs: parseChromeArgs(),
    },
    summary: {
      totalRequests: requests.length,
      loadRequests: requests.filter((request) => request.stage === "load").length,
      benchmarkRequests: benchmarkRequests.length,
      benchmarkHosts: summarizeHosts(benchmarkRequests),
      benchmarkMethods: summarizeMethods(benchmarkRequests),
      serverInferenceSuspects: suspects.length,
      benchmarkErrors: null,
      missingTasks: [],
    },
    suspects,
    requests,
  };
  await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`);
  console.log(`Wrote client-side smoke failure JSON: ${resultPath}`);
}

async function runBenchmarkTask(client, task) {
  const normalized = task.toLowerCase();
  const loopbackOptions = {
    timeoutMs: 180000,
    ...(process.env.BROWSER_SPEAK_LOOPBACK_TEXT ? { text: process.env.BROWSER_SPEAK_LOOPBACK_TEXT } : {}),
    ...(Number.isFinite(Number(process.env.BROWSER_SPEAK_LOOPBACK_SPEED))
      ? { speed: Number(process.env.BROWSER_SPEAK_LOOPBACK_SPEED) }
      : {}),
  };
  const expression = {
    tts: "window.browserSpeakBench.runTts({ timeoutMs: 90000 })",
    identity: "window.browserSpeakBench.runIdentity({ timeoutMs: 180000 })",
    chat: "window.browserSpeakBench.runChat({ timeoutMs: 180000 })",
    loopback: `window.browserSpeakBench.runLoopback(${JSON.stringify(loopbackOptions)})`,
    barge: "window.browserSpeakBench.runBargeIn({ timeoutMs: 90000 })",
    "barge-in": "window.browserSpeakBench.runBargeIn({ timeoutMs: 90000 })",
  }[normalized];
  if (!expression) throw new Error(`Unknown client-side smoke task: ${task}`);
  await runPageTask(client, expression, {
    label: `${normalized} benchmark`,
    timeoutMs: normalized === "loopback" ? Math.max(taskTimeoutMs, 240000) : taskTimeoutMs,
  });
}

function exportMetadata(snapshot) {
  if (!snapshot || typeof snapshot !== "object") return null;
  return {
    schemaVersion: snapshot.schemaVersion ?? null,
    exportId: snapshot.exportId ?? null,
    generatedAt: snapshot.generatedAt ?? null,
    hostMetadata: snapshot.hostMetadata ?? null,
    runtime: snapshot.runtime ?? null,
    evidence: snapshot.evidence ?? null,
    evidenceGuide: snapshot.evidenceGuide ?? null,
    summary: snapshot.summary ?? null,
    resultCount: Array.isArray(snapshot.results) ? snapshot.results.length : null,
  };
}

function missingTaskRows(results, requestedTasks) {
  const aliases = new Map([
    ["barge", "barge-in"],
    ["barge-in", "barge-in"],
  ]);
  const kinds = new Set(results.map((result) => result.kind));
  return requestedTasks
    .map((task) => task.toLowerCase())
    .map((task) => aliases.get(task) ?? task)
    .filter((kind, index, requestedKinds) => requestedKinds.indexOf(kind) === index)
    .filter((kind) => !kinds.has(kind));
}

function classifyRequest(request) {
  const reasons = [];
  const parsed = new URL(request.url);
  const host = parsed.hostname.toLowerCase();
  const path = parsed.pathname.toLowerCase();
  const method = request.method.toUpperCase();
  if (!["GET", "HEAD"].includes(method)) reasons.push(`non-asset HTTP method ${method}`);
  if (isForbiddenHost(host)) reasons.push(`forbidden inference host ${host}`);
  if (isInferencePath(path)) reasons.push(`inference-like path ${parsed.pathname}`);
  if (!isAllowedHost(host)) reasons.push(`unexpected host ${host}`);
  return reasons;
}

function isAllowedHost(host) {
  if (["localhost", "127.0.0.1", "::1"].includes(host)) return true;
  if (host === "cdn.jsdelivr.net") return true;
  if (host === "huggingface.co" || host.endsWith(".huggingface.co")) return true;
  if (host === "hf.co" || host.endsWith(".hf.co")) return true;
  if (host === "xethub.hf.co" || host.endsWith(".xethub.hf.co")) return true;
  return false;
}

function isForbiddenHost(host) {
  return /(^|\.)api-inference\.huggingface\.co$/.test(host) || /(^|\.)router\.hf\.co$/.test(host);
}

function isInferencePath(path) {
  return /\/(chat\/completions|completions|generate|predictions|queue\/(join|data)|run\/predict|api\/predict|v1\/)/.test(
    path,
  );
}

function summarizeHosts(requests) {
  return summarize(requests.map((request) => new URL(request.url).hostname.toLowerCase()));
}

function summarizeMethods(requests) {
  return summarize(requests.map((request) => request.method.toUpperCase()));
}

function summarize(values) {
  const counts = new Map();
  for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1);
  return Object.fromEntries([...counts.entries()].sort((a, b) => a[0].localeCompare(b[0])));
}

class NetworkRecorder {
  constructor(pageUrl) {
    this.pageOrigin = new URL(pageUrl).origin;
    this.stage = "startup";
    this.byId = new Map();
  }

  markStage(stage) {
    this.stage = stage;
  }

  requestWillBeSent(event) {
    const request = event.request;
    if (!request?.url || shouldIgnoreUrl(request.url)) return;
    this.byId.set(event.requestId, {
      stage: this.stage,
      requestId: event.requestId,
      url: request.url,
      method: request.method ?? "GET",
      resourceType: event.type ?? "",
      initiatorType: event.initiator?.type ?? "",
      hasPostData: Boolean(request.hasPostData),
      status: null,
      failed: false,
      failureText: "",
    });
  }

  responseReceived(event) {
    const item = this.byId.get(event.requestId);
    if (!item) return;
    item.status = event.response?.status ?? null;
    item.mimeType = event.response?.mimeType ?? "";
    item.fromDiskCache = Boolean(event.response?.fromDiskCache);
    item.fromServiceWorker = Boolean(event.response?.fromServiceWorker);
  }

  loadingFailed(event) {
    const item = this.byId.get(event.requestId);
    if (!item) return;
    item.failed = true;
    item.failureText = event.errorText ?? "";
  }

  requests() {
    return [...this.byId.values()].sort((a, b) => a.requestId.localeCompare(b.requestId));
  }
}

function shouldIgnoreUrl(rawUrl) {
  return /^(data|blob|devtools):/i.test(rawUrl);
}

async function ensureServer() {
  const response = await fetch(url).catch((error) => {
    throw new Error(`Could not reach ${url}: ${error.message}`);
  });
  if (!response.ok) throw new Error(`${url} returned HTTP ${response.status}`);
}

async function waitForBenchApi(client) {
  const deadline = Date.now() + 15000;
  while (Date.now() < deadline) {
    try {
      if (await client.evaluate("Boolean(window.browserSpeakBench)")) return;
    } catch {
      // The target may still be navigating.
    }
    await sleep(100);
  }
  throw new Error("window.browserSpeakBench was not installed.");
}

async function runPageTask(client, expression, { label = "page task", timeoutMs = 30000 } = {}) {
  const taskId = `task_${Date.now()}_${Math.random().toString(16).slice(2)}`;
  await client.evaluate(`(() => {
    const taskId = ${JSON.stringify(taskId)};
    window.__browserSpeakHarnessTasks ||= {};
    window.__browserSpeakHarnessTasks[taskId] = { done: false, label: ${JSON.stringify(label)} };
    Promise.resolve(${expression})
      .then((value) => {
        window.__browserSpeakHarnessTasks[taskId] = { done: true, value };
      })
      .catch((error) => {
        window.__browserSpeakHarnessTasks[taskId] = {
          done: true,
          error: error?.stack || error?.message || String(error),
        };
      });
    return true;
  })()`);
  const deadline = Date.now() + timeoutMs;
  let lastEvents = [];
  let lastPollError = "";
  let pollErrorSince = 0;
  while (Date.now() < deadline) {
    let task = null;
    try {
      task = await client.evaluate(
        `window.__browserSpeakHarnessTasks?.[${JSON.stringify(taskId)}] ?? null`,
        pollTimeoutMs,
      );
      lastPollError = "";
      pollErrorSince = 0;
    } catch (error) {
      const message = error.message ?? String(error);
      pollErrorSince ||= Date.now();
      if (message !== lastPollError) {
        console.log(`${label}: waiting for page response (${message})`);
        lastPollError = message;
      }
      if (Date.now() - pollErrorSince > pageUnresponsiveTimeoutMs) {
        throw new Error(
          `${label} page stayed unresponsive for ${(pageUnresponsiveTimeoutMs / 1000).toFixed(0)} seconds; last poll error: ${message}`,
        );
      }
      await sleep(500);
      continue;
    }
    if (task?.done) {
      await client
        .evaluate(`delete window.__browserSpeakHarnessTasks?.[${JSON.stringify(taskId)}]`, pollTimeoutMs)
        .catch(() => {});
      if (task.error) throw new Error(`${label} failed: ${task.error}`);
      return task.value;
    }
    const snapshot = await client
      .evaluate(`(() => {
        const state = window.browserSpeakBench?.state?.();
        return state ? {
          modelsLoaded: state.modelsLoaded,
          modelsLoading: state.modelsLoading,
          activeBenchmark: state.activeBenchmark?.kind ?? null,
          events: state.events?.slice(0, 3) ?? [],
        } : null;
      })()`, pollTimeoutMs)
      .catch(() => null);
    const events = snapshot?.events ?? [];
    if (events.join("\\n") !== lastEvents.join("\\n")) {
      lastEvents = events;
      if (events[0]) console.log(`${label}: ${events[0]}`);
    }
    await sleep(500);
  }
  throw new Error(`${label} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`);
}

function launchBrowser(port, profileDir) {
  const child = spawn(
    chrome,
    [
      ...(headless ? ["--headless=new"] : []),
      "--no-sandbox",
      "--disable-gpu",
      "--disable-dev-shm-usage",
      "--disable-background-networking",
      "--disable-extensions",
      "--no-default-browser-check",
      "--no-first-run",
      "--autoplay-policy=no-user-gesture-required",
      `--remote-debugging-port=${port}`,
      `--user-data-dir=${profileDir}`,
      ...parseChromeArgs(),
    ],
    { stdio: ["ignore", "pipe", "pipe"] },
  );
  child.browserLog = "";
  const appendLog = (chunk) => {
    child.browserLog = `${child.browserLog}${chunk}`;
    if (child.browserLog.length > 8000) child.browserLog = child.browserLog.slice(-8000);
  };
  child.stdout.on("data", appendLog);
  child.stderr.on("data", appendLog);
  return child;
}

async function stopBrowser(child, profileDir) {
  if (child.exitCode == null) child.kill("SIGTERM");
  await new Promise((resolve) => {
    child.once("exit", resolve);
    setTimeout(resolve, 3000);
  });
  if (child.exitCode == null) child.kill("SIGKILL");
  if (!profileDir) return;
  for (let attempt = 0; attempt < 5; attempt += 1) {
    try {
      await rm(profileDir, { recursive: true, force: true });
      return;
    } catch (error) {
      if (attempt === 4) {
        console.warn(`Could not remove ${profileDir}: ${error.message}`);
        return;
      }
      await sleep(500);
    }
  }
}

async function connectToPage(port, child) {
  const deadline = Date.now() + 60000;
  let lastError = null;
  while (Date.now() < deadline) {
    if (child.exitCode != null) {
      throw new Error(`Chrome exited before DevTools became available.\n${child.browserLog}`);
    }
    try {
      const version = await fetch(`http://127.0.0.1:${port}/json/version`).then((response) => response.json());
      if (version.webSocketDebuggerUrl) {
        const page = await createPageTarget(port);
        return new CdpClient(page.webSocketDebuggerUrl);
      }
    } catch (error) {
      lastError = error;
    }
    await sleep(250);
  }
  throw new Error(
    `Could not connect to Chrome DevTools on port ${port}: ${lastError?.message ?? "unknown error"}\n${child.browserLog}`,
  );
}

async function createPageTarget(port) {
  for (const method of ["PUT", "GET"]) {
    const response = await fetch(`http://127.0.0.1:${port}/json/new?${encodeURIComponent(url)}`, {
      method,
    }).catch(() => null);
    if (response?.ok) {
      const target = await response.json();
      if (target.webSocketDebuggerUrl) return target;
    }
  }
  const targets = await fetch(`http://127.0.0.1:${port}/json`).then((response) => response.json());
  const page = targets.find((target) => target.type === "page" && target.url === url);
  if (page?.webSocketDebuggerUrl) return page;
  throw new Error("Could not create or find a page target.");
}

class CdpClient {
  constructor(webSocketUrl) {
    this.nextId = 1;
    this.pending = new Map();
    this.listeners = new Map();
    this.socket = new WebSocket(webSocketUrl);
    this.opened = new Promise((resolve, reject) => {
      this.socket.onopen = resolve;
      this.socket.onerror = reject;
      this.socket.onmessage = (event) => this.onMessage(event);
    });
  }

  on(method, handler) {
    if (!this.listeners.has(method)) this.listeners.set(method, new Set());
    this.listeners.get(method).add(handler);
  }

  onMessage(event) {
    const message = JSON.parse(event.data);
    if (message.id && this.pending.has(message.id)) {
      const { resolve: onResolve, reject } = this.pending.get(message.id);
      this.pending.delete(message.id);
      if (message.error) reject(new Error(message.error.message));
      else onResolve(message.result);
      return;
    }
    if (message.method && this.listeners.has(message.method)) {
      for (const handler of this.listeners.get(message.method)) handler(message.params ?? {});
    }
  }

  async call(method, params = {}, timeoutMs = protocolTimeoutMs) {
    await this.opened;
    const id = this.nextId++;
    this.socket.send(JSON.stringify({ id, method, params }));
    return new Promise((resolvePromise, reject) => {
      const timer = setTimeout(() => {
        this.pending.delete(id);
        reject(new Error(`${method} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`));
      }, timeoutMs);
      this.pending.set(id, {
        resolve: (value) => {
          clearTimeout(timer);
          resolvePromise(value);
        },
        reject: (error) => {
          clearTimeout(timer);
          reject(error);
        },
      });
    });
  }

  async evaluate(expression, timeoutMs = protocolTimeoutMs) {
    const result = await this.call(
      "Runtime.evaluate",
      {
        expression,
        awaitPromise: true,
        returnByValue: true,
      },
      timeoutMs,
    );
    if (result.exceptionDetails) throw new Error(formatException(result.exceptionDetails));
    return result.result.value;
  }

  async closeBrowser() {
    await this.call("Browser.close", {}, 5000).catch(() => {});
    try {
      this.socket.close();
    } catch {
      // Ignore close races.
    }
  }
}

function formatException(exceptionDetails) {
  const exception = exceptionDetails.exception;
  return exception?.description ?? exception?.value ?? exceptionDetails.text ?? "Evaluation failed.";
}

function parseList(value) {
  return String(value ?? "")
    .split(",")
    .map((item) => item.trim())
    .filter(Boolean);
}

function parseChromeArgs() {
  const raw = process.env.BROWSER_SPEAK_CHROME_ARGS ?? "";
  return (
    raw
      .match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g)
      ?.map((arg) => arg.replace(/^["']|["']$/g, "")) ?? []
  );
}

function sleep(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

main().catch((error) => {
  console.error(error.stack ?? error.message);
  process.exitCode = 1;
});