tools/run-loopback-series.mjs

#!/usr/bin/env node
import { spawn } from "node:child_process";
import { readFile, writeFile, mkdir } from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { sourceFingerprint } from "./source-fingerprint.mjs";

const scriptDir = dirname(fileURLToPath(import.meta.url));
const resultPath = resolve(process.env.BROWSER_SPEAK_LOOPBACK_JSON ?? `${tmpdir()}/browser-speak-loopback-series.json`);
const rawPath = resolve(
  process.env.BROWSER_SPEAK_LOOPBACK_RAW_JSON ?? `${tmpdir()}/browser-speak-loopback-series-raw.json`,
);
const count = Math.max(1, Number(process.env.BROWSER_SPEAK_LOOPBACK_COUNT ?? 3));
const speed = Number(process.env.BROWSER_SPEAK_LOOPBACK_SPEED ?? 1.0);
const text = process.env.BROWSER_SPEAK_LOOPBACK_TEXT ?? "Identify this browser demo.";
const stack = {
  device: process.env.BROWSER_SPEAK_DEVICE ?? "wasm",
  llm: process.env.BROWSER_SPEAK_LLM ?? "HuggingFaceTB/SmolLM2-135M-Instruct",
  asr: process.env.BROWSER_SPEAK_ASR ?? "onnx-community/moonshine-base-ONNX",
  voice: process.env.BROWSER_SPEAK_VOICE ?? "F2",
  ttsSteps: Number(process.env.BROWSER_SPEAK_TTS_STEPS ?? 2),
  vadSilenceMs: Number(process.env.BROWSER_SPEAK_VAD_SILENCE_MS ?? 480),
  partialAsr: process.env.BROWSER_SPEAK_PARTIAL_ASR !== "false",
};

async function main() {
  await mkdir(dirname(resultPath), { recursive: true });
  await mkdir(dirname(rawPath), { recursive: true });
  const env = {
    ...process.env,
    BROWSER_SPEAK_LOCAL_JSON: rawPath,
    BROWSER_SPEAK_LOCAL_STACKS: JSON.stringify([stack]),
    BROWSER_SPEAK_LOCAL_MAX_STACKS: "1",
    BROWSER_SPEAK_LOCAL_PROFILE_DIR:
      process.env.BROWSER_SPEAK_LOCAL_PROFILE_DIR ?? `${tmpdir()}/browser-speak-loopback-series-profile`,
    BROWSER_SPEAK_LOCAL_TASKS: Array.from({ length: count }, () => "loopback").join(","),
    BROWSER_SPEAK_LOAD_TIMEOUT_MS: process.env.BROWSER_SPEAK_LOAD_TIMEOUT_MS ?? "900000",
    BROWSER_SPEAK_TASK_TIMEOUT_MS: process.env.BROWSER_SPEAK_TASK_TIMEOUT_MS ?? "240000",
    BROWSER_SPEAK_LOOPBACK_SPEED: String(speed),
    BROWSER_SPEAK_LOOPBACK_TEXT: text,
  };
  const exitCode = await runChild(process.execPath, [resolve(scriptDir, "run-local-candidate-benchmark.mjs")], env);
  const raw = await readJson(rawPath).catch(() => null);
  const payload = await summarize(raw, exitCode);
  await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`);
  console.log(`Wrote loopback series JSON: ${resultPath}`);
  printSummary(payload);
  if (!payload.passed) process.exitCode = exitCode || 1;
}

function runChild(command, args, env) {
  return new Promise((resolvePromise) => {
    const child = spawn(command, args, { env, stdio: "inherit" });
    child.on("exit", (code) => resolvePromise(code ?? 1));
  });
}

async function readJson(path) {
  return JSON.parse(await readFile(path, "utf8"));
}

async function summarize(raw, exitCode) {
  const rows = raw?.results?.filter((row) => row.kind === "loopback") ?? [];
  const completed = rows.filter((row) => !row.error);
  const errors = rows.filter((row) => row.error);
  const transcripts = completed.map((row) => row.transcript ?? "");
  return {
    generatedAt: new Date().toISOString(),
    sourceFingerprint: await sourceFingerprint(),
    passed: exitCode === 0 && rows.length >= count && errors.length === 0,
    rawPath,
    config: {
      count,
      text,
      speed,
      stack,
      taskTimeoutMs: Number(process.env.BROWSER_SPEAK_TASK_TIMEOUT_MS ?? 240000),
      url: process.env.BROWSER_SPEAK_URL ?? "http://127.0.0.1:5174/",
    },
    summary: {
      requestedRuns: count,
      producedRuns: rows.length,
      completedRuns: completed.length,
      errorRuns: errors.length,
      exactTranscriptRuns: completed.filter((row) => row.sttWer === 0).length,
      medianWer: median(completed.map((row) => row.sttWer)),
      medianCer: median(completed.map((row) => row.sttCer)),
      medianAsrMs: median(completed.map((row) => row.asrMs)),
      medianFirstTokenMs: median(completed.map((row) => row.firstTokenMs)),
      medianFirstAudioMs: median(completed.map((row) => row.firstAudioMs)),
      medianSpeechEndToFirstAudioMs: median(completed.map((row) => row.speechEndToFirstAudioMs)),
      medianSpeechEndToAudioEndMs: median(completed.map((row) => row.speechEndToAudioEndMs)),
      identityPasses: completed.filter((row) => row.llmQualityPass).length,
      transcripts,
      errors: errors.map((row) => `${row.id ?? "?"}: ${row.error}`),
    },
    rows,
    rawSummary: raw?.summary ?? null,
    rawCandidates: raw?.candidates ?? [],
  };
}

function median(values) {
  const finite = values.filter(Number.isFinite).sort((a, b) => a - b);
  if (finite.length === 0) return null;
  const middle = Math.floor(finite.length / 2);
  if (finite.length % 2 === 1) return finite[middle];
  return (finite[middle - 1] + finite[middle]) / 2;
}

function printSummary(payload) {
  const summary = payload.summary;
  console.log(
    [
      `${summary.completedRuns}/${summary.requestedRuns} completed`,
      `${summary.exactTranscriptRuns}/${summary.completedRuns} exact transcripts`,
      `median WER ${formatPercent(summary.medianWer)}`,
      `median speech-end-to-audio ${formatMs(summary.medianSpeechEndToFirstAudioMs)}`,
    ].join(", "),
  );
  if (summary.transcripts.length > 0) {
    console.log(`Transcripts: ${summary.transcripts.map((text) => JSON.stringify(text)).join(", ")}`);
  }
  for (const error of summary.errors) console.log(`Loopback error: ${error}`);
}

function formatMs(value) {
  if (!Number.isFinite(value)) return "-";
  if (value < 1000) return `${Math.round(value)} ms`;
  return `${(value / 1000).toFixed(2)} s`;
}

function formatPercent(value) {
  if (!Number.isFinite(value)) return "-";
  return `${Math.round(value * 100)}%`;
}

main().catch((error) => {
  console.error(error.stack ?? error.message);
  process.exitCode = 1;
});