tools/audit-browser-evidence.mjs

#!/usr/bin/env node
import { readFile, writeFile, mkdir } from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, resolve } from "node:path";
import { sourceFingerprint } from "./source-fingerprint.mjs";

const rawInputPath = process.argv[2] ?? process.env.BROWSER_SPEAK_BROWSER_EXPORT_JSON ?? "";
const inputPath = rawInputPath ? resolve(rawInputPath) : "";
const resultPath = resolve(
  process.env.BROWSER_SPEAK_BROWSER_AUDIT_JSON ?? `${tmpdir()}/browser-speak-browser-evidence-audit.json`,
);
const requireRealMic = process.env.BROWSER_SPEAK_BROWSER_AUDIT_REQUIRE_REAL_MIC !== "false";
const requireHardwareWebgpu = process.env.BROWSER_SPEAK_BROWSER_AUDIT_REQUIRE_HARDWARE_WEBGPU !== "false";
const minRealMicRows = Math.max(1, Number(process.env.BROWSER_SPEAK_BROWSER_AUDIT_REAL_MIC_ROWS ?? 3));
const realMicMaxWer = Number(process.env.BROWSER_SPEAK_BROWSER_AUDIT_REAL_MIC_MAX_WER ?? 0.25);
const expectedHostSuffix = process.env.BROWSER_SPEAK_BROWSER_AUDIT_HOST_SUFFIX ?? ".hf.space";

async function main() {
  if (!inputPath) {
    throw new Error(
      "Pass a downloaded benchmark JSON path, or set BROWSER_SPEAK_BROWSER_EXPORT_JSON.",
    );
  }
  const exportPayload = JSON.parse(await readFile(inputPath, "utf8"));
  const browserEvidence = evidenceSummaryFromPayload(exportPayload);
  const checks = [
    checkExportShape(exportPayload),
    checkSelfDescription(exportPayload),
    checkBrowserEvidenceSummary(exportPayload),
    checkHostedMetadata(exportPayload),
    checkRealMic(exportPayload),
    checkHardwareWebgpu(exportPayload),
  ];
  const requiredChecks = checks.filter((check) => check.required);
  const passed = requiredChecks.every((check) => check.status === "pass");
  const payload = {
    generatedAt: new Date().toISOString(),
    sourceFingerprint: await sourceFingerprint(),
    passed,
    inputPath,
    config: {
      requireRealMic,
      requireHardwareWebgpu,
      minRealMicRows,
      realMicMaxWer,
      expectedHostSuffix,
    },
    browserEvidence,
    checks,
    limitations: [
      "Downloaded browser benchmark JSON can prove the page-reported runtime, real-mic rows, and WebGPU adapter metadata.",
      "It cannot prove the absence of server inference by itself; pair it with tools/run-hosted-smoke.mjs or tools/run-client-side-smoke.mjs network evidence.",
      "It cannot prove Chrome was not launched with fake microphone capture flags; use tools/run-real-mic-series.mjs when that provenance matters.",
    ],
  };
  await mkdir(dirname(resultPath), { recursive: true });
  await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`);
  console.log(`Wrote browser evidence audit JSON: ${resultPath}`);
  for (const check of checks) {
    console.log(`${check.required ? "required" : "supporting"} ${check.status}: ${check.name} - ${check.message}`);
  }
  if (!passed) process.exitCode = 1;
}

function checkExportShape(payload) {
  const rows = rowsFromPayload(payload);
  const hostMetadata = hostMetadataFromPayload(payload);
  const passed =
    typeof generatedAtFromPayload(payload) === "string" &&
    summaryFromPayload(payload) &&
    typeof summaryFromPayload(payload) === "object" &&
    Array.isArray(rows);
  return {
    name: "browser export shape",
    required: true,
    status: passed ? "pass" : "fail",
    message: passed
      ? `Export has ${rows.length} benchmark row(s).`
      : "Export is missing generatedAt, summary, or benchmark rows.",
    evidence: {
      generatedAt: generatedAtFromPayload(payload),
      schemaVersion: schemaVersionFromPayload(payload),
      exportId: exportIdFromPayload(payload),
      rowCount: rows.length,
      summaryScopes: summaryFromPayload(payload) ? Object.keys(summaryFromPayload(payload)) : [],
      hostMetadata,
      runtimePresent: Boolean(runtimeFromPayload(payload)),
      evidenceSummaryPresent: Boolean(evidenceSummaryFromPayload(payload)),
    },
  };
}

function checkSelfDescription(payload) {
  const schemaVersion = schemaVersionFromPayload(payload);
  const exportId = exportIdFromPayload(payload);
  const guide = evidenceGuideFromPayload(payload);
  const realMic = guide?.requirements?.realMic ?? {};
  const hardwareWebgpu = guide?.requirements?.hardwareWebgpu ?? {};
  const passed =
    typeof schemaVersion === "string" &&
    schemaVersion.startsWith("browser-speak-benchmarks/") &&
    typeof exportId === "string" &&
    exportId.length > 12 &&
    typeof guide?.auditCommand === "string" &&
    guide.auditCommand.includes("audit-browser-evidence.mjs") &&
    realMic.requiredRows >= minRealMicRows &&
    hardwareWebgpu.requiredRows >= 1;
  return {
    name: "browser export self-description",
    required: true,
    status: passed ? "pass" : "fail",
    message: passed
      ? `Export ${exportId} declares ${schemaVersion} and embeds audit requirements.`
      : "Export is missing schemaVersion, exportId, or embedded audit requirements.",
    evidence: {
      schemaVersion,
      exportId,
      auditCommand: guide?.auditCommand ?? null,
      realMic,
      hardwareWebgpu,
      nextActions: guide?.nextActions ?? [],
    },
  };
}

function checkBrowserEvidenceSummary(payload) {
  const evidence = evidenceSummaryFromPayload(payload);
  if (!evidence) {
    return {
      name: "browser evidence summary",
      required: false,
      status: "missing",
      message: "Export does not include the page-generated evidence summary.",
      evidence: {},
    };
  }
  return {
    name: "browser evidence summary",
    required: false,
    status: "pass",
    message: summaryEvidenceMessage(evidence),
    evidence,
  };
}

function checkHostedMetadata(payload) {
  const hostMetadata = hostMetadataFromPayload(payload);
  const evidence = evidenceSummaryFromPayload(payload)?.hosted ?? null;
  const host = hostMetadata.host ?? "";
  const hosted = expectedHostSuffix ? host.endsWith(expectedHostSuffix) : Boolean(host);
  const commit = hostMetadata.hfSpaceCommit ?? "";
  const passed = hosted && commit.length >= 7;
  return {
    name: "hosted Space metadata",
    required: true,
    status: passed ? "pass" : "fail",
    message: passed
      ? `Export came from ${host} at Space commit ${commit}.`
      : "Export is missing hosted Space host or commit metadata.",
    evidence: { hostMetadata, expectedHostSuffix, browserEvidence: evidence },
  };
}

function checkRealMic(payload) {
  if (!requireRealMic) return skipped("real microphone rows", "Real microphone requirement disabled.");
  const evidence = evidenceSummaryFromPayload(payload)?.realMic ?? null;
  const rows = completedRows(payload).filter((row) => row.kind === "mic");
  const exactRows = rows.filter((row) => row.sttWer === 0).length;
  const identityPasses = rows.filter((row) => row.llmQualityPass === true).length;
  const inputRows = rows.filter(hasRealMicInputStats).length;
  const trackSettingsRows = rows.filter(hasInputTrackSettings).length;
  const medianWer = median(rows.map((row) => row.sttWer));
  const medianSpeechEndToFirstAudioMs = median(rows.map((row) => row.speechEndToFirstAudioMs));
  const passed =
    rows.length >= minRealMicRows &&
    inputRows >= minRealMicRows &&
    identityPasses >= minRealMicRows &&
    Number.isFinite(medianWer) &&
    medianWer <= realMicMaxWer &&
    Number.isFinite(medianSpeechEndToFirstAudioMs);
  return {
    name: "real microphone rows",
    required: true,
    status: passed ? "pass" : "missing",
    message: passed
      ? `${rows.length} completed real-mic row(s), median WER ${formatPercent(
          medianWer,
        )}, speech end to first audio ${formatMs(medianSpeechEndToFirstAudioMs)}.`
      : `Need ${minRealMicRows} completed real-mic row(s) with mic input stats, identity pass, median WER <= ${formatPercent(
          realMicMaxWer,
        )}, and finite speech-end-to-audio latency.`,
    evidence: {
      rows: rows.length,
      exactRows,
      identityPasses,
      inputRows,
      trackSettingsRows,
      medianWer,
      medianSpeechEndToFirstAudioMs,
      rowSummaries: rows.slice(0, 8).map((row) => ({
        id: row.id,
        sttWer: row.sttWer,
        transcript: row.transcript,
        llmQualityPass: row.llmQualityPass,
        speechEndToFirstAudioMs: row.speechEndToFirstAudioMs,
        micInputChunks: row.micInputChunks,
        micInputPeak: row.micInputPeak,
        inputSampleRate: row.inputSampleRate ?? row.stack?.environment?.inputSampleRate ?? null,
        inputTrackSettings: row.inputTrackSettings ?? row.stack?.environment?.inputTrackSettings ?? null,
      })),
      browserEvidence: evidence,
    },
  };
}

function checkHardwareWebgpu(payload) {
  if (!requireHardwareWebgpu) return skipped("hardware WebGPU rows", "Hardware WebGPU requirement disabled.");
  const evidence = evidenceSummaryFromPayload(payload)?.hardwareWebgpu ?? null;
  const rows = completedRows(payload).filter((row) => row.stack?.device === "webgpu");
  const hardwareRows = rows.filter((row) => {
    const environment = row.stack?.environment ?? {};
    return (
      environment.webgpuAvailable === true &&
      environment.webgpuSoftwareAdapter !== true &&
      !isSoftwareWebGpuAdapter(environment.webgpuAdapter)
    );
  });
  const passed = hardwareRows.length > 0;
  return {
    name: "hardware WebGPU rows",
    required: true,
    status: passed ? "pass" : "missing",
    message: passed
      ? `${hardwareRows.length} completed row(s) used hardware WebGPU.`
      : "No completed benchmark rows prove hardware WebGPU execution.",
    evidence: {
      webgpuRows: rows.length,
      hardwareRows: hardwareRows.length,
      adapters: uniqueAdapters(rows),
      rowKinds: rows.map((row) => row.kind),
      browserEvidence: evidence,
    },
  };
}

function generatedAtFromPayload(payload) {
  return payload?.generatedAt ?? payload?.browserExport?.generatedAt ?? null;
}

function schemaVersionFromPayload(payload) {
  return payload?.schemaVersion ?? payload?.browserExport?.schemaVersion ?? null;
}

function exportIdFromPayload(payload) {
  return payload?.exportId ?? payload?.browserExport?.exportId ?? null;
}

function evidenceGuideFromPayload(payload) {
  return payload?.evidenceGuide ?? payload?.browserExport?.evidenceGuide ?? null;
}

function summaryFromPayload(payload) {
  return payload?.summary ?? payload?.browserExport?.summary ?? payload?.benchmarkSummary ?? null;
}

function hostMetadataFromPayload(payload) {
  return (
    payload?.hostMetadata ??
    payload?.browserExport?.hostMetadata ??
    payload?.runtime?.hostMetadata ??
    payload?.browserExport?.runtime?.hostMetadata ??
    {}
  );
}

function runtimeFromPayload(payload) {
  return payload?.runtime ?? payload?.browserExport?.runtime ?? null;
}

function evidenceSummaryFromPayload(payload) {
  return payload?.evidence ?? payload?.browserExport?.evidence ?? null;
}

function rowsFromPayload(payload) {
  if (Array.isArray(payload?.results)) return payload.results;
  if (Array.isArray(payload?.benchmarkResults)) return payload.benchmarkResults;
  if (Array.isArray(payload?.browserExport?.results)) return payload.browserExport.results;
  return [];
}

function completedRows(payload) {
  return rowsFromPayload(payload).filter((row) => !row.error);
}

function hasRealMicInputStats(row) {
  return (
    Number.isFinite(row.micInputChunks) &&
    row.micInputChunks > 0 &&
    Number.isFinite(row.micInputPeak) &&
    row.micInputPeak > 0 &&
    Number.isFinite(row.inputSampleRate ?? row.stack?.environment?.inputSampleRate)
  );
}

function hasInputTrackSettings(row) {
  const settings = row.inputTrackSettings ?? row.stack?.environment?.inputTrackSettings;
  return Boolean(settings && typeof settings === "object" && Object.keys(settings).length > 0);
}

function uniqueAdapters(rows) {
  const seen = new Set();
  const adapters = [];
  for (const row of rows) {
    const adapter = row.stack?.environment?.webgpuAdapter ?? null;
    const label = formatAdapter(adapter);
    if (seen.has(label)) continue;
    seen.add(label);
    adapters.push({
      label,
      softwareAdapter: row.stack?.environment?.webgpuSoftwareAdapter ?? null,
      adapter,
    });
  }
  return adapters;
}

function isSoftwareWebGpuAdapter(adapter) {
  const text = [adapter?.vendor, adapter?.architecture, adapter?.device, adapter?.description]
    .filter(Boolean)
    .join(" ")
    .toLowerCase();
  return /\b(swiftshader|llvmpipe|software rasterizer|software adapter|warp)\b/.test(text);
}

function formatAdapter(adapter) {
  if (!adapter) return "unknown adapter";
  return (
    [adapter.vendor, adapter.architecture, adapter.device, adapter.description]
      .filter(Boolean)
      .join(" / ") || "unknown adapter"
  );
}

function skipped(name, message) {
  return { name, required: false, status: "skip", message, evidence: {} };
}

function summaryEvidenceMessage(evidence) {
  const parts = [];
  if (evidence.realMic) {
    parts.push(
      `real mic ${evidence.realMic.rows ?? 0}/${evidence.realMic.requiredRows ?? minRealMicRows}`,
    );
  }
  if (evidence.hardwareWebgpu) {
    parts.push(`hardware WebGPU rows ${evidence.hardwareWebgpu.hardwareRows ?? 0}`);
  }
  if (evidence.hosted) {
    parts.push(evidence.hosted.passed ? `hosted ${evidence.hosted.hfSpaceCommit ?? ""}` : "hosted missing");
  }
  return parts.length > 0 ? `Browser summary: ${parts.join(", ")}.` : "Browser evidence summary is present.";
}

function median(values) {
  const finite = values.filter(Number.isFinite).sort((a, b) => a - b);
  if (finite.length === 0) return null;
  const middle = Math.floor(finite.length / 2);
  if (finite.length % 2 === 1) return finite[middle];
  return (finite[middle - 1] + finite[middle]) / 2;
}

function formatMs(value) {
  if (!Number.isFinite(value)) return "-";
  if (value < 1000) return `${Math.round(value)} ms`;
  return `${(value / 1000).toFixed(2)} s`;
}

function formatPercent(value) {
  if (!Number.isFinite(value)) return "-";
  return `${Math.round(value * 100)}%`;
}

main().catch((error) => {
  console.error(error.stack ?? error.message);
  process.exitCode = 1;
});