tools/summarize-evidence.mjs

#!/usr/bin/env node
import { readFile, writeFile, mkdir } from "node:fs/promises";
import { existsSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, resolve } from "node:path";
import { sourceFingerprint } from "./source-fingerprint.mjs";

const paths = {
  finalValidation: resolve(
    process.env.BROWSER_SPEAK_FINAL_JSON ?? `${tmpdir()}/browser-speak-final-validation.json`,
  ),
  validationAudit: resolve(
    process.env.BROWSER_SPEAK_AUDIT_JSON ?? `${tmpdir()}/browser-speak-validation-audit.json`,
  ),
  hostedSmoke: resolve(
    process.env.BROWSER_SPEAK_HOSTED_SMOKE_JSON ?? `${tmpdir()}/browser-speak-hosted-smoke.json`,
  ),
  browserEvidenceAudit: resolve(
    process.env.BROWSER_SPEAK_BROWSER_AUDIT_JSON ?? `${tmpdir()}/browser-speak-browser-evidence-audit.json`,
  ),
  realMic: resolve(process.env.BROWSER_SPEAK_REAL_MIC_JSON ?? `${tmpdir()}/browser-speak-real-mic-series.json`),
  webgpu: resolve(process.env.BROWSER_SPEAK_WEBGPU_JSON ?? `${tmpdir()}/browser-speak-webgpu-results.json`),
};
const resultPath = resolve(
  process.env.BROWSER_SPEAK_EVIDENCE_SUMMARY_JSON ?? `${tmpdir()}/browser-speak-evidence-summary.json`,
);

async function main() {
  const currentSourceFingerprint = await sourceFingerprint();
  const artifacts = Object.fromEntries(
    await Promise.all(Object.entries(paths).map(async ([key, path]) => [key, await readArtifact(path, currentSourceFingerprint)])),
  );
  const checks = [
    ...validationChecks(artifacts.validationAudit.value ?? artifacts.finalValidation.value?.audit),
    hostedSmokeCheck(artifacts.hostedSmoke.value),
    browserEvidenceCheck(artifacts.browserEvidenceAudit.value),
    realMicArtifactCheck(artifacts.realMic.value),
    webgpuArtifactCheck(artifacts.webgpu.value),
    freshnessCheck(artifacts),
  ];
  const requiredChecks = dedupeChecks(checks).filter((check) => check.required);
  const passed = requiredChecks.every((check) => check.status === "pass");
  const payload = {
    generatedAt: new Date().toISOString(),
    sourceFingerprint: currentSourceFingerprint,
    passed,
    artifacts: Object.fromEntries(Object.entries(artifacts).map(([key, artifact]) => [key, summarizeArtifact(artifact)])),
    checks: requiredChecks,
    nextActions: nextActions(requiredChecks),
  };
  await mkdir(dirname(resultPath), { recursive: true });
  await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`);
  console.log(`Wrote evidence summary JSON: ${resultPath}`);
  console.log(`Source fingerprint: ${currentSourceFingerprint.hash}`);
  for (const check of requiredChecks) {
    console.log(`${check.status}: ${check.name} - ${check.message}`);
  }
  if (!passed) {
    console.log("Next actions:");
    for (const action of payload.nextActions) console.log(`- ${action}`);
  }
  if (!passed) process.exitCode = 1;
}

function validationChecks(audit) {
  const checks = Array.isArray(audit?.checks)
    ? audit.checks
    : Array.isArray(audit?.required)
      ? audit.required.map((check) => ({ ...check, required: true }))
      : [];
  return checks
    .filter((check) => check.required !== false)
    .map((check) => ({
      name: check.name,
      required: true,
      status: check.status ?? "unknown",
      message: check.message ?? "",
      source: "validation audit",
    }));
}

function hostedSmokeCheck(hostedSmoke) {
  if (!hostedSmoke) {
    return {
      name: "hosted Space no-server smoke",
      required: true,
      status: "missing",
      message: "Hosted smoke JSON is missing.",
      source: "hosted smoke",
    };
  }
  const passed =
    hostedSmoke.passed === true &&
    hostedSmoke.commitMatch === true &&
    hostedSmoke.noServerSmoke === true &&
    hostedSmoke.clientSideSummary?.benchmarkRequestCount === 0 &&
    hostedSmoke.clientSideSummary?.serverInferenceSuspectCount === 0;
  return {
    name: "hosted Space no-server smoke",
    required: true,
    status: passed ? "pass" : "fail",
    message: passed
      ? `Hosted Space commit ${
          hostedSmoke.hfInfo?.sha ?? hostedSmoke.hubApiInfo?.sha ?? "unknown"
        } passed UI and no-server smoke.`
      : "Hosted Space smoke did not prove the deployed no-server path.",
    source: "hosted smoke",
    evidence: {
      spaceSha: hostedSmoke.hfInfo?.sha ?? null,
      hubApiSha: hostedSmoke.hubApiInfo?.sha ?? null,
      hostCommit: hostedSmoke.hostHead?.hfSpaceCommit ?? null,
      commitMatch: hostedSmoke.commitMatch ?? null,
      spaceConfigParsed: hostedSmoke.spaceConfigParsed ?? null,
      runtimeStage: hostedSmoke.hubApiInfo?.runtime?.stage ?? hostedSmoke.hfInfo?.runtime?.stage ?? null,
      noServerSmoke: hostedSmoke.noServerSmoke ?? null,
      benchmarkRequestCount: hostedSmoke.clientSideSummary?.benchmarkRequestCount ?? null,
      serverInferenceSuspectCount: hostedSmoke.clientSideSummary?.serverInferenceSuspectCount ?? null,
      hostMetadata: hostedSmoke.clientSideSummary?.hostMetadata ?? null,
    },
  };
}

function browserEvidenceCheck(audit) {
  if (!audit) {
    return {
      name: "downloaded browser evidence audit",
      required: true,
      status: "missing",
      message: "Browser-downloaded benchmark evidence has not been audited.",
      source: "browser evidence audit",
    };
  }
  return {
    name: "downloaded browser evidence audit",
    required: true,
    status: audit.passed === true ? "pass" : "missing",
    message:
      audit.passed === true
        ? "Downloaded browser evidence meets real-mic and hardware-WebGPU gates."
        : "Downloaded browser evidence is missing real-mic rows, hardware-WebGPU rows, or hosted metadata.",
    source: "browser evidence audit",
    evidence: {
      inputPath: audit.inputPath ?? null,
      checks: audit.checks?.map(({ name, status, message }) => ({ name, status, message })) ?? [],
    },
  };
}

function realMicArtifactCheck(realMic) {
  if (!realMic) {
    return {
      name: "real microphone artifact",
      required: true,
      status: "missing",
      message: "Real microphone artifact is missing.",
      source: "real mic",
    };
  }
  return {
    name: "real microphone artifact",
    required: true,
    status: realMic.passed === true ? "pass" : "fail",
    message: realMic.passed === true ? "Real microphone artifact passed." : "Real microphone artifact did not pass.",
    source: "real mic",
    evidence: {
      generatedAt: realMic.generatedAt ?? null,
      summary: realMic.summary ?? null,
      config: realMic.config ?? null,
    },
  };
}

function webgpuArtifactCheck(webgpu) {
  if (!webgpu) {
    return {
      name: "hardware WebGPU artifact",
      required: true,
      status: "missing",
      message: "WebGPU benchmark artifact is missing.",
      source: "webgpu",
    };
  }
  const completeCandidates = Array.isArray(webgpu.candidates)
    ? webgpu.candidates.filter((candidate) => candidate.status === "complete")
    : [];
  const passed = webgpu.skipped !== true && webgpu.webgpu?.available === true && webgpu.webgpu?.softwareAdapter !== true && completeCandidates.length > 0;
  return {
    name: "hardware WebGPU artifact",
    required: true,
    status: passed ? "pass" : "missing",
    message: passed
      ? `${completeCandidates.length} hardware WebGPU candidate(s) completed.`
      : webgpu.reason ?? "No hardware WebGPU candidate completed.",
    source: "webgpu",
    evidence: {
      generatedAt: webgpu.generatedAt ?? null,
      webgpu: webgpu.webgpu ?? null,
      candidates: webgpu.candidates ?? [],
    },
  };
}

function freshnessCheck(artifacts) {
  const stale = Object.entries(artifacts)
    .filter(([, artifact]) => artifact.exists && artifact.stale)
    .map(([key, artifact]) => ({ key, path: artifact.path, artifactHash: artifact.sourceHash }));
  return {
    name: "artifact source freshness",
    required: true,
    status: stale.length === 0 ? "pass" : "stale",
    message:
      stale.length === 0
        ? "Readable artifacts with source fingerprints match current files."
        : `${stale.length} artifact(s) were generated from an older source fingerprint.`,
    source: "summary",
    evidence: { stale },
  };
}

async function readArtifact(path, currentSourceFingerprint) {
  if (!existsSync(path)) {
    return { exists: false, path, ok: false, error: "missing" };
  }
  try {
    const value = JSON.parse(await readFile(path, "utf8"));
    const sourceHash = value.sourceFingerprint?.hash ?? null;
    return {
      exists: true,
      ok: true,
      path,
      value,
      generatedAt: value.generatedAt ?? null,
      sourceHash,
      stale: Boolean(sourceHash && sourceHash !== currentSourceFingerprint.hash),
    };
  } catch (error) {
    return { exists: true, ok: false, path, error: error.message };
  }
}

function summarizeArtifact(artifact) {
  return {
    exists: artifact.exists,
    ok: artifact.ok,
    path: artifact.path,
    generatedAt: artifact.generatedAt ?? null,
    sourceHash: artifact.sourceHash ?? null,
    stale: artifact.stale ?? false,
    error: artifact.error ?? null,
  };
}

function dedupeChecks(checks) {
  const merged = new Map();
  for (const check of checks) {
    if (!check?.name) continue;
    const previous = merged.get(check.name);
    if (!previous || statusRank(check.status) < statusRank(previous.status)) {
      merged.set(check.name, check);
    }
  }
  return [...merged.values()];
}

function statusRank(status) {
  return {
    fail: 0,
    stale: 1,
    missing: 2,
    unknown: 3,
    warn: 4,
    pass: 5,
    skip: 6,
  }[status] ?? 3;
}

function nextActions(checks) {
  const actions = [];
  const needsRealMic = checks.some((check) => check.status !== "pass" && check.name.includes("real microphone"));
  const needsHardwareWebgpu = checks.some((check) => check.status !== "pass" && check.name.includes("WebGPU"));
  const needsBrowserEvidence = checks.some(
    (check) => check.status !== "pass" && check.name === "downloaded browser evidence audit",
  );
  if (needsRealMic || needsHardwareWebgpu || needsBrowserEvidence) {
    actions.push(
      "Run node tools/run-hosted-evidence-capture.mjs from a desktop Chrome session with a real microphone; use a machine exposing hardware WebGPU to satisfy both hosted browser evidence gates.",
    );
  }
  for (const check of checks) {
    if (check.status === "pass") continue;
    if (check.name.includes("real microphone")) {
      actions.push("Run node tools/run-real-mic-series.mjs on a machine with a real microphone.");
    } else if (check.name.includes("WebGPU")) {
      actions.push("Run node tools/run-webgpu-benchmark.mjs in Chrome on a machine exposing a hardware WebGPU adapter.");
    } else if (check.name === "downloaded browser evidence audit") {
      actions.push("Download JSON from the hosted Space after real-mic/WebGPU runs, then run node tools/audit-browser-evidence.mjs <file>.");
    } else if (check.name === "hosted Space no-server smoke") {
      actions.push("Run node tools/run-hosted-smoke.mjs.");
    } else if (check.name === "artifact source freshness") {
      actions.push("Rerun the stale harnesses so their source fingerprints match current files.");
    } else {
      actions.push(`Resolve evidence check: ${check.name}.`);
    }
  }
  return [...new Set(actions)];
}

main().catch((error) => {
  console.error(error.stack ?? error.message);
  process.exitCode = 1;
});