tools/run-final-validation.mjs

#!/usr/bin/env node
import { spawn } from "node:child_process";
import { readFile, writeFile, mkdir } from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { sourceFingerprint } from "./source-fingerprint.mjs";

const scriptDir = dirname(fileURLToPath(import.meta.url));
const resultPath = resolve(process.env.BROWSER_SPEAK_FINAL_JSON ?? `${tmpdir()}/browser-speak-final-validation.json`);
const auditPath = resolve(process.env.BROWSER_SPEAK_AUDIT_JSON ?? `${tmpdir()}/browser-speak-validation-audit.json`);
const soft = process.env.BROWSER_SPEAK_FINAL_SOFT === "true";
const skipLocal = process.env.BROWSER_SPEAK_FINAL_SKIP_LOCAL === "true";
const skipUi = skipLocal || process.env.BROWSER_SPEAK_FINAL_SKIP_UI === "true";
const skipEvidenceExport = skipLocal || process.env.BROWSER_SPEAK_FINAL_SKIP_EVIDENCE_EXPORT === "true";
const skipClientSide = skipLocal || process.env.BROWSER_SPEAK_FINAL_SKIP_CLIENT_SIDE === "true";
const skipLoopback = skipLocal || process.env.BROWSER_SPEAK_FINAL_SKIP_LOOPBACK === "true";
const skipWebgpu = process.env.BROWSER_SPEAK_FINAL_SKIP_WEBGPU === "true";
const realMicMode = parseRealMicMode(process.env.BROWSER_SPEAK_FINAL_REAL_MIC ?? "skip");

const envDefaults = {
  BROWSER_SPEAK_CLIENT_SIDE_REUSE_PROFILE: "true",
  BROWSER_SPEAK_LOCAL_REUSE_PROFILE: "true",
  BROWSER_SPEAK_LOAD_TIMEOUT_MS: "900000",
  BROWSER_SPEAK_TASK_TIMEOUT_MS: "240000",
  BROWSER_SPEAK_VOICE_PRELOAD_TIMEOUT_MS: "240000",
};

const steps = [
  {
    name: "UI smoke",
    script: "run-ui-smoke.mjs",
    artifactEnv: "BROWSER_SPEAK_UI_JSON",
    defaultArtifact: `${tmpdir()}/browser-speak-ui-smoke.json`,
    skipped: skipUi,
    skipReason: "disabled by BROWSER_SPEAK_FINAL_SKIP_UI or BROWSER_SPEAK_FINAL_SKIP_LOCAL",
  },
  {
    name: "evidence export smoke",
    script: "run-evidence-export-smoke.mjs",
    artifactEnv: "BROWSER_SPEAK_EVIDENCE_EXPORT_JSON",
    defaultArtifact: `${tmpdir()}/browser-speak-evidence-export-smoke.json`,
    skipped: skipEvidenceExport,
    skipReason: "disabled by BROWSER_SPEAK_FINAL_SKIP_EVIDENCE_EXPORT or BROWSER_SPEAK_FINAL_SKIP_LOCAL",
  },
  {
    name: "client-side/no-server smoke",
    script: "run-client-side-smoke.mjs",
    artifactEnv: "BROWSER_SPEAK_CLIENT_SIDE_JSON",
    defaultArtifact: `${tmpdir()}/browser-speak-client-side-smoke.json`,
    skipped: skipClientSide,
    skipReason: "disabled by BROWSER_SPEAK_FINAL_SKIP_CLIENT_SIDE or BROWSER_SPEAK_FINAL_SKIP_LOCAL",
  },
  {
    name: "loopback stability",
    script: "run-loopback-series.mjs",
    artifactEnv: "BROWSER_SPEAK_LOOPBACK_JSON",
    defaultArtifact: `${tmpdir()}/browser-speak-loopback-series.json`,
    skipped: skipLoopback,
    skipReason: "disabled by BROWSER_SPEAK_FINAL_SKIP_LOOPBACK or BROWSER_SPEAK_FINAL_SKIP_LOCAL",
  },
  {
    name: "hardware WebGPU benchmark",
    script: "run-webgpu-benchmark.mjs",
    artifactEnv: "BROWSER_SPEAK_WEBGPU_JSON",
    defaultArtifact: `${tmpdir()}/browser-speak-webgpu-results.json`,
    skipped: skipWebgpu,
    skipReason: "disabled by BROWSER_SPEAK_FINAL_SKIP_WEBGPU",
  },
  {
    name: realMicMode === "dry-run" ? "real microphone preflight" : "real microphone validation",
    script: "run-real-mic-series.mjs",
    artifactEnv: "BROWSER_SPEAK_REAL_MIC_JSON",
    defaultArtifact:
      realMicMode === "dry-run"
        ? `${tmpdir()}/browser-speak-real-mic-series-dry-run.json`
        : `${tmpdir()}/browser-speak-real-mic-series.json`,
    skipped: realMicMode === "skip",
    skipReason: "set BROWSER_SPEAK_FINAL_REAL_MIC=true to collect rows, or dry-run for preflight only",
    env: realMicMode === "dry-run" ? { BROWSER_SPEAK_REAL_MIC_DRY_RUN: "true" } : {},
  },
  {
    name: "validation audit",
    script: "audit-validation.mjs",
    artifactEnv: "BROWSER_SPEAK_AUDIT_JSON",
    defaultArtifact: `${tmpdir()}/browser-speak-validation-audit.json`,
    env: soft ? { BROWSER_SPEAK_AUDIT_SOFT: "true" } : {},
  },
];

const commandResults = [];

async function main() {
  await mkdir(dirname(resultPath), { recursive: true });
  await writeSummary({ status: "running" });

  for (const step of steps) {
    const result = await runStep(step);
    commandResults.push(result);
    await writeSummary({ status: "running" });
  }

  const audit = await readJson(auditPath);
  const commandFailures = commandResults.filter((result) => result.commandStatus === "fail");
  const auditPassed = audit?.passed === true;
  await writeSummary({
    status: "complete",
    audit,
    commandFailures,
    auditPassed,
  });

  console.log(`Wrote final validation JSON: ${resultPath}`);
  printFinalSummary(audit, commandFailures);
  if (commandFailures.length > 0 || (!soft && !auditPassed)) process.exitCode = 1;
}

async function runStep(step) {
  const artifactPath = resolve(process.env[step.artifactEnv] ?? step.defaultArtifact);
  const command = `${process.execPath} tools/${step.script}`;
  if (step.skipped) {
    console.log(`Skipping ${step.name}: ${step.skipReason}.`);
    return {
      name: step.name,
      status: "skip",
      commandStatus: "skip",
      evidenceStatus: "skip",
      command,
      artifactPath,
      reason: step.skipReason,
      durationMs: 0,
    };
  }

  console.log(`Running ${step.name}: ${command}`);
  const startedAt = Date.now();
  const exitCode = await runChild(process.execPath, [resolve(scriptDir, step.script)], envWithDefaults(step.env));
  const durationMs = Date.now() - startedAt;
  const commandStatus = exitCode === 0 ? "pass" : "fail";
  const evidence = await evidenceForStep(step, artifactPath);
  console.log(formatStepResult(step.name, commandStatus, evidence?.status, durationMs));
  return {
    name: step.name,
    status: evidence?.status ?? commandStatus,
    commandStatus,
    evidenceStatus: evidence?.status ?? null,
    command,
    artifactPath,
    exitCode,
    durationMs,
    evidence,
  };
}

function runChild(command, args, env) {
  return new Promise((resolvePromise) => {
    const child = spawn(command, args, { env, stdio: "inherit" });
    child.on("error", (error) => {
      console.error(`${command} failed to start: ${error.message}`);
      resolvePromise(1);
    });
    child.on("exit", (code) => resolvePromise(code ?? 1));
  });
}

function envWithDefaults(overrides = {}) {
  const env = { ...process.env };
  for (const [key, value] of Object.entries(envDefaults)) {
    if (!(key in env)) env[key] = value;
  }
  return { ...env, ...overrides };
}

async function writeSummary({ status, audit = null, commandFailures = [], auditPassed = null } = {}) {
  const currentAudit = audit ?? (await readJson(auditPath));
  const payload = {
    generatedAt: new Date().toISOString(),
    sourceFingerprint: await sourceFingerprint(),
    status,
    passed: currentAudit?.passed === true,
    completed: status === "complete" && commandFailures.length === 0,
    soft,
    config: {
      resultPath,
      auditPath,
      skipLocal,
      skipUi,
      skipEvidenceExport,
      skipClientSide,
      skipLoopback,
      skipWebgpu,
      realMicMode,
      envDefaults,
    },
    commands: commandResults,
    audit: summarizeAudit(currentAudit),
  };
  if (auditPassed !== null) payload.auditPassed = auditPassed;
  if (commandFailures.length > 0) payload.commandFailures = commandFailures;
  await writeFile(resultPath, `${JSON.stringify(payload, null, 2)}\n`);
}

function summarizeAudit(audit) {
  if (!audit || typeof audit !== "object") return null;
  const checks = Array.isArray(audit.checks) ? audit.checks : [];
  return {
    generatedAt: audit.generatedAt ?? null,
    passed: audit.passed === true,
    required: checks.filter((check) => check.required).map(summarizeCheck),
    supporting: checks.filter((check) => !check.required).map(summarizeCheck),
    nextActions: Array.isArray(audit.nextActions) ? audit.nextActions : [],
  };
}

function summarizeCheck(check) {
  return {
    name: check.name,
    status: check.status,
    message: check.message,
  };
}

async function readJson(path) {
  try {
    return JSON.parse(await readFile(path, "utf8"));
  } catch {
    return null;
  }
}

async function evidenceForStep(step, artifactPath) {
  const artifact = await readJson(artifactPath);
  if (!artifact) return { status: "missing", message: "Artifact JSON was not readable after the step." };
  if (step.script === "audit-validation.mjs") {
    return {
      status: artifact.passed === true ? "pass" : "fail",
      message: artifact.passed === true ? "Final audit passed." : "Final audit did not pass all required evidence gates.",
    };
  }
  if (step.script === "run-webgpu-benchmark.mjs") {
    if (artifact.skipped) {
      return {
        status: "missing",
        message: artifact.reason ?? "WebGPU benchmark was skipped.",
      };
    }
    const completed = Array.isArray(artifact.candidates)
      ? artifact.candidates.filter((candidate) => candidate.status === "complete").length
      : 0;
    return {
      status: artifact.webgpu?.available === true && artifact.webgpu?.softwareAdapter !== true && completed > 0 ? "pass" : "missing",
      message: completed > 0 ? `${completed} hardware WebGPU candidate(s) completed.` : "No hardware WebGPU candidate completed.",
    };
  }
  if (step.script === "run-real-mic-series.mjs") {
    if (artifact.dryRun) return { status: "preflight", message: "Real-mic dry run completed; no human speech rows collected." };
    return {
      status: artifact.passed === true ? "pass" : "fail",
      message: artifact.passed === true ? "Real microphone rows passed." : "Real microphone rows are missing or below threshold.",
    };
  }
  if ("passed" in artifact) {
    return {
      status: artifact.passed === true ? "pass" : "fail",
      message: artifact.passed === true ? "Step artifact reports passed." : "Step artifact reports failure.",
    };
  }
  return { status: "unknown", message: "Step artifact did not expose a passed flag." };
}

function parseRealMicMode(value) {
  const normalized = String(value).trim().toLowerCase();
  if (["1", "true", "yes", "real", "run"].includes(normalized)) return "real";
  if (["dry-run", "dryrun", "preflight"].includes(normalized)) return "dry-run";
  return "skip";
}

function printFinalSummary(audit, commandFailures) {
  if (commandFailures.length > 0) {
    console.log(`Command failures: ${commandFailures.map((result) => result.name).join(", ")}`);
  }
  if (!audit) {
    console.log("Validation audit JSON was not available.");
    return;
  }
  const required = Array.isArray(audit.checks) ? audit.checks.filter((check) => check.required) : [];
  console.log(`Final audit: ${audit.passed ? "pass" : "fail"}`);
  for (const check of required) {
    console.log(`${check.status}: ${check.name} - ${check.message}`);
  }
  if (!audit.passed && soft) console.log("Soft mode is enabled, so missing external gates do not fail this runner.");
}

function formatStepResult(name, commandStatus, evidenceStatus, durationMs) {
  const evidenceSuffix = evidenceStatus && evidenceStatus !== commandStatus ? `, evidence ${evidenceStatus}` : "";
  return `${name}: command ${commandStatus}${evidenceSuffix} (${formatDuration(durationMs)})`;
}

function formatDuration(ms) {
  if (ms < 1000) return `${ms} ms`;
  return `${(ms / 1000).toFixed(1)} s`;
}

main().catch((error) => {
  console.error(error.stack ?? error.message);
  process.exitCode = 1;
});