scripts/sync-results.mjs

#!/usr/bin/env node
/**
 * sync-results.mjs — pull platform run outputs into src/data/results.json.
 *
 * Reads, for each known model and each split (test|hard):
 *   ../benchmark/results/<model>/<split>/metrics.json
 *   ../benchmark/results/<model>/<split>/run_manifest.json   (optional, logged)
 *
 * metrics.json keys: wer, wer_ci95 [lo,hi], cer, cs_wer, cs_wer_ci95,
 * n_wer, mtr_exact, mtr_fuzzy, rtfx_infer.
 *
 * Upserts the matching slug's metrics into the vimedcss entry of
 * src/data/results.json (shape: { datasets: [{ info, models }] }) and sets
 * provenance to "platform-verified". All other rows and fields are preserved.
 *
 * Usage: npm run sync-results
 */

import { existsSync, readFileSync, writeFileSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";

const __dirname = dirname(fileURLToPath(import.meta.url));
const PLATFORM_ROOT = resolve(__dirname, "..");
const RESULTS_ROOT = resolve(PLATFORM_ROOT, "..", "benchmark", "results");
const DATA_FILE = join(PLATFORM_ROOT, "src", "data", "results.json");

/** benchmark results dir name → leaderboard slug */
const MODEL_MAP = {
  zipformer: "zipformer-30m",
  qwen3asr: "qwen3-asr-0.6b",
  gemini: "gemini-3.5-flash",
  deepgram: "deepgram-nova-3",
};

const SPLITS = ["test", "hard"];

/** Platform runs land on the anchor dataset's leaderboard. */
const TARGET_DATASET_ID = "vimedcss";

/** metrics.json key → SplitMetrics field */
function toSplitMetrics(raw) {
  const out = {};
  if (typeof raw.wer === "number") out.wer = raw.wer;
  if (Array.isArray(raw.wer_ci95)) out.werCi = raw.wer_ci95;
  if (typeof raw.cer === "number") out.cer = raw.cer;
  if (typeof raw.cs_wer === "number") out.csWer = raw.cs_wer;
  if (Array.isArray(raw.cs_wer_ci95)) out.csWerCi = raw.cs_wer_ci95;
  if (typeof raw.n_wer === "number") out.nWer = raw.n_wer;
  if (typeof raw.mtr_exact === "number") out.mtrExact = raw.mtr_exact;
  if (typeof raw.mtr_fuzzy === "number") out.mtrFuzzy = raw.mtr_fuzzy;
  if (typeof raw.rtfx_infer === "number") out.rtfx = raw.rtfx_infer;
  return out;
}

function readJson(path) {
  return JSON.parse(readFileSync(path, "utf8"));
}

function main() {
  if (!existsSync(RESULTS_ROOT)) {
    console.log(`[sync-results] no results dir at ${RESULTS_ROOT} — nothing to do`);
    return;
  }

  const data = readJson(DATA_FILE);
  const entry = (data.datasets ?? []).find(
    (d) => d.info?.id === TARGET_DATASET_ID,
  );
  if (!entry) {
    console.error(
      `[sync-results] dataset "${TARGET_DATASET_ID}" not found in ${DATA_FILE} — aborting`,
    );
    process.exitCode = 1;
    return;
  }
  let updates = 0;

  for (const [dir, slug] of Object.entries(MODEL_MAP)) {
    const row = entry.models.find((m) => m.slug === slug);
    if (!row) {
      console.warn(`[sync-results] slug ${slug} not found in results.json — skipped`);
      continue;
    }

    for (const split of SPLITS) {
      const metricsPath = join(RESULTS_ROOT, dir, split, "metrics.json");
      if (!existsSync(metricsPath)) continue;

      let raw;
      try {
        raw = readJson(metricsPath);
      } catch (err) {
        console.warn(`[sync-results] unreadable ${metricsPath}: ${err.message} — skipped`);
        continue;
      }

      const manifestPath = join(RESULTS_ROOT, dir, split, "run_manifest.json");
      let manifest;
      if (existsSync(manifestPath)) {
        try {
          manifest = readJson(manifestPath);
          console.log(
            `[sync-results] ${slug}/${split} manifest:`,
            JSON.stringify(manifest),
          );
        } catch {
          console.warn(`[sync-results] unreadable manifest at ${manifestPath}`);
        }
      }

      row.metrics = row.metrics ?? {};
      row.metrics[split] = toSplitMetrics(raw);
      /* RTFx: the manifest computes audio_s / infer_s at run time and is the
         authoritative timing record; metrics.json only snapshots it at score
         time and can go stale across re-runs (observed on qwen3asr). API
         runners report infer_total=0 -> manifest rtfx_infer null, so RTFx
         stays absent (network latency ≠ local-hardware throughput). */
      if (typeof manifest?.rtfx_infer === "number") {
        row.metrics[split].rtfx = manifest.rtfx_infer;
      }
      /* Provenance: locally-run models are platform-verified (pinned
         revision, our hardware, SOTA-eligible). Commercial APIs run through
         the same harness but on the vendor's service with no pinned revision,
         so they are recorded as a date-stamped "api-snapshot" instead. */
      const isApi = row.category === "api-intl" || row.category === "api-vn";
      row.provenance = isApi ? "api-snapshot" : "platform-verified";
      updates += 1;
      console.log(`[sync-results] upserted ${slug}/${split}`);
    }
  }

  if (updates === 0) {
    console.log("[sync-results] no metrics found — results.json untouched");
    return;
  }

  writeFileSync(DATA_FILE, `${JSON.stringify(data, null, 2)}\n`, "utf8");
  console.log(`[sync-results] wrote ${updates} split result(s) to ${DATA_FILE}`);
}

main();