import resultsJson from "@/data/results.json"; import type { Category, DatasetEntry, DatasetInfo, MetricKey, ModelRow, Provenance, ResultsFile, Split, Track, } from "./types"; /* JSON tuples ([lo, hi] CIs) widen to number[] under resolveJsonModule, so we narrow through unknown once, here, and nowhere else. */ const results = resultsJson as unknown as ResultsFile; /* ------------------------------------------------------------------ */ /* Platform version strings — single source of truth. */ /* Every surface (run manifests, methodology exemplar, nav badge) */ /* must reference these, never a literal. */ /* ------------------------------------------------------------------ */ export const NORMALIZER_VERSION = "v0.1"; export const HARNESS_VERSION = "medasr-bench v0.4"; /* ------------------------------------------------------------------ */ /* Dataset registry */ /* ------------------------------------------------------------------ */ /** The anchor (first live) dataset — what no-arg accessors resolve to. */ export const ANCHOR_DATASET_ID = "vimedcss"; /** All dataset profiles, in registry order. */ export function getDatasets(): DatasetInfo[] { return results.datasets.map((d) => d.info); } /** Only datasets with leaderboards running today. */ export function getLiveDatasets(): DatasetInfo[] { return results.datasets .filter((d) => d.info.status === "live") .map((d) => d.info); } /** Full registry entry (profile + model rows). No-arg returns the anchor. */ export function getDatasetEntry(id: string = ANCHOR_DATASET_ID): DatasetEntry { const entry = results.datasets.find((d) => d.info.id === id); if (!entry) { const known = results.datasets.map((d) => d.info.id).join(", "); throw new Error( `Unknown dataset id "${id}". Registered datasets: ${known}.`, ); } return entry; } /** Dataset profile by id. No-arg returns the anchor (ViMedCSS). */ export function getDataset(id: string = ANCHOR_DATASET_ID): DatasetInfo { return getDatasetEntry(id).info; } /** All model rows for a dataset, in seed order. No-arg: anchor dataset. */ export function getModels(datasetId: string = ANCHOR_DATASET_ID): ModelRow[] { return getDatasetEntry(datasetId).models; } /** Single model by slug within a dataset, or undefined. */ export function getModel( slug: string, datasetId: string = ANCHOR_DATASET_ID, ): ModelRow | undefined { return getDatasetEntry(datasetId).models.find((m) => m.slug === slug); } /** Metrics where a higher value is better (everything else: lower wins). */ export const HIGHER_IS_BETTER: ReadonlySet = new Set([ "mtrExact", "mtrFuzzy", "rtfx", ]); /** * Models that have `metric` measured on `split`, sorted best-first * (ascending for WER-family, descending for MTR/RTFx). * Rows without the metric (running/planned) are excluded. */ export function getRanked( split: Split, metric: MetricKey = "wer", datasetId: string = ANCHOR_DATASET_ID, ): ModelRow[] { const dir = HIGHER_IS_BETTER.has(metric) ? -1 : 1; return getDatasetEntry(datasetId) .models.filter((m) => typeof m.metrics?.[split]?.[metric] === "number") .sort( (a, b) => dir * ((a.metrics![split]![metric] as number) - (b.metrics![split]![metric] as number)), ); } /* ------------------------------------------------------------------ */ /* Label maps */ /* ------------------------------------------------------------------ */ export const categoryLabels: Record = { "vn-open": "Vietnamese open", "multilingual-open": "Multilingual open", medical: "Medical-specialized", "api-intl": "Commercial API (intl)", "api-vn": "Commercial API (VN)", }; export const trackLabels: Record = { A: "Track A — Zero-shot", B: "Track B — Contextual biasing", C: "Track C — Adapted", }; export const provenanceLabels: Record = { "platform-verified": "Platform-verified", "api-snapshot": "API snapshot", "self-reported": "Self-reported", "paper-imported": "Paper-imported", running: "Running", planned: "Planned", }; export const metricLabels: Record = { wer: "WER", cer: "CER", csWer: "CS-WER", nWer: "N-WER", mtrExact: "MTR (exact)", mtrFuzzy: "MTR (fuzzy)", rtfx: "RTFx", }; export const splitLabels: Record = { test: "Test", hard: "Hard", }; /* ------------------------------------------------------------------ */ /* Formatters */ /* ------------------------------------------------------------------ */ /** Fixed-point number, or em-dash when missing. fmt(23.5) → "23.50". */ export function fmt(x: number | null | undefined, digits = 2): string { if (x === null || x === undefined || Number.isNaN(x)) return "—"; return x.toFixed(digits); } /** 95% CI as "lo–hi" (en dash), or "" when missing. fmtCI([22.8, 24.5]) → "22.80–24.50". */ export function fmtCI(ci: [number, number] | undefined, digits = 2): string { if (!ci) return ""; return `${ci[0].toFixed(digits)}–${ci[1].toFixed(digits)}`; }