Spaces:
Sleeping
Sleeping
| import resultsJson from "@/data/results.json"; | |
| import type { | |
| Category, | |
| DatasetEntry, | |
| DatasetInfo, | |
| MetricKey, | |
| ModelRow, | |
| Provenance, | |
| ResultsFile, | |
| Split, | |
| Track, | |
| } from "./types"; | |
| /* JSON tuples ([lo, hi] CIs) widen to number[] under resolveJsonModule, | |
| so we narrow through unknown once, here, and nowhere else. */ | |
| const results = resultsJson as unknown as ResultsFile; | |
| /* ------------------------------------------------------------------ */ | |
| /* Platform version strings — single source of truth. */ | |
| /* Every surface (run manifests, methodology exemplar, nav badge) */ | |
| /* must reference these, never a literal. */ | |
| /* ------------------------------------------------------------------ */ | |
| export const NORMALIZER_VERSION = "v0.1"; | |
| export const HARNESS_VERSION = "medasr-bench v0.4"; | |
| /* ------------------------------------------------------------------ */ | |
| /* Dataset registry */ | |
| /* ------------------------------------------------------------------ */ | |
| /** The anchor (first live) dataset — what no-arg accessors resolve to. */ | |
| export const ANCHOR_DATASET_ID = "vimedcss"; | |
| /** All dataset profiles, in registry order. */ | |
| export function getDatasets(): DatasetInfo[] { | |
| return results.datasets.map((d) => d.info); | |
| } | |
| /** Only datasets with leaderboards running today. */ | |
| export function getLiveDatasets(): DatasetInfo[] { | |
| return results.datasets | |
| .filter((d) => d.info.status === "live") | |
| .map((d) => d.info); | |
| } | |
| /** Full registry entry (profile + model rows). No-arg returns the anchor. */ | |
| export function getDatasetEntry(id: string = ANCHOR_DATASET_ID): DatasetEntry { | |
| const entry = results.datasets.find((d) => d.info.id === id); | |
| if (!entry) { | |
| const known = results.datasets.map((d) => d.info.id).join(", "); | |
| throw new Error( | |
| `Unknown dataset id "${id}". Registered datasets: ${known}.`, | |
| ); | |
| } | |
| return entry; | |
| } | |
| /** Dataset profile by id. No-arg returns the anchor (ViMedCSS). */ | |
| export function getDataset(id: string = ANCHOR_DATASET_ID): DatasetInfo { | |
| return getDatasetEntry(id).info; | |
| } | |
| /** All model rows for a dataset, in seed order. No-arg: anchor dataset. */ | |
| export function getModels(datasetId: string = ANCHOR_DATASET_ID): ModelRow[] { | |
| return getDatasetEntry(datasetId).models; | |
| } | |
| /** Single model by slug within a dataset, or undefined. */ | |
| export function getModel( | |
| slug: string, | |
| datasetId: string = ANCHOR_DATASET_ID, | |
| ): ModelRow | undefined { | |
| return getDatasetEntry(datasetId).models.find((m) => m.slug === slug); | |
| } | |
| /** Metrics where a higher value is better (everything else: lower wins). */ | |
| export const HIGHER_IS_BETTER: ReadonlySet<MetricKey> = new Set([ | |
| "mtrExact", | |
| "mtrFuzzy", | |
| "rtfx", | |
| ]); | |
| /** | |
| * Models that have `metric` measured on `split`, sorted best-first | |
| * (ascending for WER-family, descending for MTR/RTFx). | |
| * Rows without the metric (running/planned) are excluded. | |
| */ | |
| export function getRanked( | |
| split: Split, | |
| metric: MetricKey = "wer", | |
| datasetId: string = ANCHOR_DATASET_ID, | |
| ): ModelRow[] { | |
| const dir = HIGHER_IS_BETTER.has(metric) ? -1 : 1; | |
| return getDatasetEntry(datasetId) | |
| .models.filter((m) => typeof m.metrics?.[split]?.[metric] === "number") | |
| .sort( | |
| (a, b) => | |
| dir * ((a.metrics![split]![metric] as number) - | |
| (b.metrics![split]![metric] as number)), | |
| ); | |
| } | |
| /* ------------------------------------------------------------------ */ | |
| /* Label maps */ | |
| /* ------------------------------------------------------------------ */ | |
| export const categoryLabels: Record<Category, string> = { | |
| "vn-open": "Vietnamese open", | |
| "multilingual-open": "Multilingual open", | |
| medical: "Medical-specialized", | |
| "api-intl": "Commercial API (intl)", | |
| "api-vn": "Commercial API (VN)", | |
| }; | |
| export const trackLabels: Record<Track, string> = { | |
| A: "Track A — Zero-shot", | |
| B: "Track B — Contextual biasing", | |
| C: "Track C — Adapted", | |
| }; | |
| export const provenanceLabels: Record<Provenance, string> = { | |
| "platform-verified": "Platform-verified", | |
| "api-snapshot": "API snapshot", | |
| "self-reported": "Self-reported", | |
| "paper-imported": "Paper-imported", | |
| running: "Running", | |
| planned: "Planned", | |
| }; | |
| export const metricLabels: Record<MetricKey, string> = { | |
| wer: "WER", | |
| cer: "CER", | |
| csWer: "CS-WER", | |
| nWer: "N-WER", | |
| mtrExact: "MTR (exact)", | |
| mtrFuzzy: "MTR (fuzzy)", | |
| rtfx: "RTFx", | |
| }; | |
| export const splitLabels: Record<Split, string> = { | |
| test: "Test", | |
| hard: "Hard", | |
| }; | |
| /* ------------------------------------------------------------------ */ | |
| /* Formatters */ | |
| /* ------------------------------------------------------------------ */ | |
| /** Fixed-point number, or em-dash when missing. fmt(23.5) → "23.50". */ | |
| export function fmt(x: number | null | undefined, digits = 2): string { | |
| if (x === null || x === undefined || Number.isNaN(x)) return "—"; | |
| return x.toFixed(digits); | |
| } | |
| /** 95% CI as "lo–hi" (en dash), or "" when missing. fmtCI([22.8, 24.5]) → "22.80–24.50". */ | |
| export function fmtCI(ci: [number, number] | undefined, digits = 2): string { | |
| if (!ci) return ""; | |
| return `${ci[0].toFixed(digits)}–${ci[1].toFixed(digits)}`; | |
| } | |