MedASR-Bench / src /lib /data.ts
ngoan
MedASR Bench: full platform + Hugging Face Docker Space packaging
d70f132
Raw
History Blame Contribute Delete
5.31 kB
import resultsJson from "@/data/results.json";
import type {
Category,
DatasetEntry,
DatasetInfo,
MetricKey,
ModelRow,
Provenance,
ResultsFile,
Split,
Track,
} from "./types";
/* JSON tuples ([lo, hi] CIs) widen to number[] under resolveJsonModule,
so we narrow through unknown once, here, and nowhere else. */
const results = resultsJson as unknown as ResultsFile;
/* ------------------------------------------------------------------ */
/* Platform version strings — single source of truth. */
/* Every surface (run manifests, methodology exemplar, nav badge) */
/* must reference these, never a literal. */
/* ------------------------------------------------------------------ */
export const NORMALIZER_VERSION = "v0.1";
export const HARNESS_VERSION = "medasr-bench v0.4";
/* ------------------------------------------------------------------ */
/* Dataset registry */
/* ------------------------------------------------------------------ */
/** The anchor (first live) dataset — what no-arg accessors resolve to. */
export const ANCHOR_DATASET_ID = "vimedcss";
/** All dataset profiles, in registry order. */
export function getDatasets(): DatasetInfo[] {
return results.datasets.map((d) => d.info);
}
/** Only datasets with leaderboards running today. */
export function getLiveDatasets(): DatasetInfo[] {
return results.datasets
.filter((d) => d.info.status === "live")
.map((d) => d.info);
}
/** Full registry entry (profile + model rows). No-arg returns the anchor. */
export function getDatasetEntry(id: string = ANCHOR_DATASET_ID): DatasetEntry {
const entry = results.datasets.find((d) => d.info.id === id);
if (!entry) {
const known = results.datasets.map((d) => d.info.id).join(", ");
throw new Error(
`Unknown dataset id "${id}". Registered datasets: ${known}.`,
);
}
return entry;
}
/** Dataset profile by id. No-arg returns the anchor (ViMedCSS). */
export function getDataset(id: string = ANCHOR_DATASET_ID): DatasetInfo {
return getDatasetEntry(id).info;
}
/** All model rows for a dataset, in seed order. No-arg: anchor dataset. */
export function getModels(datasetId: string = ANCHOR_DATASET_ID): ModelRow[] {
return getDatasetEntry(datasetId).models;
}
/** Single model by slug within a dataset, or undefined. */
export function getModel(
slug: string,
datasetId: string = ANCHOR_DATASET_ID,
): ModelRow | undefined {
return getDatasetEntry(datasetId).models.find((m) => m.slug === slug);
}
/** Metrics where a higher value is better (everything else: lower wins). */
export const HIGHER_IS_BETTER: ReadonlySet<MetricKey> = new Set([
"mtrExact",
"mtrFuzzy",
"rtfx",
]);
/**
* Models that have `metric` measured on `split`, sorted best-first
* (ascending for WER-family, descending for MTR/RTFx).
* Rows without the metric (running/planned) are excluded.
*/
export function getRanked(
split: Split,
metric: MetricKey = "wer",
datasetId: string = ANCHOR_DATASET_ID,
): ModelRow[] {
const dir = HIGHER_IS_BETTER.has(metric) ? -1 : 1;
return getDatasetEntry(datasetId)
.models.filter((m) => typeof m.metrics?.[split]?.[metric] === "number")
.sort(
(a, b) =>
dir * ((a.metrics![split]![metric] as number) -
(b.metrics![split]![metric] as number)),
);
}
/* ------------------------------------------------------------------ */
/* Label maps */
/* ------------------------------------------------------------------ */
export const categoryLabels: Record<Category, string> = {
"vn-open": "Vietnamese open",
"multilingual-open": "Multilingual open",
medical: "Medical-specialized",
"api-intl": "Commercial API (intl)",
"api-vn": "Commercial API (VN)",
};
export const trackLabels: Record<Track, string> = {
A: "Track A — Zero-shot",
B: "Track B — Contextual biasing",
C: "Track C — Adapted",
};
export const provenanceLabels: Record<Provenance, string> = {
"platform-verified": "Platform-verified",
"api-snapshot": "API snapshot",
"self-reported": "Self-reported",
"paper-imported": "Paper-imported",
running: "Running",
planned: "Planned",
};
export const metricLabels: Record<MetricKey, string> = {
wer: "WER",
cer: "CER",
csWer: "CS-WER",
nWer: "N-WER",
mtrExact: "MTR (exact)",
mtrFuzzy: "MTR (fuzzy)",
rtfx: "RTFx",
};
export const splitLabels: Record<Split, string> = {
test: "Test",
hard: "Hard",
};
/* ------------------------------------------------------------------ */
/* Formatters */
/* ------------------------------------------------------------------ */
/** Fixed-point number, or em-dash when missing. fmt(23.5) → "23.50". */
export function fmt(x: number | null | undefined, digits = 2): string {
if (x === null || x === undefined || Number.isNaN(x)) return "—";
return x.toFixed(digits);
}
/** 95% CI as "lo–hi" (en dash), or "" when missing. fmtCI([22.8, 24.5]) → "22.80–24.50". */
export function fmtCI(ci: [number, number] | undefined, digits = 2): string {
if (!ci) return "";
return `${ci[0].toFixed(digits)}${ci[1].toFixed(digits)}`;
}