MedASR-Bench / src /components /compare /compare-utils.ts
ngoan
MedASR Bench: full platform + Hugging Face Docker Space packaging
d70f132
Raw
History Blame Contribute Delete
8.71 kB
import type { Category, MetricKey, ModelRow, Split } from "@/lib/types";
import { HIGHER_IS_BETTER } from "@/lib/data";
/* ------------------------------------------------------------------ */
/* Category color mapping — derived ONLY from globals.css tokens. */
/* vn-open = phosphor accent · multilingual = cool ice (accent→text) */
/* medical = sage · api-* = cool/neutral mixes drawn as DIAMONDS, so */
/* the warm --warn channel belongs exclusively to contamination. */
/* ------------------------------------------------------------------ */
export const categoryColor: Record<Category, string> = {
"vn-open": "var(--accent)",
"multilingual-open": "color-mix(in srgb, var(--accent) 65%, var(--text) 35%)",
medical: "var(--text)",
"api-intl": "color-mix(in srgb, var(--text) 55%, var(--muted) 45%)",
"api-vn": "var(--muted)",
};
/** API points render as 45°-rotated squares instead of dots. */
export function isApiCategory(c: Category): boolean {
return c === "api-intl" || c === "api-vn";
}
/* ------------------------------------------------------------------ */
/* Shared Recharts text styles — single source for both compare */
/* charts. 11px floor and token-driven fills keep SVG text legible on */
/* the lighter glass panels. */
/* ------------------------------------------------------------------ */
export const CHART_AXIS_TICK = {
fill: "var(--muted)",
fontSize: 11,
fontFamily: "var(--font-mono), 'IBM Plex Mono', monospace",
} as const;
export const CHART_AXIS_LABEL = {
fill: "var(--muted)",
fontSize: 11,
letterSpacing: "0.14em",
fontFamily: "var(--font-mono), 'IBM Plex Mono', monospace",
} as const;
/** Point labels knock out lines beneath them with a panel-toned halo —
charts now sit on glass surfaces, so the halo matches --surface,
not the page background. */
export const CHART_LABEL_KNOCKOUT = {
paintOrder: "stroke",
stroke: "var(--surface)",
strokeWidth: 4,
} as const;
/** Near-text fill for chart point labels (brighter than --muted). */
export const CHART_LABEL_FILL =
"color-mix(in srgb, var(--text) 86%, var(--muted) 14%)";
/** Selection-order palette for head-to-head columns and profile bars.
Series 2 reads as bone-white with a teal cast so it never doubles the
accent; series 3 bars render hairline-outlined with transparent fill. */
export const seriesColor: readonly string[] = [
"var(--accent)",
"color-mix(in srgb, var(--accent) 15%, var(--text) 85%)",
"var(--muted)",
];
/* ------------------------------------------------------------------ */
/* Compact display names for chart point labels */
/* ------------------------------------------------------------------ */
const SHORT_NAMES: Record<string, string> = {
"phowhisper-small-ag": "PhoWhisper-S ·AG",
"phowhisper-small-lora": "PhoWhisper-S ·LoRA",
vietasr: "VietASR 68M",
"phowhisper-large": "PhoWhisper-L",
"whisper-large-v3": "Whisper-L v3",
"phowhisper-small": "PhoWhisper-S",
"wav2vec2-base-vi": "w2v2-base-vi",
"whisper-small": "Whisper-S",
"mms-1b": "MMS-1B",
"zipformer-30m": "Zipformer-30M",
"qwen3-asr-0.6b": "Qwen3-ASR-0.6B",
"gemini-2.5-pro": "Gemini 2.5 Pro",
"gpt-4o-transcribe": "gpt-4o-transcribe",
"elevenlabs-scribe-v2": "Scribe v2",
"fpt-ai-stt": "FPT.AI STT",
};
export function shortName(m: ModelRow): string {
return SHORT_NAMES[m.slug] ?? m.name;
}
/* ------------------------------------------------------------------ */
/* Pareto math (minimize both axes; lower-left is better) */
/* ------------------------------------------------------------------ */
export interface ParetoDatum {
wer: number;
csWer: number;
werCi?: [number, number];
csWerCi?: [number, number];
model: ModelRow;
}
/** One datum per model holding BOTH wer and csWer on the split. */
export function paretoData(models: ModelRow[], split: Split): ParetoDatum[] {
return models.flatMap((m) => {
const s = m.metrics?.[split];
if (typeof s?.wer !== "number" || typeof s?.csWer !== "number") return [];
return [
{
wer: s.wer,
csWer: s.csWer,
werCi: s.werCi,
csWerCi: s.csWerCi,
model: m,
},
];
});
}
/** Non-dominated subset, sorted by WER ascending (csWer strictly falls). */
export function paretoFrontier(points: ParetoDatum[]): ParetoDatum[] {
const sorted = [...points].sort((a, b) => a.wer - b.wer || a.csWer - b.csWer);
const out: ParetoDatum[] = [];
let best = Infinity;
for (const p of sorted) {
if (p.csWer < best) {
out.push(p);
best = p.csWer;
}
}
return out;
}
export type Segment = readonly [
{ x: number; y: number },
{ x: number; y: number },
];
/**
* Staircase boundary of the region dominated by the frontier,
* extended to the chart domain edges. Steps go right, then down.
*/
export function frontierSegments(
frontier: ParetoDatum[],
xMax: number,
yMax: number,
): Segment[] {
if (frontier.length === 0) return [];
const segs: Segment[] = [];
const first = frontier[0];
const last = frontier[frontier.length - 1];
// drop in from the top edge to the best-WER point
segs.push([
{ x: first.wer, y: yMax },
{ x: first.wer, y: first.csWer },
]);
for (let i = 0; i < frontier.length - 1; i += 1) {
const a = frontier[i];
const b = frontier[i + 1];
segs.push([
{ x: a.wer, y: a.csWer },
{ x: b.wer, y: a.csWer },
]);
segs.push([
{ x: b.wer, y: a.csWer },
{ x: b.wer, y: b.csWer },
]);
}
// run out to the right edge at the best CS-WER level
segs.push([
{ x: last.wer, y: last.csWer },
{ x: xMax, y: last.csWer },
]);
return segs;
}
/**
* Non-overlapping vertical bands covering the dominated region (up-right
* of the staircase): band i spans x from frontier[i] to frontier[i+1]
* (last band runs to xMax), y from frontier[i].csWer to yMax.
*/
export function dominatedBands(
frontier: ParetoDatum[],
xMax: number,
yMax: number,
): Array<{ x1: number; x2: number; y1: number; y2: number }> {
return frontier.map((p, i) => ({
x1: p.wer,
x2: frontier[i + 1]?.wer ?? xMax,
y1: p.csWer,
y2: yMax,
}));
}
/* ------------------------------------------------------------------ */
/* Axis domain helpers (data-driven, padded, rounded to 2s or 5s) */
/* ------------------------------------------------------------------ */
/**
* Data extent padded ~10–15% and snapped outward — to multiples of 2 when
* the span is tight (< 15 units) so the points command the canvas, else 5.
*/
export function niceDomain(values: number[]): [number, number] {
if (values.length === 0) return [0, 100];
const min = Math.min(...values);
const max = Math.max(...values);
const pad = Math.max((max - min) * 0.1, 1.5);
const snap = max - min < 15 ? 2 : 5;
return [
Math.floor((min - pad) / snap) * snap,
Math.ceil((max + pad) / snap) * snap,
];
}
/** Tick positions for a niceDomain: every 2 / 5 / 10 by span width. */
export function domainTicks([lo, hi]: [number, number]): number[] {
const span = hi - lo;
const step = span <= 16 ? 2 : span <= 30 ? 5 : 10;
const out: number[] = [];
for (let t = lo; t <= hi; t += step) out.push(t);
return out;
}
/* ------------------------------------------------------------------ */
/* Head-to-head helpers */
/* ------------------------------------------------------------------ */
export const H2H_METRICS: readonly MetricKey[] = [
"wer",
"csWer",
"nWer",
"cer",
"mtrExact",
"mtrFuzzy",
"rtfx",
];
export const METRIC_UNIT: Record<MetricKey, string> = {
wer: "%",
cer: "%",
csWer: "%",
nWer: "%",
mtrExact: "%",
mtrFuzzy: "%",
rtfx: "×",
};
export type DeltaTone = "better" | "worse" | "flat";
/** Sign-aware quality of a challenger-minus-baseline delta. */
export function deltaTone(delta: number, metric: MetricKey): DeltaTone {
if (delta === 0) return "flat";
const improved = HIGHER_IS_BETTER.has(metric) ? delta > 0 : delta < 0;
return improved ? "better" : "worse";
}
/** "+3.23" / "−1.40" / "±0.00" (true minus sign for mono alignment). */
export function fmtDelta(delta: number, digits = 2): string {
const sign = delta > 0 ? "+" : delta < 0 ? "−" : "±";
return `${sign}${Math.abs(delta).toFixed(digits)}`;
}
/** True when the model has no numeric metric at all on the split. */
export function hasEvidenceGap(m: ModelRow, split: Split): boolean {
const s = m.metrics?.[split];
if (!s) return true;
return !H2H_METRICS.some((k) => typeof s[k] === "number");
}