Anton Malykhin
feat: refine leaderboard metric presentation and bucket data handling
614eca7
Raw
History Blame Contribute Delete
7.91 kB
import { isNumber } from '$lib/helpers/leaderboard';
import type { GroupScore, GuardModel } from '$lib/types/leaderboard';
import type {
ToolsLanguageFilter,
ToolsRadarScale,
ToolsSortDirection,
ToolsVizMetric
} from '$lib/types/tools';
export const toolsPalette = [
'#3a55ff',
'#0dce97',
'#6b4bf5',
'#f5a623',
'#ff4d4f',
'#21c1d6',
'#b06bf5',
'#5bd17a',
'#ff8a3d',
'#9aa3b2'
];
export function toolsDataGroups(models: readonly GuardModel[], allGroups: readonly string[]) {
return allGroups.filter((group) => models.some((model) => isNumber(model.groups[group]?.score)));
}
export function toolsModelKey(model: GuardModel) {
return model.modelId ?? model.name;
}
export function toolsModelColorMap(models: readonly GuardModel[]) {
return new Map(
models.map((model, index) => [toolsModelKey(model), toolsPalette[index % toolsPalette.length]])
);
}
export function toolsModelColor(model: GuardModel, colorMap: ReadonlyMap<string, string>) {
return colorMap.get(toolsModelKey(model)) ?? '#0dce97';
}
export function toolsSelectedModels(
models: readonly GuardModel[],
selectedModelKeys: readonly string[]
) {
return models.filter((model) => selectedModelKeys.includes(toolsModelKey(model)));
}
export function toolsActiveGroups(
selectedGroups: readonly string[],
language: ToolsLanguageFilter,
groupLanguages: Readonly<Record<string, string[]>>
) {
return selectedGroups.filter(
(group) => language === 'all' || groupLanguages[group]?.includes(language)
);
}
export function toolsPinnedModels(
models: readonly GuardModel[],
pinnedModelKeys: readonly string[]
) {
return models.filter((model) => pinnedModelKeys.includes(toolsModelKey(model)));
}
export function toolsCell(model: GuardModel, group: string) {
return model.groups[group] ?? null;
}
export function toolsMetricValue(cell: GroupScore | null, key: ToolsVizMetric) {
if (!cell) return null;
return cell[key];
}
export function toolsRadarValue(cell: GroupScore | null, key: ToolsRadarScale) {
if (!cell) return null;
return cell[key];
}
export function toolsLatencySortValue(
model: GuardModel,
key: 'model' | 'p50' | 'p95' | 'p99' | 'errorRate'
) {
if (key === 'model') return model.short.toLowerCase();
return model[key];
}
export function toolsSortLatencyRows(
rows: readonly GuardModel[],
key: 'model' | 'p50' | 'p95' | 'p99' | 'errorRate',
direction: ToolsSortDirection
) {
return [...rows].sort((a, b) => {
const aValue = toolsLatencySortValue(a, key);
const bValue = toolsLatencySortValue(b, key);
const result =
typeof aValue === 'string' && typeof bValue === 'string'
? aValue.localeCompare(bValue)
: Number(aValue ?? Number.POSITIVE_INFINITY) - Number(bValue ?? Number.POSITIVE_INFINITY);
return direction === 'asc' ? result : -result;
});
}
export function toolsHeatColor(
value: number | null | undefined,
min = 0,
max = 1,
curve: 'linear' | 'sqrt' = 'linear'
) {
if (!isNumber(value)) return 'transparent';
const range = max - min;
const normalized = range > 0 ? (value - min) / range : 0;
const t =
curve === 'sqrt'
? Math.sqrt(Math.max(0, Math.min(1, normalized)))
: Math.max(0, Math.min(1, normalized));
if (t < 0.5) {
const p = t / 0.5;
return `rgb(${Math.round(13 + 242 * p)}, ${Math.round(206 - 36 * p)}, ${Math.round(151 - 111 * p)})`;
}
const p = (t - 0.5) / 0.5;
return `rgb(${Math.round(255 - 25 * p)}, ${Math.round(170 - 93 * p)}, ${Math.round(40 + 39 * p)})`;
}
export function toolsRadarPoint(index: number, total: number, radius: number, cx = 260, cy = 214) {
const angle = -Math.PI / 2 + (2 * Math.PI * index) / total;
return { x: cx + Math.cos(angle) * radius, y: cy + Math.sin(angle) * radius };
}
export function toolsRadarPolygon(
model: GuardModel,
activeGroups: readonly string[],
radarScale: ToolsRadarScale
) {
return activeGroups
.map((group, index) => {
const value = toolsRadarValue(toolsCell(model, group), radarScale);
const point = toolsRadarPoint(
index,
activeGroups.length,
150 * Math.max(0, Math.min(1, value ?? 0))
);
return `${point.x.toFixed(1)},${point.y.toFixed(1)}`;
})
.join(' ');
}
export function toolsLineScale(d0: number, d1: number, r0: number, r1: number) {
return (value: number) => r0 + ((value - d0) / (d1 - d0 || 1)) * (r1 - r0);
}
export function toolsNiceTicks(min: number, max: number, count = 5) {
let step = Math.pow(10, Math.floor(Math.log((max - min || 1) / count) / Math.LN10));
const error = (count * step) / (max - min || 1);
if (error <= 0.15) step *= 10;
else if (error <= 0.35) step *= 5;
else if (error <= 0.75) step *= 2;
const ticks: number[] = [];
for (let tick = Math.ceil(min / step) * step; tick <= max + 1e-9; tick += step) {
ticks.push(Math.round(tick * 1e6) / 1e6);
}
return ticks;
}
export function toolsScatterX(value: number, xMax: number) {
return toolsLineScale(0, xMax, 52, 502)(value);
}
export function toolsScatterY(value: number, yMax: number) {
return toolsLineScale(0, yMax, 368, 18)(value);
}
export function toolsBarY(value: number) {
return toolsLineScale(0, 1, 268, 16)(value);
}
export function toolsBarLabelLines(label: string) {
return label.split(/\s+/).reduce<string[]>((lines, word) => {
const last = lines.at(-1);
if (!last || `${last} ${word}`.length > 12) {
lines.push(word);
return lines;
}
lines[lines.length - 1] = `${last} ${word}`;
return lines;
}, []);
}
export function toolsParetoX(value: number, xMax: number) {
return toolsLineScale(0, xMax, 56, 742)(value);
}
export function toolsParetoY(value: number, y0: number) {
return toolsLineScale(y0, 1, 330, 18)(value);
}
export function toolsScatterPoint(model: GuardModel, xMax: number, yMax: number) {
return { x: toolsScatterX(model.fpr, xMax), y: toolsScatterY(model.fnr, yMax) };
}
export function toolsParetoPoint(model: GuardModel, xMax: number, y0: number) {
return { x: toolsParetoX(model.p95, xMax), y: toolsParetoY(model.integral, y0) };
}
export function toolsGroupMean(
group: string,
key: ToolsVizMetric,
selectedModels: readonly GuardModel[]
) {
const values = selectedModels
.map((model) => toolsMetricValue(toolsCell(model, group), key))
.filter(isNumber);
if (!values.length) return null;
return values.reduce((sum, value) => sum + value, 0) / values.length;
}
export function toolsScoreDelta(model: GuardModel) {
const real = model.groups['Robustness Test (real)'];
const robust = model.groups['Robustness Test (robust)'];
if (!real || !robust) return 0;
return robust.score - real.score;
}
export function toolsFnrDelta(model: GuardModel) {
const real = model.groups['Robustness Test (real)'];
const robust = model.groups['Robustness Test (robust)'];
if (!real || !robust || !isNumber(real.fnr) || !isNumber(robust.fnr)) return null;
return robust.fnr - real.fnr;
}
export function toolsRobustRows(selectedModels: readonly GuardModel[]) {
return [...selectedModels]
.filter(
(model) => model.groups['Robustness Test (real)'] && model.groups['Robustness Test (robust)']
)
.sort((a, b) => toolsScoreDelta(a) - toolsScoreDelta(b));
}
export function toolsRowsToCsv(
selectedModels: readonly GuardModel[],
activeGroups: readonly string[]
) {
const rows = selectedModels.flatMap((model) =>
activeGroups.map((group) => {
const cell = toolsCell(model, group);
return [
model.name,
group,
cell?.score?.toFixed(4) ?? '',
cell?.fnr?.toFixed(4) ?? '',
cell?.fpr?.toFixed(4) ?? '',
cell?.f1?.toFixed(4) ?? ''
];
})
);
return [['model', 'group', 'score', 'fnr', 'fpr', 'f1'], ...rows]
.map((row) => row.map((cell) => `"${String(cell).replaceAll('"', '""')}"`).join(','))
.join('\n');
}
export function toolsCsvDataUri(
selectedModels: readonly GuardModel[],
activeGroups: readonly string[]
) {
return `data:text/csv;charset=utf-8,${encodeURIComponent(toolsRowsToCsv(selectedModels, activeGroups))}`;
}