import { isNumber, sizeNumber } from '$lib/helpers/leaderboard'; import type { DetailsDataset } from '$lib/types/details-data'; import type { BenchmarkCard, BenchmarkFamilyLanguage, BenchmarkRankingRow, DetailMetric, DetailsLanguageFilter, MatrixGroup, MatrixDataset, MatrixSortDirection, ModelGroupRow, ModelSort, ModelTypeFilter, RadarPoint, SizeFilter } from '$lib/types/details'; import type { DetailsDatasetMatrixCell } from '$lib/types/details-data'; import type { LocalizedString } from '$lib/types/i18n'; import type { GroupScore, GuardModel } from '$lib/types/leaderboard'; type SupportedLocale = keyof LocalizedString; type BenchmarkLeaderMetric = NonNullable['metric']; function localized(value: LocalizedString | null, locale: SupportedLocale, fallback: string) { return value?.[locale] ?? value?.en ?? fallback; } export function groupMatchesLanguage(dataset: MatrixDataset, language: DetailsLanguageFilter) { if (language === 'any') return true; return dataset.language === language; } export function visibleDetailMatrixDatasets( detailMatrixDatasets: readonly MatrixDataset[], selectedGroups: readonly string[], language: DetailsLanguageFilter ) { return detailMatrixDatasets.filter((dataset) => { if (!selectedGroups.includes(dataset.sourceGroup)) return false; return groupMatchesLanguage(dataset, language); }); } export function buildMatrixGroups(datasets: readonly MatrixDataset[]) { const groups: Array<{ key: string; label: string; datasets: MatrixDataset[] }> = []; for (const dataset of datasets) { const group = groups.find((item) => item.key === dataset.group); if (group) { group.datasets.push(dataset); continue; } groups.push({ key: dataset.group, label: dataset.group, datasets: [dataset] }); } return groups; } export function detailMetricValue( group: GroupScore | DetailsDatasetMatrixCell | undefined, metric: DetailMetric ) { if (!group) return null; if (metric === 'score') return group.score; if (metric === 'f1') return group.f1 ?? group.score; if (metric === 'recall') { if ('recall' in group && group.recall !== undefined) return group.recall; return isNumber(group.fnr) ? 1 - group.fnr : null; } if (metric === 'precision') { return 'precision' in group ? (group.precision ?? null) : null; } if (metric === 'accuracy') { return 'accuracy' in group ? (group.accuracy ?? null) : null; } return group[metric]; } export function matrixDatasetLabel(dataset: MatrixDataset) { return dataset.label; } function modelKey(model: GuardModel) { return model.modelId ?? model.name; } function csvCell(value: string | number | null | undefined) { return `"${String(value ?? '').replaceAll('"', '""')}"`; } function csvNumber(value: number | null | undefined, digits = 4) { return isNumber(value) ? value.toFixed(digits) : ''; } export function matrixCellValue( model: GuardModel, dataset: MatrixDataset, metric: DetailMetric, datasetCells: Record> ) { const cell = datasetCells[modelKey(model)]?.[dataset.id]; return detailMetricValue(cell, metric); } export function detailMetricSortPolarity(metric: DetailMetric) { return metric === 'fpr' || metric === 'fnr' ? -1 : 1; } export function sortMatrixRows( rows: readonly GuardModel[], metric: DetailMetric, sortDatasetName: string | null, sortDirection: MatrixSortDirection, detailMatrixDatasets: readonly MatrixDataset[], datasetCells: Record> ) { const sorted = [...rows]; const sortDataset = sortDatasetName ? detailMatrixDatasets.find((dataset) => dataset.name === sortDatasetName) : null; if (!sortDataset) { return sorted.sort((a, b) => sortDirection === -1 ? b.integral - a.integral : a.integral - b.integral ); } const polarity = detailMetricSortPolarity(metric); return sorted.sort((a, b) => { const aValue = matrixCellValue(a, sortDataset, metric, datasetCells); const bValue = matrixCellValue(b, sortDataset, metric, datasetCells); const aEmpty = !isNumber(aValue); const bEmpty = !isNumber(bValue); if (aEmpty && bEmpty) return 0; if (aEmpty) return 1; if (bEmpty) return -1; const comparison = ((bValue ?? 0) - (aValue ?? 0)) * polarity; return sortDirection === -1 ? comparison : -comparison; }); } export function buildMatrixRows( search: string, metric: DetailMetric, sortDatasetName: string | null, sortDirection: MatrixSortDirection, sourceModels: readonly GuardModel[], detailMatrixDatasets: readonly MatrixDataset[], datasetCells: Record> ) { const normalizedSearch = search.trim().toLowerCase(); const rows = normalizedSearch ? sourceModels.filter((model) => [model.short, model.name, model.family, model.org] .join(' ') .toLowerCase() .includes(normalizedSearch) ) : sourceModels; return sortMatrixRows( rows, metric, sortDatasetName, sortDirection, detailMatrixDatasets, datasetCells ); } export function benchmarkRows( groupName: string | null, models: readonly GuardModel[] ): BenchmarkRankingRow[] { if (!groupName) return []; const metric = benchmarkLeaderMetric(groupName, models); return models .map((model) => { const group = model.groups[groupName]; return { model, score: group?.f1 ?? null, fpr: group?.fpr ?? null, fnr: group?.fnr ?? null, hasData: isNumber(group?.f1) || isNumber(group?.fpr) || isNumber(group?.fnr) }; }) .sort((a, b) => { if (a.hasData !== b.hasData) return a.hasData ? -1 : 1; const aValue = benchmarkRowMetricValue(a, metric); const bValue = benchmarkRowMetricValue(b, metric); const aEmpty = !isNumber(aValue); const bEmpty = !isNumber(bValue); if (aEmpty && bEmpty) return 0; if (aEmpty) return 1; if (bEmpty) return -1; return metric === 'f1' ? (bValue ?? 0) - (aValue ?? 0) : (aValue ?? 0) - (bValue ?? 0); }); } function benchmarkLeaderMetric( groupName: string, models: readonly GuardModel[] ): BenchmarkLeaderMetric | null { const metrics: BenchmarkLeaderMetric[] = ['f1', 'fnr', 'fpr']; return ( metrics.find((metric) => models.some((model) => isNumber(model.groups[groupName]?.[metric]))) ?? null ); } function benchmarkMetricValue(group: GroupScore | undefined, metric: BenchmarkLeaderMetric | null) { if (!group || !metric) return null; return group[metric]; } function benchmarkRowMetricValue(row: BenchmarkRankingRow, metric: BenchmarkLeaderMetric | null) { if (!metric) return null; return metric === 'f1' ? row.score : row[metric]; } function benchmarkLeader( groupName: string, models: readonly GuardModel[] ): BenchmarkCard['leader'] { const metric = benchmarkLeaderMetric(groupName, models); if (!metric) return null; return models.reduce((leader, model) => { const group = model.groups[groupName]; const value = benchmarkMetricValue(group, metric); if (!isNumber(value)) return leader; if (leader && (metric === 'f1' ? leader.value >= value : leader.value <= value)) return leader; return { modelShort: model.short, value, metric }; }, null); } export function buildBenchmarkCards( search: string, language: BenchmarkFamilyLanguage, dataset: DetailsDataset, locale: SupportedLocale ) { const normalizedSearch = search.trim().toLowerCase(); return dataset.allGroups .filter((group) => { const meta = dataset.benchmarkMeta[group]; if (!meta) return false; if (language !== 'all' && !meta.languages.includes(language)) return false; if (!normalizedSearch) return true; return [ localized(dataset.groupLabels[group], locale, group), localized(meta.description, locale, '') ] .join(' ') .toLowerCase() .includes(normalizedSearch); }) .map((group) => ({ name: group, meta: dataset.benchmarkMeta[group], leader: benchmarkLeader(group, dataset.models) })); } export function matchesLanguage(model: GuardModel, language: DetailsLanguageFilter) { if (language === 'any') return true; return model.languages.includes(language); } export function matchesSize(model: GuardModel, filter: SizeFilter) { if (filter === 'all') return true; const size = sizeNumber(model.params); if (filter === 'small') return size < 1; if (filter === 'medium') return size >= 1 && size <= 8; return size > 8; } export function modelSortValue(model: GuardModel, sort: ModelSort) { if (sort === 'f1') return model.f1; if (sort === 'p95') return -model.p95; if (sort === 'size') return sizeNumber(model.params); return model.integral; } export function filterModels( search: string, language: DetailsLanguageFilter, type: ModelTypeFilter, size: SizeFilter, sort: ModelSort, models: readonly GuardModel[] ) { const normalizedSearch = search.trim().toLowerCase(); return [...models] .filter((model) => { if (type !== 'all' && model.type !== type) return false; if (!matchesSize(model, size)) return false; if (!matchesLanguage(model, language)) return false; if (!normalizedSearch) return true; return [model.short, model.name, model.family, model.org] .join(' ') .toLowerCase() .includes(normalizedSearch); }) .sort((a, b) => modelSortValue(b, sort) - modelSortValue(a, sort)); } export function detailMatrixRowsToCsv( rows: readonly GuardModel[], groups: readonly MatrixGroup[], metric: DetailMetric, datasetCells: Record> ) { const datasets = groups.flatMap((group) => group.datasets.map((dataset) => ({ ...dataset, header: `${group.label} / ${dataset.label} ${metric}` })) ); const header = [ 'rank', 'model', 'organization', 'size', 'integral', 'metric', ...datasets.map((dataset) => dataset.header) ]; const lines = rows.map((model, index) => [ index + 1, model.name, model.org, model.params, csvNumber(model.integral), metric, ...datasets.map((dataset) => csvNumber(detailMetricValue(datasetCells[modelKey(model)]?.[dataset.id], metric)) ) ]); return [header, ...lines].map((row) => row.map(csvCell).join(',')).join('\n'); } export function detailModelRowsToCsv(rows: readonly GuardModel[]) { const header = [ 'rank', 'model', 'organization', 'family', 'type', 'license', 'size', 'languages', 'integral', 'min_group', 'overall_fpr', 'overall_fnr', 'overall_f1', 'p50_ms', 'p95_ms', 'p99_ms', 'run_date' ]; const lines = rows.map((model, index) => [ index + 1, model.name, model.org, model.family, model.type, model.license, model.params, model.languages.join('; '), csvNumber(model.integral), csvNumber(model.minGroup), csvNumber(model.fpr), csvNumber(model.fnr), csvNumber(model.f1), csvNumber(model.p50, 1), csvNumber(model.p95, 1), csvNumber(model.p99, 1), model.runDate ]); return [header, ...lines].map((row) => row.map(csvCell).join(',')).join('\n'); } function groupName(key: string, dataset: DetailsDataset, locale: SupportedLocale) { return localized(dataset.groupLabels[key], locale, key); } export function modelGroupRows( model: GuardModel | null, dataset: DetailsDataset, locale: SupportedLocale ) { if (!model) return []; return Object.entries(model.groups) .map(([name, group]) => ({ name: groupName(name, dataset, locale), score: group.score, recall: group.recall ?? null, precision: group.precision ?? null })) .sort((a, b) => b.score - a.score); } export function modelRadarGroups( model: GuardModel | null, dataset: DetailsDataset, locale: SupportedLocale ) { if (!model) return []; const rows: ModelGroupRow[] = []; for (const name of dataset.allGroups) { const group = model.groups[name]; if (!group || !isNumber(group.score)) continue; rows.push({ name: groupName(name, dataset, locale), score: group.score, recall: group.recall ?? null, precision: group.precision ?? null }); } return rows; } export function weakestModelGroups(groups: readonly ModelGroupRow[]) { return [...groups].reverse().slice(0, 5); } export function radarPoint(index: number, total: number, radius: number) { if (!total) return [160, 160] as const; const angle = -Math.PI / 2 + (2 * Math.PI * index) / total; return [160 + Math.cos(angle) * radius, 160 + Math.sin(angle) * radius] as const; } export function shortRadarLabel(label: string) { return label.length > 14 ? `${label.slice(0, 13)}...` : label; } export function radarPoints(groups: readonly ModelGroupRow[]) { const total = groups.length; if (!total) return []; return groups.map((group, index) => { const [x, y] = radarPoint(index, total, 118 * Math.max(0, Math.min(1, group.score))); const [labelX, labelY] = radarPoint(index, total, 142); return { ...group, x, y, labelX, labelY, label: shortRadarLabel(group.name), anchor: labelX < 155 ? 'end' : labelX > 165 ? 'start' : 'middle' }; }); } export function radarPolygon(points: readonly RadarPoint[]) { return points.map((point) => `${point.x.toFixed(1)},${point.y.toFixed(1)}`).join(' '); } export function radarGridPolygons(groups: readonly ModelGroupRow[]) { return [0.25, 0.5, 0.75, 1].map((scale) => groups .map((_, index) => { const [x, y] = radarPoint(index, groups.length, 118 * scale); return `${x.toFixed(1)},${y.toFixed(1)}`; }) .join(' ') ); }