leaderboard / src /lib /scopeSelector.ts
dbonet's picture paumateo's picture
Publish TabBioMed leaderboard
d189f56
Raw History Blame Contribute Delete
5.34 kB
import type {
BenchmarkData,
CompetitorView,
DatasetInfo,
DomainSlug,
LeaderboardEntry,
MissingPolicyView,
MissingSlice,
PrecomputedModeResult,
PrecomputedScope,
WinRatePayload,
} from '../types/benchmark';
export interface ResolvedScopeView {
scope: PrecomputedScope;
allEntries: LeaderboardEntry[];
visibleEntries: Array<LeaderboardEntry & { visibleRank: number }>;
datasets: DatasetInfo[];
effectiveDatasetsCount: number;
winRate: WinRatePayload | null;
anchorText: string;
}
export function getMissingScopeKey(domain: DomainSlug | null, slice: Exclude<MissingSlice, 'all'>): string {
return domain ? `domain:${domain}:${slice}` : `missing:${slice}`;
}
export function resolveScopeKey(
data: BenchmarkData,
domainSlug: DomainSlug | null,
missingSlice: MissingSlice,
secondaryScope: string // 'all_tasks' | 'classification_23' | 'ehr_up_to_2000' | 'ehr_over_2000' | 'Q1'..'Q4' | 'task:...' | 'wide:over_2000' | 'cat:...'
): string {
if (missingSlice !== 'all') {
const key = getMissingScopeKey(domainSlug, missingSlice);
if (data.scopes[key]) return key;
}
// Domain selected
if (domainSlug) {
if (domainSlug === 'ehr') {
if (secondaryScope === 'ehr_up_to_2000' && data.scopes['panel:ehr_up_to_2000']) {
return 'panel:ehr_up_to_2000';
}
if (secondaryScope === 'ehr_over_2000' && data.scopes['panel:ehr_over_2000']) {
return 'panel:ehr_over_2000';
}
}
if (secondaryScope === 'classification_23' && data.scopes[`domain_cls:${domainSlug}`]) {
return `domain_cls:${domainSlug}`;
}
return `domain:${domainSlug}`;
}
// Secondary regime / task / feature scope
if (secondaryScope === 'classification_23') return 'global_classification';
if (secondaryScope.startsWith('quadrant:') || secondaryScope.startsWith('quadrant_cls:')) {
if (data.scopes[secondaryScope]) return secondaryScope;
}
if (
secondaryScope.startsWith('task:') ||
secondaryScope.startsWith('wide:') ||
secondaryScope.startsWith('cat:')
) {
if (data.scopes[secondaryScope]) return secondaryScope;
}
return 'global_all_tasks';
}
export function getModelFilter(excludedModels: ReadonlySet<string>, searchQuery: string) {
const query = searchQuery.trim().toLowerCase();
return (entry: LeaderboardEntry) =>
!excludedModels.has(entry.modelId) &&
(!query ||
entry.displayName.toLowerCase().includes(query) ||
entry.family.toLowerCase().includes(query));
}
export function getScopeResults(
scope: PrecomputedScope,
mode: 'selected' | 'paired',
policy: MissingPolicyView,
): PrecomputedModeResult | null {
if (policy === 'rf_imputed' || scope.nDatasetsNoOverlap === scope.nDatasets) {
return scope[mode];
}
return (mode === 'selected' ? scope.selectedNoOverlap : scope.pairedNoOverlap) ?? null;
}
export function datasetsForScope(
data: BenchmarkData,
scope: PrecomputedScope,
results: PrecomputedModeResult,
): DatasetInfo[] {
const ids = new Set(results.datasetIds ?? scope.datasetIds);
return data.datasets.filter((dataset) => ids.has(dataset.id));
}
function getVisibleParetoFront(
entries: LeaderboardEntry[],
field: 'medianTrainPer1k' | 'medianInferPer1k',
): Set<string> {
const points = entries.flatMap((entry) => {
const cost = entry[field];
return cost !== null && cost > 0 ? [{ key: entry.key, elo: entry.elo, cost }] : [];
});
points.sort((a, b) => a.cost - b.cost || b.elo - a.elo);
const frontier = new Set<string>();
let bestElo = -Infinity;
for (const point of points) {
if (point.elo > bestElo + 1e-9) {
frontier.add(point.key);
bestElo = point.elo;
}
}
return frontier;
}
export function getScopeView(
data: BenchmarkData,
scopeKey: string,
competitorView: CompetitorView,
missingPolicy: MissingPolicyView,
matchesModel: (entry: LeaderboardEntry) => boolean,
): ResolvedScopeView {
const scope = data.scopes[scopeKey] ?? data.scopes['global_all_tasks'];
const mode = competitorView === 'selected' ? 'selected' : 'paired';
const modeBlock = getScopeResults(scope, mode, missingPolicy) ?? scope[mode];
let baseEntries = modeBlock.entries;
if (competitorView === 'tuned_only') {
baseEntries = modeBlock.entries.filter(
(e) => e.variant === 'tuned' || (!e.hpoEnabled && e.variant === 'default')
);
} else if (competitorView === 'default_only') {
baseEntries = modeBlock.entries.filter((e) => e.variant === 'default');
}
const filteredEntries = baseEntries.filter(matchesModel);
const trainingFrontier = getVisibleParetoFront(filteredEntries, 'medianTrainPer1k');
const inferenceFrontier = getVisibleParetoFront(filteredEntries, 'medianInferPer1k');
const visibleEntries = filteredEntries.map((entry, idx) => ({
...entry,
visibleRank: idx + 1,
isParetoTrain: trainingFrontier.has(entry.key),
isParetoInfer: inferenceFrontier.has(entry.key),
}));
const datasets = datasetsForScope(data, scope, modeBlock);
const anchorText =
competitorView === 'selected'
? 'RandomForest = 1000'
: 'RandomForest (Default) = 1000';
return {
scope,
allEntries: baseEntries,
visibleEntries,
datasets,
effectiveDatasetsCount: modeBlock.nDatasets,
winRate: modeBlock.winRate ?? null,
anchorText,
};
}