Spaces:
Running
Running
File size: 5,342 Bytes
d189f56 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 | import type {
BenchmarkData,
CompetitorView,
DatasetInfo,
DomainSlug,
LeaderboardEntry,
MissingPolicyView,
MissingSlice,
PrecomputedModeResult,
PrecomputedScope,
WinRatePayload,
} from '../types/benchmark';
export interface ResolvedScopeView {
scope: PrecomputedScope;
allEntries: LeaderboardEntry[];
visibleEntries: Array<LeaderboardEntry & { visibleRank: number }>;
datasets: DatasetInfo[];
effectiveDatasetsCount: number;
winRate: WinRatePayload | null;
anchorText: string;
}
export function getMissingScopeKey(domain: DomainSlug | null, slice: Exclude<MissingSlice, 'all'>): string {
return domain ? `domain:${domain}:${slice}` : `missing:${slice}`;
}
export function resolveScopeKey(
data: BenchmarkData,
domainSlug: DomainSlug | null,
missingSlice: MissingSlice,
secondaryScope: string // 'all_tasks' | 'classification_23' | 'ehr_up_to_2000' | 'ehr_over_2000' | 'Q1'..'Q4' | 'task:...' | 'wide:over_2000' | 'cat:...'
): string {
if (missingSlice !== 'all') {
const key = getMissingScopeKey(domainSlug, missingSlice);
if (data.scopes[key]) return key;
}
// Domain selected
if (domainSlug) {
if (domainSlug === 'ehr') {
if (secondaryScope === 'ehr_up_to_2000' && data.scopes['panel:ehr_up_to_2000']) {
return 'panel:ehr_up_to_2000';
}
if (secondaryScope === 'ehr_over_2000' && data.scopes['panel:ehr_over_2000']) {
return 'panel:ehr_over_2000';
}
}
if (secondaryScope === 'classification_23' && data.scopes[`domain_cls:${domainSlug}`]) {
return `domain_cls:${domainSlug}`;
}
return `domain:${domainSlug}`;
}
// Secondary regime / task / feature scope
if (secondaryScope === 'classification_23') return 'global_classification';
if (secondaryScope.startsWith('quadrant:') || secondaryScope.startsWith('quadrant_cls:')) {
if (data.scopes[secondaryScope]) return secondaryScope;
}
if (
secondaryScope.startsWith('task:') ||
secondaryScope.startsWith('wide:') ||
secondaryScope.startsWith('cat:')
) {
if (data.scopes[secondaryScope]) return secondaryScope;
}
return 'global_all_tasks';
}
export function getModelFilter(excludedModels: ReadonlySet<string>, searchQuery: string) {
const query = searchQuery.trim().toLowerCase();
return (entry: LeaderboardEntry) =>
!excludedModels.has(entry.modelId) &&
(!query ||
entry.displayName.toLowerCase().includes(query) ||
entry.family.toLowerCase().includes(query));
}
export function getScopeResults(
scope: PrecomputedScope,
mode: 'selected' | 'paired',
policy: MissingPolicyView,
): PrecomputedModeResult | null {
if (policy === 'rf_imputed' || scope.nDatasetsNoOverlap === scope.nDatasets) {
return scope[mode];
}
return (mode === 'selected' ? scope.selectedNoOverlap : scope.pairedNoOverlap) ?? null;
}
export function datasetsForScope(
data: BenchmarkData,
scope: PrecomputedScope,
results: PrecomputedModeResult,
): DatasetInfo[] {
const ids = new Set(results.datasetIds ?? scope.datasetIds);
return data.datasets.filter((dataset) => ids.has(dataset.id));
}
function getVisibleParetoFront(
entries: LeaderboardEntry[],
field: 'medianTrainPer1k' | 'medianInferPer1k',
): Set<string> {
const points = entries.flatMap((entry) => {
const cost = entry[field];
return cost !== null && cost > 0 ? [{ key: entry.key, elo: entry.elo, cost }] : [];
});
points.sort((a, b) => a.cost - b.cost || b.elo - a.elo);
const frontier = new Set<string>();
let bestElo = -Infinity;
for (const point of points) {
if (point.elo > bestElo + 1e-9) {
frontier.add(point.key);
bestElo = point.elo;
}
}
return frontier;
}
export function getScopeView(
data: BenchmarkData,
scopeKey: string,
competitorView: CompetitorView,
missingPolicy: MissingPolicyView,
matchesModel: (entry: LeaderboardEntry) => boolean,
): ResolvedScopeView {
const scope = data.scopes[scopeKey] ?? data.scopes['global_all_tasks'];
const mode = competitorView === 'selected' ? 'selected' : 'paired';
const modeBlock = getScopeResults(scope, mode, missingPolicy) ?? scope[mode];
let baseEntries = modeBlock.entries;
if (competitorView === 'tuned_only') {
baseEntries = modeBlock.entries.filter(
(e) => e.variant === 'tuned' || (!e.hpoEnabled && e.variant === 'default')
);
} else if (competitorView === 'default_only') {
baseEntries = modeBlock.entries.filter((e) => e.variant === 'default');
}
const filteredEntries = baseEntries.filter(matchesModel);
const trainingFrontier = getVisibleParetoFront(filteredEntries, 'medianTrainPer1k');
const inferenceFrontier = getVisibleParetoFront(filteredEntries, 'medianInferPer1k');
const visibleEntries = filteredEntries.map((entry, idx) => ({
...entry,
visibleRank: idx + 1,
isParetoTrain: trainingFrontier.has(entry.key),
isParetoInfer: inferenceFrontier.has(entry.key),
}));
const datasets = datasetsForScope(data, scope, modeBlock);
const anchorText =
competitorView === 'selected'
? 'RandomForest = 1000'
: 'RandomForest (Default) = 1000';
return {
scope,
allEntries: baseEntries,
visibleEntries,
datasets,
effectiveDatasetsCount: modeBlock.nDatasets,
winRate: modeBlock.winRate ?? null,
anchorText,
};
}
|