Spaces:
Running
Running
Preserve evaluator_relationship when flattening model hierarchy
Browse filesflattenModelEvaluations rebuilt each BenchmarkEvaluation from
hierarchy_by_category and hardcoded evaluator_relationship to "other" in
the synthesized SourceMetadata, collapsing every 1st/3rd-party badge in the
UI. Use the authoritative source_metadata from evaluations_by_category when
present (older exports), and prefer a per-result source_metadata when the
pipeline carries it directly on the hierarchy row (newer exports).
- lib/hf-data.ts +63 -4
lib/hf-data.ts
CHANGED
|
@@ -529,6 +529,11 @@ export interface HFEvalModelResult {
|
|
| 529 |
evaluation_id?: string
|
| 530 |
retrieved_timestamp?: string
|
| 531 |
source_record_url?: string
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 532 |
detailed_evaluation_results?: string | null
|
| 533 |
detailed_evaluation_results_meta?: unknown
|
| 534 |
instance_level_data?: unknown
|
|
@@ -1234,7 +1239,8 @@ function flattenHierarchyNode(
|
|
| 1234 |
category: CategoryType,
|
| 1235 |
rawModelIds: Set<string>,
|
| 1236 |
variantLookup: Map<string, { variantKey: string; variantLabel: string }>,
|
| 1237 |
-
inheritedContext?: FlattenHierarchyContext
|
|
|
|
| 1238 |
): BenchmarkEvaluation[] {
|
| 1239 |
const evaluations: BenchmarkEvaluation[] = []
|
| 1240 |
const context = buildFlattenHierarchyContext(node, inheritedContext)
|
|
@@ -1257,6 +1263,7 @@ function flattenHierarchyNode(
|
|
| 1257 |
evaluationResults: EvaluationResult[]
|
| 1258 |
inlineSamples?: SampleResult[]
|
| 1259 |
latestTimestamp: string
|
|
|
|
| 1260 |
}
|
| 1261 |
>()
|
| 1262 |
|
|
@@ -1265,6 +1272,13 @@ function flattenHierarchyNode(
|
|
| 1265 |
const variantKey = variantMeta.variantKey || "default"
|
| 1266 |
const modelInfo = buildModelInfoForVariant(detail, result, variantMeta)
|
| 1267 |
const inlineSamples = parseInstanceLevelData(result.instance_level_data)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1268 |
const evaluationResult: EvaluationResult = {
|
| 1269 |
evaluation_name: metric.metric_name || metric.evaluation_name || metric.display_name,
|
| 1270 |
display_name: metric.display_name || metric.metric_name || metric.evaluation_name,
|
|
@@ -1292,6 +1306,7 @@ function flattenHierarchyNode(
|
|
| 1292 |
evaluationResults: [evaluationResult],
|
| 1293 |
inlineSamples: inlineSamples.length > 0 ? inlineSamples : undefined,
|
| 1294 |
latestTimestamp: result.retrieved_timestamp ?? detail.last_updated ?? "",
|
|
|
|
| 1295 |
})
|
| 1296 |
continue
|
| 1297 |
}
|
|
@@ -1305,6 +1320,13 @@ function flattenHierarchyNode(
|
|
| 1305 |
toComparableTimestamp(existing.latestTimestamp)
|
| 1306 |
) {
|
| 1307 |
existing.latestTimestamp = result.retrieved_timestamp ?? existing.latestTimestamp
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1308 |
}
|
| 1309 |
}
|
| 1310 |
|
|
@@ -1340,7 +1362,7 @@ function flattenHierarchyNode(
|
|
| 1340 |
slice_key: sliceKey,
|
| 1341 |
slice_name: sliceName,
|
| 1342 |
source_data: sourceData,
|
| 1343 |
-
source_metadata: sourceMetadata,
|
| 1344 |
model_info: variantGroup.modelInfo,
|
| 1345 |
evaluation_results: variantGroup.evaluationResults,
|
| 1346 |
detailed_evaluation_results_per_samples:
|
|
@@ -1353,7 +1375,15 @@ function flattenHierarchyNode(
|
|
| 1353 |
|
| 1354 |
for (const subtask of node.subtasks ?? []) {
|
| 1355 |
evaluations.push(
|
| 1356 |
-
...flattenHierarchyNode(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1357 |
)
|
| 1358 |
}
|
| 1359 |
|
|
@@ -1377,17 +1407,46 @@ export function flattenModelEvaluations(detail: HFModelDetail): BenchmarkEvaluat
|
|
| 1377 |
.filter(Boolean)
|
| 1378 |
)
|
| 1379 |
const variantLookup = buildVariantLookup(detail)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1380 |
|
| 1381 |
for (const [categoryKey, nodes] of Object.entries(detail.hierarchy_by_category ?? {})) {
|
| 1382 |
const mappedCategory = mapHFCategories([categoryKey])[0]
|
| 1383 |
for (const node of nodes) {
|
| 1384 |
-
evaluations.push(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1385 |
}
|
| 1386 |
}
|
| 1387 |
|
| 1388 |
return evaluations
|
| 1389 |
}
|
| 1390 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1391 |
/**
|
| 1392 |
* Map pipeline category labels to frontend CategoryType.
|
| 1393 |
*/
|
|
|
|
| 529 |
evaluation_id?: string
|
| 530 |
retrieved_timestamp?: string
|
| 531 |
source_record_url?: string
|
| 532 |
+
// Populated by pipeline versions that copy the parent record's provenance
|
| 533 |
+
// straight onto each hierarchy row. Older exports omit these; fall back to
|
| 534 |
+
// the evaluations_by_category index when missing.
|
| 535 |
+
source_metadata?: SourceMetadata
|
| 536 |
+
source_data?: SourceData | string[]
|
| 537 |
detailed_evaluation_results?: string | null
|
| 538 |
detailed_evaluation_results_meta?: unknown
|
| 539 |
instance_level_data?: unknown
|
|
|
|
| 1239 |
category: CategoryType,
|
| 1240 |
rawModelIds: Set<string>,
|
| 1241 |
variantLookup: Map<string, { variantKey: string; variantLabel: string }>,
|
| 1242 |
+
inheritedContext?: FlattenHierarchyContext,
|
| 1243 |
+
sourceMetadataByEvaluationId?: Map<string, SourceMetadata>
|
| 1244 |
): BenchmarkEvaluation[] {
|
| 1245 |
const evaluations: BenchmarkEvaluation[] = []
|
| 1246 |
const context = buildFlattenHierarchyContext(node, inheritedContext)
|
|
|
|
| 1263 |
evaluationResults: EvaluationResult[]
|
| 1264 |
inlineSamples?: SampleResult[]
|
| 1265 |
latestTimestamp: string
|
| 1266 |
+
sourceMetadataOverride?: SourceMetadata
|
| 1267 |
}
|
| 1268 |
>()
|
| 1269 |
|
|
|
|
| 1272 |
const variantKey = variantMeta.variantKey || "default"
|
| 1273 |
const modelInfo = buildModelInfoForVariant(detail, result, variantMeta)
|
| 1274 |
const inlineSamples = parseInstanceLevelData(result.instance_level_data)
|
| 1275 |
+
// Prefer source_metadata carried directly on the result row (populated
|
| 1276 |
+
// by newer pipeline runs). Fall back to the by-evaluation-id index built
|
| 1277 |
+
// from evaluations_by_category, which older exports still need.
|
| 1278 |
+
const evaluationIdForResult = result.evaluation_id
|
| 1279 |
+
const resolvedSourceMetadata: SourceMetadata | undefined =
|
| 1280 |
+
result.source_metadata ??
|
| 1281 |
+
(evaluationIdForResult ? sourceMetadataByEvaluationId?.get(evaluationIdForResult) : undefined)
|
| 1282 |
const evaluationResult: EvaluationResult = {
|
| 1283 |
evaluation_name: metric.metric_name || metric.evaluation_name || metric.display_name,
|
| 1284 |
display_name: metric.display_name || metric.metric_name || metric.evaluation_name,
|
|
|
|
| 1306 |
evaluationResults: [evaluationResult],
|
| 1307 |
inlineSamples: inlineSamples.length > 0 ? inlineSamples : undefined,
|
| 1308 |
latestTimestamp: result.retrieved_timestamp ?? detail.last_updated ?? "",
|
| 1309 |
+
sourceMetadataOverride: resolvedSourceMetadata,
|
| 1310 |
})
|
| 1311 |
continue
|
| 1312 |
}
|
|
|
|
| 1320 |
toComparableTimestamp(existing.latestTimestamp)
|
| 1321 |
) {
|
| 1322 |
existing.latestTimestamp = result.retrieved_timestamp ?? existing.latestTimestamp
|
| 1323 |
+
// Prefer source_metadata from the freshest submission when multiple
|
| 1324 |
+
// submissions land in the same variant bucket.
|
| 1325 |
+
if (resolvedSourceMetadata) {
|
| 1326 |
+
existing.sourceMetadataOverride = resolvedSourceMetadata
|
| 1327 |
+
}
|
| 1328 |
+
} else if (!existing.sourceMetadataOverride && resolvedSourceMetadata) {
|
| 1329 |
+
existing.sourceMetadataOverride = resolvedSourceMetadata
|
| 1330 |
}
|
| 1331 |
}
|
| 1332 |
|
|
|
|
| 1362 |
slice_key: sliceKey,
|
| 1363 |
slice_name: sliceName,
|
| 1364 |
source_data: sourceData,
|
| 1365 |
+
source_metadata: variantGroup.sourceMetadataOverride ?? sourceMetadata,
|
| 1366 |
model_info: variantGroup.modelInfo,
|
| 1367 |
evaluation_results: variantGroup.evaluationResults,
|
| 1368 |
detailed_evaluation_results_per_samples:
|
|
|
|
| 1375 |
|
| 1376 |
for (const subtask of node.subtasks ?? []) {
|
| 1377 |
evaluations.push(
|
| 1378 |
+
...flattenHierarchyNode(
|
| 1379 |
+
detail,
|
| 1380 |
+
subtask,
|
| 1381 |
+
category,
|
| 1382 |
+
rawModelIds,
|
| 1383 |
+
variantLookup,
|
| 1384 |
+
context,
|
| 1385 |
+
sourceMetadataByEvaluationId
|
| 1386 |
+
)
|
| 1387 |
)
|
| 1388 |
}
|
| 1389 |
|
|
|
|
| 1407 |
.filter(Boolean)
|
| 1408 |
)
|
| 1409 |
const variantLookup = buildVariantLookup(detail)
|
| 1410 |
+
// evaluations_by_category carries the authoritative source_metadata straight
|
| 1411 |
+
// from the pipeline. The hierarchy branch of this detail file doesn't, so
|
| 1412 |
+
// we build a (evaluation_id -> source_metadata) index here and look values
|
| 1413 |
+
// up per-result inside flattenHierarchyNode. Without this the hierarchy
|
| 1414 |
+
// fallback below hardcodes evaluator_relationship to "other" and every
|
| 1415 |
+
// 1st/3rd-party badge in the UI collapses to "Other".
|
| 1416 |
+
const sourceMetadataByEvaluationId = buildSourceMetadataIndex(detail)
|
| 1417 |
|
| 1418 |
for (const [categoryKey, nodes] of Object.entries(detail.hierarchy_by_category ?? {})) {
|
| 1419 |
const mappedCategory = mapHFCategories([categoryKey])[0]
|
| 1420 |
for (const node of nodes) {
|
| 1421 |
+
evaluations.push(
|
| 1422 |
+
...flattenHierarchyNode(
|
| 1423 |
+
detail,
|
| 1424 |
+
node,
|
| 1425 |
+
mappedCategory,
|
| 1426 |
+
rawModelIds,
|
| 1427 |
+
variantLookup,
|
| 1428 |
+
undefined,
|
| 1429 |
+
sourceMetadataByEvaluationId
|
| 1430 |
+
)
|
| 1431 |
+
)
|
| 1432 |
}
|
| 1433 |
}
|
| 1434 |
|
| 1435 |
return evaluations
|
| 1436 |
}
|
| 1437 |
|
| 1438 |
+
function buildSourceMetadataIndex(detail: HFModelDetail): Map<string, SourceMetadata> {
|
| 1439 |
+
const index = new Map<string, SourceMetadata>()
|
| 1440 |
+
for (const evals of Object.values(detail.evaluations_by_category ?? {})) {
|
| 1441 |
+
for (const evaluation of evals ?? []) {
|
| 1442 |
+
if (evaluation?.source_metadata && evaluation.evaluation_id) {
|
| 1443 |
+
index.set(evaluation.evaluation_id, evaluation.source_metadata)
|
| 1444 |
+
}
|
| 1445 |
+
}
|
| 1446 |
+
}
|
| 1447 |
+
return index
|
| 1448 |
+
}
|
| 1449 |
+
|
| 1450 |
/**
|
| 1451 |
* Map pipeline category labels to frontend CategoryType.
|
| 1452 |
*/
|