evijit HF Staff commited on
Commit
431b0cc
·
1 Parent(s): 8058fce

Preserve evaluator_relationship when flattening model hierarchy

Browse files

flattenModelEvaluations rebuilt each BenchmarkEvaluation from
hierarchy_by_category and hardcoded evaluator_relationship to "other" in
the synthesized SourceMetadata, collapsing every 1st/3rd-party badge in the
UI. Use the authoritative source_metadata from evaluations_by_category when
present (older exports), and prefer a per-result source_metadata when the
pipeline carries it directly on the hierarchy row (newer exports).

Files changed (1) hide show
  1. lib/hf-data.ts +63 -4
lib/hf-data.ts CHANGED
@@ -529,6 +529,11 @@ export interface HFEvalModelResult {
529
  evaluation_id?: string
530
  retrieved_timestamp?: string
531
  source_record_url?: string
 
 
 
 
 
532
  detailed_evaluation_results?: string | null
533
  detailed_evaluation_results_meta?: unknown
534
  instance_level_data?: unknown
@@ -1234,7 +1239,8 @@ function flattenHierarchyNode(
1234
  category: CategoryType,
1235
  rawModelIds: Set<string>,
1236
  variantLookup: Map<string, { variantKey: string; variantLabel: string }>,
1237
- inheritedContext?: FlattenHierarchyContext
 
1238
  ): BenchmarkEvaluation[] {
1239
  const evaluations: BenchmarkEvaluation[] = []
1240
  const context = buildFlattenHierarchyContext(node, inheritedContext)
@@ -1257,6 +1263,7 @@ function flattenHierarchyNode(
1257
  evaluationResults: EvaluationResult[]
1258
  inlineSamples?: SampleResult[]
1259
  latestTimestamp: string
 
1260
  }
1261
  >()
1262
 
@@ -1265,6 +1272,13 @@ function flattenHierarchyNode(
1265
  const variantKey = variantMeta.variantKey || "default"
1266
  const modelInfo = buildModelInfoForVariant(detail, result, variantMeta)
1267
  const inlineSamples = parseInstanceLevelData(result.instance_level_data)
 
 
 
 
 
 
 
1268
  const evaluationResult: EvaluationResult = {
1269
  evaluation_name: metric.metric_name || metric.evaluation_name || metric.display_name,
1270
  display_name: metric.display_name || metric.metric_name || metric.evaluation_name,
@@ -1292,6 +1306,7 @@ function flattenHierarchyNode(
1292
  evaluationResults: [evaluationResult],
1293
  inlineSamples: inlineSamples.length > 0 ? inlineSamples : undefined,
1294
  latestTimestamp: result.retrieved_timestamp ?? detail.last_updated ?? "",
 
1295
  })
1296
  continue
1297
  }
@@ -1305,6 +1320,13 @@ function flattenHierarchyNode(
1305
  toComparableTimestamp(existing.latestTimestamp)
1306
  ) {
1307
  existing.latestTimestamp = result.retrieved_timestamp ?? existing.latestTimestamp
 
 
 
 
 
 
 
1308
  }
1309
  }
1310
 
@@ -1340,7 +1362,7 @@ function flattenHierarchyNode(
1340
  slice_key: sliceKey,
1341
  slice_name: sliceName,
1342
  source_data: sourceData,
1343
- source_metadata: sourceMetadata,
1344
  model_info: variantGroup.modelInfo,
1345
  evaluation_results: variantGroup.evaluationResults,
1346
  detailed_evaluation_results_per_samples:
@@ -1353,7 +1375,15 @@ function flattenHierarchyNode(
1353
 
1354
  for (const subtask of node.subtasks ?? []) {
1355
  evaluations.push(
1356
- ...flattenHierarchyNode(detail, subtask, category, rawModelIds, variantLookup, context)
 
 
 
 
 
 
 
 
1357
  )
1358
  }
1359
 
@@ -1377,17 +1407,46 @@ export function flattenModelEvaluations(detail: HFModelDetail): BenchmarkEvaluat
1377
  .filter(Boolean)
1378
  )
1379
  const variantLookup = buildVariantLookup(detail)
 
 
 
 
 
 
 
1380
 
1381
  for (const [categoryKey, nodes] of Object.entries(detail.hierarchy_by_category ?? {})) {
1382
  const mappedCategory = mapHFCategories([categoryKey])[0]
1383
  for (const node of nodes) {
1384
- evaluations.push(...flattenHierarchyNode(detail, node, mappedCategory, rawModelIds, variantLookup))
 
 
 
 
 
 
 
 
 
 
1385
  }
1386
  }
1387
 
1388
  return evaluations
1389
  }
1390
 
 
 
 
 
 
 
 
 
 
 
 
 
1391
  /**
1392
  * Map pipeline category labels to frontend CategoryType.
1393
  */
 
529
  evaluation_id?: string
530
  retrieved_timestamp?: string
531
  source_record_url?: string
532
+ // Populated by pipeline versions that copy the parent record's provenance
533
+ // straight onto each hierarchy row. Older exports omit these; fall back to
534
+ // the evaluations_by_category index when missing.
535
+ source_metadata?: SourceMetadata
536
+ source_data?: SourceData | string[]
537
  detailed_evaluation_results?: string | null
538
  detailed_evaluation_results_meta?: unknown
539
  instance_level_data?: unknown
 
1239
  category: CategoryType,
1240
  rawModelIds: Set<string>,
1241
  variantLookup: Map<string, { variantKey: string; variantLabel: string }>,
1242
+ inheritedContext?: FlattenHierarchyContext,
1243
+ sourceMetadataByEvaluationId?: Map<string, SourceMetadata>
1244
  ): BenchmarkEvaluation[] {
1245
  const evaluations: BenchmarkEvaluation[] = []
1246
  const context = buildFlattenHierarchyContext(node, inheritedContext)
 
1263
  evaluationResults: EvaluationResult[]
1264
  inlineSamples?: SampleResult[]
1265
  latestTimestamp: string
1266
+ sourceMetadataOverride?: SourceMetadata
1267
  }
1268
  >()
1269
 
 
1272
  const variantKey = variantMeta.variantKey || "default"
1273
  const modelInfo = buildModelInfoForVariant(detail, result, variantMeta)
1274
  const inlineSamples = parseInstanceLevelData(result.instance_level_data)
1275
+ // Prefer source_metadata carried directly on the result row (populated
1276
+ // by newer pipeline runs). Fall back to the by-evaluation-id index built
1277
+ // from evaluations_by_category, which older exports still need.
1278
+ const evaluationIdForResult = result.evaluation_id
1279
+ const resolvedSourceMetadata: SourceMetadata | undefined =
1280
+ result.source_metadata ??
1281
+ (evaluationIdForResult ? sourceMetadataByEvaluationId?.get(evaluationIdForResult) : undefined)
1282
  const evaluationResult: EvaluationResult = {
1283
  evaluation_name: metric.metric_name || metric.evaluation_name || metric.display_name,
1284
  display_name: metric.display_name || metric.metric_name || metric.evaluation_name,
 
1306
  evaluationResults: [evaluationResult],
1307
  inlineSamples: inlineSamples.length > 0 ? inlineSamples : undefined,
1308
  latestTimestamp: result.retrieved_timestamp ?? detail.last_updated ?? "",
1309
+ sourceMetadataOverride: resolvedSourceMetadata,
1310
  })
1311
  continue
1312
  }
 
1320
  toComparableTimestamp(existing.latestTimestamp)
1321
  ) {
1322
  existing.latestTimestamp = result.retrieved_timestamp ?? existing.latestTimestamp
1323
+ // Prefer source_metadata from the freshest submission when multiple
1324
+ // submissions land in the same variant bucket.
1325
+ if (resolvedSourceMetadata) {
1326
+ existing.sourceMetadataOverride = resolvedSourceMetadata
1327
+ }
1328
+ } else if (!existing.sourceMetadataOverride && resolvedSourceMetadata) {
1329
+ existing.sourceMetadataOverride = resolvedSourceMetadata
1330
  }
1331
  }
1332
 
 
1362
  slice_key: sliceKey,
1363
  slice_name: sliceName,
1364
  source_data: sourceData,
1365
+ source_metadata: variantGroup.sourceMetadataOverride ?? sourceMetadata,
1366
  model_info: variantGroup.modelInfo,
1367
  evaluation_results: variantGroup.evaluationResults,
1368
  detailed_evaluation_results_per_samples:
 
1375
 
1376
  for (const subtask of node.subtasks ?? []) {
1377
  evaluations.push(
1378
+ ...flattenHierarchyNode(
1379
+ detail,
1380
+ subtask,
1381
+ category,
1382
+ rawModelIds,
1383
+ variantLookup,
1384
+ context,
1385
+ sourceMetadataByEvaluationId
1386
+ )
1387
  )
1388
  }
1389
 
 
1407
  .filter(Boolean)
1408
  )
1409
  const variantLookup = buildVariantLookup(detail)
1410
+ // evaluations_by_category carries the authoritative source_metadata straight
1411
+ // from the pipeline. The hierarchy branch of this detail file doesn't, so
1412
+ // we build a (evaluation_id -> source_metadata) index here and look values
1413
+ // up per-result inside flattenHierarchyNode. Without this the hierarchy
1414
+ // fallback below hardcodes evaluator_relationship to "other" and every
1415
+ // 1st/3rd-party badge in the UI collapses to "Other".
1416
+ const sourceMetadataByEvaluationId = buildSourceMetadataIndex(detail)
1417
 
1418
  for (const [categoryKey, nodes] of Object.entries(detail.hierarchy_by_category ?? {})) {
1419
  const mappedCategory = mapHFCategories([categoryKey])[0]
1420
  for (const node of nodes) {
1421
+ evaluations.push(
1422
+ ...flattenHierarchyNode(
1423
+ detail,
1424
+ node,
1425
+ mappedCategory,
1426
+ rawModelIds,
1427
+ variantLookup,
1428
+ undefined,
1429
+ sourceMetadataByEvaluationId
1430
+ )
1431
+ )
1432
  }
1433
  }
1434
 
1435
  return evaluations
1436
  }
1437
 
1438
+ function buildSourceMetadataIndex(detail: HFModelDetail): Map<string, SourceMetadata> {
1439
+ const index = new Map<string, SourceMetadata>()
1440
+ for (const evals of Object.values(detail.evaluations_by_category ?? {})) {
1441
+ for (const evaluation of evals ?? []) {
1442
+ if (evaluation?.source_metadata && evaluation.evaluation_id) {
1443
+ index.set(evaluation.evaluation_id, evaluation.source_metadata)
1444
+ }
1445
+ }
1446
+ }
1447
+ return index
1448
+ }
1449
+
1450
  /**
1451
  * Map pipeline category labels to frontend CategoryType.
1452
  */