Spaces:
Running
Resolve score scales from producer-stamped registry bounds
Browse filesNew snapshots stamp every comparison-index metric entry with
canonical_min_score / canonical_max_score — the effective metric's
registry bounds, i.e. the scale score_canonical sits on. Scale
resolution now consults them, with precedence:
1. mul100 seen => percent registry
2. div100 seen => fraction registry
3. registry bounds when usable (min 0 / max 100 => percent;
max <= 1.5 => fraction; anything else unusable)
4. the existing 'none'-cell unit vote, keeping both guards
(unit conflict => null, anchor required => null)
Bounds let two previously-guessed group shapes resolve exactly:
- all-'curated' groups (anchor-neutral tags, e.g. WildBench's 1-10
points folded onto [0,1] wb-score): a model whose only appearance
is the curated source now shows ~46% instead of the legacy "4.6%";
- unit-conflicted all-'none' groups (terminalbench-hard's lying
"percent" unit): now exact 11.0% instead of the legacy fallback.
The old-snapshot and no_bounds guards are unchanged. The display-axis
census now reads 'none' cells' source units and skips scale-neutral
'curated' cells (ties still break percent-ward). canonicalCellIsPercent
takes the same optional bounds for the sibling-whisker path; group call
sites (overlaps, histogram builder, signals strip) pass the metric
entry's bounds, merged across sibling metrics with conflicts treated
as unusable.
Verified against warehouse/2026-08-16T19-43-10Z through the real
buildOverlapRows path: openai/gpt-5.4 wildbench 45.97 -> "46.0%",
DeepSeek-R1 terminalbench-hard -> "11.0%".
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
- components/benchmark-detail.tsx +9 -1
- components/signals/benchmark-signals-strip.tsx +17 -2
- lib/backend-artifacts.ts +5 -0
- lib/overlaps.ts +17 -1
- lib/score-scale.ts +86 -15
- tests/overlaps.test.ts +90 -0
|
@@ -2994,6 +2994,9 @@ export function BenchmarkDetail({
|
|
| 2994 |
const currentCell = currentRow ?? byModelRow
|
| 2995 |
const scaleGroup = resolveCanonicalScaleGroup(
|
| 2996 |
[...(currentCell ? [currentCell] : []), ...peerRows].map(scaleCellOf),
|
|
|
|
|
|
|
|
|
|
| 2997 |
)
|
| 2998 |
const resolvedIsPercent = scaleGroup
|
| 2999 |
? scaleGroup.percentSourceCount >= scaleGroup.fractionSourceCount
|
|
@@ -3949,7 +3952,12 @@ export function BenchmarkDetail({
|
|
| 3949 |
// Flagged or old-snapshot sibling cells keep the legacy heuristic
|
| 3950 |
// for that row.
|
| 3951 |
const siblingCellScale =
|
| 3952 |
-
activeHist.resolvedIsPercent != null
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3953 |
const reconciledScore =
|
| 3954 |
siblingCellScale != null && siblingCell.scoreCanonical != null
|
| 3955 |
? scaleOnto(siblingCell.scoreCanonical, siblingCellScale, histIsPercent)
|
|
|
|
| 2994 |
const currentCell = currentRow ?? byModelRow
|
| 2995 |
const scaleGroup = resolveCanonicalScaleGroup(
|
| 2996 |
[...(currentCell ? [currentCell] : []), ...peerRows].map(scaleCellOf),
|
| 2997 |
+
// Metric-level registry bounds (the scale score_canonical sits
|
| 2998 |
+
// on): resolve anchor-neutral (all-curated) groups exactly.
|
| 2999 |
+
{ min: metric.canonical_min_score, max: metric.canonical_max_score },
|
| 3000 |
)
|
| 3001 |
const resolvedIsPercent = scaleGroup
|
| 3002 |
? scaleGroup.percentSourceCount >= scaleGroup.fractionSourceCount
|
|
|
|
| 3952 |
// Flagged or old-snapshot sibling cells keep the legacy heuristic
|
| 3953 |
// for that row.
|
| 3954 |
const siblingCellScale =
|
| 3955 |
+
activeHist.resolvedIsPercent != null
|
| 3956 |
+
? canonicalCellIsPercent(siblingCell, {
|
| 3957 |
+
min: siblingMetric.canonical_min_score,
|
| 3958 |
+
max: siblingMetric.canonical_max_score,
|
| 3959 |
+
})
|
| 3960 |
+
: null
|
| 3961 |
const reconciledScore =
|
| 3962 |
siblingCellScale != null && siblingCell.scoreCanonical != null
|
| 3963 |
? scaleOnto(siblingCell.scoreCanonical, siblingCellScale, histIsPercent)
|
|
@@ -11,7 +11,11 @@ import type {
|
|
| 11 |
} from "@/lib/backend-artifacts"
|
| 12 |
import type { BenchmarkEvalSummary } from "@/lib/eval-processing"
|
| 13 |
import type { ModelResultForBenchmark } from "@/lib/eval-processing"
|
| 14 |
-
import {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
type SignalId = "reproducibility" | "completeness" | "provenance" | "comparability"
|
| 17 |
|
|
@@ -779,7 +783,18 @@ function buildCrossSuiteAggregate(
|
|
| 779 |
}
|
| 780 |
}
|
| 781 |
}
|
| 782 |
-
const scaleGroup = resolveCanonicalScaleGroup(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 783 |
// Same display convention as the legacy guess (any percent-scale source
|
| 784 |
// pulls the whole table onto the 0-100 axis), read off the producer
|
| 785 |
// fields when available.
|
|
|
|
| 11 |
} from "@/lib/backend-artifacts"
|
| 12 |
import type { BenchmarkEvalSummary } from "@/lib/eval-processing"
|
| 13 |
import type { ModelResultForBenchmark } from "@/lib/eval-processing"
|
| 14 |
+
import {
|
| 15 |
+
mergeRegistryBounds,
|
| 16 |
+
resolveCanonicalScaleGroup,
|
| 17 |
+
type CanonicalScaleCell,
|
| 18 |
+
} from "@/lib/score-scale"
|
| 19 |
|
| 20 |
type SignalId = "reproducibility" | "completeness" | "provenance" | "comparability"
|
| 21 |
|
|
|
|
| 783 |
}
|
| 784 |
}
|
| 785 |
}
|
| 786 |
+
const scaleGroup = resolveCanonicalScaleGroup(
|
| 787 |
+
scaleCells,
|
| 788 |
+
// Registry bounds shared by the sibling metric entries — resolve
|
| 789 |
+
// anchor-neutral / unit-conflicted groups exactly (conflicting bounds
|
| 790 |
+
// merge to undefined and keep the fallback).
|
| 791 |
+
mergeRegistryBounds(
|
| 792 |
+
Array.from(metricByEval.values(), (m) => ({
|
| 793 |
+
min: m.canonical_min_score,
|
| 794 |
+
max: m.canonical_max_score,
|
| 795 |
+
})),
|
| 796 |
+
),
|
| 797 |
+
)
|
| 798 |
// Same display convention as the legacy guess (any percent-scale source
|
| 799 |
// pulls the whole table onto the 0-100 axis), read off the producer
|
| 800 |
// fields when available.
|
|
@@ -487,6 +487,11 @@ export interface ComparisonMetricEntry {
|
|
| 487 |
group_order: number
|
| 488 |
lower_is_better: boolean
|
| 489 |
unit: string | null
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 490 |
scores: ComparisonScoreEntry[]
|
| 491 |
}
|
| 492 |
|
|
|
|
| 487 |
group_order: number
|
| 488 |
lower_is_better: boolean
|
| 489 |
unit: string | null
|
| 490 |
+
/** Registry bounds of the EFFECTIVE metric — the scale `score_canonical`
|
| 491 |
+
* sits on (e.g. [0,1] for a fold target like wb-score). Null when the
|
| 492 |
+
* registry declares no bounds; absent on snapshots predating the stamp. */
|
| 493 |
+
canonical_min_score?: number | null
|
| 494 |
+
canonical_max_score?: number | null
|
| 495 |
scores: ComparisonScoreEntry[]
|
| 496 |
}
|
| 497 |
|
|
@@ -22,6 +22,7 @@ import type {
|
|
| 22 |
} from "./backend-artifacts"
|
| 23 |
import {
|
| 24 |
isPercentUnit,
|
|
|
|
| 25 |
resolveCanonicalScaleGroup,
|
| 26 |
type CanonicalScaleCell,
|
| 27 |
} from "./score-scale"
|
|
@@ -47,6 +48,10 @@ export interface OverlapAppearance {
|
|
| 47 |
* (spec F5); undefined on old snapshots and summary-sourced appearances. */
|
| 48 |
scoreCanonical?: number | null
|
| 49 |
scaleConversion?: ScaleConversion | null
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
}
|
| 51 |
|
| 52 |
export interface OverlapRow {
|
|
@@ -257,6 +262,8 @@ export function buildOverlapRows(input: BuildOverlapRowsInput): OverlapRow[] {
|
|
| 257 |
sourceKind: "comparison-index",
|
| 258 |
scoreCanonical: cellInfo.scoreCanonical,
|
| 259 |
scaleConversion: cellInfo.scaleConversion,
|
|
|
|
|
|
|
| 260 |
})
|
| 261 |
}
|
| 262 |
}
|
|
@@ -312,7 +319,16 @@ export function buildOverlapRows(input: BuildOverlapRowsInput): OverlapRow[] {
|
|
| 312 |
scaleConversion: c.scaleConversion,
|
| 313 |
unit: c.unit,
|
| 314 |
})
|
| 315 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 316 |
let scaled: OverlapAppearance[]
|
| 317 |
let useHigh: boolean
|
| 318 |
if (scaleGroup) {
|
|
|
|
| 22 |
} from "./backend-artifacts"
|
| 23 |
import {
|
| 24 |
isPercentUnit,
|
| 25 |
+
mergeRegistryBounds,
|
| 26 |
resolveCanonicalScaleGroup,
|
| 27 |
type CanonicalScaleCell,
|
| 28 |
} from "./score-scale"
|
|
|
|
| 48 |
* (spec F5); undefined on old snapshots and summary-sourced appearances. */
|
| 49 |
scoreCanonical?: number | null
|
| 50 |
scaleConversion?: ScaleConversion | null
|
| 51 |
+
/** Registry bounds off the owning metric entry (the scale
|
| 52 |
+
* `scoreCanonical` sits on); undefined pre-stamp / summary-sourced. */
|
| 53 |
+
canonicalMinScore?: number | null
|
| 54 |
+
canonicalMaxScore?: number | null
|
| 55 |
}
|
| 56 |
|
| 57 |
export interface OverlapRow {
|
|
|
|
| 262 |
sourceKind: "comparison-index",
|
| 263 |
scoreCanonical: cellInfo.scoreCanonical,
|
| 264 |
scaleConversion: cellInfo.scaleConversion,
|
| 265 |
+
canonicalMinScore: targetMetric.canonical_min_score,
|
| 266 |
+
canonicalMaxScore: targetMetric.canonical_max_score,
|
| 267 |
})
|
| 268 |
}
|
| 269 |
}
|
|
|
|
| 319 |
scaleConversion: c.scaleConversion,
|
| 320 |
unit: c.unit,
|
| 321 |
})
|
| 322 |
+
// Registry bounds shared by the appearances' metric entries (spec:
|
| 323 |
+
// producer-stamped `canonical_min_score` / `canonical_max_score`) —
|
| 324 |
+
// they let anchor-neutral groups (all-curated) and unit-conflicted
|
| 325 |
+
// groups resolve exactly; conflicting bounds merge to undefined.
|
| 326 |
+
const scaleGroup = resolveCanonicalScaleGroup(
|
| 327 |
+
collected.map(scaleCellOf),
|
| 328 |
+
mergeRegistryBounds(
|
| 329 |
+
collected.map((c) => ({ min: c.canonicalMinScore, max: c.canonicalMaxScore })),
|
| 330 |
+
),
|
| 331 |
+
)
|
| 332 |
let scaled: OverlapAppearance[]
|
| 333 |
let useHigh: boolean
|
| 334 |
if (scaleGroup) {
|
|
@@ -9,14 +9,19 @@
|
|
| 9 |
// `scale_conversion` — how `score` maps onto `score_canonical`.
|
| 10 |
// Old snapshots lack both keys entirely.
|
| 11 |
//
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
// `resolveCanonicalScaleGroup` decides whether a set of cells that a view
|
| 13 |
// needs on ONE common scale can be settled exactly from those fields. When
|
| 14 |
// it can, callers use the producer values and skip the legacy
|
| 15 |
// `|score| > 1.5 ⇒ percent` guess; when it can't — any cell missing the
|
| 16 |
// keys (old snapshot, or a mixed old/new group which must never be
|
| 17 |
// half-converted), a bounds-less metric (canonical === raw guarantees
|
| 18 |
-
// nothing about scale consistency), an all-flagged group
|
| 19 |
-
// conversions — callers keep the legacy
|
|
|
|
| 20 |
//
|
| 21 |
// NOTE: the comparison-index metric `unit` is source-reported, not the
|
| 22 |
// registry's — a div100 metric can say "percent" while its canonical
|
|
@@ -42,6 +47,45 @@ export function isPercentUnit(unit: string | null | undefined): boolean {
|
|
| 42 |
return /percent|%|pct/.test((unit ?? "").toLowerCase())
|
| 43 |
}
|
| 44 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
/** Legacy per-row scale guess (`|raw| > 1.5 ⇒ percent`), mapped onto the
|
| 46 |
* requested display scale. Kept for old snapshots and flagged rows. */
|
| 47 |
export function heuristicToScale(raw: number, toPercent: boolean): number {
|
|
@@ -62,7 +106,10 @@ export function scaleOnto(value: number, fromPercent: boolean, toPercent: boolea
|
|
| 62 |
* group prefer the group's `registryIsPercent` — a 'none' cell's
|
| 63 |
* source-reported unit can misstate the metric scale that its converted
|
| 64 |
* siblings pin down exactly. */
|
| 65 |
-
export function canonicalCellIsPercent(
|
|
|
|
|
|
|
|
|
|
| 66 |
if (cell.scoreCanonical == null) return null
|
| 67 |
switch (cell.scaleConversion) {
|
| 68 |
case "div100":
|
|
@@ -70,7 +117,15 @@ export function canonicalCellIsPercent(cell: CanonicalScaleCell): boolean | null
|
|
| 70 |
case "mul100":
|
| 71 |
return true // raw fraction × 100 ⇒ the registry scale is the percent
|
| 72 |
case "none":
|
| 73 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
default:
|
| 75 |
return null // 'no_bounds', 'flagged', unexpected tokens
|
| 76 |
}
|
|
@@ -91,6 +146,7 @@ export interface CanonicalScaleGroup {
|
|
| 91 |
|
| 92 |
export function resolveCanonicalScaleGroup(
|
| 93 |
cells: readonly CanonicalScaleCell[],
|
|
|
|
| 94 |
): CanonicalScaleGroup | null {
|
| 95 |
if (cells.length === 0) return null
|
| 96 |
let sawDiv100 = false
|
|
@@ -115,23 +171,38 @@ export function resolveCanonicalScaleGroup(
|
|
| 115 |
else noneFractionUnit = true
|
| 116 |
} else return null // unknown conversion token — don't guess
|
| 117 |
}
|
| 118 |
-
|
| 119 |
-
//
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
//
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 128 |
let percentSourceCount = 0
|
| 129 |
let fractionSourceCount = 0
|
| 130 |
for (const c of cells) {
|
| 131 |
if (c.scoreCanonical == null) continue
|
|
|
|
|
|
|
|
|
|
| 132 |
if (c.scaleConversion === "div100") percentSourceCount += 1
|
| 133 |
else if (c.scaleConversion === "mul100") fractionSourceCount += 1
|
| 134 |
-
else if (
|
| 135 |
else fractionSourceCount += 1
|
| 136 |
}
|
| 137 |
return {
|
|
|
|
| 9 |
// `scale_conversion` — how `score` maps onto `score_canonical`.
|
| 10 |
// Old snapshots lack both keys entirely.
|
| 11 |
//
|
| 12 |
+
// Newer snapshots additionally stamp every comparison-index METRIC entry
|
| 13 |
+
// with `canonical_min_score` / `canonical_max_score` — the effective
|
| 14 |
+
// metric's registry bounds, i.e. the scale `score_canonical` sits on.
|
| 15 |
+
//
|
| 16 |
// `resolveCanonicalScaleGroup` decides whether a set of cells that a view
|
| 17 |
// needs on ONE common scale can be settled exactly from those fields. When
|
| 18 |
// it can, callers use the producer values and skip the legacy
|
| 19 |
// `|score| > 1.5 ⇒ percent` guess; when it can't — any cell missing the
|
| 20 |
// keys (old snapshot, or a mixed old/new group which must never be
|
| 21 |
// half-converted), a bounds-less metric (canonical === raw guarantees
|
| 22 |
+
// nothing about scale consistency), an all-flagged group with no usable
|
| 23 |
+
// registry bounds, or contradictory conversions — callers keep the legacy
|
| 24 |
+
// heuristic byte-for-byte.
|
| 25 |
//
|
| 26 |
// NOTE: the comparison-index metric `unit` is source-reported, not the
|
| 27 |
// registry's — a div100 metric can say "percent" while its canonical
|
|
|
|
| 47 |
return /percent|%|pct/.test((unit ?? "").toLowerCase())
|
| 48 |
}
|
| 49 |
|
| 50 |
+
/** The effective metric's registry bounds off a comparison-index metric
|
| 51 |
+
* entry (`canonical_min_score` / `canonical_max_score`); fields are
|
| 52 |
+
* undefined on snapshots predating the stamp, null when the registry
|
| 53 |
+
* declares no bounds. */
|
| 54 |
+
export interface RegistryBounds {
|
| 55 |
+
min?: number | null
|
| 56 |
+
max?: number | null
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
/** Scale the registry bounds pin down: true ⇒ percent ([0,100]), false ⇒
|
| 60 |
+
* fraction (max ≤ 1.5), null ⇒ absent or not usable (open-ended scales
|
| 61 |
+
* like Elo, unusual ranges). */
|
| 62 |
+
export function registryBoundsIsPercent(
|
| 63 |
+
bounds: RegistryBounds | null | undefined,
|
| 64 |
+
): boolean | null {
|
| 65 |
+
if (!bounds) return null
|
| 66 |
+
if (bounds.min === 0 && bounds.max === 100) return true
|
| 67 |
+
if (bounds.max != null && bounds.max <= 1.5) return false
|
| 68 |
+
return null
|
| 69 |
+
}
|
| 70 |
+
|
| 71 |
+
/** Merge the per-metric registry bounds behind a cross-metric cell group:
|
| 72 |
+
* the shared bounds when every stamped metric agrees, undefined when they
|
| 73 |
+
* conflict (sibling metrics on different registry scales must not share
|
| 74 |
+
* one anchor) or when no metric carries the stamp. */
|
| 75 |
+
export function mergeRegistryBounds(
|
| 76 |
+
boundsList: Iterable<RegistryBounds | null | undefined>,
|
| 77 |
+
): RegistryBounds | undefined {
|
| 78 |
+
let merged: { min: number | null; max: number | null } | undefined
|
| 79 |
+
for (const b of boundsList) {
|
| 80 |
+
if (!b || (b.min === undefined && b.max === undefined)) continue // pre-stamp metric
|
| 81 |
+
const min = b.min ?? null
|
| 82 |
+
const max = b.max ?? null
|
| 83 |
+
if (!merged) merged = { min, max }
|
| 84 |
+
else if (merged.min !== min || merged.max !== max) return undefined
|
| 85 |
+
}
|
| 86 |
+
return merged
|
| 87 |
+
}
|
| 88 |
+
|
| 89 |
/** Legacy per-row scale guess (`|raw| > 1.5 ⇒ percent`), mapped onto the
|
| 90 |
* requested display scale. Kept for old snapshots and flagged rows. */
|
| 91 |
export function heuristicToScale(raw: number, toPercent: boolean): number {
|
|
|
|
| 106 |
* group prefer the group's `registryIsPercent` — a 'none' cell's
|
| 107 |
* source-reported unit can misstate the metric scale that its converted
|
| 108 |
* siblings pin down exactly. */
|
| 109 |
+
export function canonicalCellIsPercent(
|
| 110 |
+
cell: CanonicalScaleCell,
|
| 111 |
+
registryBounds?: RegistryBounds | null,
|
| 112 |
+
): boolean | null {
|
| 113 |
if (cell.scoreCanonical == null) return null
|
| 114 |
switch (cell.scaleConversion) {
|
| 115 |
case "div100":
|
|
|
|
| 117 |
case "mul100":
|
| 118 |
return true // raw fraction × 100 ⇒ the registry scale is the percent
|
| 119 |
case "none":
|
| 120 |
+
case "curated": {
|
| 121 |
+
// The producer-stamped registry bounds settle the canonical scale
|
| 122 |
+
// exactly when usable. Otherwise 'none' falls back to the
|
| 123 |
+
// source-reported unit (raw already sits on the registry scale) and
|
| 124 |
+
// 'curated' stays underivable — the tag alone can't anchor.
|
| 125 |
+
const fromBounds = registryBoundsIsPercent(registryBounds)
|
| 126 |
+
if (fromBounds != null) return fromBounds
|
| 127 |
+
return cell.scaleConversion === "none" ? isPercentUnit(cell.unit) : null
|
| 128 |
+
}
|
| 129 |
default:
|
| 130 |
return null // 'no_bounds', 'flagged', unexpected tokens
|
| 131 |
}
|
|
|
|
| 146 |
|
| 147 |
export function resolveCanonicalScaleGroup(
|
| 148 |
cells: readonly CanonicalScaleCell[],
|
| 149 |
+
registryBounds?: RegistryBounds | null,
|
| 150 |
): CanonicalScaleGroup | null {
|
| 151 |
if (cells.length === 0) return null
|
| 152 |
let sawDiv100 = false
|
|
|
|
| 171 |
else noneFractionUnit = true
|
| 172 |
} else return null // unknown conversion token — don't guess
|
| 173 |
}
|
| 174 |
+
if (sawDiv100 && sawMul100) return null // contradictory conversions
|
| 175 |
+
// Scale precedence: a conversion pins the registry scale exactly; next
|
| 176 |
+
// the metric's stamped registry bounds; last the 'none' cells' unit
|
| 177 |
+
// vote. Bounds let anchor-neutral groups (all-curated, all-flagged) and
|
| 178 |
+
// unit-conflicted groups resolve exactly instead of falling back.
|
| 179 |
+
let registryIsPercent: boolean
|
| 180 |
+
if (sawMul100) registryIsPercent = true
|
| 181 |
+
else if (sawDiv100) registryIsPercent = false
|
| 182 |
+
else {
|
| 183 |
+
const boundsScale = registryBoundsIsPercent(registryBounds)
|
| 184 |
+
if (boundsScale != null) registryIsPercent = boundsScale
|
| 185 |
+
else {
|
| 186 |
+
// Need at least one 'none' cell to anchor the unit vote…
|
| 187 |
+
if (noneCount === 0) return null
|
| 188 |
+
// …and when the 'none' cells' source units DISAGREE about
|
| 189 |
+
// percent-ness, the units are lying about at least one cell
|
| 190 |
+
// (they're source-reported, not registry data) — a lone mislabeled
|
| 191 |
+
// "percent" must not flip the whole group 100x. Stay legacy.
|
| 192 |
+
if (nonePercentUnit && noneFractionUnit) return null
|
| 193 |
+
registryIsPercent = nonePercentUnit
|
| 194 |
+
}
|
| 195 |
+
}
|
| 196 |
let percentSourceCount = 0
|
| 197 |
let fractionSourceCount = 0
|
| 198 |
for (const c of cells) {
|
| 199 |
if (c.scoreCanonical == null) continue
|
| 200 |
+
// 'curated' sources sit on neither scale (e.g. 1-10 points) — they
|
| 201 |
+
// don't vote; ties break percent-ward at the call sites.
|
| 202 |
+
if (c.scaleConversion === "curated") continue
|
| 203 |
if (c.scaleConversion === "div100") percentSourceCount += 1
|
| 204 |
else if (c.scaleConversion === "mul100") fractionSourceCount += 1
|
| 205 |
+
else if (isPercentUnit(c.unit)) percentSourceCount += 1
|
| 206 |
else fractionSourceCount += 1
|
| 207 |
}
|
| 208 |
return {
|
|
@@ -322,6 +322,96 @@ describe("buildOverlapRows: producer canonical scale (spec F5)", () => {
|
|
| 322 |
expect(rows[0].appearances.map((a) => a.displayScore)).toEqual(["11.0%", "0.0%"])
|
| 323 |
})
|
| 324 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 325 |
it("reads the canonical fields off by_model cells too", () => {
|
| 326 |
const rows = build({
|
| 327 |
benchmarkIndex: [MMLU_INDEX],
|
|
|
|
| 322 |
expect(rows[0].appearances.map((a) => a.displayScore)).toEqual(["11.0%", "0.0%"])
|
| 323 |
})
|
| 324 |
|
| 325 |
+
it("resolves an all-curated group from producer registry bounds", () => {
|
| 326 |
+
// WildBench shape: raw 1-10 'points' scores curated onto a [0,1]
|
| 327 |
+
// registry metric. The 'curated' tag alone can't anchor
|
| 328 |
+
// fraction-vs-percent, but the metric-level registry bounds can — a
|
| 329 |
+
// model whose only appearance is the curated source now shows ~46%
|
| 330 |
+
// (matching the merged page) instead of the legacy magnitude guess
|
| 331 |
+
// reading raw 4.597 as "4.6%".
|
| 332 |
+
const rows = build({
|
| 333 |
+
benchmarkIndex: [MMLU_INDEX],
|
| 334 |
+
comparisonIndex: comparisonIndexOf({
|
| 335 |
+
"fam-a%2Fmmlu": evalEntry("fam-a%2Fmmlu", [
|
| 336 |
+
metricEntry({
|
| 337 |
+
unit: "points",
|
| 338 |
+
canonical_min_score: 0,
|
| 339 |
+
canonical_max_score: 1,
|
| 340 |
+
scores: [ownRow(4.597, { score_canonical: 0.4597, scale_conversion: "curated" })],
|
| 341 |
+
}),
|
| 342 |
+
]),
|
| 343 |
+
"fam-b%2Fmmlu": evalEntry("fam-b%2Fmmlu", [metricEntry({ scores: [] })]),
|
| 344 |
+
}),
|
| 345 |
+
})
|
| 346 |
+
expect(rows).toHaveLength(1)
|
| 347 |
+
expect(rows[0].appearances[0].score).toBeCloseTo(45.97, 10)
|
| 348 |
+
expect(rows[0].appearances[0].displayScore).toBe("46.0%")
|
| 349 |
+
expect(rows[0].isPercentScale).toBe(true)
|
| 350 |
+
})
|
| 351 |
+
|
| 352 |
+
it("resolves a unit-conflicted group exactly when registry bounds are stamped", () => {
|
| 353 |
+
// Same terminalbench-hard shape as the legacy-fallback test above,
|
| 354 |
+
// but with the [0,1] registry bounds stamped on both metric entries:
|
| 355 |
+
// the group resolves as fraction-registry and both canonicals map
|
| 356 |
+
// exactly onto the percent display axis — no guessing.
|
| 357 |
+
const rows = build({
|
| 358 |
+
benchmarkIndex: [MMLU_INDEX],
|
| 359 |
+
comparisonIndex: comparisonIndexOf({
|
| 360 |
+
"fam-a%2Fmmlu": evalEntry("fam-a%2Fmmlu", [
|
| 361 |
+
metricEntry({
|
| 362 |
+
unit: "proportion",
|
| 363 |
+
canonical_min_score: 0,
|
| 364 |
+
canonical_max_score: 1,
|
| 365 |
+
scores: [ownRow(0.11, { score_canonical: 0.11, scale_conversion: "none" })],
|
| 366 |
+
}),
|
| 367 |
+
]),
|
| 368 |
+
"fam-b%2Fmmlu": evalEntry("fam-b%2Fmmlu", [
|
| 369 |
+
metricEntry({
|
| 370 |
+
unit: "percent",
|
| 371 |
+
canonical_min_score: 0,
|
| 372 |
+
canonical_max_score: 1,
|
| 373 |
+
scores: [ownRow(0.0, { score_canonical: 0.0, scale_conversion: "none" })],
|
| 374 |
+
}),
|
| 375 |
+
]),
|
| 376 |
+
}),
|
| 377 |
+
})
|
| 378 |
+
expect(rows[0].appearances[0].score).toBeCloseTo(11, 10)
|
| 379 |
+
expect(rows[0].appearances[1].score).toBeCloseTo(0, 10)
|
| 380 |
+
expect(rows[0].appearances.map((a) => a.displayScore)).toEqual(["11.0%", "0.0%"])
|
| 381 |
+
expect(rows[0].isPercentScale).toBe(true)
|
| 382 |
+
})
|
| 383 |
+
|
| 384 |
+
it("resolves a percent-registry metric from [0,100] bounds without double-scaling", () => {
|
| 385 |
+
// Bounds [0,100] pin the percent registry even though one cell's unit
|
| 386 |
+
// is missing (the unit conflict would otherwise force the legacy
|
| 387 |
+
// fallback). Canonical values already sit on the percent scale — they
|
| 388 |
+
// must pass through unscaled, not get multiplied or divided again.
|
| 389 |
+
const rows = build({
|
| 390 |
+
benchmarkIndex: [MMLU_INDEX],
|
| 391 |
+
comparisonIndex: comparisonIndexOf({
|
| 392 |
+
"fam-a%2Fmmlu": evalEntry("fam-a%2Fmmlu", [
|
| 393 |
+
metricEntry({
|
| 394 |
+
unit: "percent",
|
| 395 |
+
canonical_min_score: 0,
|
| 396 |
+
canonical_max_score: 100,
|
| 397 |
+
scores: [ownRow(65, { score_canonical: 65, scale_conversion: "none" })],
|
| 398 |
+
}),
|
| 399 |
+
]),
|
| 400 |
+
"fam-b%2Fmmlu": evalEntry("fam-b%2Fmmlu", [
|
| 401 |
+
metricEntry({
|
| 402 |
+
unit: null,
|
| 403 |
+
canonical_min_score: 0,
|
| 404 |
+
canonical_max_score: 100,
|
| 405 |
+
scores: [ownRow(60, { score_canonical: 60, scale_conversion: "none" })],
|
| 406 |
+
}),
|
| 407 |
+
]),
|
| 408 |
+
}),
|
| 409 |
+
})
|
| 410 |
+
expect(rows[0].appearances.map((a) => a.score)).toEqual([65, 60])
|
| 411 |
+
expect(rows[0].appearances.map((a) => a.displayScore)).toEqual(["65.0%", "60.0%"])
|
| 412 |
+
expect(rows[0].isPercentScale).toBe(true)
|
| 413 |
+
})
|
| 414 |
+
|
| 415 |
it("reads the canonical fields off by_model cells too", () => {
|
| 416 |
const rows = build({
|
| 417 |
benchmarkIndex: [MMLU_INDEX],
|