j-chim commited on
Commit
478ae6c
·
1 Parent(s): 294d3a2

Add validated evaluator badge

Browse files
app/evals/page.tsx CHANGED
@@ -4,6 +4,7 @@ import { Suspense, useCallback, useDeferredValue, useEffect, useMemo, useState }
4
  import { useRouter, useSearchParams } from "next/navigation"
5
  import { Search } from "lucide-react"
6
 
 
7
  import { FamilyTable, getFamilyNavId, type FamilySortCol } from "@/components/family-table"
8
  import { InfiniteScrollSentinel } from "@/components/infinite-scroll"
9
  import { Navigation } from "@/components/navigation"
@@ -13,6 +14,7 @@ import { fetchBenchmarkMetadata, fetchEvalHierarchy, fetchEvalList } from "@/lib
13
  import type { BenchmarkEvalListItem } from "@/lib/eval-processing"
14
  import type { BenchmarkCard } from "@/lib/benchmark-schema"
15
  import { formatTagLabel } from "@/lib/benchmark-tags"
 
16
 
17
  const PAGE_SIZE = 60
18
 
@@ -39,6 +41,8 @@ function EvalsPageInner() {
39
  const searchParams = useSearchParams()
40
  const familyParam = searchParams.get("family")
41
  const queryParam = searchParams.get("q")
 
 
42
 
43
  const [hierarchy, setHierarchy] = useState<EvalHierarchy | null>(null)
44
  const [totalModels, setTotalModels] = useState<number>(0)
@@ -54,8 +58,27 @@ function EvalsPageInner() {
54
  const [agentMode, setAgentMode] = useState<"all" | "agentic" | "non-agentic">("all")
55
  const [sortCol, setSortCol] = useState<FamilySortCol>("name")
56
  const [sortDir, setSortDir] = useState<"asc" | "desc">("asc")
 
 
 
 
 
 
 
 
57
  const deferredSearchQuery = useDeferredValue(searchQuery)
58
 
 
 
 
 
 
 
 
 
 
 
 
59
  const handleSort = useCallback((col: FamilySortCol) => {
60
  if (sortCol === col) {
61
  setSortDir((d) => (d === "asc" ? "desc" : "asc"))
@@ -122,6 +145,47 @@ function EvalsPageInner() {
122
  setSearchQuery(fam.display_name || fam.key)
123
  }, [familyParam, hierarchy, benchmarkCards, router])
124
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
125
  const families = hierarchy?.families ?? []
126
 
127
  // Tags per family — union of derivedTags across the family and every
@@ -244,16 +308,21 @@ function EvalsPageInner() {
244
 
245
  useEffect(() => {
246
  setVisibleCount(PAGE_SIZE)
247
- }, [deferredSearchQuery, selectedCategories, agentMode, sortCol, sortDir])
248
 
249
  const visibleFamilies = useMemo(
250
  () => filteredFamilies.slice(0, visibleCount),
251
  [filteredFamilies, visibleCount],
252
  )
253
- const hasMore = visibleCount < filteredFamilies.length
 
 
 
 
 
254
  const handleLoadMore = useCallback(() => {
255
- setVisibleCount((current) => Math.min(current + PAGE_SIZE, filteredFamilies.length))
256
- }, [filteredFamilies.length])
257
 
258
  return (
259
  <div className="min-h-screen bg-background">
@@ -264,24 +333,76 @@ function EvalsPageInner() {
264
  <div className="kicker">Index</div>
265
  <h1 className="ec-page-h1">Evaluations</h1>
266
  <p className="ec-page-lede">
267
- Evaluations are grouped into <strong>families</strong>. A family may hold a single
268
- standalone benchmark or many related ones; each benchmark has one or more slices, and
269
- each slice reports one or more metrics.
 
 
 
 
 
 
 
 
 
270
  </p>
271
 
272
- {/* SEARCH ROW ---------------------------------------------- */}
273
- <div className="mb-6 flex items-center border-b border-[color:var(--border-soft)] pb-5">
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
274
  <div className="relative ml-auto min-w-[180px] flex-1 sm:max-w-[360px]">
275
  <Search className="pointer-events-none absolute left-3 top-1/2 h-3.5 w-3.5 -translate-y-1/2 text-[color:var(--fg-subtle)]" />
276
  <input
277
  className="ec-input pl-9"
278
  value={searchQuery}
279
  onChange={(event) => setSearchQuery(event.target.value)}
280
- placeholder="Search family, benchmark, or category…"
 
 
 
 
281
  />
282
  </div>
283
  </div>
284
 
 
 
 
 
 
285
  {/* INTERACTION STYLE TOGGLE — orthogonal axis from category,
286
  surfaced on its own so users don't mix "is this an agent
287
  benchmark?" with "what category is this in?" */}
@@ -348,6 +469,8 @@ function EvalsPageInner() {
348
  })}
349
  </div>
350
  )}
 
 
351
 
352
  {/* TABLE ---------------------------------------------------- */}
353
  {loading ? (
@@ -357,10 +480,12 @@ function EvalsPageInner() {
357
  stages={loadingStages}
358
  className="py-14"
359
  />
360
- ) : filteredFamilies.length === 0 ? (
361
  <div className="py-16 text-center border border-dashed border-[color:var(--border-soft)] bg-[color:var(--bg-warm)]">
362
  <p className="mb-4 text-base text-[color:var(--fg-muted)]">
363
- No families found matching your filters.
 
 
364
  </p>
365
  <button
366
  type="button"
@@ -368,11 +493,20 @@ function EvalsPageInner() {
368
  onClick={() => {
369
  setSearchQuery("")
370
  setSelectedCategories([])
 
371
  }}
372
  >
373
  Reset filters
374
  </button>
375
  </div>
 
 
 
 
 
 
 
 
376
  ) : (
377
  <FamilyTable
378
  families={visibleFamilies}
@@ -380,6 +514,7 @@ function EvalsPageInner() {
380
  benchmarkCards={benchmarkCards}
381
  categoryFilter={new Set(selectedCategories)}
382
  searchQuery={deferredSearchQuery}
 
383
  sortCol={sortCol}
384
  sortDir={sortDir}
385
  onSort={handleSort}
@@ -390,7 +525,7 @@ function EvalsPageInner() {
390
  hasMore={hasMore}
391
  onLoadMore={handleLoadMore}
392
  loadingLabel="Loading more…"
393
- endLabel={`Showing ${Math.min(visibleCount, filteredFamilies.length).toLocaleString()} of ${filteredFamilies.length.toLocaleString()} families`}
394
  />
395
  </main>
396
  </div>
 
4
  import { useRouter, useSearchParams } from "next/navigation"
5
  import { Search } from "lucide-react"
6
 
7
+ import { EvaluatorTable, type EvaluatorTableSortCol } from "@/components/evaluator-table"
8
  import { FamilyTable, getFamilyNavId, type FamilySortCol } from "@/components/family-table"
9
  import { InfiniteScrollSentinel } from "@/components/infinite-scroll"
10
  import { Navigation } from "@/components/navigation"
 
14
  import type { BenchmarkEvalListItem } from "@/lib/eval-processing"
15
  import type { BenchmarkCard } from "@/lib/benchmark-schema"
16
  import { formatTagLabel } from "@/lib/benchmark-tags"
17
+ import { groupEvalsByEvaluator, verifiedEvalIds } from "@/lib/evaluators"
18
 
19
  const PAGE_SIZE = 60
20
 
 
41
  const searchParams = useSearchParams()
42
  const familyParam = searchParams.get("family")
43
  const queryParam = searchParams.get("q")
44
+ const groupByParam = searchParams.get("groupBy")
45
+ const verifiedParam = searchParams.get("verified")
46
 
47
  const [hierarchy, setHierarchy] = useState<EvalHierarchy | null>(null)
48
  const [totalModels, setTotalModels] = useState<number>(0)
 
58
  const [agentMode, setAgentMode] = useState<"all" | "agentic" | "non-agentic">("all")
59
  const [sortCol, setSortCol] = useState<FamilySortCol>("name")
60
  const [sortDir, setSortDir] = useState<"asc" | "desc">("asc")
61
+ const [groupBy, setGroupBy] = useState<"family" | "evaluator">(
62
+ groupByParam === "evaluator" ? "evaluator" : "family",
63
+ )
64
+ const [verifiedOnly, setVerifiedOnly] = useState<boolean>(
65
+ verifiedParam === "1" || verifiedParam === "true",
66
+ )
67
+ const [evaluatorSortCol, setEvaluatorSortCol] = useState<EvaluatorTableSortCol>("evals")
68
+ const [evaluatorSortDir, setEvaluatorSortDir] = useState<"asc" | "desc">("desc")
69
  const deferredSearchQuery = useDeferredValue(searchQuery)
70
 
71
+ const handleEvaluatorSort = useCallback((col: EvaluatorTableSortCol) => {
72
+ setEvaluatorSortCol((current) => {
73
+ if (current === col) {
74
+ setEvaluatorSortDir((dir) => (dir === "asc" ? "desc" : "asc"))
75
+ return current
76
+ }
77
+ setEvaluatorSortDir(col === "name" ? "asc" : "desc")
78
+ return col
79
+ })
80
+ }, [])
81
+
82
  const handleSort = useCallback((col: FamilySortCol) => {
83
  if (sortCol === col) {
84
  setSortDir((d) => (d === "asc" ? "desc" : "asc"))
 
145
  setSearchQuery(fam.display_name || fam.key)
146
  }, [familyParam, hierarchy, benchmarkCards, router])
147
 
148
+ // Reflect groupBy / verified into the URL (nice-to-have deep link).
149
+ // Shallow replace so the back button isn't spammed and data isn't refetched.
150
+ useEffect(() => {
151
+ const params = new URLSearchParams(searchParams.toString())
152
+ if (groupBy === "evaluator") params.set("groupBy", "evaluator")
153
+ else params.delete("groupBy")
154
+ if (verifiedOnly) params.set("verified", "1")
155
+ else params.delete("verified")
156
+ const qs = params.toString()
157
+ router.replace(qs ? `/evals?${qs}` : "/evals", { scroll: false })
158
+ // searchParams intentionally omitted — we only push when our own toggles change.
159
+ // eslint-disable-next-line react-hooks/exhaustive-deps
160
+ }, [groupBy, verifiedOnly])
161
+
162
+ const allEvals = useMemo(() => Array.from(evalItems.values()), [evalItems])
163
+
164
+ // Verified-eval id universe — drives the Family-mode "Verified only" gate.
165
+ const verifiedIds = useMemo(() => verifiedEvalIds(allEvals), [allEvals])
166
+
167
+ // Evaluator groups (group-by-Evaluator mode). Verified filter is
168
+ // evaluator-aware: counts only (eval, org) pairs where org is verified.
169
+ const evaluatorGroups = useMemo(
170
+ () => groupEvalsByEvaluator(allEvals, { verifiedOnly }),
171
+ [allEvals, verifiedOnly],
172
+ )
173
+
174
+ const filteredEvaluators = useMemo(() => {
175
+ const query = deferredSearchQuery.trim().toLowerCase()
176
+ let list = evaluatorGroups
177
+ if (query) list = list.filter((g) => g.name.toLowerCase().includes(query))
178
+ const dirMul = evaluatorSortDir === "asc" ? 1 : -1
179
+ return list.slice().sort((a, b) => {
180
+ let cmp = 0
181
+ if (evaluatorSortCol === "name") cmp = a.name.localeCompare(b.name)
182
+ else if (evaluatorSortCol === "verified") cmp = a.verifiedCount - b.verifiedCount
183
+ else cmp = a.evalCount - b.evalCount
184
+ if (cmp === 0) cmp = a.name.localeCompare(b.name)
185
+ return cmp * dirMul
186
+ })
187
+ }, [evaluatorGroups, deferredSearchQuery, evaluatorSortCol, evaluatorSortDir])
188
+
189
  const families = hierarchy?.families ?? []
190
 
191
  // Tags per family — union of derivedTags across the family and every
 
308
 
309
  useEffect(() => {
310
  setVisibleCount(PAGE_SIZE)
311
+ }, [deferredSearchQuery, selectedCategories, agentMode, sortCol, sortDir, groupBy, verifiedOnly, evaluatorSortCol, evaluatorSortDir])
312
 
313
  const visibleFamilies = useMemo(
314
  () => filteredFamilies.slice(0, visibleCount),
315
  [filteredFamilies, visibleCount],
316
  )
317
+ const visibleEvaluators = useMemo(
318
+ () => filteredEvaluators.slice(0, visibleCount),
319
+ [filteredEvaluators, visibleCount],
320
+ )
321
+ const totalRows = groupBy === "evaluator" ? filteredEvaluators.length : filteredFamilies.length
322
+ const hasMore = visibleCount < totalRows
323
  const handleLoadMore = useCallback(() => {
324
+ setVisibleCount((current) => Math.min(current + PAGE_SIZE, totalRows))
325
+ }, [totalRows])
326
 
327
  return (
328
  <div className="min-h-screen bg-background">
 
333
  <div className="kicker">Index</div>
334
  <h1 className="ec-page-h1">Evaluations</h1>
335
  <p className="ec-page-lede">
336
+ {groupBy === "evaluator" ? (
337
+ <>
338
+ Evaluations grouped by the <strong>organisation that reported them</strong>. A
339
+ verified evaluator submitted the results from the org that ran the evaluation.
340
+ </>
341
+ ) : (
342
+ <>
343
+ Evaluations are grouped into <strong>families</strong>. A family may hold a single
344
+ standalone benchmark or many related ones; each benchmark has one or more slices, and
345
+ each slice reports one or more metrics.
346
+ </>
347
+ )}
348
  </p>
349
 
350
+ {/* MODE + FILTER ROW --------------------------------------- */}
351
+ <div className="mb-6 flex flex-wrap items-center gap-x-6 gap-y-3 border-y border-[color:var(--border-soft)] py-4">
352
+ <div className="ec-mode-toggle" role="group" aria-label="Group evaluations by">
353
+ <button
354
+ type="button"
355
+ className={groupBy === "family" ? "on" : ""}
356
+ onClick={() => setGroupBy("family")}
357
+ >
358
+ Family
359
+ </button>
360
+ <button
361
+ type="button"
362
+ className={groupBy === "evaluator" ? "on" : ""}
363
+ onClick={() => setGroupBy("evaluator")}
364
+ >
365
+ Evaluator
366
+ </button>
367
+ </div>
368
+
369
+ <div className="ec-mode-toggle" role="group" aria-label="Verified only filter">
370
+ <button
371
+ type="button"
372
+ className={!verifiedOnly ? "on" : ""}
373
+ onClick={() => setVerifiedOnly(false)}
374
+ >
375
+ All
376
+ </button>
377
+ <button
378
+ type="button"
379
+ className={verifiedOnly ? "on" : ""}
380
+ onClick={() => setVerifiedOnly(true)}
381
+ >
382
+ Verified only
383
+ </button>
384
+ </div>
385
+
386
  <div className="relative ml-auto min-w-[180px] flex-1 sm:max-w-[360px]">
387
  <Search className="pointer-events-none absolute left-3 top-1/2 h-3.5 w-3.5 -translate-y-1/2 text-[color:var(--fg-subtle)]" />
388
  <input
389
  className="ec-input pl-9"
390
  value={searchQuery}
391
  onChange={(event) => setSearchQuery(event.target.value)}
392
+ placeholder={
393
+ groupBy === "evaluator"
394
+ ? "Search evaluator…"
395
+ : "Search family, benchmark, or category…"
396
+ }
397
  />
398
  </div>
399
  </div>
400
 
401
+ {/* Family-mode-only filters: interaction style + category pills.
402
+ These operate on the family hierarchy and have no meaning in
403
+ the evaluator grouping. */}
404
+ {groupBy === "family" && (
405
+ <>
406
  {/* INTERACTION STYLE TOGGLE — orthogonal axis from category,
407
  surfaced on its own so users don't mix "is this an agent
408
  benchmark?" with "what category is this in?" */}
 
469
  })}
470
  </div>
471
  )}
472
+ </>
473
+ )}
474
 
475
  {/* TABLE ---------------------------------------------------- */}
476
  {loading ? (
 
480
  stages={loadingStages}
481
  className="py-14"
482
  />
483
+ ) : totalRows === 0 ? (
484
  <div className="py-16 text-center border border-dashed border-[color:var(--border-soft)] bg-[color:var(--bg-warm)]">
485
  <p className="mb-4 text-base text-[color:var(--fg-muted)]">
486
+ {groupBy === "evaluator"
487
+ ? "No evaluators found matching your filters."
488
+ : "No families found matching your filters."}
489
  </p>
490
  <button
491
  type="button"
 
493
  onClick={() => {
494
  setSearchQuery("")
495
  setSelectedCategories([])
496
+ setVerifiedOnly(false)
497
  }}
498
  >
499
  Reset filters
500
  </button>
501
  </div>
502
+ ) : groupBy === "evaluator" ? (
503
+ <EvaluatorTable
504
+ rows={visibleEvaluators}
505
+ sortCol={evaluatorSortCol}
506
+ sortDir={evaluatorSortDir}
507
+ onSort={handleEvaluatorSort}
508
+ verifiedOnly={verifiedOnly}
509
+ />
510
  ) : (
511
  <FamilyTable
512
  families={visibleFamilies}
 
514
  benchmarkCards={benchmarkCards}
515
  categoryFilter={new Set(selectedCategories)}
516
  searchQuery={deferredSearchQuery}
517
+ verifiedEvalIds={verifiedOnly ? verifiedIds : null}
518
  sortCol={sortCol}
519
  sortDir={sortDir}
520
  onSort={handleSort}
 
525
  hasMore={hasMore}
526
  onLoadMore={handleLoadMore}
527
  loadingLabel="Loading more…"
528
+ endLabel={`Showing ${Math.min(visibleCount, totalRows).toLocaleString()} of ${totalRows.toLocaleString()} ${groupBy === "evaluator" ? "evaluators" : "families"}`}
529
  />
530
  </main>
531
  </div>
app/evaluators/[...id]/page.tsx ADDED
@@ -0,0 +1,248 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ "use client"
2
+
3
+ import { Suspense, useCallback, useEffect, useMemo, useState } from "react"
4
+ import { useParams, useRouter, useSearchParams } from "next/navigation"
5
+ import { ArrowLeft, Search } from "lucide-react"
6
+
7
+ import { EvalCard } from "@/components/eval-card"
8
+ import { InfiniteScrollSentinel } from "@/components/infinite-scroll"
9
+ import { Navigation } from "@/components/navigation"
10
+ import { VerifiedBadge } from "@/components/signals/verified-badge"
11
+ import { fetchEvalList } from "@/lib/dashboard-data-client"
12
+ import type { BenchmarkEvalListItem } from "@/lib/eval-processing"
13
+ import { getEvalsForEvaluator } from "@/lib/evaluators"
14
+
15
+ const PAGE_SIZE = 24
16
+
17
+ function EvaluatorDetailInner() {
18
+ const params = useParams()
19
+ const router = useRouter()
20
+ const searchParams = useSearchParams()
21
+ const verifiedOnly = searchParams.get("verified") === "1" || searchParams.get("verified") === "true"
22
+
23
+ // The slug is a single URL-safe segment, but the route is a catch-all
24
+ // ([...id]) to match the developers/models pattern. Join just in case.
25
+ const slug = useMemo(() => {
26
+ const raw = params.id as string | string[] | undefined
27
+ const joined = Array.isArray(raw) ? raw.join("/") : (raw ?? "")
28
+ return decodeURIComponent(joined)
29
+ }, [params.id])
30
+
31
+ const [allEvals, setAllEvals] = useState<BenchmarkEvalListItem[]>([])
32
+ const [loading, setLoading] = useState(true)
33
+ const [error, setError] = useState<string | null>(null)
34
+ const [searchQuery, setSearchQuery] = useState("")
35
+ const [visibleCount, setVisibleCount] = useState(PAGE_SIZE)
36
+
37
+ useEffect(() => {
38
+ fetchEvalList()
39
+ .then((list) => setAllEvals(list.evals))
40
+ .catch((err) => {
41
+ console.error(err)
42
+ setError("Failed to load evaluations")
43
+ })
44
+ .finally(() => setLoading(false))
45
+ }, [])
46
+
47
+ const { name, isVerified, evals } = useMemo(
48
+ () => getEvalsForEvaluator(allEvals, slug, { verifiedOnly }),
49
+ [allEvals, slug, verifiedOnly],
50
+ )
51
+
52
+ const filteredEvals = useMemo(() => {
53
+ const query = searchQuery.trim().toLowerCase()
54
+ const list = query
55
+ ? evals.filter((ev) => {
56
+ const haystacks = [
57
+ ev.evaluation_name,
58
+ ev.family_display_name,
59
+ ev.composite_benchmark_name,
60
+ ]
61
+ return haystacks.some((v) => v?.toLowerCase().includes(query))
62
+ })
63
+ : evals
64
+ return list.slice().sort((a, b) => a.evaluation_name.localeCompare(b.evaluation_name))
65
+ }, [evals, searchQuery])
66
+
67
+ // Quantified facts for the header, derived from the org's owned evals.
68
+ // familyCount = distinct benchmark families covered; verifiedCount = evals
69
+ // where this org is a verified evaluator.
70
+ const { familyCount, verifiedCount } = useMemo(() => {
71
+ const families = new Set<string>()
72
+ let verified = 0
73
+ for (const ev of evals) {
74
+ const fam = ev.family_display_name?.trim()
75
+ if (fam) families.add(fam)
76
+ if (name && (ev.verified_evaluator_names ?? []).includes(name)) verified += 1
77
+ }
78
+ return {
79
+ familyCount: families.size,
80
+ verifiedCount: verified,
81
+ }
82
+ }, [evals, name])
83
+
84
+ useEffect(() => {
85
+ setVisibleCount(PAGE_SIZE)
86
+ }, [searchQuery, slug, verifiedOnly])
87
+
88
+ const visibleEvals = useMemo(
89
+ () => filteredEvals.slice(0, visibleCount),
90
+ [filteredEvals, visibleCount],
91
+ )
92
+ const hasMore = visibleCount < filteredEvals.length
93
+ const handleLoadMore = useCallback(() => {
94
+ setVisibleCount((current) => Math.min(current + PAGE_SIZE, filteredEvals.length))
95
+ }, [filteredEvals.length])
96
+
97
+ const handleBack = useCallback(() => {
98
+ router.push(verifiedOnly ? "/evals?groupBy=evaluator&verified=1" : "/evals?groupBy=evaluator")
99
+ }, [router, verifiedOnly])
100
+
101
+ if (loading) {
102
+ return (
103
+ <div className="min-h-screen bg-background">
104
+ <Navigation />
105
+ <main className="ec-page">
106
+ <div className="flex h-96 items-center justify-center">
107
+ <div className="kicker">Loading evaluator…</div>
108
+ </div>
109
+ </main>
110
+ </div>
111
+ )
112
+ }
113
+
114
+ // Slug resolved to no org (bad/expired link) — or the org has no evals
115
+ // under the active verified filter.
116
+ if (error || !name) {
117
+ return (
118
+ <div className="min-h-screen bg-background">
119
+ <Navigation />
120
+ <main className="ec-page">
121
+ <div className="flex flex-col items-center justify-center h-96 space-y-4">
122
+ <div className="kicker">{error ?? "Evaluator not found"}</div>
123
+ <button type="button" onClick={handleBack} className="btn-ec outline">
124
+ <ArrowLeft className="mr-2 h-4 w-4" />
125
+ Back
126
+ </button>
127
+ </div>
128
+ </main>
129
+ </div>
130
+ )
131
+ }
132
+
133
+ return (
134
+ <div className="min-h-screen bg-background">
135
+ <Navigation />
136
+
137
+ <main className="mx-auto w-full max-w-[96rem] px-4 pt-12 pb-24 sm:px-8">
138
+ {/* BREADCRUMB ----------------------------------------------- */}
139
+ <button
140
+ type="button"
141
+ onClick={handleBack}
142
+ className="ec-crumb mb-4 inline-flex items-center gap-1.5"
143
+ >
144
+ <ArrowLeft className="h-3 w-3" />
145
+ Evaluators
146
+ </button>
147
+
148
+ {/* HEADER --------------------------------------------------- */}
149
+ <div className="kicker">Evaluator</div>
150
+ <h1 className="ec-page-h1 inline-flex items-center gap-2">
151
+ {name}
152
+ {isVerified && <VerifiedBadge verified size="md" />}
153
+ </h1>
154
+ <div
155
+ className="mb-5 flex flex-wrap items-center gap-3 font-mono text-[11px] uppercase tracking-[0.12em]"
156
+ style={{ color: "var(--fg-muted)" }}
157
+ >
158
+ <span>Reporting organisation</span>
159
+ <span style={{ color: "var(--fg-subtle)" }}>·</span>
160
+ <span>
161
+ {familyCount} {familyCount === 1 ? "family" : "families"}
162
+ </span>
163
+ <span style={{ color: "var(--fg-subtle)" }}>·</span>
164
+ <span>{verifiedCount} verified</span>
165
+ </div>
166
+ <p className="ec-page-lede">
167
+ Reported <strong>{filteredEvals.length.toLocaleString()}</strong>{" "}
168
+ {filteredEvals.length === 1 ? "evaluation" : "evaluations"} across{" "}
169
+ <strong>{familyCount.toLocaleString()}</strong>{" "}
170
+ {familyCount === 1 ? "benchmark family" : "benchmark families"}
171
+ {verifiedCount > 0 && (
172
+ <>
173
+ , <strong>{verifiedCount.toLocaleString()}</strong> verified
174
+ </>
175
+ )}
176
+ {verifiedOnly ? " (verified submissions only)" : ""}.
177
+ </p>
178
+
179
+ <div className="ec-page-meta mt-2">
180
+ <div className="ec-page-meta-item">
181
+ <span className="ec-page-meta-item-l">Evaluations</span>
182
+ <span className="ec-page-meta-item-v">{filteredEvals.length.toLocaleString()}</span>
183
+ </div>
184
+ <div className="ec-page-meta-item">
185
+ <span className="ec-page-meta-item-l">Verified</span>
186
+ <span className="ec-page-meta-item-v">{verifiedCount.toLocaleString()}</span>
187
+ </div>
188
+ <div className="ec-page-meta-item">
189
+ <span className="ec-page-meta-item-l">Families</span>
190
+ <span className="ec-page-meta-item-v">{familyCount.toLocaleString()}</span>
191
+ </div>
192
+ </div>
193
+
194
+ {/* META + FILTER BAR --------------------------------------- */}
195
+ <div className="mb-6 flex flex-wrap items-center gap-x-8 gap-y-3 border-y border-[color:var(--border-soft)] py-4">
196
+ <div className="flex shrink-0 flex-wrap items-baseline gap-x-4 gap-y-1 font-mono text-[11px] tracking-[0.1em] uppercase text-[color:var(--fg-subtle)]">
197
+ <span>
198
+ <span className="text-[color:var(--fg)] tabular-nums font-semibold mr-1">
199
+ {filteredEvals.length.toLocaleString()}
200
+ </span>
201
+ {filteredEvals.length === 1 ? "evaluation" : "evaluations"}
202
+ </span>
203
+ </div>
204
+
205
+ <span className="hidden h-5 w-px bg-[color:var(--border-soft)] sm:block" />
206
+
207
+ <div className="relative min-w-[200px] flex-1 sm:max-w-[300px]">
208
+ <Search className="pointer-events-none absolute left-3 top-1/2 h-3.5 w-3.5 -translate-y-1/2 text-[color:var(--fg-subtle)]" />
209
+ <input
210
+ className="ec-input pl-9"
211
+ value={searchQuery}
212
+ onChange={(event) => setSearchQuery(event.target.value)}
213
+ placeholder="Search evaluations…"
214
+ />
215
+ </div>
216
+ </div>
217
+
218
+ {/* EVAL CARDS ---------------------------------------------- */}
219
+ {filteredEvals.length === 0 ? (
220
+ <div className="border border-dashed border-[color:var(--border-soft)] bg-[color:var(--bg-warm)] py-12 text-center font-mono text-[11px] uppercase tracking-[0.2em] text-[color:var(--fg-subtle)]">
221
+ No evaluations match the current filters
222
+ </div>
223
+ ) : (
224
+ <div className="grid grid-cols-1 gap-4 md:grid-cols-2 xl:grid-cols-3">
225
+ {visibleEvals.map((ev, i) => (
226
+ <EvalCard key={ev.evaluation_id} summary={ev} delayMs={Math.min(i, 8) * 40} />
227
+ ))}
228
+ </div>
229
+ )}
230
+
231
+ <InfiniteScrollSentinel
232
+ hasMore={hasMore}
233
+ onLoadMore={handleLoadMore}
234
+ loadingLabel="Loading more…"
235
+ endLabel={`Showing ${Math.min(visibleCount, filteredEvals.length).toLocaleString()} of ${filteredEvals.length.toLocaleString()} evaluations`}
236
+ />
237
+ </main>
238
+ </div>
239
+ )
240
+ }
241
+
242
+ export default function EvaluatorDetailPage() {
243
+ return (
244
+ <Suspense fallback={null}>
245
+ <EvaluatorDetailInner />
246
+ </Suspense>
247
+ )
248
+ }
components/benchmark-detail.tsx CHANGED
@@ -24,6 +24,7 @@ import {
24
  } from "@/components/signals/provenance-badge"
25
  import { SignalsRowBadges } from "@/components/signals/signals-row-badges"
26
  import { SignalTooltip } from "@/components/signals/signal-tooltip"
 
27
  import {
28
  DropdownMenu,
29
  DropdownMenuContent,
@@ -9443,8 +9444,9 @@ function CategoryStatsView({
9443
  : normalizeDisplayLabel(eval_.source_data.dataset_name))}
9444
  </div>
9445
  </div>
9446
- <div className="font-mono font-semibold">
9447
  {formatRawScoreValue(result.score_details.score, result.metric_config.unit)}
 
9448
  </div>
9449
  </div>
9450
  ))
 
24
  } from "@/components/signals/provenance-badge"
25
  import { SignalsRowBadges } from "@/components/signals/signals-row-badges"
26
  import { SignalTooltip } from "@/components/signals/signal-tooltip"
27
+ import { VerifiedBadge } from "@/components/signals/verified-badge"
28
  import {
29
  DropdownMenu,
30
  DropdownMenuContent,
 
9444
  : normalizeDisplayLabel(eval_.source_data.dataset_name))}
9445
  </div>
9446
  </div>
9447
+ <div className="flex items-center gap-1.5 font-mono font-semibold">
9448
  {formatRawScoreValue(result.score_details.score, result.metric_config.unit)}
9449
+ <VerifiedBadge verified={result.is_verified_evaluator} />
9450
  </div>
9451
  </div>
9452
  ))
components/eval-card.tsx CHANGED
@@ -4,6 +4,7 @@ import type { ComponentType, CSSProperties } from "react"
4
  import { useAudienceMode } from "@/components/audience-mode-provider"
5
  import { Badge } from "@/components/ui/badge"
6
  import { Card, CardContent, CardHeader } from "@/components/ui/card"
 
7
  import { useRouter } from "next/navigation"
8
  import {
9
  AlertTriangle,
@@ -16,7 +17,9 @@ import {
16
  Scale,
17
  Users,
18
  } from "lucide-react"
 
19
  import { routeIdToPath } from "@/lib/utils"
 
20
  import type { BenchmarkEvalListItem } from "@/lib/eval-processing"
21
  import { getTagColor, tagLabel } from "@/lib/benchmark-schema"
22
 
@@ -78,6 +81,8 @@ export function EvalCard({ summary, delayMs = 0 }: EvalCardProps) {
78
  const rawSimilar = card?.benchmark_details?.similar_benchmarks
79
  const similarBenchmarks: string[] = Array.isArray(rawSimilar) ? rawSimilar : rawSimilar ? [rawSimilar] : []
80
  const domainPreview = domains.slice(0, 2)
 
 
81
  // Source provenance pulled from the pipeline's source_data
82
  const sourceData = summary.source_data
83
  const reproducibilitySummary = summary.reproducibility_summary
@@ -258,7 +263,28 @@ export function EvalCard({ summary, delayMs = 0 }: EvalCardProps) {
258
  <div className="rounded-xl border bg-muted/10 p-3">
259
  <div className="space-y-1.5 text-sm">
260
  <DataRow label="Avg score" value={scorePercent} />
261
- <DataRow label="Reported by" value={summary.evaluator_names.join(", ") || "Unknown"} />
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
262
  {reproducibilityGapCount > 0 && (
263
  <p className="pt-1 text-xs text-muted-foreground">
264
  {reproducibilityGapCount} of {reproducibilityResultsTotal} reported scores are not fully documented.
 
4
  import { useAudienceMode } from "@/components/audience-mode-provider"
5
  import { Badge } from "@/components/ui/badge"
6
  import { Card, CardContent, CardHeader } from "@/components/ui/card"
7
+ import { VerifiedBadge } from "@/components/signals/verified-badge"
8
  import { useRouter } from "next/navigation"
9
  import {
10
  AlertTriangle,
 
17
  Scale,
18
  Users,
19
  } from "lucide-react"
20
+ import Link from "next/link"
21
  import { routeIdToPath } from "@/lib/utils"
22
+ import { evaluatorSlug } from "@/lib/evaluators"
23
  import type { BenchmarkEvalListItem } from "@/lib/eval-processing"
24
  import { getTagColor, tagLabel } from "@/lib/benchmark-schema"
25
 
 
81
  const rawSimilar = card?.benchmark_details?.similar_benchmarks
82
  const similarBenchmarks: string[] = Array.isArray(rawSimilar) ? rawSimilar : rawSimilar ? [rawSimilar] : []
83
  const domainPreview = domains.slice(0, 2)
84
+ // Validated evaluators (subset of evaluator_names) for the badge.
85
+ const verifiedEvaluators = new Set(summary.verified_evaluator_names ?? [])
86
  // Source provenance pulled from the pipeline's source_data
87
  const sourceData = summary.source_data
88
  const reproducibilitySummary = summary.reproducibility_summary
 
263
  <div className="rounded-xl border bg-muted/10 p-3">
264
  <div className="space-y-1.5 text-sm">
265
  <DataRow label="Avg score" value={scorePercent} />
266
+ <div className="flex items-start justify-between gap-3">
267
+ <span className="shrink-0 text-muted-foreground">Reported by</span>
268
+ <span className="text-right font-medium text-foreground">
269
+ {summary.evaluator_names.length === 0
270
+ ? "Unknown"
271
+ : summary.evaluator_names.map((name, i) => (
272
+ <span key={name} className="inline-flex items-center">
273
+ {i > 0 ? ", " : null}
274
+ <Link
275
+ href={`/evaluators/${evaluatorSlug(name)}`}
276
+ className="hover:text-[color:var(--accent)] hover:underline"
277
+ onClick={(e) => e.stopPropagation()}
278
+ >
279
+ {name}
280
+ </Link>
281
+ {verifiedEvaluators.has(name) ? (
282
+ <VerifiedBadge verified size="sm" className="ml-1 align-middle" />
283
+ ) : null}
284
+ </span>
285
+ ))}
286
+ </span>
287
+ </div>
288
  {reproducibilityGapCount > 0 && (
289
  <p className="pt-1 text-xs text-muted-foreground">
290
  {reproducibilityGapCount} of {reproducibilityResultsTotal} reported scores are not fully documented.
components/eval-detail.tsx CHANGED
@@ -5,6 +5,7 @@ import { Fragment, useEffect, useMemo, useState } from "react"
5
  import Link from "next/link"
6
  import { BenchmarkSignalsStrip } from "@/components/signals/benchmark-signals-strip"
7
  import { SignalsRowBadges } from "@/components/signals/signals-row-badges"
 
8
  import { getCompletenessPopulatedCount } from "@/components/signals/signal-utils"
9
  import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"
10
  import { ScoreDistribution } from "@/components/score-distribution"
@@ -53,6 +54,7 @@ import {
53
  import type { BenchmarkCard, SourceData } from "@/lib/benchmark-schema"
54
  import { tagLabel } from "@/lib/benchmark-schema"
55
  import type { BenchmarkEvalSummary, ModelResultForBenchmark } from "@/lib/eval-processing"
 
56
  import type { ComparisonIndex, EvalHierarchy } from "@/lib/backend-artifacts"
57
  import type { HierarchyEvalLocation } from "@/lib/hierarchy-lookup"
58
  import { PolicyOverview } from "@/components/policy-overview"
@@ -406,6 +408,38 @@ function formatMetadataValue(value: unknown): string {
406
  // and other surfaces format identically.
407
  const formatDate = formatDateISO
408
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
409
  /**
410
  * Render a benchmark-card field path (e.g. `methodology.metrics`,
411
  * `purpose_and_intended_users.goal`) as a human-readable label —
@@ -535,11 +569,42 @@ function getCompactMetricLabel(value: string | undefined): string {
535
  }
536
 
537
  /**
538
- * Build a chip-friendly label for a leaderboard metric. Prefers
539
- * display_name, then metric_name, then a humanised tail of metric_id /
540
- * column_key. The upstream pipeline frequently leaves display_name
541
- * blank (e.g. inspect_evals/avg_full_score), in which case the
542
- * column_key tail ('avg_full_score') is what we want to surface.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
543
  */
544
  function getMetricChipLabel(metric: {
545
  display_name?: string | null
@@ -547,17 +612,24 @@ function getMetricChipLabel(metric: {
547
  metric_id?: string | null
548
  column_key?: string | null
549
  }): string {
 
 
 
 
 
 
550
  const candidates = [
551
- metric.display_name,
552
- metric.metric_name,
553
- metric.metric_id,
554
- metric.column_key,
555
  ]
556
  for (const c of candidates) {
557
  if (c && String(c).trim()) {
558
  return compactizePath(String(c)).replace(/_/g, " ")
559
  }
560
  }
 
 
 
561
  return "Metric"
562
  }
563
 
@@ -931,17 +1003,6 @@ export function EvalDetail({
931
  return userRowSort.dir === "asc" ? "↑" : "↓"
932
  }
933
 
934
- // Hide the "Updated" column when no row has a usable timestamp —
935
- // every cell would say "Unknown" otherwise. formatDate returns the
936
- // string "Unknown" for null / empty / unparseable inputs.
937
- const hasAnyUpdatedTimestamp = useMemo(
938
- () =>
939
- leaderboardRows.some(
940
- ({ modelResult }) => formatDate(modelResult.evaluation_timestamp) !== "Unknown",
941
- ),
942
- [leaderboardRows],
943
- )
944
-
945
  const avgScoreLabel = formatRawScore(lb.avg_score, lb.metric_config.unit)
946
  const scoreDirectionLabel = lb.metric_config.lower_is_better ? "Lower scores rank higher" : "Higher scores rank higher"
947
  const leaderboardTitle = isResearchView ? "Leaderboard" : "Reporting Comparison"
@@ -989,12 +1050,41 @@ export function EvalDetail({
989
  // identical in chrome — naming the evaluator up-front is the cheapest
990
  // way to make the pages visually distinct.
991
  const evaluatorList = summary.evaluator_names ?? []
992
- const reporterLabel = (() => {
993
- if (evaluatorList.length === 0) return null
994
- const head = evaluatorList.slice(0, 2)
995
- const extra = evaluatorList.length - head.length
996
- return extra > 0 ? `${head.join(", ")} +${extra} more` : head.join(", ")
997
- })()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
998
 
999
  const heroLede = isResearchView
1000
  ? summary.metric_config.evaluation_description
@@ -1006,7 +1096,7 @@ export function EvalDetail({
1006
  <div className="space-y-12">
1007
  {/* HERO ------------------------------------------------ */}
1008
  <header className="motion-academic-enter">
1009
- {reporterLabel && (
1010
  <div
1011
  className="font-mono uppercase"
1012
  style={{
@@ -1017,7 +1107,18 @@ export function EvalDetail({
1017
  }}
1018
  >
1019
  <span style={{ color: "var(--fg-muted)" }}>Reported by </span>
1020
- <span style={{ color: "var(--fg)" }}>{reporterLabel}</span>
 
 
 
 
 
 
 
 
 
 
 
1021
  </div>
1022
  )}
1023
  <h1
@@ -1684,17 +1785,6 @@ export function EvalDetail({
1684
  title="Sort by model release date"
1685
  />
1686
  </th>
1687
- {hasAnyUpdatedTimestamp && (
1688
- <th className="hidden xl:table-cell num" style={{ width: 110 }}>
1689
- <SortableTh
1690
- label="Updated"
1691
- active={userRowSort.key === "updated"}
1692
- indicator={rowSortIndicator("updated")}
1693
- onClick={() => cycleRowSort("updated")}
1694
- title="Sort by report timestamp"
1695
- />
1696
- </th>
1697
- )}
1698
  </tr>
1699
  </thead>
1700
  <tbody>
@@ -1735,6 +1825,13 @@ export function EvalDetail({
1735
  (!Array.isArray(modelResult.source_data) && modelResult.source_data.source_type) ||
1736
  modelResult.source_metadata.source_type ||
1737
  ""
 
 
 
 
 
 
 
1738
  const familyLabel = modelResult.model_info.architecture
1739
  ?? modelResult.model_info.parameter_count
1740
  ?? null
@@ -1898,25 +1995,35 @@ export function EvalDetail({
1898
 
1899
  <td className="num hidden lg:table-cell align-top">
1900
  {sourceTypeLabel ? (
1901
- modelResult.source_metadata.source_url ? (
1902
- <a
1903
- href={modelResult.source_metadata.source_url}
1904
- target="_blank"
1905
- rel="noreferrer"
1906
- className="font-mono lowercase hover:text-[color:var(--accent)]"
1907
- style={{ fontSize: 11, color: "var(--fg-muted)" }}
1908
- onClick={(e) => e.stopPropagation()}
1909
- >
1910
- {sourceTypeLabel}
1911
- </a>
1912
- ) : (
1913
- <span
1914
- className="font-mono lowercase"
1915
- style={{ fontSize: 11, color: "var(--fg-muted)" }}
1916
- >
1917
- {sourceTypeLabel}
1918
- </span>
1919
- )
 
 
 
 
 
 
 
 
 
 
1920
  ) : (
1921
  <span style={{ color: "var(--fg-subtle)" }}>—</span>
1922
  )}
@@ -1931,16 +2038,11 @@ export function EvalDetail({
1931
  : <span style={{ color: "var(--fg-subtle)" }}>—</span>}
1932
  </td>
1933
 
1934
- {hasAnyUpdatedTimestamp && (
1935
- <td className="num hidden xl:table-cell align-top font-mono tabular-nums" style={{ fontSize: 11, color: "var(--fg-muted)" }}>
1936
- {formatDate(modelResult.evaluation_timestamp)}
1937
- </td>
1938
- )}
1939
  </tr>
1940
 
1941
  {isExpanded && (
1942
  <tr>
1943
- <td colSpan={hasAnyUpdatedTimestamp ? 8 : 7} style={{ background: "var(--bg-warm)", padding: 0 }}>
1944
  <div className="space-y-5 px-4 py-5 sm:px-6">
1945
  {/* The Model Profile / Provenance / Score Breakdown
1946
  panels were removed — model metadata lives on
@@ -2134,7 +2236,7 @@ export function EvalDetail({
2134
  })}
2135
  {leaderboardRows.length === 0 && (
2136
  <tr>
2137
- <td colSpan={hasAnyUpdatedTimestamp ? 8 : 7} style={{ padding: "32px 16px", textAlign: "center", color: "var(--fg-muted)" }}>
2138
  No leaderboard entries match the selected parameter range.
2139
  </td>
2140
  </tr>
@@ -2193,6 +2295,24 @@ function MultiMetricLeaderboard({
2193
  const [maxParamStep, setMaxParamStep] = useState(PARAM_RANGE_MAX_INDEX)
2194
  const [expandedRows, setExpandedRows] = useState<Record<string, boolean>>({})
2195
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2196
  // Index ModelResultForBenchmark entries by model_info.id so we can power the
2197
  // research-mode reproducibility card from a multi-metric row. There may be
2198
  // several entries per model (one per metric); we prefer one with a recorded
@@ -2251,6 +2371,40 @@ function MultiMetricLeaderboard({
2251
 
2252
  const hasSliceTabs = sliceTabs.length > 1
2253
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2254
  const visibleMetrics = useMemo(
2255
  () =>
2256
  leaderboardMetrics.filter((metric) => {
@@ -2258,6 +2412,10 @@ function MultiMetricLeaderboard({
2258
  return false
2259
  }
2260
 
 
 
 
 
2261
  if (!hasSliceTabs || activeSliceTab === "all") {
2262
  // "All" / no-slice-filter case: only show root metrics so the
2263
  // chips stay one-per-metric instead of one-per-(metric, slice).
@@ -2266,7 +2424,7 @@ function MultiMetricLeaderboard({
2266
 
2267
  return metric.scope === "subtask" && metric.subtask_key === activeSliceTab
2268
  }),
2269
- [activeSliceTab, hasSliceTabs, leaderboardMetrics, visibleMetricKeySet]
2270
  )
2271
  const visibleMetricColumnKeySet = useMemo(
2272
  () => new Set(visibleMetrics.map((metric) => metric.column_key)),
@@ -2683,7 +2841,7 @@ function MultiMetricLeaderboard({
2683
  // name is already shown above the table — no need to
2684
  // repeat it as a per-column topline.
2685
  const showSliceTopline = false
2686
- const mainLabel = getCompactMetricLabel(metric.display_name)
2687
  return (
2688
  <th
2689
  key={metric.column_key}
@@ -2710,6 +2868,8 @@ function MultiMetricLeaderboard({
2710
  </th>
2711
  )
2712
  })}
 
 
2713
  <th
2714
  className="num hidden lg:table-cell"
2715
  style={{ width: 110, cursor: "pointer" }}
@@ -2718,13 +2878,6 @@ function MultiMetricLeaderboard({
2718
  >
2719
  Released{getSortIndicator("released")}
2720
  </th>
2721
- <th
2722
- className="num hidden xl:table-cell"
2723
- style={{ width: 110, cursor: "pointer" }}
2724
- onClick={() => handleSort("updated")}
2725
- >
2726
- Updated{getSortIndicator("updated")}
2727
- </th>
2728
  </tr>
2729
  </thead>
2730
  <tbody>
@@ -2818,6 +2971,42 @@ function MultiMetricLeaderboard({
2818
  )
2819
  })}
2820
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2821
  <td
2822
  className="num hidden lg:table-cell align-top font-mono tabular-nums"
2823
  style={{ fontSize: 11, color: "var(--fg-muted)" }}
@@ -2826,14 +3015,11 @@ function MultiMetricLeaderboard({
2826
  ? formatDate(row.model_info.release_date).split(",")[0]
2827
  : <span style={{ color: "var(--fg-subtle)" }}>—</span>}
2828
  </td>
2829
- <td className="num hidden xl:table-cell align-top font-mono tabular-nums" style={{ fontSize: 11, color: "var(--fg-muted)" }}>
2830
- {formatDate(row.evaluation_timestamp)}
2831
- </td>
2832
  </tr>
2833
  {isResearchView && isExpanded && matchingResult && (
2834
  <tr>
2835
  <td
2836
- colSpan={visibleMetrics.length + 6}
2837
  style={{ background: "var(--bg-warm)", padding: "20px 24px" }}
2838
  >
2839
  <div className="space-y-3">
@@ -2863,7 +3049,7 @@ function MultiMetricLeaderboard({
2863
 
2864
  {filteredRows.length === 0 && (
2865
  <tr>
2866
- <td colSpan={visibleMetrics.length + 5} style={{ padding: "32px 16px", textAlign: "center", color: "var(--fg-muted)" }}>
2867
  No models match the selected parameter range.
2868
  </td>
2869
  </tr>
 
5
  import Link from "next/link"
6
  import { BenchmarkSignalsStrip } from "@/components/signals/benchmark-signals-strip"
7
  import { SignalsRowBadges } from "@/components/signals/signals-row-badges"
8
+ import { VerifiedBadge } from "@/components/signals/verified-badge"
9
  import { getCompletenessPopulatedCount } from "@/components/signals/signal-utils"
10
  import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"
11
  import { ScoreDistribution } from "@/components/score-distribution"
 
54
  import type { BenchmarkCard, SourceData } from "@/lib/benchmark-schema"
55
  import { tagLabel } from "@/lib/benchmark-schema"
56
  import type { BenchmarkEvalSummary, ModelResultForBenchmark } from "@/lib/eval-processing"
57
+ import { evaluatorSlug } from "@/lib/evaluators"
58
  import type { ComparisonIndex, EvalHierarchy } from "@/lib/backend-artifacts"
59
  import type { HierarchyEvalLocation } from "@/lib/hierarchy-lookup"
60
  import { PolicyOverview } from "@/components/policy-overview"
 
408
  // and other surfaces format identically.
409
  const formatDate = formatDateISO
410
 
411
+ /**
412
+ * Render an evaluator org name. When `linkName` is a known evaluator (a
413
+ * de-aliased name with a /evaluators/<slug> page) the name links there;
414
+ * otherwise it renders as plain text so we never emit a broken link. The
415
+ * slug uses the shared, deterministic `evaluatorSlug` base helper.
416
+ */
417
+ function EvaluatorName({
418
+ display,
419
+ linkName,
420
+ className,
421
+ style,
422
+ }: {
423
+ display: React.ReactNode
424
+ linkName: string | null
425
+ className?: string
426
+ style?: React.CSSProperties
427
+ }) {
428
+ if (!linkName) {
429
+ return <span className={className} style={style}>{display}</span>
430
+ }
431
+ return (
432
+ <Link
433
+ href={`/evaluators/${evaluatorSlug(linkName)}`}
434
+ className={cn("hover:text-[color:var(--accent)] hover:underline", className)}
435
+ style={style}
436
+ onClick={(e) => e.stopPropagation()}
437
+ >
438
+ {display}
439
+ </Link>
440
+ )
441
+ }
442
+
443
  /**
444
  * Render a benchmark-card field path (e.g. `methodology.metrics`,
445
  * `purpose_and_intended_users.goal`) as a human-readable label —
 
569
  }
570
 
571
  /**
572
+ * Humanise a raw metric identifier (metric_id / column_key) into a
573
+ * compact, readable label.
574
+ *
575
+ * Two shapes the upstream view layer leaves un-curated:
576
+ * - path-ish keys (`inspect_evals/avg_full_score`) keep the tail
577
+ * (`avg full score`).
578
+ * - `<benchmark-slug>.<stat>` keys (`cyse2-vulnerability-exploit.mean`,
579
+ * `swebench-…-mariushobbhahn.mean`) → the slug prefix just repeats the
580
+ * eval name, so collapse to the trailing stat (`Mean`). Without this the
581
+ * column header echoes the raw UPPER.SLUG.
582
+ */
583
+ function humanizeMetricKey(raw: string): string {
584
+ const tail = compactizePath(raw)
585
+ // `<slug>.mean` / `.std` / `.stderr` → just the trailing stat.
586
+ const dotMatch = /^(.+)\.([a-z0-9_]+)$/i.exec(tail)
587
+ if (dotMatch) {
588
+ const [, prefix, stat] = dotMatch
589
+ // Only collapse when the prefix looks like a slug (has a hyphen or is
590
+ // long), not a genuinely dotted metric name.
591
+ if (prefix.includes("-") || prefix.length > 6) {
592
+ return humanizeMetricKey(stat)
593
+ }
594
+ }
595
+ const spaced = tail.replace(/_/g, " ").trim()
596
+ if (!spaced) return tail
597
+ return spaced.charAt(0).toUpperCase() + spaced.slice(1)
598
+ }
599
+
600
+ /**
601
+ * Build a chip-friendly label for a leaderboard metric. Prefers a curated
602
+ * display_name / metric_name, then a humanised tail of metric_id /
603
+ * column_key. The upstream pipeline frequently leaves display_name blank
604
+ * (e.g. inspect_evals/avg_full_score → ifeval's final_acc, inst_loose_acc)
605
+ * or echoes the raw column key as the "display name" (cyse2's
606
+ * `cyse2-vulnerability-exploit.mean`). In both cases the humanised key tail
607
+ * is what we want to surface — never the literal 'Metric' or a raw slug.
608
  */
609
  function getMetricChipLabel(metric: {
610
  display_name?: string | null
 
612
  metric_id?: string | null
613
  column_key?: string | null
614
  }): string {
615
+ const key = metric.metric_id ?? metric.column_key ?? null
616
+ // Treat a display/metric name that merely echoes the raw column key as
617
+ // absent — it carries no more information than the key itself.
618
+ const isRawEcho = (value: string | null | undefined) =>
619
+ !!value && !!key && value.trim() === key.trim()
620
+
621
  const candidates = [
622
+ isRawEcho(metric.display_name) ? null : metric.display_name,
623
+ isRawEcho(metric.metric_name) ? null : metric.metric_name,
 
 
624
  ]
625
  for (const c of candidates) {
626
  if (c && String(c).trim()) {
627
  return compactizePath(String(c)).replace(/_/g, " ")
628
  }
629
  }
630
+ if (key && key.trim()) {
631
+ return humanizeMetricKey(key)
632
+ }
633
  return "Metric"
634
  }
635
 
 
1003
  return userRowSort.dir === "asc" ? "↑" : "↓"
1004
  }
1005
 
 
 
 
 
 
 
 
 
 
 
 
1006
  const avgScoreLabel = formatRawScore(lb.avg_score, lb.metric_config.unit)
1007
  const scoreDirectionLabel = lb.metric_config.lower_is_better ? "Lower scores rank higher" : "Higher scores rank higher"
1008
  const leaderboardTitle = isResearchView ? "Leaderboard" : "Reporting Comparison"
 
1050
  // identical in chrome — naming the evaluator up-front is the cheapest
1051
  // way to make the pages visually distinct.
1052
  const evaluatorList = summary.evaluator_names ?? []
1053
+ // Validated evaluators (de-aliased names, same space as evaluator_names),
1054
+ // straight from the backend rollup — used to badge the "Reported by" names.
1055
+ const verifiedEvaluators = useMemo(
1056
+ () => new Set(summary.verified_evaluator_names ?? []),
1057
+ [summary.verified_evaluator_names],
1058
+ )
1059
+
1060
+ // De-aliased evaluator names that have a /evaluators/<slug> page. The header
1061
+ // renders these names directly (always linkable); the per-row Source column
1062
+ // shows a *raw* source name, so we resolve it case-insensitively against the
1063
+ // known evaluator names and only link when it maps to a real evaluator page.
1064
+ // We key on the union of summary + active-split evaluator names so a split
1065
+ // view's Source cells still resolve.
1066
+ const knownEvaluatorByLower = useMemo(() => {
1067
+ const m = new Map<string, string>()
1068
+ for (const n of [...(summary.evaluator_names ?? []), ...(lb.evaluator_names ?? [])]) {
1069
+ const t = (n ?? "").trim()
1070
+ if (t) m.set(t.toLowerCase(), t)
1071
+ }
1072
+ return m
1073
+ }, [summary.evaluator_names, lb.evaluator_names])
1074
+
1075
+ // Resolve a (raw or de-aliased) org name to a known evaluator's de-aliased
1076
+ // name, or null when it isn't a known evaluator (→ render plain text). The
1077
+ // per-row Source value is the *raw* source name (e.g. "crfm"), which can be
1078
+ // an alias of a de-aliased evaluator ("Stanford CRFM"); when it doesn't match
1079
+ // directly but the eval has exactly one evaluator, that row unambiguously
1080
+ // belongs to it, so we link to the sole evaluator.
1081
+ const soleEvaluator =
1082
+ knownEvaluatorByLower.size === 1 ? Array.from(knownEvaluatorByLower.values())[0] : null
1083
+ const resolveEvaluatorName = (raw: string | undefined | null): string | null => {
1084
+ const t = (raw ?? "").trim()
1085
+ if (!t) return null
1086
+ return knownEvaluatorByLower.get(t.toLowerCase()) ?? soleEvaluator
1087
+ }
1088
 
1089
  const heroLede = isResearchView
1090
  ? summary.metric_config.evaluation_description
 
1096
  <div className="space-y-12">
1097
  {/* HERO ------------------------------------------------ */}
1098
  <header className="motion-academic-enter">
1099
+ {evaluatorList.length > 0 && (
1100
  <div
1101
  className="font-mono uppercase"
1102
  style={{
 
1107
  }}
1108
  >
1109
  <span style={{ color: "var(--fg-muted)" }}>Reported by </span>
1110
+ {evaluatorList.slice(0, 2).map((name, i) => (
1111
+ <span key={name} style={{ color: "var(--fg)" }}>
1112
+ {i > 0 ? ", " : null}
1113
+ <EvaluatorName display={name} linkName={resolveEvaluatorName(name)} />
1114
+ {verifiedEvaluators.has(name) ? (
1115
+ <VerifiedBadge verified size="sm" className="ml-1 align-middle" />
1116
+ ) : null}
1117
+ </span>
1118
+ ))}
1119
+ {evaluatorList.length > 2 ? (
1120
+ <span style={{ color: "var(--fg)" }}> +{evaluatorList.length - 2} more</span>
1121
+ ) : null}
1122
  </div>
1123
  )}
1124
  <h1
 
1785
  title="Sort by model release date"
1786
  />
1787
  </th>
 
 
 
 
 
 
 
 
 
 
 
1788
  </tr>
1789
  </thead>
1790
  <tbody>
 
1825
  (!Array.isArray(modelResult.source_data) && modelResult.source_data.source_type) ||
1826
  modelResult.source_metadata.source_type ||
1827
  ""
1828
+ // Link the Source value to its evaluator page when the row's
1829
+ // org resolves to a known (de-aliased) evaluator. Try the org
1830
+ // name first, then the displayed source label.
1831
+ const sourceEvaluatorName =
1832
+ resolveEvaluatorName(modelResult.source_metadata.source_organization_name) ??
1833
+ resolveEvaluatorName(modelResult.source_metadata.source_name) ??
1834
+ resolveEvaluatorName(sourceTypeLabel)
1835
  const familyLabel = modelResult.model_info.architecture
1836
  ?? modelResult.model_info.parameter_count
1837
  ?? null
 
1995
 
1996
  <td className="num hidden lg:table-cell align-top">
1997
  {sourceTypeLabel ? (
1998
+ <span className="inline-flex items-center justify-end gap-1">
1999
+ {sourceEvaluatorName ? (
2000
+ <EvaluatorName
2001
+ display={sourceTypeLabel}
2002
+ linkName={sourceEvaluatorName}
2003
+ className="font-mono lowercase"
2004
+ style={{ fontSize: 11, color: "var(--fg-muted)" }}
2005
+ />
2006
+ ) : modelResult.source_metadata.source_url ? (
2007
+ <a
2008
+ href={modelResult.source_metadata.source_url}
2009
+ target="_blank"
2010
+ rel="noreferrer"
2011
+ className="font-mono lowercase hover:text-[color:var(--accent)]"
2012
+ style={{ fontSize: 11, color: "var(--fg-muted)" }}
2013
+ onClick={(e) => e.stopPropagation()}
2014
+ >
2015
+ {sourceTypeLabel}
2016
+ </a>
2017
+ ) : (
2018
+ <span
2019
+ className="font-mono lowercase"
2020
+ style={{ fontSize: 11, color: "var(--fg-muted)" }}
2021
+ >
2022
+ {sourceTypeLabel}
2023
+ </span>
2024
+ )}
2025
+ <VerifiedBadge verified={modelResult.result?.is_verified_evaluator} size="sm" />
2026
+ </span>
2027
  ) : (
2028
  <span style={{ color: "var(--fg-subtle)" }}>—</span>
2029
  )}
 
2038
  : <span style={{ color: "var(--fg-subtle)" }}>—</span>}
2039
  </td>
2040
 
 
 
 
 
 
2041
  </tr>
2042
 
2043
  {isExpanded && (
2044
  <tr>
2045
+ <td colSpan={7} style={{ background: "var(--bg-warm)", padding: 0 }}>
2046
  <div className="space-y-5 px-4 py-5 sm:px-6">
2047
  {/* The Model Profile / Provenance / Score Breakdown
2048
  panels were removed — model metadata lives on
 
2236
  })}
2237
  {leaderboardRows.length === 0 && (
2238
  <tr>
2239
+ <td colSpan={7} style={{ padding: "32px 16px", textAlign: "center", color: "var(--fg-muted)" }}>
2240
  No leaderboard entries match the selected parameter range.
2241
  </td>
2242
  </tr>
 
2295
  const [maxParamStep, setMaxParamStep] = useState(PARAM_RANGE_MAX_INDEX)
2296
  const [expandedRows, setExpandedRows] = useState<Record<string, boolean>>({})
2297
 
2298
+ // Resolve a raw Source-column org name to a known (de-aliased) evaluator name
2299
+ // so the cell can link to /evaluators/<slug>; null → render plain text.
2300
+ const knownEvaluatorByLower = useMemo(() => {
2301
+ const m = new Map<string, string>()
2302
+ for (const n of summary.evaluator_names ?? []) {
2303
+ const t = (n ?? "").trim()
2304
+ if (t) m.set(t.toLowerCase(), t)
2305
+ }
2306
+ return m
2307
+ }, [summary.evaluator_names])
2308
+ const soleEvaluator =
2309
+ knownEvaluatorByLower.size === 1 ? Array.from(knownEvaluatorByLower.values())[0] : null
2310
+ const resolveEvaluatorName = (raw: string | undefined | null): string | null => {
2311
+ const t = (raw ?? "").trim()
2312
+ if (!t) return null
2313
+ return knownEvaluatorByLower.get(t.toLowerCase()) ?? soleEvaluator
2314
+ }
2315
+
2316
  // Index ModelResultForBenchmark entries by model_info.id so we can power the
2317
  // research-mode reproducibility card from a multi-metric row. There may be
2318
  // several entries per model (one per metric); we prefer one with a recorded
 
2371
 
2372
  const hasSliceTabs = sliceTabs.length > 1
2373
 
2374
+ // A `<benchmark-slug>.mean` column whose per-row values are identical to an
2375
+ // earlier column (e.g. cyse2's `cyse2-vulnerability-exploit.mean` mirrors
2376
+ // `accuracy`) is a redundant alias of the primary score, not a distinct
2377
+ // measure. Suppress it so the matrix doesn't render two columns of the same
2378
+ // numbers under a humanised "Mean" header next to the real metric.
2379
+ const duplicateMeanColumnKeys = useMemo(() => {
2380
+ const dupes = new Set<string>()
2381
+ const meanMetrics = leaderboardMetrics.filter(
2382
+ (m) => m.scope !== "subtask" && /\.mean$/i.test(m.column_key),
2383
+ )
2384
+ if (meanMetrics.length === 0) return dupes
2385
+ const others = leaderboardMetrics.filter((m) => m.scope !== "subtask")
2386
+ const valuesEqual = (a: string, b: string) => {
2387
+ let comparable = 0
2388
+ for (const row of leaderboardRows) {
2389
+ const va = row.values[a]
2390
+ const vb = row.values[b]
2391
+ const aNum = isNumericScore(va)
2392
+ const bNum = isNumericScore(vb)
2393
+ if (aNum !== bNum) return false
2394
+ if (aNum && bNum && Math.abs(va - vb) > 1e-6) return false
2395
+ if (aNum && bNum) comparable += 1
2396
+ }
2397
+ return comparable > 0
2398
+ }
2399
+ for (const mean of meanMetrics) {
2400
+ const twin = others.find(
2401
+ (o) => o.column_key !== mean.column_key && valuesEqual(mean.column_key, o.column_key),
2402
+ )
2403
+ if (twin) dupes.add(mean.column_key)
2404
+ }
2405
+ return dupes
2406
+ }, [leaderboardMetrics, leaderboardRows])
2407
+
2408
  const visibleMetrics = useMemo(
2409
  () =>
2410
  leaderboardMetrics.filter((metric) => {
 
2412
  return false
2413
  }
2414
 
2415
+ if (duplicateMeanColumnKeys.has(metric.column_key)) {
2416
+ return false
2417
+ }
2418
+
2419
  if (!hasSliceTabs || activeSliceTab === "all") {
2420
  // "All" / no-slice-filter case: only show root metrics so the
2421
  // chips stay one-per-metric instead of one-per-(metric, slice).
 
2424
 
2425
  return metric.scope === "subtask" && metric.subtask_key === activeSliceTab
2426
  }),
2427
+ [activeSliceTab, duplicateMeanColumnKeys, hasSliceTabs, leaderboardMetrics, visibleMetricKeySet]
2428
  )
2429
  const visibleMetricColumnKeySet = useMemo(
2430
  () => new Set(visibleMetrics.map((metric) => metric.column_key)),
 
2841
  // name is already shown above the table — no need to
2842
  // repeat it as a per-column topline.
2843
  const showSliceTopline = false
2844
+ const mainLabel = getMetricChipLabel(metric)
2845
  return (
2846
  <th
2847
  key={metric.column_key}
 
2868
  </th>
2869
  )
2870
  })}
2871
+ <th className="hidden lg:table-cell" style={{ width: 110 }}>Evaluator</th>
2872
+ <th className="num hidden lg:table-cell" style={{ width: 100 }}>Source</th>
2873
  <th
2874
  className="num hidden lg:table-cell"
2875
  style={{ width: 110, cursor: "pointer" }}
 
2878
  >
2879
  Released{getSortIndicator("released")}
2880
  </th>
 
 
 
 
 
 
 
2881
  </tr>
2882
  </thead>
2883
  <tbody>
 
2971
  )
2972
  })}
2973
 
2974
+ <td className="hidden lg:table-cell align-top">
2975
+ <span className="font-mono uppercase" style={{ fontSize: 11, color: "var(--fg-muted)" }}>
2976
+ {row.source_metadata?.evaluator_relationship === "first_party"
2977
+ ? "SELF"
2978
+ : row.source_metadata?.evaluator_relationship === "third_party"
2979
+ ? "THIRD-PARTY"
2980
+ : "—"}
2981
+ </span>
2982
+ </td>
2983
+ <td className="num hidden lg:table-cell align-top">
2984
+ {(() => {
2985
+ const sourceLabel =
2986
+ row.source_metadata?.source_name?.trim()
2987
+ || row.source_metadata?.source_organization_name?.trim()
2988
+ const sourceEvaluatorName =
2989
+ resolveEvaluatorName(row.source_metadata?.source_organization_name)
2990
+ ?? resolveEvaluatorName(row.source_metadata?.source_name)
2991
+ return sourceLabel ? (
2992
+ <span className="inline-flex items-center justify-end gap-1">
2993
+ <EvaluatorName
2994
+ display={sourceLabel}
2995
+ linkName={sourceEvaluatorName}
2996
+ className="font-mono lowercase"
2997
+ style={{ fontSize: 11, color: "var(--fg-muted)" }}
2998
+ />
2999
+ <VerifiedBadge
3000
+ verified={Object.values(row.verified ?? {}).some(Boolean)}
3001
+ size="sm"
3002
+ />
3003
+ </span>
3004
+ ) : (
3005
+ <span style={{ color: "var(--fg-subtle)" }}>—</span>
3006
+ )
3007
+ })()}
3008
+ </td>
3009
+
3010
  <td
3011
  className="num hidden lg:table-cell align-top font-mono tabular-nums"
3012
  style={{ fontSize: 11, color: "var(--fg-muted)" }}
 
3015
  ? formatDate(row.model_info.release_date).split(",")[0]
3016
  : <span style={{ color: "var(--fg-subtle)" }}>—</span>}
3017
  </td>
 
 
 
3018
  </tr>
3019
  {isResearchView && isExpanded && matchingResult && (
3020
  <tr>
3021
  <td
3022
+ colSpan={visibleMetrics.length + 7}
3023
  style={{ background: "var(--bg-warm)", padding: "20px 24px" }}
3024
  >
3025
  <div className="space-y-3">
 
3049
 
3050
  {filteredRows.length === 0 && (
3051
  <tr>
3052
+ <td colSpan={visibleMetrics.length + 6} style={{ padding: "32px 16px", textAlign: "center", color: "var(--fg-muted)" }}>
3053
  No models match the selected parameter range.
3054
  </td>
3055
  </tr>
components/evaluator-table.tsx ADDED
@@ -0,0 +1,115 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ "use client"
2
+
3
+ import Link from "next/link"
4
+ import { ArrowUpRight, ChevronDown, ChevronUp, ChevronsUpDown } from "lucide-react"
5
+
6
+ import { VerifiedBadge } from "@/components/signals/verified-badge"
7
+ import type { EvaluatorGroup } from "@/lib/evaluators"
8
+ import { cn } from "@/lib/utils"
9
+
10
+ export type EvaluatorTableSortCol = "name" | "evals" | "verified"
11
+
12
+ interface EvaluatorTableProps {
13
+ rows: EvaluatorGroup[]
14
+ sortCol: EvaluatorTableSortCol
15
+ sortDir: "asc" | "desc"
16
+ onSort: (col: EvaluatorTableSortCol) => void
17
+ /** When true, the verified filter is active — propagate it into the link. */
18
+ verifiedOnly?: boolean
19
+ }
20
+
21
+ export function EvaluatorTable({ rows, sortCol, sortDir, onSort, verifiedOnly }: EvaluatorTableProps) {
22
+ function SortIcon({ col }: { col: EvaluatorTableSortCol }) {
23
+ if (sortCol !== col) return <ChevronsUpDown className="h-3 w-3 opacity-30" aria-hidden />
24
+ return sortDir === "asc"
25
+ ? <ChevronUp className="h-3 w-3" aria-hidden />
26
+ : <ChevronDown className="h-3 w-3" aria-hidden />
27
+ }
28
+
29
+ function SortTh({
30
+ col,
31
+ children,
32
+ className,
33
+ style,
34
+ }: {
35
+ col: EvaluatorTableSortCol
36
+ children: React.ReactNode
37
+ className?: string
38
+ style?: React.CSSProperties
39
+ }) {
40
+ const active = sortCol === col
41
+ return (
42
+ <th
43
+ className={className}
44
+ style={{
45
+ ...style,
46
+ cursor: "pointer",
47
+ userSelect: "none",
48
+ color: active ? "var(--fg)" : undefined,
49
+ }}
50
+ onClick={() => onSort(col)}
51
+ >
52
+ <span className={cn("inline-flex items-center gap-1", className?.includes("num") && "justify-end")}>
53
+ {children}
54
+ <SortIcon col={col} />
55
+ </span>
56
+ </th>
57
+ )
58
+ }
59
+
60
+ const hrefFor = (slug: string) =>
61
+ verifiedOnly ? `/evaluators/${slug}?verified=1` : `/evaluators/${slug}`
62
+
63
+ return (
64
+ <div className="overflow-x-auto">
65
+ <table className="ec-htable">
66
+ <thead>
67
+ <tr>
68
+ <SortTh col="name" style={{ width: "55%" }}>Evaluator</SortTh>
69
+ <SortTh col="evals" className="num">Evaluations reported</SortTh>
70
+ <SortTh col="verified" className="num">Verified</SortTh>
71
+ <th style={{ width: 90 }} />
72
+ </tr>
73
+ </thead>
74
+ <tbody>
75
+ {rows.map((row) => (
76
+ <tr key={row.slug}>
77
+ <td>
78
+ <Link href={hrefFor(row.slug)} className="block min-w-0 group">
79
+ <div className="flex items-center gap-1.5">
80
+ <span className="font-semibold text-[14px] text-[color:var(--fg)] group-hover:text-[color:var(--accent)] transition-colors">
81
+ {row.name}
82
+ </span>
83
+ {row.isVerified && <VerifiedBadge verified size="sm" />}
84
+ </div>
85
+ </Link>
86
+ </td>
87
+ <td className="num font-mono text-[13px]">
88
+ {row.evalCount.toLocaleString()}
89
+ </td>
90
+ <td className="num font-mono text-[13px]">
91
+ {row.verifiedCount > 0 ? (
92
+ <span className="inline-flex items-center gap-1 text-[color:var(--accent)]">
93
+ {row.verifiedCount.toLocaleString()}
94
+ <VerifiedBadge verified size="sm" withTooltip={false} />
95
+ </span>
96
+ ) : (
97
+ <span className="text-[color:var(--fg-subtle)]">—</span>
98
+ )}
99
+ </td>
100
+ <td>
101
+ <Link
102
+ href={hrefFor(row.slug)}
103
+ className="font-mono text-[10px] tracking-[0.12em] uppercase text-[color:var(--accent)] hover:text-[color:var(--accent-hover)] inline-flex items-center gap-1"
104
+ >
105
+ Open
106
+ <ArrowUpRight className="h-3 w-3" aria-hidden />
107
+ </Link>
108
+ </td>
109
+ </tr>
110
+ ))}
111
+ </tbody>
112
+ </table>
113
+ </div>
114
+ )
115
+ }
components/family-table.tsx CHANGED
@@ -24,6 +24,11 @@ interface FamilyTableProps {
24
  * decided which families pass; FamilyTable uses the same query to
25
  * narrow each row's visible leaves to the matches and auto-expand. */
26
  searchQuery?: string
 
 
 
 
 
27
  sortCol?: FamilySortCol
28
  sortDir?: "asc" | "desc"
29
  onSort?: (col: FamilySortCol) => void
@@ -52,6 +57,8 @@ function humanizeFamilyKey(key: string): string {
52
 
53
  interface LeafEntry {
54
  id: string
 
 
55
  leafKey: string
56
  leafName: string
57
  evalsCount: number
@@ -98,6 +105,7 @@ function buildLeafEntry(
98
 
99
  return {
100
  id: ids[0],
 
101
  leafKey: benchmark.key,
102
  leafName: benchmark.display_name || benchmark.key,
103
  evalsCount: ids.length,
@@ -205,6 +213,7 @@ export function FamilyTable({
205
  domainFilter,
206
  categoryFilter,
207
  searchQuery,
 
208
  sortCol,
209
  sortDir,
210
  onSort,
@@ -214,9 +223,10 @@ export function FamilyTable({
214
 
215
  const domainFilterActive = Boolean(domainFilter && domainFilter.size > 0)
216
  const categoryFilterActive = Boolean(categoryFilter && categoryFilter.size > 0)
 
217
  const normalizedQuery = (searchQuery ?? "").trim().toLowerCase()
218
  const searchActive = normalizedQuery.length > 0
219
- const filterActive = domainFilterActive || categoryFilterActive || searchActive
220
 
221
  function leafMatchesDomain(leaf: LeafEntry): boolean {
222
  if (!domainFilterActive || !domainFilter) return true
@@ -237,9 +247,15 @@ export function FamilyTable({
237
  return false
238
  }
239
 
 
 
 
 
 
240
  function leafMatchesFilter(leaf: LeafEntry, opts?: { skipQuery?: boolean }): boolean {
241
  if (!leafMatchesDomain(leaf)) return false
242
  if (!leafMatchesCategory(leaf)) return false
 
243
  if (!opts?.skipQuery && !leafMatchesQuery(leaf)) return false
244
  return true
245
  }
@@ -263,8 +279,14 @@ export function FamilyTable({
263
  if (leafEntries.some((leaf) => leafMatchesFilter(leaf))) return true
264
  // Family-level search match keeps the row even if no leaf survives
265
  // the leaf-query filter (the row will fall back to showing all
266
- // leaves).
267
- if (searchActive && familyMatchedAtFamilyLevel(fam)) return true
 
 
 
 
 
 
268
  if (categoryFilterActive && categoryFilter) {
269
  for (const tag of fam.derivedTags ?? []) {
270
  if (categoryFilter.has(tag) && !domainFilterActive) return true
 
24
  * decided which families pass; FamilyTable uses the same query to
25
  * narrow each row's visible leaves to the matches and auto-expand. */
26
  searchQuery?: string
27
+ /** When provided, restrict leaves to those mapping to one of these
28
+ * evaluation_ids (drives the /evals "Verified only" filter in Family
29
+ * mode — the set is the verified-eval id universe). Null/undefined =
30
+ * no restriction. */
31
+ verifiedEvalIds?: Set<string> | null
32
  sortCol?: FamilySortCol
33
  sortDir?: "asc" | "desc"
34
  onSort?: (col: FamilySortCol) => void
 
57
 
58
  interface LeafEntry {
59
  id: string
60
+ /** All evaluation_ids this leaf maps to (constituent ids or fallbacks). */
61
+ evalIds: string[]
62
  leafKey: string
63
  leafName: string
64
  evalsCount: number
 
105
 
106
  return {
107
  id: ids[0],
108
+ evalIds: ids,
109
  leafKey: benchmark.key,
110
  leafName: benchmark.display_name || benchmark.key,
111
  evalsCount: ids.length,
 
213
  domainFilter,
214
  categoryFilter,
215
  searchQuery,
216
+ verifiedEvalIds,
217
  sortCol,
218
  sortDir,
219
  onSort,
 
223
 
224
  const domainFilterActive = Boolean(domainFilter && domainFilter.size > 0)
225
  const categoryFilterActive = Boolean(categoryFilter && categoryFilter.size > 0)
226
+ const verifiedFilterActive = Boolean(verifiedEvalIds)
227
  const normalizedQuery = (searchQuery ?? "").trim().toLowerCase()
228
  const searchActive = normalizedQuery.length > 0
229
+ const filterActive = domainFilterActive || categoryFilterActive || searchActive || verifiedFilterActive
230
 
231
  function leafMatchesDomain(leaf: LeafEntry): boolean {
232
  if (!domainFilterActive || !domainFilter) return true
 
247
  return false
248
  }
249
 
250
+ function leafMatchesVerified(leaf: LeafEntry): boolean {
251
+ if (!verifiedFilterActive || !verifiedEvalIds) return true
252
+ return leaf.evalIds.some((id) => verifiedEvalIds.has(id))
253
+ }
254
+
255
  function leafMatchesFilter(leaf: LeafEntry, opts?: { skipQuery?: boolean }): boolean {
256
  if (!leafMatchesDomain(leaf)) return false
257
  if (!leafMatchesCategory(leaf)) return false
258
+ if (!leafMatchesVerified(leaf)) return false
259
  if (!opts?.skipQuery && !leafMatchesQuery(leaf)) return false
260
  return true
261
  }
 
279
  if (leafEntries.some((leaf) => leafMatchesFilter(leaf))) return true
280
  // Family-level search match keeps the row even if no leaf survives
281
  // the leaf-query filter (the row will fall back to showing all
282
+ // leaves). But the verified filter is a hard leaf-level gate: never
283
+ // resurrect a family that has zero verified leaves.
284
+ if (
285
+ searchActive &&
286
+ familyMatchedAtFamilyLevel(fam) &&
287
+ leafEntries.some((leaf) => leafMatchesVerified(leaf))
288
+ )
289
+ return true
290
  if (categoryFilterActive && categoryFilter) {
291
  for (const tag of fam.derivedTags ?? []) {
292
  if (categoryFilter.has(tag) && !domainFilterActive) return true
components/signals/signal-tooltip.tsx CHANGED
@@ -3,12 +3,17 @@
3
  import type { ReactNode } from "react"
4
  import * as TooltipPrimitive from "@radix-ui/react-tooltip"
5
 
 
 
6
  export function SignalTooltip({
7
  children,
8
  content,
 
9
  }: {
10
  children: ReactNode
11
  content: ReactNode
 
 
12
  }) {
13
  return (
14
  <TooltipPrimitive.Provider delayDuration={150}>
@@ -19,7 +24,10 @@ export function SignalTooltip({
19
  side="top"
20
  align="center"
21
  sideOffset={8}
22
- className="z-50 max-w-80 rounded-md border border-border/70 bg-popover px-3 py-2 text-xs leading-5 text-popover-foreground shadow-lg"
 
 
 
23
  >
24
  {content}
25
  <TooltipPrimitive.Arrow className="fill-popover" />
 
3
  import type { ReactNode } from "react"
4
  import * as TooltipPrimitive from "@radix-ui/react-tooltip"
5
 
6
+ import { cn } from "@/lib/utils"
7
+
8
  export function SignalTooltip({
9
  children,
10
  content,
11
+ contentClassName,
12
  }: {
13
  children: ReactNode
14
  content: ReactNode
15
+ /** Per-instance overrides for the tooltip box (e.g. tighter one-line copy). */
16
+ contentClassName?: string
17
  }) {
18
  return (
19
  <TooltipPrimitive.Provider delayDuration={150}>
 
24
  side="top"
25
  align="center"
26
  sideOffset={8}
27
+ className={cn(
28
+ "z-50 max-w-80 rounded-md border border-border/70 bg-popover px-3 py-2 text-xs leading-5 text-popover-foreground shadow-lg",
29
+ contentClassName,
30
+ )}
31
  >
32
  {content}
33
  <TooltipPrimitive.Arrow className="fill-popover" />
components/signals/verified-badge.tsx ADDED
@@ -0,0 +1,245 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ "use client"
2
+
3
+ import { Check, CheckCircle2, ShieldCheck } from "lucide-react"
4
+
5
+ import { useAudienceMode } from "@/components/audience-mode-provider"
6
+ import { Badge } from "@/components/ui/badge"
7
+ import { cn } from "@/lib/utils"
8
+ import { SignalTooltip } from "./signal-tooltip"
9
+
10
+ /**
11
+ * VerifiedBadge — a small, simple "this result is verified" marker shown
12
+ * next to an individual evaluation result / metric value.
13
+ *
14
+ * Driven by a single boolean (`is_verified`) the producer emits per atomic
15
+ * result (see docs/verified-badge/README.md). When the flag is false / absent
16
+ * the component renders nothing, so unverified rows stay visually quiet —
17
+ * mirroring the existing row-signal badges (RowSignalsCompact, etc.).
18
+ *
19
+ * Several `design` options are provided so the look can be chosen during
20
+ * review (see /verified-badge-preview). They split into two families:
21
+ *
22
+ * Icon-only (lowest footprint — best inside dense leaderboards/score cells):
23
+ * - "check" (default): clean geometric check-in-circle. Neutral, not
24
+ * social-media-y.
25
+ * - "shield": shield + check — reads as governance / "validated".
26
+ * Matches the ShieldCheck already used for clean row signals.
27
+ * - "glyph": hairline rounded box + check — matches the editorial
28
+ * .sig-glyph signal-letter vocabulary. Most on-brand.
29
+ * - "tick": bare accent check with a hairline underline. Minimal,
30
+ * proofreading-mark feel.
31
+ * - "seal": scalloped lucide BadgeCheck. Closest to a social verified
32
+ * badge — kept for comparison.
33
+ *
34
+ * Labelled (carry the word "Verified"):
35
+ * - "tag": hairline mono-caps outline pill. Editorial label.
36
+ * - "chip": filled accent pill. Highest emphasis; use sparingly.
37
+ */
38
+
39
+ export type VerifiedBadgeDesign =
40
+ | "tile"
41
+ | "check"
42
+ | "shield"
43
+ | "glyph"
44
+ | "tick"
45
+ | "seal"
46
+ | "tag"
47
+ | "chip"
48
+ export type VerifiedBadgeSize = "sm" | "md"
49
+
50
+ const ICON_SIZE: Record<VerifiedBadgeSize, string> = {
51
+ sm: "h-3.5 w-3.5",
52
+ md: "h-4 w-4",
53
+ }
54
+
55
+ const ICON_ONLY: VerifiedBadgeDesign[] = ["tile", "check", "shield", "glyph", "tick", "seal"]
56
+
57
+ export function verifiedTooltipCopy(_mode: "research" | "policy"): string {
58
+ // The badge is a provenance signal, not an accuracy/verification claim: it
59
+ // marks results contributed by the org that ran the evaluation. Same copy in
60
+ // both audience modes; aria-label mirrors it (see VerifiedBadge).
61
+ return "Submitted by the organization that ran this evaluation."
62
+ }
63
+
64
+ function IconOnly({
65
+ design,
66
+ size,
67
+ className,
68
+ }: {
69
+ design: VerifiedBadgeDesign
70
+ size: VerifiedBadgeSize
71
+ className?: string
72
+ }) {
73
+ const sz = ICON_SIZE[size]
74
+ const base = "inline-flex shrink-0 items-center justify-center align-middle text-[var(--accent)]"
75
+
76
+ switch (design) {
77
+ case "tile":
78
+ // Filled accent rounded-square + white check — echoes the EvalEval
79
+ // brand mark (rounded-square accent tile), so it reads as "our"
80
+ // verification rather than a generic/social one. Square, not circular.
81
+ return (
82
+ <span
83
+ className={cn(
84
+ "inline-flex shrink-0 items-center justify-center rounded-[3px] bg-[var(--accent)] align-middle text-[var(--accent-fg)]",
85
+ size === "md" ? "h-[18px] w-[18px]" : "h-4 w-4",
86
+ className
87
+ )}
88
+ >
89
+ <Check className={size === "md" ? "h-3 w-3" : "h-2.5 w-2.5"} strokeWidth={3.5} />
90
+ </span>
91
+ )
92
+ case "check":
93
+ // Clean geometric check-in-circle. Geometric, not scalloped.
94
+ return (
95
+ <span className={cn(base, className)}>
96
+ <CheckCircle2 className={sz} strokeWidth={2.25} />
97
+ </span>
98
+ )
99
+ case "shield":
100
+ // Governance / "validated" read.
101
+ return (
102
+ <span className={cn(base, className)}>
103
+ <ShieldCheck className={sz} strokeWidth={2} />
104
+ </span>
105
+ )
106
+ case "glyph":
107
+ // Hairline rounded box + check — matches the .sig-glyph editorial
108
+ // signal vocabulary (bordered mark, sharp corners).
109
+ return (
110
+ <span
111
+ className={cn(
112
+ "inline-flex shrink-0 items-center justify-center rounded-[3px] border border-[var(--accent)]/55 align-middle text-[var(--accent)]",
113
+ size === "md" ? "h-[18px] w-[18px]" : "h-4 w-4",
114
+ className
115
+ )}
116
+ >
117
+ <Check className={size === "md" ? "h-3 w-3" : "h-2.5 w-2.5"} strokeWidth={3} />
118
+ </span>
119
+ )
120
+ case "tick":
121
+ // Bare accent check with a hairline underline — minimal proofreading mark.
122
+ return (
123
+ <span
124
+ className={cn(
125
+ "inline-flex shrink-0 items-center align-middle text-[var(--accent)] border-b border-[var(--accent)]/40 leading-none",
126
+ className
127
+ )}
128
+ >
129
+ <Check className={sz} strokeWidth={3} />
130
+ </span>
131
+ )
132
+ case "seal":
133
+ default:
134
+ // Original scalloped social-style mark, kept for comparison.
135
+ return (
136
+ <span className={cn(base, className)} role="img">
137
+ <BadgeCheckScalloped className={sz} />
138
+ </span>
139
+ )
140
+ }
141
+ }
142
+
143
+ // Inline copy of lucide's BadgeCheck so "seal" stays available without adding
144
+ // it to the active icon imports above.
145
+ function BadgeCheckScalloped({ className }: { className?: string }) {
146
+ return (
147
+ <svg
148
+ className={className}
149
+ viewBox="0 0 24 24"
150
+ fill="none"
151
+ stroke="currentColor"
152
+ strokeWidth={2}
153
+ strokeLinecap="round"
154
+ strokeLinejoin="round"
155
+ aria-hidden
156
+ >
157
+ <path d="M3.85 8.62a4 4 0 0 1 4.78-4.77 4 4 0 0 1 6.74 0 4 4 0 0 1 4.78 4.78 4 4 0 0 1 0 6.74 4 4 0 0 1-4.77 4.78 4 4 0 0 1-6.75 0 4 4 0 0 1-4.78-4.77 4 4 0 0 1 0-6.76Z" />
158
+ <path d="m9 12 2 2 4-4" />
159
+ </svg>
160
+ )
161
+ }
162
+
163
+ export function VerifiedBadge({
164
+ verified,
165
+ design = "check",
166
+ size = "sm",
167
+ withTooltip = true,
168
+ label = "Verified",
169
+ className,
170
+ }: {
171
+ /** The per-result `is_verified` boolean. Null/false/undefined → renders nothing. */
172
+ verified?: boolean | null
173
+ design?: VerifiedBadgeDesign
174
+ size?: VerifiedBadgeSize
175
+ withTooltip?: boolean
176
+ /** Visible text for the "tag" / "chip" designs. */
177
+ label?: string
178
+ className?: string
179
+ }) {
180
+ // Hooks must run unconditionally; guard on the value afterwards.
181
+ const { mode } = useAudienceMode()
182
+
183
+ if (!verified) {
184
+ return null
185
+ }
186
+
187
+ const tooltip = verifiedTooltipCopy(mode)
188
+ // Screen-reader text mirrors the tooltip so the two never drift. `label`
189
+ // ("Verified evaluator") only surfaces visually on the tag/chip designs.
190
+ const ariaLabel = tooltip
191
+
192
+ let node: React.ReactNode
193
+
194
+ if (ICON_ONLY.includes(design)) {
195
+ node = (
196
+ <span aria-label={ariaLabel} role="img" className="inline-flex">
197
+ <IconOnly design={design} size={size} className={className} />
198
+ </span>
199
+ )
200
+ } else if (design === "tag") {
201
+ // Hairline mono-caps outline pill — matches the .ec-tag editorial
202
+ // vocabulary (sharp corners, IBM Plex Mono, wide tracking).
203
+ node = (
204
+ <span
205
+ className={cn(
206
+ "inline-flex shrink-0 items-center gap-1 rounded-[2px] border px-1.5 py-0.5 align-middle",
207
+ "border-[var(--accent)]/40 text-[var(--accent)]",
208
+ "font-mono text-[10px] font-semibold uppercase tracking-[0.12em]",
209
+ className
210
+ )}
211
+ aria-label={ariaLabel}
212
+ >
213
+ <Check className={size === "md" ? "h-3.5 w-3.5" : "h-3 w-3"} strokeWidth={3} />
214
+ {label}
215
+ </span>
216
+ )
217
+ } else {
218
+ // "chip" — filled accent pill, highest-emphasis.
219
+ node = (
220
+ <Badge
221
+ className={cn(
222
+ "shrink-0 gap-1 rounded-full border-transparent align-middle",
223
+ "bg-[var(--accent)] text-[var(--accent-fg)] hover:bg-[var(--accent-hover)]",
224
+ className
225
+ )}
226
+ aria-label={ariaLabel}
227
+ >
228
+ <Check className={ICON_SIZE[size]} strokeWidth={3} />
229
+ {label}
230
+ </Badge>
231
+ )
232
+ }
233
+
234
+ if (!withTooltip) {
235
+ return node
236
+ }
237
+
238
+ // Short single-sentence copy → render it on one tight line instead of
239
+ // wrapping inside the default 320px box (which left a lot of empty space).
240
+ return (
241
+ <SignalTooltip content={tooltip} contentClassName="max-w-none whitespace-nowrap px-2.5 py-1.5">
242
+ {node}
243
+ </SignalTooltip>
244
+ )
245
+ }
lib/benchmark-schema.ts CHANGED
@@ -110,6 +110,13 @@ export interface EvaluationResult {
110
  detailed_evaluation_results_url?: string
111
  generation_config?: GenerationConfig
112
  evalcards?: { annotations?: RowAnnotations }
 
 
 
 
 
 
 
113
  }
114
 
115
  export interface MetricConfig {
 
110
  detailed_evaluation_results_url?: string
111
  generation_config?: GenerationConfig
112
  evalcards?: { annotations?: RowAnnotations }
113
+ /** Per-result verification flag emitted by the producer's
114
+ * `eval_results_view.is_verified_evaluator` column (one boolean per
115
+ * (model, benchmark, metric) triple). When true the UI renders a
116
+ * small VerifiedBadge next to the metric value. Nullable/absent for
117
+ * snapshots produced before the column shipped — treated as
118
+ * unverified. */
119
+ is_verified_evaluator?: boolean
120
  }
121
 
122
  export interface MetricConfig {
lib/eval-processing.ts CHANGED
@@ -36,6 +36,8 @@ export interface ModelResultForBenchmark {
36
  evaluation_timestamp: string
37
  source_metadata: SourceMetadata
38
  source_data: BenchmarkEvaluation['source_data']
 
 
39
  result: EvaluationResult
40
  /** URL to the underlying record JSON in the upstream HF dataset, when known. */
41
  source_record_url?: string
@@ -68,6 +70,8 @@ export interface BenchmarkEvalSummary extends SignalSummaries {
68
  models_count: number
69
  /** Unique evaluator organisation names */
70
  evaluator_names: string[]
 
 
71
  source_types: SourceMetadata["source_type"][]
72
  latest_source_name?: string
73
  third_party_ratio: number
@@ -177,6 +181,8 @@ export interface BenchmarkLeaderboardRow {
177
  source_metadata: SourceMetadata
178
  source_data: BenchmarkEvaluation["source_data"]
179
  values: Record<string, number | null>
 
 
180
  annotations_by_metric?: Record<string, RowAnnotations | null | undefined>
181
  metrics_present: number
182
  }
 
36
  evaluation_timestamp: string
37
  source_metadata: SourceMetadata
38
  source_data: BenchmarkEvaluation['source_data']
39
+ /** Per-result verification flag; mirrors `result.is_verified_evaluator`. */
40
+ is_verified_evaluator?: boolean
41
  result: EvaluationResult
42
  /** URL to the underlying record JSON in the upstream HF dataset, when known. */
43
  source_record_url?: string
 
70
  models_count: number
71
  /** Unique evaluator organisation names */
72
  evaluator_names: string[]
73
+ /** Subset of `evaluator_names` that are validated submitters (badge). */
74
+ verified_evaluator_names?: string[]
75
  source_types: SourceMetadata["source_type"][]
76
  latest_source_name?: string
77
  third_party_ratio: number
 
181
  source_metadata: SourceMetadata
182
  source_data: BenchmarkEvaluation["source_data"]
183
  values: Record<string, number | null>
184
+ /** Per-column verified-evaluator flag, keyed identically to `values`. */
185
+ verified?: Record<string, boolean>
186
  annotations_by_metric?: Record<string, RowAnnotations | null | undefined>
187
  metrics_present: number
188
  }
lib/evaluators.ts ADDED
@@ -0,0 +1,199 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /**
2
+ * Evaluator (reporting-org) grouping + slug helpers for the /evals
3
+ * "group by Evaluator" view and the /evaluators/<slug> detail route.
4
+ *
5
+ * Frontend-only: every function operates on the already-loaded eval list
6
+ * (`BenchmarkEvalListItem[]` from /api/eval-list-lite). An eval carries
7
+ * `evaluator_names` (de-aliased reporting orgs) and a `verified_evaluator_names`
8
+ * subset (validated submitters). A single eval may name multiple evaluators.
9
+ *
10
+ * Membership rule:
11
+ * - normal: (eval, org) counts when org ∈ eval.evaluator_names
12
+ * - verified-only: (eval, org) counts when org ∈ eval.verified_evaluator_names
13
+ *
14
+ * Slugs are derived deterministically from the full org universe so the
15
+ * list page and the detail route agree on slug→org without a server round
16
+ * trip. Junk org strings (a long university list, "CapArena (Cheng et al.,
17
+ * 2025)") still produce a valid, unique, non-empty slug.
18
+ */
19
+
20
+ import type { BenchmarkEvalListItem } from "@/lib/eval-processing"
21
+
22
+ export interface EvaluatorGroup {
23
+ /** Canonical org name as it appears in `evaluator_names`. */
24
+ name: string
25
+ /** URL-safe slug for /evaluators/<slug>. Unique across the corpus. */
26
+ slug: string
27
+ /** Number of evals this org reported (respecting the verified filter). */
28
+ evalCount: number
29
+ /** Number of those evals where this org is a *verified* evaluator. */
30
+ verifiedCount: number
31
+ /** True when the org is a verified evaluator for ≥1 eval (regardless of filter). */
32
+ isVerified: boolean
33
+ }
34
+
35
+ /**
36
+ * Base slug from an arbitrary org string: lowercase, non-alphanumerics →
37
+ * single hyphen, trimmed. Falls back to "evaluator" when the string has no
38
+ * usable characters so we never emit an empty path segment. Length-capped
39
+ * so a pathological org (e.g. a multi-institution author list) doesn't
40
+ * produce an absurd URL — uniqueness is restored by the suffix step.
41
+ */
42
+ export function evaluatorSlug(name: string): string {
43
+ const slug = (name ?? "")
44
+ .toLowerCase()
45
+ .replace(/[^a-z0-9]+/g, "-")
46
+ .replace(/^-+|-+$/g, "")
47
+ .slice(0, 64)
48
+ .replace(/-+$/g, "")
49
+ return slug || "evaluator"
50
+ }
51
+
52
+ /** @deprecated internal alias — use {@link evaluatorSlug}. */
53
+ const baseSlug = evaluatorSlug
54
+
55
+ /**
56
+ * Build a deterministic, collision-free slug↔org map over every org that
57
+ * appears in any eval's `evaluator_names`. Orgs are processed in a stable
58
+ * order (name-sorted) so the same corpus always yields the same slugs;
59
+ * collisions get a numeric suffix (`-2`, `-3`, …).
60
+ */
61
+ export function buildEvaluatorSlugMap(evals: BenchmarkEvalListItem[]): {
62
+ slugToName: Map<string, string>
63
+ nameToSlug: Map<string, string>
64
+ } {
65
+ const names = new Set<string>()
66
+ for (const ev of evals) {
67
+ for (const org of ev.evaluator_names ?? []) {
68
+ const trimmed = (org ?? "").trim()
69
+ if (trimmed) names.add(trimmed)
70
+ }
71
+ }
72
+
73
+ const slugToName = new Map<string, string>()
74
+ const nameToSlug = new Map<string, string>()
75
+ const used = new Map<string, number>()
76
+
77
+ for (const name of Array.from(names).sort((a, b) => a.localeCompare(b))) {
78
+ const base = baseSlug(name)
79
+ const seen = used.get(base) ?? 0
80
+ const slug = seen === 0 ? base : `${base}-${seen + 1}`
81
+ used.set(base, seen + 1)
82
+ slugToName.set(slug, name)
83
+ nameToSlug.set(name, slug)
84
+ }
85
+
86
+ return { slugToName, nameToSlug }
87
+ }
88
+
89
+ /**
90
+ * Group the eval list by evaluator org. When `verifiedOnly` is true, an
91
+ * (eval, org) membership only counts where org ∈ verified_evaluator_names,
92
+ * so the result collapses to the genuine validated runners. Returns groups
93
+ * sorted by eval count descending (name as the tie-break).
94
+ */
95
+ export function groupEvalsByEvaluator(
96
+ evals: BenchmarkEvalListItem[],
97
+ opts?: { verifiedOnly?: boolean },
98
+ ): EvaluatorGroup[] {
99
+ const verifiedOnly = opts?.verifiedOnly ?? false
100
+ const { nameToSlug } = buildEvaluatorSlugMap(evals)
101
+
102
+ const acc = new Map<string, { evalCount: number; verifiedCount: number; isVerified: boolean }>()
103
+
104
+ for (const ev of evals) {
105
+ // Slices (is_slice=true) are within-benchmark child rows; the evaluator
106
+ // surfaces count/show only root benchmarks. The /evals family-tree view
107
+ // nests slices under their parent, so excluding them here keeps the flat
108
+ // evaluator list to root benchmarks.
109
+ if (ev.is_slice) continue
110
+ const orgs = ev.evaluator_names ?? []
111
+ const verifiedSet = new Set(ev.verified_evaluator_names ?? [])
112
+ for (const raw of orgs) {
113
+ const org = (raw ?? "").trim()
114
+ if (!org) continue
115
+ const isVerifiedHere = verifiedSet.has(org)
116
+ // In verified-only mode, the membership itself only counts when the
117
+ // org is verified *for this eval*.
118
+ if (verifiedOnly && !isVerifiedHere) continue
119
+ const cur = acc.get(org) ?? { evalCount: 0, verifiedCount: 0, isVerified: false }
120
+ cur.evalCount += 1
121
+ if (isVerifiedHere) {
122
+ cur.verifiedCount += 1
123
+ cur.isVerified = true
124
+ }
125
+ acc.set(org, cur)
126
+ }
127
+ }
128
+
129
+ const groups: EvaluatorGroup[] = []
130
+ for (const [name, v] of acc) {
131
+ groups.push({
132
+ name,
133
+ slug: nameToSlug.get(name) ?? baseSlug(name),
134
+ evalCount: v.evalCount,
135
+ verifiedCount: v.verifiedCount,
136
+ isVerified: v.isVerified,
137
+ })
138
+ }
139
+
140
+ groups.sort((a, b) => b.evalCount - a.evalCount || a.name.localeCompare(b.name))
141
+ return groups
142
+ }
143
+
144
+ /**
145
+ * Evals reported by a single evaluator org (resolved from its slug). When
146
+ * `verifiedOnly` is true, restrict to evals where the org is a verified
147
+ * evaluator. Returns `{ name, evals }`; `name` is null when the slug
148
+ * resolves to no org (404 on the detail page).
149
+ */
150
+ export function getEvalsForEvaluator(
151
+ allEvals: BenchmarkEvalListItem[],
152
+ slug: string,
153
+ opts?: { verifiedOnly?: boolean },
154
+ ): { name: string | null; isVerified: boolean; evals: BenchmarkEvalListItem[] } {
155
+ const verifiedOnly = opts?.verifiedOnly ?? false
156
+ const { slugToName } = buildEvaluatorSlugMap(allEvals)
157
+ // Direct hit on the (possibly collision-suffixed) full slug, else fall back
158
+ // to base-slug matching so links built from the shared `evaluatorSlug(name)`
159
+ // helper — which can't see the suffix step — still resolve. The fallback is
160
+ // deterministic via the same name-sorted order buildEvaluatorSlugMap uses.
161
+ let name = slugToName.get(slug) ?? null
162
+ if (!name) {
163
+ for (const [, candidate] of Array.from(slugToName.entries()).sort((a, b) =>
164
+ a[1].localeCompare(b[1]),
165
+ )) {
166
+ if (evaluatorSlug(candidate) === slug) {
167
+ name = candidate
168
+ break
169
+ }
170
+ }
171
+ }
172
+ if (!name) return { name: null, isVerified: false, evals: [] }
173
+
174
+ let isVerified = false
175
+ const evals = allEvals.filter((ev) => {
176
+ // Exclude slices — evaluator surfaces show root benchmarks only.
177
+ if (ev.is_slice) return false
178
+ const orgs = ev.evaluator_names ?? []
179
+ if (!orgs.includes(name)) return false
180
+ const verifiedHere = (ev.verified_evaluator_names ?? []).includes(name)
181
+ if (verifiedHere) isVerified = true
182
+ if (verifiedOnly) return verifiedHere
183
+ return true
184
+ })
185
+
186
+ return { name, isVerified, evals }
187
+ }
188
+
189
+ /**
190
+ * Evaluation_ids that have ≥1 verified evaluator — drives the Family-mode
191
+ * "Verified only" filter (restrict the family tree to verified evals).
192
+ */
193
+ export function verifiedEvalIds(evals: BenchmarkEvalListItem[]): Set<string> {
194
+ const out = new Set<string>()
195
+ for (const ev of evals) {
196
+ if ((ev.verified_evaluator_names ?? []).length > 0) out.add(ev.evaluation_id)
197
+ }
198
+ return out
199
+ }
lib/view-data.ts CHANGED
@@ -67,7 +67,7 @@ const EVAL_LIST_COLUMNS = `
67
  family_display_name AS benchmark_family_name,
68
  derived_tags,
69
  CAST(to_json(metric_config) AS VARCHAR) AS metric_config,
70
- models_count, evaluator_names, source_types,
71
  latest_source_name, third_party_ratio,
72
  missing_generation_config_count, best_model, worst_model,
73
  avg_score, avg_score_norm, has_card, CAST(to_json(benchmark_card) AS VARCHAR) AS benchmark_card,
@@ -111,6 +111,7 @@ const MODEL_CELL_JOIN_COLUMNS = `
111
  CAST(to_json(r.eval_library) AS VARCHAR) AS eval_library,
112
  CAST(to_json(r.evalcards_annotations) AS VARCHAR) AS evalcards_annotations,
113
  r.instance_file_path,
 
114
  e.evaluation_name AS eval_evaluation_name,
115
  e.canonical_display_name AS eval_canonical_display_name,
116
  e.family_id AS eval_family_id,
@@ -148,6 +149,7 @@ const EVAL_CELL_JOIN_COLUMNS = `
148
  CAST(to_json(r.aggregate_components) AS VARCHAR) AS aggregate_components,
149
  CAST(to_json(r.evalcards_annotations) AS VARCHAR) AS evalcards_annotations,
150
  r.instance_file_path,
 
151
  e.evaluation_name AS eval_evaluation_name,
152
  e.canonical_display_name AS eval_canonical_display_name,
153
  e.family_id AS eval_family_id,
@@ -539,6 +541,11 @@ function resultFromCell(row: Row): EvaluationResult {
539
  generation_config: generationConfig,
540
  detailed_evaluation_results_url: optionalString(row.instance_file_path),
541
  evalcards: annotations ? { annotations } : undefined,
 
 
 
 
 
542
  }
543
  }
544
 
@@ -822,7 +829,7 @@ export async function getModelSummaryById(routeId: string): Promise<ModelEvaluat
822
  // nobody ran `pnpm build-eval-matrices` yet), we fall through and the
823
  // summary degrades to single-metric exactly like before.
824
  type MatrixEntry = {
825
- leaderboard_rows: Array<{ model_route_id: string; values: Record<string, number | null> }>
826
  subtask_metrics: Array<Record<string, unknown>>
827
  }
828
 
@@ -928,6 +935,7 @@ export async function getEvalSummaryById(evalId: string): Promise<BenchmarkEvalS
928
  source_metadata: base.source_metadata,
929
  source_data: base.source_data,
930
  values: row.values,
 
931
  metrics_present: Object.values(row.values).filter(
932
  (v): v is number => typeof v === "number" && Number.isFinite(v),
933
  ).length,
@@ -980,6 +988,9 @@ export async function getEvalSummaryById(evalId: string): Promise<BenchmarkEvalS
980
  source_metadata: mr.source_metadata,
981
  source_data: mr.source_data,
982
  values: { [columnKey]: mr.score as number },
 
 
 
983
  metrics_present: 1,
984
  })) as BenchmarkEvalSummary["leaderboard_rows"]
985
  }
 
67
  family_display_name AS benchmark_family_name,
68
  derived_tags,
69
  CAST(to_json(metric_config) AS VARCHAR) AS metric_config,
70
+ models_count, evaluator_names, verified_evaluator_names, source_types,
71
  latest_source_name, third_party_ratio,
72
  missing_generation_config_count, best_model, worst_model,
73
  avg_score, avg_score_norm, has_card, CAST(to_json(benchmark_card) AS VARCHAR) AS benchmark_card,
 
111
  CAST(to_json(r.eval_library) AS VARCHAR) AS eval_library,
112
  CAST(to_json(r.evalcards_annotations) AS VARCHAR) AS evalcards_annotations,
113
  r.instance_file_path,
114
+ r.is_verified_evaluator,
115
  e.evaluation_name AS eval_evaluation_name,
116
  e.canonical_display_name AS eval_canonical_display_name,
117
  e.family_id AS eval_family_id,
 
149
  CAST(to_json(r.aggregate_components) AS VARCHAR) AS aggregate_components,
150
  CAST(to_json(r.evalcards_annotations) AS VARCHAR) AS evalcards_annotations,
151
  r.instance_file_path,
152
+ r.is_verified_evaluator,
153
  e.evaluation_name AS eval_evaluation_name,
154
  e.canonical_display_name AS eval_canonical_display_name,
155
  e.family_id AS eval_family_id,
 
541
  generation_config: generationConfig,
542
  detailed_evaluation_results_url: optionalString(row.instance_file_path),
543
  evalcards: annotations ? { annotations } : undefined,
544
+ // Per-result verification flag (eval_results_view.is_verified_evaluator).
545
+ // Coerced to a strict boolean; absent on pre-rollout snapshots →
546
+ // undefined → treated as unverified by the UI.
547
+ is_verified_evaluator:
548
+ row.is_verified_evaluator == null ? undefined : Boolean(row.is_verified_evaluator),
549
  }
550
  }
551
 
 
829
  // nobody ran `pnpm build-eval-matrices` yet), we fall through and the
830
  // summary degrades to single-metric exactly like before.
831
  type MatrixEntry = {
832
+ leaderboard_rows: Array<{ model_route_id: string; values: Record<string, number | null>; verified?: Record<string, boolean> }>
833
  subtask_metrics: Array<Record<string, unknown>>
834
  }
835
 
 
935
  source_metadata: base.source_metadata,
936
  source_data: base.source_data,
937
  values: row.values,
938
+ verified: row.verified,
939
  metrics_present: Object.values(row.values).filter(
940
  (v): v is number => typeof v === "number" && Number.isFinite(v),
941
  ).length,
 
988
  source_metadata: mr.source_metadata,
989
  source_data: mr.source_data,
990
  values: { [columnKey]: mr.score as number },
991
+ verified: mr.result?.is_verified_evaluator
992
+ ? { [columnKey]: true }
993
+ : undefined,
994
  metrics_present: 1,
995
  })) as BenchmarkEvalSummary["leaderboard_rows"]
996
  }
scripts/build-eval-matrices.mjs CHANGED
@@ -132,7 +132,8 @@ async function main() {
132
  r.evaluation_id,
133
  r.metric_id,
134
  r.model_route_id,
135
- r.score
 
136
  FROM read_parquet(${fileRef("eval_results_view.parquet")}) r
137
  WHERE r.score IS NOT NULL
138
  AND r.model_route_id IS NOT NULL
@@ -160,7 +161,8 @@ async function main() {
160
  f.slice_key,
161
  f.slice_name,
162
  f.model_id,
163
- AVG(f.score) AS score
 
164
  FROM read_parquet(${fileRef("fact_results.parquet")}) f
165
  WHERE f.score IS NOT NULL
166
  AND f.slice_key IS NOT NULL
@@ -258,14 +260,25 @@ async function main() {
258
  return out[evalId]
259
  }
260
 
 
 
 
 
 
 
 
 
 
 
 
 
261
  for (const row of metricRows.getRowObjects().map(normalizeDuck)) {
262
  const bucket = ensureEval(row.evaluation_id)
263
- let modelEntry = bucket.leaderboard_rows.get(row.model_route_id)
264
- if (!modelEntry) {
265
- modelEntry = {}
266
- bucket.leaderboard_rows.set(row.model_route_id, modelEntry)
267
  }
268
- modelEntry[row.metric_id] = Number(row.score)
269
  }
270
 
271
  // Plant slice scores. Each (metric_id, slice_key) becomes a column
@@ -289,12 +302,11 @@ async function main() {
289
 
290
  for (const evalId of evalIds) {
291
  const bucket = ensureEval(evalId)
292
- let modelEntry = bucket.leaderboard_rows.get(route)
293
- if (!modelEntry) {
294
- modelEntry = {}
295
- bucket.leaderboard_rows.set(route, modelEntry)
296
  }
297
- modelEntry[columnKey] = score
298
 
299
  if (!bucket.subtask_metric_keys.has(columnKey)) {
300
  bucket.subtask_metric_keys.add(columnKey)
@@ -328,8 +340,8 @@ async function main() {
328
  const finalEvals = {}
329
  for (const [evalId, bucket] of Object.entries(out)) {
330
  const rows = []
331
- for (const [routeId, values] of bucket.leaderboard_rows) {
332
- rows.push({ model_route_id: routeId, values })
333
  }
334
  // Skip evals where every model has at most one metric and no
335
  // subtask data — adds no information beyond the existing summary.
 
132
  r.evaluation_id,
133
  r.metric_id,
134
  r.model_route_id,
135
+ r.score,
136
+ r.is_verified_evaluator
137
  FROM read_parquet(${fileRef("eval_results_view.parquet")}) r
138
  WHERE r.score IS NOT NULL
139
  AND r.model_route_id IS NOT NULL
 
161
  f.slice_key,
162
  f.slice_name,
163
  f.model_id,
164
+ AVG(f.score) AS score,
165
+ bool_or(f.is_verified_evaluator) AS is_verified_evaluator
166
  FROM read_parquet(${fileRef("fact_results.parquet")}) f
167
  WHERE f.score IS NOT NULL
168
  AND f.slice_key IS NOT NULL
 
260
  return out[evalId]
261
  }
262
 
263
+ // modelEntry holds parallel maps: `values` (column_key → score) and
264
+ // `verified` (column_key → bool), so the per-cell verified-evaluator flag
265
+ // rides alongside each non-primary metric / slice column.
266
+ const ensureModelEntry = (bucket, route) => {
267
+ let modelEntry = bucket.leaderboard_rows.get(route)
268
+ if (!modelEntry) {
269
+ modelEntry = { values: {}, verified: {} }
270
+ bucket.leaderboard_rows.set(route, modelEntry)
271
+ }
272
+ return modelEntry
273
+ }
274
+
275
  for (const row of metricRows.getRowObjects().map(normalizeDuck)) {
276
  const bucket = ensureEval(row.evaluation_id)
277
+ const modelEntry = ensureModelEntry(bucket, row.model_route_id)
278
+ modelEntry.values[row.metric_id] = Number(row.score)
279
+ if (row.is_verified_evaluator != null) {
280
+ modelEntry.verified[row.metric_id] = Boolean(row.is_verified_evaluator)
281
  }
 
282
  }
283
 
284
  // Plant slice scores. Each (metric_id, slice_key) becomes a column
 
302
 
303
  for (const evalId of evalIds) {
304
  const bucket = ensureEval(evalId)
305
+ const modelEntry = ensureModelEntry(bucket, route)
306
+ modelEntry.values[columnKey] = score
307
+ if (row.is_verified_evaluator != null) {
308
+ modelEntry.verified[columnKey] = Boolean(row.is_verified_evaluator)
309
  }
 
310
 
311
  if (!bucket.subtask_metric_keys.has(columnKey)) {
312
  bucket.subtask_metric_keys.add(columnKey)
 
340
  const finalEvals = {}
341
  for (const [evalId, bucket] of Object.entries(out)) {
342
  const rows = []
343
+ for (const [routeId, entry] of bucket.leaderboard_rows) {
344
+ rows.push({ model_route_id: routeId, values: entry.values, verified: entry.verified })
345
  }
346
  // Skip evals where every model has at most one metric and no
347
  // subtask data — adds no information beyond the existing summary.
tests/evaluators.test.ts ADDED
@@ -0,0 +1,116 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { describe, it, expect } from "vitest"
2
+
3
+ import {
4
+ buildEvaluatorSlugMap,
5
+ groupEvalsByEvaluator,
6
+ getEvalsForEvaluator,
7
+ verifiedEvalIds,
8
+ } from "../lib/evaluators"
9
+ import type { BenchmarkEvalListItem } from "../lib/eval-processing"
10
+
11
+ // Minimal eval rows — only the fields the evaluator grouping reads. The
12
+ // cast keeps the fixtures terse without re-declaring the full view shape.
13
+ function evalRow(
14
+ id: string,
15
+ evaluators: string[],
16
+ verified: string[] = [],
17
+ ): BenchmarkEvalListItem {
18
+ return {
19
+ evaluation_id: id,
20
+ evaluation_name: id,
21
+ evaluator_names: evaluators,
22
+ verified_evaluator_names: verified,
23
+ } as unknown as BenchmarkEvalListItem
24
+ }
25
+
26
+ const corpus: BenchmarkEvalListItem[] = [
27
+ evalRow("e1", ["Stanford CRFM"], ["Stanford CRFM"]),
28
+ evalRow("e2", ["Stanford CRFM", "OpenAI"], ["Stanford CRFM"]),
29
+ evalRow("e3", ["OpenAI"]),
30
+ // Two junk org strings that must still slug cleanly + uniquely.
31
+ evalRow("e4", ["CapArena (Cheng et al., 2025)"], ["CapArena (Cheng et al., 2025)"]),
32
+ evalRow("e5", ["University of A, University of B, University of C and many others"]),
33
+ ]
34
+
35
+ describe("groupEvalsByEvaluator", () => {
36
+ it("groups by org and sorts by eval count desc", () => {
37
+ const groups = groupEvalsByEvaluator(corpus)
38
+ const byName = Object.fromEntries(groups.map((g) => [g.name, g]))
39
+ expect(byName["Stanford CRFM"].evalCount).toBe(2)
40
+ expect(byName["OpenAI"].evalCount).toBe(2)
41
+ expect(byName["Stanford CRFM"].verifiedCount).toBe(2)
42
+ expect(byName["Stanford CRFM"].isVerified).toBe(true)
43
+ expect(byName["OpenAI"].verifiedCount).toBe(0)
44
+ expect(byName["OpenAI"].isVerified).toBe(false)
45
+ // descending eval count
46
+ expect(groups[0].evalCount).toBeGreaterThanOrEqual(groups[groups.length - 1].evalCount)
47
+ })
48
+
49
+ it("verified-only collapses to verified runners with verified counts", () => {
50
+ const groups = groupEvalsByEvaluator(corpus, { verifiedOnly: true })
51
+ const names = groups.map((g) => g.name).sort()
52
+ expect(names).toEqual(["CapArena (Cheng et al., 2025)", "Stanford CRFM"])
53
+ const crfm = groups.find((g) => g.name === "Stanford CRFM")!
54
+ expect(crfm.evalCount).toBe(2)
55
+ // OpenAI never verified → absent
56
+ expect(groups.find((g) => g.name === "OpenAI")).toBeUndefined()
57
+ })
58
+ })
59
+
60
+ describe("buildEvaluatorSlugMap", () => {
61
+ it("produces valid, non-empty, unique slugs for junk strings", () => {
62
+ const { slugToName, nameToSlug } = buildEvaluatorSlugMap(corpus)
63
+ const slugs = Array.from(slugToName.keys())
64
+ // unique
65
+ expect(new Set(slugs).size).toBe(slugs.length)
66
+ for (const s of slugs) {
67
+ expect(s.length).toBeGreaterThan(0)
68
+ expect(s).toMatch(/^[a-z0-9-]+$/)
69
+ }
70
+ // round-trips
71
+ const cap = nameToSlug.get("CapArena (Cheng et al., 2025)")!
72
+ expect(slugToName.get(cap)).toBe("CapArena (Cheng et al., 2025)")
73
+ })
74
+
75
+ it("disambiguates colliding base slugs with a numeric suffix", () => {
76
+ const collide = [
77
+ evalRow("a", ["Foo Bar!"]),
78
+ evalRow("b", ["Foo Bar"]),
79
+ ]
80
+ const { slugToName } = buildEvaluatorSlugMap(collide)
81
+ const slugs = Array.from(slugToName.keys()).sort()
82
+ expect(new Set(slugs).size).toBe(2)
83
+ expect(slugs.some((s) => /-2$/.test(s))).toBe(true)
84
+ })
85
+ })
86
+
87
+ describe("getEvalsForEvaluator", () => {
88
+ it("resolves a slug to the org's evals", () => {
89
+ const { nameToSlug } = buildEvaluatorSlugMap(corpus)
90
+ const slug = nameToSlug.get("OpenAI")!
91
+ const { name, evals, isVerified } = getEvalsForEvaluator(corpus, slug)
92
+ expect(name).toBe("OpenAI")
93
+ expect(evals.map((e) => e.evaluation_id).sort()).toEqual(["e2", "e3"])
94
+ expect(isVerified).toBe(false)
95
+ })
96
+
97
+ it("restricts to verified evals when verifiedOnly", () => {
98
+ const { nameToSlug } = buildEvaluatorSlugMap(corpus)
99
+ const slug = nameToSlug.get("Stanford CRFM")!
100
+ const { evals } = getEvalsForEvaluator(corpus, slug, { verifiedOnly: true })
101
+ expect(evals.map((e) => e.evaluation_id).sort()).toEqual(["e1", "e2"])
102
+ })
103
+
104
+ it("returns null name for an unknown slug", () => {
105
+ const { name, evals } = getEvalsForEvaluator(corpus, "does-not-exist")
106
+ expect(name).toBeNull()
107
+ expect(evals).toEqual([])
108
+ })
109
+ })
110
+
111
+ describe("verifiedEvalIds", () => {
112
+ it("collects ids with at least one verified evaluator", () => {
113
+ const ids = verifiedEvalIds(corpus)
114
+ expect(Array.from(ids).sort()).toEqual(["e1", "e2", "e4"])
115
+ })
116
+ })
tests/view-data.test.ts CHANGED
@@ -46,6 +46,7 @@ async function writeSyntheticStageJSnapshot(snapshotDir: string) {
46
  1::INTEGER AS variant_count,
47
  1::BIGINT AS evaluator_count,
48
  ['OpenAI']::VARCHAR[] AS evaluator_names,
 
49
  1::INTEGER AS source_type_count,
50
  ['documentation']::VARCHAR[] AS source_types,
51
  0::BIGINT AS third_party_eval_count,
@@ -126,6 +127,7 @@ async function writeSyntheticStageJSnapshot(snapshotDir: string) {
126
  ) AS metric_config,
127
  1::BIGINT AS models_count,
128
  ['OpenAI']::VARCHAR[] AS evaluator_names,
 
129
  ['documentation']::VARCHAR[] AS source_types,
130
  'OpenAI' AS latest_source_name,
131
  0.0 AS third_party_ratio,
@@ -288,7 +290,8 @@ async function writeSyntheticStageJSnapshot(snapshotDir: string) {
288
  NULL AS evalcards_annotations,
289
  NULL::VARCHAR AS instance_file_path,
290
  NULL::VARCHAR AS instance_file_format,
291
- 0::INTEGER AS instance_rows
 
292
  `,
293
  path.join(snapshotDir, "eval_results_view.parquet")
294
  )
@@ -447,7 +450,7 @@ describe("Stage J view-layer backend", () => {
447
  expect(evalSummary?.model_results[0]).toMatchObject({
448
  model_route_id: "openai%2Fgpt-5",
449
  score: 0.8,
450
- result: { metric_summary_id: "mmlu%3Aaccuracy" },
451
  })
452
  expect(evalSummary?.model_results[0]?.result.generation_config).toMatchObject({
453
  temperature: 0.2,
 
46
  1::INTEGER AS variant_count,
47
  1::BIGINT AS evaluator_count,
48
  ['OpenAI']::VARCHAR[] AS evaluator_names,
49
+ ['OpenAI']::VARCHAR[] AS verified_evaluator_names,
50
  1::INTEGER AS source_type_count,
51
  ['documentation']::VARCHAR[] AS source_types,
52
  0::BIGINT AS third_party_eval_count,
 
127
  ) AS metric_config,
128
  1::BIGINT AS models_count,
129
  ['OpenAI']::VARCHAR[] AS evaluator_names,
130
+ ['OpenAI']::VARCHAR[] AS verified_evaluator_names,
131
  ['documentation']::VARCHAR[] AS source_types,
132
  'OpenAI' AS latest_source_name,
133
  0.0 AS third_party_ratio,
 
290
  NULL AS evalcards_annotations,
291
  NULL::VARCHAR AS instance_file_path,
292
  NULL::VARCHAR AS instance_file_format,
293
+ 0::INTEGER AS instance_rows,
294
+ true AS is_verified_evaluator
295
  `,
296
  path.join(snapshotDir, "eval_results_view.parquet")
297
  )
 
450
  expect(evalSummary?.model_results[0]).toMatchObject({
451
  model_route_id: "openai%2Fgpt-5",
452
  score: 0.8,
453
+ result: { metric_summary_id: "mmlu%3Aaccuracy", is_verified_evaluator: true },
454
  })
455
  expect(evalSummary?.model_results[0]?.result.generation_config).toMatchObject({
456
  temperature: 0.2,