Fix embeds to filter by canonical benchmark only (no splits in embed until we decide how to display splits) + suppress popup on embed routes

app/embed/eval/distribution/[...id]/page.tsx

@@ -4,7 +4,10 @@ import { useEffect, useMemo, useState } from "react"
 import { useParams, useSearchParams } from "next/navigation"
 import { ScoreDistribution } from "@/components/score-distribution"
 import { fetchEvalSummary } from "@/lib/dashboard-data-client"
-import { getMetricChipLabel } from "@/lib/metric-labels"
+import {
+  buildDistributionSeries,
+  buildDistributionSliceAxis,
+} from "@/lib/distribution-series"
 import { routeIdFromSegments } from "@/lib/utils"
 import type { BenchmarkEvalSummary } from "@/lib/eval-processing"
 
@@ -53,26 +56,10 @@ export default function EmbedEvalDistribution() {
 
   // Slice axis — present when the eval has multiple subtask-scope metrics
   // sharing a primary root metric (e.g. Global MMLU's 24 language splits).
-  const sliceAxis = useMemo(() => {
-    if (!summary) return null
-    const metrics = summary.leaderboard_metrics ?? []
-    const primary = metrics.find((m) => m.scope !== "subtask")
-    if (!primary?.column_key) return null
-    const seen = new Map<string, string>()
-    for (const m of metrics) {
-      if (m.scope === "subtask" && m.subtask_key && !seen.has(m.subtask_key)) {
-        seen.set(m.subtask_key, m.subtask_name ?? m.subtask_key)
-      }
-    }
-    if (seen.size <= 1) return null
-    return {
-      primaryColumn: primary.column_key,
-      primaryLabel: getMetricChipLabel(primary),
-      unit: primary.unit ?? summary.metric_config.unit,
-      lowerIsBetter: Boolean(primary.lower_is_better ?? summary.metric_config.lower_is_better),
-      slices: Array.from(seen, ([key, label]) => ({ key, label })),
-    }
-  }, [summary])
+  const sliceAxis = useMemo(
+    () => (summary ? buildDistributionSliceAxis(summary) : null),
+    [summary],
+  )
 
   const ALL_SLICE_KEY = "__all__"
   const [activeSlice, setActiveSlice] = useState<string>(() => {
@@ -89,78 +76,10 @@ export default function EmbedEvalDistribution() {
     }
   }, [activeSlice, sliceAxis])
 
-  const series = useMemo(() => {
-    if (!summary) return null
-    const rows = summary.leaderboard_rows ?? []
-
-    // Slice-axis path: render one series for the active slice (Overall or
-    // a specific subtask). Drives the SPLIT dropdown UX.
-    if (sliceAxis) {
-      const columnKey =
-        activeSlice === ALL_SLICE_KEY
-          ? sliceAxis.primaryColumn
-          : `${sliceAxis.primaryColumn}::${activeSlice}`
-      const points: { score: number; releaseDate: string | null; modelName: string }[] = []
-      for (const row of rows) {
-        const raw = (row.values as Record<string, unknown> | undefined)?.[columnKey]
-        const numeric = typeof raw === "number" ? raw : Number(raw)
-        if (!Number.isFinite(numeric)) continue
-        const modelInfo = (row as { model_info?: { name?: string; release_date?: string | null } }).model_info
-        points.push({
-          score: numeric,
-          modelName: modelInfo?.name ?? "",
-          releaseDate: modelInfo?.release_date ?? null,
-        })
-      }
-      if (points.length < 3) return null
-      const sliceLabel =
-        activeSlice === ALL_SLICE_KEY
-          ? "Overall"
-          : sliceAxis.slices.find((s) => s.key === activeSlice)?.label ?? activeSlice
-      return [
-        {
-          key: `${sliceAxis.primaryColumn}::${activeSlice}`,
-          label: `${sliceAxis.primaryLabel} · ${sliceLabel}`,
-          values: points.map((p) => p.score),
-          unit: sliceAxis.unit,
-          lowerIsBetter: sliceAxis.lowerIsBetter,
-          points,
-        },
-      ]
-    }
-
-    // Non-slice path: one series per metric (e.g. agentharm's multi-metric
-    // histogram). ScoreDistribution surfaces a metric chip picker.
-    const metrics = summary.leaderboard_metrics ?? []
-    const built = metrics
-      .map((metric) => {
-        const columnKey = metric.column_key ?? metric.metric_summary_id
-        if (!columnKey) return null
-        const points: { score: number; releaseDate: string | null; modelName: string }[] = []
-        for (const row of rows) {
-          const raw = (row.values as Record<string, unknown> | undefined)?.[columnKey]
-          const numeric = typeof raw === "number" ? raw : Number(raw)
-          if (!Number.isFinite(numeric)) continue
-          const modelInfo = (row as { model_info?: { name?: string; release_date?: string | null } }).model_info
-          points.push({
-            score: numeric,
-            modelName: modelInfo?.name ?? "",
-            releaseDate: modelInfo?.release_date ?? null,
-          })
-        }
-        if (points.length < 3) return null
-        return {
-          key: columnKey,
-          label: getMetricChipLabel(metric),
-          values: points.map((p) => p.score),
-          unit: metric.unit ?? summary.metric_config.unit,
-          lowerIsBetter: Boolean(metric.lower_is_better ?? summary.metric_config.lower_is_better),
-          points,
-        }
-      })
-      .filter((s): s is NonNullable<typeof s> => s !== null)
-    return built.length > 0 ? built : null
-  }, [summary, sliceAxis, activeSlice])
+  const series = useMemo(
+    () => (summary ? buildDistributionSeries(summary, sliceAxis, activeSlice, ALL_SLICE_KEY) : null),
+    [summary, sliceAxis, activeSlice],
+  )
 
   if (error) {
     return (

components/quick-start.tsx

@@ -2,6 +2,7 @@
 
 import { createContext, useContext, useEffect, useMemo, useState } from "react"
 import Link from "next/link"
+import { usePathname } from "next/navigation"
 import {
   ArrowRight,
   BarChart3,
@@ -87,11 +88,17 @@ const SLIDES: Slide[] = [
 
 export function QuickStartProvider({ children }: { children: React.ReactNode }) {
   const [open, setOpen] = useState(false)
+  const pathname = usePathname()
 
   // Auto-open on a visitor's first time, mirroring the audience-mode pattern.
   useEffect(() => {
-    // Embedded views (iframes) never get the tour: storage partitioning means
-    // the seen-flag can't persist there, so it would replay on every load.
+    // Embed surfaces never get the tour. Two independent guards:
+    //  1. Route-based — anything under /embed/* is a self-contained card meant
+    //     for iframing, so the tour must not appear even if the URL is opened
+    //     directly as a top-level tab (e.g. a "preview" link).
+    //  2. Frame-based — when iframed, storage partitioning means the seen-flag
+    //     can't persist, so the tour would replay on every load.
+    if (pathname?.startsWith("/embed")) return
     if (window.self !== window.top) return
     try {
       if (!window.localStorage.getItem(STORAGE_KEY)) {
@@ -100,7 +107,7 @@ export function QuickStartProvider({ children }: { children: React.ReactNode })
     } catch {
       // localStorage unavailable (e.g. privacy mode) — skip the tour silently.
     }
-  }, [])
+  }, [pathname])
 
   const markSeen = () => {
     try {

lib/distribution-series.ts

@@ -0,0 +1,151 @@
+import { getMetricChipLabel } from "@/lib/metric-labels"
+import type { BenchmarkEvalSummary } from "@/lib/eval-processing"
+
+/**
+ * Series-building logic for the score-distribution embed
+ * (`app/embed/eval/distribution/[...id]/page.tsx`), extracted as pure
+ * functions so the chip-selection rules can be unit-tested against real
+ * eval-summary payloads without spinning up the React page.
+ */
+
+export type DistributionPoint = {
+  score: number
+  releaseDate: string | null
+  modelName: string
+}
+
+export type DistributionSeries = {
+  key: string
+  label: string
+  values: number[]
+  unit?: string
+  lowerIsBetter: boolean
+  points: DistributionPoint[]
+}
+
+export type DistributionSliceAxis = {
+  primaryColumn: string
+  primaryLabel: string
+  unit?: string
+  lowerIsBetter: boolean
+  slices: Array<{ key: string; label: string }>
+}
+
+/**
+ * A slice axis exists only when the eval carries MORE THAN ONE distinct
+ * subtask slice sharing a root primary metric (e.g. Global MMLU's per-language
+ * splits). A lone self-slice (one subtask key that just echoes the eval) does
+ * NOT qualify — it falls through to the non-slice path where it is dropped as a
+ * redundant twin of the root metric.
+ */
+export function buildDistributionSliceAxis(
+  summary: BenchmarkEvalSummary,
+): DistributionSliceAxis | null {
+  const metrics = summary.leaderboard_metrics ?? []
+  const primary = metrics.find((m) => m.scope !== "subtask")
+  if (!primary?.column_key) return null
+  const seen = new Map<string, string>()
+  for (const m of metrics) {
+    if (m.scope === "subtask" && m.subtask_key && !seen.has(m.subtask_key)) {
+      seen.set(m.subtask_key, m.subtask_name ?? m.subtask_key)
+    }
+  }
+  if (seen.size <= 1) return null
+  return {
+    primaryColumn: primary.column_key,
+    primaryLabel: getMetricChipLabel(primary),
+    unit: primary.unit ?? summary.metric_config.unit,
+    lowerIsBetter: Boolean(primary.lower_is_better ?? summary.metric_config.lower_is_better),
+    slices: Array.from(seen, ([key, label]) => ({ key, label })),
+  }
+}
+
+function pointsForColumn(
+  rows: BenchmarkEvalSummary["leaderboard_rows"],
+  columnKey: string,
+): DistributionPoint[] {
+  const points: DistributionPoint[] = []
+  for (const row of rows ?? []) {
+    const raw = (row.values as Record<string, unknown> | undefined)?.[columnKey]
+    const numeric = typeof raw === "number" ? raw : Number(raw)
+    if (!Number.isFinite(numeric)) continue
+    const modelInfo = (row as { model_info?: { name?: string; release_date?: string | null } }).model_info
+    points.push({
+      score: numeric,
+      modelName: modelInfo?.name ?? "",
+      releaseDate: modelInfo?.release_date ?? null,
+    })
+  }
+  return points
+}
+
+/**
+ * Build the score-distribution series for the embed. Returns null when there
+ * is nothing renderable (fewer than 3 data points everywhere).
+ */
+export function buildDistributionSeries(
+  summary: BenchmarkEvalSummary,
+  sliceAxis: DistributionSliceAxis | null,
+  activeSlice: string,
+  allSliceKey: string,
+): DistributionSeries[] | null {
+  const rows = summary.leaderboard_rows ?? []
+
+  // Slice-axis path: render one series for the active slice (Overall or a
+  // specific subtask). Drives the SPLIT dropdown UX.
+  if (sliceAxis) {
+    const columnKey =
+      activeSlice === allSliceKey
+        ? sliceAxis.primaryColumn
+        : `${sliceAxis.primaryColumn}::${activeSlice}`
+    const points = pointsForColumn(rows, columnKey)
+    if (points.length < 3) return null
+    const sliceLabel =
+      activeSlice === allSliceKey
+        ? "Overall"
+        : sliceAxis.slices.find((s) => s.key === activeSlice)?.label ?? activeSlice
+    return [
+      {
+        key: `${sliceAxis.primaryColumn}::${activeSlice}`,
+        label: `${sliceAxis.primaryLabel} · ${sliceLabel}`,
+        values: points.map((p) => p.score),
+        unit: sliceAxis.unit,
+        lowerIsBetter: sliceAxis.lowerIsBetter,
+        points,
+      },
+    ]
+  }
+
+  // Non-slice path: one series per ROOT metric (e.g. agentharm's multi-metric
+  // histogram). ScoreDistribution surfaces a metric chip picker. We mirror the
+  // full eval page (eval-detail.tsx) which builds chips only from root-scope
+  // metrics: the producer also emits a redundant self-slice subtask for some
+  // evals — a metric whose subtask_key is just the slugified eval (e.g.
+  // vals-ai/math-500 carries both root `accuracy` and subtask
+  // `accuracy::vals ai math500`, same metric_summary_id and label). Multi-slice
+  // evals (distinct subtask keys) are handled by the sliceAxis path above, so
+  // the only subtasks reaching here are these redundant twins; rendering them
+  // would duplicate the chip (two identical "Accuracy" buttons). Fall back to
+  // the full set only if an eval somehow carries no root metric, so a
+  // subtask-only eval still renders rather than going blank.
+  const allMetrics = summary.leaderboard_metrics ?? []
+  const rootMetrics = allMetrics.filter((m) => m.scope !== "subtask")
+  const metrics = rootMetrics.length > 0 ? rootMetrics : allMetrics
+  const built = metrics
+    .map((metric) => {
+      const columnKey = metric.column_key ?? metric.metric_summary_id
+      if (!columnKey) return null
+      const points = pointsForColumn(rows, columnKey)
+      if (points.length < 3) return null
+      return {
+        key: columnKey,
+        label: getMetricChipLabel(metric),
+        values: points.map((p) => p.score),
+        unit: metric.unit ?? summary.metric_config.unit,
+        lowerIsBetter: Boolean(metric.lower_is_better ?? summary.metric_config.lower_is_better),
+        points,
+      }
+    })
+    .filter((s): s is NonNullable<typeof s> => s !== null)
+  return built.length > 0 ? built : null
+}

tests/distribution-series.test.ts

@@ -0,0 +1,51 @@
+import { describe, expect, it } from "vitest"
+
+import {
+  buildDistributionSeries,
+  buildDistributionSliceAxis,
+} from "@/lib/distribution-series"
+import type { BenchmarkEvalSummary } from "@/lib/eval-processing"
+import fixtures from "./fixtures/distribution-eval-summaries.json"
+
+// Fixtures are trimmed real payloads captured from the live
+// /api/eval-summary route (evalcards.evalevalai.com), so these assertions
+// exercise the actual deployed data shape, not a hand-built mock.
+const ALL = "__all__"
+const summaryOf = (k: keyof typeof fixtures) =>
+  fixtures[k] as unknown as BenchmarkEvalSummary
+
+describe("buildDistributionSeries — metric chip selection", () => {
+  it("vals-ai/math-500: collapses the redundant self-slice subtask to a single Accuracy chip", () => {
+    const summary = summaryOf("math500")
+    // The live payload carries two metrics with the same metric_summary_id:
+    // root `accuracy` + subtask `accuracy::vals ai math500`. The subtask is a
+    // self-slice (its key is just the slugified eval), so it must NOT become a
+    // second chip.
+    expect((summary.leaderboard_metrics ?? []).map((m) => m.scope).sort()).toEqual([
+      "root",
+      "subtask",
+    ])
+    const sliceAxis = buildDistributionSliceAxis(summary)
+    // A lone self-slice is not a real slice axis.
+    expect(sliceAxis).toBeNull()
+    const series = buildDistributionSeries(summary, sliceAxis, ALL, ALL)
+    expect(series).not.toBeNull()
+    expect(series!.length).toBe(1)
+    expect(series![0].label).toBe("Accuracy")
+  })
+
+  it("agentharm: keeps every distinct root metric (multi-metric embed unbroken)", () => {
+    const summary = summaryOf("agentharm")
+    const rootCount = (summary.leaderboard_metrics ?? []).filter(
+      (m) => m.scope !== "subtask",
+    ).length
+    expect(rootCount).toBeGreaterThan(1)
+    const sliceAxis = buildDistributionSliceAxis(summary)
+    const series = buildDistributionSeries(summary, sliceAxis, ALL, ALL)
+    expect(series).not.toBeNull()
+    expect(series!.length).toBe(rootCount)
+    // No two chips share a label (the duplication the fix targets).
+    const labels = series!.map((s) => s.label)
+    expect(new Set(labels).size).toBe(labels.length)
+  })
+})

tests/fixtures/distribution-eval-summaries.json

@@ -0,0 +1,282 @@
+{
+  "math500": {
+    "evaluation_name": "MATH-500",
+    "metric_config": {
+      "evaluation_description": "Accuracy",
+      "lower_is_better": false,
+      "score_type": null,
+      "min_score": 0,
+      "max_score": 1,
+      "unit": "percent"
+    },
+    "leaderboard_metrics": [
+      {
+        "column_key": "accuracy",
+        "metric_summary_id": "math-500%3Aaccuracy",
+        "metric_id": "accuracy",
+        "metric_name": "Accuracy",
+        "display_name": "Accuracy",
+        "canonical_display_name": "Accuracy",
+        "lower_is_better": false,
+        "unit": "percent",
+        "scope": "root",
+        "subtask_key": null,
+        "subtask_name": null
+      },
+      {
+        "column_key": "accuracy::vals ai math500",
+        "metric_summary_id": "math-500%3Aaccuracy",
+        "metric_id": "accuracy",
+        "metric_name": "Accuracy",
+        "display_name": "Accuracy",
+        "canonical_display_name": "Accuracy",
+        "lower_is_better": false,
+        "unit": "percent",
+        "scope": "subtask",
+        "subtask_key": "vals ai math500",
+        "subtask_name": "vals ai math500"
+      }
+    ],
+    "leaderboard_rows": [
+      {
+        "values": {
+          "accuracy": 89,
+          "accuracy::vals ai math500": 89
+        },
+        "model_info": {
+          "name": "MiniMax M2.1",
+          "release_date": "2025-12-20"
+        }
+      },
+      {
+        "values": {
+          "accuracy": 54.8,
+          "accuracy::vals ai math500": 54.8
+        },
+        "model_info": {
+          "name": "Jamba Large 1.6",
+          "release_date": null
+        }
+      },
+      {
+        "values": {
+          "accuracy": 25.4,
+          "accuracy::vals ai math500": 25.4
+        },
+        "model_info": {
+          "name": "Jamba Mini 1.6",
+          "release_date": null
+        }
+      },
+      {
+        "values": {
+          "accuracy": 76.8
+        },
+        "model_info": {
+          "name": "Claude 3.7 Sonnet",
+          "release_date": "2025-02-19"
+        }
+      },
+      {
+        "values": {
+          "accuracy": 64.2
+        },
+        "model_info": {
+          "name": "Claude 3 5 Haiku",
+          "release_date": "2024-10-22"
+        }
+      },
+      {
+        "values": {
+          "accuracy": 91.6,
+          "accuracy::vals ai math500": 91.6
+        },
+        "model_info": {
+          "name": "Claude 3 7 Sonnet 20250219 Thinking",
+          "release_date": null
+        }
+      },
+      {
+        "values": {
+          "accuracy": 72.4
+        },
+        "model_info": {
+          "name": "Claude 3.5 Sonnet",
+          "release_date": "2024-10-22"
+        }
+      },
+      {
+        "values": {
+          "accuracy": 90.4
+        },
+        "model_info": {
+          "name": "Claude Opus 4",
+          "release_date": "2025-05-22"
+        }
+      }
+    ]
+  },
+  "agentharm": {
+    "evaluation_name": "AgentHarm",
+    "metric_config": {
+      "evaluation_description": null,
+      "lower_is_better": false,
+      "score_type": null,
+      "min_score": null,
+      "max_score": null,
+      "unit": "proportion"
+    },
+    "leaderboard_metrics": [
+      {
+        "column_key": "inspect_evals/avg_full_score",
+        "metric_summary_id": "agentharm%3Ainspect_evals%2Favg_full_score",
+        "metric_id": "inspect_evals/avg_full_score",
+        "metric_name": null,
+        "display_name": null,
+        "canonical_display_name": null,
+        "lower_is_better": false,
+        "unit": "proportion",
+        "scope": "root",
+        "subtask_key": null,
+        "subtask_name": null
+      },
+      {
+        "column_key": "inspect_evals/avg_refusals",
+        "metric_summary_id": "agentharm%3Ainspect_evals%2Favg_refusals",
+        "metric_id": "inspect_evals/avg_refusals",
+        "metric_name": null,
+        "display_name": null,
+        "canonical_display_name": null,
+        "lower_is_better": false,
+        "unit": "proportion",
+        "scope": "root",
+        "subtask_key": null,
+        "subtask_name": null
+      },
+      {
+        "column_key": "inspect_evals/avg_score",
+        "metric_summary_id": "agentharm%3Ainspect_evals%2Favg_score",
+        "metric_id": "inspect_evals/avg_score",
+        "metric_name": null,
+        "display_name": null,
+        "canonical_display_name": null,
+        "lower_is_better": false,
+        "unit": "proportion",
+        "scope": "root",
+        "subtask_key": null,
+        "subtask_name": null
+      },
+      {
+        "column_key": "inspect_evals/avg_score_non_refusals",
+        "metric_summary_id": "agentharm%3Ainspect_evals%2Favg_score_non_refusals",
+        "metric_id": "inspect_evals/avg_score_non_refusals",
+        "metric_name": null,
+        "display_name": null,
+        "canonical_display_name": null,
+        "lower_is_better": false,
+        "unit": "proportion",
+        "scope": "root",
+        "subtask_key": null,
+        "subtask_name": null
+      }
+    ],
+    "leaderboard_rows": [
+      {
+        "values": {
+          "inspect_evals/avg_full_score": 0.23863636363636365,
+          "inspect_evals/avg_refusals": 0.6534090909090909,
+          "inspect_evals/avg_score": 0.339788715072806,
+          "inspect_evals/avg_score_non_refusals": 0.8732825917252146
+        },
+        "model_info": {
+          "name": "Claude 3.7 Sonnet",
+          "release_date": "2025-02-19"
+        }
+      },
+      {
+        "values": {
+          "inspect_evals/avg_full_score": 0.375,
+          "inspect_evals/avg_refusals": 0.022727272727272728,
+          "inspect_evals/avg_score": 0.7212594696969696,
+          "inspect_evals/avg_score_non_refusals": 0.7380329457364341
+        },
+        "model_info": {
+          "name": "DeepSeek",
+          "release_date": "2025-12-01"
+        }
+      },
+      {
+        "values": {
+          "inspect_evals/avg_full_score": 0.29545454545454547,
+          "inspect_evals/avg_refusals": 0.3352272727272727,
+          "inspect_evals/avg_score": 0.47781385281385286,
+          "inspect_evals/avg_score_non_refusals": 0.7170533170533171
+        },
+        "model_info": {
+          "name": "Gemini 2.0 Flash",
+          "release_date": "2024-12-11"
+        }
+      },
+      {
+        "values": {
+          "inspect_evals/avg_full_score": 0.125,
+          "inspect_evals/avg_refusals": 0.38636363636363635,
+          "inspect_evals/avg_score": 0.3750411993309721,
+          "inspect_evals/avg_score_non_refusals": 0.582783189033189
+        },
+        "model_info": {
+          "name": "Llama 3.3",
+          "release_date": null
+        }
+      },
+      {
+        "values": {
+          "inspect_evals/avg_full_score": 0.4034090909090909,
+          "inspect_evals/avg_refusals": 0.045454545454545456,
+          "inspect_evals/avg_score": 0.6345533254624164,
+          "inspect_evals/avg_score_non_refusals": 0.6624317151102865
+        },
+        "model_info": {
+          "name": "Mistral Large",
+          "release_date": "2024-11-01"
+        }
+      },
+      {
+        "values": {
+          "inspect_evals/avg_full_score": 0.3125,
+          "inspect_evals/avg_refusals": 0.4772727272727273,
+          "inspect_evals/avg_score": 0.4302323150334514,
+          "inspect_evals/avg_score_non_refusals": 0.8007963956333523
+        },
+        "model_info": {
+          "name": "Mistral Small",
+          "release_date": "2026-03-16"
+        }
+      },
+      {
+        "values": {
+          "inspect_evals/avg_full_score": 0.35795454545454547,
+          "inspect_evals/avg_refusals": 0.42613636363636365,
+          "inspect_evals/avg_score": 0.4916531385281386,
+          "inspect_evals/avg_score_non_refusals": 0.8254125412541253
+        },
+        "model_info": {
+          "name": "GPT-4o",
+          "release_date": "2024-05-13"
+        }
+      },
+      {
+        "values": {
+          "inspect_evals/avg_full_score": 0.32386363636363635,
+          "inspect_evals/avg_refusals": 0.19318181818181818,
+          "inspect_evals/avg_score": 0.5942812868949233,
+          "inspect_evals/avg_score_non_refusals": 0.7107523931467593
+        },
+        "model_info": {
+          "name": "GPT-4o mini",
+          "release_date": "2024-07-18"
+        }
+      }
+    ]
+  }
+}