Spaces:

usmar
/

overlift

Running

App Files Files Community

usmar commited on 10 days ago

Commit

ba312df

verified ·

1 Parent(s): 77f66e1

Publish overlift interface

Browse files

Files changed (3) hide show

README.md +18 -5
data.json +0 -0
index.html +484 -17

README.md CHANGED Viewed

@@ -1,10 +1,23 @@
 ---
-title: Overlift
-emoji: 😻
-colorFrom: gray
-colorTo: yellow
 sdk: static
 pinned: false
 ---
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

 ---
+title: overlift
+emoji: 📉
+colorFrom: blue
+colorTo: gray
 sdk: static
 pinned: false
+license: mit
+short_description: The winner of your A/B test is not as good as it looked
 ---
+# overlift
+The winner's curse, peeking and power, measured on **32,363 real randomised
+experiments** from the [Upworthy Research Archive](https://osf.io/jd64p/)
+(Matias, Munger, Le Quere & Ebersole, CC BY 4.0).
+Every arm's impressions are dealt at random into a selection half and a held-out
+half: pick the best arm on one, score it on the other. Nothing is simulated —
+the clicks are the archive's clicks. The page re-runs that split in your browser,
+so "re-split the traffic" really does re-deal the impressions.
+Code, method and the full write-up: <https://github.com/UsmarHaider/overlift>

data.json ADDED Viewed

The diff for this file is too large to render. See raw diff

index.html CHANGED Viewed

@@ -1,19 +1,486 @@
 <!doctype html>
-<html>
-	<head>
-		<meta charset="utf-8" />
-		<meta name="viewport" content="width=device-width" />
-		<title>My static Space</title>
-		<link rel="stylesheet" href="style.css" />
-	</head>
-	<body>
-		<div class="card">
-			<h1>Welcome to your static Space!</h1>
-			<p>You can modify this app directly by editing <i>index.html</i> in the Files and versions tab.</p>
-			<p>
-				Also don't forget to check the
-				<a href="https://huggingface.co/docs/hub/spaces" target="_blank">Spaces documentation</a>.
-			</p>
-		</div>
-	</body>
 </html>

 <!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>overlift — the winner of your A/B test is not that good</title>
+<meta name="description" content="32,363 real randomised experiments from the Upworthy Research Archive, re-split to show how much of a winner's lift is noise.">
+<style>
+  :root {
+    --surface: #fcfcfb;
+    --plane: #f4f3ef;
+    --card: #ffffff;
+    --ink: #0b0b0b;
+    --ink2: #52514e;
+    --muted: #898781;
+    --grid: #e1e0d9;
+    --axis: #c3c2b7;
+    --blue: #2a78d6;
+    --orange: #eb6834;
+    --aqua: #1baf7a;
+    --yellow: #a8730a;
+    --shadow: 0 1px 2px rgba(11,11,11,.06), 0 8px 24px rgba(11,11,11,.05);
+  }
+  @media (prefers-color-scheme: dark) {
+    :root {
+      --surface: #1a1a19;
+      --plane: #121211;
+      --card: #232322;
+      --ink: #ffffff;
+      --ink2: #c3c2b7;
+      --muted: #898781;
+      --grid: #2c2c2a;
+      --axis: #383835;
+      --blue: #3987e5;
+      --orange: #d95926;
+      --aqua: #199e70;
+      --yellow: #c98500;
+      --shadow: 0 1px 2px rgba(0,0,0,.4), 0 8px 24px rgba(0,0,0,.3);
+    }
+  }
+  * { box-sizing: border-box; }
+  body {
+    margin: 0;
+    background: var(--plane);
+    color: var(--ink);
+    font: 15px/1.55 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
+    -webkit-font-smoothing: antialiased;
+  }
+  .wrap { max-width: 1060px; margin: 0 auto; padding: 34px 20px 60px; }
+  header { margin-bottom: 26px; }
+  h1 {
+    font-size: 30px; line-height: 1.15; margin: 0 0 6px; letter-spacing: -0.02em;
+  }
+  h1 .dot { color: var(--orange); }
+  .lede { color: var(--ink2); margin: 0; max-width: 66ch; }
+  .lede b { color: var(--ink); font-weight: 600; }
+  h2 { font-size: 15px; margin: 0 0 4px; letter-spacing: -0.01em; }
+  .sub { color: var(--muted); font-size: 12.5px; margin: 0 0 14px; }
+  .tiles { display: grid; grid-template-columns: repeat(4, 1fr); gap: 12px; margin: 22px 0 26px; }
+  .tile {
+    background: var(--card); border: 1px solid var(--grid); border-radius: 12px;
+    padding: 14px 15px; box-shadow: var(--shadow);
+  }
+  .tile .k { font-size: 26px; font-weight: 650; letter-spacing: -0.02em; line-height: 1.1; }
+  .tile .k.warn { color: var(--orange); }
+  .tile .k.good { color: var(--blue); }
+  .tile .t { font-size: 12px; color: var(--ink2); margin-top: 5px; }
+  .card {
+    background: var(--card); border: 1px solid var(--grid); border-radius: 14px;
+    padding: 18px 20px 20px; box-shadow: var(--shadow); margin-bottom: 18px;
+  }
+  .row { display: flex; gap: 10px; flex-wrap: wrap; align-items: center; margin-bottom: 14px; }
+  button {
+    font: inherit; font-size: 13.5px; font-weight: 550; color: #fff; background: var(--blue);
+    border: 0; border-radius: 8px; padding: 8px 14px; cursor: pointer;
+  }
+  button:hover { filter: brightness(1.07); }
+  button.ghost { background: transparent; color: var(--ink2); border: 1px solid var(--axis); }
+  button:focus-visible { outline: 2px solid var(--orange); outline-offset: 2px; }
+  .pill {
+    font-size: 12px; color: var(--ink2); background: var(--plane);
+    border: 1px solid var(--grid); border-radius: 999px; padding: 4px 10px;
+  }
+  table { width: 100%; border-collapse: collapse; font-size: 13px; }
+  th {
+    text-align: right; font-weight: 550; color: var(--muted); font-size: 11.5px;
+    text-transform: uppercase; letter-spacing: .04em; padding: 0 8px 7px; white-space: nowrap;
+  }
+  th:first-child, td:first-child { text-align: left; }
+  td { padding: 8px; border-top: 1px solid var(--grid); text-align: right; vertical-align: top; }
+  td.head { color: var(--ink); max-width: 380px; }
+  tr.win td { background: color-mix(in srgb, var(--blue) 9%, transparent); }
+  tr.win td.head::before {
+    content: "picked"; display: inline-block; font-size: 10px; font-weight: 650;
+    letter-spacing: .05em; text-transform: uppercase; color: #fff; background: var(--blue);
+    border-radius: 4px; padding: 1px 5px; margin-right: 7px; vertical-align: 1px;
+  }
+  .bar { height: 4px; border-radius: 2px; background: var(--axis); margin-top: 5px; }
+  .num { font-variant-numeric: tabular-nums; }
+  .verdict { display: flex; gap: 22px; flex-wrap: wrap; align-items: flex-end; margin: 16px 0 4px; }
+  .metric .v { font-size: 22px; font-weight: 650; letter-spacing: -0.02em; font-variant-numeric: tabular-nums; }
+  .metric .l { font-size: 11.5px; color: var(--muted); }
+  .badge {
+    margin-left: auto; font-size: 12.5px; font-weight: 600; border-radius: 8px;
+    padding: 6px 11px; border: 1px solid;
+  }
+  .badge.held { color: var(--blue); border-color: var(--blue); background: color-mix(in srgb, var(--blue) 10%, transparent); }
+  .badge.gone { color: var(--orange); border-color: var(--orange); background: color-mix(in srgb, var(--orange) 10%, transparent); }
+  .grid2 { display: grid; grid-template-columns: 1fr 1fr; gap: 18px; }
+  figure { margin: 0; }
+  figcaption { font-size: 12px; color: var(--muted); margin-top: 8px; }
+  svg { display: block; width: 100%; height: auto; overflow: visible; }
+  .axis-t { font-size: 10px; fill: var(--muted); }
+  .lbl { font-size: 11px; font-weight: 550; }
+  footer { color: var(--muted); font-size: 12px; margin-top: 26px; line-height: 1.7; }
+  footer a { color: var(--ink2); }
+  .scroller { overflow-x: auto; }
+  @media (max-width: 860px) {
+    .tiles { grid-template-columns: repeat(2, 1fr); }
+    .grid2 { grid-template-columns: 1fr; }
+  }
+</style>
+</head>
+<body>
+<div class="wrap">
+  <header>
+    <h1>overlift<span class="dot">.</span></h1>
+    <p class="lede">
+      <b>32,363 real randomised experiments</b> from the Upworthy Research Archive, re-split so the
+      arm you would have picked gets scored on traffic it never saw. Every number below is measured
+      on real clicks — no effects are simulated.
+    </p>
+  </header>
+  <div class="tiles" id="tiles"></div>
+  <section class="card">
+    <h2>Pick a winner, then check it against held-out traffic</h2>
+    <p class="sub">
+      Half of each arm's impressions choose the winner; the other half scores it. Re-split to see how
+      much of the verdict was the coin, not the copy.
+    </p>
+    <div class="row">
+      <button id="sample">Try a sample test</button>
+      <button id="resplit" class="ghost">Re-split the traffic</button>
+      <button id="next" class="ghost">Another test</button>
+      <span class="pill" id="meta">—</span>
+    </div>
+    <div class="scroller">
+      <table id="arms">
+        <thead>
+          <tr>
+            <th>Headline</th><th>Impressions</th><th>CTR</th>
+            <th>Selection half</th><th>Held-out half</th><th>Shrunk</th>
+          </tr>
+        </thead>
+        <tbody></tbody>
+      </table>
+    </div>
+    <div class="verdict">
+      <div class="metric"><div class="v" id="v-claim" style="color:var(--orange)">—</div><div class="l">lift you would have claimed</div></div>
+      <div class="metric"><div class="v" id="v-real" style="color:var(--blue)">—</div><div class="l">lift on held-out traffic</div></div>
+      <div class="metric"><div class="v" id="v-eb">—</div><div class="l">shrunk prediction</div></div>
+      <div class="metric"><div class="v" id="v-p">—</div><div class="l">p-value, winner vs rest</div></div>
+      <div class="badge" id="badge">—</div>
+    </div>
+  </section>
+  <section class="card">
+    <h2>What the dashboard would have shown you along the way</h2>
+    <p class="sub">
+      The z-statistic for the top two arms as traffic accumulates over ten looks. Inside the grey band
+      nothing is significant at 5%; the dashed line is the O'Brien-Fleming boundary that keeps ten
+      looks honest.
+    </p>
+    <figure>
+      <svg id="zpath" viewBox="0 0 720 240" role="img" aria-label="Sequential z-statistic across ten looks"></svg>
+      <figcaption id="zcap">—</figcaption>
+    </figure>
+  </section>
+  <div class="grid2">
+    <section class="card">
+      <h2>Peeking, on real A/A splits</h2>
+      <p class="sub">One arm dealt into two. True difference: exactly zero.</p>
+      <figure>
+        <svg id="c-peek" viewBox="0 0 420 210" role="img" aria-label="False positive rate against number of looks"></svg>
+        <figcaption>False positives when nothing is there.</figcaption>
+      </figure>
+    </section>
+    <section class="card">
+      <h2>Arms, on real null tests</h2>
+      <p class="sub">One arm dealt into k. Every arm identical.</p>
+      <figure>
+        <svg id="c-mult" viewBox="0 0 420 210" role="img" aria-label="False positive rate against number of arms"></svg>
+        <figcaption>A “significant” winner where none exists.</figcaption>
+      </figure>
+    </section>
+  </div>
+  <footer>
+    Data: <a href="https://osf.io/jd64p/">Upworthy Research Archive</a> (Matias, Munger, Le Quere &amp;
+    Ebersole), CC BY 4.0 — headlines shown as published in the archive.<br>
+    Code and method: <a href="https://github.com/UsmarHaider/overlift">github.com/UsmarHaider/overlift</a>.
+  </footer>
+</div>
+<script>
+"use strict";
+const pct = (v, d = 0) => (v === null || v === undefined || !isFinite(v)) ? "—" : (v * 100).toFixed(d) + "%";
+const num = (v) => v.toLocaleString("en-US");
+// A seeded PRNG so "re-split" is reproducible from the URL, and an exact deal of
+// c clicks into cells of the given sizes -- the same multivariate hypergeometric
+// the Python pipeline uses, written out as sampling without replacement.
+function mulberry32(a) {
+  return function () {
+    a |= 0; a = (a + 0x6D2B79F5) | 0;
+    let t = Math.imul(a ^ (a >>> 15), 1 | a);
+    t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+  };
+}
+function deal(n, c, sizes, rng) {
+  const out = new Array(sizes.length).fill(0);
+  let remainingTotal = n, remainingClicks = c;
+  const room = sizes.slice();
+  for (let i = 0; i < sizes.length - 1; i++) {
+    let got = 0;
+    let poolLeft = remainingTotal, cellLeft = room[i], clicks = remainingClicks;
+    // Deal the remaining clicks one at a time into "this cell" or "the rest".
+    for (let k = 0; k < clicks; k++) {
+      if (poolLeft <= 0) break;
+      if (rng() < cellLeft / poolLeft) { got++; cellLeft--; }
+      poolLeft--;
+    }
+    out[i] = got;
+    remainingClicks -= got;
+    remainingTotal -= sizes[i];
+  }
+  out[sizes.length - 1] = remainingClicks;
+  return out;
+}
+function equalSizes(n, m) {
+  const base = Math.floor(n / m), rem = n - base * m;
+  return Array.from({ length: m }, (_, i) => base + (i < rem ? 1 : 0));
+}
+function ctr(c, n) { return n > 0 ? c / n : 0; }
+function zTest(c1, n1, c2, n2) {
+  const p1 = ctr(c1, n1), p2 = ctr(c2, n2), p = ctr(c1 + c2, n1 + n2);
+  const se = Math.sqrt(p * (1 - p) * (1 / n1 + 1 / n2));
+  return se > 0 ? (p1 - p2) / se : 0;
+}
+function normSf(z) {  // Abramowitz & Stegun 7.1.26, plenty for a displayed p-value
+  const t = 1 / (1 + 0.2316419 * Math.abs(z));
+  const d = 0.3989422804014327 * Math.exp(-z * z / 2);
+  const p = d * t * (0.319381530 + t * (-0.356563782 + t * (1.781477937 + t * (-1.821255978 + t * 1.330274429))));
+  return z > 0 ? p : 1 - p;
+}
+let DATA = null, INDEX = 0, SEED = 1;
+async function boot() {
+  DATA = await (await fetch("data.json", { cache: "no-store" })).json();
+  const url = new URL(location.href);
+  INDEX = Math.max(0, Math.min(DATA.tests.length - 1, parseInt(url.searchParams.get("sample") ?? "6", 10) || 0));
+  renderTiles();
+  renderPeek();
+  renderMult();
+  render();
+  document.getElementById("sample").onclick = () => { INDEX = (INDEX + 7) % DATA.tests.length; SEED = 1; render(); };
+  document.getElementById("next").onclick = () => { INDEX = (INDEX + 1) % DATA.tests.length; SEED = 1; render(); };
+  document.getElementById("resplit").onclick = () => { SEED++; render(); };
+  // boot() already renders a fully populated state, so ?demo=1 only fixes the
+  // seed -- it exists so screenshots are reproducible.
+  if (url.searchParams.get("demo") === "1") { SEED = 1; render(); }
+}
+function renderTiles() {
+  const c = DATA.curse.vs_test_average, p = DATA.power;
+  const peek = DATA.peeking.aa_nulls.by_looks["10"];
+  const tiles = [
+    { k: pct(c.median_in_sample), cls: "warn", t: "median lift the winning arm appears to have" },
+    { k: pct(c.median_out_of_sample), cls: "good", t: "median lift it actually has on held-out traffic" },
+    { k: pct(peek.naive), cls: "warn", t: "false positives from peeking 10 times at a null test" },
+    { k: pct(p.median_mde_relative), cls: "", t: "smallest lift the median test could even detect" },
+  ];
+  document.getElementById("tiles").innerHTML = tiles.map(t =>
+    `<div class="tile"><div class="k ${t.cls}">${t.k}</div><div class="t">${t.t}</div></div>`).join("");
+}
+function splitTest(t, seed) {
+  const rng = mulberry32(seed * 7919 + t.impressions);
+  const arms = t.arms.map(a => {
+    const nA = Math.round(a.impressions / 2);
+    const [cA, cB] = deal(a.impressions, a.clicks, [nA, a.impressions - nA], rng);
+    return { ...a, nA, cA, nB: a.impressions - nA, cB, sel: ctr(cA, nA), hold: ctr(cB, a.impressions - nA) };
+  });
+  const selC = arms.reduce((s, a) => s + a.cA, 0), selN = arms.reduce((s, a) => s + a.nA, 0);
+  const holC = arms.reduce((s, a) => s + a.cB, 0), holN = arms.reduce((s, a) => s + a.nB, 0);
+  const poolSel = ctr(selC, selN), poolHold = ctr(holC, holN);
+  let w = 0;
+  arms.forEach((a, i) => { if (a.sel > arms[w].sel) w = i; });
+  const kappa = DATA.curse.config.kappa;
+  arms.forEach(a => { a.eb = (a.cA + kappa * poolSel) / (a.nA + kappa); });
+  const restC = selC - arms[w].cA, restN = selN - arms[w].nA;
+  const z = zTest(arms[w].cA, arms[w].nA, restC, restN);
+  const bestHold = arms.reduce((b, a, i) => (a.hold > arms[b].hold ? i : b), 0);
+  const order = arms.map((a, i) => i).sort((x, y) => arms[y].sel - arms[x].sel);
+  const top = order[0], second = order[1] ?? order[0];
+  const looks = DATA.meta.looks;
+  const path = [];
+  const pair = [top, second].map(i => {
+    const sizes = equalSizes(t.arms[i].impressions, looks);
+    return { sizes, inc: deal(t.arms[i].impressions, t.arms[i].clicks, sizes, rng) };
+  });
+  let n1 = 0, c1 = 0, n2 = 0, c2 = 0;
+  for (let k = 0; k < looks; k++) {
+    n1 += pair[0].sizes[k]; c1 += pair[0].inc[k];
+    n2 += pair[1].sizes[k]; c2 += pair[1].inc[k];
+    path.push(zTest(c1, n1, c2, n2));
+  }
+  return {
+    arms, w, poolSel, poolHold, z, path,
+    p: 2 * normSf(Math.abs(z)),
+    liftSel: poolSel > 0 ? arms[w].sel / poolSel - 1 : null,
+    liftHold: poolHold > 0 ? arms[w].hold / poolHold - 1 : null,
+    ebLift: poolSel > 0 ? arms[w].eb / poolSel - 1 : null,
+    heldUp: bestHold === w,
+  };
+}
+function render() {
+  const t = DATA.tests[INDEX];
+  const s = splitTest(t, SEED);
+  const maxCtr = Math.max(...s.arms.map(a => a.ctr)) || 1;
+  document.getElementById("meta").textContent =
+    `${t.arms.length} arms · ${num(t.impressions)} impressions · varied the ${t.variation} · test ${INDEX + 1}/${DATA.tests.length}`;
+  document.querySelector("#arms tbody").innerHTML = s.arms.map((a, i) => `
+    <tr class="${i === s.w ? "win" : ""}">
+      <td class="head">${escapeHtml(a.headline || "(headline not recorded)")}
+        <div class="bar" style="width:${(a.ctr / maxCtr * 100).toFixed(1)}%;background:${i === s.w ? "var(--blue)" : "var(--axis)"}"></div>
+      </td>
+      <td class="num">${num(a.impressions)}</td>
+      <td class="num">${pct(a.ctr, 2)}</td>
+      <td class="num" style="color:var(--orange)">${pct(a.sel, 2)}</td>
+      <td class="num" style="color:var(--blue)">${pct(a.hold, 2)}</td>
+      <td class="num">${pct(a.eb, 2)}</td>
+    </tr>`).join("");
+  document.getElementById("v-claim").textContent = s.liftSel === null ? "—" : (s.liftSel >= 0 ? "+" : "") + pct(s.liftSel, 1);
+  document.getElementById("v-real").textContent = s.liftHold === null ? "—" : (s.liftHold >= 0 ? "+" : "") + pct(s.liftHold, 1);
+  document.getElementById("v-eb").textContent = s.ebLift === null ? "—" : (s.ebLift >= 0 ? "+" : "") + pct(s.ebLift, 1);
+  document.getElementById("v-p").textContent = s.p < 0.001 ? "<0.001" : s.p.toFixed(3);
+  const badge = document.getElementById("badge");
+  badge.className = "badge " + (s.heldUp ? "held" : "gone");
+  badge.textContent = s.heldUp ? "still the best arm on held-out traffic" : "a different arm wins on held-out traffic";
+  renderZ(s.path);
+}
+function escapeHtml(str) {
+  return str.replace(/[&<>"']/g, ch => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" }[ch]));
+}
+function renderZ(path) {
+  const svg = document.getElementById("zpath");
+  const W = 720, H = 240, L = 44, R = 16, T = 14, B = 30;
+  const obf = DATA.meta.obf_bounds;
+  // Zoom to the action: the O'Brien-Fleming boundary starts near z = 6 and would
+  // otherwise squash every real path into a flat line near the axis.
+  const lim = Math.min(5.2, Math.max(2.9, ...path.map(Math.abs)) * 1.2);
+  const x = i => L + (W - L - R) * (i / (path.length - 1));
+  const y = v => T + (H - T - B) * (1 - (v + lim) / (2 * lim));
+  const line = pts => pts.map((p, i) => (i ? "L" : "M") + p[0].toFixed(1) + " " + p[1].toFixed(1)).join(" ");
+  const zf = DATA.meta.z_fixed;
+  const cross = path.findIndex(v => Math.abs(v) >= zf);
+  const parts = [];
+  parts.push(`<rect x="${L}" y="${y(zf)}" width="${W - L - R}" height="${y(-zf) - y(zf)}" fill="var(--grid)" opacity=".55"/>`);
+  parts.push(`<line x1="${L}" x2="${W - R}" y1="${y(0)}" y2="${y(0)}" stroke="var(--axis)" stroke-width="1"/>`);
+  for (const sgn of [1, -1]) {
+    parts.push(`<path d="${line(obf.map((b, i) => [x(i), y(sgn * Math.min(b, lim))]))}" fill="none"
+      stroke="var(--muted)" stroke-width="1.4" stroke-dasharray="4 4"/>`);
+  }
+  parts.push(`<path d="${line(path.map((v, i) => [x(i), y(v)]))}" fill="none" stroke="var(--blue)" stroke-width="2.4" stroke-linejoin="round"/>`);
+  path.forEach((v, i) => {
+    const hit = Math.abs(v) >= zf;
+    parts.push(`<circle cx="${x(i)}" cy="${y(v)}" r="${hit ? 5 : 3.6}" fill="${hit ? "var(--orange)" : "var(--blue)"}"
+      stroke="var(--card)" stroke-width="1.6"><title>look ${i + 1}: z = ${v.toFixed(2)}</title></circle>`);
+    parts.push(`<text class="axis-t" x="${x(i)}" y="${H - 10}" text-anchor="middle">${i + 1}</text>`);
+  });
+  for (const v of [-6, -4, -2, 0, 2, 4, 6].filter(v => Math.abs(v) < lim)) {
+    parts.push(`<text class="axis-t" x="${L - 8}" y="${y(v) + 3.5}" text-anchor="end">${v}</text>`);
+  }
+  parts.push(`<text class="lbl" x="${L + 6}" y="${y(Math.min(obf[0], lim)) + 15}" fill="var(--muted)">O'Brien-Fleming</text>`);
+  parts.push(`<text class="lbl" x="${L + 6}" y="${y(-zf) - 7}" fill="var(--muted)">not significant at 5%</text>`);
+  parts.push(`<text class="axis-t" x="${L}" y="${H - 10}" text-anchor="middle"></text>`);
+  svg.innerHTML = parts.join("");
+  const final = path[path.length - 1];
+  document.getElementById("zcap").textContent = cross >= 0
+    ? `An analyst peeking every look would have stopped at look ${cross + 1} (z = ${path[cross].toFixed(2)}). The test ended at z = ${final.toFixed(2)}${Math.abs(final) < zf ? " — not significant." : "."}`
+    : `This test never crossed 1.96 at any look; it ended at z = ${final.toFixed(2)}.`;
+}
+// A reference label sits on top of marks, so it carries a card-coloured halo.
+function refLabel(parts, x, y, text, anchor = "start") {
+  const w = text.length * 5.4 + 8;
+  const left = anchor === "end" ? x - w : x - 4;
+  parts.push(`<rect x="${left.toFixed(1)}" y="${(y - 9).toFixed(1)}" width="${w.toFixed(1)}" height="12" rx="3" fill="var(--card)" opacity=".92"/>`);
+  parts.push(`<text class="axis-t" x="${x}" y="${y}" text-anchor="${anchor}">${text}</text>`);
+}
+function axes(parts, W, H, L, R, T, B, xs, ys, xlab, fmt) {
+  const x = v => L + (W - L - R) * ((v - xs[0]) / (xs[1] - xs[0]));
+  const y = v => T + (H - T - B) * (1 - (v - ys[0]) / (ys[1] - ys[0]));
+  for (let i = 0; i <= 4; i++) {
+    const v = ys[0] + (ys[1] - ys[0]) * i / 4;
+    parts.push(`<line x1="${L}" x2="${W - R}" y1="${y(v).toFixed(1)}" y2="${y(v).toFixed(1)}" stroke="var(--grid)" stroke-width="1"/>`);
+    parts.push(`<text class="axis-t" x="${L - 7}" y="${(y(v) + 3.5).toFixed(1)}" text-anchor="end">${fmt(v)}</text>`);
+  }
+  parts.push(`<text class="axis-t" x="${(L + W - R) / 2}" y="${H - 4}" text-anchor="middle">${xlab}</text>`);
+  return { x, y };
+}
+function renderPeek() {
+  const by = DATA.peeking.aa_nulls.by_looks;
+  const looks = Object.keys(by).map(Number).sort((a, b) => a - b);
+  const W = 420, H = 210, L = 40, R = 14, T = 12, B = 30;
+  const parts = [];
+  const ymax = Math.max(...looks.map(k => by[k].naive)) * 1.15;
+  const sc = axes(parts, W, H, L, R, T, B, [0, looks.length - 1], [0, ymax], "times the analyst looks", v => (v * 100).toFixed(0) + "%");
+  const series = [["naive", "var(--orange)", "peek"], ["obrien_fleming", "var(--blue)", "O'B-F"], ["pocock", "var(--aqua)", "Pocock"]];
+  parts.push(`<line x1="${L}" x2="${W - R}" y1="${sc.y(0.05)}" y2="${sc.y(0.05)}" stroke="var(--muted)" stroke-dasharray="4 4" stroke-width="1.2"/>`);
+  refLabel(parts, L + 6, sc.y(0.05) - 5, "nominal 5%");
+  const labDy = { naive: -9, obrien_fleming: -9, pocock: 15 };
+  for (const [key, col, lab] of series) {
+    const pts = looks.map((k, i) => [sc.x(i), sc.y(by[k][key])]);
+    parts.push(`<path d="${pts.map((p, i) => (i ? "L" : "M") + p[0].toFixed(1) + " " + p[1].toFixed(1)).join(" ")}" fill="none" stroke="${col}" stroke-width="2.2"/>`);
+    pts.forEach((p, i) => parts.push(`<circle cx="${p[0].toFixed(1)}" cy="${p[1].toFixed(1)}" r="3.4" fill="${col}"><title>${lab}, ${looks[i]} looks: ${(by[looks[i]][key] * 100).toFixed(1)}%</title></circle>`));
+    parts.push(`<text class="lbl" x="${pts[pts.length - 1][0] - 4}" y="${pts[pts.length - 1][1] + labDy[key]}" text-anchor="end" fill="${col}">${lab}</text>`);
+  }
+  looks.forEach((k, i) => parts.push(`<text class="axis-t" x="${sc.x(i)}" y="${H - 17}" text-anchor="middle">${k}</text>`));
+  document.getElementById("c-peek").innerHTML = parts.join("");
+}
+function renderMult() {
+  const by = DATA.multiplicity.by_arms;
+  const ks = Object.keys(by).map(Number).sort((a, b) => a - b);
+  const W = 420, H = 210, L = 40, R = 14, T = 12, B = 30;
+  const parts = [];
+  const ymax = Math.max(...ks.map(k => by[k].false_positive_rate)) * 1.15;
+  const sc = axes(parts, W, H, L, R, T, B, [0, ks.length - 1], [0, ymax], "arms in the test", v => (v * 100).toFixed(0) + "%");
+  const bw = (W - L - R) / ks.length * 0.62;
+  ks.forEach((k, i) => {
+    const v = by[k].false_positive_rate;
+    parts.push(`<rect x="${(sc.x(i) - bw / 2).toFixed(1)}" y="${sc.y(v).toFixed(1)}" width="${bw.toFixed(1)}"
+      height="${(sc.y(0) - sc.y(v)).toFixed(1)}" rx="3" fill="var(--orange)"><title>${k} arms: ${(v * 100).toFixed(1)}% of null tests declare a winner</title></rect>`);
+    parts.push(`<text class="axis-t" x="${sc.x(i)}" y="${H - 17}" text-anchor="middle">${k}</text>`);
+  });
+  parts.push(`<line x1="${L}" x2="${W - R}" y1="${sc.y(0.05)}" y2="${sc.y(0.05)}" stroke="var(--muted)" stroke-dasharray="4 4" stroke-width="1.2"/>`);
+  refLabel(parts, W - R, sc.y(0.05) - 5, "nominal 5%", "end");
+  document.getElementById("c-mult").innerHTML = parts.join("");
+}
+boot().catch(err => {
+  document.querySelector(".wrap").insertAdjacentHTML("afterbegin",
+    `<div class="card" style="border-color:var(--orange)"><b>Could not load data.json.</b><br>
+     Run <code>make all</code> to build it, then start the service with <code>make serve</code>.</div>`);
+  console.error(err);
+});
+</script>
+</body>
 </html>