| <!doctype html> |
| <html lang="en"> |
| <head> |
| <meta charset="utf-8"> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <title>overlift — the winner of your A/B test is not that good</title> |
| <meta name="description" content="32,363 real randomised experiments from the Upworthy Research Archive, re-split to show how much of a winner's lift is noise."> |
| <style> |
| :root { |
| --surface: #fcfcfb; |
| --plane: #f4f3ef; |
| --card: #ffffff; |
| --ink: #0b0b0b; |
| --ink2: #52514e; |
| --muted: #898781; |
| --grid: #e1e0d9; |
| --axis: #c3c2b7; |
| --blue: #2a78d6; |
| --orange: #eb6834; |
| --aqua: #1baf7a; |
| --yellow: #a8730a; |
| --shadow: 0 1px 2px rgba(11,11,11,.06), 0 8px 24px rgba(11,11,11,.05); |
| } |
| @media (prefers-color-scheme: dark) { |
| :root { |
| --surface: #1a1a19; |
| --plane: #121211; |
| --card: #232322; |
| --ink: #ffffff; |
| --ink2: #c3c2b7; |
| --muted: #898781; |
| --grid: #2c2c2a; |
| --axis: #383835; |
| --blue: #3987e5; |
| --orange: #d95926; |
| --aqua: #199e70; |
| --yellow: #c98500; |
| --shadow: 0 1px 2px rgba(0,0,0,.4), 0 8px 24px rgba(0,0,0,.3); |
| } |
| } |
| * { box-sizing: border-box; } |
| body { |
| margin: 0; |
| background: var(--plane); |
| color: var(--ink); |
| font: 15px/1.55 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; |
| -webkit-font-smoothing: antialiased; |
| } |
| .wrap { max-width: 1060px; margin: 0 auto; padding: 34px 20px 60px; } |
| header { margin-bottom: 26px; } |
| h1 { |
| font-size: 30px; line-height: 1.15; margin: 0 0 6px; letter-spacing: -0.02em; |
| } |
| h1 .dot { color: var(--orange); } |
| .lede { color: var(--ink2); margin: 0; max-width: 66ch; } |
| .lede b { color: var(--ink); font-weight: 600; } |
| h2 { font-size: 15px; margin: 0 0 4px; letter-spacing: -0.01em; } |
| .sub { color: var(--muted); font-size: 12.5px; margin: 0 0 14px; } |
| |
| .tiles { display: grid; grid-template-columns: repeat(4, 1fr); gap: 12px; margin: 22px 0 26px; } |
| .tile { |
| background: var(--card); border: 1px solid var(--grid); border-radius: 12px; |
| padding: 14px 15px; box-shadow: var(--shadow); |
| } |
| .tile .k { font-size: 26px; font-weight: 650; letter-spacing: -0.02em; line-height: 1.1; } |
| .tile .k.warn { color: var(--orange); } |
| .tile .k.good { color: var(--blue); } |
| .tile .t { font-size: 12px; color: var(--ink2); margin-top: 5px; } |
| |
| .card { |
| background: var(--card); border: 1px solid var(--grid); border-radius: 14px; |
| padding: 18px 20px 20px; box-shadow: var(--shadow); margin-bottom: 18px; |
| } |
| .row { display: flex; gap: 10px; flex-wrap: wrap; align-items: center; margin-bottom: 14px; } |
| button { |
| font: inherit; font-size: 13.5px; font-weight: 550; color: #fff; background: var(--blue); |
| border: 0; border-radius: 8px; padding: 8px 14px; cursor: pointer; |
| } |
| button:hover { filter: brightness(1.07); } |
| button.ghost { background: transparent; color: var(--ink2); border: 1px solid var(--axis); } |
| button:focus-visible { outline: 2px solid var(--orange); outline-offset: 2px; } |
| .pill { |
| font-size: 12px; color: var(--ink2); background: var(--plane); |
| border: 1px solid var(--grid); border-radius: 999px; padding: 4px 10px; |
| } |
| |
| table { width: 100%; border-collapse: collapse; font-size: 13px; } |
| th { |
| text-align: right; font-weight: 550; color: var(--muted); font-size: 11.5px; |
| text-transform: uppercase; letter-spacing: .04em; padding: 0 8px 7px; white-space: nowrap; |
| } |
| th:first-child, td:first-child { text-align: left; } |
| td { padding: 8px; border-top: 1px solid var(--grid); text-align: right; vertical-align: top; } |
| td.head { color: var(--ink); max-width: 380px; } |
| tr.win td { background: color-mix(in srgb, var(--blue) 9%, transparent); } |
| tr.win td.head::before { |
| content: "picked"; display: inline-block; font-size: 10px; font-weight: 650; |
| letter-spacing: .05em; text-transform: uppercase; color: #fff; background: var(--blue); |
| border-radius: 4px; padding: 1px 5px; margin-right: 7px; vertical-align: 1px; |
| } |
| .bar { height: 4px; border-radius: 2px; background: var(--axis); margin-top: 5px; } |
| .num { font-variant-numeric: tabular-nums; } |
| |
| .verdict { display: flex; gap: 22px; flex-wrap: wrap; align-items: flex-end; margin: 16px 0 4px; } |
| .metric .v { font-size: 22px; font-weight: 650; letter-spacing: -0.02em; font-variant-numeric: tabular-nums; } |
| .metric .l { font-size: 11.5px; color: var(--muted); } |
| .badge { |
| margin-left: auto; font-size: 12.5px; font-weight: 600; border-radius: 8px; |
| padding: 6px 11px; border: 1px solid; |
| } |
| .badge.held { color: var(--blue); border-color: var(--blue); background: color-mix(in srgb, var(--blue) 10%, transparent); } |
| .badge.gone { color: var(--orange); border-color: var(--orange); background: color-mix(in srgb, var(--orange) 10%, transparent); } |
| |
| .grid2 { display: grid; grid-template-columns: 1fr 1fr; gap: 18px; } |
| figure { margin: 0; } |
| figcaption { font-size: 12px; color: var(--muted); margin-top: 8px; } |
| svg { display: block; width: 100%; height: auto; overflow: visible; } |
| .axis-t { font-size: 10px; fill: var(--muted); } |
| .lbl { font-size: 11px; font-weight: 550; } |
| footer { color: var(--muted); font-size: 12px; margin-top: 26px; line-height: 1.7; } |
| footer a { color: var(--ink2); } |
| .scroller { overflow-x: auto; } |
| @media (max-width: 860px) { |
| .tiles { grid-template-columns: repeat(2, 1fr); } |
| .grid2 { grid-template-columns: 1fr; } |
| } |
| </style> |
| </head> |
| <body> |
| <div class="wrap"> |
| <header> |
| <h1>overlift<span class="dot">.</span></h1> |
| <p class="lede"> |
| <b>32,363 real randomised experiments</b> from the Upworthy Research Archive, re-split so the |
| arm you would have picked gets scored on traffic it never saw. Every number below is measured |
| on real clicks — no effects are simulated. |
| </p> |
| </header> |
|
|
| <div class="tiles" id="tiles"></div> |
|
|
| <section class="card"> |
| <h2>Pick a winner, then check it against held-out traffic</h2> |
| <p class="sub"> |
| Half of each arm's impressions choose the winner; the other half scores it. Re-split to see how |
| much of the verdict was the coin, not the copy. |
| </p> |
| <div class="row"> |
| <button id="sample">Try a sample test</button> |
| <button id="resplit" class="ghost">Re-split the traffic</button> |
| <button id="next" class="ghost">Another test</button> |
| <span class="pill" id="meta">—</span> |
| </div> |
| <div class="scroller"> |
| <table id="arms"> |
| <thead> |
| <tr> |
| <th>Headline</th><th>Impressions</th><th>CTR</th> |
| <th>Selection half</th><th>Held-out half</th><th>Shrunk</th> |
| </tr> |
| </thead> |
| <tbody></tbody> |
| </table> |
| </div> |
| <div class="verdict"> |
| <div class="metric"><div class="v" id="v-claim" style="color:var(--orange)">—</div><div class="l">lift you would have claimed</div></div> |
| <div class="metric"><div class="v" id="v-real" style="color:var(--blue)">—</div><div class="l">lift on held-out traffic</div></div> |
| <div class="metric"><div class="v" id="v-eb">—</div><div class="l">shrunk prediction</div></div> |
| <div class="metric"><div class="v" id="v-p">—</div><div class="l">p-value, winner vs rest</div></div> |
| <div class="badge" id="badge">—</div> |
| </div> |
| </section> |
|
|
| <section class="card"> |
| <h2>What the dashboard would have shown you along the way</h2> |
| <p class="sub"> |
| The z-statistic for the top two arms as traffic accumulates over ten looks. Inside the grey band |
| nothing is significant at 5%; the dashed line is the O'Brien-Fleming boundary that keeps ten |
| looks honest. |
| </p> |
| <figure> |
| <svg id="zpath" viewBox="0 0 720 240" role="img" aria-label="Sequential z-statistic across ten looks"></svg> |
| <figcaption id="zcap">—</figcaption> |
| </figure> |
| </section> |
|
|
| <div class="grid2"> |
| <section class="card"> |
| <h2>Peeking, on real A/A splits</h2> |
| <p class="sub">One arm dealt into two. True difference: exactly zero.</p> |
| <figure> |
| <svg id="c-peek" viewBox="0 0 420 210" role="img" aria-label="False positive rate against number of looks"></svg> |
| <figcaption>False positives when nothing is there.</figcaption> |
| </figure> |
| </section> |
| <section class="card"> |
| <h2>Arms, on real null tests</h2> |
| <p class="sub">One arm dealt into k. Every arm identical.</p> |
| <figure> |
| <svg id="c-mult" viewBox="0 0 420 210" role="img" aria-label="False positive rate against number of arms"></svg> |
| <figcaption>A “significant” winner where none exists.</figcaption> |
| </figure> |
| </section> |
| </div> |
|
|
| <footer> |
| Data: <a href="https://osf.io/jd64p/">Upworthy Research Archive</a> (Matias, Munger, Le Quere & |
| Ebersole), CC BY 4.0 — headlines shown as published in the archive.<br> |
| Code and method: <a href="https://github.com/UsmarHaider/overlift">github.com/UsmarHaider/overlift</a>. |
| </footer> |
| </div> |
|
|
| <script> |
| "use strict"; |
| |
| const pct = (v, d = 0) => (v === null || v === undefined || !isFinite(v)) ? "—" : (v * 100).toFixed(d) + "%"; |
| const num = (v) => v.toLocaleString("en-US"); |
| |
| |
| |
| |
| function mulberry32(a) { |
| return function () { |
| a |= 0; a = (a + 0x6D2B79F5) | 0; |
| let t = Math.imul(a ^ (a >>> 15), 1 | a); |
| t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; |
| return ((t ^ (t >>> 14)) >>> 0) / 4294967296; |
| }; |
| } |
| function deal(n, c, sizes, rng) { |
| const out = new Array(sizes.length).fill(0); |
| let remainingTotal = n, remainingClicks = c; |
| const room = sizes.slice(); |
| for (let i = 0; i < sizes.length - 1; i++) { |
| let got = 0; |
| let poolLeft = remainingTotal, cellLeft = room[i], clicks = remainingClicks; |
| |
| for (let k = 0; k < clicks; k++) { |
| if (poolLeft <= 0) break; |
| if (rng() < cellLeft / poolLeft) { got++; cellLeft--; } |
| poolLeft--; |
| } |
| out[i] = got; |
| remainingClicks -= got; |
| remainingTotal -= sizes[i]; |
| } |
| out[sizes.length - 1] = remainingClicks; |
| return out; |
| } |
| function equalSizes(n, m) { |
| const base = Math.floor(n / m), rem = n - base * m; |
| return Array.from({ length: m }, (_, i) => base + (i < rem ? 1 : 0)); |
| } |
| function ctr(c, n) { return n > 0 ? c / n : 0; } |
| function zTest(c1, n1, c2, n2) { |
| const p1 = ctr(c1, n1), p2 = ctr(c2, n2), p = ctr(c1 + c2, n1 + n2); |
| const se = Math.sqrt(p * (1 - p) * (1 / n1 + 1 / n2)); |
| return se > 0 ? (p1 - p2) / se : 0; |
| } |
| function normSf(z) { |
| const t = 1 / (1 + 0.2316419 * Math.abs(z)); |
| const d = 0.3989422804014327 * Math.exp(-z * z / 2); |
| const p = d * t * (0.319381530 + t * (-0.356563782 + t * (1.781477937 + t * (-1.821255978 + t * 1.330274429)))); |
| return z > 0 ? p : 1 - p; |
| } |
| |
| let DATA = null, INDEX = 0, SEED = 1; |
| |
| async function boot() { |
| DATA = await (await fetch("data.json", { cache: "no-store" })).json(); |
| const url = new URL(location.href); |
| INDEX = Math.max(0, Math.min(DATA.tests.length - 1, parseInt(url.searchParams.get("sample") ?? "6", 10) || 0)); |
| renderTiles(); |
| renderPeek(); |
| renderMult(); |
| render(); |
| document.getElementById("sample").onclick = () => { INDEX = (INDEX + 7) % DATA.tests.length; SEED = 1; render(); }; |
| document.getElementById("next").onclick = () => { INDEX = (INDEX + 1) % DATA.tests.length; SEED = 1; render(); }; |
| document.getElementById("resplit").onclick = () => { SEED++; render(); }; |
| |
| |
| if (url.searchParams.get("demo") === "1") { SEED = 1; render(); } |
| } |
| |
| function renderTiles() { |
| const c = DATA.curse.vs_test_average, p = DATA.power; |
| const peek = DATA.peeking.aa_nulls.by_looks["10"]; |
| const tiles = [ |
| { k: pct(c.median_in_sample), cls: "warn", t: "median lift the winning arm appears to have" }, |
| { k: pct(c.median_out_of_sample), cls: "good", t: "median lift it actually has on held-out traffic" }, |
| { k: pct(peek.naive), cls: "warn", t: "false positives from peeking 10 times at a null test" }, |
| { k: pct(p.median_mde_relative), cls: "", t: "smallest lift the median test could even detect" }, |
| ]; |
| document.getElementById("tiles").innerHTML = tiles.map(t => |
| `<div class="tile"><div class="k ${t.cls}">${t.k}</div><div class="t">${t.t}</div></div>`).join(""); |
| } |
| |
| function splitTest(t, seed) { |
| const rng = mulberry32(seed * 7919 + t.impressions); |
| const arms = t.arms.map(a => { |
| const nA = Math.round(a.impressions / 2); |
| const [cA, cB] = deal(a.impressions, a.clicks, [nA, a.impressions - nA], rng); |
| return { ...a, nA, cA, nB: a.impressions - nA, cB, sel: ctr(cA, nA), hold: ctr(cB, a.impressions - nA) }; |
| }); |
| const selC = arms.reduce((s, a) => s + a.cA, 0), selN = arms.reduce((s, a) => s + a.nA, 0); |
| const holC = arms.reduce((s, a) => s + a.cB, 0), holN = arms.reduce((s, a) => s + a.nB, 0); |
| const poolSel = ctr(selC, selN), poolHold = ctr(holC, holN); |
| let w = 0; |
| arms.forEach((a, i) => { if (a.sel > arms[w].sel) w = i; }); |
| const kappa = DATA.curse.config.kappa; |
| arms.forEach(a => { a.eb = (a.cA + kappa * poolSel) / (a.nA + kappa); }); |
| |
| const restC = selC - arms[w].cA, restN = selN - arms[w].nA; |
| const z = zTest(arms[w].cA, arms[w].nA, restC, restN); |
| const bestHold = arms.reduce((b, a, i) => (a.hold > arms[b].hold ? i : b), 0); |
| |
| const order = arms.map((a, i) => i).sort((x, y) => arms[y].sel - arms[x].sel); |
| const top = order[0], second = order[1] ?? order[0]; |
| const looks = DATA.meta.looks; |
| const path = []; |
| const pair = [top, second].map(i => { |
| const sizes = equalSizes(t.arms[i].impressions, looks); |
| return { sizes, inc: deal(t.arms[i].impressions, t.arms[i].clicks, sizes, rng) }; |
| }); |
| let n1 = 0, c1 = 0, n2 = 0, c2 = 0; |
| for (let k = 0; k < looks; k++) { |
| n1 += pair[0].sizes[k]; c1 += pair[0].inc[k]; |
| n2 += pair[1].sizes[k]; c2 += pair[1].inc[k]; |
| path.push(zTest(c1, n1, c2, n2)); |
| } |
| return { |
| arms, w, poolSel, poolHold, z, path, |
| p: 2 * normSf(Math.abs(z)), |
| liftSel: poolSel > 0 ? arms[w].sel / poolSel - 1 : null, |
| liftHold: poolHold > 0 ? arms[w].hold / poolHold - 1 : null, |
| ebLift: poolSel > 0 ? arms[w].eb / poolSel - 1 : null, |
| heldUp: bestHold === w, |
| }; |
| } |
| |
| function render() { |
| const t = DATA.tests[INDEX]; |
| const s = splitTest(t, SEED); |
| const maxCtr = Math.max(...s.arms.map(a => a.ctr)) || 1; |
| |
| document.getElementById("meta").textContent = |
| `${t.arms.length} arms · ${num(t.impressions)} impressions · varied the ${t.variation} · test ${INDEX + 1}/${DATA.tests.length}`; |
| |
| document.querySelector("#arms tbody").innerHTML = s.arms.map((a, i) => ` |
| <tr class="${i === s.w ? "win" : ""}"> |
| <td class="head">${escapeHtml(a.headline || "(headline not recorded)")} |
| <div class="bar" style="width:${(a.ctr / maxCtr * 100).toFixed(1)}%;background:${i === s.w ? "var(--blue)" : "var(--axis)"}"></div> |
| </td> |
| <td class="num">${num(a.impressions)}</td> |
| <td class="num">${pct(a.ctr, 2)}</td> |
| <td class="num" style="color:var(--orange)">${pct(a.sel, 2)}</td> |
| <td class="num" style="color:var(--blue)">${pct(a.hold, 2)}</td> |
| <td class="num">${pct(a.eb, 2)}</td> |
| </tr>`).join(""); |
| |
| document.getElementById("v-claim").textContent = s.liftSel === null ? "—" : (s.liftSel >= 0 ? "+" : "") + pct(s.liftSel, 1); |
| document.getElementById("v-real").textContent = s.liftHold === null ? "—" : (s.liftHold >= 0 ? "+" : "") + pct(s.liftHold, 1); |
| document.getElementById("v-eb").textContent = s.ebLift === null ? "—" : (s.ebLift >= 0 ? "+" : "") + pct(s.ebLift, 1); |
| document.getElementById("v-p").textContent = s.p < 0.001 ? "<0.001" : s.p.toFixed(3); |
| |
| const badge = document.getElementById("badge"); |
| badge.className = "badge " + (s.heldUp ? "held" : "gone"); |
| badge.textContent = s.heldUp ? "still the best arm on held-out traffic" : "a different arm wins on held-out traffic"; |
| |
| renderZ(s.path); |
| } |
| |
| function escapeHtml(str) { |
| return str.replace(/[&<>"']/g, ch => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[ch])); |
| } |
| |
| function renderZ(path) { |
| const svg = document.getElementById("zpath"); |
| const W = 720, H = 240, L = 44, R = 16, T = 14, B = 30; |
| const obf = DATA.meta.obf_bounds; |
| |
| |
| const lim = Math.min(5.2, Math.max(2.9, ...path.map(Math.abs)) * 1.2); |
| const x = i => L + (W - L - R) * (i / (path.length - 1)); |
| const y = v => T + (H - T - B) * (1 - (v + lim) / (2 * lim)); |
| const line = pts => pts.map((p, i) => (i ? "L" : "M") + p[0].toFixed(1) + " " + p[1].toFixed(1)).join(" "); |
| |
| const zf = DATA.meta.z_fixed; |
| const cross = path.findIndex(v => Math.abs(v) >= zf); |
| const parts = []; |
| parts.push(`<rect x="${L}" y="${y(zf)}" width="${W - L - R}" height="${y(-zf) - y(zf)}" fill="var(--grid)" opacity=".55"/>`); |
| parts.push(`<line x1="${L}" x2="${W - R}" y1="${y(0)}" y2="${y(0)}" stroke="var(--axis)" stroke-width="1"/>`); |
| for (const sgn of [1, -1]) { |
| parts.push(`<path d="${line(obf.map((b, i) => [x(i), y(sgn * Math.min(b, lim))]))}" fill="none" |
| stroke="var(--muted)" stroke-width="1.4" stroke-dasharray="4 4"/>`); |
| } |
| parts.push(`<path d="${line(path.map((v, i) => [x(i), y(v)]))}" fill="none" stroke="var(--blue)" stroke-width="2.4" stroke-linejoin="round"/>`); |
| path.forEach((v, i) => { |
| const hit = Math.abs(v) >= zf; |
| parts.push(`<circle cx="${x(i)}" cy="${y(v)}" r="${hit ? 5 : 3.6}" fill="${hit ? "var(--orange)" : "var(--blue)"}" |
| stroke="var(--card)" stroke-width="1.6"><title>look ${i + 1}: z = ${v.toFixed(2)}</title></circle>`); |
| parts.push(`<text class="axis-t" x="${x(i)}" y="${H - 10}" text-anchor="middle">${i + 1}</text>`); |
| }); |
| for (const v of [-6, -4, -2, 0, 2, 4, 6].filter(v => Math.abs(v) < lim)) { |
| parts.push(`<text class="axis-t" x="${L - 8}" y="${y(v) + 3.5}" text-anchor="end">${v}</text>`); |
| } |
| parts.push(`<text class="lbl" x="${L + 6}" y="${y(Math.min(obf[0], lim)) + 15}" fill="var(--muted)">O'Brien-Fleming</text>`); |
| parts.push(`<text class="lbl" x="${L + 6}" y="${y(-zf) - 7}" fill="var(--muted)">not significant at 5%</text>`); |
| parts.push(`<text class="axis-t" x="${L}" y="${H - 10}" text-anchor="middle"></text>`); |
| svg.innerHTML = parts.join(""); |
| |
| const final = path[path.length - 1]; |
| document.getElementById("zcap").textContent = cross >= 0 |
| ? `An analyst peeking every look would have stopped at look ${cross + 1} (z = ${path[cross].toFixed(2)}). The test ended at z = ${final.toFixed(2)}${Math.abs(final) < zf ? " — not significant." : "."}` |
| : `This test never crossed 1.96 at any look; it ended at z = ${final.toFixed(2)}.`; |
| } |
| |
| |
| function refLabel(parts, x, y, text, anchor = "start") { |
| const w = text.length * 5.4 + 8; |
| const left = anchor === "end" ? x - w : x - 4; |
| parts.push(`<rect x="${left.toFixed(1)}" y="${(y - 9).toFixed(1)}" width="${w.toFixed(1)}" height="12" rx="3" fill="var(--card)" opacity=".92"/>`); |
| parts.push(`<text class="axis-t" x="${x}" y="${y}" text-anchor="${anchor}">${text}</text>`); |
| } |
| |
| function axes(parts, W, H, L, R, T, B, xs, ys, xlab, fmt) { |
| const x = v => L + (W - L - R) * ((v - xs[0]) / (xs[1] - xs[0])); |
| const y = v => T + (H - T - B) * (1 - (v - ys[0]) / (ys[1] - ys[0])); |
| for (let i = 0; i <= 4; i++) { |
| const v = ys[0] + (ys[1] - ys[0]) * i / 4; |
| parts.push(`<line x1="${L}" x2="${W - R}" y1="${y(v).toFixed(1)}" y2="${y(v).toFixed(1)}" stroke="var(--grid)" stroke-width="1"/>`); |
| parts.push(`<text class="axis-t" x="${L - 7}" y="${(y(v) + 3.5).toFixed(1)}" text-anchor="end">${fmt(v)}</text>`); |
| } |
| parts.push(`<text class="axis-t" x="${(L + W - R) / 2}" y="${H - 4}" text-anchor="middle">${xlab}</text>`); |
| return { x, y }; |
| } |
| |
| function renderPeek() { |
| const by = DATA.peeking.aa_nulls.by_looks; |
| const looks = Object.keys(by).map(Number).sort((a, b) => a - b); |
| const W = 420, H = 210, L = 40, R = 14, T = 12, B = 30; |
| const parts = []; |
| const ymax = Math.max(...looks.map(k => by[k].naive)) * 1.15; |
| const sc = axes(parts, W, H, L, R, T, B, [0, looks.length - 1], [0, ymax], "times the analyst looks", v => (v * 100).toFixed(0) + "%"); |
| const series = [["naive", "var(--orange)", "peek"], ["obrien_fleming", "var(--blue)", "O'B-F"], ["pocock", "var(--aqua)", "Pocock"]]; |
| parts.push(`<line x1="${L}" x2="${W - R}" y1="${sc.y(0.05)}" y2="${sc.y(0.05)}" stroke="var(--muted)" stroke-dasharray="4 4" stroke-width="1.2"/>`); |
| refLabel(parts, L + 6, sc.y(0.05) - 5, "nominal 5%"); |
| const labDy = { naive: -9, obrien_fleming: -9, pocock: 15 }; |
| for (const [key, col, lab] of series) { |
| const pts = looks.map((k, i) => [sc.x(i), sc.y(by[k][key])]); |
| parts.push(`<path d="${pts.map((p, i) => (i ? "L" : "M") + p[0].toFixed(1) + " " + p[1].toFixed(1)).join(" ")}" fill="none" stroke="${col}" stroke-width="2.2"/>`); |
| pts.forEach((p, i) => parts.push(`<circle cx="${p[0].toFixed(1)}" cy="${p[1].toFixed(1)}" r="3.4" fill="${col}"><title>${lab}, ${looks[i]} looks: ${(by[looks[i]][key] * 100).toFixed(1)}%</title></circle>`)); |
| parts.push(`<text class="lbl" x="${pts[pts.length - 1][0] - 4}" y="${pts[pts.length - 1][1] + labDy[key]}" text-anchor="end" fill="${col}">${lab}</text>`); |
| } |
| looks.forEach((k, i) => parts.push(`<text class="axis-t" x="${sc.x(i)}" y="${H - 17}" text-anchor="middle">${k}</text>`)); |
| document.getElementById("c-peek").innerHTML = parts.join(""); |
| } |
| |
| function renderMult() { |
| const by = DATA.multiplicity.by_arms; |
| const ks = Object.keys(by).map(Number).sort((a, b) => a - b); |
| const W = 420, H = 210, L = 40, R = 14, T = 12, B = 30; |
| const parts = []; |
| const ymax = Math.max(...ks.map(k => by[k].false_positive_rate)) * 1.15; |
| const sc = axes(parts, W, H, L, R, T, B, [0, ks.length - 1], [0, ymax], "arms in the test", v => (v * 100).toFixed(0) + "%"); |
| const bw = (W - L - R) / ks.length * 0.62; |
| ks.forEach((k, i) => { |
| const v = by[k].false_positive_rate; |
| parts.push(`<rect x="${(sc.x(i) - bw / 2).toFixed(1)}" y="${sc.y(v).toFixed(1)}" width="${bw.toFixed(1)}" |
| height="${(sc.y(0) - sc.y(v)).toFixed(1)}" rx="3" fill="var(--orange)"><title>${k} arms: ${(v * 100).toFixed(1)}% of null tests declare a winner</title></rect>`); |
| parts.push(`<text class="axis-t" x="${sc.x(i)}" y="${H - 17}" text-anchor="middle">${k}</text>`); |
| }); |
| parts.push(`<line x1="${L}" x2="${W - R}" y1="${sc.y(0.05)}" y2="${sc.y(0.05)}" stroke="var(--muted)" stroke-dasharray="4 4" stroke-width="1.2"/>`); |
| refLabel(parts, W - R, sc.y(0.05) - 5, "nominal 5%", "end"); |
| document.getElementById("c-mult").innerHTML = parts.join(""); |
| } |
| |
| boot().catch(err => { |
| document.querySelector(".wrap").insertAdjacentHTML("afterbegin", |
| `<div class="card" style="border-color:var(--orange)"><b>Could not load data.json.</b><br> |
| Run <code>make all</code> to build it, then start the service with <code>make serve</code>.</div>`); |
| console.error(err); |
| }); |
| </script> |
| </body> |
| </html> |
|
|
