| <!doctype html> |
| <html lang="en"> |
| <head> |
| <meta charset="utf-8"> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <title>outliar — the metric is the bug</title> |
| <style> |
| :root { |
| --blue: #2a78d6; |
| --orange: #eb6834; |
| --ink: #14202c; |
| --muted: #66748a; |
| --bg: #f6f8fb; |
| --card: #ffffff; |
| --line: #e2e8f1; |
| --shadow: 0 1px 2px rgba(20, 32, 44, .06), 0 8px 24px rgba(20, 32, 44, .06); |
| } |
| @media (prefers-color-scheme: dark) { |
| :root { |
| --ink: #e8edf4; |
| --muted: #93a1b5; |
| --bg: #0f151d; |
| --card: #171f2a; |
| --line: #26313f; |
| --shadow: 0 1px 2px rgba(0, 0, 0, .4), 0 8px 24px rgba(0, 0, 0, .3); |
| } |
| } |
| * { box-sizing: border-box; } |
| body { |
| margin: 0; padding: 28px 20px 48px; background: var(--bg); color: var(--ink); |
| font: 15px/1.55 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; |
| -webkit-font-smoothing: antialiased; |
| } |
| .wrap { max-width: 1080px; margin: 0 auto; } |
| header { margin-bottom: 20px; } |
| h1 { font-size: 25px; margin: 0 0 4px; letter-spacing: -.02em; } |
| h1 span { color: var(--orange); } |
| .tag { color: var(--muted); font-size: 14.5px; margin: 0; max-width: 66ch; } |
| .card { |
| background: var(--card); border: 1px solid var(--line); border-radius: 12px; |
| box-shadow: var(--shadow); padding: 18px; margin-bottom: 16px; |
| } |
| .controls { display: flex; flex-wrap: wrap; gap: 12px; align-items: flex-end; } |
| .field { display: flex; flex-direction: column; gap: 5px; min-width: 190px; flex: 1 1 200px; } |
| label { font-size: 11.5px; text-transform: uppercase; letter-spacing: .07em; color: var(--muted); font-weight: 600; } |
| select, button { |
| font: inherit; color: var(--ink); background: var(--bg); |
| border: 1px solid var(--line); border-radius: 8px; padding: 9px 11px; |
| } |
| select { width: 100%; } |
| button { |
| background: var(--blue); color: #fff; border-color: transparent; cursor: pointer; |
| font-weight: 600; padding: 9px 16px; white-space: nowrap; transition: filter .15s; |
| } |
| button:hover { filter: brightness(1.08); } |
| button:disabled { opacity: .55; cursor: default; } |
| |
| .verdict { |
| display: flex; gap: 12px; align-items: flex-start; |
| border-left: 3px solid var(--blue); padding: 10px 0 10px 14px; margin-top: 16px; |
| font-size: 14px; |
| } |
| .verdict.control { border-left-color: var(--orange); } |
| .verdict b { display: block; margin-bottom: 2px; } |
| .verdict .why { color: var(--muted); } |
| |
| canvas { width: 100%; display: block; } |
| .slider-row { display: flex; align-items: center; gap: 14px; margin-top: 14px; } |
| .slider-row input[type=range] { flex: 1; accent-color: var(--orange); } |
| .thr { font-variant-numeric: tabular-nums; font-size: 13px; color: var(--muted); min-width: 128px; } |
| .presets { display: flex; gap: 8px; } |
| .presets button { background: transparent; color: var(--muted); border: 1px solid var(--line); font-weight: 500; font-size: 12.5px; padding: 6px 10px; } |
| .presets button:hover { color: var(--ink); filter: none; border-color: var(--muted); } |
| |
| .metrics { display: grid; grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); gap: 12px; } |
| .metric { border: 1px solid var(--line); border-radius: 10px; padding: 12px 13px; background: var(--bg); } |
| .metric.hero { border-color: var(--orange); background: color-mix(in srgb, var(--orange) 8%, var(--card)); } |
| .metric.good { border-color: var(--blue); background: color-mix(in srgb, var(--blue) 8%, var(--card)); } |
| .metric .k { font-size: 11px; text-transform: uppercase; letter-spacing: .06em; color: var(--muted); font-weight: 600; } |
| .metric .v { font-size: 27px; font-weight: 650; font-variant-numeric: tabular-nums; letter-spacing: -.02em; margin: 3px 0 1px; } |
| .metric .s { font-size: 12px; color: var(--muted); font-variant-numeric: tabular-nums; } |
| |
| .ledger { margin-top: 16px; padding-top: 14px; border-top: 1px solid var(--line); font-size: 13.5px; color: var(--muted); } |
| .ledger b { color: var(--ink); font-variant-numeric: tabular-nums; } |
| .legend { display: flex; gap: 16px; flex-wrap: wrap; font-size: 12.5px; color: var(--muted); margin-bottom: 10px; } |
| .swatch { display: inline-block; width: 11px; height: 11px; border-radius: 3px; vertical-align: -1px; margin-right: 5px; } |
| footer { color: var(--muted); font-size: 12.5px; text-align: center; margin-top: 22px; } |
| footer a { color: var(--blue); } |
| .err { color: var(--orange); font-size: 13.5px; } |
| </style> |
| </head> |
| <body> |
|
|
| <div class="wrap"> |
| <header> |
| <h1>outl<span>iar</span></h1> |
| <p class="tag"> |
| Anomaly detection on the Numenta benchmark, scored four ways. Pick a detector — including |
| one that is literally uniform noise — drag the threshold, and watch the metrics disagree |
| about whether it works. |
| </p> |
| </header> |
|
|
| <div class="card"> |
| <div class="controls"> |
| <div class="field"> |
| <label for="series">Series</label> |
| <select id="series"></select> |
| </div> |
| <div class="field"> |
| <label for="detector">Detector</label> |
| <select id="detector"></select> |
| </div> |
| <button id="sample">Try a sample</button> |
| </div> |
| <div class="verdict" id="verdict"> |
| <div><b id="verdict-title">Loading…</b><span class="why" id="verdict-why"></span></div> |
| </div> |
| </div> |
|
|
| <div class="card"> |
| <div class="legend"> |
| <span><i class="swatch" style="background:#2a78d6;opacity:.35"></i>ground-truth anomaly window</span> |
| <span><i class="swatch" style="background:#eb6834"></i>alarm raised</span> |
| <span><i class="swatch" style="background:#66748a"></i>signal & anomaly score</span> |
| </div> |
| <canvas id="chart" height="360"></canvas> |
| <div class="slider-row"> |
| <input type="range" id="threshold" min="0" max="1000" value="500"> |
| <span class="thr" id="thr-label">threshold —</span> |
| <div class="presets"> |
| <button id="opt-pa" title="The threshold that maximises point-adjusted F1">PA optimum</button> |
| <button id="opt-comp" title="The threshold that maximises composite F1">Composite optimum</button> |
| </div> |
| </div> |
| </div> |
|
|
| <div class="card"> |
| <div class="metrics" id="metrics"></div> |
| <div class="ledger" id="ledger"></div> |
| </div> |
|
|
| <footer id="footer"></footer> |
| </div> |
|
|
| <script src="outliar.js"></script> |
| <script src="shim.js"></script> |
| <script> |
| const BLUE = "#2a78d6", ORANGE = "#eb6834"; |
| const $ = (id) => document.getElementById(id); |
| const params = new URLSearchParams(location.search); |
| |
| let state = { data: null, catalog: null, sampleIndex: 0, busy: false }; |
| |
| function ink() { |
| return getComputedStyle(document.body).getPropertyValue("--ink").trim(); |
| } |
| function muted() { |
| return getComputedStyle(document.body).getPropertyValue("--muted").trim(); |
| } |
| function lineColor() { |
| return getComputedStyle(document.body).getPropertyValue("--line").trim(); |
| } |
| |
| async function getJSON(url) { |
| const response = await fetch(url); |
| if (!response.ok) throw new Error(`${response.status} ${await response.text()}`); |
| return response.json(); |
| } |
| |
| |
| function draw() { |
| const d = state.data; |
| const canvas = $("chart"); |
| const ratio = window.devicePixelRatio || 1; |
| const width = canvas.clientWidth, height = 360; |
| canvas.width = width * ratio; |
| canvas.height = height * ratio; |
| canvas.style.height = height + "px"; |
| const ctx = canvas.getContext("2d"); |
| ctx.setTransform(ratio, 0, 0, ratio, 0, 0); |
| ctx.clearRect(0, 0, width, height); |
| if (!d) return; |
| |
| const padL = 8, padR = 8; |
| const topH = 210, gap = 26, botH = 96; |
| const plotW = width - padL - padR; |
| const n = d.values.length; |
| const x = (i) => padL + (i / Math.max(1, n - 1)) * plotW; |
| |
| |
| |
| const scale = n / d.n; |
| ctx.fillStyle = BLUE; |
| ctx.globalAlpha = 0.3; |
| for (const [s, e] of d.windows) { |
| const x0 = x(s * scale), x1 = x(e * scale); |
| ctx.fillRect(x0, 0, Math.max(1.5, x1 - x0), topH + gap + botH); |
| } |
| ctx.globalAlpha = 1; |
| |
| |
| const probX = x(d.probation * scale); |
| ctx.fillStyle = muted(); |
| ctx.globalAlpha = 0.09; |
| ctx.fillRect(padL, 0, probX - padL, topH + gap + botH); |
| ctx.globalAlpha = 1; |
| |
| const drawSeries = (arr, y0, h, color, lw) => { |
| let lo = Infinity, hi = -Infinity; |
| for (const v of arr) { if (v < lo) lo = v; if (v > hi) hi = v; } |
| if (!(hi > lo)) { hi = lo + 1; } |
| const y = (v) => y0 + h - ((v - lo) / (hi - lo)) * h; |
| ctx.beginPath(); |
| for (let i = 0; i < arr.length; i++) { |
| const px = x(i), py = y(arr[i]); |
| i ? ctx.lineTo(px, py) : ctx.moveTo(px, py); |
| } |
| ctx.strokeStyle = color; ctx.lineWidth = lw; ctx.stroke(); |
| return { y, lo, hi }; |
| }; |
| |
| |
| const sig = drawSeries(d.values, 0, topH, muted(), 0.8); |
| ctx.fillStyle = ORANGE; |
| for (let i = 0; i < n; i++) { |
| if (d.scores[i] >= d.threshold && i * scale >= d.probation) { |
| ctx.fillRect(x(i) - 0.6, topH - 8, 1.6, 8); |
| } |
| } |
| |
| |
| const y0 = topH + gap; |
| const sc = drawSeries(d.scores, y0, botH, muted(), 0.8); |
| const ty = Math.min(y0 + botH, Math.max(y0, sc.y(d.threshold))); |
| ctx.beginPath(); |
| ctx.setLineDash([5, 4]); |
| ctx.moveTo(padL, ty); ctx.lineTo(width - padR, ty); |
| ctx.strokeStyle = ORANGE; ctx.lineWidth = 1.4; ctx.stroke(); |
| ctx.setLineDash([]); |
| |
| ctx.fillStyle = muted(); |
| ctx.font = "11px -apple-system, system-ui, sans-serif"; |
| ctx.textAlign = "left"; |
| ctx.fillText("signal", padL + 2, 12); |
| |
| |
| |
| const capY = ty < y0 + 22 ? y0 + botH - 4 : y0 + 12; |
| ctx.fillText("anomaly score", padL + 2, capY); |
| ctx.textAlign = "right"; |
| const labY = Math.min(y0 + botH - 4, Math.max(y0 + 10, ty - 5)); |
| ctx.fillText("threshold", width - padR - 4, labY); |
| ctx.textAlign = "left"; |
| } |
| |
| |
| function metricCard(key, label, value, sub, cls) { |
| return `<div class="metric ${cls}"> |
| <div class="k">${label}</div> |
| <div class="v">${value.toFixed(3)}</div> |
| <div class="s">${sub}</div> |
| </div>`; |
| } |
| |
| function render() { |
| const d = state.data; |
| if (!d) return; |
| |
| $("verdict").className = "verdict" + (d.is_control ? " control" : ""); |
| $("verdict-title").textContent = |
| `${d.detector} — ${d.description}` + (d.is_control ? " · not a detector" : ""); |
| $("verdict-why").textContent = d.note || |
| (d.is_control |
| ? "This is a control. Any score it earns is a property of the metric, not of the method." |
| : "A genuine detector: it fits on the warm-up period only and never sees a label."); |
| |
| const m = d.metrics; |
| $("metrics").innerHTML = [ |
| metricCard("point", "Point-wise F1", m.point.f1, |
| `P ${m.point.precision.toFixed(2)} · R ${m.point.recall.toFixed(2)}`, ""), |
| metricCard("pa", "Point-adjusted F1", m.pa.f1, |
| "the number papers report", "hero"), |
| metricCard("pa20", "PA%K, K=20", m.pa20.f1, |
| "needs >20% of the window", ""), |
| metricCard("composite", "Composite F1", m.composite.f1, |
| `precision ${m.composite.precision.toFixed(2)} · events ${d.windows_caught}/${d.windows_total}`, "good"), |
| ].join(""); |
| |
| const ratio = d.alarms_inside > 0 ? (d.pa_credited / d.alarms_inside) : 0; |
| $("ledger").innerHTML = |
| `<b>${d.alarms.toLocaleString()}</b> alarms raised · ` + |
| `<b>${d.alarms_inside.toLocaleString()}</b> landed inside a real anomaly window · ` + |
| `point adjustment credits <b>${d.pa_credited.toLocaleString()}</b> of them as true positives` + |
| (ratio > 1.5 ? ` — a <b>${ratio.toFixed(0)}×</b> markup` : "") + |
| ` · <b>${d.false_alarms_per_day.toFixed(1)}</b> false alarms per day.`; |
| |
| const range = d.score_range; |
| const pos = Math.round(1000 * (d.threshold - range[0]) / Math.max(1e-9, range[1] - range[0])); |
| $("threshold").value = Math.max(0, Math.min(1000, pos)); |
| $("thr-label").textContent = `threshold ${d.threshold.toFixed(3)}`; |
| draw(); |
| } |
| |
| |
| async function load(threshold) { |
| if (state.busy) return; |
| state.busy = true; |
| $("sample").disabled = true; |
| try { |
| const series = $("series").value, detector = $("detector").value; |
| let url = `/api/evaluate?series=${encodeURIComponent(series)}&detector=${detector}`; |
| if (threshold !== undefined && threshold !== null) url += `&threshold=${threshold}`; |
| state.data = await getJSON(url); |
| render(); |
| } catch (err) { |
| $("verdict-title").textContent = "Could not evaluate"; |
| $("verdict-why").innerHTML = `<span class="err">${err.message}</span>`; |
| } finally { |
| state.busy = false; |
| $("sample").disabled = false; |
| } |
| } |
| |
| async function loadSample() { |
| $("sample").disabled = true; |
| try { |
| state.data = await getJSON(`/sample?index=${state.sampleIndex++}`); |
| $("series").value = state.data.series; |
| $("detector").value = state.data.detector; |
| render(); |
| } catch (err) { |
| $("verdict-title").textContent = "No sample available"; |
| $("verdict-why").innerHTML = `<span class="err">${err.message}</span>`; |
| } finally { |
| $("sample").disabled = false; |
| } |
| } |
| |
| async function boot() { |
| state.catalog = await getJSON("/api/catalog"); |
| $("series").innerHTML = state.catalog.series |
| .map((s) => `<option value="${s.key}">${s.name} — ${s.corpus} (${s.windows} windows)</option>`) |
| .join(""); |
| $("detector").innerHTML = state.catalog.detectors |
| .map((d) => `<option value="${d.name}">${d.name}${d.is_control ? " (control)" : ""}</option>`) |
| .join(""); |
| |
| $("series").onchange = () => load(); |
| $("detector").onchange = () => load(); |
| $("sample").onclick = loadSample; |
| $("opt-pa").onclick = () => load(state.data && state.data.pa_optimal_threshold); |
| $("opt-comp").onclick = () => load(state.data && state.data.composite_optimal_threshold); |
| |
| $("threshold").oninput = (e) => { |
| if (!state.data) return; |
| const [lo, hi] = state.data.score_range; |
| state.data.threshold = lo + (e.target.value / 1000) * (hi - lo); |
| $("thr-label").textContent = `threshold ${state.data.threshold.toFixed(3)}`; |
| draw(); |
| }; |
| $("threshold").onchange = (e) => { |
| if (!state.data) return; |
| const [lo, hi] = state.data.score_range; |
| load(lo + (e.target.value / 1000) * (hi - lo)); |
| }; |
| window.addEventListener("resize", draw); |
| matchMedia("(prefers-color-scheme: dark)").addEventListener("change", draw); |
| |
| try { |
| const f = await getJSON("/api/findings"); |
| const pa = f.noise_rank.pa, comp = f.noise_rank.composite; |
| $("footer").innerHTML = |
| `Across all ${f.n_series} real NAB series: uniform noise scores ` + |
| `<b>${pa.noise_f1.toFixed(3)}</b> point-adjusted F1 (rank ${pa.noise_rank}/${pa.n_detectors}) ` + |
| `and <b>${comp.noise_f1.toFixed(3)}</b> composite F1 (rank ${comp.noise_rank}/${comp.n_detectors}).`; |
| } catch { |
| $("footer").textContent = "Run `make benchmark` to populate the benchmark-wide summary."; |
| } |
| |
| await loadSample(); |
| if (params.get("demo") === "1") { |
| |
| const n = parseInt(params.get("sample") || "0", 10); |
| state.sampleIndex = n; |
| await loadSample(); |
| } |
| } |
| |
| boot(); |
| </script> |
| </body> |
| </html> |
|
|
