Spaces:

usmar
/

outliar

Running

File size: 15,666 Bytes

<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>outliar — the metric is the bug</title>
<style>
  :root {
    --blue: #2a78d6;
    --orange: #eb6834;
    --ink: #14202c;
    --muted: #66748a;
    --bg: #f6f8fb;
    --card: #ffffff;
    --line: #e2e8f1;
    --shadow: 0 1px 2px rgba(20, 32, 44, .06), 0 8px 24px rgba(20, 32, 44, .06);
  }
  @media (prefers-color-scheme: dark) {
    :root {
      --ink: #e8edf4;
      --muted: #93a1b5;
      --bg: #0f151d;
      --card: #171f2a;
      --line: #26313f;
      --shadow: 0 1px 2px rgba(0, 0, 0, .4), 0 8px 24px rgba(0, 0, 0, .3);
    }
  }
  * { box-sizing: border-box; }
  body {
    margin: 0; padding: 28px 20px 48px; background: var(--bg); color: var(--ink);
    font: 15px/1.55 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
    -webkit-font-smoothing: antialiased;
  }
  .wrap { max-width: 1080px; margin: 0 auto; }
  header { margin-bottom: 20px; }
  h1 { font-size: 25px; margin: 0 0 4px; letter-spacing: -.02em; }
  h1 span { color: var(--orange); }
  .tag { color: var(--muted); font-size: 14.5px; margin: 0; max-width: 66ch; }
  .card {
    background: var(--card); border: 1px solid var(--line); border-radius: 12px;
    box-shadow: var(--shadow); padding: 18px; margin-bottom: 16px;
  }
  .controls { display: flex; flex-wrap: wrap; gap: 12px; align-items: flex-end; }
  .field { display: flex; flex-direction: column; gap: 5px; min-width: 190px; flex: 1 1 200px; }
  label { font-size: 11.5px; text-transform: uppercase; letter-spacing: .07em; color: var(--muted); font-weight: 600; }
  select, button {
    font: inherit; color: var(--ink); background: var(--bg);
    border: 1px solid var(--line); border-radius: 8px; padding: 9px 11px;
  }
  select { width: 100%; }
  button {
    background: var(--blue); color: #fff; border-color: transparent; cursor: pointer;
    font-weight: 600; padding: 9px 16px; white-space: nowrap; transition: filter .15s;
  }
  button:hover { filter: brightness(1.08); }
  button:disabled { opacity: .55; cursor: default; }

  .verdict {
    display: flex; gap: 12px; align-items: flex-start;
    border-left: 3px solid var(--blue); padding: 10px 0 10px 14px; margin-top: 16px;
    font-size: 14px;
  }
  .verdict.control { border-left-color: var(--orange); }
  .verdict b { display: block; margin-bottom: 2px; }
  .verdict .why { color: var(--muted); }

  canvas { width: 100%; display: block; }
  .slider-row { display: flex; align-items: center; gap: 14px; margin-top: 14px; }
  .slider-row input[type=range] { flex: 1; accent-color: var(--orange); }
  .thr { font-variant-numeric: tabular-nums; font-size: 13px; color: var(--muted); min-width: 128px; }
  .presets { display: flex; gap: 8px; }
  .presets button { background: transparent; color: var(--muted); border: 1px solid var(--line); font-weight: 500; font-size: 12.5px; padding: 6px 10px; }
  .presets button:hover { color: var(--ink); filter: none; border-color: var(--muted); }

  .metrics { display: grid; grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); gap: 12px; }
  .metric { border: 1px solid var(--line); border-radius: 10px; padding: 12px 13px; background: var(--bg); }
  .metric.hero { border-color: var(--orange); background: color-mix(in srgb, var(--orange) 8%, var(--card)); }
  .metric.good { border-color: var(--blue); background: color-mix(in srgb, var(--blue) 8%, var(--card)); }
  .metric .k { font-size: 11px; text-transform: uppercase; letter-spacing: .06em; color: var(--muted); font-weight: 600; }
  .metric .v { font-size: 27px; font-weight: 650; font-variant-numeric: tabular-nums; letter-spacing: -.02em; margin: 3px 0 1px; }
  .metric .s { font-size: 12px; color: var(--muted); font-variant-numeric: tabular-nums; }

  .ledger { margin-top: 16px; padding-top: 14px; border-top: 1px solid var(--line); font-size: 13.5px; color: var(--muted); }
  .ledger b { color: var(--ink); font-variant-numeric: tabular-nums; }
  .legend { display: flex; gap: 16px; flex-wrap: wrap; font-size: 12.5px; color: var(--muted); margin-bottom: 10px; }
  .swatch { display: inline-block; width: 11px; height: 11px; border-radius: 3px; vertical-align: -1px; margin-right: 5px; }
  footer { color: var(--muted); font-size: 12.5px; text-align: center; margin-top: 22px; }
  footer a { color: var(--blue); }
  .err { color: var(--orange); font-size: 13.5px; }
</style>
</head>
<body>

<div class="wrap">
  <header>
    <h1>outl<span>iar</span></h1>
    <p class="tag">
      Anomaly detection on the Numenta benchmark, scored four ways. Pick a detector — including
      one that is literally uniform noise — drag the threshold, and watch the metrics disagree
      about whether it works.
    </p>
  </header>

  <div class="card">
    <div class="controls">
      <div class="field">
        <label for="series">Series</label>
        <select id="series"></select>
      </div>
      <div class="field">
        <label for="detector">Detector</label>
        <select id="detector"></select>
      </div>
      <button id="sample">Try a sample</button>
    </div>
    <div class="verdict" id="verdict">
      <div><b id="verdict-title">Loading…</b><span class="why" id="verdict-why"></span></div>
    </div>
  </div>

  <div class="card">
    <div class="legend">
      <span><i class="swatch" style="background:#2a78d6;opacity:.35"></i>ground-truth anomaly window</span>
      <span><i class="swatch" style="background:#eb6834"></i>alarm raised</span>
      <span><i class="swatch" style="background:#66748a"></i>signal &amp; anomaly score</span>
    </div>
    <canvas id="chart" height="360"></canvas>
    <div class="slider-row">
      <input type="range" id="threshold" min="0" max="1000" value="500">
      <span class="thr" id="thr-label">threshold —</span>
      <div class="presets">
        <button id="opt-pa" title="The threshold that maximises point-adjusted F1">PA optimum</button>
        <button id="opt-comp" title="The threshold that maximises composite F1">Composite optimum</button>
      </div>
    </div>
  </div>

  <div class="card">
    <div class="metrics" id="metrics"></div>
    <div class="ledger" id="ledger"></div>
  </div>

  <footer id="footer"></footer>
</div>

<script src="outliar.js"></script>
<script src="shim.js"></script>
<script>
const BLUE = "#2a78d6", ORANGE = "#eb6834";
const $ = (id) => document.getElementById(id);
const params = new URLSearchParams(location.search);

let state = { data: null, catalog: null, sampleIndex: 0, busy: false };

function ink() {
  return getComputedStyle(document.body).getPropertyValue("--ink").trim();
}
function muted() {
  return getComputedStyle(document.body).getPropertyValue("--muted").trim();
}
function lineColor() {
  return getComputedStyle(document.body).getPropertyValue("--line").trim();
}

async function getJSON(url) {
  const response = await fetch(url);
  if (!response.ok) throw new Error(`${response.status} ${await response.text()}`);
  return response.json();
}

// ---------------------------------------------------------------- chart
function draw() {
  const d = state.data;
  const canvas = $("chart");
  const ratio = window.devicePixelRatio || 1;
  const width = canvas.clientWidth, height = 360;
  canvas.width = width * ratio;
  canvas.height = height * ratio;
  canvas.style.height = height + "px";
  const ctx = canvas.getContext("2d");
  ctx.setTransform(ratio, 0, 0, ratio, 0, 0);
  ctx.clearRect(0, 0, width, height);
  if (!d) return;

  const padL = 8, padR = 8;
  const topH = 210, gap = 26, botH = 96;
  const plotW = width - padL - padR;
  const n = d.values.length;
  const x = (i) => padL + (i / Math.max(1, n - 1)) * plotW;

  // Ground-truth windows, drawn first so everything else sits on top. Window
  // bounds are full-resolution indices; the drawn series is decimated.
  const scale = n / d.n;
  ctx.fillStyle = BLUE;
  ctx.globalAlpha = 0.3;
  for (const [s, e] of d.windows) {
    const x0 = x(s * scale), x1 = x(e * scale);
    ctx.fillRect(x0, 0, Math.max(1.5, x1 - x0), topH + gap + botH);
  }
  ctx.globalAlpha = 1;

  // Probation shading — nothing there is scored.
  const probX = x(d.probation * scale);
  ctx.fillStyle = muted();
  ctx.globalAlpha = 0.09;
  ctx.fillRect(padL, 0, probX - padL, topH + gap + botH);
  ctx.globalAlpha = 1;

  const drawSeries = (arr, y0, h, color, lw) => {
    let lo = Infinity, hi = -Infinity;
    for (const v of arr) { if (v < lo) lo = v; if (v > hi) hi = v; }
    if (!(hi > lo)) { hi = lo + 1; }
    const y = (v) => y0 + h - ((v - lo) / (hi - lo)) * h;
    ctx.beginPath();
    for (let i = 0; i < arr.length; i++) {
      const px = x(i), py = y(arr[i]);
      i ? ctx.lineTo(px, py) : ctx.moveTo(px, py);
    }
    ctx.strokeStyle = color; ctx.lineWidth = lw; ctx.stroke();
    return { y, lo, hi };
  };

  // Panel 1 — the signal, with an alarm tick under every flagged point.
  const sig = drawSeries(d.values, 0, topH, muted(), 0.8);
  ctx.fillStyle = ORANGE;
  for (let i = 0; i < n; i++) {
    if (d.scores[i] >= d.threshold && i * scale >= d.probation) {
      ctx.fillRect(x(i) - 0.6, topH - 8, 1.6, 8);
    }
  }

  // Panel 2 — the anomaly score and where the threshold cuts it.
  const y0 = topH + gap;
  const sc = drawSeries(d.scores, y0, botH, muted(), 0.8);
  const ty = Math.min(y0 + botH, Math.max(y0, sc.y(d.threshold)));
  ctx.beginPath();
  ctx.setLineDash([5, 4]);
  ctx.moveTo(padL, ty); ctx.lineTo(width - padR, ty);
  ctx.strokeStyle = ORANGE; ctx.lineWidth = 1.4; ctx.stroke();
  ctx.setLineDash([]);

  ctx.fillStyle = muted();
  ctx.font = "11px -apple-system, system-ui, sans-serif";
  ctx.textAlign = "left";
  ctx.fillText("signal", padL + 2, 12);
  // Drop the caption to the floor of the panel when the threshold line is
  // sitting where the caption would go — which is exactly what happens at the
  // point-adjusted optimum, i.e. in every interesting screenshot.
  const capY = ty < y0 + 22 ? y0 + botH - 4 : y0 + 12;
  ctx.fillText("anomaly score", padL + 2, capY);
  ctx.textAlign = "right";
  const labY = Math.min(y0 + botH - 4, Math.max(y0 + 10, ty - 5));
  ctx.fillText("threshold", width - padR - 4, labY);
  ctx.textAlign = "left";
}

// ---------------------------------------------------------------- render
function metricCard(key, label, value, sub, cls) {
  return `<div class="metric ${cls}">
    <div class="k">${label}</div>
    <div class="v">${value.toFixed(3)}</div>
    <div class="s">${sub}</div>
  </div>`;
}

function render() {
  const d = state.data;
  if (!d) return;

  $("verdict").className = "verdict" + (d.is_control ? " control" : "");
  $("verdict-title").textContent =
    `${d.detector} — ${d.description}` + (d.is_control ? "  ·  not a detector" : "");
  $("verdict-why").textContent = d.note ||
    (d.is_control
      ? "This is a control. Any score it earns is a property of the metric, not of the method."
      : "A genuine detector: it fits on the warm-up period only and never sees a label.");

  const m = d.metrics;
  $("metrics").innerHTML = [
    metricCard("point", "Point-wise F1", m.point.f1,
      `P ${m.point.precision.toFixed(2)} · R ${m.point.recall.toFixed(2)}`, ""),
    metricCard("pa", "Point-adjusted F1", m.pa.f1,
      "the number papers report", "hero"),
    metricCard("pa20", "PA%K, K=20", m.pa20.f1,
      "needs >20% of the window", ""),
    metricCard("composite", "Composite F1", m.composite.f1,
      `precision ${m.composite.precision.toFixed(2)} · events ${d.windows_caught}/${d.windows_total}`, "good"),
  ].join("");

  const ratio = d.alarms_inside > 0 ? (d.pa_credited / d.alarms_inside) : 0;
  $("ledger").innerHTML =
    `<b>${d.alarms.toLocaleString()}</b> alarms raised · ` +
    `<b>${d.alarms_inside.toLocaleString()}</b> landed inside a real anomaly window · ` +
    `point adjustment credits <b>${d.pa_credited.toLocaleString()}</b> of them as true positives` +
    (ratio > 1.5 ? ` — a <b>${ratio.toFixed(0)}×</b> markup` : "") +
    ` · <b>${d.false_alarms_per_day.toFixed(1)}</b> false alarms per day.`;

  const range = d.score_range;
  const pos = Math.round(1000 * (d.threshold - range[0]) / Math.max(1e-9, range[1] - range[0]));
  $("threshold").value = Math.max(0, Math.min(1000, pos));
  $("thr-label").textContent = `threshold ${d.threshold.toFixed(3)}`;
  draw();
}

// ---------------------------------------------------------------- data
async function load(threshold) {
  if (state.busy) return;
  state.busy = true;
  $("sample").disabled = true;
  try {
    const series = $("series").value, detector = $("detector").value;
    let url = `/api/evaluate?series=${encodeURIComponent(series)}&detector=${detector}`;
    if (threshold !== undefined && threshold !== null) url += `&threshold=${threshold}`;
    state.data = await getJSON(url);
    render();
  } catch (err) {
    $("verdict-title").textContent = "Could not evaluate";
    $("verdict-why").innerHTML = `<span class="err">${err.message}</span>`;
  } finally {
    state.busy = false;
    $("sample").disabled = false;
  }
}

async function loadSample() {
  $("sample").disabled = true;
  try {
    state.data = await getJSON(`/sample?index=${state.sampleIndex++}`);
    $("series").value = state.data.series;
    $("detector").value = state.data.detector;
    render();
  } catch (err) {
    $("verdict-title").textContent = "No sample available";
    $("verdict-why").innerHTML = `<span class="err">${err.message}</span>`;
  } finally {
    $("sample").disabled = false;
  }
}

async function boot() {
  state.catalog = await getJSON("/api/catalog");
  $("series").innerHTML = state.catalog.series
    .map((s) => `<option value="${s.key}">${s.name} — ${s.corpus} (${s.windows} windows)</option>`)
    .join("");
  $("detector").innerHTML = state.catalog.detectors
    .map((d) => `<option value="${d.name}">${d.name}${d.is_control ? " (control)" : ""}</option>`)
    .join("");

  $("series").onchange = () => load();
  $("detector").onchange = () => load();
  $("sample").onclick = loadSample;
  $("opt-pa").onclick = () => load(state.data && state.data.pa_optimal_threshold);
  $("opt-comp").onclick = () => load(state.data && state.data.composite_optimal_threshold);

  $("threshold").oninput = (e) => {
    if (!state.data) return;
    const [lo, hi] = state.data.score_range;
    state.data.threshold = lo + (e.target.value / 1000) * (hi - lo);
    $("thr-label").textContent = `threshold ${state.data.threshold.toFixed(3)}`;
    draw();
  };
  $("threshold").onchange = (e) => {
    if (!state.data) return;
    const [lo, hi] = state.data.score_range;
    load(lo + (e.target.value / 1000) * (hi - lo));
  };
  window.addEventListener("resize", draw);
  matchMedia("(prefers-color-scheme: dark)").addEventListener("change", draw);

  try {
    const f = await getJSON("/api/findings");
    const pa = f.noise_rank.pa, comp = f.noise_rank.composite;
    $("footer").innerHTML =
      `Across all ${f.n_series} real NAB series: uniform noise scores ` +
      `<b>${pa.noise_f1.toFixed(3)}</b> point-adjusted F1 (rank ${pa.noise_rank}/${pa.n_detectors}) ` +
      `and <b>${comp.noise_f1.toFixed(3)}</b> composite F1 (rank ${comp.noise_rank}/${comp.n_detectors}).`;
  } catch {
    $("footer").textContent = "Run `make benchmark` to populate the benchmark-wide summary.";
  }

  await loadSample();
  if (params.get("demo") === "1") {
    // Headless screenshots land on a populated, argument-making state.
    const n = parseInt(params.get("sample") || "0", 10);
    state.sampleIndex = n;
    await loadSample();
  }
}

boot();
</script>
</body>
</html>