Spaces:

SaylorTwift
/

llm-benchmark-usage-explorer

Running

File size: 39,575 Bytes

"""Explore the SaylorTwift/llm-benchmark-usage dataset: which benchmarks are popular,
when they emerged, which models (open/closed) use them, and how benchmark categories
have evolved over time.

The whole UI is built from `gr.HTML`: custom HTML input components (pills, select,
range, search) subclassed from gr.HTML, and server-rendered HTML/SVG for every
table, card, and chart. No Dataframe/Dropdown/Radio/Slider/Plot components.
"""

print("app.py: starting imports...", flush=True)
import html as html_lib
import math
from urllib.parse import urlparse

import gradio as gr
import pandas as pd

from data import CATEGORY_ORDER, load_data

print("app.py: imports done, loading data...", flush=True)
MODELS_DF, USAGE_DF = load_data()
print("app.py: data loaded, building UI...", flush=True)

ALL_BENCHMARKS = sorted(USAGE_DF["benchmark"].unique())
ALL_MODELS = sorted(USAGE_DF["model_id"].unique())
ALL_CATEGORIES = sorted(USAGE_DF["bucket"].unique())
PERIODS = sorted(MODELS_DF["period"].unique())


def esc(s) -> str:
    return html_lib.escape(str(s), quote=True)


# ---------------------------------------------------------------------------
# Styles. Category/openness colors are defined once as CSS variables (with
# dark-mode steps under `.dark`, which Gradio toggles on the page), and every
# chart mark and chip references them, so both themes come from one place.
# ---------------------------------------------------------------------------
CUSTOM_CSS = """
:root {
    --cat-knowledge: #2a78d6; --cat-coding: #1baf7a; --cat-agentic: #eda100;
    --cat-math: #008300; --cat-safety: #4a3aa7; --cat-vision: #e34948;
    --cat-multilingual: #e87ba4; --cat-long_context: #eb6834; --cat-other: #86847d;
    --open: #2a78d6; --closed: #e34948; --accent: #2a78d6;
}
/* Gradio sets the light theme variables on <html> and the dark ones on
   .gradio-container, and var() resolves where a property is DEFINED — so these
   aliases must live on .gradio-container to pick up the active theme. */
.gradio-container {
    --ink: var(--body-text-color, #0b0b0b);
    --muted: var(--body-text-color-subdued, #898781);
    --surf: var(--background-fill-primary, #fcfcfb);
    --hair: var(--border-color-primary, #e1e0d9);
    --grid: color-mix(in srgb, var(--hair) 55%, transparent);
}
.dark {
    --cat-knowledge: #3987e5; --cat-coding: #199e70; --cat-agentic: #c98500;
    --cat-math: #008300; --cat-safety: #9085e9; --cat-vision: #e66767;
    --cat-multilingual: #d55181; --cat-long_context: #d95926; --cat-other: #898781;
    --open: #3987e5; --closed: #e66767; --accent: #3987e5;
}

/* Gradio 6's theme cascades flex-grow:1 from .gradio-container down through
   .main.fillable.app -> .wrap -> main.contain, stretching the whole app shell
   to the viewport height regardless of actual content height. Force those
   containers back to their natural height. */
.gradio-container, .main.fillable.app, .wrap.svelte-zxu34v, main.contain.svelte-zxu34v {
    flex-grow: 0 !important;
    height: auto !important;
}

/* --- hero ------------------------------------------------------------- */
.hero h1 { font-size: 26px; margin: 4px 0 6px; color: var(--ink); }
.hero p { color: var(--muted); font-size: 14px; margin: 0; }
.hero a { color: var(--accent); }
.tiles { display: flex; gap: 12px; flex-wrap: wrap; margin-top: 14px; }
.tile { flex: 1; min-width: 150px; border: 1px solid var(--hair); border-radius: 10px;
        padding: 10px 14px; background: var(--surf); }
.tile .lab { font-size: 12px; color: var(--muted); }
.tile .val { font-size: 26px; font-weight: 600; color: var(--ink); }
.tile .val.sm { font-size: 15px; padding-top: 8px; }

/* --- controls ---------------------------------------------------------- */
.ctl { display: flex; flex-direction: column; gap: 6px; margin: 2px 0 8px; }
.ctl-label { font-size: 12px; font-weight: 600; color: var(--muted); }
.pillrow { display: flex; flex-wrap: wrap; gap: 6px; }
.pill { border: 1px solid var(--hair); background: transparent; color: var(--ink);
        border-radius: 999px; padding: 5px 12px; font-size: 12.5px; cursor: pointer; }
.pill:hover { border-color: var(--accent); }
.pill.on { background: var(--accent); border-color: var(--accent); color: #fff; }
.cbx-wrap { position: relative; max-width: 460px; }
input.cbx-input { width: 100%; box-sizing: border-box; padding: 7px 30px 7px 10px;
        border: 1px solid var(--hair); border-radius: 8px; background: var(--surf);
        color: var(--ink); font-size: 13px; }
input.cbx-input:focus { outline: none; border-color: var(--accent); }
.cbx-caret { position: absolute; right: 10px; top: 50%; transform: translateY(-50%);
        color: var(--muted); pointer-events: none; font-size: 11px; }
.cbx-list { position: absolute; top: calc(100% + 4px); left: 0; right: 0; max-height: 320px;
        overflow: auto; background: var(--surf); border: 1px solid var(--hair);
        border-radius: 8px; z-index: 100; box-shadow: 0 6px 18px rgba(0, 0, 0, .15); }
.cbx-item { padding: 7px 10px; font-size: 13px; color: var(--ink); cursor: pointer; }
.cbx-item:hover, .cbx-item.active { background: color-mix(in srgb, var(--accent) 14%, transparent); }
.cbx-item.sel { font-weight: 600; }
.cbx-empty { padding: 8px 10px; color: var(--muted); font-size: 12.5px; }
.sliderrow { display: flex; align-items: center; gap: 10px; }
input.ctl-range { accent-color: var(--accent); width: 200px; max-width: 100%; }
.ctl-range-val { font-size: 13px; font-weight: 600; color: var(--ink); min-width: 2ch; }
input.ctl-search, input.tbl-filter { padding: 7px 10px; border: 1px solid var(--hair);
        border-radius: 8px; background: var(--surf); color: var(--ink); font-size: 13px; width: 230px; }

/* --- cards, chips, notes ------------------------------------------------ */
.tab-note { color: var(--muted); font-size: 13.5px; margin: 2px 0 10px; }
.card { border: 1px solid var(--hair); border-radius: 10px; padding: 12px 16px;
        background: var(--surf); margin: 4px 0 10px; }
.card-title { font-size: 17px; font-weight: 600; color: var(--ink); margin-bottom: 4px; }
.card-sub { color: var(--muted); font-size: 13px; line-height: 1.7; }
.card-sub strong { color: var(--ink); font-weight: 600; }
.card-sub a { color: var(--accent); }
.chip { display: inline-flex; align-items: center; gap: 5px; padding: 2px 9px;
        border: 1px solid var(--hair); border-radius: 999px; font-size: 11.5px;
        margin: 2px 4px 2px 0; color: var(--muted); white-space: nowrap; }
.chip i { width: 8px; height: 8px; border-radius: 50%; display: inline-block; flex: none; }
.rel-hd { display: flex; align-items: center; gap: 8px; flex-wrap: wrap; }
.rel-name { font-size: 15px; font-weight: 600; color: var(--ink); }
.rel-meta { color: var(--muted); font-size: 12.5px; margin-left: auto; }
.rel-chips { margin-top: 8px; }
.rel-src { color: var(--muted); font-size: 12.5px; margin-top: 6px; }
.rel-src a { color: var(--accent); }
.cat-sec { margin: 10px 0 14px; }
.cat-hd { display: flex; align-items: center; gap: 7px; font-size: 13.5px;
          font-weight: 600; color: var(--ink); margin-bottom: 6px; }
.cat-hd i { width: 10px; height: 10px; border-radius: 50%; display: inline-block; }
.cat-hd .n { color: var(--muted); font-weight: 400; }

/* --- tables ------------------------------------------------------------- */
.tbl-tools { margin: 2px 0 8px; }
.tbl-wrap { max-height: 480px; overflow: auto; border: 1px solid var(--hair); border-radius: 8px; }
.tbl { width: 100%; border-collapse: collapse; font-size: 13px; color: var(--ink); }
.tbl thead th { position: sticky; top: 0; z-index: 1; background: var(--surf);
        text-align: left; padding: 8px 10px; border-bottom: 1px solid var(--hair);
        color: var(--muted); font-weight: 600; white-space: nowrap; }
.tbl td { padding: 7px 10px; border-bottom: 1px solid var(--hair); vertical-align: middle; }
.tbl tbody tr:last-child td { border-bottom: none; }
.tbl tbody tr:hover { background: color-mix(in srgb, var(--ink) 5%, transparent); }
.tbl .num { font-variant-numeric: tabular-nums; text-align: right; }
.tbl .mut { color: var(--muted); font-size: 12.5px; white-space: nowrap; }
.tbl a { color: var(--accent); text-decoration: none; }
.tbl a:hover { text-decoration: underline; }
.match-note { color: var(--muted); font-size: 12.5px; margin: 6px 2px; }

.cellbar { display: inline-block; vertical-align: middle; width: 110px; height: 8px;
        background: color-mix(in srgb, var(--ink) 7%, transparent); border-radius: 4px;
        overflow: hidden; margin-left: 8px; }
.cellbar span { display: block; height: 100%; background: var(--accent);
        border-radius: 0 4px 4px 0; }
.splitbar { display: inline-flex; vertical-align: middle; width: 90px; height: 8px;
        gap: 2px; margin-left: 8px; }
.splitbar .o { background: var(--open); height: 100%; border-radius: 4px 0 0 4px; }
.splitbar .c { background: var(--closed); height: 100%; border-radius: 0 4px 4px 0; }

/* --- charts ------------------------------------------------------------- */
.viz-title { font-size: 14px; font-weight: 600; color: var(--ink); margin: 8px 2px 2px; }
.legend { display: flex; gap: 16px; flex-wrap: wrap; margin: 6px 2px 8px;
          font-size: 12px; color: var(--muted); }
.legend .key { display: inline-flex; gap: 6px; align-items: center; }
.legend .key i { width: 10px; height: 10px; border-radius: 3px; display: inline-block; }
.legend .key i.round { border-radius: 50%; }
.viz { width: 100%; height: auto; display: block; }
.viz text { font-family: system-ui, -apple-system, "Segoe UI", sans-serif; }
.viz .dot { transition: r .1s; }
.viz .dot:hover { r: 7.5px; }
.viz .seg:hover { opacity: .82; }
"""


# ---------------------------------------------------------------------------
# Custom HTML input components (replace Radio / Dropdown / Slider / Textbox)
# ---------------------------------------------------------------------------
class PillGroup(gr.HTML):
    """Single-select pill buttons (replaces gr.Radio)."""

    def __init__(self, options, value, label, **kwargs):
        super().__init__(
            value=value,
            options=list(options),
            label_text=label,
            html_template="""
<div class="ctl">
  <div class="ctl-label">${label_text}</div>
  <div class="pillrow">
    ${options.map(o => `<button type="button" class="pill${o === value ? ' on' : ''}" data-v="${o}">${o}</button>`).join('')}
  </div>
</div>""",
            js_on_load="""
element.addEventListener('click', (e) => {
  const btn = e.target.closest('.pill');
  if (!btn) return;
  props.value = btn.dataset.v;
  trigger('change');
});""",
            **kwargs,
        )

    def api_info(self):
        return {"type": "string"}


class SelectBox(gr.HTML):
    """Type-to-filter combobox (replaces gr.Dropdown with filterable=True):
    a text input that opens a filtered option list, with arrow/Enter/Escape
    keyboard support. All listeners are delegated on the component root so
    they survive the re-render that follows each value sync."""

    def __init__(self, options, value, label, **kwargs):
        super().__init__(
            value=value,
            options=list(options),
            label_text=label,
            html_template="""
<div class="ctl">
  <div class="ctl-label">${label_text}</div>
  <div class="cbx-wrap">
    <input type="text" class="cbx-input" role="combobox" aria-expanded="false"
           autocomplete="off" spellcheck="false" placeholder="Type to search…"
           value="${value.replace(/&/g,'&amp;').replace(/"/g,'&quot;')}">
    <span class="cbx-caret">&#9662;</span>
    <div class="cbx-list" hidden>
      ${options.map(o => `<div class="cbx-item${o === value ? ' sel' : ''}" data-v="${o.replace(/&/g,'&amp;').replace(/"/g,'&quot;')}">${o.replace(/&/g,'&amp;').replace(/</g,'&lt;')}</div>`).join('')}
      <div class="cbx-empty" hidden>No match</div>
    </div>
  </div>
</div>""",
            js_on_load="""
const list = () => element.querySelector('.cbx-list');
const input = () => element.querySelector('.cbx-input');
const items = () => [...element.querySelectorAll('.cbx-item')];
const visible = () => items().filter(it => !it.hidden);

function open(showAll) {
  if (showAll) { items().forEach(it => { it.hidden = false; }); element.querySelector('.cbx-empty').hidden = true; }
  list().hidden = false;
  input().setAttribute('aria-expanded', 'true');
}
function close(restore) {
  list().hidden = true;
  input().setAttribute('aria-expanded', 'false');
  items().forEach(it => it.classList.remove('active'));
  if (restore) input().value = props.value;
}
function applyFilter() {
  const q = input().value.toLowerCase();
  let any = false;
  items().forEach(it => {
    it.hidden = !it.textContent.toLowerCase().includes(q);
    it.classList.remove('active');
    if (!it.hidden) any = true;
  });
  element.querySelector('.cbx-empty').hidden = any;
}
function move(dir) {
  const vis = visible();
  if (!vis.length) return;
  const cur = vis.findIndex(it => it.classList.contains('active'));
  const next = Math.min(Math.max(cur + dir, 0), vis.length - 1);
  vis.forEach(it => it.classList.remove('active'));
  vis[next].classList.add('active');
  vis[next].scrollIntoView({ block: 'nearest' });
}
function pick(v) {
  close(false);
  props.value = v;
  trigger('change');
}

element.addEventListener('focusin', (e) => {
  if (!e.target.classList.contains('cbx-input')) return;
  e.target.select();
  open(true);
});
element.addEventListener('input', (e) => {
  if (!e.target.classList.contains('cbx-input')) return;
  open(false);
  applyFilter();
});
element.addEventListener('keydown', (e) => {
  if (!e.target.classList.contains('cbx-input')) return;
  if (e.key === 'ArrowDown') { e.preventDefault(); if (list().hidden) open(true); move(1); }
  else if (e.key === 'ArrowUp') { e.preventDefault(); move(-1); }
  else if (e.key === 'Enter') {
    e.preventDefault();
    const target = visible().find(it => it.classList.contains('active')) || visible()[0];
    if (target) pick(target.dataset.v);
  }
  else if (e.key === 'Escape') { close(true); e.target.blur(); }
});
element.addEventListener('pointerdown', (e) => {
  const it = e.target.closest('.cbx-item');
  if (!it) return;
  e.preventDefault();  // beat the input's blur
  pick(it.dataset.v);
});
element.addEventListener('focusout', (e) => {
  if (!element.contains(e.relatedTarget)) close(true);
});""",
            **kwargs,
        )

    def api_info(self):
        return {"type": "string"}


class RangeSlider(gr.HTML):
    """Native range input with a value readout (replaces gr.Slider)."""

    def __init__(self, minimum, maximum, value, step, label, **kwargs):
        super().__init__(
            value=value,
            minimum=minimum,
            maximum=maximum,
            step=step,
            label_text=label,
            html_template="""
<div class="ctl">
  <div class="ctl-label">${label_text}</div>
  <div class="sliderrow">
    <input type="range" class="ctl-range" min="${minimum}" max="${maximum}" step="${step}" value="${value}">
    <span class="ctl-range-val">${value}</span>
  </div>
</div>""",
            js_on_load="""
element.addEventListener('input', (e) => {
  if (!e.target.classList.contains('ctl-range')) return;
  const out = element.querySelector('.ctl-range-val');
  if (out) out.textContent = e.target.value;
});
element.addEventListener('change', (e) => {
  if (!e.target.classList.contains('ctl-range')) return;
  props.value = Number(e.target.value);
  trigger('change');
});""",
            **kwargs,
        )

    def api_info(self):
        return {"type": "integer"}


class SearchBox(gr.HTML):
    """Debounced text search (replaces gr.Textbox). Focus is restored after the
    value-sync re-render so typing isn't interrupted."""

    def __init__(self, value, label, placeholder, **kwargs):
        super().__init__(
            value=value,
            label_text=label,
            placeholder=placeholder,
            html_template="""
<div class="ctl">
  <div class="ctl-label">${label_text}</div>
  <input type="search" class="ctl-search" placeholder="${placeholder}" value="${value.replace(/&/g,'&amp;').replace(/"/g,'&quot;')}">
</div>""",
            js_on_load="""
let t = null;
element.addEventListener('input', (e) => {
  if (!e.target.classList.contains('ctl-search')) return;
  clearTimeout(t);
  const v = e.target.value;
  t = setTimeout(() => {
    props.value = v;
    trigger('change');
    setTimeout(() => {
      const inp = element.querySelector('.ctl-search');
      if (inp && document.activeElement !== inp) {
        inp.focus();
        inp.setSelectionRange(inp.value.length, inp.value.length);
      }
    }, 60);
  }, 250);
});""",
            **kwargs,
        )

    def api_info(self):
        return {"type": "string"}


# Client-side row filter for rendered tables: filters <tbody> rows against the
# .tbl-filter input, no server round trip. Attached via js_on_load so the
# delegated listener survives value updates.
TABLE_FILTER_JS = """
element.addEventListener('input', (e) => {
  if (!e.target.classList.contains('tbl-filter')) return;
  const q = e.target.value.toLowerCase();
  element.querySelectorAll('tbody tr').forEach(tr => {
    tr.style.display = tr.textContent.toLowerCase().includes(q) ? '' : 'none';
  });
});"""


# ---------------------------------------------------------------------------
# Small HTML helpers
# ---------------------------------------------------------------------------
def cat_chip(bucket: str) -> str:
    return f'<span class="chip"><i style="background:var(--cat-{esc(bucket)})"></i>{esc(bucket)}</span>'


def openness_chip(is_open: bool) -> str:
    color, text = ("var(--open)", "open") if is_open else ("var(--closed)", "closed")
    return f'<span class="chip"><i style="background:{color}"></i>{text}</span>'


def source_label(url: str) -> str:
    """The dataset no longer carries a source title/type, just a link — show its
    host as a short, still-meaningful label (e.g. "arxiv.org", "huggingface.co")."""
    return urlparse(url).netloc or url


def nice_ceil(v: float) -> float:
    if v <= 0:
        return 1
    exp = 10 ** math.floor(math.log10(v))
    frac = v / exp
    for m in (1, 2, 2.5, 5, 10):
        if frac <= m:
            return m * exp
    return v


def month_ticks(t0: pd.Timestamp, t1: pd.Timestamp) -> list[pd.Timestamp]:
    span = max((t1 - t0).days, 1)
    step = next((s for s in (1, 2, 3, 6, 12, 24) if span / (30.4 * s) <= 7), 24)
    m0 = ((t0.month - 1) // step) * step + 1
    d = pd.Timestamp(year=t0.year, month=m0, day=1)
    ticks = []
    while d <= t1:
        if d >= t0:
            ticks.append(d)
        total = (d.month - 1) + step
        d = pd.Timestamp(year=d.year + total // 12, month=total % 12 + 1, day=1)
    return ticks


# ---------------------------------------------------------------------------
# Hero header
# ---------------------------------------------------------------------------
def render_hero() -> str:
    d0, d1 = MODELS_DF["release_date"].min().date(), MODELS_DF["release_date"].max().date()
    tiles = [
        ("Models", str(len(MODELS_DF)), ""),
        ("Labs", str(MODELS_DF["lab"].nunique()), ""),
        ("Distinct benchmarks", str(USAGE_DF["benchmark"].nunique()), ""),
        ("Coverage", f"{d0} → {d1}", "sm"),
    ]
    tiles_html = "".join(
        f'<div class="tile"><div class="lab">{esc(lab)}</div><div class="val {cls}">{esc(val)}</div></div>'
        for lab, val, cls in tiles
    )
    return (
        '<div class="hero"><h1>LLM Benchmark Usage Explorer</h1>'
        '<p>Exploring <a href="https://huggingface.co/datasets/SaylorTwift/llm-benchmark-usage" '
        'target="_blank">SaylorTwift/llm-benchmark-usage</a> — which benchmarks labs report, '
        "who uses them, and how the mix shifts over time.</p>"
        f'<div class="tiles">{tiles_html}</div></div>'
    )


# ---------------------------------------------------------------------------
# Tab 0: Latest releases (landing view)
# ---------------------------------------------------------------------------
def rel_date(d: pd.Timestamp) -> str:
    days = (pd.Timestamp.now().normalize() - d.normalize()).days
    if days <= 0:
        return "today"
    if days == 1:
        return "yesterday"
    if days < 7:
        return f"{days} days ago"
    if days < 60:
        return f"{days // 7} week{'s' if days >= 14 else ''} ago"
    if days < 365:
        return f"{days // 30} months ago"
    return f"{days // 365} year{'s' if days >= 730 else ''} ago"


MAX_RELEASE_CHIPS = 16


def latest_releases_view(count) -> str:
    recent = MODELS_DF.sort_values("release_date", ascending=False).head(int(count))
    bucket_rank = {b: i for i, b in enumerate(CATEGORY_ORDER)}
    cards = []
    for m in recent.itertuples():
        usage = (
            USAGE_DF[USAGE_DF["model_id"] == m.model_id]
            .drop_duplicates(subset=["benchmark"])
            .sort_values(["bucket", "benchmark"], key=lambda s: s.map(bucket_rank) if s.name == "bucket" else s)
        )
        if usage.empty:
            body = '<div class="card-sub">No benchmark data recorded for this model.</div>'
        else:
            shown = usage.head(MAX_RELEASE_CHIPS)
            chips = "".join(
                f'<span class="chip" title="{esc(r.bucket)} · reported category: {esc(r.category)}">'
                f'<i style="background:var(--cat-{esc(r.bucket)})"></i>{esc(r.benchmark)}</span>'
                for r in shown.itertuples()
            )
            if len(usage) > MAX_RELEASE_CHIPS:
                chips += f'<span class="chip">+{len(usage) - MAX_RELEASE_CHIPS} more</span>'
            src_url = usage.iloc[0]["source"]
            body = (
                f'<div class="rel-chips">{chips}</div>'
                f'<div class="rel-src">{len(usage)} benchmarks · Source: '
                f'<a href="{esc(src_url)}" target="_blank">{esc(source_label(src_url))}</a></div>'
            )
        cards.append(
            f'<div class="card"><div class="rel-hd">'
            f'<span class="rel-name">{esc(m.model_id)}</span>'
            f"{openness_chip(bool(m.is_open))}"
            f'<span class="rel-meta">{esc(m.lab)} · {m.release_date.date()} · {rel_date(m.release_date)}</span>'
            f"</div>{body}</div>"
        )
    return "".join(cards)


# ---------------------------------------------------------------------------
# Tab 1: Benchmark popularity
# ---------------------------------------------------------------------------
def popularity_view(category_filter, openness_filter, min_models, search) -> str:
    df = USAGE_DF
    if category_filter and category_filter != "All":
        df = df[df["bucket"] == category_filter]
    if openness_filter == "Open only":
        df = df[df["is_open"]]
    elif openness_filter == "Closed only":
        df = df[~df["is_open"]]

    # A model can appear several times for one benchmark (multiple sources report
    # it), so count on deduplicated (benchmark, model) pairs; categories keep all rows.
    uniq = df.drop_duplicates(subset=["benchmark", "model_id"])
    agg = (
        uniq.groupby("benchmark")
        .agg(
            models=("model_id", "nunique"),
            open_models=("is_open", "sum"),
            first_seen=("release_date", "min"),
            last_seen=("release_date", "max"),
            labs=("lab", "nunique"),
        )
        .reset_index()
    )
    buckets = df.groupby("benchmark")["bucket"].agg(lambda s: sorted(set(s))).rename("buckets")
    agg = agg.merge(buckets, on="benchmark")
    agg["closed_models"] = agg["models"] - agg["open_models"]
    agg = agg[agg["models"] >= int(min_models)]
    if search:
        agg = agg[agg["benchmark"].str.contains(search, case=False, na=False, regex=False)]
    agg = agg.sort_values(["models", "benchmark"], ascending=[False, True])

    if agg.empty:
        return '<div class="card"><div class="card-sub">No benchmarks match the current filters.</div></div>'

    max_models = int(agg["models"].max())
    rows = []
    for i, r in enumerate(agg.itertuples(), start=1):
        bar_pct = r.models / max_models * 100
        tot = max(r.models, 1)
        o_pct, c_pct = r.open_models / tot * 100, r.closed_models / tot * 100
        chips = "".join(cat_chip(b) for b in r.buckets)
        rows.append(
            f"<tr>"
            f'<td class="num mut">{i}</td>'
            f"<td><strong>{esc(r.benchmark)}</strong></td>"
            f'<td class="num">{r.models}<span class="cellbar"><span style="width:{bar_pct:.1f}%"></span></span></td>'
            f'<td class="num">{int(r.open_models)} / {int(r.closed_models)}'
            f'<span class="splitbar" title="{int(r.open_models)} open · {int(r.closed_models)} closed">'
            f'<span class="o" style="width:{o_pct:.0f}%"></span><span class="c" style="width:{c_pct:.0f}%"></span></span></td>'
            f'<td class="num">{r.labs}</td>'
            f"<td>{chips}</td>"
            f'<td class="mut">{r.first_seen.date()}</td>'
            f'<td class="mut">{r.last_seen.date()}</td>'
            f"</tr>"
        )
    return (
        f'<div class="match-note">{len(agg)} benchmarks match · sorted by number of models</div>'
        '<div class="tbl-wrap"><table class="tbl"><thead><tr>'
        "<th>#</th><th>Benchmark</th><th>Models</th><th>Open / closed</th><th>Labs</th>"
        "<th>Categories</th><th>First seen</th><th>Last seen</th>"
        f'</tr></thead><tbody>{"".join(rows)}</tbody></table></div>'
    )


# ---------------------------------------------------------------------------
# Tab 2: Benchmark -> Models (summary card, monthly-usage SVG bars, table)
# ---------------------------------------------------------------------------
def _rounded_top_rect(x, y, w, h, r) -> str:
    r = min(r, h / 2, w / 2)
    return (
        f'M {x:.1f} {y + h:.1f} L {x:.1f} {y + r:.1f} Q {x:.1f} {y:.1f} {x + r:.1f} {y:.1f} '
        f'L {x + w - r:.1f} {y:.1f} Q {x + w:.1f} {y:.1f} {x + w:.1f} {y + r:.1f} L {x + w:.1f} {y + h:.1f} Z'
    )


def svg_benchmark_weekly_usage(df: pd.DataFrame) -> str:
    """Bars of how many model releases reported this benchmark each week, spanning
    the whole dataset time range (empty weeks stay visible as gaps)."""
    t0 = MODELS_DF["release_date"].min().to_period("W")
    t1 = MODELS_DF["release_date"].max().to_period("W")
    weeks = pd.period_range(t0, t1, freq="W")
    counts = df["release_date"].dt.to_period("W").value_counts().reindex(weeks, fill_value=0)

    W, H, ML, MR, MT, MB = 920, 300, 44, 10, 12, 36
    plot_h = H - MT - MB
    base = MT + plot_h
    vmax = int(counts.max())
    step = max(1, math.ceil(vmax / 4))
    ymax = step * 4
    scale = plot_h / ymax
    band = (W - ML - MR) / len(weeks)
    bw = max(band - 1, 1.0)  # 1px surface gap between adjacent bars (weekly bands are narrow)

    parts = [
        f'<svg class="viz" viewBox="0 0 {W} {H}" role="img" '
        f'aria-label="Weekly count of model releases reporting this benchmark, over the whole dataset period">'
    ]
    for k in range(5):
        v = step * k
        y = base - v * scale
        parts.append(f'<line x1="{ML}" y1="{y:.1f}" x2="{W - MR}" y2="{y:.1f}" stroke="var(--grid)" stroke-width="1"/>')
        parts.append(f'<text x="{ML - 7}" y="{y + 4:.1f}" text-anchor="end" font-size="11" fill="var(--muted)">{v}</text>')

    week0_start = weeks[0].start_time
    for tick in month_ticks(week0_start, weeks[-1].end_time):
        idx = (tick - week0_start).days // 7
        if 0 <= idx < len(weeks):
            cx = ML + idx * band + band / 2
            parts.append(
                f'<text x="{cx:.1f}" y="{base + 20}" text-anchor="middle" font-size="11" fill="var(--muted)">'
                f"{tick.strftime('%b %Y')}</text>"
            )
    for i, week in enumerate(weeks):
        n = int(counts.loc[week])
        if n == 0:
            continue
        h = n * scale
        label = f"Week of {week.start_time.strftime('%b %-d, %Y')}: {n} model{'s' if n != 1 else ''}"
        parts.append(
            f'<path class="seg" d="{_rounded_top_rect(ML + i * band + 0.5, base - h, bw, h, 2)}" fill="var(--accent)">'
            f"<title>{esc(label)}</title></path>"
        )
    parts.append(f'<line x1="{ML}" y1="{base}" x2="{W - MR}" y2="{base}" stroke="var(--muted)" stroke-width="1"/>')
    parts.append("</svg>")
    return "".join(parts)


def benchmark_view(benchmark):
    if not benchmark:
        return "", "", '<div class="card"><div class="card-sub">Pick a benchmark above.</div></div>'
    df = (
        USAGE_DF[USAGE_DF["benchmark"] == benchmark]
        .drop_duplicates(subset=["model_id"])
        .sort_values("release_date")
    )
    n_open = int(df["is_open"].sum())
    n_closed = len(df) - n_open

    summary = (
        f'<div class="card"><div class="card-title">{esc(benchmark)}</div>'
        f'<div class="card-sub"><strong>{len(df)} models</strong> from '
        f"<strong>{df['lab'].nunique()} labs</strong> report it — "
        f"{n_open} open-weight, {n_closed} closed. First seen "
        f"<strong>{df['release_date'].min().date()}</strong>, most recent "
        f"<strong>{df['release_date'].max().date()}</strong>.</div></div>"
    )

    chart = (
        f'<div class="viz-title">How often is “{esc(benchmark)}” used? Model releases reporting it per week</div>'
        + svg_benchmark_weekly_usage(df)
    )

    rows = []
    for r in df.itertuples():
        rows.append(
            "<tr>"
            f"<td><strong>{esc(r.model_id)}</strong></td>"
            f"<td>{esc(r.lab)}</td>"
            f'<td class="mut">{r.release_date.date()}</td>'
            f"<td>{openness_chip(r.is_open)}</td>"
            f'<td class="mut">{esc(r.category)}</td>'
            f'<td class="mut"><a href="{esc(r.source)}" target="_blank">{esc(source_label(r.source))}</a></td>'
            "</tr>"
        )
    table = (
        '<div class="tbl-tools"><input class="tbl-filter" placeholder="Filter rows…"></div>'
        '<div class="tbl-wrap"><table class="tbl"><thead><tr>'
        "<th>Model</th><th>Lab</th><th>Released</th><th>Openness</th><th>Reported category</th><th>Source</th>"
        f'</tr></thead><tbody>{"".join(rows)}</tbody></table></div>'
    )
    return summary, chart, table


# ---------------------------------------------------------------------------
# Tab 3: Model -> Benchmarks (summary card + benchmarks grouped by category)
# ---------------------------------------------------------------------------
def model_view(model_id):
    if not model_id:
        return "", '<div class="card"><div class="card-sub">Pick a model above.</div></div>'
    df = (
        USAGE_DF[USAGE_DF["model_id"] == model_id]
        .drop_duplicates(subset=["benchmark", "bucket"])
        .sort_values(["bucket", "benchmark"])
    )
    if df.empty:
        return "", '<div class="card"><div class="card-sub">No data for this model.</div></div>'
    meta = MODELS_DF[MODELS_DF["model_id"] == model_id].iloc[0]
    src = df.iloc[0]

    summary = (
        f'<div class="card"><div class="card-title">{esc(model_id)} {openness_chip(bool(meta["is_open"]))}</div>'
        f'<div class="card-sub"><strong>{esc(meta["lab"])}</strong>, released '
        f'<strong>{meta["release_date"].date()}</strong> · <strong>{df["benchmark"].nunique()} benchmarks</strong> '
        f"across {df['bucket'].nunique()} categories · Source: "
        f'<a href="{esc(src["source"])}" target="_blank">{esc(source_label(src["source"]))}</a></div></div>'
    )

    sections = []
    for bucket in CATEGORY_ORDER:
        sub = df[df["bucket"] == bucket]
        if sub.empty:
            continue
        chips = "".join(
            f'<span class="chip" title="reported category: {esc(r.category)}">'
            f'<i style="background:var(--cat-{esc(bucket)})"></i>{esc(r.benchmark)}</span>'
            for r in sub.itertuples()
        )
        sections.append(
            f'<div class="cat-sec"><div class="cat-hd"><i style="background:var(--cat-{esc(bucket)})"></i>'
            f'{esc(bucket)} <span class="n">({len(sub)})</span></div><div>{chips}</div></div>'
        )
    return summary, "".join(sections)


# ---------------------------------------------------------------------------
# Tab 4: Category evolution (SVG stacked bars + table)
# ---------------------------------------------------------------------------
def svg_category_bars(counts: pd.DataFrame, normalize: bool) -> str:
    periods = counts.index.tolist()
    plot = counts.copy()
    if normalize:
        plot = plot.div(plot.sum(axis=1).replace(0, 1), axis=0) * 100

    W, H, ML, MR, MT, MB, BW = 920, 430, 52, 10, 10, 36, 24
    plot_h = H - MT - MB
    base = MT + plot_h
    ymax = 100.0 if normalize else float(nice_ceil(plot.sum(axis=1).max()))
    scale = plot_h / ymax
    band = (W - ML - MR) / max(len(periods), 1)

    parts = [
        f'<svg class="viz" viewBox="0 0 {W} {H}" role="img" '
        f'aria-label="Stacked bars of benchmark category mix per half-year period">'
    ]
    for k in range(5):
        v = ymax * k / 4
        y = base - v * scale
        parts.append(f'<line x1="{ML}" y1="{y:.1f}" x2="{W - MR}" y2="{y:.1f}" stroke="var(--grid)" stroke-width="1"/>')
        lbl = f"{v:.0f}%" if normalize else f"{v:,.0f}"
        parts.append(f'<text x="{ML - 7}" y="{y + 4:.1f}" text-anchor="end" font-size="11" fill="var(--muted)">{lbl}</text>')

    for i, period in enumerate(periods):
        x0 = ML + i * band + band / 2 - BW / 2
        row = plot.loc[period]
        nonzero = [c for c in plot.columns if row[c] > 0]
        cum = 0.0
        for cat in nonzero:
            val = float(row[cat])
            seg_bottom = base - cum * scale
            seg_top = base - (cum + val) * scale
            cum += val
            h = seg_bottom - seg_top
            disp = f"{val:.1f}%" if normalize else f"{val:,.0f} uses"
            title = f"<title>{esc(period)} · {esc(cat)}: {disp}</title>"
            fill = f"var(--cat-{esc(cat)})"
            is_top = cat == nonzero[-1]
            gy, gh = seg_top + 1, h - 2  # 2px surface gap between segments
            if gh < 1:
                gy, gh = seg_top, max(h, 0.8)
            if is_top and gh > 6:
                parts.append(f'<path class="seg" d="{_rounded_top_rect(x0, gy, BW, gh, 4)}" fill="{fill}">{title}</path>')
            else:
                parts.append(f'<rect class="seg" x="{x0:.1f}" y="{gy:.1f}" width="{BW}" height="{gh:.1f}" fill="{fill}">{title}</rect>')
        parts.append(
            f'<text x="{ML + i * band + band / 2:.1f}" y="{base + 20}" text-anchor="middle" '
            f'font-size="11" fill="var(--muted)">{esc(period)}</text>'
        )
    parts.append(f'<line x1="{ML}" y1="{base}" x2="{W - MR}" y2="{base}" stroke="var(--muted)" stroke-width="1"/>')
    parts.append("</svg>")
    return "".join(parts)


def category_view(openness_filter, mode):
    df = USAGE_DF
    if openness_filter == "Open only":
        df = df[df["is_open"]]
    elif openness_filter == "Closed only":
        df = df[~df["is_open"]]
    normalize = mode == "Share of period (%)"

    df = df.drop_duplicates(subset=["model_id", "benchmark", "bucket"])
    counts = df.groupby(["period", "bucket"]).size().unstack(fill_value=0)
    counts = counts.reindex(columns=[c for c in CATEGORY_ORDER if c in counts.columns], fill_value=0)
    counts = counts.reindex(PERIODS, fill_value=0)

    legend = '<div class="legend">' + "".join(
        f'<span class="key"><i style="background:var(--cat-{esc(c)})"></i>{esc(c)}</span>' for c in counts.columns
    ) + "</div>"
    title = "Benchmark category mix over time" + (" (% of period total)" if normalize else " (raw benchmark-use count)")
    chart = f'<div class="viz-title">{esc(title)}</div>' + legend + svg_category_bars(counts, normalize)

    rows = []
    for period, row in counts.iterrows():
        cells = "".join(f'<td class="num">{int(v)}</td>' for v in row)
        rows.append(f"<tr><td><strong>{esc(period)}</strong></td>{cells}</tr>")
    head = "<th>Period</th>" + "".join(f"<th>{esc(c)}</th>" for c in counts.columns)
    table = (
        '<div class="tbl-wrap"><table class="tbl"><thead><tr>'
        f'{head}</tr></thead><tbody>{"".join(rows)}</tbody></table></div>'
    )
    return chart, table


# ---------------------------------------------------------------------------
# UI
# ---------------------------------------------------------------------------
DEFAULT_BENCH = "GPQA-Diamond" if "GPQA-Diamond" in ALL_BENCHMARKS else ALL_BENCHMARKS[0]

with gr.Blocks(title="LLM Benchmark Usage Explorer") as demo:
    gr.HTML(render_hero())

    with gr.Tab("🆕 Latest releases"):
        gr.HTML('<p class="tab-note">The most recent model releases, the benchmarks they report, and where the numbers come from.</p>')
        rel_count_pills = PillGroup(["10", "20", "50"], "10", "Show")
        rel_out = gr.HTML(latest_releases_view("10"))
        rel_count_pills.change(latest_releases_view, rel_count_pills, rel_out)

    with gr.Tab("📊 Popularity & timeline"):
        gr.HTML('<p class="tab-note">Which benchmarks are used the most, and when did they first show up?</p>')
        with gr.Row():
            cat_pills = PillGroup(["All"] + ALL_CATEGORIES, "All", "Category", scale=3)
            open_pills = PillGroup(["All", "Open only", "Closed only"], "All", "Model type", scale=2)
        with gr.Row():
            min_models_sl = RangeSlider(1, 20, 1, 1, "Min. # models using it")
            search_box = SearchBox("", "Search benchmark name", "e.g. GPQA")

        pop_out = gr.HTML(popularity_view("All", "All", 1, ""))
        for ctrl in [cat_pills, open_pills, min_models_sl, search_box]:
            ctrl.change(popularity_view, [cat_pills, open_pills, min_models_sl, search_box], pop_out)

    with gr.Tab("🔎 Benchmark → Models"):
        gr.HTML('<p class="tab-note">Pick a benchmark to see every model that reports it, and whether that model is open or closed.</p>')
        bench_sel = SelectBox(ALL_BENCHMARKS, DEFAULT_BENCH, "Benchmark")
        _s, _c, _t = benchmark_view(DEFAULT_BENCH)
        bench_summary = gr.HTML(_s)
        bench_chart = gr.HTML(_c)
        bench_table = gr.HTML(_t, js_on_load=TABLE_FILTER_JS)
        bench_sel.change(benchmark_view, bench_sel, [bench_summary, bench_chart, bench_table])

    with gr.Tab("🧬 Model → Benchmarks"):
        gr.HTML('<p class="tab-note">Pick a model to see its full evaluation suite, grouped by category.</p>')
        model_sel = SelectBox(ALL_MODELS, ALL_MODELS[0], "Model")
        _s, _b = model_view(ALL_MODELS[0])
        model_summary = gr.HTML(_s)
        model_body = gr.HTML(_b)
        model_sel.change(model_view, model_sel, [model_summary, model_body])

    with gr.Tab("📈 Category evolution"):
        gr.HTML('<p class="tab-note">How has the <em>type</em> of benchmark being reported shifted over time?</p>')
        with gr.Row():
            cat_open_pills = PillGroup(["All", "Open only", "Closed only"], "All", "Model type")
            norm_pills = PillGroup(["Share of period (%)", "Raw counts"], "Share of period (%)", "Y-axis")
        _c, _t = category_view("All", "Share of period (%)")
        cat_chart = gr.HTML(_c)
        cat_table = gr.HTML(_t)
        for ctrl in [cat_open_pills, norm_pills]:
            ctrl.change(category_view, [cat_open_pills, norm_pills], [cat_chart, cat_table])

if __name__ == "__main__":
    demo.launch(css=CUSTOM_CSS)