SaylorTwift HF Staff
Adapt to simplified dataset schema (drop sources table, inline source link + benchmarks)
01f17ff verified | """Explore the SaylorTwift/llm-benchmark-usage dataset: which benchmarks are popular, | |
| when they emerged, which models (open/closed) use them, and how benchmark categories | |
| have evolved over time. | |
| The whole UI is built from `gr.HTML`: custom HTML input components (pills, select, | |
| range, search) subclassed from gr.HTML, and server-rendered HTML/SVG for every | |
| table, card, and chart. No Dataframe/Dropdown/Radio/Slider/Plot components. | |
| """ | |
| print("app.py: starting imports...", flush=True) | |
| import html as html_lib | |
| import math | |
| from urllib.parse import urlparse | |
| import gradio as gr | |
| import pandas as pd | |
| from data import CATEGORY_ORDER, load_data | |
| print("app.py: imports done, loading data...", flush=True) | |
| MODELS_DF, USAGE_DF = load_data() | |
| print("app.py: data loaded, building UI...", flush=True) | |
| ALL_BENCHMARKS = sorted(USAGE_DF["benchmark"].unique()) | |
| ALL_MODELS = sorted(USAGE_DF["model_id"].unique()) | |
| ALL_CATEGORIES = sorted(USAGE_DF["bucket"].unique()) | |
| PERIODS = sorted(MODELS_DF["period"].unique()) | |
| def esc(s) -> str: | |
| return html_lib.escape(str(s), quote=True) | |
| # --------------------------------------------------------------------------- | |
| # Styles. Category/openness colors are defined once as CSS variables (with | |
| # dark-mode steps under `.dark`, which Gradio toggles on the page), and every | |
| # chart mark and chip references them, so both themes come from one place. | |
| # --------------------------------------------------------------------------- | |
| CUSTOM_CSS = """ | |
| :root { | |
| --cat-knowledge: #2a78d6; --cat-coding: #1baf7a; --cat-agentic: #eda100; | |
| --cat-math: #008300; --cat-safety: #4a3aa7; --cat-vision: #e34948; | |
| --cat-multilingual: #e87ba4; --cat-long_context: #eb6834; --cat-other: #86847d; | |
| --open: #2a78d6; --closed: #e34948; --accent: #2a78d6; | |
| } | |
| /* Gradio sets the light theme variables on <html> and the dark ones on | |
| .gradio-container, and var() resolves where a property is DEFINED — so these | |
| aliases must live on .gradio-container to pick up the active theme. */ | |
| .gradio-container { | |
| --ink: var(--body-text-color, #0b0b0b); | |
| --muted: var(--body-text-color-subdued, #898781); | |
| --surf: var(--background-fill-primary, #fcfcfb); | |
| --hair: var(--border-color-primary, #e1e0d9); | |
| --grid: color-mix(in srgb, var(--hair) 55%, transparent); | |
| } | |
| .dark { | |
| --cat-knowledge: #3987e5; --cat-coding: #199e70; --cat-agentic: #c98500; | |
| --cat-math: #008300; --cat-safety: #9085e9; --cat-vision: #e66767; | |
| --cat-multilingual: #d55181; --cat-long_context: #d95926; --cat-other: #898781; | |
| --open: #3987e5; --closed: #e66767; --accent: #3987e5; | |
| } | |
| /* Gradio 6's theme cascades flex-grow:1 from .gradio-container down through | |
| .main.fillable.app -> .wrap -> main.contain, stretching the whole app shell | |
| to the viewport height regardless of actual content height. Force those | |
| containers back to their natural height. */ | |
| .gradio-container, .main.fillable.app, .wrap.svelte-zxu34v, main.contain.svelte-zxu34v { | |
| flex-grow: 0 !important; | |
| height: auto !important; | |
| } | |
| /* --- hero ------------------------------------------------------------- */ | |
| .hero h1 { font-size: 26px; margin: 4px 0 6px; color: var(--ink); } | |
| .hero p { color: var(--muted); font-size: 14px; margin: 0; } | |
| .hero a { color: var(--accent); } | |
| .tiles { display: flex; gap: 12px; flex-wrap: wrap; margin-top: 14px; } | |
| .tile { flex: 1; min-width: 150px; border: 1px solid var(--hair); border-radius: 10px; | |
| padding: 10px 14px; background: var(--surf); } | |
| .tile .lab { font-size: 12px; color: var(--muted); } | |
| .tile .val { font-size: 26px; font-weight: 600; color: var(--ink); } | |
| .tile .val.sm { font-size: 15px; padding-top: 8px; } | |
| /* --- controls ---------------------------------------------------------- */ | |
| .ctl { display: flex; flex-direction: column; gap: 6px; margin: 2px 0 8px; } | |
| .ctl-label { font-size: 12px; font-weight: 600; color: var(--muted); } | |
| .pillrow { display: flex; flex-wrap: wrap; gap: 6px; } | |
| .pill { border: 1px solid var(--hair); background: transparent; color: var(--ink); | |
| border-radius: 999px; padding: 5px 12px; font-size: 12.5px; cursor: pointer; } | |
| .pill:hover { border-color: var(--accent); } | |
| .pill.on { background: var(--accent); border-color: var(--accent); color: #fff; } | |
| .cbx-wrap { position: relative; max-width: 460px; } | |
| input.cbx-input { width: 100%; box-sizing: border-box; padding: 7px 30px 7px 10px; | |
| border: 1px solid var(--hair); border-radius: 8px; background: var(--surf); | |
| color: var(--ink); font-size: 13px; } | |
| input.cbx-input:focus { outline: none; border-color: var(--accent); } | |
| .cbx-caret { position: absolute; right: 10px; top: 50%; transform: translateY(-50%); | |
| color: var(--muted); pointer-events: none; font-size: 11px; } | |
| .cbx-list { position: absolute; top: calc(100% + 4px); left: 0; right: 0; max-height: 320px; | |
| overflow: auto; background: var(--surf); border: 1px solid var(--hair); | |
| border-radius: 8px; z-index: 100; box-shadow: 0 6px 18px rgba(0, 0, 0, .15); } | |
| .cbx-item { padding: 7px 10px; font-size: 13px; color: var(--ink); cursor: pointer; } | |
| .cbx-item:hover, .cbx-item.active { background: color-mix(in srgb, var(--accent) 14%, transparent); } | |
| .cbx-item.sel { font-weight: 600; } | |
| .cbx-empty { padding: 8px 10px; color: var(--muted); font-size: 12.5px; } | |
| .sliderrow { display: flex; align-items: center; gap: 10px; } | |
| input.ctl-range { accent-color: var(--accent); width: 200px; max-width: 100%; } | |
| .ctl-range-val { font-size: 13px; font-weight: 600; color: var(--ink); min-width: 2ch; } | |
| input.ctl-search, input.tbl-filter { padding: 7px 10px; border: 1px solid var(--hair); | |
| border-radius: 8px; background: var(--surf); color: var(--ink); font-size: 13px; width: 230px; } | |
| /* --- cards, chips, notes ------------------------------------------------ */ | |
| .tab-note { color: var(--muted); font-size: 13.5px; margin: 2px 0 10px; } | |
| .card { border: 1px solid var(--hair); border-radius: 10px; padding: 12px 16px; | |
| background: var(--surf); margin: 4px 0 10px; } | |
| .card-title { font-size: 17px; font-weight: 600; color: var(--ink); margin-bottom: 4px; } | |
| .card-sub { color: var(--muted); font-size: 13px; line-height: 1.7; } | |
| .card-sub strong { color: var(--ink); font-weight: 600; } | |
| .card-sub a { color: var(--accent); } | |
| .chip { display: inline-flex; align-items: center; gap: 5px; padding: 2px 9px; | |
| border: 1px solid var(--hair); border-radius: 999px; font-size: 11.5px; | |
| margin: 2px 4px 2px 0; color: var(--muted); white-space: nowrap; } | |
| .chip i { width: 8px; height: 8px; border-radius: 50%; display: inline-block; flex: none; } | |
| .rel-hd { display: flex; align-items: center; gap: 8px; flex-wrap: wrap; } | |
| .rel-name { font-size: 15px; font-weight: 600; color: var(--ink); } | |
| .rel-meta { color: var(--muted); font-size: 12.5px; margin-left: auto; } | |
| .rel-chips { margin-top: 8px; } | |
| .rel-src { color: var(--muted); font-size: 12.5px; margin-top: 6px; } | |
| .rel-src a { color: var(--accent); } | |
| .cat-sec { margin: 10px 0 14px; } | |
| .cat-hd { display: flex; align-items: center; gap: 7px; font-size: 13.5px; | |
| font-weight: 600; color: var(--ink); margin-bottom: 6px; } | |
| .cat-hd i { width: 10px; height: 10px; border-radius: 50%; display: inline-block; } | |
| .cat-hd .n { color: var(--muted); font-weight: 400; } | |
| /* --- tables ------------------------------------------------------------- */ | |
| .tbl-tools { margin: 2px 0 8px; } | |
| .tbl-wrap { max-height: 480px; overflow: auto; border: 1px solid var(--hair); border-radius: 8px; } | |
| .tbl { width: 100%; border-collapse: collapse; font-size: 13px; color: var(--ink); } | |
| .tbl thead th { position: sticky; top: 0; z-index: 1; background: var(--surf); | |
| text-align: left; padding: 8px 10px; border-bottom: 1px solid var(--hair); | |
| color: var(--muted); font-weight: 600; white-space: nowrap; } | |
| .tbl td { padding: 7px 10px; border-bottom: 1px solid var(--hair); vertical-align: middle; } | |
| .tbl tbody tr:last-child td { border-bottom: none; } | |
| .tbl tbody tr:hover { background: color-mix(in srgb, var(--ink) 5%, transparent); } | |
| .tbl .num { font-variant-numeric: tabular-nums; text-align: right; } | |
| .tbl .mut { color: var(--muted); font-size: 12.5px; white-space: nowrap; } | |
| .tbl a { color: var(--accent); text-decoration: none; } | |
| .tbl a:hover { text-decoration: underline; } | |
| .match-note { color: var(--muted); font-size: 12.5px; margin: 6px 2px; } | |
| .cellbar { display: inline-block; vertical-align: middle; width: 110px; height: 8px; | |
| background: color-mix(in srgb, var(--ink) 7%, transparent); border-radius: 4px; | |
| overflow: hidden; margin-left: 8px; } | |
| .cellbar span { display: block; height: 100%; background: var(--accent); | |
| border-radius: 0 4px 4px 0; } | |
| .splitbar { display: inline-flex; vertical-align: middle; width: 90px; height: 8px; | |
| gap: 2px; margin-left: 8px; } | |
| .splitbar .o { background: var(--open); height: 100%; border-radius: 4px 0 0 4px; } | |
| .splitbar .c { background: var(--closed); height: 100%; border-radius: 0 4px 4px 0; } | |
| /* --- charts ------------------------------------------------------------- */ | |
| .viz-title { font-size: 14px; font-weight: 600; color: var(--ink); margin: 8px 2px 2px; } | |
| .legend { display: flex; gap: 16px; flex-wrap: wrap; margin: 6px 2px 8px; | |
| font-size: 12px; color: var(--muted); } | |
| .legend .key { display: inline-flex; gap: 6px; align-items: center; } | |
| .legend .key i { width: 10px; height: 10px; border-radius: 3px; display: inline-block; } | |
| .legend .key i.round { border-radius: 50%; } | |
| .viz { width: 100%; height: auto; display: block; } | |
| .viz text { font-family: system-ui, -apple-system, "Segoe UI", sans-serif; } | |
| .viz .dot { transition: r .1s; } | |
| .viz .dot:hover { r: 7.5px; } | |
| .viz .seg:hover { opacity: .82; } | |
| """ | |
| # --------------------------------------------------------------------------- | |
| # Custom HTML input components (replace Radio / Dropdown / Slider / Textbox) | |
| # --------------------------------------------------------------------------- | |
| class PillGroup(gr.HTML): | |
| """Single-select pill buttons (replaces gr.Radio).""" | |
| def __init__(self, options, value, label, **kwargs): | |
| super().__init__( | |
| value=value, | |
| options=list(options), | |
| label_text=label, | |
| html_template=""" | |
| <div class="ctl"> | |
| <div class="ctl-label">${label_text}</div> | |
| <div class="pillrow"> | |
| ${options.map(o => `<button type="button" class="pill${o === value ? ' on' : ''}" data-v="${o}">${o}</button>`).join('')} | |
| </div> | |
| </div>""", | |
| js_on_load=""" | |
| element.addEventListener('click', (e) => { | |
| const btn = e.target.closest('.pill'); | |
| if (!btn) return; | |
| props.value = btn.dataset.v; | |
| trigger('change'); | |
| });""", | |
| **kwargs, | |
| ) | |
| def api_info(self): | |
| return {"type": "string"} | |
| class SelectBox(gr.HTML): | |
| """Type-to-filter combobox (replaces gr.Dropdown with filterable=True): | |
| a text input that opens a filtered option list, with arrow/Enter/Escape | |
| keyboard support. All listeners are delegated on the component root so | |
| they survive the re-render that follows each value sync.""" | |
| def __init__(self, options, value, label, **kwargs): | |
| super().__init__( | |
| value=value, | |
| options=list(options), | |
| label_text=label, | |
| html_template=""" | |
| <div class="ctl"> | |
| <div class="ctl-label">${label_text}</div> | |
| <div class="cbx-wrap"> | |
| <input type="text" class="cbx-input" role="combobox" aria-expanded="false" | |
| autocomplete="off" spellcheck="false" placeholder="Type to search…" | |
| value="${value.replace(/&/g,'&').replace(/"/g,'"')}"> | |
| <span class="cbx-caret">▾</span> | |
| <div class="cbx-list" hidden> | |
| ${options.map(o => `<div class="cbx-item${o === value ? ' sel' : ''}" data-v="${o.replace(/&/g,'&').replace(/"/g,'"')}">${o.replace(/&/g,'&').replace(/</g,'<')}</div>`).join('')} | |
| <div class="cbx-empty" hidden>No match</div> | |
| </div> | |
| </div> | |
| </div>""", | |
| js_on_load=""" | |
| const list = () => element.querySelector('.cbx-list'); | |
| const input = () => element.querySelector('.cbx-input'); | |
| const items = () => [...element.querySelectorAll('.cbx-item')]; | |
| const visible = () => items().filter(it => !it.hidden); | |
| function open(showAll) { | |
| if (showAll) { items().forEach(it => { it.hidden = false; }); element.querySelector('.cbx-empty').hidden = true; } | |
| list().hidden = false; | |
| input().setAttribute('aria-expanded', 'true'); | |
| } | |
| function close(restore) { | |
| list().hidden = true; | |
| input().setAttribute('aria-expanded', 'false'); | |
| items().forEach(it => it.classList.remove('active')); | |
| if (restore) input().value = props.value; | |
| } | |
| function applyFilter() { | |
| const q = input().value.toLowerCase(); | |
| let any = false; | |
| items().forEach(it => { | |
| it.hidden = !it.textContent.toLowerCase().includes(q); | |
| it.classList.remove('active'); | |
| if (!it.hidden) any = true; | |
| }); | |
| element.querySelector('.cbx-empty').hidden = any; | |
| } | |
| function move(dir) { | |
| const vis = visible(); | |
| if (!vis.length) return; | |
| const cur = vis.findIndex(it => it.classList.contains('active')); | |
| const next = Math.min(Math.max(cur + dir, 0), vis.length - 1); | |
| vis.forEach(it => it.classList.remove('active')); | |
| vis[next].classList.add('active'); | |
| vis[next].scrollIntoView({ block: 'nearest' }); | |
| } | |
| function pick(v) { | |
| close(false); | |
| props.value = v; | |
| trigger('change'); | |
| } | |
| element.addEventListener('focusin', (e) => { | |
| if (!e.target.classList.contains('cbx-input')) return; | |
| e.target.select(); | |
| open(true); | |
| }); | |
| element.addEventListener('input', (e) => { | |
| if (!e.target.classList.contains('cbx-input')) return; | |
| open(false); | |
| applyFilter(); | |
| }); | |
| element.addEventListener('keydown', (e) => { | |
| if (!e.target.classList.contains('cbx-input')) return; | |
| if (e.key === 'ArrowDown') { e.preventDefault(); if (list().hidden) open(true); move(1); } | |
| else if (e.key === 'ArrowUp') { e.preventDefault(); move(-1); } | |
| else if (e.key === 'Enter') { | |
| e.preventDefault(); | |
| const target = visible().find(it => it.classList.contains('active')) || visible()[0]; | |
| if (target) pick(target.dataset.v); | |
| } | |
| else if (e.key === 'Escape') { close(true); e.target.blur(); } | |
| }); | |
| element.addEventListener('pointerdown', (e) => { | |
| const it = e.target.closest('.cbx-item'); | |
| if (!it) return; | |
| e.preventDefault(); // beat the input's blur | |
| pick(it.dataset.v); | |
| }); | |
| element.addEventListener('focusout', (e) => { | |
| if (!element.contains(e.relatedTarget)) close(true); | |
| });""", | |
| **kwargs, | |
| ) | |
| def api_info(self): | |
| return {"type": "string"} | |
| class RangeSlider(gr.HTML): | |
| """Native range input with a value readout (replaces gr.Slider).""" | |
| def __init__(self, minimum, maximum, value, step, label, **kwargs): | |
| super().__init__( | |
| value=value, | |
| minimum=minimum, | |
| maximum=maximum, | |
| step=step, | |
| label_text=label, | |
| html_template=""" | |
| <div class="ctl"> | |
| <div class="ctl-label">${label_text}</div> | |
| <div class="sliderrow"> | |
| <input type="range" class="ctl-range" min="${minimum}" max="${maximum}" step="${step}" value="${value}"> | |
| <span class="ctl-range-val">${value}</span> | |
| </div> | |
| </div>""", | |
| js_on_load=""" | |
| element.addEventListener('input', (e) => { | |
| if (!e.target.classList.contains('ctl-range')) return; | |
| const out = element.querySelector('.ctl-range-val'); | |
| if (out) out.textContent = e.target.value; | |
| }); | |
| element.addEventListener('change', (e) => { | |
| if (!e.target.classList.contains('ctl-range')) return; | |
| props.value = Number(e.target.value); | |
| trigger('change'); | |
| });""", | |
| **kwargs, | |
| ) | |
| def api_info(self): | |
| return {"type": "integer"} | |
| class SearchBox(gr.HTML): | |
| """Debounced text search (replaces gr.Textbox). Focus is restored after the | |
| value-sync re-render so typing isn't interrupted.""" | |
| def __init__(self, value, label, placeholder, **kwargs): | |
| super().__init__( | |
| value=value, | |
| label_text=label, | |
| placeholder=placeholder, | |
| html_template=""" | |
| <div class="ctl"> | |
| <div class="ctl-label">${label_text}</div> | |
| <input type="search" class="ctl-search" placeholder="${placeholder}" value="${value.replace(/&/g,'&').replace(/"/g,'"')}"> | |
| </div>""", | |
| js_on_load=""" | |
| let t = null; | |
| element.addEventListener('input', (e) => { | |
| if (!e.target.classList.contains('ctl-search')) return; | |
| clearTimeout(t); | |
| const v = e.target.value; | |
| t = setTimeout(() => { | |
| props.value = v; | |
| trigger('change'); | |
| setTimeout(() => { | |
| const inp = element.querySelector('.ctl-search'); | |
| if (inp && document.activeElement !== inp) { | |
| inp.focus(); | |
| inp.setSelectionRange(inp.value.length, inp.value.length); | |
| } | |
| }, 60); | |
| }, 250); | |
| });""", | |
| **kwargs, | |
| ) | |
| def api_info(self): | |
| return {"type": "string"} | |
| # Client-side row filter for rendered tables: filters <tbody> rows against the | |
| # .tbl-filter input, no server round trip. Attached via js_on_load so the | |
| # delegated listener survives value updates. | |
| TABLE_FILTER_JS = """ | |
| element.addEventListener('input', (e) => { | |
| if (!e.target.classList.contains('tbl-filter')) return; | |
| const q = e.target.value.toLowerCase(); | |
| element.querySelectorAll('tbody tr').forEach(tr => { | |
| tr.style.display = tr.textContent.toLowerCase().includes(q) ? '' : 'none'; | |
| }); | |
| });""" | |
| # --------------------------------------------------------------------------- | |
| # Small HTML helpers | |
| # --------------------------------------------------------------------------- | |
| def cat_chip(bucket: str) -> str: | |
| return f'<span class="chip"><i style="background:var(--cat-{esc(bucket)})"></i>{esc(bucket)}</span>' | |
| def openness_chip(is_open: bool) -> str: | |
| color, text = ("var(--open)", "open") if is_open else ("var(--closed)", "closed") | |
| return f'<span class="chip"><i style="background:{color}"></i>{text}</span>' | |
| def source_label(url: str) -> str: | |
| """The dataset no longer carries a source title/type, just a link — show its | |
| host as a short, still-meaningful label (e.g. "arxiv.org", "huggingface.co").""" | |
| return urlparse(url).netloc or url | |
| def nice_ceil(v: float) -> float: | |
| if v <= 0: | |
| return 1 | |
| exp = 10 ** math.floor(math.log10(v)) | |
| frac = v / exp | |
| for m in (1, 2, 2.5, 5, 10): | |
| if frac <= m: | |
| return m * exp | |
| return v | |
| def month_ticks(t0: pd.Timestamp, t1: pd.Timestamp) -> list[pd.Timestamp]: | |
| span = max((t1 - t0).days, 1) | |
| step = next((s for s in (1, 2, 3, 6, 12, 24) if span / (30.4 * s) <= 7), 24) | |
| m0 = ((t0.month - 1) // step) * step + 1 | |
| d = pd.Timestamp(year=t0.year, month=m0, day=1) | |
| ticks = [] | |
| while d <= t1: | |
| if d >= t0: | |
| ticks.append(d) | |
| total = (d.month - 1) + step | |
| d = pd.Timestamp(year=d.year + total // 12, month=total % 12 + 1, day=1) | |
| return ticks | |
| # --------------------------------------------------------------------------- | |
| # Hero header | |
| # --------------------------------------------------------------------------- | |
| def render_hero() -> str: | |
| d0, d1 = MODELS_DF["release_date"].min().date(), MODELS_DF["release_date"].max().date() | |
| tiles = [ | |
| ("Models", str(len(MODELS_DF)), ""), | |
| ("Labs", str(MODELS_DF["lab"].nunique()), ""), | |
| ("Distinct benchmarks", str(USAGE_DF["benchmark"].nunique()), ""), | |
| ("Coverage", f"{d0} → {d1}", "sm"), | |
| ] | |
| tiles_html = "".join( | |
| f'<div class="tile"><div class="lab">{esc(lab)}</div><div class="val {cls}">{esc(val)}</div></div>' | |
| for lab, val, cls in tiles | |
| ) | |
| return ( | |
| '<div class="hero"><h1>LLM Benchmark Usage Explorer</h1>' | |
| '<p>Exploring <a href="https://huggingface.co/datasets/SaylorTwift/llm-benchmark-usage" ' | |
| 'target="_blank">SaylorTwift/llm-benchmark-usage</a> — which benchmarks labs report, ' | |
| "who uses them, and how the mix shifts over time.</p>" | |
| f'<div class="tiles">{tiles_html}</div></div>' | |
| ) | |
| # --------------------------------------------------------------------------- | |
| # Tab 0: Latest releases (landing view) | |
| # --------------------------------------------------------------------------- | |
| def rel_date(d: pd.Timestamp) -> str: | |
| days = (pd.Timestamp.now().normalize() - d.normalize()).days | |
| if days <= 0: | |
| return "today" | |
| if days == 1: | |
| return "yesterday" | |
| if days < 7: | |
| return f"{days} days ago" | |
| if days < 60: | |
| return f"{days // 7} week{'s' if days >= 14 else ''} ago" | |
| if days < 365: | |
| return f"{days // 30} months ago" | |
| return f"{days // 365} year{'s' if days >= 730 else ''} ago" | |
| MAX_RELEASE_CHIPS = 16 | |
| def latest_releases_view(count) -> str: | |
| recent = MODELS_DF.sort_values("release_date", ascending=False).head(int(count)) | |
| bucket_rank = {b: i for i, b in enumerate(CATEGORY_ORDER)} | |
| cards = [] | |
| for m in recent.itertuples(): | |
| usage = ( | |
| USAGE_DF[USAGE_DF["model_id"] == m.model_id] | |
| .drop_duplicates(subset=["benchmark"]) | |
| .sort_values(["bucket", "benchmark"], key=lambda s: s.map(bucket_rank) if s.name == "bucket" else s) | |
| ) | |
| if usage.empty: | |
| body = '<div class="card-sub">No benchmark data recorded for this model.</div>' | |
| else: | |
| shown = usage.head(MAX_RELEASE_CHIPS) | |
| chips = "".join( | |
| f'<span class="chip" title="{esc(r.bucket)} · reported category: {esc(r.category)}">' | |
| f'<i style="background:var(--cat-{esc(r.bucket)})"></i>{esc(r.benchmark)}</span>' | |
| for r in shown.itertuples() | |
| ) | |
| if len(usage) > MAX_RELEASE_CHIPS: | |
| chips += f'<span class="chip">+{len(usage) - MAX_RELEASE_CHIPS} more</span>' | |
| src_url = usage.iloc[0]["source"] | |
| body = ( | |
| f'<div class="rel-chips">{chips}</div>' | |
| f'<div class="rel-src">{len(usage)} benchmarks · Source: ' | |
| f'<a href="{esc(src_url)}" target="_blank">{esc(source_label(src_url))}</a></div>' | |
| ) | |
| cards.append( | |
| f'<div class="card"><div class="rel-hd">' | |
| f'<span class="rel-name">{esc(m.model_id)}</span>' | |
| f"{openness_chip(bool(m.is_open))}" | |
| f'<span class="rel-meta">{esc(m.lab)} · {m.release_date.date()} · {rel_date(m.release_date)}</span>' | |
| f"</div>{body}</div>" | |
| ) | |
| return "".join(cards) | |
| # --------------------------------------------------------------------------- | |
| # Tab 1: Benchmark popularity | |
| # --------------------------------------------------------------------------- | |
| def popularity_view(category_filter, openness_filter, min_models, search) -> str: | |
| df = USAGE_DF | |
| if category_filter and category_filter != "All": | |
| df = df[df["bucket"] == category_filter] | |
| if openness_filter == "Open only": | |
| df = df[df["is_open"]] | |
| elif openness_filter == "Closed only": | |
| df = df[~df["is_open"]] | |
| # A model can appear several times for one benchmark (multiple sources report | |
| # it), so count on deduplicated (benchmark, model) pairs; categories keep all rows. | |
| uniq = df.drop_duplicates(subset=["benchmark", "model_id"]) | |
| agg = ( | |
| uniq.groupby("benchmark") | |
| .agg( | |
| models=("model_id", "nunique"), | |
| open_models=("is_open", "sum"), | |
| first_seen=("release_date", "min"), | |
| last_seen=("release_date", "max"), | |
| labs=("lab", "nunique"), | |
| ) | |
| .reset_index() | |
| ) | |
| buckets = df.groupby("benchmark")["bucket"].agg(lambda s: sorted(set(s))).rename("buckets") | |
| agg = agg.merge(buckets, on="benchmark") | |
| agg["closed_models"] = agg["models"] - agg["open_models"] | |
| agg = agg[agg["models"] >= int(min_models)] | |
| if search: | |
| agg = agg[agg["benchmark"].str.contains(search, case=False, na=False, regex=False)] | |
| agg = agg.sort_values(["models", "benchmark"], ascending=[False, True]) | |
| if agg.empty: | |
| return '<div class="card"><div class="card-sub">No benchmarks match the current filters.</div></div>' | |
| max_models = int(agg["models"].max()) | |
| rows = [] | |
| for i, r in enumerate(agg.itertuples(), start=1): | |
| bar_pct = r.models / max_models * 100 | |
| tot = max(r.models, 1) | |
| o_pct, c_pct = r.open_models / tot * 100, r.closed_models / tot * 100 | |
| chips = "".join(cat_chip(b) for b in r.buckets) | |
| rows.append( | |
| f"<tr>" | |
| f'<td class="num mut">{i}</td>' | |
| f"<td><strong>{esc(r.benchmark)}</strong></td>" | |
| f'<td class="num">{r.models}<span class="cellbar"><span style="width:{bar_pct:.1f}%"></span></span></td>' | |
| f'<td class="num">{int(r.open_models)} / {int(r.closed_models)}' | |
| f'<span class="splitbar" title="{int(r.open_models)} open · {int(r.closed_models)} closed">' | |
| f'<span class="o" style="width:{o_pct:.0f}%"></span><span class="c" style="width:{c_pct:.0f}%"></span></span></td>' | |
| f'<td class="num">{r.labs}</td>' | |
| f"<td>{chips}</td>" | |
| f'<td class="mut">{r.first_seen.date()}</td>' | |
| f'<td class="mut">{r.last_seen.date()}</td>' | |
| f"</tr>" | |
| ) | |
| return ( | |
| f'<div class="match-note">{len(agg)} benchmarks match · sorted by number of models</div>' | |
| '<div class="tbl-wrap"><table class="tbl"><thead><tr>' | |
| "<th>#</th><th>Benchmark</th><th>Models</th><th>Open / closed</th><th>Labs</th>" | |
| "<th>Categories</th><th>First seen</th><th>Last seen</th>" | |
| f'</tr></thead><tbody>{"".join(rows)}</tbody></table></div>' | |
| ) | |
| # --------------------------------------------------------------------------- | |
| # Tab 2: Benchmark -> Models (summary card, monthly-usage SVG bars, table) | |
| # --------------------------------------------------------------------------- | |
| def _rounded_top_rect(x, y, w, h, r) -> str: | |
| r = min(r, h / 2, w / 2) | |
| return ( | |
| f'M {x:.1f} {y + h:.1f} L {x:.1f} {y + r:.1f} Q {x:.1f} {y:.1f} {x + r:.1f} {y:.1f} ' | |
| f'L {x + w - r:.1f} {y:.1f} Q {x + w:.1f} {y:.1f} {x + w:.1f} {y + r:.1f} L {x + w:.1f} {y + h:.1f} Z' | |
| ) | |
| def svg_benchmark_weekly_usage(df: pd.DataFrame) -> str: | |
| """Bars of how many model releases reported this benchmark each week, spanning | |
| the whole dataset time range (empty weeks stay visible as gaps).""" | |
| t0 = MODELS_DF["release_date"].min().to_period("W") | |
| t1 = MODELS_DF["release_date"].max().to_period("W") | |
| weeks = pd.period_range(t0, t1, freq="W") | |
| counts = df["release_date"].dt.to_period("W").value_counts().reindex(weeks, fill_value=0) | |
| W, H, ML, MR, MT, MB = 920, 300, 44, 10, 12, 36 | |
| plot_h = H - MT - MB | |
| base = MT + plot_h | |
| vmax = int(counts.max()) | |
| step = max(1, math.ceil(vmax / 4)) | |
| ymax = step * 4 | |
| scale = plot_h / ymax | |
| band = (W - ML - MR) / len(weeks) | |
| bw = max(band - 1, 1.0) # 1px surface gap between adjacent bars (weekly bands are narrow) | |
| parts = [ | |
| f'<svg class="viz" viewBox="0 0 {W} {H}" role="img" ' | |
| f'aria-label="Weekly count of model releases reporting this benchmark, over the whole dataset period">' | |
| ] | |
| for k in range(5): | |
| v = step * k | |
| y = base - v * scale | |
| parts.append(f'<line x1="{ML}" y1="{y:.1f}" x2="{W - MR}" y2="{y:.1f}" stroke="var(--grid)" stroke-width="1"/>') | |
| parts.append(f'<text x="{ML - 7}" y="{y + 4:.1f}" text-anchor="end" font-size="11" fill="var(--muted)">{v}</text>') | |
| week0_start = weeks[0].start_time | |
| for tick in month_ticks(week0_start, weeks[-1].end_time): | |
| idx = (tick - week0_start).days // 7 | |
| if 0 <= idx < len(weeks): | |
| cx = ML + idx * band + band / 2 | |
| parts.append( | |
| f'<text x="{cx:.1f}" y="{base + 20}" text-anchor="middle" font-size="11" fill="var(--muted)">' | |
| f"{tick.strftime('%b %Y')}</text>" | |
| ) | |
| for i, week in enumerate(weeks): | |
| n = int(counts.loc[week]) | |
| if n == 0: | |
| continue | |
| h = n * scale | |
| label = f"Week of {week.start_time.strftime('%b %-d, %Y')}: {n} model{'s' if n != 1 else ''}" | |
| parts.append( | |
| f'<path class="seg" d="{_rounded_top_rect(ML + i * band + 0.5, base - h, bw, h, 2)}" fill="var(--accent)">' | |
| f"<title>{esc(label)}</title></path>" | |
| ) | |
| parts.append(f'<line x1="{ML}" y1="{base}" x2="{W - MR}" y2="{base}" stroke="var(--muted)" stroke-width="1"/>') | |
| parts.append("</svg>") | |
| return "".join(parts) | |
| def benchmark_view(benchmark): | |
| if not benchmark: | |
| return "", "", '<div class="card"><div class="card-sub">Pick a benchmark above.</div></div>' | |
| df = ( | |
| USAGE_DF[USAGE_DF["benchmark"] == benchmark] | |
| .drop_duplicates(subset=["model_id"]) | |
| .sort_values("release_date") | |
| ) | |
| n_open = int(df["is_open"].sum()) | |
| n_closed = len(df) - n_open | |
| summary = ( | |
| f'<div class="card"><div class="card-title">{esc(benchmark)}</div>' | |
| f'<div class="card-sub"><strong>{len(df)} models</strong> from ' | |
| f"<strong>{df['lab'].nunique()} labs</strong> report it — " | |
| f"{n_open} open-weight, {n_closed} closed. First seen " | |
| f"<strong>{df['release_date'].min().date()}</strong>, most recent " | |
| f"<strong>{df['release_date'].max().date()}</strong>.</div></div>" | |
| ) | |
| chart = ( | |
| f'<div class="viz-title">How often is “{esc(benchmark)}” used? Model releases reporting it per week</div>' | |
| + svg_benchmark_weekly_usage(df) | |
| ) | |
| rows = [] | |
| for r in df.itertuples(): | |
| rows.append( | |
| "<tr>" | |
| f"<td><strong>{esc(r.model_id)}</strong></td>" | |
| f"<td>{esc(r.lab)}</td>" | |
| f'<td class="mut">{r.release_date.date()}</td>' | |
| f"<td>{openness_chip(r.is_open)}</td>" | |
| f'<td class="mut">{esc(r.category)}</td>' | |
| f'<td class="mut"><a href="{esc(r.source)}" target="_blank">{esc(source_label(r.source))}</a></td>' | |
| "</tr>" | |
| ) | |
| table = ( | |
| '<div class="tbl-tools"><input class="tbl-filter" placeholder="Filter rows…"></div>' | |
| '<div class="tbl-wrap"><table class="tbl"><thead><tr>' | |
| "<th>Model</th><th>Lab</th><th>Released</th><th>Openness</th><th>Reported category</th><th>Source</th>" | |
| f'</tr></thead><tbody>{"".join(rows)}</tbody></table></div>' | |
| ) | |
| return summary, chart, table | |
| # --------------------------------------------------------------------------- | |
| # Tab 3: Model -> Benchmarks (summary card + benchmarks grouped by category) | |
| # --------------------------------------------------------------------------- | |
| def model_view(model_id): | |
| if not model_id: | |
| return "", '<div class="card"><div class="card-sub">Pick a model above.</div></div>' | |
| df = ( | |
| USAGE_DF[USAGE_DF["model_id"] == model_id] | |
| .drop_duplicates(subset=["benchmark", "bucket"]) | |
| .sort_values(["bucket", "benchmark"]) | |
| ) | |
| if df.empty: | |
| return "", '<div class="card"><div class="card-sub">No data for this model.</div></div>' | |
| meta = MODELS_DF[MODELS_DF["model_id"] == model_id].iloc[0] | |
| src = df.iloc[0] | |
| summary = ( | |
| f'<div class="card"><div class="card-title">{esc(model_id)} {openness_chip(bool(meta["is_open"]))}</div>' | |
| f'<div class="card-sub"><strong>{esc(meta["lab"])}</strong>, released ' | |
| f'<strong>{meta["release_date"].date()}</strong> · <strong>{df["benchmark"].nunique()} benchmarks</strong> ' | |
| f"across {df['bucket'].nunique()} categories · Source: " | |
| f'<a href="{esc(src["source"])}" target="_blank">{esc(source_label(src["source"]))}</a></div></div>' | |
| ) | |
| sections = [] | |
| for bucket in CATEGORY_ORDER: | |
| sub = df[df["bucket"] == bucket] | |
| if sub.empty: | |
| continue | |
| chips = "".join( | |
| f'<span class="chip" title="reported category: {esc(r.category)}">' | |
| f'<i style="background:var(--cat-{esc(bucket)})"></i>{esc(r.benchmark)}</span>' | |
| for r in sub.itertuples() | |
| ) | |
| sections.append( | |
| f'<div class="cat-sec"><div class="cat-hd"><i style="background:var(--cat-{esc(bucket)})"></i>' | |
| f'{esc(bucket)} <span class="n">({len(sub)})</span></div><div>{chips}</div></div>' | |
| ) | |
| return summary, "".join(sections) | |
| # --------------------------------------------------------------------------- | |
| # Tab 4: Category evolution (SVG stacked bars + table) | |
| # --------------------------------------------------------------------------- | |
| def svg_category_bars(counts: pd.DataFrame, normalize: bool) -> str: | |
| periods = counts.index.tolist() | |
| plot = counts.copy() | |
| if normalize: | |
| plot = plot.div(plot.sum(axis=1).replace(0, 1), axis=0) * 100 | |
| W, H, ML, MR, MT, MB, BW = 920, 430, 52, 10, 10, 36, 24 | |
| plot_h = H - MT - MB | |
| base = MT + plot_h | |
| ymax = 100.0 if normalize else float(nice_ceil(plot.sum(axis=1).max())) | |
| scale = plot_h / ymax | |
| band = (W - ML - MR) / max(len(periods), 1) | |
| parts = [ | |
| f'<svg class="viz" viewBox="0 0 {W} {H}" role="img" ' | |
| f'aria-label="Stacked bars of benchmark category mix per half-year period">' | |
| ] | |
| for k in range(5): | |
| v = ymax * k / 4 | |
| y = base - v * scale | |
| parts.append(f'<line x1="{ML}" y1="{y:.1f}" x2="{W - MR}" y2="{y:.1f}" stroke="var(--grid)" stroke-width="1"/>') | |
| lbl = f"{v:.0f}%" if normalize else f"{v:,.0f}" | |
| parts.append(f'<text x="{ML - 7}" y="{y + 4:.1f}" text-anchor="end" font-size="11" fill="var(--muted)">{lbl}</text>') | |
| for i, period in enumerate(periods): | |
| x0 = ML + i * band + band / 2 - BW / 2 | |
| row = plot.loc[period] | |
| nonzero = [c for c in plot.columns if row[c] > 0] | |
| cum = 0.0 | |
| for cat in nonzero: | |
| val = float(row[cat]) | |
| seg_bottom = base - cum * scale | |
| seg_top = base - (cum + val) * scale | |
| cum += val | |
| h = seg_bottom - seg_top | |
| disp = f"{val:.1f}%" if normalize else f"{val:,.0f} uses" | |
| title = f"<title>{esc(period)} · {esc(cat)}: {disp}</title>" | |
| fill = f"var(--cat-{esc(cat)})" | |
| is_top = cat == nonzero[-1] | |
| gy, gh = seg_top + 1, h - 2 # 2px surface gap between segments | |
| if gh < 1: | |
| gy, gh = seg_top, max(h, 0.8) | |
| if is_top and gh > 6: | |
| parts.append(f'<path class="seg" d="{_rounded_top_rect(x0, gy, BW, gh, 4)}" fill="{fill}">{title}</path>') | |
| else: | |
| parts.append(f'<rect class="seg" x="{x0:.1f}" y="{gy:.1f}" width="{BW}" height="{gh:.1f}" fill="{fill}">{title}</rect>') | |
| parts.append( | |
| f'<text x="{ML + i * band + band / 2:.1f}" y="{base + 20}" text-anchor="middle" ' | |
| f'font-size="11" fill="var(--muted)">{esc(period)}</text>' | |
| ) | |
| parts.append(f'<line x1="{ML}" y1="{base}" x2="{W - MR}" y2="{base}" stroke="var(--muted)" stroke-width="1"/>') | |
| parts.append("</svg>") | |
| return "".join(parts) | |
| def category_view(openness_filter, mode): | |
| df = USAGE_DF | |
| if openness_filter == "Open only": | |
| df = df[df["is_open"]] | |
| elif openness_filter == "Closed only": | |
| df = df[~df["is_open"]] | |
| normalize = mode == "Share of period (%)" | |
| df = df.drop_duplicates(subset=["model_id", "benchmark", "bucket"]) | |
| counts = df.groupby(["period", "bucket"]).size().unstack(fill_value=0) | |
| counts = counts.reindex(columns=[c for c in CATEGORY_ORDER if c in counts.columns], fill_value=0) | |
| counts = counts.reindex(PERIODS, fill_value=0) | |
| legend = '<div class="legend">' + "".join( | |
| f'<span class="key"><i style="background:var(--cat-{esc(c)})"></i>{esc(c)}</span>' for c in counts.columns | |
| ) + "</div>" | |
| title = "Benchmark category mix over time" + (" (% of period total)" if normalize else " (raw benchmark-use count)") | |
| chart = f'<div class="viz-title">{esc(title)}</div>' + legend + svg_category_bars(counts, normalize) | |
| rows = [] | |
| for period, row in counts.iterrows(): | |
| cells = "".join(f'<td class="num">{int(v)}</td>' for v in row) | |
| rows.append(f"<tr><td><strong>{esc(period)}</strong></td>{cells}</tr>") | |
| head = "<th>Period</th>" + "".join(f"<th>{esc(c)}</th>" for c in counts.columns) | |
| table = ( | |
| '<div class="tbl-wrap"><table class="tbl"><thead><tr>' | |
| f'{head}</tr></thead><tbody>{"".join(rows)}</tbody></table></div>' | |
| ) | |
| return chart, table | |
| # --------------------------------------------------------------------------- | |
| # UI | |
| # --------------------------------------------------------------------------- | |
| DEFAULT_BENCH = "GPQA-Diamond" if "GPQA-Diamond" in ALL_BENCHMARKS else ALL_BENCHMARKS[0] | |
| with gr.Blocks(title="LLM Benchmark Usage Explorer") as demo: | |
| gr.HTML(render_hero()) | |
| with gr.Tab("🆕 Latest releases"): | |
| gr.HTML('<p class="tab-note">The most recent model releases, the benchmarks they report, and where the numbers come from.</p>') | |
| rel_count_pills = PillGroup(["10", "20", "50"], "10", "Show") | |
| rel_out = gr.HTML(latest_releases_view("10")) | |
| rel_count_pills.change(latest_releases_view, rel_count_pills, rel_out) | |
| with gr.Tab("📊 Popularity & timeline"): | |
| gr.HTML('<p class="tab-note">Which benchmarks are used the most, and when did they first show up?</p>') | |
| with gr.Row(): | |
| cat_pills = PillGroup(["All"] + ALL_CATEGORIES, "All", "Category", scale=3) | |
| open_pills = PillGroup(["All", "Open only", "Closed only"], "All", "Model type", scale=2) | |
| with gr.Row(): | |
| min_models_sl = RangeSlider(1, 20, 1, 1, "Min. # models using it") | |
| search_box = SearchBox("", "Search benchmark name", "e.g. GPQA") | |
| pop_out = gr.HTML(popularity_view("All", "All", 1, "")) | |
| for ctrl in [cat_pills, open_pills, min_models_sl, search_box]: | |
| ctrl.change(popularity_view, [cat_pills, open_pills, min_models_sl, search_box], pop_out) | |
| with gr.Tab("🔎 Benchmark → Models"): | |
| gr.HTML('<p class="tab-note">Pick a benchmark to see every model that reports it, and whether that model is open or closed.</p>') | |
| bench_sel = SelectBox(ALL_BENCHMARKS, DEFAULT_BENCH, "Benchmark") | |
| _s, _c, _t = benchmark_view(DEFAULT_BENCH) | |
| bench_summary = gr.HTML(_s) | |
| bench_chart = gr.HTML(_c) | |
| bench_table = gr.HTML(_t, js_on_load=TABLE_FILTER_JS) | |
| bench_sel.change(benchmark_view, bench_sel, [bench_summary, bench_chart, bench_table]) | |
| with gr.Tab("🧬 Model → Benchmarks"): | |
| gr.HTML('<p class="tab-note">Pick a model to see its full evaluation suite, grouped by category.</p>') | |
| model_sel = SelectBox(ALL_MODELS, ALL_MODELS[0], "Model") | |
| _s, _b = model_view(ALL_MODELS[0]) | |
| model_summary = gr.HTML(_s) | |
| model_body = gr.HTML(_b) | |
| model_sel.change(model_view, model_sel, [model_summary, model_body]) | |
| with gr.Tab("📈 Category evolution"): | |
| gr.HTML('<p class="tab-note">How has the <em>type</em> of benchmark being reported shifted over time?</p>') | |
| with gr.Row(): | |
| cat_open_pills = PillGroup(["All", "Open only", "Closed only"], "All", "Model type") | |
| norm_pills = PillGroup(["Share of period (%)", "Raw counts"], "Share of period (%)", "Y-axis") | |
| _c, _t = category_view("All", "Share of period (%)") | |
| cat_chart = gr.HTML(_c) | |
| cat_table = gr.HTML(_t) | |
| for ctrl in [cat_open_pills, norm_pills]: | |
| ctrl.change(category_view, [cat_open_pills, norm_pills], [cat_chart, cat_table]) | |
| if __name__ == "__main__": | |
| demo.launch(css=CUSTOM_CSS) | |