""",
js_on_load="""
element.addEventListener('click', (e) => {
const btn = e.target.closest('.pill');
if (!btn) return;
props.value = btn.dataset.v;
trigger('change');
});""",
**kwargs,
)
def api_info(self):
return {"type": "string"}
class SelectBox(gr.HTML):
"""Type-to-filter combobox (replaces gr.Dropdown with filterable=True):
a text input that opens a filtered option list, with arrow/Enter/Escape
keyboard support. All listeners are delegated on the component root so
they survive the re-render that follows each value sync."""
def __init__(self, options, value, label, **kwargs):
super().__init__(
value=value,
options=list(options),
label_text=label,
html_template="""
${label_text}
▾
${options.map(o => `
${o.replace(/&/g,'&').replace(/`).join('')}
No match
""",
js_on_load="""
const list = () => element.querySelector('.cbx-list');
const input = () => element.querySelector('.cbx-input');
const items = () => [...element.querySelectorAll('.cbx-item')];
const visible = () => items().filter(it => !it.hidden);
function open(showAll) {
if (showAll) { items().forEach(it => { it.hidden = false; }); element.querySelector('.cbx-empty').hidden = true; }
list().hidden = false;
input().setAttribute('aria-expanded', 'true');
}
function close(restore) {
list().hidden = true;
input().setAttribute('aria-expanded', 'false');
items().forEach(it => it.classList.remove('active'));
if (restore) input().value = props.value;
}
function applyFilter() {
const q = input().value.toLowerCase();
let any = false;
items().forEach(it => {
it.hidden = !it.textContent.toLowerCase().includes(q);
it.classList.remove('active');
if (!it.hidden) any = true;
});
element.querySelector('.cbx-empty').hidden = any;
}
function move(dir) {
const vis = visible();
if (!vis.length) return;
const cur = vis.findIndex(it => it.classList.contains('active'));
const next = Math.min(Math.max(cur + dir, 0), vis.length - 1);
vis.forEach(it => it.classList.remove('active'));
vis[next].classList.add('active');
vis[next].scrollIntoView({ block: 'nearest' });
}
function pick(v) {
close(false);
props.value = v;
trigger('change');
}
element.addEventListener('focusin', (e) => {
if (!e.target.classList.contains('cbx-input')) return;
e.target.select();
open(true);
});
element.addEventListener('input', (e) => {
if (!e.target.classList.contains('cbx-input')) return;
open(false);
applyFilter();
});
element.addEventListener('keydown', (e) => {
if (!e.target.classList.contains('cbx-input')) return;
if (e.key === 'ArrowDown') { e.preventDefault(); if (list().hidden) open(true); move(1); }
else if (e.key === 'ArrowUp') { e.preventDefault(); move(-1); }
else if (e.key === 'Enter') {
e.preventDefault();
const target = visible().find(it => it.classList.contains('active')) || visible()[0];
if (target) pick(target.dataset.v);
}
else if (e.key === 'Escape') { close(true); e.target.blur(); }
});
element.addEventListener('pointerdown', (e) => {
const it = e.target.closest('.cbx-item');
if (!it) return;
e.preventDefault(); // beat the input's blur
pick(it.dataset.v);
});
element.addEventListener('focusout', (e) => {
if (!element.contains(e.relatedTarget)) close(true);
});""",
**kwargs,
)
def api_info(self):
return {"type": "string"}
class RangeSlider(gr.HTML):
"""Native range input with a value readout (replaces gr.Slider)."""
def __init__(self, minimum, maximum, value, step, label, **kwargs):
super().__init__(
value=value,
minimum=minimum,
maximum=maximum,
step=step,
label_text=label,
html_template="""
${label_text}
${value}
""",
js_on_load="""
element.addEventListener('input', (e) => {
if (!e.target.classList.contains('ctl-range')) return;
const out = element.querySelector('.ctl-range-val');
if (out) out.textContent = e.target.value;
});
element.addEventListener('change', (e) => {
if (!e.target.classList.contains('ctl-range')) return;
props.value = Number(e.target.value);
trigger('change');
});""",
**kwargs,
)
def api_info(self):
return {"type": "integer"}
class SearchBox(gr.HTML):
"""Debounced text search (replaces gr.Textbox). Focus is restored after the
value-sync re-render so typing isn't interrupted."""
def __init__(self, value, label, placeholder, **kwargs):
super().__init__(
value=value,
label_text=label,
placeholder=placeholder,
html_template="""
${label_text}
""",
js_on_load="""
let t = null;
element.addEventListener('input', (e) => {
if (!e.target.classList.contains('ctl-search')) return;
clearTimeout(t);
const v = e.target.value;
t = setTimeout(() => {
props.value = v;
trigger('change');
setTimeout(() => {
const inp = element.querySelector('.ctl-search');
if (inp && document.activeElement !== inp) {
inp.focus();
inp.setSelectionRange(inp.value.length, inp.value.length);
}
}, 60);
}, 250);
});""",
**kwargs,
)
def api_info(self):
return {"type": "string"}
# Client-side row filter for rendered tables: filters rows against the
# .tbl-filter input, no server round trip. Attached via js_on_load so the
# delegated listener survives value updates.
TABLE_FILTER_JS = """
element.addEventListener('input', (e) => {
if (!e.target.classList.contains('tbl-filter')) return;
const q = e.target.value.toLowerCase();
element.querySelectorAll('tbody tr').forEach(tr => {
tr.style.display = tr.textContent.toLowerCase().includes(q) ? '' : 'none';
});
});"""
# ---------------------------------------------------------------------------
# Small HTML helpers
# ---------------------------------------------------------------------------
def cat_chip(bucket: str) -> str:
return f'{esc(bucket)}'
def openness_chip(is_open: bool) -> str:
color, text = ("var(--open)", "open") if is_open else ("var(--closed)", "closed")
return f'{text}'
def source_label(url: str) -> str:
"""The dataset no longer carries a source title/type, just a link — show its
host as a short, still-meaningful label (e.g. "arxiv.org", "huggingface.co")."""
return urlparse(url).netloc or url
def nice_ceil(v: float) -> float:
if v <= 0:
return 1
exp = 10 ** math.floor(math.log10(v))
frac = v / exp
for m in (1, 2, 2.5, 5, 10):
if frac <= m:
return m * exp
return v
def month_ticks(t0: pd.Timestamp, t1: pd.Timestamp) -> list[pd.Timestamp]:
span = max((t1 - t0).days, 1)
step = next((s for s in (1, 2, 3, 6, 12, 24) if span / (30.4 * s) <= 7), 24)
m0 = ((t0.month - 1) // step) * step + 1
d = pd.Timestamp(year=t0.year, month=m0, day=1)
ticks = []
while d <= t1:
if d >= t0:
ticks.append(d)
total = (d.month - 1) + step
d = pd.Timestamp(year=d.year + total // 12, month=total % 12 + 1, day=1)
return ticks
# ---------------------------------------------------------------------------
# Hero header
# ---------------------------------------------------------------------------
def render_hero() -> str:
d0, d1 = MODELS_DF["release_date"].min().date(), MODELS_DF["release_date"].max().date()
tiles = [
("Models", str(len(MODELS_DF)), ""),
("Labs", str(MODELS_DF["lab"].nunique()), ""),
("Distinct benchmarks", str(USAGE_DF["benchmark"].nunique()), ""),
("Coverage", f"{d0} → {d1}", "sm"),
]
tiles_html = "".join(
f'
{esc(lab)}
{esc(val)}
'
for lab, val, cls in tiles
)
return (
'
LLM Benchmark Usage Explorer
'
'
Exploring SaylorTwift/llm-benchmark-usage — which benchmarks labs report, '
"who uses them, and how the mix shifts over time.
"
f'
{tiles_html}
'
)
# ---------------------------------------------------------------------------
# Tab 0: Latest releases (landing view)
# ---------------------------------------------------------------------------
def rel_date(d: pd.Timestamp) -> str:
days = (pd.Timestamp.now().normalize() - d.normalize()).days
if days <= 0:
return "today"
if days == 1:
return "yesterday"
if days < 7:
return f"{days} days ago"
if days < 60:
return f"{days // 7} week{'s' if days >= 14 else ''} ago"
if days < 365:
return f"{days // 30} months ago"
return f"{days // 365} year{'s' if days >= 730 else ''} ago"
MAX_RELEASE_CHIPS = 16
def latest_releases_view(count) -> str:
recent = MODELS_DF.sort_values("release_date", ascending=False).head(int(count))
bucket_rank = {b: i for i, b in enumerate(CATEGORY_ORDER)}
cards = []
for m in recent.itertuples():
usage = (
USAGE_DF[USAGE_DF["model_id"] == m.model_id]
.drop_duplicates(subset=["benchmark"])
.sort_values(["bucket", "benchmark"], key=lambda s: s.map(bucket_rank) if s.name == "bucket" else s)
)
if usage.empty:
body = '
No benchmark data recorded for this model.
'
else:
shown = usage.head(MAX_RELEASE_CHIPS)
chips = "".join(
f''
f'{esc(r.benchmark)}'
for r in shown.itertuples()
)
if len(usage) > MAX_RELEASE_CHIPS:
chips += f'+{len(usage) - MAX_RELEASE_CHIPS} more'
src_url = usage.iloc[0]["source"]
body = (
f'
"
)
return "".join(cards)
# ---------------------------------------------------------------------------
# Tab 1: Benchmark popularity
# ---------------------------------------------------------------------------
def popularity_view(category_filter, openness_filter, min_models, search) -> str:
df = USAGE_DF
if category_filter and category_filter != "All":
df = df[df["bucket"] == category_filter]
if openness_filter == "Open only":
df = df[df["is_open"]]
elif openness_filter == "Closed only":
df = df[~df["is_open"]]
# A model can appear several times for one benchmark (multiple sources report
# it), so count on deduplicated (benchmark, model) pairs; categories keep all rows.
uniq = df.drop_duplicates(subset=["benchmark", "model_id"])
agg = (
uniq.groupby("benchmark")
.agg(
models=("model_id", "nunique"),
open_models=("is_open", "sum"),
first_seen=("release_date", "min"),
last_seen=("release_date", "max"),
labs=("lab", "nunique"),
)
.reset_index()
)
buckets = df.groupby("benchmark")["bucket"].agg(lambda s: sorted(set(s))).rename("buckets")
agg = agg.merge(buckets, on="benchmark")
agg["closed_models"] = agg["models"] - agg["open_models"]
agg = agg[agg["models"] >= int(min_models)]
if search:
agg = agg[agg["benchmark"].str.contains(search, case=False, na=False, regex=False)]
agg = agg.sort_values(["models", "benchmark"], ascending=[False, True])
if agg.empty:
return '
No benchmarks match the current filters.
'
max_models = int(agg["models"].max())
rows = []
for i, r in enumerate(agg.itertuples(), start=1):
bar_pct = r.models / max_models * 100
tot = max(r.models, 1)
o_pct, c_pct = r.open_models / tot * 100, r.closed_models / tot * 100
chips = "".join(cat_chip(b) for b in r.buckets)
rows.append(
f"
"
f'
{i}
'
f"
{esc(r.benchmark)}
"
f'
{r.models}
'
f'
{int(r.open_models)} / {int(r.closed_models)}'
f''
f'
'
f'
{r.labs}
'
f"
{chips}
"
f'
{r.first_seen.date()}
'
f'
{r.last_seen.date()}
'
f"
"
)
return (
f'
{len(agg)} benchmarks match · sorted by number of models
'
'
'
"
#
Benchmark
Models
Open / closed
Labs
"
"
Categories
First seen
Last seen
"
f'
{"".join(rows)}
'
)
# ---------------------------------------------------------------------------
# Tab 2: Benchmark -> Models (summary card, monthly-usage SVG bars, table)
# ---------------------------------------------------------------------------
def _rounded_top_rect(x, y, w, h, r) -> str:
r = min(r, h / 2, w / 2)
return (
f'M {x:.1f} {y + h:.1f} L {x:.1f} {y + r:.1f} Q {x:.1f} {y:.1f} {x + r:.1f} {y:.1f} '
f'L {x + w - r:.1f} {y:.1f} Q {x + w:.1f} {y:.1f} {x + w:.1f} {y + r:.1f} L {x + w:.1f} {y + h:.1f} Z'
)
def svg_benchmark_weekly_usage(df: pd.DataFrame) -> str:
"""Bars of how many model releases reported this benchmark each week, spanning
the whole dataset time range (empty weeks stay visible as gaps)."""
t0 = MODELS_DF["release_date"].min().to_period("W")
t1 = MODELS_DF["release_date"].max().to_period("W")
weeks = pd.period_range(t0, t1, freq="W")
counts = df["release_date"].dt.to_period("W").value_counts().reindex(weeks, fill_value=0)
W, H, ML, MR, MT, MB = 920, 300, 44, 10, 12, 36
plot_h = H - MT - MB
base = MT + plot_h
vmax = int(counts.max())
step = max(1, math.ceil(vmax / 4))
ymax = step * 4
scale = plot_h / ymax
band = (W - ML - MR) / len(weeks)
bw = max(band - 1, 1.0) # 1px surface gap between adjacent bars (weekly bands are narrow)
parts = [
f'")
return "".join(parts)
def benchmark_view(benchmark):
if not benchmark:
return "", "", '
{len(df)} models from '
f"{df['lab'].nunique()} labs report it — "
f"{n_open} open-weight, {n_closed} closed. First seen "
f"{df['release_date'].min().date()}, most recent "
f"{df['release_date'].max().date()}.
"
)
chart = (
f'
How often is “{esc(benchmark)}” used? Model releases reporting it per week
'
+ svg_benchmark_weekly_usage(df)
)
rows = []
for r in df.itertuples():
rows.append(
"
'
)
sections = []
for bucket in CATEGORY_ORDER:
sub = df[df["bucket"] == bucket]
if sub.empty:
continue
chips = "".join(
f''
f'{esc(r.benchmark)}'
for r in sub.itertuples()
)
sections.append(
f'
'
f'{esc(bucket)} ({len(sub)})
{chips}
'
)
return summary, "".join(sections)
# ---------------------------------------------------------------------------
# Tab 4: Category evolution (SVG stacked bars + table)
# ---------------------------------------------------------------------------
def svg_category_bars(counts: pd.DataFrame, normalize: bool) -> str:
periods = counts.index.tolist()
plot = counts.copy()
if normalize:
plot = plot.div(plot.sum(axis=1).replace(0, 1), axis=0) * 100
W, H, ML, MR, MT, MB, BW = 920, 430, 52, 10, 10, 36, 24
plot_h = H - MT - MB
base = MT + plot_h
ymax = 100.0 if normalize else float(nice_ceil(plot.sum(axis=1).max()))
scale = plot_h / ymax
band = (W - ML - MR) / max(len(periods), 1)
parts = [
f'")
return "".join(parts)
def category_view(openness_filter, mode):
df = USAGE_DF
if openness_filter == "Open only":
df = df[df["is_open"]]
elif openness_filter == "Closed only":
df = df[~df["is_open"]]
normalize = mode == "Share of period (%)"
df = df.drop_duplicates(subset=["model_id", "benchmark", "bucket"])
counts = df.groupby(["period", "bucket"]).size().unstack(fill_value=0)
counts = counts.reindex(columns=[c for c in CATEGORY_ORDER if c in counts.columns], fill_value=0)
counts = counts.reindex(PERIODS, fill_value=0)
legend = '
' + "".join(
f'{esc(c)}' for c in counts.columns
) + "
"
title = "Benchmark category mix over time" + (" (% of period total)" if normalize else " (raw benchmark-use count)")
chart = f'
{esc(title)}
' + legend + svg_category_bars(counts, normalize)
rows = []
for period, row in counts.iterrows():
cells = "".join(f'
{int(v)}
' for v in row)
rows.append(f"
{esc(period)}
{cells}
")
head = "
Period
" + "".join(f"
{esc(c)}
" for c in counts.columns)
table = (
'
'
f'{head}
{"".join(rows)}
'
)
return chart, table
# ---------------------------------------------------------------------------
# UI
# ---------------------------------------------------------------------------
DEFAULT_BENCH = "GPQA-Diamond" if "GPQA-Diamond" in ALL_BENCHMARKS else ALL_BENCHMARKS[0]
with gr.Blocks(title="LLM Benchmark Usage Explorer") as demo:
gr.HTML(render_hero())
with gr.Tab("🆕 Latest releases"):
gr.HTML('
The most recent model releases, the benchmarks they report, and where the numbers come from.