Spaces:
Running
Running
| """Gradio component builders for the leaderboard. | |
| Each function creates one piece of the UI inside the surrounding Blocks/render | |
| context. No data-path logic lives here (see ``data_loading.py``) and no copy | |
| lives here (see ``website_texts.py``). | |
| """ | |
| from __future__ import annotations | |
| import html | |
| import re | |
| from itertools import groupby | |
| from pathlib import Path | |
| import gradio as gr | |
| import pandas as pd | |
| from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns | |
| import website_texts | |
| from constants import Constants, model_type_color, model_type_emoji | |
| from data_loading import ( | |
| BEYOND_SUBSET_LABELS, | |
| DATASET_SIZE_NOTE, | |
| BeyondSubset, | |
| LBContainer, | |
| Subset, | |
| load_leaderboard_csv, | |
| subset_name, | |
| unzip_png, | |
| ) | |
| # --------------------------------------------------------------------------- # | |
| # Full per-subset leaderboard table | |
| # --------------------------------------------------------------------------- # | |
| _IMPUTED_INFO = ( | |
| "We impute the performance for models that cannot run on all datasets due to" | |
| " task or dataset size constraints. We impute with the performance of a" | |
| " default RandomForest. We add a postfix [X% IMPUTED] to the model if any" | |
| " results were imputed. The X% shows the percentage of datasets that were" | |
| " imputed. In general, imputation negatively represents the model" | |
| " performance, punishing the model for not being able to run on all datasets." | |
| ) | |
| def make_leaderboard(lb: LBContainer) -> Leaderboard: | |
| df = lb.load_df() | |
| # -- Derived filter columns | |
| df["TypeFiler"] = df["TypeName"].apply(lambda m: f"{m} {model_type_emoji[m]}") | |
| df["Only Default"] = df["Model"].str.contains("(default)", regex=False) | |
| df["Only Tuned"] = df["Model"].str.contains("(tuned)", regex=False) | |
| df["Only Tuned + Ensembled"] = df["Model"].str.contains( | |
| "(tuned + ensembled)", regex=False | |
| ) | df["Model"].str.contains("AutoGluon", regex=False) | |
| filter_columns = [ | |
| ColumnFilter("TypeFiler", type="checkboxgroup", label="🤖 Model Types"), | |
| ColumnFilter("Only Default", type="boolean", default=False), | |
| ColumnFilter("Only Tuned", type="boolean", default=False), | |
| ColumnFilter("Only Tuned + Ensembled", type="boolean", default=False), | |
| ] | |
| # -- Imputation column (only meaningful when some rows are imputed) | |
| if df["Imputed"].any(): | |
| df["Imputed"] = df["Imputed"].replace({True: "Imputed", False: "Not Imputed"}) | |
| filter_columns.append( | |
| ColumnFilter( | |
| "Imputed", | |
| type="checkboxgroup", | |
| label="(Not) Imputed Models", | |
| info=_IMPUTED_INFO, | |
| ) | |
| ) | |
| else: | |
| df = df.drop(columns=["Imputed (%) [⬇️]"]) | |
| # -- Color model names by type so the table shares the overview's hue. | |
| def _color_model(row: pd.Series) -> str: | |
| color = model_type_color.get(row["TypeName"], "#dddddd") | |
| link = re.match(r"\[(.*?)\]\((.*?)\)(.*)", row["Model"]) | |
| if link: | |
| text, url, rest = link.groups() | |
| return ( | |
| f'<a href="{url}" target="_blank" rel="noopener" ' | |
| f'style="color:{color};font-weight:600;">{text}</a>{rest}' | |
| ) | |
| return f'<span style="color:{color};font-weight:600;">{row["Model"]}</span>' | |
| df["Model"] = df.apply(_color_model, axis=1) | |
| # -- Column datatypes (read straight from dtypes; transposing collapses to | |
| # object and mistypes every numeric column as markdown). | |
| datatypes = [] | |
| for dtype in df.dtypes: | |
| if pd.api.types.is_bool_dtype(dtype): # check bool first (it is also "numeric") | |
| datatypes.append("bool") | |
| elif pd.api.types.is_numeric_dtype(dtype): | |
| datatypes.append("number") | |
| else: | |
| datatypes.append("markdown") | |
| return Leaderboard( | |
| elem_id=f"lb_{lb.subset.rel_path.replace('/', '_')}", | |
| value=df, | |
| datatype=datatypes, | |
| select_columns=SelectColumns( | |
| default_selection=list(df.columns), | |
| cant_deselect=["Type", "Model"], | |
| label="Select Columns to Display:", | |
| ), | |
| hide_columns=[ | |
| "TypeName", | |
| "TypeFiler", | |
| "RefModel", | |
| "Only Default", | |
| "Only Tuned", | |
| "Only Tuned + Ensembled", | |
| "Imputed", | |
| ], | |
| search_columns=["Model", "TypeName"], | |
| filter_columns=filter_columns, | |
| bool_checkboxgroup_label="Custom Views (exclusive, only toggle one at a time):", | |
| height=800, | |
| ) | |
| # --------------------------------------------------------------------------- # | |
| # Per-subset figures | |
| # --------------------------------------------------------------------------- # | |
| def make_overview_images(lb: LBContainer) -> None: | |
| name = subset_name(lb.subset) | |
| gr.Image( | |
| lb.image_path("tuning-impact-elo"), | |
| label=f"Leaderboard Overview [{name}]", | |
| show_label=True, | |
| height=500, | |
| show_share_button=True, | |
| ) | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| gr.Image( | |
| value=lb.image_path("pareto_front_improvability_vs_time_infer"), | |
| label=f"Inference Time Pareto Front [{name}]", | |
| height=400, | |
| show_label=True, | |
| show_share_button=True, | |
| ) | |
| with gr.Column(scale=1): | |
| gr.Image( | |
| value=lb.image_path("pareto_n_configs_imp"), | |
| label=f"Tuning Trajectories [{name}]", | |
| height=400, | |
| show_label=True, | |
| show_share_button=True, | |
| ) | |
| def make_winrate_image(lb: LBContainer) -> None: | |
| # Uses ``lb.name`` (the caller-supplied subset name) so it is benchmark-agnostic and reusable | |
| # across both the TabArena and BeyondArena tabs. | |
| gr.Image( | |
| lb.image_path("winrate_matrix"), | |
| label=f"Win-rate Matrix [{lb.name}]", | |
| show_label=True, | |
| height=800, | |
| show_share_button=True, | |
| ) | |
| # --------------------------------------------------------------------------- # | |
| # Cross-subset overview (Elo heatmap, rendered as HTML for links + grouping) | |
| # --------------------------------------------------------------------------- # | |
| # Columns of the overview: (label, group, subset). Always imputation=yes/splits=all. | |
| _OVERVIEW_COLUMNS: list[tuple[str, str, Subset]] = [ | |
| ("Overall", "overall", Subset(tasks="all", datasets="all")), | |
| ("Class.", "task", Subset(tasks="classification", datasets="all")), | |
| ("Regr.", "task", Subset(tasks="regression", datasets="all")), | |
| ("Binary", "task", Subset(tasks="binary", datasets="all")), | |
| ("Multi.", "task", Subset(tasks="multiclass", datasets="all")), | |
| ("Small", "size", Subset(tasks="all", datasets="small")), | |
| ("Medium", "size", Subset(tasks="all", datasets="medium")), | |
| ] | |
| _GROUP_TITLE = {"task": "By Task", "size": "By Dataset Size"} | |
| # Hover tooltips for the overview's subset column headers (rendered as a native `title=`). | |
| # Size definitions are reused from DATASET_SIZE_NOTE so they can't drift from the subset blurbs. | |
| _COLUMN_TOOLTIPS: dict[str, str] = { | |
| "Overall": "All tasks across every dataset size — the headline ranking.", | |
| "Class.": "Classification tasks only (binary + multiclass).", | |
| "Regr.": "Regression tasks only.", | |
| "Binary": "Binary classification tasks only.", | |
| "Multi.": "Multiclass classification tasks only.", | |
| "Small": DATASET_SIZE_NOTE["small"], | |
| "Medium": DATASET_SIZE_NOTE["medium"], | |
| } | |
| # Top-3 per column get a medal in a fixed-width slot reserved in every number cell, so the | |
| # numbers line up whether or not a medal is present (see `.ta-medal`/`.ta-val` in `_OVERVIEW_CSS`). | |
| _MEDALS = {1: "🥇", 2: "🥈", 3: "🥉"} | |
| _VARIANT_RE = re.compile(r"\((default|tuned \+ ensembled|tuned)\)") | |
| # Metrics selectable in the overview: label -> (csv column, higher_is_better, formatter). | |
| # Order = display order in the selector; the two headline metrics come first. | |
| _OVERVIEW_METRIC_SPECS: dict[str, tuple] = { | |
| "Elo": ("Elo [⬆️]", True, lambda v: str(int(round(v)))), | |
| "Improvability (%)": ("Improvability (%) [⬇️]", False, lambda v: f"{v:.1f}"), | |
| "Score": ("Score [⬆️]", True, lambda v: f"{v:.3f}"), | |
| "Average Rank": ("Rank [⬇️]", False, lambda v: f"{v:.2f}"), | |
| "Harmonic Rank": ("Harmonic Rank [⬇️]", False, lambda v: f"{v:.2f}"), | |
| } | |
| OVERVIEW_METRIC_CHOICES = list(_OVERVIEW_METRIC_SPECS) | |
| # One-line TL;DR shown next to the overview metric selector. | |
| OVERVIEW_METRIC_TLDR = { | |
| "Elo": "Pairwise win-rate rating (a 400-point gap ≈ a 91% win rate). Higher is better.", | |
| "Improvability (%)": "How much lower the best model's error is than this one's, per dataset. Lower is better.", | |
| "Score": "Error rescaled per dataset to 1 (best) … 0 (median), then averaged. Higher is better.", | |
| "Average Rank": "The model's mean rank across datasets. Lower is better.", | |
| "Harmonic Rank": "Harmonic mean of per-dataset ranks — rewards being excellent on some datasets. Lower is better.", | |
| } | |
| def _parse_model(model: str) -> tuple[str, str, str | None]: | |
| """Split a Model cell into (base name, variant, url).""" | |
| link = re.match(r"\[(.*?)\]\((.*?)\)", model) | |
| text, url = (link.group(1), link.group(2)) if link else (model, None) | |
| text = text.split("[")[0].strip() # drop any [X% IMPUTED] tag | |
| variant_match = _VARIANT_RE.search(text) | |
| variant = variant_match.group(1) if variant_match else "" | |
| base = _VARIANT_RE.sub("", text).strip() | |
| return base, variant, url | |
| def _subset_best(df: pd.DataFrame, column: str, higher_is_better: bool) -> pd.DataFrame: | |
| """Best variant per model in a subset for `column`; excludes reference pipelines.""" | |
| df = df[~df["TypeName"].isin([Constants.reference])].copy() | |
| df = df.dropna(subset=[column]) | |
| parsed = df["Model"].map(_parse_model) | |
| df["base"] = [p[0] for p in parsed] | |
| df["variant"] = [p[1] for p in parsed] | |
| df["url"] = [p[2] for p in parsed] | |
| df["imputed"] = df["Imputed"].astype(bool) if "Imputed" in df.columns else False | |
| df["verified"] = df["Verified"] if "Verified" in df.columns else "" | |
| grouped = df.groupby("base")[column] | |
| best = df.loc[grouped.idxmax() if higher_is_better else grouped.idxmin()] | |
| return best[ | |
| ["base", "TypeName", "Type", "variant", "url", column, "verified", "imputed"] | |
| ].rename(columns={column: "val"}) | |
| def _overview_th(label: str, *, rowspan: int | None = None) -> str: | |
| """A subset column header `<th>`; gets a hover tooltip + 'help' affordance when defined.""" | |
| attrs = f' rowspan="{rowspan}"' if rowspan else "" | |
| tooltip = _COLUMN_TOOLTIPS.get(label) | |
| if tooltip: | |
| attrs += f' class="ta-th-info" title="{html.escape(tooltip, quote=True)}"' | |
| return f"<th{attrs}>{label}</th>" | |
| def _interp_color(frac: float) -> str: | |
| """Map 0 (best) .. 1 (worst) to a green->olive->red hex (readable on dark bg).""" | |
| stops = [(0.0, (28, 120, 62)), (0.5, (138, 122, 36)), (1.0, (160, 58, 58))] | |
| frac = max(0.0, min(1.0, frac)) | |
| for (f0, c0), (f1, c1) in zip(stops, stops[1:]): | |
| if frac <= f1: | |
| t = 0 if f1 == f0 else (frac - f0) / (f1 - f0) | |
| r, g, b = (round(a + (b_ - a) * t) for a, b_ in zip(c0, c1)) | |
| return f"#{r:02x}{g:02x}{b:02x}" | |
| return "#a03a3a" | |
| _OVERVIEW_CSS = """ | |
| <style> | |
| .ta-overview { border-collapse: collapse; width: 100%; font-size: 1.08em; } | |
| .ta-overview th, .ta-overview td { padding: 5px 9px; text-align: center; border: 1px solid #ffffff14; } | |
| .ta-overview thead th { background: #1b1b22; font-weight: 600; position: sticky; z-index: 5; box-shadow: inset 0 1px 0 #ffffff14, inset 0 -1px 0 #ffffff14; } | |
| /* Subset column headers carry an explanatory tooltip (title=); hint it with a help cursor + dotted underline. */ | |
| .ta-overview thead th.ta-th-info { cursor: help; text-decoration: underline; text-decoration-style: dotted; text-decoration-color: #ffffff66; text-underline-offset: 3px; } | |
| .ta-overview thead tr:first-child th { top: 0; } | |
| .ta-overview thead tr:nth-child(2) th { top: 33px; } | |
| .ta-overview .ta-group-h { border-bottom: 2px solid #ffffff33; } | |
| .ta-overview td.ta-model-cell { text-align: left; white-space: nowrap; } | |
| .ta-overview .ta-variant { opacity: 0.5; font-weight: 400; font-size: 0.9em; } | |
| /* Numbers match the rest of the table text in size; bold keeps them emphasized. */ | |
| .ta-overview td.ta-num { font-weight: 600; } | |
| /* Reserve a fixed-width medal slot in every number cell so the digits line up in their | |
| own sub-column whether or not a top-3 medal is present. */ | |
| .ta-overview td.ta-num .ta-cell { display: inline-flex; align-items: baseline; } | |
| .ta-overview td.ta-num .ta-medal { flex: 0 0 1.45em; width: 1.45em; text-align: left; font-size: 0.78em; } | |
| .ta-overview td.ta-na { color: #777; } | |
| .ta-overview td.ta-model-cell a { text-decoration: underline; text-decoration-style: dotted; text-underline-offset: 3px; } | |
| .ta-overview td.ta-model-cell a:hover { text-decoration-style: solid; } | |
| .ta-link-icon { font-size: 0.78em; opacity: 0.65; margin-left: 2px; } | |
| .ta-imp { color: #e6c14d; font-weight: 700; margin-left: 1px; } | |
| .ta-verified { font-size: 0.85em; } | |
| .ta-pill { padding: 1px 7px; border-radius: 999px; font-size: 0.95em; } | |
| .ta-scroll { | |
| overflow: auto; | |
| max-height: 680px; | |
| border: 1px solid #ffffff1f; | |
| border-radius: 10px; | |
| scrollbar-width: thin; | |
| scrollbar-color: #ffffff3a transparent; | |
| scrollbar-gutter: stable; | |
| } | |
| .ta-scroll::-webkit-scrollbar { width: 11px; height: 11px; } | |
| .ta-scroll::-webkit-scrollbar-track { background: transparent; } | |
| .ta-scroll::-webkit-scrollbar-thumb { background: #ffffff33; border-radius: 8px; border: 3px solid transparent; background-clip: content-box; } | |
| .ta-scroll::-webkit-scrollbar-thumb:hover { background: #ffffff5c; background-clip: content-box; } | |
| .ta-scroll::-webkit-scrollbar-corner { background: transparent; } | |
| .ta-cap { font-size: 0.85em; opacity: 0.8; margin: 6px 0 4px 0; } | |
| .ta-legend { margin: 0 0 10px 0; } | |
| </style> | |
| """ | |
| def type_legend_html(include_reference: bool = True) -> str: | |
| """A small legend explaining the model-type symbols (shared across tables). | |
| Baseline and Other share one color, so they are shown as a single entry. | |
| `include_reference` is False for the overview, which excludes reference pipelines. | |
| """ | |
| e = model_type_emoji | |
| entries = [ | |
| (Constants.foundational, e[Constants.foundational], "Foundation Model"), | |
| (Constants.neural_network, e[Constants.neural_network], "Neural Network"), | |
| (Constants.tree, e[Constants.tree], "Tree-based"), | |
| (Constants.baseline, f"{e[Constants.baseline]} {e[Constants.other]}", "Baseline / Other"), | |
| ] | |
| if include_reference: | |
| entries.append((Constants.reference, e[Constants.reference], "Reference Pipeline")) | |
| chips = [] | |
| for type_name, symbols, label in entries: | |
| color = model_type_color.get(type_name, "#9e9e9e") | |
| chips.append( | |
| f'<span style="display:inline-block;margin:2px 12px 2px 0;white-space:nowrap;">' | |
| f'<span class="ta-pill" style="background:{color}22;color:{color};border:1px solid {color}66;">{symbols}</span> ' | |
| f'<span style="color:{color};font-size:0.85em;">{label}</span></span>' | |
| ) | |
| return f'{_OVERVIEW_CSS}<div class="ta-legend">{"".join(chips)}</div>' | |
| def make_cross_subset_overview(data_root: Path, metric: str = "Elo") -> gr.HTML: | |
| """Heatmap of the best `metric` per model (rows) and subset (columns).""" | |
| if metric not in _OVERVIEW_METRIC_SPECS: | |
| metric = "Elo" | |
| column, higher_is_better, fmt = _OVERVIEW_METRIC_SPECS[metric] | |
| val_by_col: dict[str, dict[str, float]] = {} | |
| imp_by_col: dict[str, dict[str, bool]] = {} | |
| meta: dict[str, dict] = {} | |
| present: list[tuple[str, str]] = [] # (label, group) | |
| for label, group, subset in _OVERVIEW_COLUMNS: | |
| path = Path(data_root) / subset.rel_path / "website_leaderboard.csv" | |
| if not path.exists(): | |
| continue | |
| best = _subset_best(load_leaderboard_csv(str(path.resolve())), column, higher_is_better) | |
| val_by_col[label] = dict(zip(best["base"], best["val"])) | |
| imp_by_col[label] = dict(zip(best["base"], best["imputed"])) | |
| present.append((label, group)) | |
| for _, row in best.iterrows(): | |
| meta.setdefault( | |
| row["base"], | |
| { | |
| "type_name": row["TypeName"], | |
| "emoji": row["Type"], | |
| "variant": row["variant"], | |
| "url": row["url"], | |
| "verified": row["verified"], | |
| }, | |
| ) | |
| if not present: | |
| return gr.HTML("<p>No overview data available.</p>") | |
| sort_label = present[0][0] | |
| worst_sort = float("-inf") if higher_is_better else float("inf") | |
| bases = sorted( | |
| meta, key=lambda b: val_by_col[sort_label].get(b, worst_sort), reverse=higher_is_better | |
| ) | |
| rank_by_col, bounds = {}, {} | |
| for label, _ in present: | |
| col = val_by_col[label] | |
| ranked = sorted(col, key=lambda b: col[b], reverse=higher_is_better) | |
| rank_by_col[label] = {b: i + 1 for i, b in enumerate(ranked[:3])} | |
| bounds[label] = (min(col.values()), max(col.values())) if col else (0.0, 1.0) | |
| # -- Grouped header (two rows) | |
| row1 = ['<th rowspan="2">Type</th>', '<th rowspan="2">Model</th>'] | |
| row2 = [] | |
| for group, items in groupby(present, key=lambda x: x[1]): | |
| items = list(items) | |
| if group == "overall": | |
| for label, _ in items: | |
| row1.append(_overview_th(label, rowspan=2)) | |
| else: | |
| row1.append(f'<th colspan="{len(items)}" class="ta-group-h">{_GROUP_TITLE[group]}</th>') | |
| row2.extend(_overview_th(label) for label, _ in items) | |
| header = f"<thead><tr>{''.join(row1)}</tr><tr>{''.join(row2)}</tr></thead>" | |
| # -- Body | |
| body = [] | |
| for base in bases: | |
| m = meta[base] | |
| color = model_type_color.get(m["type_name"], "#9e9e9e") | |
| name = html.escape(base) | |
| if m["variant"]: | |
| name += f' <span class="ta-variant">({html.escape(m["variant"])})</span>' | |
| if m.get("verified") == "✔️": | |
| name += ' <span class="ta-verified" title="Verified implementation">✔️</span>' | |
| if m["url"]: | |
| name_html = ( | |
| f'<a href="{html.escape(m["url"])}" target="_blank" rel="noopener" ' | |
| f'style="color:{color};font-weight:600;">{name}<span class="ta-link-icon">↗</span></a>' | |
| ) | |
| else: | |
| name_html = f'<span style="color:{color};font-weight:600;">{name}</span>' | |
| cells = [ | |
| f'<td><span class="ta-pill" style="background:{color}22;color:{color};' | |
| f'border:1px solid {color}66;">{m["emoji"]}</span></td>', | |
| f'<td class="ta-model-cell">{name_html}</td>', | |
| ] | |
| for label, _ in present: | |
| val = val_by_col[label].get(base) | |
| if val is None: | |
| cells.append('<td class="ta-na">–</td>') | |
| continue | |
| lo, hi = bounds[label] | |
| if hi <= lo: | |
| frac_best = 0.5 | |
| else: | |
| frac_best = (val - lo) / (hi - lo) if higher_is_better else (hi - val) / (hi - lo) | |
| bg = _interp_color(1 - frac_best) | |
| medal = _MEDALS.get(rank_by_col[label].get(base), "") | |
| imp = ( | |
| '<sup class="ta-imp" title="Score is (partly) imputed">*</sup>' | |
| if imp_by_col[label].get(base) | |
| else "" | |
| ) | |
| cells.append( | |
| f'<td class="ta-num" style="background:{bg};color:#f7f7f7;">' | |
| f'<span class="ta-cell"><span class="ta-medal">{medal}</span>' | |
| f'<span class="ta-val">{fmt(val)}{imp}</span></span></td>' | |
| ) | |
| body.append(f"<tr>{''.join(cells)}</tr>") | |
| direction = "Higher is better" if higher_is_better else "Lower is better" | |
| caption = ( | |
| f'<div class="ta-cap">Best <b>{html.escape(metric)}</b> per model across subsets ' | |
| f"(with imputation, all repeats). {direction}; 🥇🥈🥉 mark the top 3 in each column. " | |
| "Each model shows its best-performing variant.</div>" | |
| '<div class="ta-cap ta-cap-legend">✔️ = verified implementation · ' | |
| '<span class="ta-imp">*</span> = (partly) imputed score · ' | |
| "💡 Click any <u>underlined</u> model name (↗) to open its paper or code.</div>" | |
| ) | |
| table = f'<table class="ta-overview">{header}<tbody>{"".join(body)}</tbody></table>' | |
| return gr.HTML( | |
| f'{type_legend_html(include_reference=False)}{caption}<div class="ta-scroll">{table}</div>', | |
| elem_classes="ta-overview-block", | |
| ) | |
| # --------------------------------------------------------------------------- # | |
| # Metric reference and agentic guide | |
| # --------------------------------------------------------------------------- # | |
| def make_metric_reference() -> None: | |
| """Compact column key — pills that reveal a single shared panel (one open at a time). | |
| Uses a CSS-only radio group (no JS): selecting a pill checks its hidden radio, which | |
| reveals just that definition in the shared panel below; the ✕ rechecks a hidden | |
| "none" radio to close. | |
| """ | |
| radios = ['<input type="radio" name="ta-mk" id="ta-mk-none" class="ta-mk-radio" checked>'] | |
| labels, defs, rules = [], [], [] | |
| for i, metric in enumerate(website_texts.METRICS): | |
| rid = f"ta-mk-{i}" | |
| name = html.escape(metric["name"]) | |
| definition = html.escape(f"{metric['details']} · Why we use it: {metric['why']}") | |
| radios.append(f'<input type="radio" name="ta-mk" id="{rid}" class="ta-mk-radio">') | |
| labels.append(f'<label for="{rid}" class="ta-mk-chip">{name}</label>') | |
| defs.append( | |
| f'<div class="ta-mk-def" id="{rid}-def">' | |
| f'<label for="ta-mk-none" class="ta-mk-close" title="Close">✕</label>{definition}</div>' | |
| ) | |
| rules.append(f"#{rid}:checked ~ .ta-mk-panel #{rid}-def{{display:block}}") | |
| rules.append( | |
| f"#{rid}:checked ~ .ta-mk-chips label[for='{rid}']" | |
| "{background:rgba(110,140,245,0.22);border-color:rgba(110,140,245,0.9);color:#cdd7ff}" | |
| ) | |
| gr.HTML( | |
| "<style>" + "".join(rules) + "</style>" | |
| '<div class="ta-metric-key">' + "".join(radios) | |
| + '<div class="ta-mk-head">' | |
| '<span class="ta-mk-title">📊 Column key</span>' | |
| '<span class="ta-mk-hint">ranked by the <b>Elo</b> aggregation · click any column for its definition</span>' | |
| "</div>" | |
| + '<div class="ta-mk-chips">' + "".join(labels) + "</div>" | |
| + '<div class="ta-mk-panel">' + "".join(defs) + "</div>" | |
| + "</div>" | |
| ) | |
| def make_agentic_guide() -> None: | |
| gr.Markdown(website_texts.AGENTIC_GUIDE, elem_classes="markdown-text-box") | |
| def make_hero_stats(data_root: Path) -> gr.HTML: | |
| """A compact strip of headline-fact cards shown above the info boxes.""" | |
| lb = LBContainer(data_root, Subset(), "") | |
| n_datasets = lb.n_datasets or "—" | |
| df = lb.load_df() | |
| df = df[~df["TypeName"].isin([Constants.reference])] | |
| n_models = len({_parse_model(m)[0] for m in df["Model"]}) | |
| paper = "https://tabarena.ai/paper-tabular-ml-iid-study" | |
| code = "https://tabarena.ai/code" | |
| cards = [ | |
| ( | |
| "🧾", | |
| f"{n_datasets} datasets", | |
| f'curated from 1,053 (<a href="{paper}" target="_blank" rel="noopener">see paper</a>)', | |
| ), | |
| ("🤖", f"{n_models}+ models", "state-of-the-art, each tuned to its peak"), | |
| ( | |
| "✅", | |
| "Open-source", | |
| f'verified implementations (<a href="{code}" target="_blank" rel="noopener">see code</a>)', | |
| ), | |
| ("⚖️", "Scientifically rigorous", "strong validation, reproducible"), | |
| ] | |
| chips = "".join( | |
| f'<div class="ta-card"><div class="ta-card-ico">{ico}</div>' | |
| f'<div class="ta-card-body"><div class="ta-card-num">{num}</div>' | |
| f'<div class="ta-card-lbl">{lbl}</div></div></div>' | |
| for ico, num, lbl in cards | |
| ) | |
| # Tagline rendered as a full-width card in the same group, above the stat boxes. | |
| intro = " ".join(website_texts.INTRODUCTION_TEXT.split()) | |
| intro = re.sub(r"\*\*(.+?)\*\*", r"<b>\1</b>", intro) | |
| return gr.HTML(f'<div class="ta-hero"><div class="ta-intro">{intro}</div>{chips}</div>') | |
| # --------------------------------------------------------------------------- # | |
| # BeyondArena components | |
| # | |
| # BeyondArena reuses the per-subset leaderboard table (make_leaderboard) and the | |
| # win-rate figure (make_winrate_image) unchanged; only the hero strip, the | |
| # cross-subset overview (an image, not the TabArena HTML heatmap) and the | |
| # per-subset figure set differ (no HPO tuning-trajectory figure — BeyondArena is | |
| # evaluated on a single `core` protocol). | |
| # --------------------------------------------------------------------------- # | |
| # A distinct teal→violet accent for the BeyondArena hero, so its top reads clearly different from | |
| # TabArena's neutral cards while staying in the same visual family (scoped to `.beyond-hero`). | |
| _BEYOND_HERO_CSS = """ | |
| <style> | |
| .beyond-hero .ta-card { | |
| border-color: #2dd4bf33; | |
| border-left: 3px solid #2dd4bf; | |
| background: linear-gradient(135deg, #2dd4bf1f, #7c5cf00d); | |
| } | |
| .beyond-hero .ta-card:hover { border-color: #2dd4bf99; } | |
| .beyond-hero .ta-card-ico { color: #2dd4bf; } | |
| .beyond-hero .ta-intro { border-left: 3px solid #7c5cf0; } | |
| </style> | |
| """ | |
| def make_beyond_hero_stats(data_root: Path) -> gr.HTML: | |
| """A compact strip of headline-fact cards shown above the BeyondArena info boxes. | |
| Deliberately ordered / worded / colored differently from TabArena's hero: it leads with the | |
| beyond-IID identity and the curated subsets (the benchmark's focus) and uses a teal→violet accent. | |
| """ | |
| lb = LBContainer(data_root, BeyondSubset("full"), "") | |
| n_datasets = lb.n_datasets or "—" | |
| df = lb.load_df() | |
| df = df[~df["TypeName"].isin([Constants.reference])] | |
| n_models = len({_parse_model(m)[0] for m in df["Model"]}) | |
| # Curated subsets = every subset tab except the "full" (whole-benchmark) view. | |
| n_subsets = len(BEYOND_SUBSET_LABELS) - 1 | |
| paper = "https://arxiv.org/abs/2606.30410" | |
| code = "https://tabarena.ai/code" | |
| cards = [ | |
| ("🌍", "Beyond IID", "non-IID, temporal & grouped tabular data"), | |
| ("🧩", f"{n_subsets} subsets", "curated subsets of the benchmark"), | |
| ( | |
| "🧾", | |
| f"{n_datasets} datasets", | |
| f'across sizes & dimensionalities (<a href="{paper}" target="_blank" rel="noopener">see paper</a>)', | |
| ), | |
| ( | |
| "🤖", | |
| f"{n_models} models", | |
| f'tuned pipelines with preprocessing, beyond IID (<a href="{code}" target="_blank" rel="noopener">see code</a>)', | |
| ), | |
| ] | |
| chips = "".join( | |
| f'<div class="ta-card"><div class="ta-card-ico">{ico}</div>' | |
| f'<div class="ta-card-body"><div class="ta-card-num">{num}</div>' | |
| f'<div class="ta-card-lbl">{lbl}</div></div></div>' | |
| for ico, num, lbl in cards | |
| ) | |
| intro = " ".join(website_texts.BEYOND_INTRODUCTION_TEXT.split()) | |
| intro = re.sub(r"\*\*(.+?)\*\*", r"<b>\1</b>", intro) | |
| return gr.HTML(f'{_BEYOND_HERO_CSS}<div class="ta-hero beyond-hero"><div class="ta-intro">{intro}</div>{chips}</div>') | |
| def make_beyond_overview_figure(data_root: Path) -> None: | |
| """The cross-subset overview: best Elo per model / per family across every subset. | |
| Unlike TabArena's HTML heatmap, BeyondArena's overview is the ``plot_subset_results`` image | |
| (``result_plots/per_model_elo`` + ``per_family_elo``). Missing images are skipped gracefully. | |
| """ | |
| result_dir = Path(data_root) / "result_plots" | |
| images = [ | |
| ("per_family_elo", "Best Elo per model type (family) across subsets"), | |
| ("per_model_elo", "Best Elo per model across subsets"), | |
| ] | |
| shown = False | |
| for name, label in images: | |
| if (result_dir / f"{name}.png").exists() or (result_dir / f"{name}.png.zip").exists(): | |
| gr.Image( | |
| unzip_png(result_dir, name), | |
| label=label, | |
| show_label=True, | |
| height=520, | |
| show_share_button=True, | |
| ) | |
| shown = True | |
| if not shown: | |
| gr.Markdown("_The cross-subset overview figure is not available yet._", elem_classes="markdown-text") | |
| def make_beyond_subset_figures(lb: LBContainer) -> None: | |
| """Per-subset figures for BeyondArena: the Elo overview and the inference-time Pareto front.""" | |
| name = lb.name | |
| gr.Image( | |
| lb.image_path("tuning-impact-elo"), | |
| label=f"Leaderboard Overview [{name}]", | |
| show_label=True, | |
| height=500, | |
| show_share_button=True, | |
| ) | |
| gr.Image( | |
| value=lb.image_path("pareto_front_improvability_vs_time_infer"), | |
| label=f"Inference Time Pareto Front [{name}]", | |
| height=450, | |
| show_label=True, | |
| show_share_button=True, | |
| ) | |