"""Gradio component builders for the leaderboard. Each function creates one piece of the UI inside the surrounding Blocks/render context. No data-path logic lives here (see ``data_loading.py``) and no copy lives here (see ``website_texts.py``). """ from __future__ import annotations import html import re from itertools import groupby from pathlib import Path import gradio as gr import pandas as pd from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns import website_texts from constants import Constants, model_type_color, model_type_emoji from data_loading import ( BEYOND_SUBSET_LABELS, DATASET_SIZE_NOTE, BeyondSubset, LBContainer, Subset, load_leaderboard_csv, subset_name, unzip_png, ) # --------------------------------------------------------------------------- # # Full per-subset leaderboard table # --------------------------------------------------------------------------- # _IMPUTED_INFO = ( "We impute the performance for models that cannot run on all datasets due to" " task or dataset size constraints. We impute with the performance of a" " default RandomForest. We add a postfix [X% IMPUTED] to the model if any" " results were imputed. The X% shows the percentage of datasets that were" " imputed. In general, imputation negatively represents the model" " performance, punishing the model for not being able to run on all datasets." ) def make_leaderboard(lb: LBContainer) -> Leaderboard: df = lb.load_df() # -- Derived filter columns df["TypeFiler"] = df["TypeName"].apply(lambda m: f"{m} {model_type_emoji[m]}") df["Only Default"] = df["Model"].str.contains("(default)", regex=False) df["Only Tuned"] = df["Model"].str.contains("(tuned)", regex=False) df["Only Tuned + Ensembled"] = df["Model"].str.contains( "(tuned + ensembled)", regex=False ) | df["Model"].str.contains("AutoGluon", regex=False) filter_columns = [ ColumnFilter("TypeFiler", type="checkboxgroup", label="🤖 Model Types"), ColumnFilter("Only Default", type="boolean", default=False), ColumnFilter("Only Tuned", type="boolean", default=False), ColumnFilter("Only Tuned + Ensembled", type="boolean", default=False), ] # -- Imputation column (only meaningful when some rows are imputed) if df["Imputed"].any(): df["Imputed"] = df["Imputed"].replace({True: "Imputed", False: "Not Imputed"}) filter_columns.append( ColumnFilter( "Imputed", type="checkboxgroup", label="(Not) Imputed Models", info=_IMPUTED_INFO, ) ) else: df = df.drop(columns=["Imputed (%) [⬇️]"]) # -- Color model names by type so the table shares the overview's hue. def _color_model(row: pd.Series) -> str: color = model_type_color.get(row["TypeName"], "#dddddd") link = re.match(r"\[(.*?)\]\((.*?)\)(.*)", row["Model"]) if link: text, url, rest = link.groups() return ( f'{text}{rest}' ) return f'{row["Model"]}' df["Model"] = df.apply(_color_model, axis=1) # -- Column datatypes (read straight from dtypes; transposing collapses to # object and mistypes every numeric column as markdown). datatypes = [] for dtype in df.dtypes: if pd.api.types.is_bool_dtype(dtype): # check bool first (it is also "numeric") datatypes.append("bool") elif pd.api.types.is_numeric_dtype(dtype): datatypes.append("number") else: datatypes.append("markdown") return Leaderboard( elem_id=f"lb_{lb.subset.rel_path.replace('/', '_')}", value=df, datatype=datatypes, select_columns=SelectColumns( default_selection=list(df.columns), cant_deselect=["Type", "Model"], label="Select Columns to Display:", ), hide_columns=[ "TypeName", "TypeFiler", "RefModel", "Only Default", "Only Tuned", "Only Tuned + Ensembled", "Imputed", ], search_columns=["Model", "TypeName"], filter_columns=filter_columns, bool_checkboxgroup_label="Custom Views (exclusive, only toggle one at a time):", height=800, ) # --------------------------------------------------------------------------- # # Per-subset figures # --------------------------------------------------------------------------- # def make_overview_images(lb: LBContainer) -> None: name = subset_name(lb.subset) gr.Image( lb.image_path("tuning-impact-elo"), label=f"Leaderboard Overview [{name}]", show_label=True, height=500, show_share_button=True, ) with gr.Row(): with gr.Column(scale=1): gr.Image( value=lb.image_path("pareto_front_improvability_vs_time_infer"), label=f"Inference Time Pareto Front [{name}]", height=400, show_label=True, show_share_button=True, ) with gr.Column(scale=1): gr.Image( value=lb.image_path("pareto_n_configs_imp"), label=f"Tuning Trajectories [{name}]", height=400, show_label=True, show_share_button=True, ) def make_winrate_image(lb: LBContainer) -> None: # Uses ``lb.name`` (the caller-supplied subset name) so it is benchmark-agnostic and reusable # across both the TabArena and BeyondArena tabs. gr.Image( lb.image_path("winrate_matrix"), label=f"Win-rate Matrix [{lb.name}]", show_label=True, height=800, show_share_button=True, ) # --------------------------------------------------------------------------- # # Cross-subset overview (Elo heatmap, rendered as HTML for links + grouping) # --------------------------------------------------------------------------- # # Columns of the overview: (label, group, subset). Always imputation=yes/splits=all. _OVERVIEW_COLUMNS: list[tuple[str, str, Subset]] = [ ("Overall", "overall", Subset(tasks="all", datasets="all")), ("Class.", "task", Subset(tasks="classification", datasets="all")), ("Regr.", "task", Subset(tasks="regression", datasets="all")), ("Binary", "task", Subset(tasks="binary", datasets="all")), ("Multi.", "task", Subset(tasks="multiclass", datasets="all")), ("Small", "size", Subset(tasks="all", datasets="small")), ("Medium", "size", Subset(tasks="all", datasets="medium")), ] _GROUP_TITLE = {"task": "By Task", "size": "By Dataset Size"} # Hover tooltips for the overview's subset column headers (rendered as a native `title=`). # Size definitions are reused from DATASET_SIZE_NOTE so they can't drift from the subset blurbs. _COLUMN_TOOLTIPS: dict[str, str] = { "Overall": "All tasks across every dataset size — the headline ranking.", "Class.": "Classification tasks only (binary + multiclass).", "Regr.": "Regression tasks only.", "Binary": "Binary classification tasks only.", "Multi.": "Multiclass classification tasks only.", "Small": DATASET_SIZE_NOTE["small"], "Medium": DATASET_SIZE_NOTE["medium"], } # Top-3 per column get a medal in a fixed-width slot reserved in every number cell, so the # numbers line up whether or not a medal is present (see `.ta-medal`/`.ta-val` in `_OVERVIEW_CSS`). _MEDALS = {1: "🥇", 2: "🥈", 3: "🥉"} _VARIANT_RE = re.compile(r"\((default|tuned \+ ensembled|tuned)\)") # Metrics selectable in the overview: label -> (csv column, higher_is_better, formatter). # Order = display order in the selector; the two headline metrics come first. _OVERVIEW_METRIC_SPECS: dict[str, tuple] = { "Elo": ("Elo [⬆️]", True, lambda v: str(int(round(v)))), "Improvability (%)": ("Improvability (%) [⬇️]", False, lambda v: f"{v:.1f}"), "Score": ("Score [⬆️]", True, lambda v: f"{v:.3f}"), "Average Rank": ("Rank [⬇️]", False, lambda v: f"{v:.2f}"), "Harmonic Rank": ("Harmonic Rank [⬇️]", False, lambda v: f"{v:.2f}"), } OVERVIEW_METRIC_CHOICES = list(_OVERVIEW_METRIC_SPECS) # One-line TL;DR shown next to the overview metric selector. OVERVIEW_METRIC_TLDR = { "Elo": "Pairwise win-rate rating (a 400-point gap ≈ a 91% win rate). Higher is better.", "Improvability (%)": "How much lower the best model's error is than this one's, per dataset. Lower is better.", "Score": "Error rescaled per dataset to 1 (best) … 0 (median), then averaged. Higher is better.", "Average Rank": "The model's mean rank across datasets. Lower is better.", "Harmonic Rank": "Harmonic mean of per-dataset ranks — rewards being excellent on some datasets. Lower is better.", } def _parse_model(model: str) -> tuple[str, str, str | None]: """Split a Model cell into (base name, variant, url).""" link = re.match(r"\[(.*?)\]\((.*?)\)", model) text, url = (link.group(1), link.group(2)) if link else (model, None) text = text.split("[")[0].strip() # drop any [X% IMPUTED] tag variant_match = _VARIANT_RE.search(text) variant = variant_match.group(1) if variant_match else "" base = _VARIANT_RE.sub("", text).strip() return base, variant, url def _subset_best(df: pd.DataFrame, column: str, higher_is_better: bool) -> pd.DataFrame: """Best variant per model in a subset for `column`; excludes reference pipelines.""" df = df[~df["TypeName"].isin([Constants.reference])].copy() df = df.dropna(subset=[column]) parsed = df["Model"].map(_parse_model) df["base"] = [p[0] for p in parsed] df["variant"] = [p[1] for p in parsed] df["url"] = [p[2] for p in parsed] df["imputed"] = df["Imputed"].astype(bool) if "Imputed" in df.columns else False df["verified"] = df["Verified"] if "Verified" in df.columns else "" grouped = df.groupby("base")[column] best = df.loc[grouped.idxmax() if higher_is_better else grouped.idxmin()] return best[ ["base", "TypeName", "Type", "variant", "url", column, "verified", "imputed"] ].rename(columns={column: "val"}) def _overview_th(label: str, *, rowspan: int | None = None) -> str: """A subset column header ``; gets a hover tooltip + 'help' affordance when defined.""" attrs = f' rowspan="{rowspan}"' if rowspan else "" tooltip = _COLUMN_TOOLTIPS.get(label) if tooltip: attrs += f' class="ta-th-info" title="{html.escape(tooltip, quote=True)}"' return f"{label}" def _interp_color(frac: float) -> str: """Map 0 (best) .. 1 (worst) to a green->olive->red hex (readable on dark bg).""" stops = [(0.0, (28, 120, 62)), (0.5, (138, 122, 36)), (1.0, (160, 58, 58))] frac = max(0.0, min(1.0, frac)) for (f0, c0), (f1, c1) in zip(stops, stops[1:]): if frac <= f1: t = 0 if f1 == f0 else (frac - f0) / (f1 - f0) r, g, b = (round(a + (b_ - a) * t) for a, b_ in zip(c0, c1)) return f"#{r:02x}{g:02x}{b:02x}" return "#a03a3a" _OVERVIEW_CSS = """ """ def type_legend_html(include_reference: bool = True) -> str: """A small legend explaining the model-type symbols (shared across tables). Baseline and Other share one color, so they are shown as a single entry. `include_reference` is False for the overview, which excludes reference pipelines. """ e = model_type_emoji entries = [ (Constants.foundational, e[Constants.foundational], "Foundation Model"), (Constants.neural_network, e[Constants.neural_network], "Neural Network"), (Constants.tree, e[Constants.tree], "Tree-based"), (Constants.baseline, f"{e[Constants.baseline]} {e[Constants.other]}", "Baseline / Other"), ] if include_reference: entries.append((Constants.reference, e[Constants.reference], "Reference Pipeline")) chips = [] for type_name, symbols, label in entries: color = model_type_color.get(type_name, "#9e9e9e") chips.append( f'' f'{symbols} ' f'{label}' ) return f'{_OVERVIEW_CSS}
{"".join(chips)}
' def make_cross_subset_overview(data_root: Path, metric: str = "Elo") -> gr.HTML: """Heatmap of the best `metric` per model (rows) and subset (columns).""" if metric not in _OVERVIEW_METRIC_SPECS: metric = "Elo" column, higher_is_better, fmt = _OVERVIEW_METRIC_SPECS[metric] val_by_col: dict[str, dict[str, float]] = {} imp_by_col: dict[str, dict[str, bool]] = {} meta: dict[str, dict] = {} present: list[tuple[str, str]] = [] # (label, group) for label, group, subset in _OVERVIEW_COLUMNS: path = Path(data_root) / subset.rel_path / "website_leaderboard.csv" if not path.exists(): continue best = _subset_best(load_leaderboard_csv(str(path.resolve())), column, higher_is_better) val_by_col[label] = dict(zip(best["base"], best["val"])) imp_by_col[label] = dict(zip(best["base"], best["imputed"])) present.append((label, group)) for _, row in best.iterrows(): meta.setdefault( row["base"], { "type_name": row["TypeName"], "emoji": row["Type"], "variant": row["variant"], "url": row["url"], "verified": row["verified"], }, ) if not present: return gr.HTML("

No overview data available.

") sort_label = present[0][0] worst_sort = float("-inf") if higher_is_better else float("inf") bases = sorted( meta, key=lambda b: val_by_col[sort_label].get(b, worst_sort), reverse=higher_is_better ) rank_by_col, bounds = {}, {} for label, _ in present: col = val_by_col[label] ranked = sorted(col, key=lambda b: col[b], reverse=higher_is_better) rank_by_col[label] = {b: i + 1 for i, b in enumerate(ranked[:3])} bounds[label] = (min(col.values()), max(col.values())) if col else (0.0, 1.0) # -- Grouped header (two rows) row1 = ['Type', 'Model'] row2 = [] for group, items in groupby(present, key=lambda x: x[1]): items = list(items) if group == "overall": for label, _ in items: row1.append(_overview_th(label, rowspan=2)) else: row1.append(f'{_GROUP_TITLE[group]}') row2.extend(_overview_th(label) for label, _ in items) header = f"{''.join(row1)}{''.join(row2)}" # -- Body body = [] for base in bases: m = meta[base] color = model_type_color.get(m["type_name"], "#9e9e9e") name = html.escape(base) if m["variant"]: name += f' ({html.escape(m["variant"])})' if m.get("verified") == "✔️": name += ' ✔️' if m["url"]: name_html = ( f'{name}' ) else: name_html = f'{name}' cells = [ f'{m["emoji"]}', f'{name_html}', ] for label, _ in present: val = val_by_col[label].get(base) if val is None: cells.append('–') continue lo, hi = bounds[label] if hi <= lo: frac_best = 0.5 else: frac_best = (val - lo) / (hi - lo) if higher_is_better else (hi - val) / (hi - lo) bg = _interp_color(1 - frac_best) medal = _MEDALS.get(rank_by_col[label].get(base), "") imp = ( '*' if imp_by_col[label].get(base) else "" ) cells.append( f'' f'{medal}' f'{fmt(val)}{imp}' ) body.append(f"{''.join(cells)}") direction = "Higher is better" if higher_is_better else "Lower is better" caption = ( f'
Best {html.escape(metric)} per model across subsets ' f"(with imputation, all repeats). {direction}; 🥇🥈🥉 mark the top 3 in each column. " "Each model shows its best-performing variant.
" '
✔️ = verified implementation  ·  ' '* = (partly) imputed score  ·  ' "💡 Click any underlined model name (↗) to open its paper or code.
" ) table = f'{header}{"".join(body)}
' return gr.HTML( f'{type_legend_html(include_reference=False)}{caption}
{table}
', elem_classes="ta-overview-block", ) # --------------------------------------------------------------------------- # # Metric reference and agentic guide # --------------------------------------------------------------------------- # def make_metric_reference() -> None: """Compact column key — pills that reveal a single shared panel (one open at a time). Uses a CSS-only radio group (no JS): selecting a pill checks its hidden radio, which reveals just that definition in the shared panel below; the ✕ rechecks a hidden "none" radio to close. """ radios = [''] labels, defs, rules = [], [], [] for i, metric in enumerate(website_texts.METRICS): rid = f"ta-mk-{i}" name = html.escape(metric["name"]) definition = html.escape(f"{metric['details']} · Why we use it: {metric['why']}") radios.append(f'') labels.append(f'') defs.append( f'
' f'{definition}
' ) rules.append(f"#{rid}:checked ~ .ta-mk-panel #{rid}-def{{display:block}}") rules.append( f"#{rid}:checked ~ .ta-mk-chips label[for='{rid}']" "{background:rgba(110,140,245,0.22);border-color:rgba(110,140,245,0.9);color:#cdd7ff}" ) gr.HTML( "" '
' + "".join(radios) + '
' '📊 Column key' 'ranked by the Elo aggregation · click any column for its definition' "
" + '
' + "".join(labels) + "
" + '
' + "".join(defs) + "
" + "
" ) def make_agentic_guide() -> None: gr.Markdown(website_texts.AGENTIC_GUIDE, elem_classes="markdown-text-box") def make_hero_stats(data_root: Path) -> gr.HTML: """A compact strip of headline-fact cards shown above the info boxes.""" lb = LBContainer(data_root, Subset(), "") n_datasets = lb.n_datasets or "—" df = lb.load_df() df = df[~df["TypeName"].isin([Constants.reference])] n_models = len({_parse_model(m)[0] for m in df["Model"]}) paper = "https://tabarena.ai/paper-tabular-ml-iid-study" code = "https://tabarena.ai/code" cards = [ ( "🧾", f"{n_datasets} datasets", f'curated from 1,053 (see paper)', ), ("🤖", f"{n_models}+ models", "state-of-the-art, each tuned to its peak"), ( "✅", "Open-source", f'verified implementations (see code)', ), ("⚖️", "Scientifically rigorous", "strong validation, reproducible"), ] chips = "".join( f'
{ico}
' f'
{num}
' f'
{lbl}
' for ico, num, lbl in cards ) # Tagline rendered as a full-width card in the same group, above the stat boxes. intro = " ".join(website_texts.INTRODUCTION_TEXT.split()) intro = re.sub(r"\*\*(.+?)\*\*", r"\1", intro) return gr.HTML(f'
{intro}
{chips}
') # --------------------------------------------------------------------------- # # BeyondArena components # # BeyondArena reuses the per-subset leaderboard table (make_leaderboard) and the # win-rate figure (make_winrate_image) unchanged; only the hero strip, the # cross-subset overview (an image, not the TabArena HTML heatmap) and the # per-subset figure set differ (no HPO tuning-trajectory figure — BeyondArena is # evaluated on a single `core` protocol). # --------------------------------------------------------------------------- # # A distinct teal→violet accent for the BeyondArena hero, so its top reads clearly different from # TabArena's neutral cards while staying in the same visual family (scoped to `.beyond-hero`). _BEYOND_HERO_CSS = """ """ def make_beyond_hero_stats(data_root: Path) -> gr.HTML: """A compact strip of headline-fact cards shown above the BeyondArena info boxes. Deliberately ordered / worded / colored differently from TabArena's hero: it leads with the beyond-IID identity and the curated subsets (the benchmark's focus) and uses a teal→violet accent. """ lb = LBContainer(data_root, BeyondSubset("full"), "") n_datasets = lb.n_datasets or "—" df = lb.load_df() df = df[~df["TypeName"].isin([Constants.reference])] n_models = len({_parse_model(m)[0] for m in df["Model"]}) # Curated subsets = every subset tab except the "full" (whole-benchmark) view. n_subsets = len(BEYOND_SUBSET_LABELS) - 1 paper = "https://arxiv.org/abs/2606.30410" code = "https://tabarena.ai/code" cards = [ ("🌍", "Beyond IID", "non-IID, temporal & grouped tabular data"), ("🧩", f"{n_subsets} subsets", "curated subsets of the benchmark"), ( "🧾", f"{n_datasets} datasets", f'across sizes & dimensionalities (see paper)', ), ( "🤖", f"{n_models} models", f'tuned pipelines with preprocessing, beyond IID (see code)', ), ] chips = "".join( f'
{ico}
' f'
{num}
' f'
{lbl}
' for ico, num, lbl in cards ) intro = " ".join(website_texts.BEYOND_INTRODUCTION_TEXT.split()) intro = re.sub(r"\*\*(.+?)\*\*", r"\1", intro) return gr.HTML(f'{_BEYOND_HERO_CSS}
{intro}
{chips}
') def make_beyond_overview_figure(data_root: Path) -> None: """The cross-subset overview: best Elo per model / per family across every subset. Unlike TabArena's HTML heatmap, BeyondArena's overview is the ``plot_subset_results`` image (``result_plots/per_model_elo`` + ``per_family_elo``). Missing images are skipped gracefully. """ result_dir = Path(data_root) / "result_plots" images = [ ("per_family_elo", "Best Elo per model type (family) across subsets"), ("per_model_elo", "Best Elo per model across subsets"), ] shown = False for name, label in images: if (result_dir / f"{name}.png").exists() or (result_dir / f"{name}.png.zip").exists(): gr.Image( unzip_png(result_dir, name), label=label, show_label=True, height=520, show_share_button=True, ) shown = True if not shown: gr.Markdown("_The cross-subset overview figure is not available yet._", elem_classes="markdown-text") def make_beyond_subset_figures(lb: LBContainer) -> None: """Per-subset figures for BeyondArena: the Elo overview and the inference-time Pareto front.""" name = lb.name gr.Image( lb.image_path("tuning-impact-elo"), label=f"Leaderboard Overview [{name}]", show_label=True, height=500, show_share_button=True, ) gr.Image( value=lb.image_path("pareto_front_improvability_vs_time_infer"), label=f"Inference Time Pareto Front [{name}]", height=450, show_label=True, show_share_button=True, )