import gradio as gr
from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
from gradio_rangeslider import RangeSlider
import pandas as pd
import json
import random
import plotly.express as px
import seaborn as sns
from io import BytesIO
import base64
from utils import (
scores_to_battles,
compute_bt_elo,
compute_pct_improvement_over_baseline,
)
# TabBench currently supports the following models, with new additions that keep coming:
# - **NICL (Neuralk In-Context-Learning)**: Our in-house tabular foundation model based on an in-context learning architecture (proprietary).
# - **TabICL**: A transformer-based model that performs feature compression before doing in-context learning on tabular data by conditioning on labeled support examples to predict unseen queries without task-specific training.
# - **TabPFNv2**: A transformer-based model that performs in-context learning by approximating Bayesian inference for tabular classification on small datasets.
# - **XGBoost**: An optimized distributed gradient boosting library designed to be highly efficient, flexible, and portable.
# - **CatBoost**: A gradient boosting on decision trees library, particularly strong with categorical features.
# - **LightGBM**: A fast gradient boosting framework that builds decision trees using histogram-based learning for scalable, high-performance tabular modeling.
# - **ModernNCA**: An advanced extension of Neighborhood Component Analysis (NCA) that employs deep neural networks with stochastic neighborhood sampling to learn complex feature interactions for tabular data.
# - **RealMLP**: An enhanced MLP optimized with strong default parameters, achieving competitive performance on tabular data.
# - **TabM**: A deep learning model that efficiently imitates an ensemble of MLPs, enhancing performance on tabular data tasks without the overhead of training multiple models.
PUBLIC_DATASETS_TEXT = """
We curate a list of **186 datasets** from OpenML that span various industries such as **retail, healthcare, finance, and more**.
🔎 You can easily filter datasets by industry below or explore the industry specific leaderboards via the sidebar.
"""
MODELS_TEXT = """
TabBench currently supports the following models, with more additions planned:
- **Tabular foundation models**: NICL, TabICL, TabPFNv2.5
- **Tree-based**: XGBoost, CatBoost, LightGBM
- **Neural networks**: RealMLP, TabM, ModernNCA
__Note__ : Tree-based models are also ensembled according to best practices.
"""
BENCHMARKING_TEXT = """
TabBench currently focuses on **classification** tasks.
Evaluation is optimized for ROC-AUC and preprocessing steps vary following each model’s recommended practices:
- **Tabular foundation models**: These models include their own preprocessing pipelines, which we use as-is.
- **Tree-based**: Since these models support native handling of categorical features, we apply ordinal encoding to categorical columns and leave numerical values unchanged.
- **Neural networks**: Categorical features are embedded via learned embeddings, while numerical features are standardized using z-score normalization.
**Evaluation protocol**: Performance is evaluated using a 5-fold stratified shuffle split. For models that require hyperparameter tuning (tree-based models and MLPs), we conduct 100 Optuna trials on a 5-fold shuffle split of the training set.
"""
METRICS_TEXT = """
TabBench uses different types of metrics to evaluate model performance:
- **Classic metrics**: Accuracy, Precision, Recall, F1 score, AUC
- **Ranking metrics**: Rank, Elo score
"""
TITLE = """
TabBench
: A Classification Benchmark for Tabular Foundation Models
"""
INTRODUCTION_TEXT = """
Tracking the state of the art in pretrained models for tabular classification
"""
DATA_MAP_TEXT = """
- 🔎 **Explore different settings of dataset generation**
- 📈 **Compare Tabular Foundation Models on specific situations and identify strengths & weaknesses**
"""
PUBLIC_BENCHMARK_TEXT = """
- 📊 **A curation of 60 public datasets** across industries such as retail, healthcare, finance, energy, etc
- 🤖 **Evaluations on 10+ powerful models** from gradient boosting and decision trees to tabular foundation models
- 💥 **Integration of NICL**, the Neuralk AI tabular foundation model designed for Enterprise-grade use cases
- 🏅 **Leaderboards by sector** to discover which models perform best for each industry
"""
PRIVATE_BENCHMARK_TEXT = """
- 🏢 **Benchmarking on 60+ proprietary datasets** in the retail sector for industry-relevant insights (more sectors coming soon)
- ⚙️ **Evaluation of end-to-end pipelines** to measure model performance on real industry use cases (with new cases continuously added)
"""
# rewrite new uses continuously being added
PRIVATE_DATA_TEXT = """
We curate a list of **63 proprietary datasets** from different retailers focused on the **use case of product categorization**.
👚 Each dataset contains product titles, descriptions, and hierarchical product categories with four levels.
👉 The goal is to automatically assign **products** to the **right category**. For example, a product like a jigsaw could be categorized as:
> Tools/Equipment → Tools/Equipment → Saws → Jigsaw
You can read more about product categorization and its challenges [here](https://www.neuralk-ai.com/post/automating-product-categorization-with-tabular-ai-why-retailers-cant-afford-to-ignore-it).
"""
PRIVATE_MODEL_TEXT = """
TabBench Enterprise currently supports the following models, with more additions planned:
- **Tabular foundation models**: NICL, TabICL, TabPFNv2
- **Tree-based**: XGBoost
- **Neural networks**: RealMLP, TabM, ModernNCA
"""
PRIVATE_BENCHMARKING_TEXT = """
The evaluation results are generated using an end-to-end **TabBench pipeline** (also called **Workflow**) for product categorization, where different models are used for the classifier step.
Below is an example illustrating the Workflow process for XGBoost: the same logic applies to all other models assessed.
For a deeper understanding, you can explore our [Notebooks](https://github.com/Neuralk-AI/TabBench/tree/main/tutorials) and see the Workflows in action.
"""
PRIVATE_METRICS_TEXT = """
TabBench Enterprise uses different types of metrics to evaluate model performance:
- **Classic metrics**: Accuracy, Precision, Recall, F1 score, AUC
- **Ranking metrics**: Rank, Elo score
"""
CITATION_BUTTON_LABEL = "Citation"
CITATION_BUTTON_TEXT = """
@article{neuralk2025tabench,
title={TabBench: an industry-first benchmark for Machine Learning on tabular data},
author={Neuralk-AI},
year={2025},
publisher={GitHub},
url={https://github.com/Neuralk-AI/TabBench}
}
"""
def format_two_decimals(df: pd.DataFrame):
df_copy = df.copy()
for col in df_copy.select_dtypes(include="number"):
# Round up the rest as you requested
df_copy[col] = df_copy[col].apply(lambda x: f"{x:.2f}" if pd.notnull(x) else "")
return df_copy
def format_4_decimals_no_trailing_zeros(df: pd.DataFrame):
df = df.copy()
for col in df.select_dtypes(include="number").columns:
df[col] = df[col].map(
lambda x: f"{x:.4f}".rstrip("0").rstrip(".") if pd.notnull(x) else x
)
return df
def highlight_max(df: pd.DataFrame):
def style_column(col):
if not pd.api.types.is_numeric_dtype(col):
return [""] * len(col)
sorted_vals = col.dropna().sort_values(ascending=False).unique()
top1, top2, top3 = (list(sorted_vals) + [None, None, None])[:3]
# Medal palette tuned for legibility on BOTH light and dark dashboards:
# the bg colors are mid-dark, so dark ink-coloured text + slight bold
# reads better than the previous white text in both themes.
return [
(
"background-color: #d9b800; color: #1a1525; font-weight: 600;"
if v == top1
else (
"background-color: #c7c7c7; color: #1a1525; font-weight: 600;"
if v == top2
else (
"background-color: #d97a3a; color: #1a1525; font-weight: 600;"
if v == top3
else ""
)
)
)
for v in col
]
return df.style.apply(style_column, axis=0).format(
lambda x: f"{x:.4f}".rstrip("0").rstrip("."),
subset=df.select_dtypes(include="number").columns,
)
def compute_elo_for_subset(df: pd.DataFrame, metric: str = "Accuracy") -> pd.DataFrame:
"""
Recompute Elo scores for a filtered subset of results.
Takes a DataFrame with per-dataset model scores and computes Elo ratings
based only on the battles within that subset.
Args:
df: DataFrame with columns 'model', 'dataset_name', and the metric column
metric: The metric to use for determining battle winners (default: "Accuracy")
Returns:
DataFrame with updated 'Elo_score' column
"""
if df.empty:
return df
# Convert scores to pairwise battles
battles = scores_to_battles(df, metric)
if not battles:
return df
# Compute Elo scores from battles
elo_dict = compute_bt_elo(battles)
# Map Elo scores back to each row
df = df.copy()
df["Elo_score"] = df["model"].map(elo_dict)
return df
# --- Load new academic results from results_academic/*.json ---
# Each file: {model, summary, datasets: {dataset_id: {folds: [...], mean_auc, mean_accuracy}}}
# We build BOTH a per-fold long dataframe (for proper CI computation) and a
# per-dataset mean dataframe (matching the legacy schema used downstream).
import glob, os
def _load_academic_results(directory="results_academic_new"):
"""Load per-model JSONs into long dataframes.
Supports both schemas:
OLD (small files): fold has {"accuracy", "auc"}; dataset has mean_accuracy, mean_auc
NEW (rich files): fold has {"test_accuracy", "test_roc_auc", "test_f1_score",
"test_precision", "test_recall", "test_cross_entropy"};
dataset has mean_accuracy, mean_auc, mean_f1, mean_precision,
mean_recall, mean_cross_entropy
"""
# Per-dataset mean column -> our internal name (single-fit)
DS_METRICS = {
"mean_accuracy": "Accuracy",
"mean_auc": "AUC",
"mean_f1": "F1_score",
"mean_precision": "Precision",
"mean_recall": "Recall",
"mean_cross_entropy": "Cross_entropy",
}
# Same metrics, but reading the *_ensemble_* columns (only GBDTs publish these)
DS_METRICS_ENS = {
"mean_ensemble_accuracy": "Accuracy",
"mean_ensemble_roc_auc": "AUC",
"mean_ensemble_f1_score": "F1_score",
"mean_ensemble_precision": "Precision",
"mean_ensemble_recall": "Recall",
"mean_ensemble_cross_entropy": "Cross_entropy",
}
FOLD_METRICS = {
("test_accuracy", "accuracy"): "Accuracy",
("test_roc_auc", "auc"): "AUC",
("test_f1_score",): "F1_score",
("test_precision",): "Precision",
("test_recall",): "Recall",
("test_cross_entropy",): "Cross_entropy",
}
FOLD_METRICS_ENS = {
("test_ensemble_accuracy",): "Accuracy",
("test_ensemble_roc_auc",): "AUC",
("test_ensemble_f1_score",): "F1_score",
("test_ensemble_precision",): "Precision",
("test_ensemble_recall",): "Recall",
("test_ensemble_cross_entropy",): "Cross_entropy",
}
# Models for which we also emit a virtual _ensemble row.
ENSEMBLE_MODELS = {"xgboost", "catboost", "lightgbm"}
per_fold_rows = []
per_dataset_rows = []
for path in sorted(glob.glob(os.path.join(directory, "*.json"))):
with open(path) as f:
d = json.load(f)
model_id = os.path.splitext(os.path.basename(path))[0]
for dataset_id, ds in d.get("datasets", {}).items():
ds_name = dataset_id
ds_id = (
dataset_id.replace("openml-", "")
if dataset_id.startswith("openml-")
else dataset_id
)
# Single-fit row
row = {"model": model_id, "dataset_name": ds_name, "dataset_id": ds_id}
for src_key, our_name in DS_METRICS.items():
if src_key in ds:
row[our_name] = ds[src_key]
per_dataset_rows.append(row)
for fold in ds.get("folds", []):
fr = {
"model": model_id,
"dataset_name": ds_name,
"dataset_id": ds_id,
"fold": fold.get("fold"),
"imputed": bool(fold.get("imputed", False)),
}
for src_keys, our_name in FOLD_METRICS.items():
for sk in src_keys:
if sk in fold:
fr[our_name] = fold[sk]
break
per_fold_rows.append(fr)
# Ensemble row (only for GBDTs and only if the dataset has ens metrics)
if model_id in ENSEMBLE_MODELS:
ens_model_id = f"{model_id}_ensemble"
has_any = any(k in ds for k in DS_METRICS_ENS)
if has_any:
erow = {
"model": ens_model_id,
"dataset_name": ds_name,
"dataset_id": ds_id,
}
for src_key, our_name in DS_METRICS_ENS.items():
if src_key in ds:
erow[our_name] = ds[src_key]
per_dataset_rows.append(erow)
for fold in ds.get("folds", []):
if not any(
sk in fold for sks in FOLD_METRICS_ENS for sk in sks
):
continue
efr = {
"model": ens_model_id,
"dataset_name": ds_name,
"dataset_id": ds_id,
"fold": fold.get("fold"),
"imputed": bool(fold.get("imputed", False)),
}
for src_keys, our_name in FOLD_METRICS_ENS.items():
for sk in src_keys:
if sk in fold:
efr[our_name] = fold[sk]
break
per_fold_rows.append(efr)
return pd.DataFrame(per_dataset_rows), pd.DataFrame(per_fold_rows)
def _reimpute_best_gbdt(
per_fold,
gbdts=("xgboost", "catboost", "lightgbm"),
metrics=("Accuracy", "AUC", "F1_score", "Precision", "Recall", "Cross_entropy"),
):
"""Make every model cover the union (dataset, fold) universe, filling
BOTH (a) existing imputed rows AND (b) entirely missing rows with the
mean GBDT score for that (dataset, fold), per metric.
Selection rule per metric:
- Mean of NON-imputed GBDT scores on that exact (dataset, fold) cell.
- If no GBDT has a real score for the cell, fall back to that model's
per-dataset mean over its surviving (non-imputed) folds.
"""
if per_fold.empty:
return per_fold
# Mean non-imputed GBDT score per (dataset, fold, metric)
if "imputed" in per_fold.columns:
gbdt_pool = per_fold[(per_fold["model"].isin(gbdts)) & (~per_fold["imputed"])]
else:
gbdt_pool = per_fold[per_fold["model"].isin(gbdts)]
best_per_cell = {}
for metric in metrics:
if metric not in per_fold.columns:
continue
sub = gbdt_pool[["dataset_id", "fold", metric]].dropna(subset=[metric])
if sub.empty:
continue
best_per_cell[metric] = sub.groupby(["dataset_id", "fold"])[metric].mean()
# Per-model per-dataset mean from NON-imputed folds (tertiary fallback)
real = per_fold[~per_fold["imputed"]] if "imputed" in per_fold.columns else per_fold
model_dataset_mean = {}
for metric in metrics:
if metric not in per_fold.columns:
continue
model_dataset_mean[metric] = real.groupby(["model", "dataset_id"])[
metric
].mean()
# (1) Patch existing imputed rows
out = per_fold.copy()
if "imputed" in out.columns:
imputed_mask = out["imputed"].astype(bool)
if imputed_mask.any():
for metric in metrics:
if metric not in out.columns:
continue
best_series = best_per_cell.get(metric)
md_mean = model_dataset_mean.get(metric)
for ridx in out.index[imputed_mask]:
ds_id = out.at[ridx, "dataset_id"]
fold = out.at[ridx, "fold"]
model = out.at[ridx, "model"]
new_val = None
if best_series is not None and (ds_id, fold) in best_series.index:
new_val = float(best_series.loc[(ds_id, fold)])
elif md_mean is not None and (model, ds_id) in md_mean.index:
fb = md_mean.loc[(model, ds_id)]
if pd.notna(fb):
new_val = float(fb)
if new_val is not None:
out.at[ridx, metric] = new_val
# (2) Fill entirely-missing (model, dataset, fold) cells using
# the union of (dataset, fold, dataset_name) cells across ALL models.
universe = (
per_fold.groupby(["dataset_id", "fold"])
.agg(dataset_name=("dataset_name", "first"))
.reset_index()
)
all_models = sorted(per_fold["model"].unique())
existing = set(zip(out["model"], out["dataset_id"], out["fold"]))
new_rows = []
for model in all_models:
for _, urow in universe.iterrows():
key = (model, urow["dataset_id"], urow["fold"])
if key in existing:
continue
row = {
"model": model,
"dataset_name": urow["dataset_name"],
"dataset_id": urow["dataset_id"],
"fold": urow["fold"],
"imputed": True,
}
for metric in metrics:
if metric not in per_fold.columns:
continue
best_series = best_per_cell.get(metric)
md_mean = model_dataset_mean.get(metric)
new_val = None
if (
best_series is not None
and (urow["dataset_id"], urow["fold"]) in best_series.index
):
new_val = float(best_series.loc[(urow["dataset_id"], urow["fold"])])
elif (
md_mean is not None and (model, urow["dataset_id"]) in md_mean.index
):
fb = md_mean.loc[(model, urow["dataset_id"])]
if pd.notna(fb):
new_val = float(fb)
if new_val is not None:
row[metric] = new_val
new_rows.append(row)
if new_rows:
out = pd.concat([out, pd.DataFrame(new_rows)], ignore_index=True)
return out
def _per_dataset_from_per_fold(
per_fold,
metrics=("Accuracy", "AUC", "F1_score", "Precision", "Recall", "Cross_entropy"),
):
"""Recompute per-dataset means from the (re-imputed) per-fold frame."""
avail = [m for m in metrics if m in per_fold.columns]
return (
per_fold.groupby(["model", "dataset_id", "dataset_name"])[avail]
.mean()
.reset_index()
)
public_per_dataset, public_per_fold = _load_academic_results()
# Restrict to the canonical leaderboard base: intersection of the 10 well-covered
# models, minus the datasets curatorially labeled "drop" in decisions.json
# (time-aware, duplicates, group-aware-split-needed). Missing (model, dataset,
# fold) cells inside this base are filled below by mean-GBDT imputation;
# datasets outside the base never reach the leaderboard.
with open("base_datasets.json") as _f:
_BASE_DATASETS = set(json.load(_f))
# dataset_id in the per-fold/per-dataset frames has the "openml-" prefix stripped
# (see _load_academic_results), so normalize the base list the same way.
_BASE_IDS = {
d.replace("openml-", "") if d.startswith("openml-") else d for d in _BASE_DATASETS
}
public_per_fold = public_per_fold[
public_per_fold["dataset_id"].isin(_BASE_IDS)
].reset_index(drop=True)
public_per_dataset = public_per_dataset[
public_per_dataset["dataset_id"].isin(_BASE_IDS)
].reset_index(drop=True)
# Re-impute imputed folds using best-of-GBDT per metric, then recompute
# the per-dataset means from the cleaned per-fold data.
public_per_fold = _reimpute_best_gbdt(public_per_fold)
public_per_dataset = _per_dataset_from_per_fold(public_per_fold)
# Recompute Elo scores for the full public dataset
public_per_dataset = compute_elo_for_subset(public_per_dataset, metric="Accuracy")
# Compute percentage improvement over XGBoost (ensemble = canonical "XGBoost")
public_per_dataset = compute_pct_improvement_over_baseline(
public_per_dataset, baseline_model="xgboost_ensemble", metric="Accuracy"
)
# Hide single-fit GBDTs from the leaderboard display: they're superseded
# by the ensemble versions (xgboost_ensemble, catboost_ensemble, lightgbm_ensemble)
# which we now show under the plain XGBoost/CatBoost/LightGBM names.
_HIDDEN_DISPLAY_MODELS = {"xgboost", "catboost", "lightgbm"}
public_enter_per_dataset = public_per_dataset[
~public_per_dataset["model"].isin(_HIDDEN_DISPLAY_MODELS)
].copy()
with open("complete_results_indus.json", "r") as f:
private_per_dataset = pd.json_normalize(json.load(f))
# Recompute Elo scores for the full private dataset
private_per_dataset = compute_elo_for_subset(private_per_dataset, metric="Accuracy")
# Compute percentage improvement over XGBoost (ensemble = canonical "XGBoost")
private_per_dataset = compute_pct_improvement_over_baseline(
private_per_dataset, baseline_model="xgboost_ensemble", metric="Accuracy"
)
with open("qrt.json", "r") as f:
qrt_scores = json.load(f)
models = ["Models"] + list(qrt_scores.keys())
accuracies = ["Test Accuracy"] + list(qrt_scores.values())
# Create a DataFrame with two rows: "Model" and "Test Accuracy"
df_qrt = format_4_decimals_no_trailing_zeros(
pd.DataFrame([accuracies], index=["Test Accuracy"], columns=models)
)
# Optional: highlight best accuracy
def highlight_best_qrt(df):
styles = pd.DataFrame("", index=df.index, columns=df.columns)
styles.loc["Test Accuracy", df.loc["Test Accuracy"] == "51.7"] = (
"background-color: #a38a00; color: white;"
)
styles.loc["Test Accuracy", df.loc["Test Accuracy"] == "51.58"] = (
"background-color: #a6a6a6; color: white;"
)
styles.loc["Test Accuracy", df.loc["Test Accuracy"] == "51.18"] = (
"background-color: #ad5803; color: white;"
)
return styles
styled_df_qrt = df_qrt.style.apply(highlight_best_qrt, axis=None)
# -------------------------
# Leaderboard builder
# -------------------------
def init_leaderboard(dataframe, include_model_type_filter=True, industry_filter=False):
filter_columns = []
if include_model_type_filter:
filter_columns.append(
ColumnFilter("model_type", type="checkboxgroup", label="🤖 Model Types")
)
if industry_filter:
filter_columns.append(
ColumnFilter("industry", type="checkboxgroup", label="🏭 Industry")
)
# Set search column based on context
if industry_filter and "dataset_name" in dataframe.columns:
search_columns = ["dataset_name"]
elif "model" in dataframe.columns:
search_columns = ["model"]
else:
search_columns = []
return Leaderboard(
value=dataframe,
datatype=["str"] + ["number"] * (len(dataframe.columns) - 1),
select_columns=SelectColumns(
default_selection=dataframe.columns.tolist(),
cant_deselect=["model"] if "model" in dataframe.columns else [],
label="Select Columns to Display:",
),
search_columns=search_columns,
hide_columns=[],
filter_columns=filter_columns,
interactive=False,
)
# -------------------------
# Gradio UI
# -------------------------
css = """
/* Neuralk brand fonts (served from neuralk.ai's webflow CDN) */
@font-face {
font-family: 'BDO Grotesk';
src: url('https://cdn.prod.website-files.com/69ba8904321d710987dcaa88/69bc04d0285062a2fe5ccf9b_BDOGrotesk-Regular.woff2') format('woff2');
font-weight: 400; font-style: normal; font-display: swap;
}
@font-face {
font-family: 'BDO Grotesk';
src: url('https://cdn.prod.website-files.com/69ba8904321d710987dcaa88/69bc04d0f2e00107d2a42fb2_BDOGrotesk-Medium.woff2') format('woff2');
font-weight: 500; font-style: normal; font-display: swap;
}
@font-face {
font-family: 'Geist Mono';
src: url('https://cdn.prod.website-files.com/69ba8904321d710987dcaa88/69bc052ed8e314b78821dbb7_GeistMono-Medium.woff2') format('woff2');
font-weight: 500; font-style: normal; font-display: swap;
}
/* ===== Theme tokens: light defaults, overridden under body.dark ===== */
:root {
--nk-bg: #fafafc;
--nk-surface: #ffffff;
--nk-ink: #060510;
--nk-muted: #5d6c7b;
--nk-line: #e6e4ef;
--nk-hover: #f3f1fb;
--nk-violet: #4e29ff;
--nk-violet-soft: #a28fff;
--nk-hl-1-bg: rgba(78, 41, 255, 0.12);
--nk-hl-1-fg: #4e29ff;
--nk-hl-2-bg: rgba(162, 143, 255, 0.18);
--nk-hl-3-bg: rgba(162, 143, 255, 0.10);
}
/* NOTE: we deliberately do NOT key dark theming off prefers-color-scheme.
The theme is driven exclusively by Gradio (?__theme=dark → body.dark),
so a user whose OS prefers dark won't see dark tokens unless Gradio
itself is in dark mode. */
body.dark {
--nk-bg: #0a0814;
--nk-surface: #14101f;
--nk-ink: #f3f1fb;
--nk-muted: #9a96b0;
--nk-line: #2a2440;
--nk-hover: #1c1730;
--nk-violet: #8c75ff;
--nk-violet-soft: #a28fff;
--nk-hl-1-bg: rgba(140, 117, 255, 0.22);
--nk-hl-1-fg: #c7baff;
--nk-hl-2-bg: rgba(162, 143, 255, 0.18);
--nk-hl-3-bg: rgba(162, 143, 255, 0.10);
}
/* Base typography */
body, .gradio-container, .prose, button, input, select, textarea {
font-family: 'BDO Grotesk', 'Helvetica Neue', Helvetica, Arial, sans-serif;
}
.gradio-container, body {
background: var(--nk-bg);
color: var(--nk-ink);
}
.prose, .prose p, .prose li, .prose span, .markdown {
color: var(--nk-ink);
}
.prose h1, .prose h2, .prose h3, .prose h4 {
font-family: 'Geist Mono', ui-monospace, SFMono-Regular, Menlo, monospace;
color: var(--nk-ink);
letter-spacing: -0.01em;
font-weight: 500;
}
/* Tables: model column in BDO Grotesk, numerics in Geist Mono */
.dataframe td, .dataframe th, table.dataframe td, table.dataframe th {
font-family: 'Geist Mono', ui-monospace, SFMono-Regular, Menlo, monospace;
font-feature-settings: 'tnum' 1;
color: var(--nk-ink);
}
.dataframe th {
font-family: 'BDO Grotesk', sans-serif;
font-weight: 500;
letter-spacing: 0.02em;
text-transform: uppercase;
font-size: 12px;
color: var(--nk-muted);
border-bottom: 1px solid var(--nk-line);
background: transparent;
text-align: left;
}
.dataframe table { table-layout: fixed; width: 100%; word-wrap: break-word; white-space: normal; }
.dataframe, .dataframe .table-wrap { height: auto; max-height: none; }
.dataframe td, .dataframe th { max-width: 200px; overflow-wrap: break-word; word-wrap: break-word; white-space: normal; }
.prose h2, .prose h1 { margin: 0; }
.prose h1 { font-size: 34px; padding-top: 40px; }
.sidebar.svelte-1hez9vf.svelte-1hez9vf {
width: 180px;
min-width: 120px;
max-width: 100vw;
overflow-x: auto;
background-color: var(--nk-bg);
border-right: 1px solid var(--nk-line);
}
/* Buttons (existing classes used in app.py) */
.white-btn {
background-color: var(--nk-surface);
color: var(--nk-ink);
border: 1px solid var(--nk-line);
border-radius: 10px;
box-shadow: none;
transition: transform 0.1s, background-color 0.15s, border-color 0.15s, color 0.15s;
}
.white-btn:active { transform: translateY(1px); }
.white-btn:hover {
background-color: var(--nk-hover);
border-color: var(--nk-violet-soft);
color: var(--nk-violet);
}
.selected-btn {
background-color: var(--nk-violet);
color: #fff;
border-color: var(--nk-violet);
}
/* Tabs, accordions */
.tab-nav button.selected, .tabitem.selected, [role="tab"][aria-selected="true"] {
color: var(--nk-violet);
border-bottom-color: var(--nk-violet);
}
.gradio-container details > summary:hover,
.gradio-container .label-wrap:hover { color: var(--nk-violet); }
.gradio-container details, .gradio-container .panel { border-color: var(--nk-line); }
.gradio-container details > summary {
font-family: 'BDO Grotesk', sans-serif;
font-weight: 500;
}
/* Links */
.prose a, a { color: var(--nk-violet); text-decoration: none; }
.prose a:hover, a:hover { text-decoration: underline; }
/* ===== Neuralk "Competition" section vocabulary ===== */
.nk-tag {
display: inline-flex;
align-items: center;
gap: 0.45em;
font-family: 'Geist Mono', ui-monospace, SFMono-Regular, monospace;
font-size: 11px;
font-weight: 500;
letter-spacing: 0.08em;
text-transform: uppercase;
color: var(--nk-muted);
padding: 0.25em 0.6em;
border: 1px solid var(--nk-line);
border-radius: 999px;
background: var(--nk-surface);
line-height: 1;
}
.nk-tag::before {
content: "";
display: inline-block;
width: 6px;
height: 6px;
border-radius: 999px;
background: var(--nk-violet);
}
.nk-tag.nk-tag--orange::before { background: #ff4e33; }
.nk-tag.nk-tag--lime::before { background: #89e500; }
.nk-tag.nk-tag--ink::before { background: var(--nk-ink); }
.nk-num {
display: inline-block;
font-family: 'Geist Mono', ui-monospace, SFMono-Regular, monospace;
font-size: 0.42em;
font-weight: 500;
letter-spacing: 0.04em;
color: #ffffff !important;
background: var(--nk-violet);
border-radius: 3px;
padding: 0.2em 0.35em;
line-height: 1;
vertical-align: super;
margin-right: 0.35em;
position: relative;
top: -0.1em;
}
/* Subsection numbering pill: same violet styling as nk-num but sized for
the smaller h3 headings so the label stays legible. */
.nk-subnum {
display: inline-block;
font-family: 'Geist Mono', ui-monospace, SFMono-Regular, monospace;
/* Match nk-num pixel height: nk-num is 0.42em of an h2 (≈1.5em),
so the absolute pill text size is ≈0.63em. Inside an h3 (≈1.17em),
the equivalent ratio is 0.63 / 1.17 ≈ 0.54em. */
font-size: 0.54em;
font-weight: 500;
letter-spacing: 0.04em;
color: #ffffff !important;
background: #b03f7b; /* magenta */
border-radius: 3px;
padding: 0.2em 0.35em;
line-height: 1;
vertical-align: super;
margin-right: 0.35em;
position: relative;
top: -0.1em;
}
/* Cluster card: two-column row. Left column is the identity (name + size
facts), right column carries the interpretation. */
.nk-cluster-row {
display: grid;
grid-template-columns: minmax(220px, 38%) 1fr;
gap: 2em;
align-items: start;
padding: 0.9em 0;
border-top: 1px solid var(--nk-line);
}
.nk-cluster-row:last-child { border-bottom: 1px solid var(--nk-line); }
.nk-cluster-left .nk-cluster-name {
display: block;
font-weight: 600;
color: var(--nk-ink);
font-size: 0.98em;
line-height: 1.35;
margin-bottom: 0.45em;
}
.nk-cluster-left .nk-cluster-count {
font-family: 'Geist Mono', ui-monospace, SFMono-Regular, monospace;
font-size: 12px;
color: var(--nk-violet);
letter-spacing: 0.04em;
font-weight: 600;
}
.nk-cluster-left .nk-cluster-stats {
font-size: 12px;
color: var(--nk-muted);
line-height: 1.5;
margin-top: 0.25em;
}
.nk-cluster-right {
color: var(--nk-ink);
line-height: 1.55;
font-size: 0.96em;
}
.nk-cluster-right p { margin: 0; }
/* Section card: groups the cluster-row entries inside a single visual block
that sits indented from the page margin with a faint tinted background.
Used to separate '04.A Salient clusters' from '04.B Trends'. */
.nk-section-card {
margin: 0.6em 0 2em 24px;
padding: 0.2em 1.1em;
border-radius: 6px;
}
.nk-section-card--clusters {
background: rgba(78, 41, 255, 0.045);
}
.nk-section-card--trends {
background: rgba(176, 63, 123, 0.045);
}
/* The first/last row inside a card shouldn't paint its hairline rule. */
.nk-section-card > .nk-cluster-row:first-child { border-top: none; }
.nk-section-card > .nk-cluster-row:last-child { border-bottom: none; }
/* The hairline between two adjacent rows inside the card needs more
contrast against the tinted background. Use a darker neutral. */
.nk-section-card > .nk-cluster-row { border-top-color: rgba(0,0,0,0.18); }
/* Trend rows: same row geometry as cluster rows but the left column carries
a scalar name, a one-line definition, and a numeric-range chip pair. */
.nk-trend-range {
display: inline-flex;
align-items: center;
gap: 0.4em;
font-family: 'Geist Mono', ui-monospace, SFMono-Regular, monospace;
font-size: 12px;
font-weight: 600;
letter-spacing: 0.02em;
margin-top: 0.45em;
}
.nk-trend-range .nk-trend-min,
.nk-trend-range .nk-trend-max {
padding: 0.18em 0.5em;
border-radius: 4px;
line-height: 1.2;
}
.nk-trend-range .nk-trend-min { background: rgba(33,102,172,0.12); color: #14457a; }
.nk-trend-range .nk-trend-max { background: rgba(178,24,43,0.12); color: #962235; }
.nk-trend-range .nk-trend-sep { color: var(--nk-line); }
.nk-trend-range.nk-trend-flat .nk-trend-min,
.nk-trend-range.nk-trend-flat .nk-trend-max {
background: rgba(120,120,120,0.10);
color: var(--nk-muted);
}
/* Dark mode: brighten the chip text + bg so both reds and blues pop against
the dark cluster-card background. (Driven by body.dark, NOT the OS-level
prefers-color-scheme.) */
body.dark .nk-trend-range .nk-trend-min { background: rgba(99,160,230,0.22); color: #a6cdfa; }
body.dark .nk-trend-range .nk-trend-max { background: rgba(244,114,140,0.22); color: #ffadbe; }
body.dark .nk-trend-range.nk-trend-flat .nk-trend-min,
body.dark .nk-trend-range.nk-trend-flat .nk-trend-max {
background: rgba(180,180,200,0.16); color: #d0ccda;
}
.nk-trend-def {
font-size: 12px;
color: var(--nk-muted);
line-height: 1.45;
margin-top: 0.2em;
}
/* Highlighted Seldon callout: pale violet wash with a violet left rule. */
.nk-seldon-frame {
background: rgba(78, 41, 255, 0.06);
border: 1px solid rgba(78, 41, 255, 0.25);
border-radius: 8px;
padding: 14px 20px !important;
margin: 16px 0 !important;
}
.nk-seldon-frame h3 {
color: var(--nk-violet);
margin-top: 0 !important;
}
/* Dark mode cartouche contrast (body.dark only). */
body.dark .nk-seldon-frame {
background: rgba(140, 117, 255, 0.13);
border: 1px solid rgba(140, 117, 255, 0.45);
}
/* =====================================================================
DARK-MODE PASS for non-themable surfaces: cartouches, plot containers,
inline Plotly SVG (axes, legends, gridlines, tick text), section cards,
leaderboard hairlines and medal cells, tree node text.
All rules below key off body.dark only — Gradio is the source of truth
for theme, the OS-level prefers-color-scheme is ignored.
===================================================================== */
/* ---- Plot containers: elevated surface so figures don't blend into the
page bg. body.dark only.
IMPORTANT: do NOT apply padding to .js-plotly-plot itself — Plotly
uses a ResizeObserver on its own container and a padded container
creates a positive-feedback resize loop (plot keeps growing every
time the parent reflows). Padding belongs on the outer .gr-plot
wrapper, not on the Plotly node. */
body.dark .gradio-container .gr-plot {
background: #1d182d !important;
border: 1px solid #3d375c !important;
border-radius: 8px;
padding: 4px;
box-sizing: border-box;
}
body.dark .gradio-container .plot-container,
body.dark .gradio-container .js-plotly-plot {
background: #1d182d !important;
border-radius: 6px;
}
/* ---- Plotly SVG text (legend, tick labels, colorbar titles, axis titles).
Plotly bakes black `fill` into ; override via CSS. NOTE: we do
NOT recolour annotation text — those carry their own font.color set
by the figure builder (e.g. cluster labels in the perfmap, leaf labels
in the trees), and overriding them here would clobber those choices. */
body.dark .js-plotly-plot .main-svg .legendtext,
body.dark .js-plotly-plot .main-svg .xtitle, body.dark .js-plotly-plot .main-svg .ytitle,
body.dark .js-plotly-plot .main-svg .gtitle,
body.dark .js-plotly-plot .main-svg .xtick text,
body.dark .js-plotly-plot .main-svg .ytick text,
body.dark .js-plotly-plot .main-svg .colorbar text { fill: #d8d4ec !important; }
body.dark .js-plotly-plot .main-svg .xaxis path.domain,
body.dark .js-plotly-plot .main-svg .yaxis path.domain,
body.dark .js-plotly-plot .main-svg .xaxis .tick line,
body.dark .js-plotly-plot .main-svg .yaxis .tick line {
stroke: #5a5478 !important;
}
body.dark .js-plotly-plot .main-svg .gridlayer path { stroke: #4d4670 !important; }
/* ---- Tree yes/no badge bg: server-side bgcolor is white (matches light
page). When Gradio is rendering its DARK theme (`body.dark`),
repaint that with the plot-card surface colour so the label
"cuts" the Bezier curve. NOTE: we do NOT use @media (prefers-color-
scheme: dark) here, because the user's OS preference is independent
of Gradio's actual theme (which is driven by ?__theme= and toggled
via body.dark). */
body.dark .nk-tree-plot .js-plotly-plot .annotation rect.bg,
body.dark .nk-tree-plot .js-plotly-plot .annotation rect[class="bg"],
body.dark .nk-tree-plot .js-plotly-plot g.annotation rect {
fill: #1d182d !important;
stroke: none !important;
}
/* ---- Section cards (clusters / trends): tints are too subtle on dark bg */
body.dark .nk-section-card--clusters { background: rgba(140, 117, 255, 0.10); }
body.dark .nk-section-card--trends { background: rgba(212, 122, 174, 0.10); }
body.dark .nk-section-card > .nk-cluster-row { border-top-color: rgba(255,255,255,0.16); }
/* ---- Leaderboard hairlines: in dark, the default --nk-line (#2a2440) is
too close to the bg. Override to a clearly lighter grey. */
body.dark .dataframe tbody td,
body.dark .dataframe th { border-bottom: 1px solid #5a5478 !important; }
body.dark table.nk-lb tbody td,
body.dark table.nk-lb tbody th { border-bottom: 1px solid #5a5478 !important; }
body.dark .nk-section-card > .nk-cluster-row { border-top-color: rgba(255,255,255,0.28) !important; }
/* Leaderboard body cells */
.dataframe tbody td {
font-variant-numeric: tabular-nums;
border-bottom: 1px solid var(--nk-line);
border-left: none;
border-right: none;
border-top: none;
padding: 10px 12px;
background: transparent;
}
.dataframe tbody td:first-child {
font-family: 'BDO Grotesk', sans-serif;
font-weight: 500;
text-align: left;
}
.dataframe tbody td:not(:first-child) { text-align: right; }
.dataframe tbody tr:hover td { background: var(--nk-hover); }
/* Highlight cells from highlight_max(): we only re-style by detecting its inline colors */
.dataframe td[style*="#a38a00"] {
background: var(--nk-hl-1-bg) !important;
color: var(--nk-hl-1-fg) !important;
font-weight: 500;
border-radius: 6px;
}
.dataframe td[style*="#a6a6a6"] {
background: var(--nk-hl-2-bg) !important;
color: var(--nk-ink) !important;
}
.dataframe td[style*="#ad5803"] {
background: var(--nk-hl-3-bg) !important;
color: var(--nk-ink) !important;
}
/* ===== Floating sun/moon toggle =====
Targets the gr.Button's wrapping
"
return html
# ---------------------------------------------------------------------------
# Per-model colors + class assignment shared across views.
# (Bars draw by class for restraint; line/radar views need per-model hues
# because they put up to 14 traces on the same axes.)
# ---------------------------------------------------------------------------
MODEL_CLASS = {
"xgboost": "tree",
"catboost": "tree",
"lightgbm": "tree",
"xgboost_ensemble": "tree",
"catboost_ensemble": "tree",
"lightgbm_ensemble": "tree",
"tabm": "nn",
"realmlp": "nn",
"modernnca": "nn",
"modern_nca": "nn",
"tabpfn": "fm1",
"tabpfn_2_5": "fm1",
"tabicl": "fm1",
"tabicl_v1": "fm1",
"tabdpt": "fm1",
"mitra": "fm1",
"limix": "fm1",
"nicl": "fm1",
"sap": "fm1",
"tabpfn_v3": "fm2",
"tabicl_v2": "fm2",
"seldon": "fm2",
}
# Stable per-model color (picked to keep classes visually clustered).
MODEL_COLOR = {
"xgboost": "#1f4c98",
"catboost": "#2860b0",
"lightgbm": "#5680c4",
"xgboost_ensemble": "#3a6abf",
"catboost_ensemble": "#4f7fc8",
"lightgbm_ensemble": "#7898cf",
"realmlp": "#b03f7b",
"tabm": "#ec9bca",
"modern_nca": "#7d3a5c",
"tabpfn": "#37a3a3",
"tabicl": "#5dc4c4",
"tabdpt": "#92d4d4",
"mitra": "#1e5f5f",
"limix": "#0d3838",
"tabpfn_v3": "#4d2cdc",
"tabicl_v2": "#a18cff",
"seldon": "#5a30c2",
"nicl": "#7b62de",
"sap": "#9c8ce0",
}
_MODEL_DISPLAY_PRETTY = {
"xgboost": "XGBoost",
"catboost": "CatBoost",
"lightgbm": "LightGBM",
"xgboost_ensemble": "XGBoost",
"catboost_ensemble": "CatBoost",
"lightgbm_ensemble": "LightGBM",
"realmlp": "RealMLP",
"tabm": "TabM",
"modern_nca": "ModernNCA",
"modernnca": "ModernNCA",
"tabpfn": "TabPFN v2.5",
"tabpfn_2_5": "TabPFN v2.5",
"tabpfn_v3": "TabPFN v3.0",
"tabicl": "TabICL v1.1",
"tabicl_v1": "TabICL v1.1",
"tabicl_v2": "TabICL v2.0",
"tabdpt": "TabDPT",
"limix": "LimiX",
"mitra": "Mitra",
"seldon": "Seldon",
"nicl": "NICL",
"sap": "SAP RPT-1",
}
_LOWER_IS_BETTER_METRICS = {"Cross_entropy"}
def _classes_bin_label(c):
"""Semantic bins for n_classes; distribution is too skewed for quantiles."""
if c == 2:
return ("=2 (binary)", 0)
if c == 3:
return ("=3", 1)
if c <= 5:
return ("4–5", 2)
if c <= 7:
return ("6–7", 3)
if c <= 10:
return ("8–10", 4)
if c <= 25:
return ("11–25", 5)
return ("26+", 6)
def _quantile_bin(values, n_bins=10):
"""Return (label, sort_key) per value from pd.qcut; drops duplicate edges."""
import pandas as _pd, numpy as _np
s = _pd.Series(values).dropna()
if len(s) == 0:
return {}, []
n_bins = min(n_bins, int(s.nunique()))
try:
binned, edges = _pd.qcut(s, q=n_bins, retbins=True, duplicates="drop")
except ValueError:
return {v: (str(int(v)), int(v)) for v in s.unique()}, []
label_for = {}
order = []
seen = set()
for v, b in zip(s, binned):
if _pd.isna(b):
continue
lo, hi = int(_np.floor(b.left)), int(_np.ceil(b.right))
label = f"{lo}–{hi}" if lo != hi else str(lo)
label_for[v] = (label, float(b.left))
if label not in seen:
order.append((float(b.left), label))
seen.add(label)
order.sort()
return label_for, [lbl for _, lbl in order]
def plot_dataset_comparison(per_dataset_df, metric="Accuracy", min_per_bin=3):
"""Three side-by-side line plots: model performance vs. n_features / n_samples / n_classes.
`per_dataset_df` must have columns: model, dataset_id, ,
features, rows, n_classes.
"""
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import numpy as np, pandas as pd
axes = [
("features", "Features", 10, None),
("rows", "Samples (rows)", 10, None),
("n_classes", "Classes", 7, _classes_bin_label),
]
higher_better = metric not in _LOWER_IS_BETTER_METRICS
metric_pretty = {
"Accuracy": "Accuracy",
"AUC": "AUC",
"F1_score": "F1",
"Precision": "Precision",
"Recall": "Recall",
"Cross_entropy": "Cross-entropy",
"Elo_score": "Elo",
"%↗ over XGBoost": "% over XGBoost",
}.get(metric, metric)
fig = make_subplots(
rows=1,
cols=3,
subplot_titles=[a[1] for a in axes],
horizontal_spacing=0.08,
)
models_present = sorted(
per_dataset_df["model"].unique(),
key=lambda m: per_dataset_df[per_dataset_df["model"] == m][metric].mean(),
reverse=higher_better,
)
for col_i, (col, _title, n_bins, custom_labeler) in enumerate(axes, start=1):
df = per_dataset_df[["model", "dataset_id", metric, col]].dropna(
subset=[col, metric]
)
if df.empty:
continue
# One bin per dataset (collapse across models for binning).
ds_props = df[["dataset_id", col]].drop_duplicates(subset=["dataset_id"])
if custom_labeler is not None:
ds_props["_bin_key"] = ds_props[col].apply(lambda v: custom_labeler(v)[1])
ds_props["_bin_label"] = ds_props[col].apply(lambda v: custom_labeler(v)[0])
order_pairs = sorted(set(zip(ds_props["_bin_key"], ds_props["_bin_label"])))
x_order = [lbl for _, lbl in order_pairs]
else:
label_for, x_order = _quantile_bin(ds_props[col], n_bins=n_bins)
ds_props["_bin_label"] = ds_props[col].map(
lambda v: label_for.get(v, (str(v), 0))[0]
)
ds_props["_bin_key"] = ds_props[col].map(
lambda v: label_for.get(v, ("", 0))[1]
)
df = df.merge(
ds_props[["dataset_id", "_bin_label", "_bin_key"]], on="dataset_id"
)
for model in models_present:
sub = df[df["model"] == model]
if sub.empty:
continue
# Per-bin: mean metric, n datasets. Hide sparse bins.
grouped = (
sub.groupby(["_bin_key", "_bin_label"])
.agg(
mean=(metric, "mean"),
n=("dataset_id", "nunique"),
)
.reset_index()
.sort_values("_bin_key")
)
grouped = grouped[grouped["n"] >= min_per_bin]
if grouped.empty:
continue
color = MODEL_COLOR.get(model, "#888888")
pretty = _MODEL_DISPLAY_PRETTY.get(model, model)
cls = MODEL_CLASS.get(model, "other")
line_dash = {
"tree": "solid",
"nn": "dot",
"fm1": "dash",
"fm2": "solid",
"other": "longdash",
}[cls]
fig.add_trace(
go.Scatter(
x=grouped["_bin_label"],
y=grouped["mean"],
name=pretty,
legendgroup=pretty,
showlegend=(col_i == 1),
mode="lines+markers",
line=dict(
color=color, width=2.4 if cls == "fm2" else 1.8, dash=line_dash
),
marker=dict(size=7 if cls == "fm2" else 5, color=color),
hovertemplate=f"{pretty} "
+ _title
+ "=%{x} "
+ metric_pretty
+ "=%{y:.4f} n=%{customdata}",
customdata=grouped["n"],
),
row=1,
col=col_i,
)
fig.update_xaxes(categoryorder="array", categoryarray=x_order, row=1, col=col_i)
if not higher_better:
fig.update_yaxes(autorange="reversed")
fig.update_layout(
height=440,
margin=dict(l=40, r=20, t=50, b=40),
legend=dict(
orientation="v",
yanchor="top",
y=1,
xanchor="left",
x=1.02,
font=dict(size=11),
),
template="simple_white",
plot_bgcolor="rgba(0,0,0,0)",
paper_bgcolor="rgba(0,0,0,0)",
)
fig.update_yaxes(title=metric_pretty, col=1)
for axis_idx in (1, 2, 3):
fig.update_xaxes(showgrid=False, row=1, col=axis_idx)
fig.update_yaxes(
showgrid=True, gridcolor="rgba(0,0,0,0.06)", row=1, col=axis_idx
)
return fig
def _radar_overlay(norm, metrics, models_sorted, theta_loop, title_text):
"""Build the all-models overlay radar from a [0,1] normalized df."""
import plotly.graph_objects as go
fig = go.Figure()
for model in models_sorted:
r = norm.loc[norm["model"] == model, metrics].values[0].tolist()
r += [r[0]]
color = MODEL_COLOR.get(model, "#888888")
pretty = _MODEL_DISPLAY_PRETTY.get(model, model)
cls = MODEL_CLASS.get(model, "other")
fig.add_trace(
go.Scatterpolar(
r=r,
theta=theta_loop,
name=pretty,
legendgroup=pretty,
mode="lines",
line=dict(color=color, width=2.2 if cls == "fm2" else 1.4),
opacity=0.85 if cls == "fm2" else 0.55,
hovertemplate=f"{pretty} %{{theta}}: %{{r:.2f}}",
)
)
fig.update_layout(
polar=dict(
radialaxis=dict(
visible=True,
range=[0, 1],
tickfont=dict(size=9),
gridcolor="rgba(0,0,0,0.08)",
),
angularaxis=dict(tickfont=dict(size=11)),
bgcolor="rgba(0,0,0,0)",
),
showlegend=True,
legend=dict(
orientation="v",
yanchor="top",
y=1,
xanchor="left",
x=1.15,
font=dict(size=10),
),
height=520,
margin=dict(l=40, r=160, t=40, b=20),
title=dict(text=title_text, x=0.5, xanchor="center", font=dict(size=13)),
template="simple_white",
paper_bgcolor="rgba(0,0,0,0)",
)
return fig
def _radar_small_multiples(norm, metrics, models_sorted, theta_loop, title_text):
"""Build the per-model filled small multiples grid from a [0,1] normalized df."""
import plotly.graph_objects as go
from plotly.subplots import make_subplots
n = len(models_sorted)
cols = 4
rows = (n + cols - 1) // cols
fig = make_subplots(
rows=rows,
cols=cols,
specs=[[{"type": "polar"}] * cols for _ in range(rows)],
subplot_titles=[_MODEL_DISPLAY_PRETTY.get(m, m) for m in models_sorted]
+ [""] * (rows * cols - n),
horizontal_spacing=0.04,
vertical_spacing=0.10,
)
for i, model in enumerate(models_sorted):
r = norm.loc[norm["model"] == model, metrics].values[0].tolist()
r += [r[0]]
color = MODEL_COLOR.get(model, "#888888")
row, col = i // cols + 1, i % cols + 1
fig.add_trace(
go.Scatterpolar(
r=r,
theta=theta_loop,
name=_MODEL_DISPLAY_PRETTY.get(model, model),
mode="lines",
line=dict(color=color, width=1.8),
fill="toself",
fillcolor=color,
opacity=0.40,
showlegend=False,
hovertemplate=f"{_MODEL_DISPLAY_PRETTY.get(model, model)} %{{theta}}: %{{r:.2f}}",
),
row=row,
col=col,
)
fig.update_polars(
radialaxis=dict(visible=False, range=[0, 1]),
angularaxis=dict(tickfont=dict(size=8)),
bgcolor="rgba(0,0,0,0)",
)
fig.update_layout(
height=max(220, rows * 220),
margin=dict(l=10, r=10, t=40, b=10),
template="simple_white",
paper_bgcolor="rgba(0,0,0,0)",
title=dict(text=title_text, x=0.5, xanchor="center", font=dict(size=13)),
)
return fig
def plot_model_radars(
per_dataset_df,
primary_metrics=(
"Accuracy",
"AUC",
"F1_score",
"Precision",
"Recall",
"Cross_entropy",
),
):
"""Return four radar figures: (overlay_minmax, smallmult_minmax, overlay_rank, smallmult_rank).
Two normalizations:
- min-max per axis (so the winning model touches the outer ring; the
loser collapses to 0)
- rank per axis (so the loser is at 1/N, never zero)
Both invert Cross_entropy so 'more area = better' holds.
"""
import plotly.graph_objects as go
import numpy as np, pandas as pd
metrics = [m for m in primary_metrics if m in per_dataset_df.columns]
if not metrics:
empty = go.Figure().update_layout(title="No metrics available")
return empty, empty, empty, empty
metric_pretty_map = {
"Accuracy": "Accuracy",
"AUC": "AUC",
"F1_score": "F1",
"Precision": "Precision",
"Recall": "Recall",
"Cross_entropy": "1 - Cross-entropy",
}
agg = per_dataset_df.groupby("model")[list(metrics)].mean().reset_index()
# Invert Cross_entropy so higher = better.
if "Cross_entropy" in metrics and agg["Cross_entropy"].notna().any():
agg["Cross_entropy"] = -agg["Cross_entropy"]
# --- Min-max normalization (current behaviour) ---
norm_mm = agg[metrics].copy()
for m in metrics:
col = agg[m]
lo, hi = col.min(), col.max()
norm_mm[m] = (col - lo) / (hi - lo) if (hi - lo) > 1e-9 else 0.5
norm_mm["model"] = agg["model"]
# --- Rank-based normalization: 1 = best, N = worst → score = 1 - (rank-1)/(N-1) ---
n_models = len(agg)
norm_rk = agg[metrics].copy()
for m in metrics:
# rank: 1 = highest value (best after the CE inversion above)
ranks = agg[m].rank(method="min", ascending=False)
norm_rk[m] = 1.0 - (ranks - 1.0) / (n_models - 1.0) if n_models > 1 else 0.5
norm_rk["model"] = agg["model"]
# Sort models by mean of non-CE metrics for legend ordering.
non_ce = [m for m in metrics if m != "Cross_entropy"]
if non_ce:
order_key = agg.set_index("model")[non_ce].mean(axis=1)
else:
order_key = agg.set_index("model")[metrics].mean(axis=1)
models_sorted = list(order_key.sort_values(ascending=False).index)
theta = [metric_pretty_map.get(m, m) for m in metrics]
theta_loop = theta + [theta[0]]
overlay_mm = _radar_overlay(
norm_mm,
metrics,
models_sorted,
theta_loop,
"All models: per-metric profile (min-max normalized per axis)",
)
sm_mm = _radar_small_multiples(
norm_mm,
metrics,
models_sorted,
theta_loop,
"Per-model profile: min-max normalized per axis",
)
overlay_rk = _radar_overlay(
norm_rk,
metrics,
models_sorted,
theta_loop,
"All models: rank-based (1 = best on that axis, 0 = worst)",
)
sm_rk = _radar_small_multiples(
norm_rk, metrics, models_sorted, theta_loop, "Per-model profile: rank-based"
)
return overlay_mm, sm_mm, overlay_rk, sm_rk
# ---------------------------------------------------------------------------
# Performance Map: co-clustering + PCA on the (datasets × model-metrics) matrix.
#
# Builds a rank-pct matrix M [datasets × (model,metric)] where each cell is
# 1.0 if that model wins on that dataset for that metric, 0.0 if it's last.
# Co-clusters with SpectralCoclustering → 5 dataset clusters × 5 model-cluster
# columns. Projects datasets to 2D via PCA on M.
#
# Findings (on the full base):
# PC1 correlates with avg_card (+0.39), pct_cat (+0.30), (Acc-F1) gap (+0.35),
# n_classes (−0.34) → "production-tabular character"
# • PC1 ≪ 0 = academic numeric balanced multiclass (TFM-favourable)
# • PC1 ≫ 0 = production categorical imbalanced binary (GBDT-favourable)
# PC2 correlates with rows (+0.33), n_classes (+0.35) → "size & complexity"
#
# We label three zones on the scatter:
# - TFM perf zone (PC1 < 0) : foundation models dominate, 4/5 clusters
# - GBDT zone (LightGBM cluster D1): only zone GBDTs win outright
# - Convergence zone (PC2 > 0.6) : large multiclass; NNs close on FMs
# ---------------------------------------------------------------------------
# Sentinel: precomputed full-base perf-map cached at boot so the static
# narrative section doesn't trigger sklearn on every reload. The interactive
# view recomputes on the filtered subset.
_PERFMAP_CACHE = {"full": None}
_METRICS_FOR_PERFMAP = (
"Accuracy",
"AUC",
"F1_score",
"Precision",
"Recall",
"Cross_entropy",
)
_PERFMAP_LOWER_IS_BETTER = {"Cross_entropy"}
# Canonical cluster centroids in PC1/PC2 space, calibrated on the curated base.
# These are FIXED: names get assigned by matching each cluster's actual
# centroid to its closest canonical target. Stable across reruns.
PERFMAP_CANONICAL_CENTROIDS = [
("Foundation-model territory", -0.60, 0.00),
("Small Industry/Science", -0.45, -0.40),
("Finance/classical mid-size", 0.80, -0.40),
("Convergence zone", 0.20, 1.20),
("LGBM niche", 2.10, 0.00),
]
def _assign_cluster_names(scores, row_labels):
"""Return dict raw_label → canonical name, by matching each cluster's
centroid to the nearest canonical target. Hungarian-style 1:1 matching
so two clusters never share a name."""
import numpy as np
unique = sorted(set(row_labels))
centroids = {}
for k in unique:
mask = row_labels == k
if mask.sum():
centroids[k] = (scores[mask, 0].mean(), scores[mask, 1].mean())
# Build cost matrix (cluster × canonical) of squared distances
canonical = PERFMAP_CANONICAL_CENTROIDS
n_cl = len(unique)
n_ca = len(canonical)
cost = np.zeros((n_cl, n_ca))
for i, k in enumerate(unique):
cx, cy = centroids[k]
for j, (_, tx, ty) in enumerate(canonical):
cost[i, j] = (cx - tx) ** 2 + (cy - ty) ** 2
try:
from scipy.optimize import linear_sum_assignment
row_ind, col_ind = linear_sum_assignment(cost)
return {unique[i]: canonical[j][0] for i, j in zip(row_ind, col_ind)}
except Exception:
# Greedy fallback
used = set()
out = {}
for i, k in enumerate(unique):
order = sorted(range(n_ca), key=lambda j: cost[i, j])
for j in order:
if j not in used:
out[k] = canonical[j][0]
used.add(j)
break
return out
def _build_perfmap(per_dataset_df, n_clusters=5):
"""Build the (datasets × model-metric) rank matrix, run SpectralCoclustering
+ PCA(2). Returns a dict with everything downstream plotting needs.
Applies manual overrides from cluster_overrides.json (if present) so the
user can hand-correct stray points without touching the algorithm.
Returns None if there's not enough data for stable clustering.
"""
import numpy as np, pandas as pd, json as _json, os as _os
from sklearn.cluster import SpectralCoclustering
from sklearn.decomposition import PCA
metrics = [m for m in _METRICS_FOR_PERFMAP if m in per_dataset_df.columns]
if not metrics:
return None
rank_frames = []
for metric in metrics:
pivot = per_dataset_df.pivot_table(
index="dataset_id", columns="model", values=metric
)
asc = metric not in _PERFMAP_LOWER_IS_BETTER
ranks = pivot.rank(axis=1, ascending=not asc, method="average")
n = pivot.notna().sum(axis=1).clip(lower=1)
rp = (ranks.sub(1, axis=0)).div(n - 1, axis=0).fillna(0.5)
rp.columns = [f"{metric}/{m}" for m in rp.columns]
rank_frames.append(rp)
M = pd.concat(rank_frames, axis=1)
M = 1 - M # high = wins
# Need enough rows + cols for spectral coclustering to be stable
if M.shape[0] < 25 or M.shape[1] < 15:
return None
# Load the FROZEN basis (PCA components + cluster labels) so adding new
# datasets doesn't re-derive the axes we already tuned. New datasets
# without a saved label inherit the label of their nearest known dataset
# in PC-space.
freeze_path = "perfmap_freeze.json"
frozen = None
if _os.path.exists(freeze_path):
try:
frozen = _json.load(open(freeze_path))
except Exception as _e:
print(f"[perfmap] could not load {freeze_path}: {_e}")
if frozen is not None:
# Align matrix columns to the frozen feature order
feat_cols = frozen["feature_cols"]
missing_cols = [c for c in feat_cols if c not in M.columns]
if missing_cols:
for c in missing_cols:
M[c] = 0.5 # neutral (no info)
M = M[feat_cols]
# Row labels: known datasets keep their saved labels; new ones get
# assigned via PCA-space nearest-neighbour to the frozen reference.
saved_labels = dict(zip(frozen["dataset_ids"], frozen["row_labels"]))
known_ids = [did for did in M.index if did in saved_labels]
new_ids = [did for did in M.index if did not in saved_labels]
row_labels = np.array([saved_labels.get(did, -1) for did in M.index])
col_labels = np.array(frozen["col_labels"])
if new_ids:
# Project everything into the frozen PCA basis, assign new points
# to the closest known-dataset label.
comps = np.array(frozen["pca_components"]) # (2, F)
pca_mean = np.array(frozen["pca_mean"]) # (F,)
scores_all = (M.values - pca_mean) @ comps.T # (N, 2)
id_to_idx = {did: i for i, did in enumerate(M.index)}
known_pts = np.array([scores_all[id_to_idx[d]] for d in known_ids])
known_lbls = np.array([saved_labels[d] for d in known_ids])
for did in new_ids:
p = scores_all[id_to_idx[did]]
d2 = np.sum((known_pts - p) ** 2, axis=1)
row_labels[id_to_idx[did]] = int(known_lbls[int(np.argmin(d2))])
print(
f"[perfmap] frozen PCA: {len(known_ids)} known, "
f"{len(new_ids)} new datasets projected into existing basis"
)
else:
try:
cc = SpectralCoclustering(n_clusters=n_clusters, random_state=0).fit(
M.values
)
except Exception:
return None
row_labels = cc.row_labels_.copy()
col_labels = cc.column_labels_
# Apply manual overrides from cluster_overrides.json. Format:
# {"": "", ...}
# cluster_name must match one of CLUSTER_NAMES in plot_perfmap_scatter
# (Foundation-model territory / Small Industry/Science / Finance/classical
# mid-size / Convergence zone / LGBM niche), OR "drop" to exclude entirely.
# Internally we map name → the raw row-cluster index that corresponds to
# that rank (largest cluster = rank 0 = first name, etc.).
override_path = "cluster_overrides.json"
overridden_ids = set()
blacklisted_ids = set() # dataset_ids that no cluster's hull can claim
anchored_ids = {} # dataset_id → cluster_rank (must appear on hull)
anchor_after = {} # dataset_id → dataset_id of hull vertex to insert AFTER
if _os.path.exists(override_path):
try:
overrides = _json.load(open(override_path))
# Build name → raw-label mapping by current rank order (largest first)
sizes = pd.Series(row_labels).value_counts()
rank_to_raw = sizes.index.tolist()
# Updated 2026-06-01 after the PCA was refit on the DISPLAY models
# only (single-fit GBDTs dropped). 5 clusters; ranks by size desc.
# Rank 1 is the GBDT-dominant cluster (the new finding).
name_to_rank = {
"2nd-gen FM Supremacy": 0,
"GBDTs rival FMs": 1,
"Tuned NNs rival FMs": 2,
}
id_to_pos = {did: i for i, did in enumerate(M.index)}
drop_mask = np.ones(len(row_labels), dtype=bool)
n_applied = 0
# First: pull the blacklist (special key "_blacklist": [id, id, ...])
for did in overrides.get("_blacklist", []) or []:
if did in id_to_pos:
blacklisted_ids.add(did)
# Pull anchors (special key "_anchor": {id: target, ...}).
# Anchored points keep their original cluster membership; the
# named cluster's hull just additionally includes this point as
# a vertex. So the anchored point can show up on multiple hulls.
# `target` accepts three shapes:
# → cluster rank 0..n-1
# "" → full cluster name from name_to_rank
# {"to": ,
# "after": ""} → also pin the insertion position:
# injected RIGHT AFTER `after`
# in the target cluster's hull
# vertex sequence.
for did, target in (overrides.get("_anchor", {}) or {}).items():
if did not in id_to_pos:
continue
after_id = None
# Extract `to` (rank) + optional `after`
if isinstance(target, dict):
after_id = target.get("after")
target = target.get("to")
if isinstance(target, int):
rank = target
elif isinstance(target, str) and target.isdigit():
rank = int(target)
elif isinstance(target, str) and target in name_to_rank:
rank = name_to_rank[target]
else:
continue
if 0 <= rank < len(rank_to_raw):
anchored_ids[did] = rank
if after_id:
anchor_after[did] = str(after_id)
for did, target_name in overrides.items():
if did.startswith("_"): # comment keys
continue
if did not in id_to_pos:
continue
pos = id_to_pos[did]
if target_name == "drop":
drop_mask[pos] = False
n_applied += 1
elif target_name == "blacklist":
blacklisted_ids.add(did)
n_applied += 1
elif target_name in name_to_rank:
rank = name_to_rank[target_name]
if rank < len(rank_to_raw):
row_labels[pos] = rank_to_raw[rank]
overridden_ids.add(did)
n_applied += 1
if n_applied or anchored_ids or blacklisted_ids:
print(
f"[perfmap] applied {n_applied} cluster override(s), "
f"blacklist={len(blacklisted_ids)}, anchors={len(anchored_ids)} "
f"from {override_path}"
)
# Drop the "drop"-labeled points from M and row_labels so they
# disappear from the plot entirely.
if not drop_mask.all():
M = M.loc[drop_mask]
row_labels = row_labels[drop_mask]
except Exception as _e:
print(f"[perfmap] could not load {override_path}: {_e}")
# PCA scores: use the frozen basis if available so axes stay stable.
if frozen is not None:
comps = np.array(frozen["pca_components"])
pca_mean = np.array(frozen["pca_mean"])
scores = (M.values - pca_mean) @ comps.T
pca_components = comps
else:
pca = PCA(n_components=2).fit(M.values)
scores = pca.transform(M.values)
pca_components = pca.components_
# 5×5 block matrix
unique_r = sorted(set(row_labels))
unique_c = sorted(set(col_labels))
B = np.zeros((len(unique_r), len(unique_c)))
for i, ki in enumerate(unique_r):
for j, kj in enumerate(unique_c):
rm = row_labels == ki
cm = col_labels == kj
if rm.sum() and cm.sum():
B[i, j] = M.values[rm][:, cm].mean()
# Per-model-cluster label = top-3 models in the cluster
from collections import Counter
col_labels_pretty = {}
for kj in unique_c:
cols = M.columns[col_labels == kj].tolist()
by_model = Counter(c.split("/")[1] for c in cols)
col_labels_pretty[kj] = ", ".join(
_MODEL_DISPLAY_PRETTY.get(m, m) for m, _ in by_model.most_common(3)
)
return {
"M": M,
"row_labels": row_labels,
"col_labels": col_labels,
"pca_scores": scores,
"pca_components": pca_components,
"block": B,
"unique_r": unique_r,
"unique_c": unique_c,
"col_labels_pretty": col_labels_pretty,
"n_datasets": M.shape[0],
"overridden_ids": overridden_ids,
"blacklisted_ids": blacklisted_ids,
"anchored_ids": anchored_ids,
"anchor_after": anchor_after,
"_per_dataset_df": per_dataset_df,
}
def _ellipse_shape(cx, cy, rx, ry, angle_deg=0, n=80):
"""Return SVG path tracing an ellipse (for plotly add_shape via path)."""
import numpy as np
t = np.linspace(0, 2 * np.pi, n)
cos_a = np.cos(np.radians(angle_deg))
sin_a = np.sin(np.radians(angle_deg))
x = cx + rx * np.cos(t) * cos_a - ry * np.sin(t) * sin_a
y = cy + rx * np.cos(t) * sin_a + ry * np.sin(t) * cos_a
pts = (
[f"M {x[0]} {y[0]}"] + [f"L {xi} {yi}" for xi, yi in zip(x[1:], y[1:])] + ["Z"]
)
return " ".join(pts)
def _prune_geometric_outliers(own_pts, max_z=2.0, protected_idx=None):
"""Drop any point whose Mahalanobis-normalized distance to the cluster
median exceeds `max_z`. Returns the array of kept indices into own_pts."""
import numpy as np
pts = np.asarray(own_pts, dtype=float)
n = len(pts)
if n < 4:
return np.arange(n)
if protected_idx is None:
protected_idx = set()
else:
protected_idx = set(protected_idx)
center = np.median(pts, axis=0)
spread = np.maximum(np.std(pts, axis=0), 1e-6)
z = np.linalg.norm((pts - center) / spread, axis=1)
keep = np.array([(i in protected_idx) or (z[i] <= max_z) for i in range(n)])
return np.where(keep)[0]
def _prune_for_purity(
own_pts, foreign_pts, min_evicted=3, protected_idx=None, max_iter=50
):
"""Iteratively prune CONVEX hull vertices whose removal evicts the most
foreign points from the convex hull's interior."""
import numpy as np
from scipy.spatial import ConvexHull
from matplotlib.path import Path
own = np.asarray(own_pts, dtype=float)
foreign = np.asarray(foreign_pts, dtype=float).reshape(-1, 2)
if len(own) < 4:
return np.arange(len(own))
if protected_idx is None:
protected_idx = set()
else:
protected_idx = set(protected_idx)
kept_idx = np.arange(len(own))
for _ in range(max_iter):
try:
hull = ConvexHull(own[kept_idx])
except Exception:
return kept_idx
hull_vertices_local = hull.vertices
if len(hull_vertices_local) < 4 or len(kept_idx) <= 4:
return kept_idx
hull_polygon = Path(own[kept_idx][hull_vertices_local])
if len(foreign) == 0:
return kept_idx
inside_now = hull_polygon.contains_points(foreign)
n_inside_now = int(inside_now.sum())
if n_inside_now == 0:
return kept_idx
best_v = None
best_evicted = 0
for v_local in hull_vertices_local:
global_v_idx = int(kept_idx[v_local])
if global_v_idx in protected_idx:
continue
trial = np.delete(kept_idx, np.where(kept_idx == global_v_idx)[0])
if len(trial) < 4:
continue
try:
trial_hull = ConvexHull(own[trial])
except Exception:
continue
trial_polygon = Path(own[trial][trial_hull.vertices])
n_inside_after = int(trial_polygon.contains_points(foreign).sum())
evicted = n_inside_now - n_inside_after
if evicted > best_evicted:
best_evicted = evicted
best_v = global_v_idx
if best_v is None or best_evicted < min_evicted:
return kept_idx
kept_idx = np.delete(kept_idx, np.where(kept_idx == best_v)[0])
return kept_idx
def _adopt_orphans(hull_points, candidate_points, max_dist=0.6):
"""If unclustered/foreign points sit within `max_dist` of the hull's centroid
AND within the hull's bounding-box-plus-margin, include them.
Returns the augmented point set.
`hull_points`: the (kept) cluster points that will form the hull.
`candidate_points`: external points that *might* belong with them.
"""
import numpy as np
pts = np.asarray(hull_points, dtype=float)
if len(pts) < 3:
return pts
cx, cy = pts.mean(axis=0)
# Adopt only points whose distance to the centroid is within the
# cluster's own 90th-percentile radius (so we don't pull from far away).
radii = np.linalg.norm(pts - [cx, cy], axis=1)
radius_limit = min(max_dist, float(np.quantile(radii, 0.90)) * 1.15)
cand = np.asarray(candidate_points, dtype=float).reshape(-1, 2)
if len(cand) == 0:
return pts
d = np.linalg.norm(cand - [cx, cy], axis=1)
keep = cand[d <= radius_limit]
return np.vstack([pts, keep]) if len(keep) else pts
def _concave_hull_indices(points, alpha_pct=85):
"""
Concave hull (alpha shape) via Delaunay triangulation.
Returns ordered indices of boundary points (a closed polygon).
`alpha_pct`: percentile (0..100) of triangle circumradius used as the
rejection threshold. Higher = more concave (tighter), lower = more
convex (looser). 100 = convex hull.
"""
import numpy as np
from scipy.spatial import Delaunay, ConvexHull
from collections import defaultdict
pts = np.asarray(points, dtype=float)
n = len(pts)
if n < 4:
try:
return ConvexHull(pts).vertices.tolist()
except Exception:
return list(range(n))
try:
tri = Delaunay(pts)
except Exception:
return ConvexHull(pts).vertices.tolist()
# Per-triangle circumradius
def circumradius(p0, p1, p2):
a = np.linalg.norm(p1 - p2)
b = np.linalg.norm(p0 - p2)
c = np.linalg.norm(p0 - p1)
s = (a + b + c) / 2
area = max(np.sqrt(max(s * (s - a) * (s - b) * (s - c), 0)), 1e-12)
return (a * b * c) / (4 * area)
radii = np.array(
[circumradius(pts[a], pts[b], pts[c]) for a, b, c in tri.simplices]
)
if alpha_pct >= 100:
return ConvexHull(pts).vertices.tolist()
threshold = np.percentile(radii, alpha_pct)
kept = tri.simplices[radii <= threshold]
if len(kept) == 0:
return ConvexHull(pts).vertices.tolist()
# Edges appearing exactly once = boundary edges
edge_count = defaultdict(int)
for a, b, c in kept:
for u, v in [(a, b), (b, c), (a, c)]:
edge_count[(min(u, v), max(u, v))] += 1
boundary_edges = [e for e, n_used in edge_count.items() if n_used == 1]
if not boundary_edges:
return ConvexHull(pts).vertices.tolist()
# Chain edges into ordered ring (pick the longest closed loop if multiple)
adj = defaultdict(list)
for u, v in boundary_edges:
adj[u].append(v)
adj[v].append(u)
# Greedy walk starting from any boundary vertex
visited = set()
rings = []
for start in list(adj.keys()):
if start in visited:
continue
ring = [start]
visited.add(start)
cur = start
prev = None
while True:
nbrs = [x for x in adj[cur] if x != prev]
if not nbrs:
break
nxt = next((x for x in nbrs if x not in visited), None)
if nxt is None:
# Closed the loop?
if start in adj[cur]:
break
break
ring.append(nxt)
visited.add(nxt)
prev = cur
cur = nxt
rings.append(ring)
longest = max(rings, key=len)
if len(longest) < 3:
return ConvexHull(pts).vertices.tolist()
return longest
def _raw_hull_path(points, alpha_pct=85, anchor_pts=None,
anchor_after_pts=None, smooth=0.0, expand=0.0,
use_as_is=False):
"""Concave hull (alpha-shape) polygon, with convex-hull fallback if the
alpha-shape would leave any member point outside.
`anchor_pts`: optional (n,2) array of points that MUST appear as hull
vertices. Each anchor not already on the natural boundary is inserted.
`anchor_after_pts`: optional list of (x,y) tuples, parallel to
`anchor_pts`. If provided, anchor i is inserted IMMEDIATELY AFTER the
hull vertex closest to `anchor_after_pts[i]`, instead of using the
default angular-order placement. Set an entry to None to fall back to
angular-order placement for that anchor.
`smooth`: 0..0.5; mild Chaikin corner-rounding applied AFTER the hull
is built. 0 = sharp corners (original behavior). 0.15 = gentle rounding.
"""
import numpy as np
from scipy.spatial import ConvexHull
from matplotlib.path import Path
pts = np.asarray(points, dtype=float)
if len(pts) < 3:
cx, cy = pts.mean(axis=0) if len(pts) else (0, 0)
return _ellipse_shape(cx, cy, 0.3, 0.3), []
# use_as_is=True ⇒ `points` is already an ordered polyline (typically
# from a hand-edited override). Skip alpha-shape/convex-hull computation,
# skip anchor injection, but still apply the optional outward push +
# smoothing.
# vertex_order: indices into `points` matching the order the boundary
# is traversed. Used so the caller can look up dataset IDs in render
# order for the dump file.
if use_as_is:
hull_xy = pts.copy()
vertex_order = list(range(len(pts)))
anchor_pts = None # bypass anchor insertion
else:
hull_xy = None
vertex_order = None
try:
boundary_idx = _concave_hull_indices(pts, alpha_pct=alpha_pct)
candidate = pts[boundary_idx]
poly = Path(candidate)
inside = poly.contains_points(pts, radius=1e-6) | poly.contains_points(
pts, radius=-1e-6
)
if inside.all():
hull_xy = candidate
vertex_order = list(boundary_idx)
except Exception:
hull_xy = None
if hull_xy is None:
try:
ch_verts = ConvexHull(pts).vertices
hull_xy = pts[ch_verts]
vertex_order = list(ch_verts)
except Exception:
return _ellipse_shape(pts[:, 0].mean(), pts[:, 1].mean(), 0.3, 0.3), []
# Inject anchor points that aren't already on the boundary.
# Default: insert in ANGULAR ORDER around the hull centroid (natural
# position at that bearing). Override per-anchor via `anchor_after_pts`:
# if provided, insert RIGHT AFTER the hull vertex closest to that point.
if anchor_pts is not None and len(anchor_pts) > 0:
anchors = np.asarray(anchor_pts, dtype=float).reshape(-1, 2)
# Normalise anchor_after_pts to same length as anchors
afters = list(anchor_after_pts) if anchor_after_pts is not None else []
while len(afters) < len(anchors):
afters.append(None)
cx, cy = hull_xy.mean(axis=0)
def angle(p):
return np.arctan2(p[1] - cy, p[0] - cx)
# Iterate in angular order. Stable: we never reorder afters.
order = sorted(range(len(anchors)), key=lambda i: angle(anchors[i]))
for idx in order:
anchor = anchors[idx]
after_pt = afters[idx]
on_boundary = np.any(np.all(np.isclose(hull_xy, anchor, atol=1e-6), axis=1))
if on_boundary:
continue
if after_pt is not None:
# Find the closest existing hull vertex to `after_pt`
ap = np.asarray(after_pt, dtype=float)
dists = np.linalg.norm(hull_xy - ap, axis=1)
best_i = int(np.argmin(dists))
else:
# Default angular-order placement
a_ang = angle(anchor)
angles = [angle(p) for p in hull_xy]
n = len(hull_xy)
best_i = 0
for i in range(n):
a1 = angles[i]
a2 = angles[(i + 1) % n]
if a2 < a1:
a2_n = a2 + 2 * np.pi
else:
a2_n = a2
a_norm = a_ang
if a_norm < a1:
a_norm += 2 * np.pi
if a1 < a_norm <= a2_n:
best_i = i
break
hull_xy = np.vstack([hull_xy[: best_i + 1], anchor, hull_xy[best_i + 1:]])
# Mirror the insertion in vertex_order so the index list stays
# aligned with the polyline. -1 = "anchor, not in `points`".
if vertex_order is not None:
vertex_order = vertex_order[: best_i + 1] + [-1] + vertex_order[best_i + 1:]
# Optional per-vertex outward push so corners aren't right on the data point
if expand and expand > 0 and len(hull_xy) >= 3:
n_in = len(hull_xy)
out = np.empty_like(hull_xy)
for i in range(n_in):
p_prev = hull_xy[(i - 1) % n_in]
p_curr = hull_xy[i]
p_next = hull_xy[(i + 1) % n_in]
e1 = p_curr - p_prev
e2 = p_next - p_curr
n1 = np.array([e1[1], -e1[0]])
n1 = n1 / (np.linalg.norm(n1) + 1e-9)
n2 = np.array([e2[1], -e2[0]])
n2 = n2 / (np.linalg.norm(n2) + 1e-9)
bisect = n1 + n2
bn = np.linalg.norm(bisect) + 1e-9
out[i] = p_curr + (bisect / bn) * expand
hull_xy = out
# Catmull-Rom subdivision: spline passes through every hull vertex,
# corners get implicit rounding from the in-between sub-samples.
if smooth and smooth > 0 and len(hull_xy) >= 3:
n_in = len(hull_xy)
n_samples = max(4, int(round(smooth * 20)))
new_pts = []
for i in range(n_in):
p0 = hull_xy[(i - 1) % n_in]
p1 = hull_xy[i]
p2 = hull_xy[(i + 1) % n_in]
p3 = hull_xy[(i + 2) % n_in]
ts = np.linspace(0, 1, n_samples, endpoint=False).reshape(-1, 1)
t2 = ts * ts
t3 = t2 * ts
c0 = -0.5 * t3 + t2 - 0.5 * ts
c1 = 1.5 * t3 - 2.5 * t2 + 1.0
c2 = -1.5 * t3 + 2.0 * t2 + 0.5 * ts
c3 = 0.5 * t3 - 0.5 * t2
new_pts.append(c0 * p0 + c1 * p1 + c2 * p2 + c3 * p3)
hull_xy = np.vstack(new_pts)
parts = (
[f"M {hull_xy[0, 0]} {hull_xy[0, 1]}"]
+ [f"L {p[0]} {p[1]}" for p in hull_xy[1:]]
+ ["Z"]
)
return " ".join(parts), (vertex_order if vertex_order is not None else [])
def _smoothed_hull_path(
points, expand=0.06, alpha_pct=85, _smoothing=None, _passes=None, **_kwargs
):
"""
Concave hull (alpha shape) → small per-vertex outward push → Catmull-Rom
spline through all hull vertices.
"""
import numpy as np
from scipy.spatial import ConvexHull
pts = np.asarray(points, dtype=float)
if len(pts) < 3:
cx, cy = pts.mean(axis=0) if len(pts) else (0, 0)
return _ellipse_shape(cx, cy, 0.3, 0.3)
try:
boundary_idx = _concave_hull_indices(pts, alpha_pct=alpha_pct)
hull_xy = pts[boundary_idx].astype(float)
except Exception:
try:
hull = ConvexHull(pts)
hull_xy = pts[hull.vertices].astype(float)
except Exception:
cx, cy = pts.mean(axis=0)
return _ellipse_shape(
cx, cy, max(0.3, pts[:, 0].std()), max(0.3, pts[:, 1].std())
)
n = len(hull_xy)
if expand and expand > 0:
out = np.empty_like(hull_xy)
for i in range(n):
p_prev = hull_xy[(i - 1) % n]
p_curr = hull_xy[i]
p_next = hull_xy[(i + 1) % n]
e1 = p_curr - p_prev
e2 = p_next - p_curr
n1 = np.array([e1[1], -e1[0]])
n1 = n1 / (np.linalg.norm(n1) + 1e-9)
n2 = np.array([e2[1], -e2[0]])
n2 = n2 / (np.linalg.norm(n2) + 1e-9)
bisect = n1 + n2
bn = np.linalg.norm(bisect) + 1e-9
# Curvature-aware boost: at sharp vertices (isolated points where
# the two adjacent edges form a tight angle), push outward MORE so
# the Catmull-Rom spline has room to curve smoothly instead of
# collapsing into a cusp. sharpness = 1 at a 0° spike, 0 at a
# straight (180°) edge.
cos_inner = float(np.clip(n1 @ n2, -1.0, 1.0)) # 1=flat, -1=spike
sharpness = (1.0 - cos_inner) * 0.5 # in [0, 1]
boost = 1.0 + 4.0 * sharpness # up to 5× at spikes
out[i] = p_curr + (bisect / bn) * expand * boost
hull_xy = out
def catmull_rom_segment(p0, p1, p2, p3, n_samples=16):
t = np.linspace(0, 1, n_samples, endpoint=False).reshape(-1, 1)
t2 = t * t
t3 = t2 * t
c0 = -0.5 * t3 + t2 - 0.5 * t
c1 = 1.5 * t3 - 2.5 * t2 + 1.0
c2 = -1.5 * t3 + 2.0 * t2 + 0.5 * t
c3 = 0.5 * t3 - 0.5 * t2
return c0 * p0 + c1 * p1 + c2 * p2 + c3 * p3
samples = []
for i in range(n):
p0 = hull_xy[(i - 1) % n]
p1 = hull_xy[i]
p2 = hull_xy[(i + 1) % n]
p3 = hull_xy[(i + 2) % n]
samples.append(catmull_rom_segment(p0, p1, p2, p3, n_samples=16))
smoothed = np.vstack(samples)
pts_path = (
[f"M {smoothed[0, 0]} {smoothed[0, 1]}"]
+ [f"L {p[0]} {p[1]}" for p in smoothed[1:]]
+ ["Z"]
)
return " ".join(pts_path)
def _apply_low_repulsion(point_groups, strength=0.25, iters=20):
"""
Adjust cluster centroids by a small repulsion vector when two clusters'
centroids are within their combined std radii. Only moves the OFFSETS;
the underlying points stay where they are. Returns per-cluster (dx, dy)
offsets which we then apply to the hull centroid only — used to nudge
overlapping cluster *labels* apart, not the hulls themselves.
"""
import numpy as np
centroids = np.array([g["pts"].mean(axis=0) for g in point_groups])
radii = np.array([max(g["pts"].std(axis=0).mean(), 0.3) for g in point_groups])
offsets = np.zeros_like(centroids)
for _ in range(iters):
moved = False
for i in range(len(centroids)):
for j in range(i + 1, len(centroids)):
p_i = centroids[i] + offsets[i]
p_j = centroids[j] + offsets[j]
d = np.linalg.norm(p_i - p_j) + 1e-6
r = radii[i] + radii[j]
if d < r:
push = (p_i - p_j) / d
delta = strength * (r - d) * 0.5
offsets[i] += push * delta
offsets[j] -= push * delta
moved = True
if not moved:
break
return offsets
def _smooth_scalar_field(
fig, scores, scalar, zmin, zmax, colorscale, colorbar_title, sigma=0.45
):
"""Render a kernel-smoothed per-dataset scalar field as a background heatmap."""
import numpy as np
import plotly.graph_objects as go
from scipy.spatial.distance import cdist
x_min, x_max = scores[:, 0].min() - 0.5, scores[:, 0].max() + 0.5
y_min, y_max = scores[:, 1].min() - 0.5, scores[:, 1].max() + 0.5
nx, ny = 60, 60
xs = np.linspace(x_min, x_max, nx)
ys = np.linspace(y_min, y_max, ny)
XX, YY = np.meshgrid(xs, ys)
grid = np.column_stack([XX.ravel(), YY.ravel()])
d2 = cdist(grid, scores, metric="sqeuclidean")
w = np.exp(-d2 / (2 * sigma**2))
w_sum = w.sum(axis=1)
w_sum[w_sum < 1e-6] = 1e-6
avg = (w @ scalar) / w_sum
Z = avg.reshape(ny, nx)
fig.add_trace(
go.Heatmap(
z=Z,
x=xs,
y=ys,
colorscale=colorscale,
zmin=zmin,
zmax=zmax,
opacity=0.45,
showscale=True,
colorbar=dict(title=colorbar_title, thickness=10, len=0.5, y=0.5),
hoverinfo="skip",
)
)
# Map of background key → (scalar-builder, zmin, zmax, colorscale, label).
# Scalar-builder takes the perfmap dict, returns a per-dataset numpy array
# aligned to perfmap["M"].index ordering.
import numpy as _np
def _bg_fm_gen(perfmap, model_keys):
cols = [
f"Cross_entropy/{m}"
for m in model_keys
if f"Cross_entropy/{m}" in perfmap["M"].columns
]
return perfmap["M"][cols].mean(axis=1).values if cols else None
def _bg_fm_minus_gbdt_accuracy(perfmap):
M = perfmap["M"]
fm_cols = [
f"Accuracy/{m}"
for m in (
"seldon",
"tabpfn_v3",
"tabicl_v2",
"tabicl",
"tabpfn",
"mitra",
"limix",
"tabdpt",
)
if f"Accuracy/{m}" in M.columns
]
gbdt_cols = [
f"Accuracy/{m}"
for m in (
"xgboost_ensemble",
"catboost_ensemble",
"lightgbm_ensemble",
"xgboost",
"catboost",
"lightgbm",
)
if f"Accuracy/{m}" in M.columns
]
if not fm_cols or not gbdt_cols:
return None
return (M[fm_cols].mean(axis=1) - M[gbdt_cols].mean(axis=1)).values
def _bg_fm_minus_nn_accuracy(perfmap):
M = perfmap["M"]
fm_cols = [
f"Accuracy/{m}"
for m in (
"seldon",
"tabpfn_v3",
"tabicl_v2",
"tabicl",
"tabpfn",
"mitra",
"limix",
"tabdpt",
)
if f"Accuracy/{m}" in M.columns
]
nn_cols = [
f"Accuracy/{m}"
for m in ("realmlp", "tabm", "modern_nca")
if f"Accuracy/{m}" in M.columns
]
if not fm_cols or not nn_cols:
return None
return (M[fm_cols].mean(axis=1) - M[nn_cols].mean(axis=1)).values
def _bg_modern_minus_legacy_fm(perfmap):
M = perfmap["M"]
m2 = [
f"Accuracy/{m}"
for m in ("seldon", "tabpfn_v3", "tabicl_v2")
if f"Accuracy/{m}" in M.columns
]
m1 = [
f"Accuracy/{m}"
for m in ("tabpfn", "mitra", "limix")
if f"Accuracy/{m}" in M.columns
]
if not m2 or not m1:
return None
return (M[m2].mean(axis=1) - M[m1].mean(axis=1)).values
def _bg_f1_minus_accuracy(perfmap):
"""Mean (F1 rank − Accuracy rank) across all models — proxy for imbalance."""
M = perfmap["M"]
f1_cols = [c for c in M.columns if c.startswith("F1_score/")]
acc_cols = [c for c in M.columns if c.startswith("Accuracy/")]
if not f1_cols or not acc_cols:
return None
return (M[f1_cols].mean(axis=1) - M[acc_cols].mean(axis=1)).values
def _bg_ce_spread(perfmap):
"""Cross-model std of CE rank — disagreement on calibration."""
M = perfmap["M"]
ce_cols = [c for c in M.columns if c.startswith("Cross_entropy/")]
if not ce_cols:
return None
return M[ce_cols].std(axis=1).values
def _bg_best_accuracy(perfmap):
"""Best model's accuracy rank per dataset — proxy for difficulty."""
M = perfmap["M"]
acc_cols = [c for c in M.columns if c.startswith("Accuracy/")]
if not acc_cols:
return None
return M[acc_cols].max(axis=1).values
def _bg_gbdt_minus_nn_accuracy(perfmap):
"""GBDT (ensemble) − Tuned NN accuracy rank. Positive almost everywhere
except the multi-class large cluster, where NNs flip the order."""
M = perfmap["M"]
gbdt_cols = [
f"Accuracy/{m}"
for m in (
"xgboost_ensemble",
"catboost_ensemble",
"lightgbm_ensemble",
"xgboost",
"catboost",
"lightgbm",
)
if f"Accuracy/{m}" in M.columns
]
nn_cols = [
f"Accuracy/{m}"
for m in ("realmlp", "tabm", "modern_nca")
if f"Accuracy/{m}" in M.columns
]
if not gbdt_cols or not nn_cols:
return None
return (M[gbdt_cols].mean(axis=1) - M[nn_cols].mean(axis=1)).values
def _bg_tabdpt_minus_other_fm1(perfmap):
"""TabDPT accuracy minus the other 1st-gen FMs' mean. Highlights where
TabDPT specifically falls behind its 1st-gen peers (≈ context-window /
pretraining-fit signal)."""
M = perfmap["M"]
if "Accuracy/tabdpt" not in M.columns:
return None
other = [
f"Accuracy/{m}"
for m in ("tabpfn", "mitra", "limix", "tabicl")
if f"Accuracy/{m}" in M.columns
]
if not other:
return None
return (M["Accuracy/tabdpt"] - M[other].mean(axis=1)).values
def _bg_imbalance(perfmap):
"""Per-dataset (mean Accuracy − mean F1) across all models.
On imbalanced datasets, Accuracy stays high while F1 tanks → positive gap.
The previous implementation was computed on the rank-normalised matrix
where row-means are identically 0.5 for every column block — so the
signal was always zero. We now read the raw Accuracy / F1_score
values from the per-dataset frame.
"""
M = perfmap["M"]
df = perfmap.get("_per_dataset_df")
if df is None or "Accuracy" not in df.columns or "F1_score" not in df.columns:
return None
per_ds = df.groupby("dataset_id")[["Accuracy", "F1_score"]].mean()
per_ds["gap"] = per_ds["Accuracy"] - per_ds["F1_score"]
# Align to M.index (string ids)
aligned = per_ds["gap"].reindex(M.index.astype(str))
return aligned.values
# Registry: key → (builder, zmin, zmax, colorscale, colorbar title)
_PERFMAP_BACKGROUNDS = {
"none": None,
"fm1_ce": (
lambda p: _bg_fm_gen(p, ("tabpfn", "mitra", "limix")),
0.0,
1.0,
"RdYlGn",
"FM 1st gen CE rank",
),
"fm2_ce": (
lambda p: _bg_fm_gen(p, ("seldon", "tabpfn_v3", "tabicl_v2")),
0.0,
1.0,
"RdYlGn",
"FM 2nd gen CE rank",
),
"fm_vs_gbdt": (
_bg_fm_minus_gbdt_accuracy,
-0.25,
0.25,
"RdBu",
"FM − GBDT accuracy rank",
),
"fm_vs_nn": (
_bg_fm_minus_nn_accuracy,
-0.20,
0.20,
"RdBu",
"FM − NN accuracy rank",
),
"modern_vs_leg": (
_bg_modern_minus_legacy_fm,
-0.25,
0.25,
"RdBu",
"2nd-gen − 1st-gen FM accuracy",
),
"gbdt_vs_nn": (
_bg_gbdt_minus_nn_accuracy,
-0.25,
0.25,
"RdBu",
"GBDT − NN accuracy rank",
),
"tabdpt_vs_fm1": (
_bg_tabdpt_minus_other_fm1,
-0.35,
0.35,
"RdBu",
"TabDPT − other 1st-gen FMs",
),
"imbalance": (
_bg_imbalance,
0.0,
0.40,
"Reds",
"Class imbalance (Acc − F1 gap)",
),
}
def plot_perfmap_scatter(
perfmap, meta_df, title_extra="", background="none", smooth=0.5, expand=0.01
):
"""PC1×PC2 dataset scatter with smoothed cluster hulls + named labels.
`background`: one of "none", "fm1", "fm2". Adds a CE-calibration heatmap
for the chosen FM family behind the cluster hulls.
"""
import plotly.graph_objects as go
import numpy as np
scores = perfmap["pca_scores"]
row_labels = perfmap["row_labels"]
M = perfmap["M"]
# ----- Per-cluster names (in size order, largest first) -----
# Defaults are tuned for the full-base clustering. If the filter changes
# what's in each cluster, the labels stay generic: that's fine; the goal
# is to communicate the shape of the map, not pin each cluster to a label.
CLUSTER_NAMES = [
("2nd-gen FM Supremacy", "#5a30c2"),
("GBDTs rival FMs", "#3970d0"),
("Tuned NNs rival FMs", "#d97706"),
]
fig = go.Figure()
# ----- Optional scalar-field background -----
spec = _PERFMAP_BACKGROUNDS.get(background)
if spec is not None:
builder, zmin, zmax, cmap, cbar_title = spec
scalar = builder(perfmap)
if scalar is not None and len(scalar) == len(scores):
_smooth_scalar_field(fig, scores, scalar, zmin, zmax, cmap, cbar_title)
# Order clusters largest-first so label-placement priorities make sense
cluster_sizes = {k: int((row_labels == k).sum()) for k in set(row_labels)}
cluster_order = sorted(cluster_sizes.keys(), key=lambda k: -cluster_sizes[k])
# Build per-cluster point groups (for hulls + label placement).
# Each group gets: pts (all points), hull_pts (after pruning + adopting),
# idx (boolean mask into scores).
point_groups = []
for rank, k in enumerate(cluster_order):
idx = row_labels == k
cluster_pts = scores[idx, :]
name, color = (
CLUSTER_NAMES[rank]
if rank < len(CLUSTER_NAMES)
else (f"Cluster {k}", "#666")
)
point_groups.append(
{
"k": k,
"pts": cluster_pts,
"name": name,
"color": color,
"idx": idx,
}
)
# For each cluster: iteratively prune hull vertices whose removal evicts
# foreign points from the hull's interior. Tight clusters keep every point;
# gerrymandered hulls shrink. Overridden points are protected from pruning.
# Blacklisted points are excluded from EVERY cluster's hull (they still
# show as dots).
overridden_ids = perfmap.get("overridden_ids", set())
blacklisted_ids = perfmap.get("blacklisted_ids", set())
anchored_ids = perfmap.get("anchored_ids", {})
anchor_after = perfmap.get("anchor_after", {})
# All-IDs-to-coord lookup for the dump + override file. We need this
# because the hand-edited override stores IDs, not coords.
_all_ids_list = M.index.astype(str).tolist()
_id_to_score = {did: scores[i] for i, did in enumerate(_all_ids_list)}
for i, g in enumerate(point_groups):
# Filter out blacklisted points from this cluster's hull input
own_ids = M.index[g["idx"]].tolist()
keep_mask = np.array([did not in blacklisted_ids for did in own_ids])
hull_input = g["pts"][keep_mask]
kept_own_ids = [did for did, k in zip(own_ids, keep_mask) if k]
if len(hull_input) < 4:
g["hull_pts"] = hull_input
g["hull_ids"] = list(kept_own_ids)
continue
# Foreign points = every point from every OTHER cluster (also blacklist-filtered)
foreign_pts = []
for j, other in enumerate(point_groups):
if j == i:
continue
other_ids = M.index[other["idx"]].tolist()
other_mask = np.array([did not in blacklisted_ids for did in other_ids])
foreign_pts.append(other["pts"][other_mask])
foreign = np.vstack(foreign_pts) if foreign_pts else np.empty((0, 2))
# Protected = overridden points in this cluster
protected_local = {
li for li, did in enumerate(kept_own_ids) if did in overridden_ids
}
kept_idx = _prune_for_purity(
hull_input, foreign, min_evicted=3, protected_idx=protected_local
)
g["hull_pts"] = hull_input[kept_idx]
g["hull_ids"] = [str(kept_own_ids[k]) for k in kept_idx]
# Anchors: any dataset whose anchor target maps to THIS cluster's rank
# gets added as a hull vertex of THIS cluster. The point keeps its
# original cluster membership; this is only about hull geometry.
g_rank = cluster_order.index(g["k"])
all_ids = M.index.tolist()
id_to_pos = {did: pos for pos, did in enumerate(all_ids)}
anchor_coords = []
anchor_after_pts = [] # parallel list: where to insert each anchor
for did, target_rank in anchored_ids.items():
if target_rank == g_rank and did in id_to_pos:
anchor_coords.append(scores[id_to_pos[did]])
after_id = anchor_after.get(did)
if after_id and after_id in id_to_pos:
anchor_after_pts.append(scores[id_to_pos[after_id]])
else:
anchor_after_pts.append(None)
g["anchor_pts"] = np.array(anchor_coords) if anchor_coords else None
g["anchor_after_pts"] = anchor_after_pts
# ----- Hand-editable hull pipeline ---------------------------------------
# The dump is written AFTER the polyline is drawn, using the ordered
# traversal indices returned by `_raw_hull_path`. That guarantees the
# dumped sequence matches what you see on the map.
import os as _os
import json as _json
# Read the override (if any) and replace hull_pts / hull_ids per cluster.
override_path = "perfmap_hulls_override.json"
if _os.path.exists(override_path):
try:
ovr = _json.load(open(override_path))
except Exception as _e:
print(f"[perfmap] could not load {override_path}: {_e}")
ovr = {}
for gi, g in enumerate(point_groups):
try:
rank = cluster_order.index(g["k"])
except (KeyError, ValueError):
rank = gi
seq = ovr.get(str(rank))
if not seq:
continue
# Resolve ID sequence to coordinates. Unknown IDs are skipped.
coords = []
valid_ids = []
for did in seq:
did_s = str(did)
if did_s in _id_to_score:
coords.append(_id_to_score[did_s])
valid_ids.append(did_s)
if len(coords) >= 3:
g["hull_pts"] = np.array(coords)
g["hull_ids"] = valid_ids
g["hand_curated"] = True
# Hand-defined hull → skip anchor injection (the user already
# placed every vertex they wanted).
g["anchor_pts"] = None
g["anchor_after_pts"] = []
# ----- Draw hulls (below points). All four current clusters are dense
# enough to support a closed hull.
# We also capture the ordered ID sequence per cluster (in render order)
# so we can write a dump file for the user to hand-edit.
dumped_seqs = {}
for gi, g in enumerate(point_groups):
hp = g["hull_pts"]
if len(hp) < 3:
continue
path, vertex_order = _raw_hull_path(
hp,
anchor_pts=g.get("anchor_pts"),
anchor_after_pts=g.get("anchor_after_pts"),
smooth=smooth,
expand=expand,
use_as_is=bool(g.get("hand_curated")),
)
# Map the vertex_order indices back to dataset_ids in render order.
hull_ids = list(g.get("hull_ids", []))
ordered_ids = []
for vi in vertex_order:
if vi == -1:
ordered_ids.append("ANCHOR") # anchored point (not in cluster's own member set)
elif 0 <= vi < len(hull_ids):
ordered_ids.append(hull_ids[vi])
try:
rank = cluster_order.index(g["k"])
except (KeyError, ValueError):
rank = gi
dumped_seqs[str(rank)] = ordered_ids
col = g["color"]
r, gg, b = int(col[1:3], 16), int(col[3:5], 16), int(col[5:7], 16)
fig.add_shape(
type="path",
path=path,
fillcolor=f"rgba({r},{gg},{b},0.22)",
line=dict(color=f"rgba({r},{gg},{b},0.75)", width=1.5),
layer="below",
)
# Write the dump file. This is what the user copies → edits → renames
# to perfmap_hulls_override.json to take effect.
try:
with open("perfmap_hulls_dump.json", "w") as _f:
_json.dump(dumped_seqs, _f, indent=2)
except Exception as _e:
print(f"[perfmap] could not write perfmap_hulls_dump.json: {_e}")
# ----- Compute label positions to avoid overlap with dataset points.
# For each cluster we pick a position within the cluster's bounding
# halo that maximises distance to the K nearest points (across ALL
# clusters). Candidates: 16 directions × 2 radii around the centroid.
all_pts = np.vstack([g["pts"] for g in point_groups if len(g["pts"]) > 0])
placed_labels = [] # list of (x, y) of labels already placed
for g in point_groups:
if len(g["pts"]) == 0:
continue
cx, cy = g["pts"].mean(axis=0)
sx = max(g["pts"][:, 0].std(), 0.25)
sy = max(g["pts"][:, 1].std(), 0.25)
# 16 directions, 2 radii (close + far) → 32 candidates
candidates = []
for angle_idx in range(16):
theta = (angle_idx / 16.0) * 2 * np.pi
for radius_mult in (1.30, 1.80, 2.20):
cand_x = cx + np.cos(theta) * sx * radius_mult
cand_y = cy + np.sin(theta) * sy * radius_mult
candidates.append((cand_x, cand_y))
# Score each candidate by min-distance to the K=8 nearest points,
# minus a small penalty for distance to already-placed labels.
best = None
best_score = -np.inf
for (cand_x, cand_y) in candidates:
d_to_pts = np.sqrt((all_pts[:, 0] - cand_x) ** 2 + (all_pts[:, 1] - cand_y) ** 2)
k_nearest = np.partition(d_to_pts, min(8, len(d_to_pts) - 1))[:min(8, len(d_to_pts))]
score = float(k_nearest.mean())
# Penalise candidates close to already-placed labels
for (px, py) in placed_labels:
dl = np.hypot(cand_x - px, cand_y - py)
if dl < 0.6:
score -= (0.6 - dl) * 2.0
if score > best_score:
best_score = score
best = (cand_x, cand_y)
lx, ly = best
placed_labels.append((lx, ly))
fig.add_annotation(
x=lx,
y=ly,
text=f"{g['name']}",
showarrow=False,
font=dict(color=g["color"], size=11),
# No solid bg: works on light and dark themes. The cluster color
# carries the label's identity; a thin tinted stroke gives shape.
bgcolor="rgba(0,0,0,0)",
bordercolor=g["color"],
borderwidth=1,
borderpad=4,
)
# ----- The dataset points themselves: small + faint -----
for g in point_groups:
if len(g["pts"]) == 0:
continue
hover_texts = []
for did in M.index[g["idx"]]:
try:
meta = meta_df.loc[did]
name = meta["name"]
ind = meta["industry"]
feats = meta["features"]
rows_n = meta["rows"]
cls_n = meta["n_classes"]
except Exception:
name = did
ind = ""
feats = rows_n = cls_n = None
parts = [f"{name} id: {did}"]
if ind:
parts.append(f"vertical: {ind}")
if feats is not None and not (isinstance(feats, float) and feats != feats):
parts.append(f"features: {int(feats)}")
if rows_n is not None and not (
isinstance(rows_n, float) and rows_n != rows_n
):
parts.append(f"rows: {int(rows_n)}")
if cls_n is not None and not (isinstance(cls_n, float) and cls_n != cls_n):
parts.append(f"classes: {int(cls_n)}")
hover_texts.append(" ".join(parts))
fig.add_trace(
go.Scatter(
x=g["pts"][:, 0],
y=g["pts"][:, 1],
mode="markers",
marker=dict(size=6, color=g["color"], opacity=0.8, line=dict(width=0)),
showlegend=False,
hovertext=hover_texts,
hoverinfo="text",
)
)
fig.update_layout(
height=600,
margin=dict(l=120, r=160, t=60, b=70),
template="simple_white",
plot_bgcolor="rgba(0,0,0,0)",
paper_bgcolor="rgba(0,0,0,0)",
xaxis=dict(
title="",
showticklabels=False,
ticks="",
zeroline=False,
showgrid=False,
showline=False,
),
yaxis=dict(
title="",
showticklabels=False,
ticks="",
zeroline=False,
showgrid=False,
showline=False,
),
title=dict(
text=f"Performance map · {perfmap['n_datasets']} datasets{title_extra}",
x=0.5,
xanchor="center",
font=dict(size=14),
),
)
# Magic-quadrant style: 4 block arrows (left, right, down, up). Each
# arrow is anchored at a paper-fraction position; its geometry (shaft
# length, thickness, head size) is entirely in PIXELS via Plotly's
# `xsizemode="pixel"` / `ysizemode="pixel"`. This is resolution- and
# aspect-ratio-independent: the figure can autosize freely, the arrows
# stay the same shape and pixel size on any screen.
AX_FONT = dict(size=10, color="#7a7a90", family="Arial")
EDGE = "#9aa3ad"
FILL = "rgba(154,163,173,0.10)"
BODY_PX = 18 # shaft thickness, pixels
HEAD_LEN_PX = 28 # head length along arrow direction, pixels
HEAD_SPAN_PX = 28 # head span perpendicular to arrow, pixels (≈square)
GAP_PX = 12 # gap from the centre crossing, pixels
AX_Y = -0.07
AX_X = -0.03
def _add_h_arrow(x_anchor_paper, y_anchor_paper, direction, shaft_px):
"""Horizontal arrow. Anchored at the TAIL (centre side).
direction: 'left' or 'right'. shaft_px = total arrow length in pixels.
"""
sign = -1 if direction == "left" else 1
tip_x = sign * shaft_px
head_base_x = sign * (shaft_px - HEAD_LEN_PX)
body = BODY_PX / 2
span = HEAD_SPAN_PX / 2
path = (
f"M 0 {-body} "
f"L {head_base_x} {-body} "
f"L {head_base_x} {-span} "
f"L {tip_x} 0 "
f"L {head_base_x} {span} "
f"L {head_base_x} {body} "
f"L 0 {body} Z"
)
fig.add_shape(
type="path",
path=path,
xref="paper",
yref="paper",
xsizemode="pixel",
ysizemode="pixel",
xanchor=x_anchor_paper,
yanchor=y_anchor_paper,
fillcolor=FILL,
line=dict(color=EDGE, width=1.0),
)
def _add_v_arrow(x_anchor_paper, y_anchor_paper, direction, shaft_px):
"""Vertical arrow. Anchored at the TAIL (centre side).
direction: 'up' or 'down'. shaft_px = total arrow length in pixels.
Note: pixel y is screen-down-positive, so 'up' = negative pixel y.
"""
sign = -1 if direction == "up" else 1 # screen pixels: +y = down
tip_y = sign * shaft_px
head_base_y = sign * (shaft_px - HEAD_LEN_PX)
body = BODY_PX / 2
span = HEAD_SPAN_PX / 2
path = (
f"M {-body} 0 "
f"L {-body} {head_base_y} "
f"L {-span} {head_base_y} "
f"L 0 {tip_y} "
f"L {span} {head_base_y} "
f"L {body} {head_base_y} "
f"L {body} 0 Z"
)
fig.add_shape(
type="path",
path=path,
xref="paper",
yref="paper",
xsizemode="pixel",
ysizemode="pixel",
xanchor=x_anchor_paper,
yanchor=y_anchor_paper,
fillcolor=FILL,
line=dict(color=EDGE, width=1.0),
)
# Compute shaft length so the arrow tip sits a fixed-pixel inset from the
# paper-edge. We do this by anchoring the tail at the centre (paper 0.5)
# and the tip extends in pixels; the rendered length therefore equals
# half-the-plot-width in pixels MINUS an edge inset. Since we want the
# tip near the edge but we don't know the plot pixel width at runtime,
# use a generous shaft length and let it clip naturally. A "long enough"
# value works because Plotly clips at the plot area when needed.
# In practice, ~480px works for the typical plot width Gradio renders
# (and matches what people saw before when the figure was 920 wide).
H_SHAFT_PX = 420
V_SHAFT_PX = 230
# Horizontal axis: both arrows anchored at the centre, pointing outward.
_add_h_arrow(0.50, AX_Y, "left", H_SHAFT_PX)
_add_h_arrow(0.50, AX_Y, "right", H_SHAFT_PX)
# Vertical axis: both arrows anchored at the centre, pointing outward.
_add_v_arrow(AX_X, 0.50, "down", V_SHAFT_PX)
_add_v_arrow(AX_X, 0.50, "up", V_SHAFT_PX)
# Labels: pinned in PIXELS to the centre crossing so they stay clear of
# the arrow tails regardless of plot size.
fig.add_annotation(
xref="paper",
yref="paper",
x=0.50,
y=AX_Y,
text="More numeric · Fewer rows",
showarrow=False,
xanchor="right",
yanchor="middle",
xshift=-GAP_PX,
font=AX_FONT,
)
fig.add_annotation(
xref="paper",
yref="paper",
x=0.50,
y=AX_Y,
text="More categorical · More rows",
showarrow=False,
xanchor="left",
yanchor="middle",
xshift=+GAP_PX,
font=AX_FONT,
)
fig.add_annotation(
xref="paper",
yref="paper",
x=AX_X,
y=0.50,
text="Multi-class · More imbalanced",
showarrow=False,
xanchor="center",
yanchor="bottom",
yshift=+GAP_PX,
textangle=-90,
font=AX_FONT,
)
fig.add_annotation(
xref="paper",
yref="paper",
x=AX_X,
y=0.50,
text="Binary · More balanced",
showarrow=False,
xanchor="center",
yanchor="top",
yshift=-GAP_PX,
textangle=-90,
font=AX_FONT,
)
return fig
def plot_perfmap_blockgrid(perfmap):
"""5×5 heatmap: rows = dataset clusters, cols = model-clusters, color = winrate."""
import plotly.graph_objects as go
B = perfmap["block"]
unique_r = perfmap["unique_r"]
unique_c = perfmap["unique_c"]
col_labels = perfmap["col_labels_pretty"]
row_sizes = {ki: int((perfmap["row_labels"] == ki).sum()) for ki in unique_r}
y_labels = [f"D{ki} n={row_sizes[ki]}" for ki in unique_r]
x_labels = [f"M{kj} {col_labels[kj]}" for kj in unique_c]
# Text annotations (rounded values)
text = [
[f"{B[i,j]:.2f}" for j in range(len(unique_c))] for i in range(len(unique_r))
]
fig = go.Figure(
data=go.Heatmap(
z=B,
x=x_labels,
y=y_labels,
text=text,
texttemplate="%{text}",
colorscale=[[0.0, "#e0dee7"], [0.5, "#a18cff"], [1.0, "#4d2cdc"]],
zmin=0.0,
zmax=1.0,
colorbar=dict(title="winrate", thickness=12),
hovertemplate="dataset cluster %{y} model cluster %{x} winrate=%{z:.3f}",
)
)
# Mark the diagonal winners with a thicker border
import numpy as np
for i in range(len(unique_r)):
j_best = int(np.argmax(B[i, :]))
fig.add_shape(
type="rect",
xref="x",
yref="y",
x0=j_best - 0.5,
x1=j_best + 0.5,
y0=i - 0.5,
y1=i + 0.5,
line=dict(color="#1a0a6b", width=2.5),
fillcolor="rgba(0,0,0,0)",
)
fig.update_layout(
height=420,
margin=dict(l=80, r=40, t=40, b=60),
template="simple_white",
plot_bgcolor="rgba(0,0,0,0)",
paper_bgcolor="rgba(0,0,0,0)",
title=dict(
text="Who wins where: dataset-cluster × model-cluster winrate",
x=0.5,
xanchor="center",
font=dict(size=13),
),
xaxis=dict(side="bottom"),
yaxis=dict(autorange="reversed"),
)
return fig
def _perfmap_meta_df():
"""Helper: load public_datasets_info.json into a metadata DataFrame keyed by dataset_id."""
import pandas as pd, json as _json
info_list = _json.load(open("public_datasets_info.json"))
rows = []
for r in info_list:
rows.append(
{
"dataset_id": str(r["dataset_id"]),
"name": r.get("dataset_name") or str(r["dataset_id"]),
"industry": r.get("dataset_industry") or "",
"features": r.get("features"),
"rows": r.get("rows"),
"n_classes": r.get("n_classes"),
}
)
return pd.DataFrame(rows).set_index("dataset_id")
def _build_perfmap_dtree_data(per_dataset_df):
"""Build feature matrix + two label vectors for the decision-tree heuristics:
1. winner: which family wins on Accuracy ('FM' / 'GBDT' / 'NN')
2. is_close: 1 if FM's lead over the runner-up family is < 0.05, else 0
Returns (X, feature_names, y_winner, y_close, kept_dataset_ids).
"""
import numpy as np, pandas as pd, json as _json
# Tree compares the three families that win clusters: 2nd-gen FMs vs
# GBDT ensembles vs Tuned NNs. 1st-gen FMs are intentionally excluded —
# they never win a regime, so including them only adds tie noise.
FAMILIES = {
"FM2": ["seldon", "tabpfn_v3", "tabicl_v2"],
"GBDT": ["xgboost_ensemble", "catboost_ensemble", "lightgbm_ensemble"],
"NN": ["realmlp", "tabm", "modern_nca"],
}
info_list = _json.load(open("public_datasets_info.json"))
info = {str(r["dataset_id"]): r for r in info_list}
# Family-mean Accuracy per dataset
family_acc = {}
for did in per_dataset_df["dataset_id"].unique():
sub = per_dataset_df[per_dataset_df["dataset_id"] == did]
d = {}
for fam, models in FAMILIES.items():
vals = sub[sub["model"].isin(models)]["Accuracy"].dropna()
if len(vals):
d[fam] = float(vals.mean())
if d:
family_acc[did] = d
# Per-dataset acc and F1 means (for imbalance proxy)
acc_f1 = per_dataset_df.groupby("dataset_id").agg(
acc=("Accuracy", "mean"), f1=("F1_score", "mean")
)
acc_f1["imbalance"] = acc_f1["acc"] - acc_f1["f1"]
X_rows, y_win, y_close, kept = [], [], [], []
for did, fa in family_acc.items():
m = info.get(did, {})
feats = m.get("features")
rows_n = m.get("rows")
cls = m.get("n_classes")
pct = m.get("pct_cat")
ac = m.get("avg_card", 0)
imb = float(acc_f1.loc[did, "imbalance"]) if did in acc_f1.index else None
if any(v is None for v in (feats, rows_n, cls, pct, imb)):
continue
# winner (FM2 / GBDT / NN: the three leaf classes the tree learns)
winner = max(fa, key=fa.get)
# close = is FM2 ahead of the best non-FM by < 0.05?
if "FM2" not in fa:
continue
fm_score = fa["FM2"]
others = [v for k, v in fa.items() if k != "FM2"]
if not others:
continue
runner_up = max(others)
is_close = int((fm_score - runner_up) < 0.05)
X_rows.append([feats, rows_n, cls, pct, ac, imb])
y_win.append(winner)
y_close.append(is_close)
kept.append(did)
return (
np.array(X_rows, dtype=float),
["n_features", "n_rows", "n_classes", "pct_cat", "avg_card", "imbalance"],
np.array(y_win),
np.array(y_close),
kept,
)
def render_perfmap_decision_tree(per_dataset_df, mode="winner"):
"""Fit a depth-3 balanced decision tree and render it as a Plotly figure
using shapes + annotations. Two modes:
- "winner": predicts the winning family (FM/GBDT/NN)
- "close" : predicts whether FM's lead is < 0.05 (alternatives competitive)
Returns a plotly Figure.
"""
import numpy as np, plotly.graph_objects as go
from sklearn.tree import DecisionTreeClassifier
X, feat_names, y_win, y_close, _ = _build_perfmap_dtree_data(per_dataset_df)
y = y_win if mode == "winner" else y_close
# Moderate class reweight: NN gets a 4× weight (much smaller than the
# "balanced" 27× that produced a fake 42-dataset NN leaf with only 6
# actual NN wins). At 4×, the tree surfaces a single small NN-favorable
# leaf (the regime where tuned NNs actually capture a non-trivial share
# of wins) without overstating their reach.
clf = DecisionTreeClassifier(
max_depth=3,
min_samples_leaf=6,
class_weight={"FM2": 1, "GBDT": 1, "NN": 4},
random_state=0,
).fit(X, y)
tree = clf.tree_
feature = tree.feature
threshold = tree.threshold
left = list(tree.children_left)
right = list(tree.children_right)
value = tree.value
classes = clf.classes_
def is_leaf(n):
return left[n] == right[n]
def leaf_class(n):
return classes[int(np.argmax(value[n][0]))]
# ---- Post-prune: collapse any split whose two children predict the same
# class. A "decide GBDT or GBDT" split adds no heuristic value and clutters
# the diagram. Walk bottom-up.
def subtree_class(n):
"""If the whole subtree at n predicts a single class, return it.
Otherwise return None."""
if is_leaf(n):
return leaf_class(n)
cl = subtree_class(left[n])
cr = subtree_class(right[n])
if cl is not None and cl == cr:
return cl
return None
def prune(n):
if is_leaf(n):
return
prune(left[n])
prune(right[n])
sc = subtree_class(n)
if sc is not None:
# Collapse: mark as leaf by making children equal to itself
left[n] = -1
right[n] = -1
prune(0)
# ---- Collect surviving nodes (BFS)
nodes = []
def bfs(n):
nodes.append(n)
if not is_leaf(n):
bfs(left[n])
bfs(right[n])
bfs(0)
# ---- Layout: walk surviving tree, give each leaf a slot, each internal
# node the midpoint of its children's slots. Avoids the "doubled-up"
# layout when one side is pruned.
leaf_order = []
def collect_leaves(n):
if is_leaf(n):
leaf_order.append(n)
return
collect_leaves(left[n])
collect_leaves(right[n])
collect_leaves(0)
leaf_slot = {n: i for i, n in enumerate(leaf_order)}
depth = {0: 0}
def assign_depth(n, d=0):
depth[n] = d
if not is_leaf(n):
assign_depth(left[n], d + 1)
assign_depth(right[n], d + 1)
assign_depth(0, 0)
def node_slot(n):
if is_leaf(n):
return float(leaf_slot[n])
return (node_slot(left[n]) + node_slot(right[n])) / 2
max_depth = max(depth.values())
n_leaves = len(leaf_order)
coords = {}
for n in nodes:
slot = node_slot(n)
# Center horizontally: spread leaves across [0.06, 0.94]
if n_leaves == 1:
x = 0.5
else:
x = 0.06 + (slot / (n_leaves - 1)) * 0.88
# Top → bottom: root at y=0.9, leaves near y=0.1
if max_depth == 0:
y = 0.5
else:
y = 0.9 - (depth[n] / max_depth) * 0.78
coords[n] = (x, y)
fig = go.Figure()
# ---- Pretty labels
FAMILY_PRETTY = {
"FM2": "2nd-gen FMs",
"GBDT": "GBDT ensembles",
"NN": "Tuned NNs",
}
FEATURE_PRETTY = {
"n_features": "# features",
"n_rows": "# rows",
"n_classes": "# classes",
"pct_cat": "categorical %",
"avg_card": "avg categorical card.",
"imbalance": "imbalance (Acc − F1)",
}
LEAF_COLORS = {
"FM2": "#5a30c2",
"GBDT": "#3970d0",
"NN": "#b03f7b",
1: "#cc6600",
0: "#5dc4c4",
}
def split_label(node):
f = feat_names[feature[node]]
thr = threshold[node]
fp = FEATURE_PRETTY.get(f, f)
# Integer-format counts; round percentages and small numbers nicely
if f in ("n_features", "n_rows", "n_classes"):
thr_str = f"{int(round(thr)):,}"
elif f == "pct_cat":
thr_str = f"{thr*100:.0f}%" if thr <= 1.5 else f"{thr:.0f}%"
elif f == "avg_card":
thr_str = f"{thr:.1f}"
elif f == "imbalance":
thr_str = f"{thr:.2f}"
else:
thr_str = f"{thr:.2f}"
return f"{fp} ≤ {thr_str}"
def leaf_label(node):
cls = leaf_class(node)
# tree.n_node_samples is the count of training rows reaching this node;
# for pruned/leaf nodes that equals the full subtree size, i.e. the
# number of datasets in this leaf.
n_here = int(tree.n_node_samples[node])
if mode == "winner":
name = FAMILY_PRETTY.get(cls, str(cls))
else:
name = (
"Alternatives competitive" if cls == 1 else "Use a Foundation Model"
)
return f"{name} n = {n_here}"
# ---- Per-node size: pixel-based. Box width is computed from
# an estimate of glyph width (font-size 11–12 monospace ≈ 7 px/glyph)
# in PIXELS, not data units, so it doesn't depend on the plot width.
# Boxes are drawn with xsizemode/ysizemode="pixel" anchored at the
# node's paper-coord centre: same trick the perfmap arrows use.
GLYPH_PX = 6 # monospace glyph width at size 11–12 (calibrated)
LEAF_PAD_PX = 8
SPLIT_PAD_PX = 10
LINE_PX = 14 # vertical pixel space per extra text line
NODE_HALF_H_PX = 18 # box half-height in pixels (single line)
MIN_HALF_W_PX = 28
import re
def text_visual_len(text):
s = re.sub(r"", "\n", text)
s = re.sub(r"<[^>]+>", "", s)
return max(len(L) for L in s.split("\n"))
def text_line_count(text):
s = re.sub(r"", "\n", text)
s = re.sub(r"<[^>]+>", "", s)
return len(s.split("\n"))
def node_half_w_px(text, is_leaf_node):
n_chars = text_visual_len(text)
pad = LEAF_PAD_PX if is_leaf_node else SPLIT_PAD_PX
return max(MIN_HALF_W_PX, n_chars * GLYPH_PX / 2 + pad)
def node_half_h_px(text):
lines = text_line_count(text)
# one line → NODE_HALF_H_PX (≈32); each extra line adds ~LINE_PX/2
return max(NODE_HALF_H_PX, NODE_HALF_H_PX + (lines - 1) * LINE_PX / 2)
node_text = {}
node_hw_px = {}
node_hh_px = {}
for n in nodes:
leaf_here = is_leaf(n)
t = leaf_label(n) if leaf_here else split_label(n)
node_text[n] = t
node_hw_px[n] = node_half_w_px(t, leaf_here)
node_hh_px[n] = node_half_h_px(t)
# Convert each node's pixel half-height to a paper-y offset using the
# figure's plot-area height (≈ 400 - top - bottom margin). We just need
# to know where the box's TOP/BOTTOM lands in paper coords so connectors
# stop AT the edge of each individual box rather than entering it.
FIG_PLOT_H = 400 - 44 - 14 # height - t margin - b margin in update_layout
def hh_paper(n):
return node_hh_px[n] / FIG_PLOT_H
# ---- Edges: gentle curved connectors with side labels (yes/no).
for n in nodes:
if is_leaf(n):
continue
x0, y0 = coords[n]
for child, side in [(left[n], "yes"), (right[n], "no")]:
x1, y1 = coords[child]
y0b = y0 - hh_paper(n)
y1t = y1 + hh_paper(child)
cy = (y0b + y1t) / 2
path = f"M {x0} {y0b} " f"C {x0} {cy}, {x1} {cy}, {x1} {y1t}"
fig.add_shape(
type="path",
path=path,
line=dict(color="#c9ced5", width=2),
fillcolor="rgba(0,0,0,0)",
xref="x",
yref="y",
layer="below",
)
edge_color = "#5a30c2" if side == "yes" else "#7a7a90"
edge_text = "yes" if side == "yes" else "no"
fig.add_annotation(
x=(x0 + x1) / 2,
y=cy + 0.005,
text=f"{edge_text}",
showarrow=False,
font=dict(size=10, color=edge_color, family="Geist Mono, monospace"),
bgcolor="#ffffff",
bordercolor="rgba(0,0,0,0)",
borderpad=2,
)
# ---- Nodes: boxes drawn in PIXEL mode so width and height are
# resolution-independent.
for n in nodes:
x, y = coords[n]
hw = node_hw_px[n]
hh = node_hh_px[n]
text = node_text[n]
if is_leaf(n):
cls = leaf_class(n)
color = LEAF_COLORS.get(cls, "#666")
fig.add_shape(
type="rect",
xref="x", yref="y",
xsizemode="pixel", ysizemode="pixel",
xanchor=x, yanchor=y,
x0=-hw, x1=+hw, y0=-hh, y1=+hh,
line=dict(color=color, width=2.5),
fillcolor=color,
opacity=0.32,
layer="below",
)
fig.add_annotation(
x=x,
y=y,
text=f"{text}",
showarrow=False,
font=dict(size=12, color=color, family="Geist Mono, monospace"),
align="center",
)
else:
fig.add_shape(
type="rect",
xref="x", yref="y",
xsizemode="pixel", ysizemode="pixel",
xanchor=x, yanchor=y,
x0=-hw, x1=+hw, y0=-hh, y1=+hh,
line=dict(color="#9aa3ad", width=1.5),
fillcolor="rgba(0,0,0,0)",
layer="below",
)
fig.add_annotation(
x=x,
y=y,
text=text,
showarrow=False,
font=dict(size=11, color="#7a7a90", family="Geist Mono, monospace"),
align="center",
)
fig.update_layout(
height=520,
margin=dict(l=10, r=10, t=44, b=14),
# Range extended past [0,1] so the box strokes never get clipped by the
# plot viewport on first paint.
xaxis=dict(visible=False, range=[-0.08, 1.08]),
yaxis=dict(visible=False, range=[-0.05, 1.05]),
plot_bgcolor="rgba(0,0,0,0)",
paper_bgcolor="rgba(0,0,0,0)",
template="simple_white",
title=dict(
text=(
"Which model family wins?"
if mode == "winner"
else "When are non-FM alternatives competitive?"
),
x=0.5,
xanchor="center",
font=dict(size=13, color="#7a7a90", family="Geist Mono, monospace"),
),
showlegend=False,
)
return fig
def plot_global_model_ranking_plotly(df, metric="Accuracy", per_dataset_df=None):
"""Horizontal bar chart of model means with per-dataset std as CI overlay.
`df` is the aggregated mean-per-model dataframe; `per_dataset_df` is the
raw per-dataset, per-model dataframe used to compute the std for CIs.
"""
import plotly.graph_objects as go
df_sorted = df.sort_values(metric, ascending=True).reset_index(drop=True)
n = len(df_sorted)
# Per-model 95% CI of the mean: ±1.96 * SEM, where SEM = std / sqrt(N)
# across the per-dataset means. Each model's CI reflects its OWN spread
# across datasets, independent of any baseline.
cis = None
if per_dataset_df is not None and metric in per_dataset_df.columns:
grouped = per_dataset_df.groupby("model")[metric]
stds = grouped.std()
counts = grouped.count()
sems = stds / counts.pow(0.5)
cis = 1.96 * sems
# Data-driven axis bounds. Extend by CI half-width so error lines aren't
# clipped, then snap to a clean step so ticks land on round numbers.
# NaN guard: with very few datasets a model may have std=NaN (N<2).
import math as _math
max_ci = 0.0
if cis is not None and len(cis) > 0:
max_ci_val = float(cis.max())
if not _math.isnan(max_ci_val):
max_ci = max_ci_val
raw_min_v = df_sorted[metric].min()
raw_max_v = df_sorted[metric].max()
if _math.isnan(raw_min_v) or _math.isnan(raw_max_v):
raw_min, raw_max = 0.0, 1.0
else:
raw_min = raw_min_v - max_ci
raw_max = raw_max_v + max_ci
spread = max(raw_max - raw_min, 1e-9)
pad = spread * 0.10
BOUNDED = {"Accuracy", "AUC", "Precision", "Recall", "F1_score"}
is_bounded = metric in BOUNDED
# Snap to a step that's small relative to the spread (~5 ticks visible).
def _pick_step(s):
# Pick a "nice" number close to s/5 from {1, 2, 2.5, 5} * 10**k.
import math
if s <= 0:
return 0.01
target = s / 5.0
mag = 10 ** math.floor(math.log10(target))
for mult in (1, 2, 2.5, 5, 10):
if mult * mag >= target:
return mult * mag
return 10 * mag
step = _pick_step(spread)
import math
x_axis_min = math.floor((raw_min - pad) / step) * step
x_axis_max = math.ceil((raw_max + pad) / step) * step
if is_bounded:
x_axis_min = max(0.0, x_axis_min)
x_axis_max = min(1.0, x_axis_max)
# Bars: violet gradient by metric value, value label OUTSIDE-RIGHT.
fig = go.Figure()
# Format the value labels based on magnitude: integers for Elo-like
# values, 3 decimals for [0, 1] metrics, 1 decimal for percentages.
def _fmt(v):
if v != v: # NaN
return ""
if abs(v) >= 100:
return f"{v:.0f}"
if abs(v) >= 10:
return f"{v:.1f}"
if abs(v) >= 1:
return f"{v:.2f}"
return f"{v:.3f}"
# Color each bar by the model's class (Tree / NN / FM-gen1 / FM-gen2).
MODEL_CLASS = {
# Trees
"xgboost": "tree",
"catboost": "tree",
"lightgbm": "tree",
"xgboost_ensemble": "tree",
"catboost_ensemble": "tree",
"lightgbm_ensemble": "tree",
# Neural networks
"tabm": "nn",
"realmlp": "nn",
"modernnca": "nn",
"modern_nca": "nn",
# Foundation models: 1st generation
"tabpfn": "fm1",
"tabpfn_2_5": "fm1",
"tabicl": "fm1",
"tabicl_v1": "fm1",
"tabdpt": "fm1",
"mitra": "fm1",
"limix": "fm1",
"nicl": "fm1",
"sap": "fm1",
# Foundation models: 2nd generation
"tabpfn_v3": "fm2",
"tabicl_v2": "fm2",
"seldon": "fm2",
}
# Display-name maps: raw model id -> pretty label, raw metric -> pretty.
MODEL_DISPLAY = {
"xgboost": "XGBoost",
"catboost": "CatBoost",
"lightgbm": "LightGBM",
"xgboost_ensemble": "XGBoost",
"catboost_ensemble": "CatBoost",
"lightgbm_ensemble": "LightGBM",
"realmlp": "RealMLP",
"tabm": "TabM",
"modern_nca": "ModernNCA",
"modernnca": "ModernNCA",
"tabpfn": "TabPFN v2.5",
"tabpfn_2_5": "TabPFN v2.5",
"tabpfn_v3": "TabPFN v3.0",
"tabicl": "TabICL v1.1",
"tabicl_v1": "TabICL v1.1",
"tabicl_v2": "TabICL v2.0",
"tabdpt": "TabDPT",
"limix": "LimiX",
"mitra": "Mitra",
"seldon": "Seldon",
"nicl": "NICL",
"sap": "SAP RPT-1",
}
METRIC_DISPLAY = {
"Accuracy": "Accuracy",
"AUC": "AUC",
"F1_score": "F1",
"Precision": "Precision",
"Recall": "Recall",
"Cross_entropy": "Cross-entropy",
"Elo_score": "Elo",
"%↗ over XGBoost": "% over XGBoost",
}
# Per-class palette: main body + light top-highlight + dark separator/outline
# + mid-dark CI tone. Soft trio (violet / blue / pink) chosen from preview28.
CLASS_PALETTE = {
"fm1": {
"main": "#5dc4c4",
"light": "#b9e3e3",
"dark": "#2a7c7c",
"ci": "#1e5f5f",
"ink": "#0d3838",
},
"fm2": {
"main": "#a18cff",
"light": "#e6dfff",
"dark": "#4d2cdc",
"ci": "#4a25c9",
"ink": "#1a0a6b",
},
"tree": {
"main": "#7daaef",
"light": "#dde9fb",
"dark": "#2860b0",
"ci": "#1f4c98",
"ink": "#0a2466",
},
"nn": {
"main": "#ec9bca",
"light": "#fce4ef",
"dark": "#b03f7b",
"ci": "#992f64",
"ink": "#5e123f",
},
"other": {
"main": "#b3afc4",
"light": "#e0dee7",
"dark": "#5d6c7b",
"ci": "#5d6c7b",
"ink": "#2a2440",
},
}
# Draw bars as TICK-ALIGNED PIXEL CHUNKS: each chunk fills one gridline
# interval (step). Style: body + thin top highlight + per-class dark
# separator between chunks + dark outline around the whole bar.
BAR_HEIGHT = 0.50
HIGHLIGHT_FRAC = 0.12 # slim top highlight band
OUTLINE_W = 1.5
SEP_W = 1
for i, row in df_sorted.iterrows():
model = row["model"]
mean = row[metric]
cls = MODEL_CLASS.get(model, "other")
pal = CLASS_PALETTE.get(cls, CLASS_PALETTE["other"])
y_top, y_bot = i + BAR_HEIGHT / 2, i - BAR_HEIGHT / 2
hl_bot = y_top - BAR_HEIGHT * HIGHLIGHT_FRAC
# Collect chunk boundaries (tick-aligned + a partial last chunk)
edges = []
x = x_axis_min
while x + step <= mean + 1e-9:
edges.append((x, x + step))
x += step
if mean - x > 1e-9:
edges.append((x, mean))
if not edges:
continue
# 1. Filled chunk bodies + top highlight strip (no stroke)
for x0, x1 in edges:
fig.add_shape(
type="rect",
x0=x0,
x1=x1,
y0=y_bot,
y1=y_top,
line=dict(width=0),
fillcolor=pal["main"],
layer="above",
)
fig.add_shape(
type="rect",
x0=x0,
x1=x1,
y0=hl_bot,
y1=y_top,
line=dict(width=0),
fillcolor=pal["light"],
layer="above",
)
# 2. Inner separator lines between adjacent chunks (per-class dark tone)
for j in range(1, len(edges)):
sep_x = edges[j][0]
fig.add_shape(
type="line",
x0=sep_x,
x1=sep_x,
y0=y_bot,
y1=y_top,
line=dict(color=pal["dark"], width=SEP_W),
layer="above",
)
# 3. Outer outline around the whole bar (per-class dark tone)
fig.add_shape(
type="rect",
x0=edges[0][0],
x1=edges[-1][1],
y0=y_bot,
y1=y_top,
line=dict(color=pal["dark"], width=OUTLINE_W),
fillcolor="rgba(0,0,0,0)",
layer="above",
)
# Value labels INSIDE the first chunk, vertically centered on the
# non-highlight portion of the bar (below the top light strip). A bit
# darker than the CI bar for legibility against the main body.
y_offset = -0.06 * BAR_HEIGHT
for i, row in df_sorted.iterrows():
model = row["model"]
mean = row[metric]
cls = MODEL_CLASS.get(model, "other")
pal = CLASS_PALETTE.get(cls, CLASS_PALETTE["other"])
pad_x = (x_axis_max - x_axis_min) * 0.005
fig.add_annotation(
x=x_axis_min + pad_x,
y=i + y_offset,
text=_fmt(mean),
showarrow=False,
xanchor="left",
yanchor="middle",
font=dict(
family="Geist Mono, ui-monospace, monospace", size=13, color=pal["ink"]
),
)
# Hover-only invisible trace at the actual mean for tooltip values.
fig.add_trace(
go.Scatter(
x=df_sorted[metric],
y=df_sorted["model"],
customdata=[MODEL_DISPLAY.get(m, m) for m in df_sorted["model"].tolist()],
mode="markers",
marker=dict(size=12, opacity=0),
cliponaxis=False,
hovertemplate="%{customdata} %{x:.4f}",
showlegend=False,
)
)
# CI bar: opaque, per-class mid-dark tone, slim band (~18% of bar height).
CI_H_FRAC = 0.18
if cis is not None:
for i, row in df_sorted.iterrows():
model = row["model"]
mean = row[metric]
ci = cis.get(model, None)
if ci is None or not (ci == ci): # NaN check
continue
cls = MODEL_CLASS.get(model, "other")
pal = CLASS_PALETTE.get(cls, CLASS_PALETTE["other"])
h = BAR_HEIGHT * CI_H_FRAC
cy = i + y_offset
fig.add_shape(
type="rect",
x0=mean - ci,
x1=mean + ci,
y0=cy - h / 2,
y1=cy + h / 2,
line=dict(width=0),
fillcolor=pal["ci"],
layer="above",
)
fig.update_layout(
template="plotly_white",
plot_bgcolor="rgba(0,0,0,0)",
paper_bgcolor="rgba(0,0,0,0)",
font=dict(family="BDO Grotesk, Helvetica Neue, sans-serif", color="#060510"),
xaxis=dict(
title=None,
automargin=True,
range=[x_axis_min, x_axis_max],
tick0=x_axis_min,
dtick=step,
tickformat=".0f" if step >= 1 else (".1f" if step >= 0.1 else ".2f"),
showgrid=True,
gridcolor="rgba(125,125,140,0.15)",
zeroline=False,
tickfont=dict(family="Geist Mono, monospace", size=11, color="#5d6c7b"),
),
yaxis=dict(
title=None,
automargin=True,
showgrid=False,
zeroline=False,
ticks="outside",
ticklen=12, # pixel gap between label and bar
tickcolor="rgba(0,0,0,0)", # invisible tick marks (just spacing)
tickfont=dict(
family="Geist Mono, ui-monospace, monospace", size=16, color="#060510"
),
),
bargap=0.55,
height=max(320, 48 * n + 80),
margin=dict(l=200, r=80, t=20, b=40),
)
# Force ALL CAPS model labels by replacing the y-axis category text.
fig.update_yaxes(
tickmode="array",
tickvals=df_sorted["model"].tolist(),
ticktext=[MODEL_DISPLAY.get(m, m).upper() for m in df_sorted["model"].tolist()],
)
# ----- Model-family legend -----
# Four ghost traces show the family / color mapping in the legend
# without affecting the bar layout.
family_legend = [
("Foundation models: 2nd gen", CLASS_PALETTE["fm2"]["main"]),
("Foundation models: 1st gen", CLASS_PALETTE["fm1"]["main"]),
("GBDT", CLASS_PALETTE["tree"]["main"]),
("Tabular NN", CLASS_PALETTE["nn"]["main"]),
]
for name, color in family_legend:
fig.add_trace(
go.Scatter(
x=[None],
y=[None],
mode="markers",
marker=dict(size=12, color=color, symbol="square"),
name=name,
hoverinfo="skip",
showlegend=True,
)
)
fig.update_layout(
showlegend=True,
legend=dict(
orientation="h",
yanchor="bottom",
y=1.02,
xanchor="left",
x=0,
# Mid-tone label fill: legible on light bg, the dark-mode CSS
# override below recolours it to light grey on dark.
font=dict(size=11, color="#1a1525"),
bgcolor="rgba(255,255,255,0)",
),
margin=dict(l=200, r=80, t=60, b=40),
)
return fig
def plot_significance_forest(per_dataset_df, metric="Accuracy"):
"""Forest-plot view of model means with 95% CI lollipops and the
Wilcoxon signed-rank + Holm-corrected "leading group" highlighted.
Top of the chart: three KPI rows (leading-group size, leaders'
metric range, top-spot verdict). The leading-group rows are drawn
over a soft blue band; everyone else stays neutral grey.
"""
import plotly.graph_objects as go
import numpy as np
import pandas as pd
from scipy.stats import wilcoxon
from statsmodels.stats.multitest import multipletests
LOWER_IS_BETTER = metric in {"Cross_entropy"}
if per_dataset_df is None or per_dataset_df.empty or metric not in per_dataset_df.columns:
return go.Figure().update_layout(title="No data")
piv = per_dataset_df.pivot_table(index="dataset_id", columns="model",
values=metric, aggfunc="mean")
piv = piv.dropna(axis=0, how="any")
if piv.shape[0] < 5 or piv.shape[1] < 2:
return go.Figure().update_layout(title="Not enough data for significance testing")
models = list(piv.columns)
pretty_map = _MODEL_DISPLAY_PRETTY if "_MODEL_DISPLAY_PRETTY" in globals() else {}
# Per-model mean + 95% paired bootstrap CI, projected onto the metric's
# native scale via the average within-dataset SD.
#
# Why: independent per-model bootstrap CIs on the mean don't reflect
# paired-test significance because dataset-difficulty variance inflates
# every model's CI in lockstep. We resample on the per-dataset z-scored
# metric (within-dataset standardised), which strips that shared
# variance. The CI is then back-projected to the original metric units
# by multiplying the z-CI by σ̄: the mean across datasets of the
# within-dataset standard deviation across models. The resulting bars
# are roughly comparable to the paired Wilcoxon-Holm test driving the
# leading-group highlight.
grouped = per_dataset_df.groupby("model")[metric]
means = grouped.mean()
# σ̄: average within-dataset SD across the models in this view
per_ds_sd = per_dataset_df.groupby("dataset_id")[metric].std()
sigma_bar = float(per_ds_sd.replace(0, np.nan).dropna().mean())
if not np.isfinite(sigma_bar) or sigma_bar == 0:
sigma_bar = float("nan")
# z-score per dataset for this metric (recomputed on the filtered subset)
sub = per_dataset_df.copy()
g = sub.groupby("dataset_id")[metric]
mu_per_ds = g.transform("mean")
sd_per_ds = g.transform("std").replace(0, np.nan)
sub["_z"] = ((sub[metric] - mu_per_ds) / sd_per_ds).fillna(0.0)
rng = np.random.default_rng(seed=0)
B = 1000
ci_low = {}
ci_high = {}
for m, vals in sub.groupby("model")["_z"]:
z_arr = vals.dropna().to_numpy()
mean_m = float(means.get(m, float("nan")))
n = len(z_arr)
if n < 2 or not np.isfinite(sigma_bar):
ci_low[m] = mean_m
ci_high[m] = mean_m
continue
idx = rng.integers(0, n, size=(B, n))
boot_z_means = z_arr[idx].mean(axis=1)
# Half-widths around 0 in z-units, projected to metric units
z_q_lo = float(np.quantile(boot_z_means, 0.025))
z_q_hi = float(np.quantile(boot_z_means, 0.975))
# Symmetric-ish projection: anchor at the model's actual mean, add
# (z - mean_z) * σ̄ for the lower / upper offset. mean_z is just the
# sample mean of z_arr; using the bootstrap median of boot_z_means
# is equivalent within B-noise.
mean_z = float(z_arr.mean())
lo_offset = (z_q_lo - mean_z) * sigma_bar
hi_offset = (z_q_hi - mean_z) * sigma_bar
ci_low[m] = mean_m + lo_offset
ci_high[m] = mean_m + hi_offset
# Sort by metric (higher better, unless Cross-entropy)
sorted_models = sorted(models,
key=lambda m: means.get(m, float("nan")),
reverse=not LOWER_IS_BETTER)
# Some metrics aren't meaningful per-dataset signals for a paired
# Wilcoxon test: Elo is a single global number per model (constant
# across all rows), %↗ is itself a baseline-relative quantity. For
# these we don't compute a leading group: just mark the top model
# as the (single) leader.
SIG_SKIP_METRICS = {"Elo_score", "%↗ over XGBoost"}
pairs = []
if metric not in SIG_SKIP_METRICS:
for i, a in enumerate(sorted_models):
for b in sorted_models[i + 1:]:
d = (piv[a] - piv[b]).values
nz = d[d != 0]
if len(nz) < 5:
p = 1.0
else:
try:
p = wilcoxon(d, zero_method="wilcox", alternative="two-sided").pvalue
except Exception:
p = 1.0
pairs.append((a, b, p))
if pairs:
_, p_adj, _, _ = multipletests([p for _, _, p in pairs], alpha=0.05, method="holm")
else:
p_adj = []
not_sig = set() # symmetric set of {(a,b)} where a < b in sorted_models order
for (a, b, _), pa in zip(pairs, p_adj):
if pa > 0.05:
not_sig.add((a, b))
# Leading group = the top model + every model not significantly worse than it
leader = sorted_models[0]
leading_group = [leader]
for m in sorted_models[1:]:
# (leader, m) pair was inserted in that order
if (leader, m) in not_sig:
leading_group.append(m)
else:
break # once we hit a significantly worse model, stop (sorted by metric)
leading_set = set(leading_group)
n_leaders = len(leading_group)
# ----- KPI card content -----
metric_pretty = {"Accuracy": "Accuracy",
"AUC": "AUC", "F1_score": "F1",
"Precision": "Precision", "Recall": "Recall",
"Cross_entropy": "Cross-entropy"}.get(metric, metric)
leader_names_pretty = [pretty_map.get(m, m) for m in leading_group]
# Card 1: Leader names
if n_leaders == 1:
kpi_group_value = leader_names_pretty[0]
kpi_group_sub = "clear leader"
elif n_leaders <= 3:
kpi_group_value = " · ".join(leader_names_pretty)
kpi_group_sub = f"{n_leaders} models tied"
else:
kpi_group_value = " · ".join(leader_names_pretty[:3]) + " …"
kpi_group_sub = f"{n_leaders} models tied"
# Card 2: Lead margin: mean(leader scores) − mean(rest of the field)
rest_models = [m for m in sorted_models if m not in leading_set]
if rest_models:
leader_mean = float(np.mean([means[m] for m in leading_group]))
rest_mean = float(np.mean([means[m] for m in rest_models]))
# margin is "how much better the leaders are on the metric scale"
margin = (leader_mean - rest_mean) * (-1 if LOWER_IS_BETTER else 1)
if metric == "Z_Accuracy":
kpi_margin_value = f"{margin:+.2f} σ"
elif metric == "%↗ over XGBoost":
kpi_margin_value = f"{margin:+.2f} pts"
elif metric == "Elo_score":
kpi_margin_value = f"{margin:+.0f} pts"
else:
kpi_margin_value = f"{margin*100:+.1f} pts"
kpi_margin_sub = "leaders above the field"
else:
kpi_margin_value = "—"
kpi_margin_sub = "no field to compare to"
# Card 3: Top-3 coverage: how often is at least ONE leader in the
# per-dataset top-3 on this metric?
asc = LOWER_IS_BETTER
per_ds_rank = piv.rank(axis=1, ascending=asc, method="min")
leaders_in_top3 = (per_ds_rank[list(leading_group)] <= 3).any(axis=1)
n_top3 = int(leaders_in_top3.sum())
n_total = int(len(piv))
kpi_top3_value = f"{n_top3}/{n_total}"
pct = (100.0 * n_top3 / n_total) if n_total else 0.0
kpi_top3_sub = f"datasets · {pct:.0f}%"
# Plot construction
fig = go.Figure()
# Reverse so the best model appears at the top of the y-axis
y_order = list(reversed(sorted_models))
y_labels = [pretty_map.get(m, m) for m in y_order]
# Background band over the leading-group rows (drawn as a shape).
# Plotly y-categories are integer-indexed in the order we feed them.
leader_positions = [y_order.index(m) for m in leading_group if m in y_order]
if leader_positions:
ymin = min(leader_positions) - 0.45
ymax = max(leader_positions) + 0.45
fig.add_shape(
type="rect", xref="paper", yref="y",
x0=0, x1=1, y0=ymin, y1=ymax,
fillcolor="rgba(78, 41, 255, 0.07)",
line=dict(color="rgba(78, 41, 255, 0.0)"),
layer="below",
)
# CI lines + lollipop dots. The CI shaft is drawn as a thin filled
# rectangle (not a line stroke) so its edges read slightly "blocky" —
# subtle pixel flavour on an otherwise normal forest plot.
LEAD_COLOR = "#4e29ff"
OTHER_COLOR = "#9aa3ad"
for y_idx, m in enumerate(y_order):
mean_v = float(means.get(m, float("nan")))
if mean_v != mean_v:
continue
lo = float(ci_low.get(m, mean_v))
hi = float(ci_high.get(m, mean_v))
is_leader = m in leading_set
c = LEAD_COLOR if is_leader else OTHER_COLOR
# Thin rectangle for the CI shaft (no stroke: keeps it understated)
bar_h = 0.05
fig.add_shape(
type="rect", xref="x", yref="y",
x0=lo, x1=hi,
y0=y_idx - bar_h, y1=y_idx + bar_h,
line=dict(color=c, width=0),
fillcolor=c,
opacity=0.85,
)
# End caps
cap_h = 0.18
for cx in (lo, hi):
fig.add_shape(
type="line", xref="x", yref="y",
x0=cx, x1=cx,
y0=y_idx - cap_h, y1=y_idx + cap_h,
line=dict(color=c, width=1.4),
)
# Round dots (back to original). Seldon (our model) gets a thin violet
# outline as an extra cue regardless of whether it's in the leading
# group.
dot_x = [float(means.get(m, float("nan"))) for m in y_order]
dot_colors = [LEAD_COLOR if m in leading_set else OTHER_COLOR for m in y_order]
dot_outline = ["#5a30c2" if m == "seldon" else "rgba(0,0,0,0)" for m in y_order]
dot_outline_w = [2 if m == "seldon" else 0 for m in y_order]
fig.add_trace(go.Scatter(
x=dot_x, y=list(range(len(y_order))),
mode="markers",
marker=dict(
size=[14 if m == "seldon" else 11 for m in y_order],
color=dot_colors,
line=dict(color=dot_outline, width=dot_outline_w),
),
hovertemplate=(
"%{customdata[0]} "
f"{metric_pretty}: %{{x:.4f}}"
),
customdata=[[lab] for lab in y_labels],
showlegend=False,
))
# Right-margin numeric labels: one per model with its metric value
for y_idx, (m, val) in enumerate(zip(y_order, dot_x)):
if val != val:
continue
c = LEAD_COLOR if m in leading_set else "#5d6c7b"
weight = "" if m in leading_set else ""
weight_end = "" if m in leading_set else ""
fig.add_annotation(
xref="paper", x=1.02, yref="y", y=y_idx,
text=f"{weight}{val:.3f}{weight_end}",
showarrow=False, xanchor="left", yanchor="middle",
font=dict(size=11, color=c, family="Geist Mono, monospace"),
)
# KPI cards as top annotations on paper coords.
# Each card has 3 stacked annotations (kicker / big value / subline).
# Plotly inline `` is unreliable, so each
# annotation sets its size via the `font=` parameter instead.
#
# Responsive sizing: the card pill is sized in paper fractions (so it
# shrinks with the plot width), but the *text* is sized in pixels. To
# avoid overflow on narrow plots, we shrink the value/sub text more
# aggressively for long content AND truncate the LEADING GROUP value
# to at most 2 model names (was 3) so common cases fit even at small
# widths.
# KPI cards: rendered as a sibling HTML block (not inside the figure),
# so they get native CSS flexbox responsiveness and resize fluidly with
# the window. The HTML is returned alongside the figure.
if n_leaders == 1:
kpi_group_value_display = leader_names_pretty[0]
elif n_leaders == 2:
kpi_group_value_display = " · ".join(leader_names_pretty)
else:
kpi_group_value_display = " · ".join(leader_names_pretty[:2]) + " …"
import html as _html
def _esc(s):
return _html.escape(str(s))
kpi_html = (
'
'
'
'
f'
LEADING GROUP
'
f'
{_esc(kpi_group_value_display)}
'
f'
{_esc(kpi_group_sub)}
'
'
'
'
'
f'
LEAD MARGIN
'
f'
{_esc(kpi_margin_value)}
'
f'
{_esc(kpi_margin_sub)}
'
'
'
'
'
f'
TOP-3 COVERAGE
'
f'
{_esc(kpi_top3_value)}
'
f'
{_esc(kpi_top3_sub)}
'
'
'
'
'
)
# Axes
raw_min = min(ci_low.get(m, v) for m, v in zip(y_order, dot_x) if v == v)
raw_max = max(ci_high.get(m, v) for m, v in zip(y_order, dot_x) if v == v)
pad = (raw_max - raw_min) * 0.10
fig.update_xaxes(
range=[raw_min - pad, raw_max + pad],
showgrid=True, gridcolor="rgba(125,125,140,0.15)",
zeroline=False,
tickfont=dict(family="Geist Mono, monospace", size=11, color="#5d6c7b"),
title=None,
)
# Hide built-in y-axis ticks; we draw model labels as annotations so
# we can control their distance from the plot via `xshift` (Plotly's
# tickpadding is unreliable across versions).
fig.update_yaxes(
tickvals=[], ticktext=[],
showgrid=False, zeroline=False, ticks="",
range=[-0.5, len(y_order) - 0.5],
)
SELDON_HIGHLIGHT = "#5a30c2" # neuralk violet for the model label
for y_idx, (m, lab) in enumerate(zip(y_order, y_labels)):
is_seldon = m == "seldon"
# Seldon: pill-shaded violet background behind the name + a
# "by Neuralk" tooltip on hover. Other models: plain ink label.
if is_seldon:
fig.add_annotation(
xref="paper", x=0, yref="y", y=y_idx,
text=f"{lab.upper()}",
showarrow=False,
xanchor="right", yanchor="middle",
xshift=-18,
font=dict(family="Geist Mono, ui-monospace, monospace",
size=12, color=SELDON_HIGHLIGHT),
bgcolor="rgba(78, 41, 255, 0.12)",
bordercolor="rgba(78, 41, 255, 0.40)",
borderwidth=1,
borderpad=4,
hovertext="by Neuralk",
)
else:
fig.add_annotation(
xref="paper", x=0, yref="y", y=y_idx,
text=lab.upper(),
showarrow=False,
xanchor="right", yanchor="middle",
xshift=-18,
font=dict(family="Geist Mono, ui-monospace, monospace",
size=12, color="#060510"),
)
fig.update_layout(
# Autosize: the figure fills its container at full width and shrinks
# down to the min-width set by .nk-bar-plot's CSS rule (820px); below
# that the wrapper scrolls horizontally rather than compressing.
autosize=True,
height=max(420, 36 * len(y_order) + 240),
# Left margin holds the model labels (drawn at paper-x=0 with
# xshift=-18). 140px is enough for the longest model name (~12 chars
# at 12px monospace ≈ 86px) plus the 18px gap. Top margin was 180px
# to host the KPI cards; now 40px since the cards are gone.
margin=dict(l=140, r=90, t=40, b=70),
template="plotly_white",
plot_bgcolor="rgba(0,0,0,0)",
paper_bgcolor="rgba(0,0,0,0)",
font=dict(family="BDO Grotesk, Helvetica Neue, sans-serif", color="#060510"),
showlegend=False,
)
# Legend strip at the bottom
fig.add_annotation(
xref="paper", yref="paper", x=0.02, y=-0.06, xanchor="left", yanchor="top",
text=(f"● "
f"Leading group "
f" ● "
f"Rest of field "
f" · 95% CI"),
showarrow=False, font=dict(size=12),
)
return fig, kpi_html
def plot_winrate_matrix(per_dataset_df, metric="Accuracy"):
"""Pairwise win-rate heatmap. Cell (row i, col j) = fraction of datasets
where model i's `metric` strictly exceeds model j's. Diagonal is masked.
Models ordered by their overall win-rate (best on top → worst at bottom).
"""
import plotly.graph_objects as go
import numpy as np
import pandas as pd
if (
per_dataset_df is None
or per_dataset_df.empty
or metric not in per_dataset_df.columns
):
return go.Figure().update_layout(title="No data")
# Pivot to dataset × model with the metric of interest.
pivot = per_dataset_df.pivot_table(
index="dataset_id", columns="model", values=metric, aggfunc="mean"
)
models = list(pivot.columns)
if len(models) < 2:
return go.Figure().update_layout(title="Need at least two models")
lower_is_better = metric in {"Cross_entropy"}
direction = -1.0 if lower_is_better else 1.0
n = len(models)
win = np.full((n, n), np.nan)
arr = pivot[models].values # (n_datasets, n_models)
for i in range(n):
for j in range(n):
if i == j:
continue
both = ~np.isnan(arr[:, i]) & ~np.isnan(arr[:, j])
if not both.any():
continue
diff = (arr[both, i] - arr[both, j]) * direction
# Ignore ties for win-rate purposes
non_tie = diff != 0
if non_tie.any():
win[i, j] = (diff[non_tie] > 0).mean()
else:
win[i, j] = 0.5
overall = np.nanmean(win, axis=1)
order = np.argsort(-overall)
models_ord = [models[k] for k in order]
win_ord = win[np.ix_(order, order)]
# Pretty model names for axis labels
pretty_map = _MODEL_DISPLAY_PRETTY if "_MODEL_DISPLAY_PRETTY" in globals() else {}
labels = [pretty_map.get(m, m) for m in models_ord]
# Single source of truth for the displayed integer percentage. Both the
# in-cell label AND the hover read from this same array, so the two can
# never disagree (which they did when one used `int(round(z*100))` and
# the other used Plotly's `:.0%` format: different rounding modes at
# the .5 boundary).
pct_text = np.empty_like(win_ord, dtype=object)
for i in range(n):
for j in range(n):
if i == j or np.isnan(win_ord[i, j]):
pct_text[i, j] = ""
else:
pct_text[i, j] = f"{int(round(win_ord[i, j] * 100))}%"
fig = go.Figure(
go.Heatmap(
z=win_ord,
x=labels,
y=labels,
zmin=0.0,
zmax=1.0,
zmid=0.5,
colorscale="RdBu",
reversescale=True,
text=pct_text,
texttemplate="%{text}",
textfont=dict(size=11, family="Geist Mono, monospace"),
# Hover uses the SAME pct string as the cell: no risk of mismatch.
hovertemplate="%{y} beats %{x} "
"Win-rate: %{text}",
colorbar=dict(
title="Row beats Col", tickformat=".0%", thickness=10, len=0.7
),
)
)
# Square cells: the heatmap is n × n. We lock the y-axis to scaleanchor
# the x-axis 1:1 so cells render as proper squares regardless of the
# container width. Height = (cell_px * n) + chrome.
cell_px = 42
chrome_px = 200
fig.update_layout(
height=max(520, cell_px * n + chrome_px),
margin=dict(l=230, r=80, t=190, b=40),
xaxis=dict(side="top", tickangle=-45, tickfont=dict(size=12), constrain="domain"),
yaxis=dict(autorange="reversed", tickfont=dict(size=12),
scaleanchor="x", scaleratio=1, constrain="domain"),
plot_bgcolor="rgba(0,0,0,0)",
paper_bgcolor="rgba(0,0,0,0)",
template="simple_white",
# No title: the section heading already labels this view, and the
# colorbar carries the "row beats col" semantic. A plotly title here
# collides with the top-side x-tick labels.
)
return fig
def update_ranking_plot(metric):
return plot_global_model_ranking_plotly(
public_enter_model_agg,
metric=metric,
per_dataset_df=public_enter_per_dataset,
)
nk_theme = gr.themes.Default(
primary_hue=gr.themes.Color(
c50="#f3f1fb",
c100="#e6e0ff",
c200="#cdbfff",
c300="#a28fff",
c400="#7e5fff",
c500="#4e29ff",
c600="#3a1ee0",
c700="#2c17b0",
c800="#1e1080",
c900="#100850",
c950="#0a0428",
),
neutral_hue=gr.themes.Color(
c50="#fafafc",
c100="#f3f1fb",
c200="#e6e4ef",
c300="#c8c6d6",
c400="#9a96b0",
c500="#5d6c7b",
c600="#3a3850",
c700="#2a2440",
c800="#1c1730",
c900="#14101f",
c950="#0a0814",
),
font=("BDO Grotesk", "Helvetica Neue", "Helvetica", "Arial", "sans-serif"),
font_mono=("Geist Mono", "ui-monospace", "SFMono-Regular", "Menlo", "monospace"),
).set(
body_background_fill="#fafafc",
body_background_fill_dark="#0a0814",
body_text_color="#060510",
body_text_color_dark="#f3f1fb",
background_fill_primary="#ffffff",
background_fill_primary_dark="#14101f",
background_fill_secondary="#fafafc",
background_fill_secondary_dark="#0a0814",
block_background_fill="#ffffff",
block_background_fill_dark="#14101f",
block_border_color="#e6e4ef",
block_border_color_dark="#2a2440",
block_label_text_color="#5d6c7b",
block_label_text_color_dark="#9a96b0",
border_color_primary="#e6e4ef",
border_color_primary_dark="#2a2440",
button_primary_background_fill="#4e29ff",
button_primary_background_fill_dark="#8c75ff",
button_primary_text_color="#ffffff",
button_primary_text_color_dark="#0a0814",
)
NAV_HEAD_JS = """
"""
with gr.Blocks(css=css, theme=nk_theme, head=NAV_HEAD_JS) as demo:
# ---- Dark mode TEMPORARILY DISABLED ----------------------------------
# The dark/light toggle and the "best viewed in light mode" nudge are
# commented out below. CSS `body.dark` rules in the stylesheet remain
# in place but never fire because Gradio's dark theme isn't reachable
# from the UI any more. To re-enable, uncomment the three blocks below.
#
# # Floating light/dark toggle. Uses Gradio's js= escape hatch
# # (not sanitized) to flip ?__theme=dark|light and reload: the only
# # path that flips ALL widgets (Gradio chrome + custom neuralk styles)
# # consistently in 5.50.
# theme_toggle = gr.Button(
# "☀ / ☾",
# elem_id="nk-theme-toggle",
# )
# # Handwritten "best viewed in light mode" nudge: visible only in dark
# # mode (CSS handles the toggle). The arrow points right at the toggle.
# gr.HTML(
# """
#
# best viewed in light mode!
#
#
# """,
# elem_id="nk-light-mode-nudge-wrap",
# visible=True,
# )
# theme_toggle.click(
# fn=None,
# inputs=None,
# outputs=None,
# js="""() => {
# const u = new URL(window.location.href);
# const cur = u.searchParams.get('__theme');
# const isDark = cur === 'dark' || (cur !== 'light' && document.body.classList.contains('dark'));
# u.searchParams.set('__theme', isDark ? 'light' : 'dark');
# window.location.href = u.toString();
# }""",
# )
# Force-light is handled in NAV_HEAD_JS at page boot (the visible=False
# gr.HTML below was a no-op: Gradio doesn't execute scripts inside
# hidden HTML components).
with gr.Column() as main_content:
# Home page content (all intro up to 'More features')
with gr.Column(visible=True) as home_intro:
gr.Markdown(TITLE)
# gr.Markdown("---")
gr.Markdown(INTRODUCTION_TEXT)
gr.Markdown(" ")
with gr.Column():
gr.Markdown('## 01Introduction')
gr.Markdown("""
Tabular foundation models (TabPFN, TabICL, TabDPT, Seldon, Mitra, Limix, and others) are the most active frontier in tabular ML. They promise general-purpose performance without per-task training, but it's hard to tell which ones actually deliver, and where.
**TabBench is a benchmark dedicated to tracking that frontier on tabular classification tasks.** Regression is intentionally out of scope: most public tabular foundation models still ship classification-only, and reliable cross-benchmark regression suites are sparse. We evaluate every released tabular foundation model on **189 classification datasets curated from OpenML**, spanning healthcare, finance, retail, and other verticals, all filtered to be IID across rows. Foundation models are compared head-to-head against the strongest classical baselines (XGBoost, CatBoost, LightGBM) and tuned neural networks (RealMLP, TabM, ModernNCA). The dashboard also provides several ways to compare them and explore the results: a filterable leaderboard, pairwise win-rate matrices, a performance map that clusters datasets by model behaviour, and pick-a-model heuristics derived from dataset metadata.
**Model families.** Beyond classical categories, we introduce the notion of 1st- and 2nd-generation foundation models for tabular ML, mainly distinguished by capacity, as it induces different empirical behaviors and failure modes. 1st-generation models (TabPFN v2.5, TabICL v1, TabDPT, Mitra, LimiX) have limited context windows and often saturate on larger datasets, whereas 2nd-generation models (Seldon, TabPFN v3, TabICL v2) scale to much larger inputs and show more stable, uniform behavior across regimes. Other families follow standard categories: gradient-boosted trees, tuned neural networks (including ModernNCA), and classical baselines.
Pick a vertical to see which model leads, browse the per-model scores in the table view, or read the methodology below for evaluation details.
""")
with gr.Column(elem_classes=["nk-seldon-frame"]):
gr.Markdown("### Seldon: our tabular foundation model")
gr.Markdown("""
**Seldon** is the tabular foundation model developed by [Neuralk-AI](https://www.neuralk.ai/). It appears in the leaderboard alongside the other models on equal footing. Available through the Neuralk API: [docs.neuralk.ai](https://docs.neuralk.ai/).
""")
gr.Markdown(" ")
# Pre-merge dataset metadata once so we can filter by industry / size.
_public_meta_df = pd.DataFrame(
json.load(open("public_datasets_info.json"))
)
# Normalize: keys we'll filter on
_public_meta_df["dataset_id"] = _public_meta_df["dataset_id"].astype(
str
)
public_per_dataset["dataset_id"] = public_per_dataset[
"dataset_id"
].astype(str)
public_per_dataset_with_meta = public_per_dataset.merge(
_public_meta_df[
[
"dataset_id",
"dataset_industry",
"rows",
"features",
"n_classes",
]
],
on="dataset_id",
how="left",
)
# 8 supercategory buckets present in the metadata.
# Values are emoji + label; filtering matches on the raw bucket name.
_INDUSTRY_BUCKETS = [
"🌐 All",
"🏥 Healthcare",
"🛒 Behavioral",
"👁 Computer Vision",
"🏭 Industry & Science",
"💰 Finance/insurance",
"🎮 Games & Synthetic",
"🏛 Social/Public",
"📦 Other",
]
# Map emoji-labeled value back to the raw industry name for filtering.
_BUCKET_FROM_LABEL = {b: b.split(" ", 1)[1] for b in _INDUSTRY_BUCKETS}
_feat_min = int(public_per_dataset_with_meta["features"].min())
_feat_max = int(public_per_dataset_with_meta["features"].max())
_rows_min = int(public_per_dataset_with_meta["rows"].min())
_rows_max = int(public_per_dataset_with_meta["rows"].max())
_classes_min = int(public_per_dataset_with_meta["n_classes"].min())
_classes_max = int(public_per_dataset_with_meta["n_classes"].max())
_initial_count = int(
public_per_dataset_with_meta["dataset_id"].nunique()
)
gr.Markdown('## 02Leaderboard')
gr.Markdown(" ")
# Filter band: ONE frame, three rows ---------------------
with gr.Group(elem_classes=["nk-filter-frame"]):
# Row 1: metric + view
with gr.Row():
metric_selector = gr.Dropdown(
choices=[
("Accuracy", "Accuracy"),
("AUC", "AUC"),
("F1", "F1_score"),
("Precision", "Precision"),
("Recall", "Recall"),
("Cross-entropy", "Cross_entropy"),
("Elo", "Elo_score"),
("% over XGBoost", "%↗ over XGBoost"),
],
value="Accuracy",
label="📊 Metric",
elem_classes=["compact-dropdown"],
)
view_selector = gr.Radio(
choices=["Bars", "Win-rate", "Table"],
value="Bars",
label="View",
elem_classes=["compact-radio"],
)
dataset_count_md = gr.Markdown(
f"📦 **{_initial_count}** datasets",
elem_classes=["nk-dataset-count"],
)
# Row 2: external-benchmark filter: TEMPORARILY HIDDEN.
# The component is kept (but invisible) so all downstream
# callbacks that read `benchmark_selector` keep working;
# value is pinned to "All" so the leaderboard always shows
# the full TabBench base. Restore by flipping visible=True.
# with gr.Row():
benchmark_selector = gr.Radio(
choices=[
("🌐 All TabBench", "All"),
("TabArena (38)", "TabArena"),
],
value="All",
label="📊 Other benchmarks",
elem_classes=["nk-industry-radio"],
visible=False,
)
# Row 3: verticals (dedicated line)
with gr.Row():
industry_selector = gr.Radio(
choices=_INDUSTRY_BUCKETS,
value="🌐 All",
label="🏷️ Verticals",
elem_classes=["nk-industry-radio"],
)
# Row 3: advanced filters in a collapsible accordion.
# gr.Accordion provides the triangle disclosure widget
# natively; collapsing/expanding it changes only its own
# visibility, not the values of the sliders inside, so the
# bar chart is NOT redrawn when toggled.
with gr.Accordion(
"Advanced filters",
open=False,
elem_classes=["nk-advanced-accordion"],
):
# Features + classes (short numeric ranges) on one row.
with gr.Row():
features_slider = RangeSlider(
minimum=_feat_min,
maximum=_feat_max,
value=(_feat_min, _feat_max),
step=1,
label=f"Features range ({_feat_min}–{_feat_max})",
)
classes_slider = RangeSlider(
minimum=_classes_min,
maximum=_classes_max,
value=(_classes_min, _classes_max),
step=1,
label=f"Classes range ({_classes_min}–{_classes_max})",
)
# Samples on its own row (wide numeric range, often
# wraps to a second line when crammed with the others).
with gr.Row():
samples_slider = RangeSlider(
minimum=_rows_min,
maximum=_rows_max,
value=(_rows_min, _rows_max),
step=100,
label=f"Samples range ({_rows_min}–{_rows_max})",
)
# Initial aggregate -------------------------------------------
public_enter_model_agg = (
public_enter_per_dataset.groupby("model")[
[
"Accuracy",
"AUC",
"F1_score",
"Precision",
"Recall",
"Cross_entropy",
"Elo_score",
"%↗ over XGBoost",
]
]
.mean()
.reset_index()
)
_init_forest_fig, _init_forest_kpi = plot_significance_forest(
public_enter_per_dataset, metric="Accuracy",
)
ranking_kpi_html = gr.HTML(
value=_init_forest_kpi,
elem_classes=["nk-bar-kpi-wrap"],
visible=True,
)
ranking_plot = gr.Plot(
value=_init_forest_fig,
show_label=False,
elem_classes=["nk-bar-plot"],
visible=True,
)
# --- Pretty-display name maps + table HTML builder ---
_MODEL_DISPLAY_FULL = {
"xgboost": "XGBoost",
"catboost": "CatBoost",
"lightgbm": "LightGBM",
"xgboost_ensemble": "XGBoost",
"catboost_ensemble": "CatBoost",
"lightgbm_ensemble": "LightGBM",
"realmlp": "RealMLP",
"tabm": "TabM",
"modern_nca": "ModernNCA",
"modernnca": "ModernNCA",
"tabpfn": "TabPFN v2.5",
"tabpfn_2_5": "TabPFN v2.5",
"tabpfn_v3": "TabPFN v3.0",
"tabicl": "TabICL v1.1",
"tabicl_v1": "TabICL v1.1",
"tabicl_v2": "TabICL v2.0",
"tabdpt": "TabDPT",
"limix": "LimiX",
"mitra": "Mitra",
"seldon": "Seldon",
"nicl": "NICL",
"sap": "SAP RPT-1",
}
_METRIC_DISPLAY_FULL = {
"Accuracy": "Accuracy",
"AUC": "AUC",
"F1_score": "F1",
"Precision": "Precision",
"Recall": "Recall",
"Cross_entropy": "Cross-entropy",
"Elo_score": "Elo",
"%↗ over XGBoost": "% over XGBoost",
}
_LOWER_IS_BETTER = {"Cross_entropy"}
def _fmt_metric(v, m):
if pd.isna(v):
return ""
if m == "Elo_score":
# Legitimate Elo lives near INIT_RATING=1000 (typical
# spread 200–2000). If the value sneaks in as a 0–1
# normalised score from some upstream rank-pipeline,
# display the raw float instead of the int-rounded
# version so the caller sees the actual number.
if -2.0 <= v <= 2.0:
return f"{v:.3f}"
return f"{int(round(v))}"
if m == "%↗ over XGBoost":
return f"{v:+.2f}"
return f"{v:.3f}"
# Model-family palette for table chips (matches the bars CLASS_PALETTE).
FAMILY_OF_MODEL = {
"xgboost": "tree",
"catboost": "tree",
"lightgbm": "tree",
"xgboost_ensemble": "tree",
"catboost_ensemble": "tree",
"lightgbm_ensemble": "tree",
"tabm": "nn",
"realmlp": "nn",
"modern_nca": "nn",
"modernnca": "nn",
"tabpfn": "fm1",
"tabpfn_2_5": "fm1",
"tabicl": "fm1",
"tabicl_v1": "fm1",
"tabdpt": "fm1",
"mitra": "fm1",
"limix": "fm1",
"nicl": "fm1",
"sap": "fm1",
"tabpfn_v3": "fm2",
"tabicl_v2": "fm2",
"seldon": "fm2",
}
FAMILY_LABEL = {
"fm2": "FM 2nd gen",
"fm1": "FM 1st gen",
"tree": "GBDT",
"nn": "Tabular NN",
}
FAMILY_COLOR = {
"fm2": "#a18cff",
"fm1": "#5dc4c4",
"tree": "#7daaef",
"nn": "#ec9bca",
}
def _leading_group_per_metric(per_dataset_df, models, metric):
"""Return set of models in the Wilcoxon-Holm leading group
for the given metric. Mirrors the Bars (forest plot)
implementation exactly so the two views agree:
1. All N(N-1)/2 pairwise Wilcoxon p-values
2. Holm correction across the full pairwise family
3. Leading group = sort by mean, walk down from leader,
include every model not significantly worse than the
leader; STOP at first significantly-worse model.
"""
from scipy.stats import wilcoxon
from statsmodels.stats.multitest import multipletests
if per_dataset_df is None or per_dataset_df.empty or metric not in per_dataset_df.columns:
return set()
piv = per_dataset_df.pivot_table(index="dataset_id", columns="model",
values=metric, aggfunc="mean")
piv = piv.dropna(axis=0, how="any")
if piv.shape[0] < 5 or piv.shape[1] < 2:
return set()
cols = [m for m in models if m in piv.columns]
if len(cols) < 2:
return set(cols)
ascending = metric in _LOWER_IS_BETTER
means_local = piv[cols].mean(axis=0)
sorted_models = sorted(cols, key=lambda m: means_local[m],
reverse=not ascending)
# All pairwise comparisons in the (a, b) order where a is
# ranked higher than b in sorted_models.
pairs = []
for i, a in enumerate(sorted_models):
for b in sorted_models[i + 1:]:
d = (piv[a] - piv[b]).values
nz = d[d != 0]
if len(nz) < 5:
p = 1.0
else:
try:
p = wilcoxon(d, zero_method="wilcox", alternative="two-sided").pvalue
except Exception:
p = 1.0
pairs.append((a, b, p))
if not pairs:
return {sorted_models[0]}
_, p_adj, _, _ = multipletests([p for _, _, p in pairs], alpha=0.05, method="holm")
not_sig = set()
for (a, b, _), pa in zip(pairs, p_adj):
if pa > 0.05:
not_sig.add((a, b))
# Greedy walk down from the leader; stop at first model that
# IS significantly worse than the leader.
leader = sorted_models[0]
leading = {leader}
for m in sorted_models[1:]:
if (leader, m) in not_sig:
leading.add(m)
else:
break
return leading
def _table_html(df, per_dataset_df=None):
"""Render the agg dataframe to HTML with leading-group
violet highlight per column.
df is expected to have raw column names (model, Accuracy, AUC, ...).
A leading "Family" column groups models by lineage.
"""
if df is None or df.empty:
return (
'
No data for this filter
'
)
metric_cols = [c for c in df.columns if c != "model"]
# Leading-group set per metric column (used for violet
# highlighting). Excluded: Elo (already a single number
# per model, no per-dataset signal) and %↗ over XGBoost
# (depends on a single baseline, not meaningful here).
SIG_SKIP = {"Elo_score", "%↗ over XGBoost"}
leaders_per_col = {}
if per_dataset_df is not None:
for m in metric_cols:
if m in SIG_SKIP:
leaders_per_col[m] = set()
continue
leaders_per_col[m] = _leading_group_per_metric(
per_dataset_df, df["model"].tolist(), m
)
else:
leaders_per_col = {m: set() for m in metric_cols}
# Sort rows by family group (FM2 → FM1 → Trees → NN) so the
# family chips form contiguous bands. Within a family, keep
# original row order.
FAM_ORDER = {"fm2": 0, "fm1": 1, "tree": 2, "nn": 3, None: 4}
df_sorted = df.assign(_fam=df["model"].map(FAMILY_OF_MODEL)).copy()
df_sorted["_fam_rank"] = df_sorted["_fam"].map(
lambda f: FAM_ORDER.get(f, 4)
)
df_sorted = (
df_sorted.sort_values("_fam_rank", kind="stable")
.drop(columns=["_fam_rank"])
.reset_index(drop=True)
)
h = ['
']
h.append(
'
Model
'
)
for m in metric_cols:
h.append(f"
{_METRIC_DISPLAY_FULL.get(m, m)}
")
h.append("
")
# Pre-compute family group sizes for rowspan merging
fams_in_order = df_sorted["_fam"].tolist()
fam_sizes = {}
for f in fams_in_order:
fam_sizes[f] = fam_sizes.get(f, 0) + 1
last_fam = None
for i in range(len(df_sorted)):
model_raw = df_sorted.iloc[i]["model"]
fam = df_sorted.iloc[i]["_fam"]
model_disp = _MODEL_DISPLAY_FULL.get(model_raw, model_raw)
if fam != last_fam:
color = FAMILY_COLOR.get(fam, "#999")
label = FAMILY_LABEL.get(fam, "")
rowspan = fam_sizes.get(fam, 1)
fam_cell = (
f'
')
for m in metric_cols:
v = df_sorted.iloc[i][m]
in_lead = model_raw in leaders_per_col.get(m, set())
cell_extra = " nk-lead-cell" if in_lead else ""
h.append(
f'
{_fmt_metric(v, m)}
'
)
h.append("
")
h.append("
")
return "".join(h)
ranking_table = gr.HTML(
value=_table_html(public_enter_model_agg.round(4),
per_dataset_df=public_enter_per_dataset),
visible=False,
elem_classes=["nk-lb-wrap"],
)
comparison_plot = gr.Plot(
show_label=False,
visible=False,
elem_classes=["nk-comparison-plot"],
)
radar_overlay_plot = gr.Plot(
show_label=False,
visible=False,
elem_classes=["nk-radar-overlay"],
)
radar_grid_plot = gr.Plot(
show_label=False,
visible=False,
elem_classes=["nk-radar-grid"],
)
radar_overlay_rank_plot = gr.Plot(
show_label=False,
visible=False,
elem_classes=["nk-radar-overlay-rank"],
)
radar_grid_rank_plot = gr.Plot(
show_label=False,
visible=False,
elem_classes=["nk-radar-grid-rank"],
)
winrate_plot = gr.Plot(
show_label=False,
visible=False,
elem_classes=["nk-winrate-plot"],
)
# Load pre-built figures cache built by build_figures_cache.py.
# Used as a fast-path when sliders are at their defaults (most
# common case: clicking an industry button or switching metric).
_figures_cache = None
try:
_figures_cache = json.load(open("figures_cache.json"))
except FileNotFoundError:
print(
"[warn] figures_cache.json not found: run build_figures_cache.py"
)
def _decode_plotly_binary(obj):
"""Plotly's `to_plotly_json()` serializes ndarrays as
{dtype: ..., bdata: , shape: "r,c"} dicts. JS
plotly handles them, but `go.Figure(dict)` Python round-trips
don't always reconstruct them correctly (Heatmap z stays
as a dict and renders broken). Walk the cached dict and
replace any such payload with a plain (nested) list.
"""
import base64, numpy as _np
if isinstance(obj, dict):
if (
"bdata" in obj and "dtype" in obj
and set(obj.keys()) <= {"bdata", "dtype", "shape"}
):
shape_s = obj.get("shape")
if isinstance(shape_s, str) and shape_s:
shape = tuple(int(x) for x in shape_s.split(","))
else:
shape = None
arr = _np.frombuffer(
base64.b64decode(obj["bdata"]), dtype=obj["dtype"]
)
if shape:
arr = arr.reshape(shape)
return arr.tolist()
return {k: _decode_plotly_binary(v) for k, v in obj.items()}
if isinstance(obj, list):
return [_decode_plotly_binary(v) for v in obj]
return obj
if _figures_cache is not None:
_figures_cache = _decode_plotly_binary(_figures_cache)
def _strip_singlefit_gbdt_winrate(cache):
"""The cache was built from `public_per_dataset` which still
contains the single-fit GBDT rows (`xgboost`, `catboost`,
`lightgbm`) alongside their `*_ensemble` siblings. The
leaderboard hides the single-fit versions, but the
pairwise win-rate matrix shows them twice: once via the
pretty name "XGBoost" mapped from `xgboost`, and once via
the same pretty name mapped from `xgboost_ensemble`. Drop
the single-fit duplicates from each cached heatmap.
"""
DROP_NAMES = {"xgboost", "catboost", "lightgbm"}
# We need to detect which DUPLICATE pretty-name index to
# drop. The cache builder uses `_MODEL_DISPLAY_PRETTY`
# mapping; both 'xgboost' and 'xgboost_ensemble' map to
# the same display name 'XGBoost'. We drop the first
# occurrence of each duplicate display name: the second
# (ensemble) is the one we want to keep.
if not isinstance(cache, dict):
return
for bench, by_industry in cache.items():
if not isinstance(by_industry, dict):
continue
for industry, payload in by_industry.items():
if not isinstance(payload, dict):
continue
wr_dict = payload.get("winrate")
if not isinstance(wr_dict, dict):
continue
for metric, fig in wr_dict.items():
try:
heat = fig["data"][0]
labels = list(heat.get("x") or [])
if not labels:
continue
# Find first-occurrence indices for any
# label that appears more than once.
seen = {}
to_drop = []
for i, lab in enumerate(labels):
if lab in seen:
# duplicate: drop the FIRST occurrence
to_drop.append(seen[lab])
seen[lab] = i # keep latest = ensemble
else:
seen[lab] = i
if not to_drop:
continue
keep = [i for i in range(len(labels)) if i not in set(to_drop)]
# Apply to x, y, z, text
heat["x"] = [labels[i] for i in keep]
yvals = list(heat.get("y") or [])
if yvals:
heat["y"] = [yvals[i] for i in keep]
z = heat.get("z")
if isinstance(z, list):
heat["z"] = [
[row[j] for j in keep]
for row in [z[i] for i in keep]
]
text = heat.get("text")
if isinstance(text, list):
heat["text"] = [
[row[j] for j in keep]
for row in [text[i] for i in keep]
]
except Exception:
continue
return cache
_strip_singlefit_gbdt_winrate(_figures_cache)
def _strip_singlefit_gbdt_bars(cache):
"""Same dedup as for win-rate, but for the bar-chart cache.
The bars are a list of go.Bar traces keyed by the y-axis
model name. We drop traces whose model id is in the
single-fit GBDT set: these are duplicates of the
`*_ensemble` siblings under the same pretty display name.
"""
if not isinstance(cache, dict):
return
DROP_PRETTY = {"XGBoost", "CatBoost", "LightGBM"}
for bench, by_industry in cache.items():
if not isinstance(by_industry, dict):
continue
for industry, payload in by_industry.items():
if not isinstance(payload, dict):
continue
metrics = payload.get("metrics")
if not isinstance(metrics, dict):
continue
for metric, fig in metrics.items():
try:
# bars come as horizontal traces with their
# y-axis categoryarray listing every model
# twice. Drop the FIRST of each duplicate
# category: that's the single-fit row.
yaxis = (fig.get("layout") or {}).get("yaxis", {})
tickvals = yaxis.get("tickvals")
ticktext = yaxis.get("ticktext")
if not isinstance(tickvals, list) or not isinstance(ticktext, list):
continue
seen = {}
to_drop_idx = []
for i, label in enumerate(ticktext):
if label in seen:
to_drop_idx.append(seen[label])
seen[label] = i
else:
seen[label] = i
if not to_drop_idx:
continue
keep = [i for i in range(len(ticktext)) if i not in set(to_drop_idx)]
drop_models = {tickvals[i] for i in to_drop_idx}
yaxis["tickvals"] = [tickvals[i] for i in keep]
yaxis["ticktext"] = [ticktext[i] for i in keep]
# Drop traces (bars + CI) that target the dropped y-values
data = fig.get("data") or []
new_data = []
for tr in data:
ys = tr.get("y")
# Each bar trace has y=[model_id]
if isinstance(ys, list) and any(y in drop_models for y in ys):
continue
new_data.append(tr)
fig["data"] = new_data
# Drop shapes that anchor to dropped y-values
shapes = ((fig.get("layout") or {}).get("shapes")) or []
if shapes:
fig["layout"]["shapes"] = [
s for s in shapes
if not (
isinstance(s, dict)
and s.get("yref") == "y"
and (s.get("y0") in drop_models or s.get("y1") in drop_models)
)
]
except Exception:
continue
return cache
_strip_singlefit_gbdt_bars(_figures_cache)
# External-benchmark dataset lists. Values are OpenML dataset
# IDs (as integers): both the canonical IDs published by each
# benchmark AND any equivalent dataset we have under a
# different OpenML ID (a few TabArena tasks are well-known
# datasets re-uploaded by TabArena under fresh IDs; we already
# had the classic low-numbered versions in our base).
_BENCHMARK_DATASET_IDS = {
"TabArena": {
# canonical TabArena IDs (46900s range): 32 we share
46905, 46906, 46908, 46910, 46911, 46912, 46916, 46919,
46920, 46922, 46924, 46927, 46929, 46930, 46932, 46933,
46935, 46937, 46938, 46940, 46941, 46947, 46950, 46955,
46956, 46958, 46960, 46962, 46963, 46969, 46979, 46980,
# equivalents for 5 TabArena IDs we have under other IDs:
# 46913 blood-transfusion-service-center → 1464
# 46915 churn → 40701
# 46918 credit-g → 31
# 46921 diabetes → 37
# 46952 qsar-biodeg → 1494
1464, 40701, 31, 37, 1494,
# kddcup09_appetency: added to the base separately
46939,
},
}
def _apply_benchmark_filter(df, benchmark):
if not benchmark or benchmark == "All":
return df
ids = _BENCHMARK_DATASET_IDS.get(benchmark)
if not ids:
return df
id_strs = {str(i) for i in ids}
return df[df["dataset_id"].astype(str).isin(id_strs)]
def _count_datasets(benchmark, industry, feat_range, samples_range, classes_range):
"""Number of distinct datasets surviving the current filters.
Datasets without metadata (NaN features/rows/classes) pass
the numeric filters by default: they only get excluded when
an industry filter is applied."""
df = public_per_dataset_with_meta
df = _apply_benchmark_filter(df, benchmark)
raw = _BUCKET_FROM_LABEL.get(industry, industry)
if raw and raw != "All":
df = df[df["dataset_industry"] == raw]
f_lo, f_hi = feat_range
s_lo, s_hi = samples_range
c_lo, c_hi = classes_range
feat_ok = df["features"].isna() | df["features"].between(f_lo, f_hi)
rows_ok = df["rows"].isna() | df["rows"].between(s_lo, s_hi)
class_ok = df["n_classes"].isna() | df["n_classes"].between(
c_lo, c_hi
)
df = df[feat_ok & rows_ok & class_ok]
return int(df["dataset_id"].nunique())
def _count_label(benchmark, industry, feat_range, samples_range, classes_range):
n = _count_datasets(
benchmark, industry, feat_range, samples_range, classes_range
)
return f"📦 **{n}** datasets"
def _filter_and_aggregate(
benchmark, industry, feat_range, samples_range, classes_range
):
"""Apply benchmark/industry/feature/sample/class filters and re-aggregate.
NaN features/rows/classes pass the numeric filters (only the
industry filter requires metadata to be present)."""
df = public_per_dataset_with_meta
df = _apply_benchmark_filter(df, benchmark)
raw = _BUCKET_FROM_LABEL.get(industry, industry)
if raw and raw != "All":
df = df[df["dataset_industry"] == raw]
f_lo, f_hi = feat_range
s_lo, s_hi = samples_range
c_lo, c_hi = classes_range
feat_ok = df["features"].isna() | df["features"].between(f_lo, f_hi)
rows_ok = df["rows"].isna() | df["rows"].between(s_lo, s_hi)
class_ok = df["n_classes"].isna() | df["n_classes"].between(
c_lo, c_hi
)
df = df[feat_ok & rows_ok & class_ok]
if df.empty:
empty = pd.DataFrame(
columns=[
"model",
"Accuracy",
"AUC",
"F1_score",
"Precision",
"Recall",
"Cross_entropy",
"Elo_score",
"%↗ over XGBoost",
]
)
return empty, df
df = compute_elo_for_subset(df, metric="Accuracy")
df = compute_pct_improvement_over_baseline(
df, baseline_model="xgboost_ensemble", metric="Accuracy"
)
# Drop single-fit GBDT rows now that Elo and %↗ have been
# computed against the ensemble baseline. Otherwise the
# live-recompute path (sliders / industry / benchmark)
# would render "XGBoost"/"CatBoost"/"LightGBM" twice in
# the bars, win-rate matrix, and table — the cache
# fast-path already dedups, but the slow path didn't.
df = df[~df["model"].isin(_HIDDEN_DISPLAY_MODELS)]
agg = (
df.groupby("model")[
[
"Accuracy",
"AUC",
"F1_score",
"Precision",
"Recall",
"Cross_entropy",
"Elo_score",
"%↗ over XGBoost",
]
]
.mean()
.reset_index()
)
return agg, df
def _update_view(
view, metric, benchmark, industry,
feat_range, samples_range, classes_range,
):
raw = _BUCKET_FROM_LABEL.get(industry, industry)
count_label = _count_label(
benchmark, industry, feat_range, samples_range, classes_range
)
# Branches return (bars, table, comp, radar_overlay,
# radar_grid, radar_overlay_rank, radar_grid_rank,
# winrate, count).
# Only the active view's widget is visible.
hide = gr.update(visible=False)
# Fast path: cache lookup only when sliders are at their
# full default range. The cache is now keyed by
# (benchmark, industry, metric): look up the benchmark
# slice if present. Falls back to slow path otherwise.
f_lo, f_hi = feat_range
s_lo, s_hi = samples_range
c_lo, c_hi = classes_range
bench_key = benchmark if benchmark else "All"
cache_root = None
if _figures_cache is not None:
# New nested shape: cache[bench][industry]
if isinstance(_figures_cache.get(bench_key), dict) and raw in _figures_cache[bench_key]:
cache_root = _figures_cache[bench_key][raw]
# Old flat shape (only valid for the All benchmark)
elif bench_key == "All" and raw in _figures_cache and "metrics" in _figures_cache[raw]:
cache_root = _figures_cache[raw]
# Cache fast-path: hit whenever the sliders are at (or near)
# the full range the cache was built under. Float equality
# was too strict — any tiny drift from gradio re-emitting
# the slider value as a slightly-different float dropped
# us into the slow path. Use a generous tolerance: the
# cache premise is just "no filtering", so as long as the
# slider window covers the full range, we can use it.
def _is_full(lo, hi, lo_ref, hi_ref):
return lo <= lo_ref + 1e-6 and hi >= hi_ref - 1e-6
use_cache = (
cache_root is not None
and _is_full(f_lo, f_hi, _feat_min, _feat_max)
and _is_full(s_lo, s_hi, _rows_min, _rows_max)
and _is_full(c_lo, c_hi, _classes_min, _classes_max)
and metric in cache_root.get("metrics", {})
)
if (
use_cache
and view == "Bars"
and metric in cache_root.get("forest", {})
):
# Bars view = significance forest plot. Cached at build
# time (build_figures_cache.py) because the live render
# runs Wilcoxon-Holm + 1000-iter paired bootstrap, ~30 s
# on Spaces CPU per metric change. The KPI HTML is
# served from the cache when present; otherwise we fall
# back to live recompute (slow path).
import plotly.graph_objects as _go
fig = _go.Figure(cache_root["forest"][metric])
kpi_html_cached = cache_root.get("forest_kpi", {}).get(metric, "")
return (
gr.update(visible=True, value=fig),
hide,
hide,
hide,
hide,
hide,
hide,
hide,
gr.update(visible=True, value=kpi_html_cached),
count_label,
)
if use_cache and view == "Table":
table = pd.DataFrame(cache_root["table"])
ascending = metric in _LOWER_IS_BETTER
sorted_df = (
(
table.sort_values(by=metric, ascending=ascending)
.round(4)
.reset_index(drop=True)
)
if not table.empty
else table
)
# For the per-cell leading-group highlight we need
# the per-dataset frame. The cache doesn't store it
# (and serialising 189×14 rows per slice would bloat
# the JSON), so reconstruct it live from the same
# filters the cache was built under.
fast_pd_df = _apply_benchmark_filter(public_enter_per_dataset, benchmark)
if raw and raw != "All":
try:
# public_enter_per_dataset is the per-dataset frame;
# filter by industry using meta if available.
_id_to_ind = (
_public_meta_df[["dataset_id", "dataset_industry"]]
.drop_duplicates("dataset_id")
.set_index("dataset_id")["dataset_industry"]
.to_dict()
)
_industries = fast_pd_df["dataset_id"].astype(str).map(_id_to_ind)
fast_pd_df = fast_pd_df[_industries == raw]
except Exception:
pass
return (
hide,
gr.update(visible=True, value=_table_html(sorted_df, per_dataset_df=fast_pd_df)),
hide,
hide,
hide,
hide,
hide,
hide,
gr.update(),
count_label,
)
# Win-rate view: cached at build time. Live recompute is
# slow on Spaces CPU (the heatmap pairs every model against
# every other across the full dataset). After any styling
# change to plot_winrate_matrix, rerun build_figures_cache.py.
if (
use_cache
and view == "Win-rate"
and metric in cache_root.get("winrate", {})
):
import plotly.graph_objects as _go
fig = _go.Figure(cache_root["winrate"][metric])
return (
hide,
hide,
hide,
hide,
hide,
hide,
hide,
gr.update(visible=True, value=fig),
gr.update(),
count_label,
)
# Slow path: compute filtered subset live.
agg, filtered_per_dataset = _filter_and_aggregate(
benchmark, industry, feat_range, samples_range, classes_range
)
if view == "Bars":
if agg.empty or filtered_per_dataset.empty:
return (
gr.update(visible=True, value=None),
hide,
hide,
hide,
hide,
hide,
hide,
hide,
gr.update(),
count_label,
)
# Bars view now renders the significance forest plot
# (Wilcoxon-Holm leading-group highlight + CI lollipops).
fig, kpi_html = plot_significance_forest(
filtered_per_dataset, metric=metric,
)
return (
gr.update(visible=True, value=fig),
hide,
hide,
hide,
hide,
hide,
hide,
hide,
gr.update(visible=True, value=kpi_html),
count_label,
)
if view == "Win-rate":
if filtered_per_dataset.empty:
return (
hide,
hide,
hide,
hide,
hide,
hide,
hide,
gr.update(visible=True, value=None),
gr.update(),
count_label,
)
fig = plot_winrate_matrix(filtered_per_dataset, metric=metric)
return (
hide,
hide,
hide,
hide,
hide,
hide,
hide,
gr.update(visible=True, value=fig),
gr.update(),
count_label,
)
if view == "Table":
ascending = metric in _LOWER_IS_BETTER
sorted_df = (
(
agg.sort_values(by=metric, ascending=ascending)
.round(4)
.reset_index(drop=True)
)
if not agg.empty
else agg
)
return (
hide,
gr.update(visible=True, value=_table_html(sorted_df, per_dataset_df=filtered_per_dataset)),
hide,
hide,
hide,
hide,
hide,
hide,
gr.update(),
count_label,
)
if view == "Dataset comparison":
if (
filtered_per_dataset.empty
or filtered_per_dataset["dataset_id"].nunique() < 10
):
empty_fig = px.scatter(
title="Not enough datasets for this view (need ≥10 after filters)"
)
return (
hide,
hide,
gr.update(visible=True, value=empty_fig),
hide,
hide,
hide,
hide,
hide,
gr.update(),
count_label,
)
fig = plot_dataset_comparison(
filtered_per_dataset, metric=metric
)
return (
hide,
hide,
gr.update(visible=True, value=fig),
hide,
hide,
hide,
hide,
hide,
gr.update(),
count_label,
)
if view == "Model radars":
if filtered_per_dataset.empty:
return (
hide,
hide,
hide,
hide,
hide,
hide,
hide,
hide,
gr.update(),
count_label,
)
overlay_mm, sm_mm, overlay_rk, sm_rk = plot_model_radars(
filtered_per_dataset
)
return (
hide,
hide,
hide,
gr.update(visible=True, value=overlay_mm),
gr.update(visible=True, value=sm_mm),
gr.update(visible=True, value=overlay_rk),
gr.update(visible=True, value=sm_rk),
hide,
gr.update(),
count_label,
)
# Fallback (unknown view): fall back to bars.
return (
gr.update(visible=True),
hide,
hide,
hide,
hide,
hide,
hide,
hide,
gr.update(),
count_label,
)
_filter_inputs = [
view_selector,
metric_selector,
benchmark_selector,
industry_selector,
features_slider,
samples_slider,
classes_slider,
]
for ctrl in (
metric_selector,
view_selector,
benchmark_selector,
industry_selector,
features_slider,
samples_slider,
classes_slider,
):
ctrl.change(
fn=_update_view,
inputs=_filter_inputs,
outputs=[
ranking_plot,
ranking_table,
comparison_plot,
radar_overlay_plot,
radar_grid_plot,
radar_overlay_rank_plot,
radar_grid_rank_plot,
winrate_plot,
ranking_kpi_html,
dataset_count_md,
],
)
# Section 03: Methodology (now BEFORE Performance Map) -----
gr.Markdown(" ")
gr.Markdown('## 03Methodology')
gr.Markdown("""
**Scope.** TabBench is a **classification-only** benchmark for now. All 189 datasets are classification tasks (binary or multi-class); regression is intentionally out of scope until the FM landscape stabilises there.
**Evaluation protocol.** Each (model, dataset) pair is run on **5 stratified shuffle splits** (80/20). The per-dataset mean across the 5 folds is what feeds the leaderboard.
**Hyperparameter tuning.**
- **Tree-based models** (XGBoost, CatBoost, LightGBM): **100 Optuna trials** on a held-out validation split of the train fold, with a **4-hour time limit per dataset**. The reported XGBoost / CatBoost / LightGBM scores are **bagged ensembles** of the top trials per dataset, not single-fit. We surface only the ensembled variant because it is the canonical way these models are deployed in practice and the strongest baseline available to compare against the foundation models.
- **Neural networks** (RealMLP, TabM, ModernNCA): run through **AutoGluon**, which handles preprocessing and training.
- **Foundation models** (TabPFNv2.5/v3, TabICL/v1/v2, TabDPT, Mitra, Seldon, Limix): used as-is, no fine-tuning, no HP search. They perform in-context learning at inference time.
**Preprocessing.**
- **Tree-based**: ordinal-encoded categoricals, raw numerics (these models handle categorical features natively).
- **Foundation models**: their own pipelines, used as-published.
**Significance forest plot (default leaderboard view).** Each row is a model. The dot is the model's mean score across the filtered datasets; the horizontal bar is a **95% confidence interval**. The CI is computed in two steps:
1. **Within-dataset z-score**: for each dataset, subtract the across-models mean and divide by the across-models standard deviation. This strips out dataset difficulty: a +0.01 gap on a hard dataset isn't drowned out by ±0.001 noise on easy ones.
2. **Paired bootstrap (B = 1 000)** on those z-scores, then **back-project** to the metric scale by multiplying by σ̄: the mean across datasets of the within-dataset standard deviation. The bar you see lives in native metric units (e.g. Accuracy points) but reflects the *paired* uncertainty, not the per-model spread across datasets. That makes the bars consistent with the significance test driving the leading-group highlight.
**Leading-group highlight.** The violet-shaded rows are the **statistically indistinguishable winners** at α = 0.05, identified by a **Wilcoxon signed-rank test** between every model pair, with **Holm correction** for multiple comparisons. The leading group is the top model plus every model whose Holm-adjusted p-value vs the top model is > 0.05. Excluded from significance testing: Elo (one global number per model: no per-dataset signal) and % over XGBoost (already baseline-relative).
**Elo score.** A separate global ranking computed by Bradley–Terry on **pairwise wins across datasets**: for each (dataset, model_A, model_B) we count a win for whichever has the higher Accuracy. The BT log-likelihood is fit with BFGS; scores are scaled to ±400 around 1000 in standard Elo fashion. Unlike the mean, Elo is invariant to absolute score levels: it just cares about who beats whom.
**% over XGBoost** is the per-dataset relative improvement vs XGBoost's score on the same dataset, averaged across datasets.
""")
# Section 04: Performance Map -----------------------------
gr.Markdown(" ")
gr.Markdown('## 04Performance Map')
gr.Markdown("""
The leaderboard tells you which model has the highest *average* score; the **Performance Map** tells you **where on the dataset landscape each model family wins or struggles**. We use it to summarize results across multiple dimensions and help build insights into model behavior. In particular, we study tabular foundation models against tuned classical baselines, analyzing where they dominate, where they collapse to parity, and which dataset properties (categorical density, imbalance, class count, size) predict each regime.
""")
# Compute the full-base performance map once at boot. Use the
# display-filtered frame so single-fit GBDTs aren't double-counted
# alongside their ensemble variants (which biases the PCA + the
# rank matrix toward GBDT structure).
_perfmap_full = _build_perfmap(public_enter_per_dataset)
_perfmap_meta = _perfmap_meta_df()
_PERFMAP_CACHE["full"] = _perfmap_full
if _perfmap_full is not None:
perfmap_bg_radio = gr.Radio(
choices=[
("None", "none"),
("FM − GBDT accuracy", "fm_vs_gbdt"),
("FM − tuned NN accuracy", "fm_vs_nn"),
("GBDT − tuned NN accuracy", "gbdt_vs_nn"),
("2nd-gen − 1st-gen FM accuracy", "modern_vs_leg"),
("TabDPT − other 1st-gen FMs", "tabdpt_vs_fm1"),
("Class imbalance (Acc − F1)", "imbalance"),
],
value="none",
label="Background level-set",
elem_classes=["compact-radio"],
)
perfmap_scatter = gr.Plot(
value=plot_perfmap_scatter(
_perfmap_full,
_perfmap_meta,
title_extra=": full base",
background="none",
),
show_label=False,
elem_classes=["nk-perfmap-scatter"],
)
def _on_bg_change(bg):
return plot_perfmap_scatter(
_perfmap_full,
_perfmap_meta,
title_extra=": full base",
background=bg,
)
perfmap_bg_radio.change(
fn=_on_bg_change,
inputs=[perfmap_bg_radio],
outputs=[perfmap_scatter],
)
with gr.Column(elem_classes=["nk-seldon-frame"]):
gr.Markdown("### How to read the map")
gr.Markdown("""
Each dot is one classification dataset. Datasets that produce similar model-performance profiles sit close together; the two axes are the directions of greatest variation across the 189-dataset benchmark.
- **Horizontal axis**: datasets shift from **mostly numeric, fewer rows** on the left to **more categorical, more rows** on the right.
- **Vertical axis**: datasets shift from **binary, more balanced** at the bottom to **multi-class, more imbalanced** at the top.
Hover over a dot for its name, vertical, row count, feature count and class count. Coloured blobs are the three emergent clusters from co-clustering the (datasets × model-metrics) matrix; dashed rings mark a cluster whose members are too spread out for a single closed region.
""")
gr.Markdown(
'### 04.ASalient clusters'
)
gr.Markdown("""
2nd-gen FM Supremacy · Numeric data · Small-to-mid size126 datasets
By far the largest cluster on the map. 2nd-gen FMs win 116/126 datasets outright. FM2 head-to-head: 91 % vs 1st-gen FMs, 98 % vs GBDTs, 99 % vs tuned NNs. Small-to-mid-size numeric data is the regime where FM training data lives, and it shows: the rest of the field is left behind.
GBDTs rival FMs · Categorical data · Mid-size binary46 datasets
The one regime where reaching for FMs is wrong. GBDT ensembles take 23/46 rank-1 wins vs 19 for 2nd-gen FMs. FM2 head-to-head vs GBDTs: only 41 %. High categorical %, mid-size binary classification: classical tabular methods still hold their ground here, particularly on behavioural / healthcare / finance data.
Tuned NNs rival FMs · Large multi-class · Many rows, many classes17 datasets
The FM context-window edge case. 2nd-gen FMs still lead (13/17) but tuned NNs claim 4 wins and FM2 head-to-head vs NNs drops to 76 %. The 1st-generation FMs collapse here (0 wins): large multi-class data sits well above their context window. GBDTs are uncompetitive too. Only RealMLP / TabM / ModernNCA stay in the fight.
Mean accuracy rank, 2nd-gen FMs minus GBDT ensembles
−0.07→+0.49
The headline trend. 2nd-gen FMs are well ahead on Numeric data (+0.49) and on Large multi-class (+0.40), but the gap nearly closes on the Categorical / GBDTs rival FMs cluster (−0.07): the only regime where classical tabular methods still edge out the FMs.
2nd-gen − 1st-gen FM accuracy
How much of the FM lead belongs to the new generation
+0.19→+0.53
The 2026 generation strictly improves over the 2024 one. Positive in every cluster, widest in Large multi-class (+0.53), where 1st-gen FMs hit their context-window wall. Narrowest on the Categorical / GBDTs rival FMs cluster (+0.19), where both generations have to share the regime with GBDTs.
2nd-gen FM − tuned NN accuracy
2nd-gen FMs minus RealMLP / TabM / ModernNCA
+0.17→+0.57
2nd-gen FMs are ahead of tuned NNs in every cluster, peaking at +0.57 on Numeric data. The narrowest gap is Large multi-class (+0.17), the regime where TabM / RealMLP / ModernNCA close the gap and pick up their few outright wins.
GBDT − tuned NN accuracy
GBDT ensembles minus tuned neural nets
−0.23→+0.34
GBDT > NN on numeric and categorical data, with the strongest GBDT edge on Categorical / GBDTs rival FMs (+0.34). Flips to −0.23 on Large multi-class: same regime where NNs catch the FMs. Multi-class is where the classical/NN ordering reverses.
""")
else:
gr.Markdown(
"_Performance map unavailable: not enough datasets in the base._"
)
# Section 05: Pick a model -------------------------------
gr.Markdown(" ")
gr.Markdown('## 05Pick a model')
gr.Markdown("""
**2nd-gen FMs win on the overwhelming majority of classification datasets** (157 out of 189), but the performance map above already shows that other families still take regimes of their own. The tree below makes that explicit on dataset metadata: where the number of rows is high and categorical features dominate, **GBDT ensembles still rival FMs**: the classical regime where structured tabular data and well-tuned trees hold their ground. And where datasets are large with many classes, **tuned NNs get some victories**: their parameter count scales naturally with the number of output classes, which 1st-gen FMs can't follow and 2nd-gen FMs only partially compensate for.
""")
gr.Plot(
value=render_perfmap_decision_tree(
public_enter_per_dataset, mode="winner"
),
show_label=False,
elem_classes=["nk-tree-plot"],
)
# Section 06: Links --------------------------------------
gr.Markdown(" ")
gr.Markdown('## 06Links')
gr.Markdown("""
- **Neuralk**: [neuralk.ai](https://www.neuralk.ai/)
- **Package documentation**: [docs.neuralk.ai](https://docs.neuralk.ai/)
- **Previous version of TabBench**: [https://dashboard.neuralk-ai.com/](https://dashboard.neuralk-ai.com/)
""")
# Hidden placeholders so the rest of the navigation wiring
# (popstate, show_content, outputs lists) keeps working without
# broken references. The Academic/Enterprise/Datamap panes are
# still reachable by URL hash but no Home button links to them.
public_btn_home = gr.Button("", visible=False, elem_id="nav-academic")
private_btn_home = gr.Button(
"", visible=False, elem_id="nav-enterprise"
)
datamap_btn_home = gr.Button("", visible=False, elem_id="nav-datamap")
public_content = gr.Column(visible=False)
with public_content:
home_btn_public = gr.Button(
"🏡 Home",
elem_classes=["white-btn", "selected-btn"],
elem_id="nav-home-from-academic",
)
label_md = gr.Markdown(
"# 🥇 TabBench Academic Leaderboard - All Industries"
)
gr.Markdown("### Features")
# -------------------------
# Top row: Models & Datasets
# -------------------------
with gr.Row():
with gr.Column():
with gr.Accordion("📂 Datasets", open=False):
gr.Markdown(PUBLIC_DATASETS_TEXT)
with gr.Column():
with gr.Accordion("🤖 Models", open=False):
gr.Markdown(MODELS_TEXT)
# -------------------------
# Second row: Metrics & Evaluation Workflow
# -------------------------
with gr.Row():
with gr.Column():
with gr.Accordion("⚙️ Benchmarking procedure", open=False):
gr.Markdown(BENCHMARKING_TEXT)
with gr.Column():
with gr.Accordion("📈 Metrics", open=False):
gr.Markdown(METRICS_TEXT)
with gr.Accordion("📙 Citation", open=False):
gr.Markdown(
"If you reference any part of TabBench and its results in your work, please cite it using the following citation:"
)
gr.Textbox(
value=CITATION_BUTTON_TEXT,
label=CITATION_BUTTON_LABEL,
lines=10,
elem_id="citation-button",
show_copy_button=True,
)
gr.Markdown("## 🏆 Overview ")
# Compute per-model averages from public_per_dataset
public_model_agg = (
public_per_dataset.groupby("model")[
[
"Accuracy",
"AUC",
"F1_score",
"Precision",
"Recall",
"Cross_entropy",
"Elo_score",
"%↗ over XGBoost",
]
]
.mean()
.reset_index()
)
public_model_agg = public_model_agg.round(3)
# Add model_type from model_info.json
with open("model_info.json", "r") as f:
model_info = pd.DataFrame(json.load(f))
public_model_agg = public_model_agg.merge(
model_info[["model_name", "model_type"]],
left_on="model",
right_on="model_name",
how="left",
).drop(columns=["model_name"])
# Only keep the required columns, with model_type second
public_model_agg = public_model_agg[
[
"model",
"model_type",
"Accuracy",
"AUC",
"F1_score",
"Precision",
"Recall",
"Cross_entropy",
"Elo_score",
"%↗ over XGBoost",
]
]
public_model_agg = public_model_agg.sort_values(
by=["model"], key=lambda x: x != "NICL"
)
model_df = gr.DataFrame(
value=highlight_max(public_model_agg),
interactive=False,
wrap=True,
type="pandas",
)
gr.Markdown("## 🔍 Explore more and filter")
with gr.Tabs() as tabs:
with gr.Tab("🤖 Average model performance"):
# Count unique datasets by id used for model performance
num_unique_datasets = (
public_per_dataset["dataset_id"].nunique()
if "dataset_id" in public_per_dataset.columns
else 0
)
gr.Markdown(
f"Estimated average model performance over {num_unique_datasets} datasets."
)
# Use per-model averages for leaderboard, allow filtering by model_type
per_model_leaderboard = init_leaderboard(
public_model_agg,
include_model_type_filter=True,
industry_filter=False,
)
with gr.Tab("📊 Performance by dataset"):
# Load dataset metadata
with open("public_datasets_info.json", "r") as f:
public_datasets_info = pd.DataFrame(json.load(f))
# Merge on dataset_id only: new results use openml-XXXX as
# dataset_name, but the metadata file uses the human-readable
# name, so a two-key merge would miss everything.
public_per_dataset_merged = public_per_dataset.merge(
public_datasets_info.drop(
columns=["dataset_name"], errors="ignore"
),
on="dataset_id",
how="left",
)
public_per_dataset_merged["industry"] = public_per_dataset_merged[
"dataset_industry"
]
public_per_dataset_merged = public_per_dataset_merged.dropna(
subset=["industry"]
)
# --- Sliders and controls ---
min_features, max_features = int(
public_per_dataset_merged["features"].min()
), int(public_per_dataset_merged["features"].max())
min_rows, max_rows = int(
public_per_dataset_merged["rows"].min()
), int(public_per_dataset_merged["rows"].max())
min_pct_cat, max_pct_cat = float(
public_per_dataset_merged["pct_cat"].min()
), float(public_per_dataset_merged["pct_cat"].max())
with gr.Row():
with gr.Column():
features_slider = gr.Slider(
minimum=min_features,
maximum=max_features,
value=min_features,
step=1,
label="Min Number of Features",
)
rows_slider = gr.Slider(
minimum=min_rows,
maximum=max_rows,
value=min_rows,
step=100,
label="Min Number of Rows",
)
pct_cat_slider = gr.Slider(
minimum=min_pct_cat,
maximum=max_pct_cat,
value=min_pct_cat,
step=0.01,
label="Min % Categorical Features",
)
with gr.Column():
task_button = gr.Radio(
choices=["Binary", "Multi-class"],
value="Binary",
label="Task Type",
)
# --- Columns ---
dataset_perf_cols = [
"model",
"Accuracy",
"AUC",
"F1_score",
"Precision",
"Recall",
"Cross_entropy",
"Elo_score",
"%↗ over XGBoost",
]
# --- Filtering & aggregation ---
def filter_and_aggregate_models(
min_features, min_rows, min_pct_cat, task_type
):
task_val = 0 if task_type == "Binary" else 1
filtered = public_per_dataset_merged[
(public_per_dataset_merged["features"] >= min_features)
& (public_per_dataset_merged["rows"] >= min_rows)
& (public_per_dataset_merged["pct_cat"] >= min_pct_cat)
& (public_per_dataset_merged["task"] == task_val)
]
if filtered.empty:
return pd.DataFrame(columns=dataset_perf_cols)
# Recompute Elo scores for this filtered subset
filtered = compute_elo_for_subset(filtered, metric="Accuracy")
# Recompute percentage improvement over XGBoost (ensemble)
filtered = compute_pct_improvement_over_baseline(
filtered, baseline_model="xgboost_ensemble", metric="Accuracy"
)
# Hide single-fit GBDTs from the displayed table (same
# rule as the leaderboard view); kept until now because
# the %↗ baseline needs the xgboost_ensemble rows.
filtered = filtered[~filtered["model"].isin(_HIDDEN_DISPLAY_MODELS)]
per_model_avg = (
filtered.groupby("model")[dataset_perf_cols[1:]]
.mean()
.reset_index()
)
return highlight_max(per_model_avg)
# --- Styled table ---
performance_table = gr.DataFrame(
value=filter_and_aggregate_models(
min_features,
min_rows,
min_pct_cat,
"Binary",
),
interactive=False,
wrap=True,
type="pandas",
)
# --- Bind controls ---
inputs = [features_slider, rows_slider, pct_cat_slider, task_button]
for control in inputs:
control.change(
fn=filter_and_aggregate_models,
inputs=inputs,
outputs=performance_table,
)
with open("public_datasets_info.json", "r") as f:
public_datasets_info = pd.DataFrame(json.load(f))
with open("model_info.json", "r") as f:
model_info = pd.DataFrame(json.load(f))
private_content = gr.Column(visible=False)
with private_content:
home_btn_private = gr.Button(
"🏡 Home",
elem_classes=["white-btn", "selected-btn"],
elem_id="nav-home-from-enterprise",
)
label_md2 = gr.Markdown("# 🥇 TabBench Enterprise Leaderboard")
gr.Markdown("## Example: comparative results on QRT Data Challenge")
gr.Markdown(
""
"Qube Research & Technologies (QRT) is a leading global quantitative and systematic investment manager. Their use of real-world business data to sponsor a data challenge with ENS Paris provided a real-world opportunity to highlight the value of Neuralk's Predictive Foundation Models on Enterprise financial data. "
"The proposed challenge aimed predicting the return of a stock in the US market using historical data over a recent period of 20 days. More info on the challenge [here](https://challengedata.ens.fr/challenges/23). "
"The models were all evaluated under the same preprocessing and feature engineering steps, thus providing a fair comparison of their predictive capabilities on this real-world financial dataset."
)
gr.DataFrame(
value=styled_df_qrt,
interactive=False,
wrap=True,
type="pandas",
)
gr.Markdown("### Features")
with gr.Row():
with gr.Column():
with gr.Accordion("📂 Datasets & use case", open=False):
gr.Markdown(PRIVATE_DATA_TEXT)
with gr.Column():
with gr.Accordion("🤖 Models", open=False):
gr.Markdown(PRIVATE_MODEL_TEXT)
with gr.Row():
with gr.Accordion("⚙️ Benchmarking procedure", open=False):
gr.Markdown(PRIVATE_BENCHMARKING_TEXT)
with gr.Row():
with gr.Column():
with gr.Accordion("📈 Metrics", open=False):
gr.Markdown(PRIVATE_METRICS_TEXT)
with gr.Accordion("📙 Citation", open=False):
gr.Markdown(
"If you reference any part of TabBench and its results in your work, please cite it using the following citation:"
)
gr.Textbox(
value=CITATION_BUTTON_TEXT,
label=CITATION_BUTTON_LABEL,
lines=10,
elem_id="citation-button",
show_copy_button=True,
)
gr.Markdown("## 🏆 Overview : Leaderboard on Retail Use-cases ")
# Compute per-model averages from private_per_dataset
private_model_agg = (
private_per_dataset.groupby("model")[
["Accuracy", "Precision", "Recall", "F1_score", "AUC", "Elo_score"]
]
.mean()
.reset_index()
)
private_model_agg = private_model_agg.round(3)
# Add model_type from model_info.json
with open("model_info.json", "r") as f:
model_info = pd.DataFrame(json.load(f))
private_model_agg = private_model_agg.merge(
model_info[["model_name", "model_type"]],
left_on="model",
right_on="model_name",
how="left",
).drop(columns=["model_name"])
# Only keep the required columns, with model_type second
private_model_agg = private_model_agg[
[
"model",
"model_type",
"Accuracy",
"Precision",
"Recall",
"F1_score",
"AUC",
"Elo_score",
]
]
private_model_agg = private_model_agg.sort_values(
by=["model"], key=lambda x: x != "NICL"
)
model_df = gr.DataFrame(
value=highlight_max(private_model_agg),
interactive=False,
wrap=True,
type="pandas",
)
gr.Markdown("## 🔍 Explore performance by dataset")
# Columns to display
dataset_perf_cols = [
"model",
"Accuracy",
"Precision",
"Recall",
"F1_score",
"AUC",
"Elo_score",
]
# Available datasets
dataset_ids = private_per_dataset["dataset_id"].dropna().unique().tolist()
dataset_ids = sorted(dataset_ids)
dataset_dropdown = gr.Dropdown(
choices=dataset_ids,
label="📂 Select a dataset",
value=dataset_ids[0],
)
dataset_leaderboard = gr.DataFrame(
value=highlight_max(
private_per_dataset[
private_per_dataset["dataset_id"] == dataset_ids[0]
][dataset_perf_cols]
),
interactive=False,
wrap=True,
type="pandas",
)
def update_dataset_leaderboard(selected_dataset_id):
return highlight_max(
private_per_dataset[
private_per_dataset["dataset_id"] == selected_dataset_id
][dataset_perf_cols]
)
dataset_dropdown.change(
fn=update_dataset_leaderboard,
inputs=dataset_dropdown,
outputs=dataset_leaderboard,
)
datamap_content = gr.Column(visible=False)
with datamap_content:
home_btn_datamap = gr.Button(
"🏡 Home",
elem_classes=["white-btn", "selected-btn"],
elem_id="nav-home-from-datamap",
)
gr.Markdown("# 🤖 Understanding tabular Foundation Models")
gr.Markdown(" ")
gr.Markdown(
"Side study using scikit-learn's make_classification data generation function to have an in-depth understanding of Tabular Foundation Models strengths and weaknesses under specific data characteristics."
)
gr.Markdown("")
PARQUET_PATH = "make_classif-parameter_space.parquet"
df_sklearn = pd.read_parquet(PARQUET_PATH)
# =====================
# Basic preprocessing
# =====================
MODEL_COLS = {
"NICL": "acc_NICL",
"TABPFN": "acc_TABPFN",
"TABICL": "acc_TABICL",
"RF": "acc_RF",
}
# =====================
# Slider ranges
# =====================
min_features, max_features = int(df_sklearn["n_features"].min()), int(
df_sklearn["n_features"].max()
)
min_samples, max_samples = int(df_sklearn["n_samples"].min()), int(
df_sklearn["n_samples"].max()
)
min_classes, max_classes = int(df_sklearn["n_classes"].min()), int(
df_sklearn["n_classes"].max()
)
min_sep, max_sep = float(df_sklearn["class_sep"].min()), float(
df_sklearn["class_sep"].max()
)
# =====================
# Filtering + aggregation
# =====================
def filter_and_aggregate(
f_min,
f_max,
s_min,
s_max,
c_min,
c_max,
sep_min,
sep_max,
):
# sécurité si sliders se croisent
if f_min > f_max or s_min > s_max or c_min > c_max or sep_min > sep_max:
return (
pd.DataFrame(columns=["Model", "Accuracy"]),
"⚠️ Invalid interval selection",
)
filtered = df_sklearn[
df_sklearn["n_features"].between(f_min, f_max)
& df_sklearn["n_samples"].between(s_min, s_max)
& df_sklearn["n_classes"].between(c_min, c_max)
& df_sklearn["class_sep"].between(sep_min, sep_max)
]
n_datasets = len(filtered)
if filtered.empty:
return (
pd.DataFrame(columns=["Model", "Accuracy"]),
"⚠️ No datasets match the current filters.",
)
rows = []
for model, col in MODEL_COLS.items():
rows.append(
{
"Model": model,
"Accuracy": filtered[col].mean(),
}
)
out = highlight_max(pd.DataFrame(rows))
return out, f"📦 **{n_datasets} datasets** considered"
gr.Markdown("## 📊 Average Model Accuracy by Dataset Properties")
with gr.Row():
with gr.Column():
features_min = gr.Slider(
min_features,
max_features,
value=min_features,
step=1,
label="Min number of features",
)
features_max = gr.Slider(
min_features,
max_features,
value=max_features,
step=1,
label="Max number of features",
)
with gr.Column():
samples_min = gr.Slider(
min_samples,
max_samples,
value=min_samples,
step=10,
label="Min number of samples",
)
samples_max = gr.Slider(
min_samples,
max_samples,
value=max_samples,
step=10,
label="Max number of samples",
)
with gr.Column():
classes_min = gr.Slider(
min_classes,
max_classes,
value=min_classes,
step=1,
label="Min number of classes",
)
classes_max = gr.Slider(
min_classes,
max_classes,
value=max_classes,
step=1,
label="Max number of classes",
)
with gr.Column():
sep_min = gr.Slider(
min_sep,
max_sep,
value=min_sep,
step=0.01,
label="Min class separation",
)
sep_max = gr.Slider(
min_sep,
max_sep,
value=max_sep,
step=0.01,
label="Max class separation",
)
perf_table = gr.DataFrame(
interactive=False,
type="pandas",
wrap=True,
)
count_md = gr.Markdown("")
controls = [
features_min,
features_max,
samples_min,
samples_max,
classes_min,
classes_max,
sep_min,
sep_max,
]
for c in controls:
c.change(
fn=filter_and_aggregate,
inputs=controls,
outputs=[perf_table, count_md],
)
def show_content(visible_content, selected_btn):
content_blocks = [
home_intro,
public_content,
private_content,
datamap_content,
]
btns = [
public_btn_home,
home_btn_public,
home_btn_private,
home_btn_datamap,
]
updates = {
block: gr.update(visible=(block == visible_content))
for block in content_blocks
}
for btn in btns:
if btn == selected_btn:
updates[btn] = gr.update(elem_classes=["white-btn", "selected-btn"])
else:
updates[btn] = gr.update(elem_classes=["white-btn"])
return updates
def show_home():
return show_content(home_intro, public_btn_home)
def show_public():
return show_content(public_content, home_btn_public)
def show_private():
return show_content(private_content, home_btn_private)
def show_datamap():
return show_content(datamap_content, home_btn_datamap)
outputs = [
home_intro,
public_content,
private_content,
datamap_content,
public_btn_home,
home_btn_public,
home_btn_private,
home_btn_datamap,
]
def _push(pane):
return (
"() => { if (window.__nk_skip_push) { window.__nk_skip_push = false; return; }"
f" history.pushState({{pane: '{pane}'}}, '', '#{pane}'); }}"
)
home_btn_public.click(fn=show_home, inputs=None, outputs=outputs, js=_push("home"))
home_btn_private.click(fn=show_home, inputs=None, outputs=outputs, js=_push("home"))
home_btn_datamap.click(fn=show_home, inputs=None, outputs=outputs, js=_push("home"))
public_btn_home.click(
fn=show_public, inputs=None, outputs=outputs, js=_push("academic")
)
private_btn_home.click(
fn=show_private, inputs=None, outputs=outputs, js=_push("enterprise")
)
datamap_btn_home.click(
fn=show_datamap, inputs=None, outputs=outputs, js=_push("datamap")
)
# On app load: install popstate listener that re-dispatches a click on the
# appropriate nav button when the user hits Back/Forward. Also, if the URL
# already has a #hash on first load, navigate to that pane.
demo.launch(ssr_mode=False, share=False, allowed_paths=["static"])