diff --git "a/app.py" "b/app.py"
--- "a/app.py"
+++ "b/app.py"
@@ -1,13 +1,16 @@
from __future__ import annotations
+import json
import re
from pathlib import Path
-from typing import Iterable
+from typing import Iterable, Sequence
import numpy as np
import pandas as pd
import plotly.graph_objects as go
from dash import Dash, Input, Output, State, dash_table, dcc, html
+from flask import abort, send_from_directory
+from plotly.colors import sample_colorscale
from plotly.subplots import make_subplots
@@ -59,128 +62,131 @@ DISPLAY_NAMES = {
"pca": "PCA",
"rnn": "RNN",
"smc_rnns": "SMC-RNN",
- "svc": "SVC",
+ "svc": "SVM/SVR",
"tndm": "TNDM",
"torchdfine": "DFINE",
"xg": "XGBoost",
}
-METHOD_FAMILY = {
- "pca": "Linear latent",
- "gpfa": "Linear latent",
- "dnn": "Supervised decoder",
- "gru": "Supervised decoder",
- "lstm": "Supervised decoder",
- "rnn": "Supervised decoder",
- "svc": "Supervised decoder",
- "xg": "Supervised decoder",
- "lfads_torch": "Sequential latent",
- "dpad": "Sequential latent",
- "torchdfine": "Sequential latent",
- "smc_rnns": "Sequential latent",
- "tndm": "Sequential latent",
- "langevinflow_ccn": "Sequential latent",
- "ldns": "Sequential latent",
- "neuro_behavior_conditioning": "Sequential latent",
- "blend": "Distillation",
- "blend_ndt": "Distillation",
- "neds": "Foundation model",
- "neds_pretrained": "Foundation model",
- "cebra": "Contrastive",
- "marble": "Geometric",
- "mint": "Non-parametric",
+DATASET_LABELS = {
+ "monkey": "Macaque center-out reaching",
+ "allen_neuropixels": "Allen Neuropixels visual coding",
+ "speech": "Attempted speech",
+ "mc_pacman": "MC PacMan force decoding",
+ "ratinabox": "RatInABox navigation",
}
-METHOD_HARDWARE = {
- "gpfa": "CPU",
- "mint": "CPU",
- "pca": "CPU",
- "svc": "CPU",
- "xg": "CPU",
+DATASET_SHORT_LABELS = {
+ "monkey": "Macaque reaching",
+ "allen_neuropixels": "Allen visual coding",
+ "speech": "Attempted speech",
+ "mc_pacman": "MC PacMan",
+ "ratinabox": "RatInABox",
}
-TABLE_LABELS = {
- "rank": "Rank",
- "method": "Method",
- "family": "Family",
- "hardware": "Hardware",
- "task_score": "Decoding score",
- "score": "Decoding score",
- "robustness_auc": "Robustness AUC",
- "alignment_score": "Cross-session alignment",
- "training_time_sec": "Training time (s)",
- "inference_time_sec": "Inference time (s)",
- "peak_ram_gb": "Peak memory (GB)",
- "peak_vram_gb": "Peak GPU memory (GB)",
- "reference_score": "Reference score",
- "highest_noise_score": "Highest-noise score",
- "average_noisy_score": "Average noisy score",
- "latent_dim": "Latent dimensions",
- "n_sessions": "Sessions",
- "n_pairwise": "Session pairs",
- "baseline_score": "Baseline score",
- "full_model_score": "Full-model score",
- "neuron_influence_auc": "Neuron influence AUC",
- "trial_influence_auc": "Trial influence AUC",
- "shap_mean_value": "Mean signed neuron contribution",
- "shap_fraction_positive": "Fraction positive",
- "n_train_trials": "Training trials",
- "n_test_trials": "Test trials",
- "n_neurons": "Neurons",
+DATASET_DESCRIPTIONS = {
+ "monkey": "Two-dimensional hand-position regression from macaque neural population activity.",
+ "allen_neuropixels": "Eight-class drifting-grating orientation classification from Allen Neuropixels units.",
+ "speech": "Eight-class attempted-word classification from threshold-crossing features.",
+ "mc_pacman": "Continuous force regression from motor-cortical population activity.",
+ "ratinabox": "Two-dimensional position regression from simulated place, head-direction and speed cells.",
}
-NUMERIC_COLUMNS = {
- "task_score",
- "score",
- "robustness_auc",
- "alignment_score",
- "training_time_sec",
- "inference_time_sec",
- "peak_ram_gb",
- "peak_vram_gb",
- "reference_score",
- "highest_noise_score",
- "average_noisy_score",
- "latent_dim",
- "n_sessions",
- "n_pairwise",
- "baseline_score",
- "full_model_score",
- "neuron_influence_auc",
- "trial_influence_auc",
- "shap_mean_value",
- "shap_fraction_positive",
- "n_train_trials",
- "n_test_trials",
- "n_neurons",
-}
-RIGHT_ALIGNED_COLUMNS = NUMERIC_COLUMNS | {"rank"}
+DATASETS = list(DATASET_LABELS)
+MODELS = PAPER_MODEL_ORDER.copy()
+MODEL_INDEX = {model: index for index, model in enumerate(MODELS)}
-# Shared figure colors copied from the paper plotting scripts.
-TASK_COLOR = "#1565C0"
+# Figure 2 color semantics.
+PREDICTION_COLOR = "#1565C0"
ROBUSTNESS_COLOR = "#2E7D32"
COMPUTE_COLOR = "#E65100"
-INFLUENCE_COLOR = "#CC79A7"
-ALIGNMENT_COLOR = "#0072B2"
-SCORE_SCALE = [[0.0, "#EFF6FF"], [1.0, TASK_COLOR]]
-ALIGNMENT_SCALE = [[0.0, "#F7FBF7"], [1.0, ROBUSTNESS_COLOR]]
-DATASET_COLORS = {
- "monkey": "#0072B2",
- "allen_neuropixels": "#E69F00",
- "speech": "#009E73",
- "mc_pacman": "#CC79A7",
- "ratinabox": "#D55E00",
+# Figure 3 uses a green-blue scale; Figures 4 and 5 use purple scales.
+CONSISTENCY_COLOR = "#007C91"
+FEATURE_COLOR = "#6A51A3"
+TRIAL_COLOR = "#6A1B9A"
+NEGATIVE_COLOR = "#B35806"
+TEXT_COLOR = "#17202A"
+MUTED_COLOR = "#607080"
+GRID_COLOR = "#E8EDF1"
+
+PREDICTION_SCALE = [[0.0, "#F3F8FD"], [1.0, PREDICTION_COLOR]]
+CONSISTENCY_SCALE = [[0.0, "#F1FAF8"], [1.0, CONSISTENCY_COLOR]]
+FEATURE_SCALE = [[0.0, "#F7F2FA"], [1.0, FEATURE_COLOR]]
+TRIAL_SCALE = [[0.0, "#F8F2FA"], [1.0, TRIAL_COLOR]]
+
+MODEL_COLORS = {
+ model: color
+ for model, color in zip(
+ MODELS,
+ sample_colorscale("Turbo", np.linspace(0.04, 0.96, len(MODELS))),
+ )
}
-CATEGORICAL_PALETTE = [
- "#E69F00",
- "#56B4E9",
- "#009E73",
- "#F0E442",
- "#0072B2",
- "#D55E00",
- "#CC79A7",
- "#000000",
-]
+
+CPU_ONLY_MODELS = {"gpfa", "mint", "pca", "svc", "xg"}
+
+CONSISTENCY_ELIGIBLE = {
+ "blend",
+ "cebra",
+ "dpad",
+ "gpfa",
+ "ldns",
+ "lfads_torch",
+ "marble",
+ "neuro_behavior_conditioning",
+ "pca",
+ "smc_rnns",
+ "tndm",
+ "torchdfine",
+}
+FEATURE_ELIGIBLE = set(MODELS) - {"marble"}
+TRIAL_ELIGIBLE = {
+ "blend",
+ "blend_ndt",
+ "cebra",
+ "dnn",
+ "gpfa",
+ "gru",
+ "langevinflow_ccn",
+ "ldns",
+ "lfads_torch",
+ "lstm",
+ "marble",
+ "neds",
+ "neds_pretrained",
+ "pca",
+ "rnn",
+ "smc_rnns",
+ "tndm",
+}
+
+CONDITION_LABELS = {
+ "monkey": {
+ "0": "90°",
+ "1": "45°",
+ "2": "0°",
+ "3": "315°",
+ "4": "270°",
+ "5": "225°",
+ "6": "180°",
+ "7": "135°",
+ },
+ "allen_neuropixels": {
+ str(index): f"{angle}°"
+ for index, angle in enumerate([0, 45, 90, 135, 180, 225, 270, 315])
+ },
+ "speech": {
+ "0": "Do nothing",
+ "1": "ban",
+ "2": "choice",
+ "3": "day",
+ "4": "feel",
+ "5": "kite",
+ "6": "though",
+ "7": "were",
+ },
+}
+
+# Exact reach-direction palette used by Figure 3.
DIRECTION_PALETTE = [
"#B23AEE",
"#3B1C54",
@@ -191,6 +197,8 @@ DIRECTION_PALETTE = [
"#5B8FF9",
"#F97316",
]
+DIRECTION_LEGEND_ORDER = [2, 1, 0, 7, 6, 5, 4, 3]
+DIRECTION_LEGEND_LABELS = ["0°", "45°", "90°", "135°", "180°", "225°", "270°", "315°"]
SPEECH_PALETTE = {
"3": "#E69F00",
"2": "#56B4E9",
@@ -201,6 +209,12 @@ SPEECH_PALETTE = {
"5": "#CC79A7",
"0": "#000000",
}
+ALLEN_PALETTE = {
+ str(index): color
+ for index, color in enumerate(
+ ["#E69F00", "#56B4E9", "#009E73", "#F0E442", "#0072B2", "#D55E00", "#CC79A7", "#000000"]
+ )
+}
RATINABOX_SCALE = [
[0.0, "#440154"],
[0.25, "#3B528B"],
@@ -209,6 +223,134 @@ RATINABOX_SCALE = [
[1.0, "#FDE725"],
]
+TABLE_LABELS = {
+ "method": "Method",
+ "workflow": "Prediction workflow",
+ "readout": "Prediction/readout implementation",
+ "hardware": "Primary hardware",
+ "prediction_status": "Prediction",
+ "robustness_status": "Robustness",
+ "compute_status": "Computational cost",
+ "consistency_status": "Latent consistency",
+ "feature_status": "Feature attribution",
+ "trial_status": "Trial valuation",
+ "coverage_notes": "Coverage notes",
+ "task_score": "Held-out task score",
+ "prediction_percentile": "Within-dataset percentile",
+ "robustness_auc": "Raw score-vs-noise AUC",
+ "training_time_sec": "Training time (s)",
+ "inference_time_sec": "Inference time (s)",
+ "peak_ram_gb": "Peak RAM (GB)",
+ "peak_vram_gb": "Peak GPU memory (GB)",
+ "unperturbed_score": "Unperturbed score",
+ "highest_noise_score": "Score at λ = 0.8",
+ "average_noisy_score": "Mean score across λ",
+ "latent_consistency_r2": "Latent-consistency R²",
+ "latent_dim": "Latent dimensions",
+ "n_recordings": "Recordings",
+ "n_pairwise": "Directional pairs",
+ "validation_target": "Validation target",
+ "validation_metric": "Validation metric",
+ "validation_score": "Validation score",
+ "shap_mean_value": "Mean signed Kernel SHAP",
+ "shap_median_value": "Median signed Kernel SHAP",
+ "shap_min_value": "Minimum signed Kernel SHAP",
+ "shap_max_value": "Maximum signed Kernel SHAP",
+ "shap_fraction_positive": "Fraction positive",
+ "shap_fraction_negative": "Fraction negative",
+ "corrupted_trial_auc": "Corrupted-trial ROC-AUC",
+ "iterations": "TMC permutations",
+ "converged": "Converged",
+ "final_error": "Final convergence error",
+ "perturbation_fraction": "Rotated-trial fraction",
+ "rotation_angle_deg": "Rotation angle (degrees)",
+ "rotation_subspace_dim_spec": "Rotation subspace",
+ "shapley_mean_value": "Mean signed Data Shapley",
+ "shapley_median_value": "Median signed Data Shapley",
+ "shapley_min_value": "Minimum signed Data Shapley",
+ "shapley_max_value": "Maximum signed Data Shapley",
+ "shapley_fraction_positive": "Fraction positive",
+ "shapley_fraction_negative": "Fraction negative",
+ "mixed_full": "Mixed trials",
+ "data_shapley": "After trial-value removal",
+ "oracle": "Oracle removal",
+ "recovery": "Recovery ΔR²",
+ "target_only": "Current session only",
+ "all_sessions": "All-session pooling",
+ "historical_selected": "Trial-value historical selection",
+}
+
+NUMERIC_COLUMNS = {
+ "task_score",
+ "prediction_percentile",
+ "robustness_auc",
+ "training_time_sec",
+ "inference_time_sec",
+ "peak_ram_gb",
+ "peak_vram_gb",
+ "unperturbed_score",
+ "highest_noise_score",
+ "average_noisy_score",
+ "latent_consistency_r2",
+ "latent_dim",
+ "n_recordings",
+ "n_pairwise",
+ "validation_score",
+ "shap_mean_value",
+ "shap_median_value",
+ "shap_min_value",
+ "shap_max_value",
+ "shap_fraction_positive",
+ "shap_fraction_negative",
+ "corrupted_trial_auc",
+ "iterations",
+ "final_error",
+ "perturbation_fraction",
+ "rotation_angle_deg",
+ "shapley_mean_value",
+ "shapley_median_value",
+ "shapley_min_value",
+ "shapley_max_value",
+ "shapley_fraction_positive",
+ "shapley_fraction_negative",
+ "mixed_full",
+ "data_shapley",
+ "oracle",
+ "recovery",
+ "target_only",
+ "all_sessions",
+ "historical_selected",
+}
+
+DOWNLOADABLE_FILES = {
+ "clean_prediction_summary.csv",
+ "robustness_summary.csv",
+ "consistency_summary.csv",
+ "scalability_summary.csv",
+ "neuron_shap_summary.csv",
+ "trial_shapley_summary.csv",
+ "trial_shapley_retrain_summary.csv",
+ "trial_historical_trajectories.csv",
+}
+
+TRIAL_HISTORICAL_TRAJECTORY_COLUMNS = [
+ "model",
+ "target_session",
+ "trial_index",
+ "trial_id",
+ "direction_index",
+ "direction_label",
+ "time_index",
+ "target_x",
+ "target_y",
+ "current_only_x",
+ "current_only_y",
+ "historical_selected_x",
+ "historical_selected_y",
+ "current_only_r2",
+ "historical_selected_r2",
+]
+
def load_csv(name: str) -> pd.DataFrame:
path = DATA_DIR / name
@@ -217,423 +359,269 @@ def load_csv(name: str) -> pd.DataFrame:
return pd.read_csv(path)
+def load_historical_trajectories() -> pd.DataFrame:
+ frame = load_csv("trial_historical_trajectories.csv")
+ if list(frame.columns) != TRIAL_HISTORICAL_TRAJECTORY_COLUMNS:
+ raise ValueError(
+ "trial_historical_trajectories.csv has an invalid schema. "
+ f"Expected exactly: {TRIAL_HISTORICAL_TRAJECTORY_COLUMNS}"
+ )
+ if frame.empty:
+ raise ValueError("trial_historical_trajectories.csv must contain the Figure 5e RNN example.")
+ if set(frame["model"].astype(str)) != {"rnn"}:
+ raise ValueError("trial_historical_trajectories.csv must contain only the Figure 5e RNN example.")
+ numeric_columns = [
+ "trial_index",
+ "direction_index",
+ "time_index",
+ "target_x",
+ "target_y",
+ "current_only_x",
+ "current_only_y",
+ "historical_selected_x",
+ "historical_selected_y",
+ "current_only_r2",
+ "historical_selected_r2",
+ ]
+ for column in numeric_columns:
+ values = pd.to_numeric(frame[column], errors="coerce")
+ if values.isna().any():
+ raise ValueError(
+ f"trial_historical_trajectories.csv contains a non-numeric or missing {column} value."
+ )
+ if not np.isfinite(values.to_numpy(dtype=float)).all():
+ raise ValueError(
+ f"trial_historical_trajectories.csv contains a non-finite {column} value."
+ )
+ frame[column] = values
+ integer_columns = ["trial_index", "direction_index", "time_index"]
+ for column in integer_columns:
+ if not np.allclose(frame[column], np.round(frame[column])):
+ raise ValueError(f"trial_historical_trajectories.csv requires integer {column} values.")
+ frame[column] = frame[column].astype(int)
+ if not frame["direction_index"].between(0, 7).all():
+ raise ValueError("trial_historical_trajectories.csv direction_index values must be in [0, 7].")
+ if set(frame["direction_index"]) != set(DIRECTION_LEGEND_ORDER):
+ raise ValueError("trial_historical_trajectories.csv must include all eight reach directions.")
+ if frame["target_session"].astype(str).nunique() != 1:
+ raise ValueError("trial_historical_trajectories.csv must contain one target session.")
+ if frame["current_only_r2"].nunique() != 1 or frame["historical_selected_r2"].nunique() != 1:
+ raise ValueError("Figure 5e R² values must be constant across trajectory rows.")
+ trial_metadata = frame.groupby("trial_index").agg(
+ trial_ids=("trial_id", "nunique"),
+ directions=("direction_index", "nunique"),
+ direction_labels=("direction_label", "nunique"),
+ )
+ if (trial_metadata != 1).any().any():
+ raise ValueError("Each Figure 5e trial must have one ID and one reach-direction label.")
+ if frame.duplicated(["trial_index", "time_index"]).any():
+ raise ValueError("Figure 5e contains duplicate trial/time rows.")
+ return frame.sort_values(["direction_index", "trial_index", "time_index"]).reset_index(drop=True)
+
+
+def load_release_manifest() -> dict:
+ path = DATA_DIR / "release_manifest.json"
+ if not path.exists():
+ return {}
+ try:
+ payload = json.loads(path.read_text(encoding="utf-8"))
+ except (OSError, json.JSONDecodeError):
+ return {}
+ return payload if isinstance(payload, dict) else {}
+
+
prediction = load_csv("clean_prediction_summary.csv")
robustness = load_csv("robustness_summary.csv")
consistency = load_csv("consistency_summary.csv")
scalability = load_csv("scalability_summary.csv")
neuron_shap = load_csv("neuron_shap_summary.csv")
trial_shapley = load_csv("trial_shapley_summary.csv")
+trial_retrain = load_csv("trial_shapley_retrain_summary.csv")
+trial_historical_trajectories = load_historical_trajectories()
latent_samples = load_csv("latent_samples.csv")
latent_trajectories = load_csv("latent_trajectories.csv")
+release_manifest = load_release_manifest()
def present_rows(df: pd.DataFrame) -> pd.DataFrame:
if df.empty or "status" not in df.columns:
return df.copy()
- return df[df["status"].fillna("") == "present"].copy()
+ return df[df["status"].fillna("").astype(str).str.lower() == "present"].copy()
def active_rows(df: pd.DataFrame) -> pd.DataFrame:
if df.empty or "is_active_model" not in df.columns:
return df.copy()
- return df[df["is_active_model"].astype(str).str.lower() == "true"].copy()
-
-
-def ordered_unique(values: Iterable[object]) -> list[str]:
- seen: set[str] = set()
- out: list[str] = []
- for value in values:
- if pd.isna(value):
- continue
- text = str(value)
- if text not in seen:
- seen.add(text)
- out.append(text)
- return out
-
-
-def build_dataset_labels() -> dict[str, str]:
- pairs = (
- prediction[["dataset", "dataset_display"]]
- .dropna(subset=["dataset"])
- .drop_duplicates(subset=["dataset"])
- )
- return {row.dataset: row.dataset_display for row in pairs.itertuples()}
-
-
-DATASET_LABELS = build_dataset_labels()
-DATASETS = ordered_unique(prediction.get("dataset", pd.Series(dtype=str)))
-CONSISTENCY_DATASETS = set(
- active_rows(consistency).get("dataset", pd.Series(dtype=str)).dropna().astype(str)
-)
-MODEL_SET = set(prediction.get("model", pd.Series(dtype=str)).dropna().astype(str))
-MODELS = [model for model in PAPER_MODEL_ORDER if model in MODEL_SET]
-MODELS += sorted(model for model in MODEL_SET if model not in set(MODELS))
-MODEL_RANK = {model: idx for idx, model in enumerate(MODELS)}
-
-CONDITION_LABELS = {
- "monkey": {
- "0": "Up",
- "1": "Up-right",
- "2": "Right",
- "3": "Down-right",
- "4": "Down",
- "5": "Down-left",
- "6": "Left",
- "7": "Up-left",
- },
- "allen_neuropixels": {
- str(i): f"{angle} deg"
- for i, angle in enumerate([0, 45, 90, 135, 180, 225, 270, 315])
- },
- "speech": {
- "0": "Do nothing",
- "1": "ban",
- "2": "choice",
- "3": "day",
- "4": "feel",
- "5": "kite",
- "6": "though",
- "7": "were",
- },
-}
+ mask = df["is_active_model"].fillna(False).astype(str).str.lower().isin({"true", "1", "yes"})
+ return df[mask].copy()
def model_label(model: object) -> str:
- if pd.isna(model):
+ if model is None or pd.isna(model):
return ""
- text = str(model)
- return DISPLAY_NAMES.get(text, text)
-
-
-def model_sort_value(model: object) -> int:
- return MODEL_RANK.get(str(model), len(MODEL_RANK))
+ return DISPLAY_NAMES.get(str(model), str(model))
-def hardware_label(model: object) -> str:
- return METHOD_HARDWARE.get(str(model), "GPU")
+def dataset_model_label(model: object, dataset: str) -> str:
+ if str(model) == "svc":
+ return "SVM" if dataset in {"allen_neuropixels", "speech"} else "SVR"
+ return model_label(model)
-def selected_models(models: list[str] | None) -> list[str]:
+def selected_models(models: Sequence[str] | None) -> list[str]:
if not models:
return MODELS.copy()
- return [model for model in MODELS if model in set(models)]
+ chosen = set(models)
+ return [model for model in MODELS if model in chosen]
-def filter_models(df: pd.DataFrame, models: list[str] | None) -> pd.DataFrame:
+def filter_models(df: pd.DataFrame, models: Sequence[str] | None) -> pd.DataFrame:
if df.empty or "model" not in df.columns:
return df.copy()
return df[df["model"].astype(str).isin(selected_models(models))].copy()
-def condition_sort_key(value: object) -> tuple[int, float | str]:
- try:
- return (0, float(value))
- except (TypeError, ValueError):
- return (1, str(value))
-
-
-def condition_label(dataset: str, condition: object) -> str:
- if pd.isna(condition):
- return "Unknown"
- text = str(condition)
- mapped = CONDITION_LABELS.get(dataset, {}).get(text)
- if mapped is not None:
- return mapped
- if dataset == "ratinabox":
- try:
- idx = int(float(text))
- return f"x{idx % 10}, y{idx // 10}"
- except ValueError:
- return text
- return text
-
-
-def condition_axis_label(dataset: str, color_mode: str = "condition") -> str:
- if dataset == "ratinabox":
- if color_mode == "x":
- return "X position bin"
- if color_mode == "y":
- return "Y position bin"
- return "Position bin"
- return {
- "monkey": "Reach direction",
- "allen_neuropixels": "Orientation",
- "speech": "Cue",
- }.get(dataset, "Condition")
-
-
-def latent_color_label(dataset: str, color_mode: str, value: object) -> str:
- if pd.isna(value):
- return "Unknown"
- if dataset == "ratinabox":
- try:
- idx = int(float(value))
- except ValueError:
- return str(value)
- if color_mode == "x":
- return f"x{idx}"
- if color_mode == "y":
- return f"y{idx}"
- return condition_label(dataset, value)
-
-
-def add_latent_color_columns(df: pd.DataFrame, dataset: str, color_mode: str) -> pd.DataFrame:
- out = df.copy()
- condition_num = pd.to_numeric(out["condition"], errors="coerce")
- if condition_num.isna().any() or (condition_num < 0).any():
- raise ValueError("Latent samples contain missing condition labels.")
-
- if dataset == "ratinabox" and color_mode == "x":
- values = (condition_num.astype(int) % 10).astype(str)
- elif dataset == "ratinabox" and color_mode == "y":
- values = (condition_num.astype(int) // 10).astype(str)
- else:
- values = condition_num.astype(int).astype(str)
-
- out["color_value"] = values
- out["color_num"] = pd.to_numeric(values, errors="coerce")
- out["color_label"] = out["color_value"].map(lambda value: latent_color_label(dataset, color_mode, value))
- return out
-
-
-def session_display_label(dataset: str, session: object) -> str:
- text = "" if pd.isna(session) else str(session)
- if dataset == "monkey":
- match = re.search(r"sub-([A-Za-z])_ses-CO-(\d{4})(\d{2})(\d{2})", text)
- if match:
- monkey, year, month, day = match.groups()
- return f"Monkey {monkey}, {year}-{month}-{day}"
- if dataset == "ratinabox":
- match = re.search(r"s(\d+)", text)
- if match:
- return f"Run {match.group(1)}"
- if dataset == "speech":
- return f"Session {text}"
- if dataset == "allen_neuropixels":
- return f"Session {text}"
- return text or "Session"
-
-
def add_method_columns(df: pd.DataFrame) -> pd.DataFrame:
out = df.copy()
if "model" in out.columns:
out["method"] = out["model"].map(model_label)
- out["model_order"] = out["model"].map(model_sort_value)
- return out
-
-
-def round_numeric(df: pd.DataFrame, columns: Iterable[str], digits: int = 3) -> pd.DataFrame:
- out = df.copy()
- for col in columns:
- if col in out.columns:
- out[col] = pd.to_numeric(out[col], errors="coerce").round(digits)
+ out["model_order"] = out["model"].map(MODEL_INDEX).fillna(len(MODELS)).astype(int)
return out
-def sort_table(df: pd.DataFrame, sort_by: list[dict] | None, default: list[tuple[str, bool]]) -> pd.DataFrame:
- if sort_by:
- sort_spec = []
- for item in sort_by:
- col = item.get("column_id")
- if col in df.columns:
- sort_spec.append((col, item.get("direction") == "asc"))
- if sort_spec:
- return df.sort_values(
- [col for col, _ in sort_spec],
- ascending=[ascending for _, ascending in sort_spec],
- na_position="last",
- )
- return df.sort_values(
- [col for col, _ in default],
- ascending=[ascending for _, ascending in default],
- na_position="last",
- )
-
-
-def supported_sort(sort_by: list[dict] | None, columns: Iterable[str]) -> list[dict] | None:
- allowed = set(columns)
- if not sort_by:
- return None
- filtered = [item for item in sort_by if item.get("column_id") in allowed]
- return filtered or None
-
-
def records(df: pd.DataFrame) -> list[dict]:
clean = df.astype(object).where(pd.notna(df), None)
return clean.to_dict("records")
-def column_defs(columns: Iterable[str]) -> list[dict]:
- out = []
- for col in columns:
- item = {"name": TABLE_LABELS.get(col, col), "id": col}
- if col in RIGHT_ALIGNED_COLUMNS:
- item["type"] = "numeric"
- out.append(item)
+def round_numeric(df: pd.DataFrame, digits: int = 4) -> pd.DataFrame:
+ out = df.copy()
+ for column in NUMERIC_COLUMNS.intersection(out.columns):
+ out[column] = pd.to_numeric(out[column], errors="coerce").round(digits)
return out
-def metric_text(value: object) -> str:
- if pd.isna(value):
- return "score"
- return str(value)
-
-
-def decoding_label(metric: object) -> str:
- text = metric_text(metric)
- if text.lower() == "r2":
- return "Decoding R2"
- if text.lower() == "accuracy":
- return "Decoding accuracy"
- return f"Decoding {text}"
-
-
-def leaderboard_columns(columns: Iterable[str], metric: object) -> list[dict]:
- defs = column_defs(columns)
- label = decoding_label(metric)
- for item in defs:
- if item["id"] == "task_score":
- item["name"] = label
- return defs
-
-
-def sort_status(
- sort_by: list[dict] | None,
- default: tuple[str, bool],
- labels: dict[str, str] | None = None,
-) -> str:
- column, ascending = default
- if sort_by:
- for item in sort_by:
- candidate = item.get("column_id")
- if candidate:
- column = candidate
- ascending = item.get("direction") == "asc"
- break
-
- merged_labels = dict(TABLE_LABELS)
- if labels:
- merged_labels.update(labels)
- label = merged_labels.get(column, column)
-
- if column in {"method", "family", "hardware"}:
- direction = "A to Z" if ascending else "Z to A"
- else:
- direction = "low to high" if ascending else "high to low"
- return f"Sorted by {label}: {direction}"
+def metric_name(metric: object, *, held_out: bool = False) -> str:
+ text = "" if metric is None or pd.isna(metric) else str(metric).lower()
+ prefix = "Held-out " if held_out else ""
+ if text == "r2":
+ return f"{prefix}R²"
+ if text == "accuracy":
+ return f"{prefix}accuracy"
+ return f"{prefix}{text or 'task score'}"
def value_text(value: object, digits: int = 3) -> str:
if value is None or pd.isna(value):
- return "Not available"
+ return "Unavailable"
number = float(value)
if abs(number) >= 1000:
return f"{number:,.0f}"
return f"{number:.{digits}f}"
-def fig_layout(fig: go.Figure, *, height: int = 420) -> go.Figure:
- fig.update_layout(
- height=height,
- paper_bgcolor="#ffffff",
- plot_bgcolor="#ffffff",
- margin=dict(l=22, r=22, t=52, b=38),
- font=dict(family="Inter, Arial, sans-serif", size=13, color="#17202a"),
- title=dict(font=dict(size=15, color="#17202a")),
- legend=dict(orientation="h", yanchor="bottom", y=1.02, xanchor="left", x=0),
- )
- fig.update_xaxes(showgrid=True, gridcolor="#edf1f4", zerolinecolor="#d7e0e6")
- fig.update_yaxes(showgrid=True, gridcolor="#edf1f4", zerolinecolor="#d7e0e6")
- return fig
-
-
-def empty_figure(message: str) -> go.Figure:
- fig = go.Figure()
- fig.add_annotation(
- text=message,
- x=0.5,
- y=0.5,
- xref="paper",
- yref="paper",
- showarrow=False,
- font=dict(size=15, color="#637381"),
- )
- fig.update_xaxes(visible=False)
- fig.update_yaxes(visible=False)
- return fig_layout(fig, height=360)
-
-
def parse_float_list(value: object) -> list[float]:
- if pd.isna(value):
+ if value is None or pd.isna(value):
return []
- out = []
+ values: list[float] = []
for part in str(value).split(";"):
- part = part.strip()
- if not part:
- continue
try:
- out.append(float(part))
+ values.append(float(part.strip()))
except ValueError:
- continue
- return out
+ raise ValueError(f"Malformed numeric sequence in dashboard data: {value!r}") from None
+ return values
+
+
+def aggregate_prediction_order() -> list[str]:
+ frame = prediction.copy()
+ frame["score"] = pd.to_numeric(frame["score"], errors="coerce")
+ grid = pd.MultiIndex.from_product([MODELS, DATASETS], names=["model", "dataset"]).to_frame(index=False)
+ values = frame[["model", "dataset", "score"]].drop_duplicates(["model", "dataset"])
+ grid = grid.merge(values, on=["model", "dataset"], how="left")
+ grid["rank"] = grid.groupby("dataset")["score"].rank(method="average", ascending=False)
+ grid["rank"] = grid["rank"].fillna(len(MODELS))
+ mean_rank = grid.groupby("model", as_index=False)["rank"].mean()
+ mean_rank["model_order"] = mean_rank["model"].map(MODEL_INDEX)
+ return mean_rank.sort_values(["rank", "model_order"])["model"].tolist()
+
+
+FIGURE_MODEL_ORDER = aggregate_prediction_order()
+
+
+def column_defs(columns: Iterable[str]) -> list[dict]:
+ definitions = []
+ for column in columns:
+ item = {"name": TABLE_LABELS.get(column, column), "id": column}
+ if column in NUMERIC_COLUMNS:
+ item["type"] = "numeric"
+ definitions.append(item)
+ return definitions
def dataframe_table(
table_id: str,
*,
- page_size: int = 8,
- max_height: str = "520px",
- sort_action: str = "native",
- sort_by: list[dict] | None = None,
+ page_size: int = 12,
+ max_height: str = "620px",
) -> dash_table.DataTable:
return dash_table.DataTable(
id=table_id,
columns=[],
data=[],
page_size=page_size,
- sort_action=sort_action,
- sort_mode="single",
- sort_by=sort_by or [],
- cell_selectable=True,
+ sort_action="native",
+ sort_mode="multi",
+ filter_action="native",
+ cell_selectable=False,
style_as_list_view=True,
fixed_rows={"headers": True},
+ tooltip_delay=250,
+ tooltip_duration=None,
style_table={"overflowX": "auto", "overflowY": "auto", "maxHeight": max_height},
style_header={
- "backgroundColor": "#f3f6f8",
+ "backgroundColor": "#F3F6F8",
"fontWeight": "700",
"border": "0",
- "borderBottom": "1px solid #cfd8df",
- "color": "#26323f",
+ "borderBottom": "1px solid #CFD8DF",
+ "color": "#26323F",
},
style_cell={
- "fontFamily": "Inter, Arial, sans-serif",
+ "fontFamily": "Arial, Helvetica, sans-serif",
"fontSize": "13px",
- "padding": "10px 12px",
+ "padding": "9px 11px",
"textAlign": "left",
- "minWidth": "90px",
- "maxWidth": "260px",
+ "minWidth": "92px",
+ "maxWidth": "300px",
"whiteSpace": "normal",
"height": "auto",
"border": "0",
- "borderBottom": "1px solid #edf1f4",
+ "borderBottom": "1px solid #EDF1F4",
},
style_cell_conditional=[
- {"if": {"column_id": col}, "textAlign": "right"} for col in RIGHT_ALIGNED_COLUMNS
+ {"if": {"column_id": column}, "textAlign": "right"}
+ for column in NUMERIC_COLUMNS
],
style_data_conditional=[
- {"if": {"row_index": "odd"}, "backgroundColor": "#fbfcfd"},
- {"if": {"state": "active"}, "backgroundColor": "#e5f3f2", "border": "1px solid #4c908b"},
+ {"if": {"row_index": "odd"}, "backgroundColor": "#FBFCFD"},
],
)
-def panel(title: str, *children, subtitle: str | None = None, className: str = "") -> html.Div:
- heading = [html.H2(title)]
+def panel(
+ title: str,
+ *children,
+ subtitle: str | None = None,
+ class_name: str = "",
+ eyebrow: str | None = None,
+) -> html.Section:
+ heading: list = []
+ if eyebrow:
+ heading.append(html.Div(eyebrow, className="section-eyebrow"))
+ heading.append(html.H2(title))
if subtitle:
heading.append(html.P(subtitle, className="panel-subtitle"))
- classes = "panel" if not className else f"panel {className}"
- return html.Div([html.Div(heading, className="panel-heading"), *children], className=classes)
+ classes = "panel" if not class_name else f"panel {class_name}"
+ return html.Section([html.Div(heading, className="panel-heading"), *children], className=classes)
def details_table(summary: str, table: dash_table.DataTable) -> html.Details:
@@ -643,678 +631,1824 @@ def details_table(summary: str, table: dash_table.DataTable) -> html.Details:
)
-def metric_card(label: str, value: str, detail: str | None = None) -> html.Div:
+def metric_card(label: str, value: str, detail: str | None = None, accent: str = "") -> html.Div:
children = [
html.Div(label, className="metric-label"),
html.Div(value, className="metric-value"),
]
if detail:
children.append(html.Div(detail, className="metric-detail"))
- return html.Div(children, className="metric-card")
+ classes = "metric-card" if not accent else f"metric-card metric-card-{accent}"
+ return html.Div(children, className=classes)
-def leaderboard_frame(dataset: str, models: list[str] | None) -> pd.DataFrame:
- chosen = selected_models(models)
- base = pd.DataFrame({"model": chosen})
- base = add_method_columns(base)
+def source_link(filename: str, label: str = "Download source CSV") -> html.A:
+ return html.A(
+ label,
+ href=f"/download/{filename}",
+ className="source-link",
+ target="_blank",
+ rel="noopener noreferrer",
+ )
- pred = filter_models(prediction, chosen)
- pred = pred[pred["dataset"].astype(str) == str(dataset)].copy()
- pred["score"] = pd.to_numeric(pred["score"], errors="coerce")
- pred = pred[["model", "metric", "score", "n_train_trials", "n_test_trials", "n_neurons", "latent_dim"]]
- pred = pred.rename(columns={"score": "task_score"})
- rob = filter_models(present_rows(robustness), chosen)
- rob = rob[rob["dataset"].astype(str) == str(dataset)].copy()
- rob["raw_auc"] = pd.to_numeric(rob["raw_auc"], errors="coerce")
- rob = rob[["model", "raw_auc"]].rename(columns={"raw_auc": "robustness_auc"})
-
- cons = filter_models(active_rows(consistency), chosen)
- cons = cons[cons["dataset"].astype(str) == str(dataset)].copy()
- cons["mean_r2"] = pd.to_numeric(cons["mean_r2"], errors="coerce")
- cons = cons[["model", "mean_r2"]].rename(columns={"mean_r2": "alignment_score"})
-
- scale = filter_models(present_rows(scalability), chosen)
- scale = scale[scale["dataset"].astype(str) == str(dataset)].copy()
- scale = scale[["model", "training_time_sec", "inference_time_sec", "peak_ram_gb", "peak_vram_gb"]]
- for col in ["training_time_sec", "inference_time_sec", "peak_ram_gb", "peak_vram_gb"]:
- scale[col] = pd.to_numeric(scale[col], errors="coerce")
-
- df = base.merge(pred, on="model", how="left")
- df = df.merge(rob, on="model", how="left")
- df = df.merge(cons, on="model", how="left")
- df = df.merge(scale, on="model", how="left")
- available = df["task_score"].notna()
- order = df.loc[available].sort_values(
- ["task_score", "model_order"], ascending=[False, True]
- ).index
- df["rank"] = None
- for rank, idx in enumerate(order, start=1):
- df.at[idx, "rank"] = rank
-
- df["id"] = df["model"]
- return round_numeric(df, NUMERIC_COLUMNS)
-
-
-def leaderboard_summary(dataset: str, table_df: pd.DataFrame) -> list[html.Div]:
- label = DATASET_LABELS.get(dataset, dataset)
- metric = metric_text(table_df["metric"].dropna().iloc[0]) if table_df["metric"].notna().any() else "score"
- available = sort_table(
- table_df, None, [("task_score", False), ("model_order", True)]
- ).dropna(subset=["task_score"]).head(3)
- cards = [metric_card("Dataset", label, f"Primary metric: {metric}")]
- for _, row in available.iterrows():
- cards.append(
- metric_card(
- f"Rank {int(row['rank'])}",
- str(row["method"]),
- f"{value_text(row['task_score'])} {metric}",
- )
- )
- if len(cards) == 1:
- cards.append(metric_card("Top method", "Not available"))
- return cards
+def graph_box(graph_id: str, label: str, *, class_name: str = "") -> html.Div:
+ classes = "graph-box" if not class_name else f"graph-box {class_name}"
+ return html.Div(
+ dcc.Graph(
+ id=graph_id,
+ config={
+ "displaylogo": False,
+ "responsive": True,
+ "toImageButtonOptions": {"format": "png", "scale": 2},
+ },
+ ),
+ className=classes,
+ role="region",
+ **{"aria-label": label},
+ )
-def performance_heatmap(dataset: str, models: list[str] | None) -> go.Figure:
- df = filter_models(prediction, models)
- if df.empty:
- return empty_figure("No decoding results are available.")
- df = add_method_columns(df)
- df["score"] = pd.to_numeric(df["score"], errors="coerce")
- df["dataset_label"] = df["dataset"].map(DATASET_LABELS).fillna(df["dataset"])
- pivot = df.pivot_table(
- index="method", columns="dataset_label", values="score", aggfunc="first"
- )
-
- selected_label = DATASET_LABELS.get(dataset, dataset)
- order_df = sort_table(
- leaderboard_frame(dataset, models),
- None,
- [("task_score", False), ("model_order", True)],
- )
- method_order = [m for m in order_df["method"] if m in set(pivot.index)]
- pivot = pivot.reindex(method_order)
- dataset_order = [DATASET_LABELS.get(ds, ds) for ds in DATASETS]
- pivot = pivot.reindex(columns=[label for label in dataset_order if label in pivot.columns])
- text = pivot.map(lambda x: "" if pd.isna(x) else f"{x:.3f}") if not pivot.empty else pivot
-
- zmin = min(0.0, float(np.nanmin(pivot.values))) if pivot.size and not np.isnan(pivot.values).all() else 0.0
- zmax = max(1.0, float(np.nanmax(pivot.values))) if pivot.size and not np.isnan(pivot.values).all() else 1.0
- fig = go.Figure(
- go.Heatmap(
- z=pivot.values if not pivot.empty else [[]],
- x=list(pivot.columns),
- y=list(pivot.index),
- text=text.values if not pivot.empty else [[]],
- texttemplate="%{text}",
- colorscale=SCORE_SCALE,
- zmin=zmin,
- zmax=zmax,
- colorbar=dict(title="Score", thickness=12),
- hovertemplate="Method=%{y}
Dataset=%{x}
Score=%{z:.4f}",
- )
+def figure_layout(
+ fig: go.Figure,
+ *,
+ height: int = 430,
+ legend_below: bool = False,
+) -> go.Figure:
+ legend = dict(
+ orientation="h",
+ yanchor="bottom",
+ y=1.02,
+ xanchor="left",
+ x=0,
+ font=dict(size=11),
)
- fig.update_layout(title=f"Decoding score matrix, sorted by {selected_label}")
- return fig_layout(fig, height=max(430, 26 * len(pivot.index) + 150))
-
+ if legend_below:
+ legend.update(yanchor="top", y=-0.18)
+ fig.update_layout(
+ height=height,
+ paper_bgcolor="#FFFFFF",
+ plot_bgcolor="#FFFFFF",
+ margin=dict(l=54, r=28, t=58, b=58 if not legend_below else 105),
+ font=dict(family="Arial, Helvetica, sans-serif", size=13, color=TEXT_COLOR),
+ title=dict(font=dict(size=16, color=TEXT_COLOR), x=0.01, xanchor="left"),
+ legend=legend,
+ hoverlabel=dict(font=dict(family="Arial, Helvetica, sans-serif", size=12)),
+ )
+ fig.update_xaxes(
+ showgrid=True,
+ gridcolor=GRID_COLOR,
+ zerolinecolor="#CDD6DD",
+ automargin=True,
+ )
+ fig.update_yaxes(
+ showgrid=True,
+ gridcolor=GRID_COLOR,
+ zerolinecolor="#CDD6DD",
+ automargin=True,
+ )
+ return fig
+
+
+def heatmap_layout(fig: go.Figure, *, height: int) -> go.Figure:
+ figure_layout(fig, height=height)
+ fig.update_layout(
+ margin=dict(l=54, r=28, t=126, b=58),
+ title=dict(y=0.985, yanchor="top", pad=dict(b=12)),
+ )
+ fig.update_xaxes(tickangle=-28)
+ return fig
+
+
+def empty_figure(message: str, *, height: int = 360) -> go.Figure:
+ fig = go.Figure()
+ fig.add_annotation(
+ text=message,
+ x=0.5,
+ y=0.5,
+ xref="paper",
+ yref="paper",
+ showarrow=False,
+ align="center",
+ font=dict(size=14, color=MUTED_COLOR),
+ )
+ fig.update_xaxes(visible=False)
+ fig.update_yaxes(visible=False)
+ return figure_layout(fig, height=height)
+
+
+def availability_note(available: int, configured: int, *, unsupported: int = 0) -> str:
+ message = f"Coverage: {available} of {configured} configured entries available"
+ if unsupported:
+ message += f"; {unsupported} selected method{'s' if unsupported != 1 else ''} outside this analysis"
+ return message if message.endswith(".") else message + "."
+
+
+def feature_coverage_note(dataset: str, models: Sequence[str] | None) -> str:
+ chosen = selected_models(models)
+ configured = [model for model in chosen if model in FEATURE_ELIGIBLE]
+ available = filter_models(active_rows(neuron_shap), chosen)
+ available = int(available["dataset"].astype(str).eq(str(dataset)).sum())
+ message = (
+ f"Main Figure 4 interface: {available} of {len(configured)} configured entries "
+ "available"
+ )
+ if "marble" in chosen:
+ message += f"; full attempted grid: {available} of {len(chosen)} available. "
+ if dataset == "allen_neuropixels":
+ message += (
+ "MARBLE exceeded the host-memory allocation before feature attribution."
+ )
+ else:
+ message += (
+ "MARBLE is outside the masked-input interface because masking changes its "
+ "transductive graph construction."
+ )
+ return message if message.endswith(".") else message + "."
+
+
+def intervention_coverage_note() -> str:
+ frame = active_rows(trial_retrain)
+ within = frame[frame["analysis"] == "within_session_cleaning"].pivot_table(
+ index="model", columns="condition", values="score", aggfunc="first"
+ )
+ historical = frame[
+ frame["analysis"] == "cross_session_old_trial_selection"
+ ].pivot_table(index="model", columns="condition", values="score", aggfunc="first")
+ removal_improved = int((within["data_shapley"] > within["mixed_full"]).sum())
+ historical_vs_current = int(
+ (historical["oldonly_dshap_negative_removal"] > historical["target_only"]).sum()
+ )
+ historical_vs_pooling = int(
+ (historical["oldonly_dshap_negative_removal"] > historical["all_sessions"]).sum()
+ )
+ return (
+ f"Full coverage: {len(within)} removal and {len(historical)} historical-selection "
+ f"entries. Removal improved {removal_improved}/{len(within)} methods "
+ f"(mean R² {within['mixed_full'].mean():.3f}→{within['data_shapley'].mean():.3f}); "
+ f"historical selection beat current-only training for "
+ f"{historical_vs_current}/{len(historical)} and pooling for "
+ f"{historical_vs_pooling}/{len(historical)} (means "
+ f"{historical['oldonly_dshap_negative_removal'].mean():.3f} vs "
+ f"{historical['target_only'].mean():.3f}/{historical['all_sessions'].mean():.3f}). "
+ "The removal comparison reuses the valuation split and is a controlled diagnostic, "
+ "not an independent generalization estimate."
+ )
+
+
+def row_exists(df: pd.DataFrame, model: str, dataset: str) -> bool:
+ if df.empty:
+ return False
+ return bool(
+ ((df["model"].astype(str) == model) & (df["dataset"].astype(str) == dataset)).any()
+ )
+
+
+def missing_reason(analysis: str, model: str, dataset: str) -> str:
+ specific = {
+ ("prediction", "marble", "allen_neuropixels"): "MARBLE graph/embedding construction exceeded the 192-GB host-memory allocation.",
+ ("prediction", "tndm", "speech"): "TNDM training returned non-finite relevant-prior/posterior KL values.",
+ ("prediction", "svc", "allen_neuropixels"): "Exact-kernel SVM fitting exceeded the 24-hour allocation.",
+ ("consistency", "marble", "allen_neuropixels"): "MARBLE graph/embedding construction exceeded the host-memory allocation.",
+ ("consistency", "tndm", "speech"): "TNDM training returned non-finite KL values before a representation was available.",
+ ("feature", "mint", "allen_neuropixels"): "The MATLAB-backed repeated masked-input procedure did not complete in the allocation.",
+ ("feature", "mint", "ratinabox"): "The MATLAB-backed repeated masked-input procedure did not complete in the allocation.",
+ ("feature", "svc", "allen_neuropixels"): "Exact-kernel SVM fitting exceeded the 24-hour allocation.",
+ ("feature", "svc", "mc_pacman"): "Repeated masked-input SVM/SVR evaluation exceeded the 24-hour allocation.",
+ ("feature", "svc", "ratinabox"): "Repeated masked-input SVM/SVR evaluation exceeded the 24-hour allocation.",
+ ("trial", "langevinflow_ccn", "allen_neuropixels"): "Repeated full-rank logistic coalition fits exceeded the execution allocation.",
+ ("trial", "marble", "allen_neuropixels"): "No Stage 1 representation was available after MARBLE exceeded host memory.",
+ ("trial", "tndm", "allen_neuropixels"): "Repeated causal-logistic coalition fits exceeded the execution allocation.",
+ ("trial", "tndm", "mc_pacman"): "Repeated causal-linear coalition fits exceeded the execution allocation.",
+ }
+ return specific.get((analysis, model, dataset), "No completed summary is available in this release.")
+
+
+def prediction_workflow(model: str, decoder: object, status: object) -> str:
+ if str(status).lower() != "present":
+ return "Unavailable"
+ decoder_name = "" if decoder is None or pd.isna(decoder) else str(decoder)
+ direct_decoders = {
+ "native",
+ "dnn",
+ "gru",
+ "lstm",
+ "mint_pipeline",
+ "rnn",
+ "neds_e2e",
+ "svc",
+ "svr",
+ "xgboost_classification",
+ "xgboost_regression",
+ }
+ if decoder_name in direct_decoders:
+ return "Predictions generated directly by trained model"
+ # Manuscript v7 deliberately distinguishes LDNS task families: continuous
+ # prediction uses the method recipe's ridge mapping on reconstructed rates
+ # (alpha = 1e-6), while classification uses the standard logistic readout.
+ if decoder_name in {"ridge", "logistic", "ldns_rate_sklearn_logistic"}:
+ return "Shared ridge/logistic readout"
+ if decoder_name in {"knn", "ole", "ldns_rate_sklearn_ridge"}:
+ return "Method-specific task mapping"
+ raise ValueError(f"Unrecognized prediction decoder for {model}: {decoder_name!r}")
+
+
+def readout_label(decoder: object) -> str:
+ if decoder is None or pd.isna(decoder):
+ return "Unavailable"
+ labels = {
+ "native": "Model-native output",
+ "ridge": "Ridge regression",
+ "logistic": "Logistic regression",
+ "knn": "k-nearest neighbors",
+ "ole": "Ordinary least squares",
+ "mint_pipeline": "MINT prediction pipeline",
+ "neds_e2e": "NEDS end-to-end head",
+ "dnn": "DNN task head",
+ "gru": "GRU task head",
+ "lstm": "LSTM task head",
+ "rnn": "RNN task head",
+ "svc": "Support-vector classifier",
+ "svr": "Support-vector regression",
+ "xgboost_classification": "XGBoost classifier",
+ "xgboost_regression": "XGBoost regressor",
+ "ldns_rate_sklearn_ridge": "Ridge on LDNS reconstructed rates",
+ "ldns_rate_sklearn_logistic": "Logistic regression on LDNS reconstructed rates",
+ }
+ return labels.get(str(decoder), str(decoder).replace("_", " "))
+
+
+def prediction_percentiles() -> pd.DataFrame:
+ frame = present_rows(prediction)[["model", "dataset", "score"]].copy()
+ frame["score"] = pd.to_numeric(frame["score"], errors="coerce")
+ frame["prediction_percentile"] = (
+ frame.groupby("dataset")["score"].rank(method="average", pct=True) * 100.0
+ )
+ return frame
+
+
+def overview_frame(dataset: str, models: Sequence[str] | None) -> pd.DataFrame:
+ chosen = selected_models(models)
+ base = add_method_columns(pd.DataFrame({"model": chosen}))
+ base["method"] = base["model"].map(lambda model: dataset_model_label(model, dataset))
+ pred = prediction[prediction["dataset"].astype(str) == str(dataset)].copy()
+ pred["score"] = pd.to_numeric(pred["score"], errors="coerce")
+ pred = pred.merge(
+ prediction_percentiles()[["model", "dataset", "prediction_percentile"]],
+ on=["model", "dataset"],
+ how="left",
+ )
+ pred = pred[
+ [
+ "model",
+ "status",
+ "metric",
+ "decoder",
+ "score",
+ "prediction_percentile",
+ "n_train_trials",
+ "n_test_trials",
+ "n_neurons",
+ ]
+ ].rename(columns={"score": "task_score", "status": "prediction_status_raw"})
+
+ rob = present_rows(robustness)
+ rob = rob[rob["dataset"].astype(str) == str(dataset)][["model", "raw_auc"]].copy()
+ rob["raw_auc"] = pd.to_numeric(rob["raw_auc"], errors="coerce")
+ rob = rob.rename(columns={"raw_auc": "robustness_auc"})
+
+ scale = present_rows(scalability)
+ scale = scale[scale["dataset"].astype(str) == str(dataset)][
+ ["model", "training_time_sec", "inference_time_sec", "peak_ram_gb", "peak_vram_gb"]
+ ].copy()
+ for column in ["training_time_sec", "inference_time_sec", "peak_ram_gb", "peak_vram_gb"]:
+ scale[column] = pd.to_numeric(scale[column], errors="coerce")
+
+ frame = base.merge(pred, on="model", how="left")
+ frame = frame.merge(rob, on="model", how="left")
+ frame = frame.merge(scale, on="model", how="left")
+ frame["workflow"] = frame.apply(
+ lambda row: prediction_workflow(row["model"], row.get("decoder"), row.get("prediction_status_raw")),
+ axis=1,
+ )
+ frame["prediction_status"] = np.where(frame["task_score"].notna(), "Available", "Unavailable")
+ frame.loc[frame["model"].isin(CPU_ONLY_MODELS), "peak_vram_gb"] = np.nan
+ return round_numeric(frame)
+
+
+def overview_cards(dataset: str, models: Sequence[str] | None) -> list[html.Div]:
+ frame = overview_frame(dataset, models)
+ available = frame.dropna(subset=["task_score"]).sort_values(
+ ["task_score", "model_order"], ascending=[False, True]
+ )
+ metric = "task score"
+ if not available.empty and available["metric"].notna().any():
+ metric = metric_name(available["metric"].dropna().iloc[0], held_out=True)
+ cards = [
+ metric_card(
+ "Dataset",
+ DATASET_SHORT_LABELS.get(dataset, dataset),
+ DATASET_DESCRIPTIONS.get(dataset),
+ "prediction",
+ ),
+ metric_card(
+ "Primary outcome",
+ metric,
+ "Higher is better; raw values are shown within each task.",
+ "prediction",
+ ),
+ ]
+ if available.empty:
+ cards.append(metric_card("Leading selected method", "Unavailable", accent="prediction"))
+ else:
+ top = available.iloc[0]
+ cards.append(
+ metric_card(
+ "Leading selected method",
+ str(top["method"]),
+ f"{metric}: {value_text(top['task_score'])}",
+ "prediction",
+ )
+ )
+ cards.append(
+ metric_card(
+ "Prediction coverage",
+ f"{len(available)}/{len(frame)}",
+ "Completed method–dataset entries in the current comparison.",
+ "prediction",
+ )
+ )
+ return cards
+
+
+def prediction_ranking_figure(dataset: str, models: Sequence[str] | None) -> go.Figure:
+ frame = overview_frame(dataset, models).dropna(subset=["task_score"])
+ if frame.empty:
+ return empty_figure("No held-out prediction results are available for this selection.")
+ frame = frame.sort_values(["task_score", "model_order"], ascending=[True, False])
+ metric = metric_name(frame["metric"].dropna().iloc[0], held_out=True)
+ fig = go.Figure(
+ go.Bar(
+ x=frame["task_score"],
+ y=frame["method"],
+ orientation="h",
+ marker=dict(color=PREDICTION_COLOR),
+ customdata=np.stack([frame["prediction_percentile"], frame["workflow"]], axis=-1),
+ hovertemplate=(
+ "Method=%{y}
Raw score=%{x:.4f}
"
+ "Within-dataset percentile=%{customdata[0]:.1f}
"
+ "Prediction workflow=%{customdata[1]}"
+ ),
+ )
+ )
+ fig.update_layout(title=f"{DATASET_SHORT_LABELS.get(dataset, dataset)} held-out prediction")
+ fig.update_xaxes(title=metric)
+ fig.update_yaxes(title="", showgrid=False)
+ return figure_layout(fig, height=max(440, 25 * len(frame) + 145))
+
+
+def mean_rank_order(
+ values: pd.DataFrame,
+ value_column: str,
+ eligible_models: set[str],
+) -> list[str]:
+ frame = values[values["model"].isin(eligible_models)].copy()
+ frame[value_column] = pd.to_numeric(frame[value_column], errors="coerce")
+ pivot = frame.pivot_table(
+ index="model", columns="dataset", values=value_column, aggfunc="first"
+ ).reindex(columns=DATASETS)
+ n_models = len(pivot)
+ ranks = pd.concat(
+ [
+ pivot[dataset].rank(ascending=False, method="average").fillna(n_models)
+ for dataset in DATASETS
+ ],
+ axis=1,
+ )
+ pivot["mean_rank"] = ranks.mean(axis=1)
+ return pivot.sort_values("mean_rank", ascending=True).index.astype(str).tolist()
+
+
+def percentile_heatmap(
+ values: pd.DataFrame,
+ models: Sequence[str] | None,
+ *,
+ title: str,
+ colorscale: list,
+ raw_column: str,
+ metric_column: str,
+ empty_message: str,
+ eligible_models: set[str] | None = None,
+ row_order: Sequence[str] | None = None,
+) -> go.Figure:
+ chosen = selected_models(models)
+ if eligible_models is not None:
+ chosen = [model for model in chosen if model in eligible_models]
+ if not chosen:
+ return empty_figure("None of the selected methods is configured for this analysis.")
+ frame = values.copy()
+ if frame.empty:
+ return empty_figure(empty_message)
+ frame[raw_column] = pd.to_numeric(frame[raw_column], errors="coerce")
+ frame["percentile"] = (
+ frame.groupby("dataset")[raw_column].rank(method="average", pct=True) * 100.0
+ )
+ frame = frame[frame["model"].isin(chosen)].copy()
+ grid = pd.MultiIndex.from_product([chosen, DATASETS], names=["model", "dataset"]).to_frame(index=False)
+ grid = grid.merge(
+ frame[["model", "dataset", raw_column, metric_column, "percentile"]],
+ on=["model", "dataset"],
+ how="left",
+ )
+ canonical_order = list(row_order) if row_order is not None else FIGURE_MODEL_ORDER
+ ordered_models = [model for model in canonical_order if model in chosen]
+ ordered_models += [model for model in chosen if model not in set(ordered_models)]
+ percentile_matrix = grid.pivot(index="model", columns="dataset", values="percentile").reindex(
+ index=ordered_models, columns=DATASETS
+ )
+ raw_matrix = grid.pivot(index="model", columns="dataset", values=raw_column).reindex(
+ index=ordered_models, columns=DATASETS
+ )
+ metric_matrix = grid.pivot(index="model", columns="dataset", values=metric_column).reindex(
+ index=ordered_models, columns=DATASETS
+ )
+ display_text = np.empty(percentile_matrix.shape, dtype=object)
+ customdata = np.empty((*percentile_matrix.shape, 3), dtype=object)
+ for row_index, model in enumerate(percentile_matrix.index):
+ for column_index, dataset in enumerate(percentile_matrix.columns):
+ percentile = percentile_matrix.iloc[row_index, column_index]
+ raw_value = raw_matrix.iloc[row_index, column_index]
+ metric = metric_matrix.iloc[row_index, column_index]
+ available = pd.notna(raw_value)
+ display_text[row_index, column_index] = "×" if not available else f"{percentile:.0f}"
+ customdata[row_index, column_index, 0] = (
+ "Unavailable" if not available else f"{float(raw_value):.4f}"
+ )
+ customdata[row_index, column_index, 1] = "Unavailable" if pd.isna(metric) else str(metric)
+ customdata[row_index, column_index, 2] = "Available" if available else "Unavailable"
+ fig = go.Figure(
+ go.Heatmap(
+ z=percentile_matrix.to_numpy(dtype=float),
+ x=[DATASET_SHORT_LABELS[dataset] for dataset in percentile_matrix.columns],
+ y=[model_label(model) for model in percentile_matrix.index],
+ text=display_text,
+ texttemplate="%{text}",
+ textfont=dict(size=11),
+ customdata=customdata,
+ colorscale=colorscale,
+ zmin=0,
+ zmax=100,
+ colorbar=dict(title="Percentile", thickness=13, ticksuffix="th"),
+ hovertemplate=(
+ "Method=%{y}
Dataset=%{x}
"
+ "Within-dataset percentile=%{z:.1f}
"
+ "Raw value=%{customdata[0]}
Metric=%{customdata[1]}
"
+ "Status=%{customdata[2]}"
+ ),
+ hoverongaps=False,
+ )
+ )
+ missing_rows, missing_columns = np.where(percentile_matrix.isna().to_numpy())
+ if len(missing_rows):
+ fig.add_trace(
+ go.Scatter(
+ x=[DATASET_SHORT_LABELS[percentile_matrix.columns[index]] for index in missing_columns],
+ y=[model_label(percentile_matrix.index[index]) for index in missing_rows],
+ mode="markers",
+ marker=dict(symbol="x", size=9, color="#7A858E", line=dict(width=1)),
+ showlegend=False,
+ hovertemplate="Method=%{y}
Dataset=%{x}
Status=Unavailable",
+ )
+ )
+ fig.update_layout(title=title)
+ fig.update_xaxes(title="", side="top", showgrid=False)
+ fig.update_yaxes(title="", showgrid=False)
+ return heatmap_layout(fig, height=max(500, 25 * len(percentile_matrix) + 185))
+
+
+def prediction_heatmap(models: Sequence[str] | None) -> go.Figure:
+ values = present_rows(prediction)[["model", "dataset", "score", "metric"]].copy()
+ values["metric_label"] = values["metric"].map(lambda value: metric_name(value, held_out=True))
+ return percentile_heatmap(
+ values,
+ models,
+ title="Prediction across tasks (within-dataset percentiles; raw scores on hover)",
+ colorscale=PREDICTION_SCALE,
+ raw_column="score",
+ metric_column="metric_label",
+ empty_message="No held-out prediction results are available.",
+ )
+
+
+def robustness_frame(dataset: str, models: Sequence[str] | None) -> pd.DataFrame:
+ chosen = selected_models(models)
+ base = add_method_columns(pd.DataFrame({"model": chosen}))
+ base["method"] = base["model"].map(lambda model: dataset_model_label(model, dataset))
+ values = present_rows(robustness)
+ values = values[values["dataset"].astype(str) == str(dataset)].copy()
+ values = values.rename(
+ columns={
+ "score_at_noise0": "unperturbed_score",
+ "score_at_max_noise": "highest_noise_score",
+ "raw_auc": "robustness_auc",
+ "mean_score": "average_noisy_score",
+ }
+ )
+ columns = [
+ "model",
+ "metric",
+ "noise_levels",
+ "scores",
+ "unperturbed_score",
+ "highest_noise_score",
+ "robustness_auc",
+ "average_noisy_score",
+ ]
+ values = values[[column for column in columns if column in values.columns]]
+ frame = base.merge(values, on="model", how="left")
+ return round_numeric(frame)
+
+
+def robustness_figure(dataset: str, models: Sequence[str] | None) -> go.Figure:
+ frame = robustness_frame(dataset, models).dropna(subset=["robustness_auc"])
+ if frame.empty:
+ return empty_figure("No robustness results are available for this selection.")
+ frame = frame.sort_values("model_order")
+ fig = go.Figure()
+ line_dashes = ["solid", "dash", "dot", "dashdot"]
+ for index, row in enumerate(frame.itertuples()):
+ levels = parse_float_list(row.noise_levels)
+ scores = parse_float_list(row.scores)
+ if len(levels) != len(scores):
+ raise ValueError(f"Noise levels and scores differ for {row.model} on {dataset}.")
+ fig.add_trace(
+ go.Scatter(
+ x=levels,
+ y=scores,
+ mode="lines+markers",
+ name=row.method,
+ line=dict(color=MODEL_COLORS[row.model], width=2.2, dash=line_dashes[index % len(line_dashes)]),
+ marker=dict(size=6, symbol=index % 8),
+ customdata=np.repeat(row.robustness_auc, len(levels)),
+ hovertemplate=(
+ f"Method={row.method}
Additive count-noise λ=%{{x:.1f}}
"
+ "Task score=%{y:.4f}
Raw score-vs-noise AUC=%{customdata:.4f}"
+ ),
+ )
+ )
+ metric = metric_name(frame["metric"].dropna().iloc[0])
+ fig.update_layout(
+ title=f"{DATASET_SHORT_LABELS.get(dataset, dataset)} robustness to noisy neural inputs",
+ hovermode="closest",
+ showlegend=len(frame) <= 12,
+ )
+ if len(frame) > 12:
+ fig.add_annotation(
+ text="Use the method comparison control to isolate curves; every curve is named on hover.",
+ x=0,
+ y=1.08,
+ xref="paper",
+ yref="paper",
+ showarrow=False,
+ xanchor="left",
+ font=dict(size=11, color=MUTED_COLOR),
+ )
+ fig.update_xaxes(title="Additive Poisson count-noise level λ", tickvals=[0, 0.2, 0.4, 0.6, 0.8])
+ fig.update_yaxes(title=metric)
+ return figure_layout(fig, height=540, legend_below=len(frame) <= 12)
+
+
+def compute_figures(
+ dataset: str, models: Sequence[str] | None
+) -> tuple[go.Figure, go.Figure, pd.DataFrame]:
+ frame = overview_frame(dataset, models).dropna(subset=["training_time_sec"])
+ if frame.empty:
+ empty = pd.DataFrame(
+ columns=[
+ "method",
+ "hardware",
+ "task_score",
+ "training_time_sec",
+ "inference_time_sec",
+ "peak_ram_gb",
+ "peak_vram_gb",
+ ]
+ )
+ return empty_figure("No runtime results are available."), empty_figure("No memory results are available."), empty
+ frame["hardware"] = np.where(frame["model"].isin(CPU_ONLY_MODELS), "CPU", "GPU")
+ frame = frame.sort_values(["training_time_sec", "model_order"], ascending=[False, True])
+
+ runtime = go.Figure()
+ runtime.add_trace(
+ go.Bar(
+ x=frame["training_time_sec"],
+ y=frame["method"],
+ orientation="h",
+ name="Training",
+ marker=dict(color=COMPUTE_COLOR),
+ customdata=frame["task_score"],
+ hovertemplate="Method=%{y}
Training time=%{x:.4g} s
Held-out score=%{customdata:.4f}",
+ )
+ )
+ runtime.add_trace(
+ go.Bar(
+ x=frame["inference_time_sec"],
+ y=frame["method"],
+ orientation="h",
+ name="Inference",
+ marker=dict(color="#F6A15D"),
+ hovertemplate="Method=%{y}
Complete held-out split=%{x:.4g} s",
+ )
+ )
+ runtime.update_layout(
+ title=f"{DATASET_SHORT_LABELS.get(dataset, dataset)} training and inference time",
+ barmode="group",
+ )
+ runtime.update_xaxes(title="Elapsed time (seconds, log scale)", type="log")
+ runtime.update_yaxes(title="", showgrid=False)
+ figure_layout(runtime, height=max(470, 27 * len(frame) + 155))
+
+ memory = go.Figure()
+ memory.add_trace(
+ go.Bar(
+ x=frame["peak_ram_gb"],
+ y=frame["method"],
+ orientation="h",
+ name="Peak RAM",
+ marker=dict(color="#E6842A"),
+ hovertemplate="Method=%{y}
Peak RAM=%{x:.3f} GB",
+ )
+ )
+ memory.add_trace(
+ go.Bar(
+ x=frame["peak_vram_gb"],
+ y=frame["method"],
+ orientation="h",
+ name="Peak GPU memory",
+ marker=dict(color="#F7C68B"),
+ hovertemplate="Method=%{y}
Peak GPU memory=%{x:.3f} GB",
+ )
+ )
+ memory.update_layout(
+ title=f"{DATASET_SHORT_LABELS.get(dataset, dataset)} peak RAM and GPU memory",
+ barmode="group",
+ )
+ memory.update_xaxes(title="Memory (GB)")
+ memory.update_yaxes(title="", showgrid=False)
+ figure_layout(memory, height=max(470, 27 * len(frame) + 155))
+
+ table = frame[
+ [
+ "method",
+ "hardware",
+ "task_score",
+ "training_time_sec",
+ "inference_time_sec",
+ "peak_ram_gb",
+ "peak_vram_gb",
+ ]
+ ].copy()
+ return runtime, memory, round_numeric(table)
+
+
+def feature_spec(dataset: str) -> tuple[str, str, str, float | None]:
+ if dataset == "allen_neuropixels":
+ return (
+ "spearman_corr",
+ "Drifting-gratings orientation selectivity",
+ "Spearman’s ρ",
+ 0.0,
+ )
+ if dataset == "ratinabox":
+ return (
+ "auc",
+ "Place cells vs head-direction and speed cells",
+ "ROC-AUC",
+ 0.5,
+ )
+ return (
+ "auc",
+ "Recorded neural features vs appended synthetic controls",
+ "ROC-AUC",
+ 0.5,
+ )
+
+
+def feature_frame(dataset: str, models: Sequence[str] | None) -> pd.DataFrame:
+ score_column, target, metric, _ = feature_spec(dataset)
+ frame = filter_models(active_rows(neuron_shap), models)
+ frame = frame[frame["dataset"].astype(str) == str(dataset)].copy()
+ if frame.empty:
+ return frame
+ frame = add_method_columns(frame)
+ frame["method"] = frame["model"].map(lambda model: dataset_model_label(model, dataset))
+ frame["validation_score"] = pd.to_numeric(frame[score_column], errors="coerce")
+ frame["validation_target"] = target
+ frame["validation_metric"] = metric
+ return round_numeric(frame)
+
+
+def feature_figures(
+ dataset: str, models: Sequence[str] | None
+) -> tuple[go.Figure, go.Figure, pd.DataFrame]:
+ frame = feature_frame(dataset, models)
+ if frame.empty:
+ columns = [
+ "method",
+ "validation_target",
+ "validation_metric",
+ "validation_score",
+ "shap_mean_value",
+ "shap_median_value",
+ "shap_min_value",
+ "shap_max_value",
+ "shap_fraction_positive",
+ "shap_fraction_negative",
+ ]
+ return empty_figure("No feature-attribution validation result is available."), empty_figure("No signed Kernel SHAP summary is available."), pd.DataFrame(columns=columns)
+ score_column, target, metric, reference = feature_spec(dataset)
+ validation = frame.dropna(subset=["validation_score"]).sort_values(
+ ["validation_score", "model_order"], ascending=[True, False]
+ )
+ validation_fig = go.Figure(
+ go.Bar(
+ x=validation["validation_score"],
+ y=validation["method"],
+ orientation="h",
+ marker=dict(color=FEATURE_COLOR),
+ hovertemplate=f"Method=%{{y}}
{metric}=%{{x:.4f}}
Target={target}",
+ )
+ )
+ if reference is not None:
+ validation_fig.add_vline(
+ x=reference,
+ line_dash="dash",
+ line_color="#6F7882",
+ annotation_text="0" if reference == 0 else "Chance = 0.5",
+ annotation_position="top",
+ )
+ validation_fig.update_layout(
+ title=f"{DATASET_SHORT_LABELS.get(dataset, dataset)} feature-attribution validation"
+ )
+ validation_fig.update_xaxes(title=metric)
+ validation_fig.update_yaxes(title="", showgrid=False)
+ figure_layout(validation_fig, height=max(430, 25 * len(validation) + 145))
+
+ signed = frame.dropna(subset=["shap_mean_value"]).sort_values(
+ ["shap_mean_value", "model_order"], ascending=[True, False]
+ )
+ signed_colors = [FEATURE_COLOR if value >= 0 else NEGATIVE_COLOR for value in signed["shap_mean_value"]]
+ signed_fig = go.Figure(
+ go.Bar(
+ x=signed["shap_mean_value"],
+ y=signed["method"],
+ orientation="h",
+ marker=dict(color=signed_colors),
+ customdata=np.stack(
+ [
+ signed["shap_median_value"],
+ signed["shap_min_value"],
+ signed["shap_max_value"],
+ signed["shap_fraction_positive"],
+ ],
+ axis=-1,
+ ),
+ hovertemplate=(
+ "Method=%{y}
Mean signed value=%{x:.5g}
"
+ "Median=%{customdata[0]:.5g}
Range=[%{customdata[1]:.5g}, %{customdata[2]:.5g}]
"
+ "Fraction positive=%{customdata[3]:.3f}"
+ ),
+ )
+ )
+ signed_fig.add_vline(x=0, line_color="#6F7882", line_width=1)
+ signed_fig.update_layout(title="Signed global Kernel SHAP summary")
+ signed_fig.update_xaxes(title="Mean signed contribution to task score")
+ signed_fig.update_yaxes(title="", showgrid=False)
+ figure_layout(signed_fig, height=max(430, 25 * len(signed) + 145))
+
+ columns = [
+ "method",
+ "validation_target",
+ "validation_metric",
+ "validation_score",
+ "shap_mean_value",
+ "shap_median_value",
+ "shap_min_value",
+ "shap_max_value",
+ "shap_fraction_positive",
+ "shap_fraction_negative",
+ ]
+ return validation_fig, signed_fig, round_numeric(frame[columns].sort_values("validation_score", ascending=False))
+
+
+def feature_heatmap(models: Sequence[str] | None) -> go.Figure:
+ rows = []
+ for dataset in DATASETS:
+ score_column, _target, metric, _reference = feature_spec(dataset)
+ frame = active_rows(neuron_shap)
+ frame = frame[frame["dataset"].astype(str) == dataset]
+ for row in frame.itertuples():
+ rows.append(
+ {
+ "model": row.model,
+ "dataset": dataset,
+ "validation_score": getattr(row, score_column),
+ "metric_label": metric,
+ }
+ )
+ values = pd.DataFrame(rows)
+ return percentile_heatmap(
+ values,
+ models,
+ title="Feature-attribution validation across tasks (within-dataset percentiles)",
+ colorscale=FEATURE_SCALE,
+ raw_column="validation_score",
+ metric_column="metric_label",
+ empty_message="No feature-attribution validation results are available.",
+ eligible_models=FEATURE_ELIGIBLE,
+ row_order=mean_rank_order(values, "validation_score", FEATURE_ELIGIBLE),
+ )
+
+
+def trial_frame(dataset: str, models: Sequence[str] | None) -> pd.DataFrame:
+ frame = filter_models(active_rows(trial_shapley), models)
+ frame = frame[frame["dataset"].astype(str) == str(dataset)].copy()
+ if frame.empty:
+ return frame
+ frame = add_method_columns(frame)
+ frame["method"] = frame["model"].map(lambda model: dataset_model_label(model, dataset))
+ frame = frame.rename(columns={"perturbation_auc": "corrupted_trial_auc"})
+ frame["converged"] = frame["converged"].fillna(False).astype(str).str.lower().map(
+ {"true": "Yes", "1": "Yes", "false": "No", "0": "No"}
+ ).fillna("No")
+ return round_numeric(frame)
+
+
+def trial_detection_figure(dataset: str, models: Sequence[str] | None) -> go.Figure:
+ frame = trial_frame(dataset, models)
+ if frame.empty:
+ return empty_figure("No corrupted-trial detection result is available for this selection.")
+ frame = frame.dropna(subset=["corrupted_trial_auc"]).sort_values(
+ ["corrupted_trial_auc", "model_order"], ascending=[True, False]
+ )
+ patterns = ["" if value == "Yes" else "/" for value in frame["converged"]]
+ fig = go.Figure(
+ go.Bar(
+ x=frame["corrupted_trial_auc"],
+ y=frame["method"],
+ orientation="h",
+ marker=dict(color=TRIAL_COLOR, pattern=dict(shape=patterns)),
+ customdata=np.stack(
+ [
+ frame["converged"],
+ frame["iterations"],
+ frame["final_error"],
+ frame["shapley_mean_value"],
+ frame["shapley_fraction_positive"],
+ ],
+ axis=-1,
+ ),
+ hovertemplate=(
+ "Method=%{y}
Corrupted-trial ROC-AUC=%{x:.4f}
"
+ "Converged=%{customdata[0]}
Permutations=%{customdata[1]:.0f}
"
+ "Final error=%{customdata[2]:.4f}
Mean signed trial value=%{customdata[3]:.5g}
"
+ "Fraction positive=%{customdata[4]:.3f}"
+ ),
+ )
+ )
+ fig.add_vline(
+ x=0.5,
+ line_dash="dash",
+ line_color="#6F7882",
+ annotation_text="Chance = 0.5",
+ annotation_position="top",
+ )
+ fig.update_layout(
+ title=f"{DATASET_SHORT_LABELS.get(dataset, dataset)} corrupted-trial detection"
+ )
+ fig.update_xaxes(title="ROC-AUC from negative trial value")
+ fig.update_yaxes(title="", showgrid=False)
+ return figure_layout(fig, height=max(430, 25 * len(frame) + 145))
+
+
+def trial_heatmap(models: Sequence[str] | None) -> go.Figure:
+ values = active_rows(trial_shapley)[["model", "dataset", "perturbation_auc"]].copy()
+ values["metric_label"] = "Corrupted-trial ROC-AUC"
+ return percentile_heatmap(
+ values,
+ models,
+ title="Corrupted-trial detection across tasks (within-dataset percentiles)",
+ colorscale=TRIAL_SCALE,
+ raw_column="perturbation_auc",
+ metric_column="metric_label",
+ empty_message="No corrupted-trial detection results are available.",
+ eligible_models=TRIAL_ELIGIBLE,
+ row_order=mean_rank_order(values, "perturbation_auc", TRIAL_ELIGIBLE),
+ )
+
+
+def retrain_frames(models: Sequence[str] | None) -> tuple[pd.DataFrame, pd.DataFrame]:
+ chosen = selected_models(models)
+ frame = active_rows(trial_retrain)
+ frame = frame[frame["model"].astype(str).isin(chosen)].copy()
+ within = frame[frame["analysis"] == "within_session_cleaning"].pivot_table(
+ index="model", columns="condition", values="score", aggfunc="first"
+ )
+ historical = frame[frame["analysis"] == "cross_session_old_trial_selection"].pivot_table(
+ index="model", columns="condition", values="score", aggfunc="first"
+ )
+ within = within.reset_index()
+ historical = historical.reset_index()
+ if not within.empty:
+ within = add_method_columns(within)
+ within["recovery"] = within.get("data_shapley") - within.get("mixed_full")
+ if not historical.empty:
+ historical = add_method_columns(historical)
+ historical = historical.rename(
+ columns={"oldonly_dshap_negative_removal": "historical_selected"}
+ )
+ return within, historical
+
+
+def equality_bounds(*series: pd.Series) -> tuple[float, float]:
+ values = pd.concat([pd.to_numeric(item, errors="coerce") for item in series]).dropna()
+ if values.empty:
+ return 0.0, 1.0
+ span = float(values.max() - values.min())
+ padding = max(span * 0.08, 0.03)
+ return float(values.min() - padding), float(values.max() + padding)
+
+
+def trial_retrain_figures(
+ models: Sequence[str] | None,
+) -> tuple[go.Figure, go.Figure, go.Figure, pd.DataFrame]:
+ within, historical = retrain_frames(models)
+ if within.empty:
+ removal = empty_figure("No macaque within-session removal summary is available.")
+ relation = empty_figure("No detection-versus-recovery summary is available.")
+ else:
+ lower, upper = equality_bounds(within["mixed_full"], within["data_shapley"])
+ removal = go.Figure()
+ removal.add_trace(
+ go.Scatter(
+ x=within["mixed_full"],
+ y=within["data_shapley"],
+ mode="markers",
+ text=within["method"],
+ marker=dict(
+ color=[MODEL_COLORS[model] for model in within["model"]],
+ size=10,
+ line=dict(color="#FFFFFF", width=1),
+ ),
+ customdata=np.stack([within["method"], within["oracle"], within["recovery"]], axis=-1),
+ hovertemplate=(
+ "Method=%{customdata[0]}
Mixed trials R²=%{x:.4f}
"
+ "After trial-value removal R²=%{y:.4f}
Oracle removal R²=%{customdata[1]:.4f}
"
+ "Recovery ΔR²=%{customdata[2]:+.4f}"
+ ),
+ )
+ )
+ removal.add_shape(type="line", x0=lower, x1=upper, y0=lower, y1=upper, line=dict(color="#69737D", dash="dash"))
+ removal_change = (
+ float(within["recovery"].mean())
+ / float(within["mixed_full"].abs().mean())
+ * 100.0
+ )
+ removal.add_annotation(
+ text=f"Mean change = {removal_change:+.0f}%",
+ x=0.03,
+ y=0.97,
+ xref="paper",
+ yref="paper",
+ xanchor="left",
+ yanchor="top",
+ showarrow=False,
+ bgcolor="rgba(255,255,255,0.88)",
+ font=dict(size=12, color="#2E7D32" if removal_change >= 0 else NEGATIVE_COLOR),
+ )
+ removal.update_layout(title="Macaque trial-value-guided removal")
+ removal.update_xaxes(title="Before filtering: mixed-trial test R²", range=[lower, upper])
+ removal.update_yaxes(title="After negative-value removal: test R²", range=[lower, upper])
+ figure_layout(removal, height=480)
+
+ detection = active_rows(trial_shapley)
+ detection = detection[detection["dataset"].astype(str) == "monkey"][["model", "perturbation_auc"]]
+ relation_frame = within.merge(detection, on="model", how="inner").dropna(
+ subset=["perturbation_auc", "recovery"]
+ )
+ relation = go.Figure()
+ if relation_frame.empty:
+ relation = empty_figure("No shared detection and recovery entries are available.")
+ else:
+ x = relation_frame["perturbation_auc"].astype(float)
+ y = relation_frame["recovery"].astype(float)
+ rho = (
+ x.rank(method="average").corr(y.rank(method="average"))
+ if len(relation_frame) >= 2
+ else np.nan
+ )
+ relation.add_trace(
+ go.Scatter(
+ x=x,
+ y=y,
+ mode="markers",
+ marker=dict(
+ color=[MODEL_COLORS[model] for model in relation_frame["model"]],
+ size=10,
+ line=dict(color="#FFFFFF", width=1),
+ ),
+ customdata=relation_frame["method"],
+ hovertemplate=(
+ "Method=%{customdata}
Detection ROC-AUC=%{x:.4f}
"
+ "Recovery ΔR²=%{y:+.4f}"
+ ),
+ )
+ )
+ if len(relation_frame) >= 2 and float(x.max() - x.min()) > 0:
+ coefficients = np.polyfit(x, y, 1)
+ line_x = np.linspace(float(x.min()), float(x.max()), 100)
+ relation.add_trace(
+ go.Scatter(
+ x=line_x,
+ y=np.polyval(coefficients, line_x),
+ mode="lines",
+ line=dict(color=TRIAL_COLOR, width=2),
+ name="Linear fit",
+ hoverinfo="skip",
+ )
+ )
+ relation.add_hline(y=0, line_dash="dash", line_color="#69737D")
+ relation_text = (
+ f"Spearman ρ = {rho:.2f}; n = {len(relation_frame)}"
+ if pd.notna(rho)
+ else f"n = {len(relation_frame)}; select at least two methods for correlation"
+ )
+ if not models and pd.notna(rho):
+ relation_text += "
one-sided permutation P = 0.035"
+ relation.add_annotation(
+ text=relation_text,
+ x=0.02,
+ y=0.98,
+ xref="paper",
+ yref="paper",
+ xanchor="left",
+ yanchor="top",
+ showarrow=False,
+ bgcolor="rgba(255,255,255,0.85)",
+ font=dict(size=12),
+ )
+ relation.update_layout(title="Detection signal versus recovery after removal", showlegend=False)
+ relation.update_xaxes(title="Corrupted-trial detection ROC-AUC")
+ relation.update_yaxes(title="Recovery after removal (ΔR²)")
+ figure_layout(relation, height=480)
+
+ if historical.empty:
+ historical_fig = empty_figure("No same-subject historical-selection summary is available.")
+ else:
+ lower, upper = equality_bounds(historical["target_only"], historical["historical_selected"])
+ historical_fig = go.Figure(
+ go.Scatter(
+ x=historical["target_only"],
+ y=historical["historical_selected"],
+ mode="markers",
+ marker=dict(
+ color=[MODEL_COLORS[model] for model in historical["model"]],
+ size=10,
+ line=dict(color="#FFFFFF", width=1),
+ ),
+ customdata=np.stack([historical["method"], historical["all_sessions"]], axis=-1),
+ hovertemplate=(
+ "Method=%{customdata[0]}
Current session only R²=%{x:.4f}
"
+ "Trial-value historical selection R²=%{y:.4f}
"
+ "All-session pooling R²=%{customdata[1]:.4f}"
+ ),
+ )
+ )
+ historical_fig.add_shape(type="line", x0=lower, x1=upper, y0=lower, y1=upper, line=dict(color="#69737D", dash="dash"))
+ historical_change = (
+ float((historical["historical_selected"] - historical["target_only"]).mean())
+ / float(historical["target_only"].abs().mean())
+ * 100.0
+ )
+ historical_fig.add_annotation(
+ text=f"Mean change = {historical_change:+.0f}%",
+ x=0.03,
+ y=0.97,
+ xref="paper",
+ yref="paper",
+ xanchor="left",
+ yanchor="top",
+ showarrow=False,
+ bgcolor="rgba(255,255,255,0.88)",
+ font=dict(size=12, color="#2E7D32" if historical_change >= 0 else NEGATIVE_COLOR),
+ )
+ historical_fig.update_layout(title="Same-subject historical-trial selection")
+ historical_fig.update_xaxes(title="Current-session trials only: test R²", range=[lower, upper])
+ historical_fig.update_yaxes(
+ title="Add nonnegative-valued historical trials: test R²",
+ range=[lower, upper],
+ )
+ figure_layout(historical_fig, height=480)
+
+ table = within[
+ [column for column in ["model", "method", "mixed_full", "data_shapley", "oracle", "recovery"] if column in within]
+ ].copy() if not within.empty else pd.DataFrame(columns=["model", "method", "mixed_full", "data_shapley", "oracle", "recovery"])
+ historical_columns = ["model", "target_only", "all_sessions", "historical_selected"]
+ if not historical.empty:
+ table = table.merge(historical[historical_columns], on="model", how="outer")
+ table["method"] = table["method"].fillna(table["model"].map(model_label))
+ table = table.drop(columns=["model"], errors="ignore")
+ return removal, relation, historical_fig, round_numeric(table)
+
+
+def historical_trajectory_figure(models: Sequence[str] | None) -> go.Figure:
+ if models and "rnn" not in set(models):
+ return empty_figure(
+ "The Figure 5e held-out trajectory example uses RNN. "
+ "Include RNN in Compare methods or clear the method filter.",
+ height=430,
+ )
+
+ frame = trial_historical_trajectories.copy()
+ current_r2 = float(frame["current_only_r2"].iloc[0])
+ historical_r2 = float(frame["historical_selected_r2"].iloc[0])
+ panels = [
+ ("target_x", "target_y", "Ground truth", 0.84, 2.2),
+ (
+ "current_only_x",
+ "current_only_y",
+ f"Current session only
R² = {current_r2:.2f}",
+ 0.64,
+ 2.5,
+ ),
+ (
+ "historical_selected_x",
+ "historical_selected_y",
+ f"Current + nonnegative-valued historical trials
R² = {historical_r2:.2f}",
+ 0.64,
+ 2.5,
+ ),
+ ]
+ fig = make_subplots(
+ rows=1,
+ cols=3,
+ horizontal_spacing=0.045,
+ subplot_titles=[panel[2] for panel in panels],
+ )
+
+ direction_labels = dict(zip(DIRECTION_LEGEND_ORDER, DIRECTION_LEGEND_LABELS))
+ for direction_rank, direction_index in enumerate(DIRECTION_LEGEND_ORDER):
+ direction_frame = frame[frame["direction_index"] == direction_index]
+ direction_label = direction_labels[direction_index]
+ for panel_index, (x_column, y_column, _title, opacity, width) in enumerate(
+ panels, start=1
+ ):
+ x_values: list[object] = []
+ y_values: list[object] = []
+ hover_values: list[list[object]] = []
+ for trial_index in sorted(direction_frame["trial_index"].unique()):
+ trial = direction_frame[
+ direction_frame["trial_index"] == trial_index
+ ].sort_values("time_index")
+ x_values.extend(trial[x_column].tolist())
+ y_values.extend(trial[y_column].tolist())
+ hover_values.extend(
+ [
+ [trial_id, direction_label, time_index]
+ for trial_id, time_index in zip(
+ trial["trial_id"].astype(str), trial["time_index"]
+ )
+ ]
+ )
+ x_values.append(None)
+ y_values.append(None)
+ hover_values.append([None, None, None])
+ fig.add_trace(
+ go.Scatter(
+ x=x_values,
+ y=y_values,
+ mode="lines",
+ name=direction_label,
+ legendgroup=f"direction-{direction_index}",
+ legendrank=direction_rank,
+ showlegend=panel_index == 1,
+ opacity=opacity,
+ line=dict(color=DIRECTION_PALETTE[direction_index], width=width),
+ customdata=hover_values,
+ connectgaps=False,
+ hovertemplate=(
+ "Trial=%{customdata[0]}
Reach direction=%{customdata[1]}
"
+ "Time bin=%{customdata[2]}
x=%{x:.3f}
y=%{y:.3f}"
+ ),
+ ),
+ row=1,
+ col=panel_index,
+ )
+
+ first_points = (
+ frame.sort_values("time_index").groupby("trial_index", as_index=False).first()
+ )
+ for panel_index, (x_column, y_column, _title, _opacity, _width) in enumerate(
+ panels, start=1
+ ):
+ fig.add_trace(
+ go.Scatter(
+ x=[float(first_points[x_column].mean())],
+ y=[float(first_points[y_column].mean())],
+ mode="markers",
+ marker=dict(size=8, color="#222222"),
+ showlegend=False,
+ hovertemplate="Mean trajectory origin",
+ ),
+ row=1,
+ col=panel_index,
+ )
+
+ x_values = pd.concat(
+ [frame["target_x"], frame["current_only_x"], frame["historical_selected_x"]],
+ ignore_index=True,
+ )
+ y_values = pd.concat(
+ [frame["target_y"], frame["current_only_y"], frame["historical_selected_y"]],
+ ignore_index=True,
+ )
+ x_span = max(float(x_values.max() - x_values.min()), 1.0)
+ y_span = max(float(y_values.max() - y_values.min()), 1.0)
+ x_range = [float(x_values.min() - 0.06 * x_span), float(x_values.max() + 0.06 * x_span)]
+ y_range = [float(y_values.min() - 0.06 * y_span), float(y_values.max() + 0.06 * y_span)]
+ for panel_index in range(1, 4):
+ x_axis_id = "x" if panel_index == 1 else f"x{panel_index}"
+ fig.update_xaxes(
+ range=x_range,
+ showgrid=False,
+ zeroline=False,
+ showticklabels=False,
+ ticks="",
+ row=1,
+ col=panel_index,
+ )
+ fig.update_yaxes(
+ range=y_range,
+ showgrid=False,
+ zeroline=False,
+ showticklabels=False,
+ ticks="",
+ scaleanchor=x_axis_id,
+ scaleratio=1,
+ row=1,
+ col=panel_index,
+ )
+
+ figure_layout(fig, height=510, legend_below=True)
+ fig.update_layout(
+ title="Held-out target-session trajectories (RNN)",
+ margin=dict(l=28, r=28, t=76, b=118),
+ legend=dict(
+ orientation="h",
+ yanchor="top",
+ y=-0.10,
+ xanchor="center",
+ x=0.5,
+ title="Reach direction",
+ traceorder="normal",
+ font=dict(size=11),
+ ),
+ )
+ fig.for_each_annotation(
+ lambda annotation: annotation.update(font=dict(size=12, color=TEXT_COLOR))
+ )
+ return fig
+
+
+def condition_sort_key(value: object) -> tuple[int, float | str]:
+ try:
+ return (0, float(value))
+ except (TypeError, ValueError):
+ return (1, str(value))
-def ranking_figure(dataset: str, table_df: pd.DataFrame) -> go.Figure:
- rank_df = table_df.dropna(subset=["task_score"]).sort_values("task_score", ascending=True)
- if rank_df.empty:
- return empty_figure(f"No decoding results are available for {DATASET_LABELS.get(dataset, dataset)}.")
- metric = metric_text(rank_df["metric"].dropna().iloc[0]) if rank_df["metric"].notna().any() else "score"
- label = decoding_label(metric)
- fig = go.Figure(
- go.Bar(
- x=rank_df["task_score"],
- y=rank_df["method"],
- orientation="h",
- marker=dict(color=TASK_COLOR),
- hovertemplate="Method=%{y}
Score=%{x:.4f}",
- )
- )
- fig.update_layout(title=f"{DATASET_LABELS.get(dataset, dataset)} {label} ranking")
- fig.update_xaxes(title=label)
- fig.update_yaxes(title="")
- return fig_layout(fig, height=max(420, 25 * len(rank_df) + 150))
+def condition_label(dataset: str, value: object, color_mode: str = "condition") -> str:
+ if value is None or pd.isna(value):
+ return "Unknown"
+ text = str(int(float(value))) if re.fullmatch(r"-?\d+(\.0+)?", str(value)) else str(value)
+ if dataset == "ratinabox":
+ index = int(float(text))
+ if color_mode == "x":
+ return f"x bin {index}"
+ if color_mode == "y":
+ return f"y bin {index}"
+ return f"x{index % 10}, y{index // 10}"
+ return CONDITION_LABELS.get(dataset, {}).get(text, text)
-def consistency_frame(dataset: str, models: list[str] | None) -> pd.DataFrame:
- chosen = selected_models(models)
- base = add_method_columns(pd.DataFrame({"model": chosen}))
- cons = filter_models(active_rows(consistency), chosen)
- cons = cons[cons["dataset"].astype(str) == str(dataset)].copy()
- for col in ["mean_r2", "latent_dim", "n_sessions", "n_pairwise"]:
- cons[col] = pd.to_numeric(cons[col], errors="coerce")
- cons = cons[["model", "mean_r2", "latent_dim", "n_sessions", "n_pairwise"]]
- cons = cons.rename(columns={"mean_r2": "alignment_score"})
-
- df = base.merge(cons, on="model", how="left")
- available = df["alignment_score"].notna()
- order = df.loc[available].sort_values(
- ["alignment_score", "model_order"], ascending=[False, True]
- ).index
- df["rank"] = None
- for rank, idx in enumerate(order, start=1):
- df.at[idx, "rank"] = rank
- df["id"] = df["model"]
- return round_numeric(df, NUMERIC_COLUMNS)
-
-
-def selected_consistency_model(df: pd.DataFrame, active_cell: dict | None) -> str | None:
- if active_cell and active_cell.get("row_id") in set(df["model"]):
- return str(active_cell["row_id"])
- available = sort_table(
- df, None, [("alignment_score", False), ("model_order", True)]
- ).dropna(subset=["alignment_score"])
- if available.empty:
- return None
- return str(available.iloc[0]["model"])
+def condition_axis_label(dataset: str, color_mode: str = "condition") -> str:
+ if dataset == "ratinabox":
+ if color_mode == "x":
+ return "X-position bin"
+ if color_mode == "y":
+ return "Y-position bin"
+ return "Spatial bin"
+ return {
+ "monkey": "Reach direction",
+ "allen_neuropixels": "Stimulus orientation",
+ "speech": "Attempted word",
+ }.get(dataset, "Task condition")
-def consistency_bar_figure(dataset: str, df: pd.DataFrame) -> go.Figure:
- bar_df = df.dropna(subset=["alignment_score"]).sort_values("alignment_score", ascending=True)
- if bar_df.empty:
- return empty_figure(f"No cross-session alignment results are available for {DATASET_LABELS.get(dataset, dataset)}.")
- fig = go.Figure(
- go.Bar(
- x=bar_df["alignment_score"],
- y=bar_df["method"],
- orientation="h",
- marker=dict(color=ALIGNMENT_COLOR),
- hovertemplate="Method=%{y}
Alignment=%{x:.4f}",
- )
- )
- fig.update_layout(title=f"{DATASET_LABELS.get(dataset, dataset)} cross-session alignment")
- fig.update_xaxes(title="Alignment score")
- fig.update_yaxes(title="")
- return fig_layout(fig, height=max(360, 25 * len(bar_df) + 130))
+def session_display_label(dataset: str, session: object) -> str:
+ text = "" if session is None or pd.isna(session) else str(session)
+ if dataset == "monkey":
+ match = re.search(r"sub-([A-Za-z])_ses-CO-", text)
+ return f"Monkey {match.group(1)}" if match else text
+ if dataset == "ratinabox":
+ sessions = ["ratinabox_nav", "ratinabox_nav_s123", "ratinabox_nav_s456", "ratinabox_nav_s789"]
+ return f"Simulation {sessions.index(text) + 1}" if text in sessions else text
+ if dataset == "speech":
+ return f"Participant {text.upper()}"
+ if dataset == "allen_neuropixels":
+ return f"Recording {text}"
+ return text or "Recording"
-def consistency_heatmap(models: list[str] | None) -> go.Figure:
- df = filter_models(active_rows(consistency), models)
- if df.empty:
- return empty_figure("No cross-session alignment results are available.")
- df = add_method_columns(df)
- df["mean_r2"] = pd.to_numeric(df["mean_r2"], errors="coerce")
- df["dataset_label"] = df["dataset"].map(DATASET_LABELS).fillna(df["dataset"])
- pivot = df.pivot_table(index="method", columns="dataset_label", values="mean_r2", aggfunc="first")
- if not pivot.empty:
- pivot = pivot.loc[pivot.mean(axis=1, skipna=True).sort_values(ascending=False).index]
- text = pivot.map(lambda x: "" if pd.isna(x) else f"{x:.2f}") if not pivot.empty else pivot
- fig = go.Figure(
- go.Heatmap(
- z=pivot.values if not pivot.empty else [[]],
- x=list(pivot.columns),
- y=list(pivot.index),
- text=text.values if not pivot.empty else [[]],
- texttemplate="%{text}",
- colorscale=ALIGNMENT_SCALE,
- colorbar=dict(title="Score", thickness=12),
- hovertemplate="Method=%{y}
Dataset=%{x}
Alignment=%{z:.4f}",
- )
- )
- fig.update_layout(title="Alignment across datasets")
- return fig_layout(fig, height=max(360, 25 * len(pivot.index) + 135))
+def add_latent_color_columns(df: pd.DataFrame, dataset: str, color_mode: str) -> pd.DataFrame:
+ out = df.copy()
+ values = pd.to_numeric(out["condition"], errors="coerce")
+ if values.isna().any() or (values < 0).any():
+ raise ValueError("Latent samples contain invalid task-condition labels.")
+ if dataset == "ratinabox" and color_mode == "x":
+ color_values = values.astype(int) % 10
+ elif dataset == "ratinabox" and color_mode == "y":
+ color_values = values.astype(int) // 10
+ else:
+ color_values = values.astype(int)
+ out["color_value"] = color_values.astype(str)
+ out["color_num"] = color_values
+ out["color_label"] = [condition_label(dataset, value, color_mode) for value in color_values]
+ return out
-def latent_space_figure(dataset: str, model: str | None, color_mode: str = "condition") -> go.Figure:
+def latent_space_figure(dataset: str, model: str | None, color_mode: str) -> go.Figure:
if not model:
- return empty_figure("No latent-space view is available for this selection.")
- if latent_samples.empty:
- return empty_figure("Latent-space samples are unavailable in this view.")
-
- plot_df = latent_samples[
+ return empty_figure("Select an available method to view aligned coordinates.", height=500)
+ samples = latent_samples[
(latent_samples["dataset"].astype(str) == str(dataset))
& (latent_samples["model"].astype(str) == str(model))
].copy()
- if plot_df.empty:
- return empty_figure(f"No latent-space samples are available for {model_label(model)} on {DATASET_LABELS.get(dataset, dataset)}.")
-
- for col in ["x", "y", "z"]:
- plot_df[col] = pd.to_numeric(plot_df[col], errors="coerce")
- try:
- plot_df = add_latent_color_columns(plot_df, dataset, color_mode)
- except ValueError as exc:
- return empty_figure(str(exc))
- plot_df["session_display"] = plot_df["session_label"].map(lambda value: session_display_label(dataset, value))
- plot_df = plot_df.dropna(subset=["x", "y", "z"])
- if plot_df.empty:
- return empty_figure("No latent-space samples match this selection.")
-
- trajectory_df = latent_trajectories[
+ if samples.empty:
+ return empty_figure("Figure 3-aligned display coordinates are unavailable for this entry.", height=500)
+ for column in ["x", "y", "z"]:
+ samples[column] = pd.to_numeric(samples[column], errors="coerce")
+ samples = add_latent_color_columns(samples, dataset, color_mode).dropna(subset=["x", "y", "z"])
+ trajectories = latent_trajectories[
(latent_trajectories["dataset"].astype(str) == str(dataset))
& (latent_trajectories["model"].astype(str) == str(model))
].copy()
- for col in ["x", "y", "z"]:
- if col in trajectory_df:
- trajectory_df[col] = pd.to_numeric(trajectory_df[col], errors="coerce")
- if not trajectory_df.empty:
- try:
- trajectory_df = add_latent_color_columns(trajectory_df, dataset, color_mode)
- except ValueError:
- trajectory_df = trajectory_df.iloc[0:0].copy()
- trajectory_df["session_display"] = trajectory_df["session_label"].map(lambda value: session_display_label(dataset, value))
- trajectory_df = trajectory_df.dropna(subset=["x", "y", "z"])
- if "color_value" not in trajectory_df.columns:
- trajectory_df["color_value"] = pd.Series(dtype=str)
-
- sessions = ordered_unique(plot_df["session_label"])
- session_titles = [session_display_label(dataset, session) for session in sessions]
- n_cols = 2 if len(sessions) > 1 else 1
- n_rows = int(np.ceil(len(sessions) / n_cols))
- specs = [[{"type": "scene"} for _ in range(n_cols)] for _ in range(n_rows)]
+ if not trajectories.empty:
+ for column in ["x", "y", "z", "time_index"]:
+ trajectories[column] = pd.to_numeric(trajectories[column], errors="coerce")
+ trajectories = add_latent_color_columns(trajectories, dataset, color_mode).dropna(
+ subset=["x", "y", "z"]
+ )
+ else:
+ trajectories["color_value"] = pd.Series(dtype=str)
+
+ sessions = list(dict.fromkeys(samples["session_label"].astype(str)))
+ columns = 2 if len(sessions) > 1 else 1
+ rows = int(np.ceil(len(sessions) / columns))
fig = make_subplots(
- rows=n_rows,
- cols=n_cols,
- specs=specs,
- subplot_titles=session_titles,
- horizontal_spacing=0.045,
- vertical_spacing=0.12,
+ rows=rows,
+ cols=columns,
+ specs=[[{"type": "scene"} for _ in range(columns)] for _ in range(rows)],
+ subplot_titles=[session_display_label(dataset, session) for session in sessions],
+ horizontal_spacing=0.04,
+ vertical_spacing=0.1,
)
-
- condition_values = sorted(plot_df["color_value"].astype(str).unique(), key=condition_sort_key)
- use_categorical = len(condition_values) <= 12 and dataset != "ratinabox"
+ condition_values = sorted(samples["color_value"].unique(), key=condition_sort_key)
+ categorical = dataset != "ratinabox"
if dataset == "monkey":
- condition_colors = {
- condition: DIRECTION_PALETTE[int(float(condition)) % len(DIRECTION_PALETTE)]
- for condition in condition_values
- }
+ colors = {value: DIRECTION_PALETTE[int(value) % len(DIRECTION_PALETTE)] for value in condition_values}
elif dataset == "speech":
- condition_colors = {
- condition: SPEECH_PALETTE.get(condition, CATEGORICAL_PALETTE[idx % len(CATEGORICAL_PALETTE)])
- for idx, condition in enumerate(condition_values)
- }
+ colors = {value: SPEECH_PALETTE.get(value, "#777777") for value in condition_values}
else:
- condition_colors = {
- condition: CATEGORICAL_PALETTE[idx % len(CATEGORICAL_PALETTE)]
- for idx, condition in enumerate(condition_values)
- }
+ colors = {value: ALLEN_PALETTE.get(value, "#777777") for value in condition_values}
condition_name = condition_axis_label(dataset, color_mode)
- for session_idx, session in enumerate(sessions):
- session_df = plot_df[plot_df["session_label"].astype(str) == str(session)]
- if session_df.empty:
- continue
- row = session_idx // n_cols + 1
- col = session_idx % n_cols + 1
+ for session_index, session in enumerate(sessions):
+ row_index = session_index // columns + 1
+ column_index = session_index % columns + 1
+ session_samples = samples[samples["session_label"].astype(str) == session]
display_session = session_display_label(dataset, session)
-
- if use_categorical:
+ if categorical:
for condition in condition_values:
- cond_df = session_df[session_df["color_value"].astype(str) == condition]
- if cond_df.empty:
+ points = session_samples[session_samples["color_value"] == condition]
+ if points.empty:
continue
- trace_name = latent_color_label(dataset, color_mode, condition)
- session_traj = trajectory_df[
- (trajectory_df["session_label"].astype(str) == str(session))
- & (trajectory_df["color_value"].astype(str) == condition)
+ means = trajectories[
+ (trajectories["session_label"].astype(str) == session)
+ & (trajectories["color_value"] == condition)
].sort_values("time_index")
+ label = condition_label(dataset, condition, color_mode)
fig.add_trace(
go.Scatter3d(
- x=cond_df["x"],
- y=cond_df["y"],
- z=cond_df["z"],
+ x=points["x"],
+ y=points["y"],
+ z=points["z"],
mode="markers",
- name=trace_name,
+ name=label,
legendgroup=condition,
- showlegend=session_idx == 0,
- marker=dict(
- size=2.4 if not session_traj.empty else 3.0,
- opacity=0.32 if not session_traj.empty else 0.78,
- color=condition_colors[condition],
- ),
+ showlegend=session_index == 0,
+ marker=dict(size=2.6, opacity=0.42 if not means.empty else 0.76, color=colors[condition]),
customdata=np.stack(
[
- np.repeat(display_session, len(cond_df)),
- cond_df["color_label"].astype(str),
- cond_df["trial_index"].astype(str),
- cond_df["time_index"].astype(str),
+ np.repeat(display_session, len(points)),
+ points["color_label"],
+ points["trial_index"],
+ points["time_index"],
],
axis=-1,
),
hovertemplate=(
- "Session=%{customdata[0]}
"
- f"{condition_name}=%{{customdata[1]}}
"
- "Trial=%{customdata[2]} time=%{customdata[3]}"
+ "Recording=%{customdata[0]}
"
+ f"{condition_name}=%{{customdata[1]}}
Trial=%{{customdata[2]}}; time bin=%{{customdata[3]}}"
""
),
),
- row=row,
- col=col,
+ row=row_index,
+ col=column_index,
)
- if not session_traj.empty:
+ if not means.empty:
fig.add_trace(
go.Scatter3d(
- x=session_traj["x"],
- y=session_traj["y"],
- z=session_traj["z"],
+ x=means["x"],
+ y=means["y"],
+ z=means["z"],
mode="lines",
- name=trace_name,
+ name=label,
legendgroup=condition,
showlegend=False,
- line=dict(color=condition_colors[condition], width=5),
- hovertemplate=(
- f"{condition_name}={trace_name}
"
- "Time=%{customdata}"
- ),
- customdata=session_traj["time_index"],
+ line=dict(color=colors[condition], width=5),
+ hovertemplate=f"{condition_name}={label}
Time bin=%{{customdata}}",
+ customdata=means["time_index"],
),
- row=row,
- col=col,
+ row=row_index,
+ col=column_index,
)
else:
fig.add_trace(
go.Scatter3d(
- x=session_df["x"],
- y=session_df["y"],
- z=session_df["z"],
+ x=session_samples["x"],
+ y=session_samples["y"],
+ z=session_samples["z"],
mode="markers",
name=display_session,
showlegend=False,
marker=dict(
size=2.8,
- opacity=0.72,
- color=session_df["color_num"],
+ opacity=0.74,
+ color=session_samples["color_num"],
colorscale=RATINABOX_SCALE,
cmin=0,
cmax=9 if color_mode in {"x", "y"} else 99,
- showscale=session_idx == 0,
+ showscale=session_index == 0,
colorbar=dict(title=condition_name, thickness=12),
),
customdata=np.stack(
[
- np.repeat(display_session, len(session_df)),
- session_df["color_label"].astype(str),
- session_df["trial_index"].astype(str),
- session_df["time_index"].astype(str),
+ np.repeat(display_session, len(session_samples)),
+ session_samples["color_label"],
+ session_samples["trial_index"],
+ session_samples["time_index"],
],
axis=-1,
),
hovertemplate=(
- "Session=%{customdata[0]}
"
- f"{condition_name}=%{{customdata[1]}}
"
- "Trial=%{customdata[2]} time=%{customdata[3]}"
+ "Recording=%{customdata[0]}
"
+ f"{condition_name}=%{{customdata[1]}}
Trial=%{{customdata[2]}}; time bin=%{{customdata[3]}}"
""
),
),
- row=row,
- col=col,
+ row=row_index,
+ col=column_index,
)
extent = max(
- float(np.nanpercentile(np.abs(plot_df["x"]), 99)),
- float(np.nanpercentile(np.abs(plot_df["y"]), 99)),
- float(np.nanpercentile(np.abs(plot_df["z"]), 99)),
+ float(np.nanpercentile(np.abs(samples[["x", "y", "z"]].to_numpy()), 99)),
1.0,
- )
- lim = extent * 1.08
- for idx in range(len(sessions)):
- scene_id = "scene" if idx == 0 else f"scene{idx + 1}"
+ ) * 1.08
+ for scene_index in range(len(sessions)):
+ scene_id = "scene" if scene_index == 0 else f"scene{scene_index + 1}"
fig.update_layout(
**{
scene_id: dict(
- xaxis=dict(title="", range=[-lim, lim], showgrid=False, zeroline=False, showticklabels=False),
- yaxis=dict(title="", range=[-lim, lim], showgrid=False, zeroline=False, showticklabels=False),
- zaxis=dict(title="", range=[-lim, lim], showgrid=False, zeroline=False, showticklabels=False),
+ xaxis=dict(title="Aligned dim. 1", range=[-extent, extent], showgrid=False, zeroline=False, showticklabels=False),
+ yaxis=dict(title="Aligned dim. 2", range=[-extent, extent], showgrid=False, zeroline=False, showticklabels=False),
+ zaxis=dict(title="Aligned dim. 3", range=[-extent, extent], showgrid=False, zeroline=False, showticklabels=False),
aspectmode="cube",
- bgcolor="#ffffff",
- camera=dict(eye=dict(x=1.55, y=1.45, z=1.05)),
+ bgcolor="#FFFFFF",
+ camera=dict(eye=dict(x=1.5, y=1.4, z=1.0)),
)
}
)
-
- score_df = active_rows(consistency)
- score_df = score_df[
- (score_df["dataset"].astype(str) == str(dataset))
- & (score_df["model"].astype(str) == str(model))
- ].copy()
- score = None
- if not score_df.empty:
- score = pd.to_numeric(score_df["mean_r2"], errors="coerce").dropna()
- score = float(score.iloc[0]) if not score.empty else None
- score_suffix = "" if score is None else f" | alignment {score:.2f}"
- fig.update_layout(
- title=f"{model_label(model)} latent space on {DATASET_LABELS.get(dataset, dataset)}{score_suffix}",
- height=760 if n_rows > 1 else 520,
- paper_bgcolor="#ffffff",
- plot_bgcolor="#ffffff",
- margin=dict(l=8, r=8, t=78, b=92),
- font=dict(family="Inter, Arial, sans-serif", size=13, color="#17202a"),
- legend=dict(orientation="h", yanchor="top", y=-0.08, xanchor="left", x=0, title=condition_name),
- )
- fig.for_each_annotation(lambda ann: ann.update(font=dict(size=12, color="#526171")))
- return fig
-
-
-def robustness_figure(dataset: str, models: list[str] | None, hover_data: dict | None = None) -> go.Figure:
- df = filter_models(present_rows(robustness), models)
- df = df[df["dataset"].astype(str) == str(dataset)].copy() if not df.empty else df
- if df.empty:
- return empty_figure("No robustness results are available for this selection.")
- df = add_method_columns(df).sort_values("model_order")
- highlighted_trace = None
- if hover_data and hover_data.get("points"):
- highlighted_trace = hover_data["points"][0].get("curveNumber")
-
- fig = go.Figure()
- trace_idx = 0
- for idx, row in enumerate(df.itertuples()):
- xs = parse_float_list(row.noise_levels)
- ys = parse_float_list(row.scores)
- if xs and len(xs) == len(ys):
- is_highlighted = highlighted_trace is None or highlighted_trace == trace_idx
- fig.add_trace(
- go.Scatter(
- x=xs,
- y=ys,
- mode="lines+markers",
- name=row.method,
- opacity=0.92 if is_highlighted else 0.16,
- line=dict(
- color=CATEGORICAL_PALETTE[idx % len(CATEGORICAL_PALETTE)],
- width=4 if highlighted_trace == trace_idx else 2,
- ),
- marker=dict(size=8 if highlighted_trace == trace_idx else 5),
- hovertemplate="Noise=%{x:.2f}
Score=%{y:.4f}",
- )
- )
- trace_idx += 1
- fig.update_layout(title=f"{DATASET_LABELS.get(dataset, dataset)} robustness curves")
- fig.update_layout(hovermode="closest")
- fig.update_xaxes(title="Noise fraction")
- metric = metric_text(df["metric"].dropna().iloc[0]) if df["metric"].notna().any() else "score"
- fig.update_yaxes(title=decoding_label(metric))
- fig_layout(fig, height=560)
+ score_rows = active_rows(consistency)
+ score_rows = score_rows[
+ (score_rows["dataset"].astype(str) == str(dataset))
+ & (score_rows["model"].astype(str) == str(model))
+ ]
+ score = pd.to_numeric(score_rows.get("mean_r2"), errors="coerce").dropna()
+ suffix = "" if score.empty else f" · latent-consistency R² = {float(score.iloc[0]):.3f}"
fig.update_layout(
- margin=dict(l=58, r=170, t=64, b=48),
- legend=dict(
- orientation="v",
- yanchor="top",
- y=1,
- xanchor="left",
- x=1.02,
- font=dict(size=11),
- title=None,
- ),
+ title=f"{model_label(model)} Figure 3-aligned representations{suffix}",
+ height=760 if rows > 1 else 540,
+ paper_bgcolor="#FFFFFF",
+ plot_bgcolor="#FFFFFF",
+ margin=dict(l=8, r=8, t=82, b=105),
+ font=dict(family="Arial, Helvetica, sans-serif", size=12, color=TEXT_COLOR),
+ legend=dict(orientation="h", yanchor="top", y=-0.07, xanchor="left", x=0, title=condition_name),
)
+ fig.for_each_annotation(lambda annotation: annotation.update(font=dict(size=12, color="#526171")))
return fig
-def robustness_table_frame(dataset: str, models: list[str] | None) -> pd.DataFrame:
- df = filter_models(present_rows(robustness), models)
- df = df[df["dataset"].astype(str) == str(dataset)].copy() if not df.empty else df
- if df.empty:
- return pd.DataFrame(columns=["method", "reference_score", "highest_noise_score", "robustness_auc", "average_noisy_score"])
- df = add_method_columns(df)
- df = df.rename(
+def consistency_frame(dataset: str, models: Sequence[str] | None) -> pd.DataFrame:
+ frame = filter_models(active_rows(consistency), models)
+ frame = frame[frame["dataset"].astype(str) == str(dataset)].copy()
+ if frame.empty:
+ return frame
+ frame = add_method_columns(frame)
+ frame = frame.rename(
columns={
- "score_at_noise0": "reference_score",
- "score_at_max_noise": "highest_noise_score",
- "raw_auc": "robustness_auc",
- "mean_score": "average_noisy_score",
+ "mean_r2": "latent_consistency_r2",
+ "n_sessions": "n_recordings",
}
)
- cols = ["method", "reference_score", "highest_noise_score", "robustness_auc", "average_noisy_score"]
- return round_numeric(df[cols].sort_values("robustness_auc", ascending=False), NUMERIC_COLUMNS)
+ return round_numeric(frame)
-def compute_figures(dataset: str, models: list[str] | None) -> tuple[go.Figure, go.Figure, pd.DataFrame]:
- df = filter_models(present_rows(scalability), models)
- df = df[df["dataset"].astype(str) == str(dataset)].copy() if not df.empty else df
- if df.empty:
- empty = pd.DataFrame(columns=["method", "hardware", "training_time_sec", "inference_time_sec", "peak_ram_gb", "peak_vram_gb"])
- return (
- empty_figure("No compute results are available for this selection."),
- empty_figure("No memory results are available for this selection."),
- empty,
+def consistency_figures(
+ dataset: str,
+ models: Sequence[str] | None,
+) -> tuple[go.Figure, go.Figure, pd.DataFrame]:
+ frame = consistency_frame(dataset, models)
+ if frame.empty:
+ message = (
+ "Cross-recording latent consistency is not defined for MC PacMan."
+ if dataset == "mc_pacman"
+ else "No latent-consistency result is available for this selection."
)
-
- pred = present_rows(prediction)[["model", "dataset", "score", "metric"]].copy()
- df = df.merge(pred, on=["model", "dataset"], how="left")
- df = add_method_columns(df)
- df["hardware"] = df["model"].map(hardware_label)
- for col in ["training_time_sec", "inference_time_sec", "peak_ram_gb", "peak_vram_gb", "score"]:
- df[col] = pd.to_numeric(df[col], errors="coerce")
-
- scatter = go.Figure(
- go.Scatter(
- x=df["training_time_sec"],
- y=df["score"],
- mode="markers",
- text=df["method"],
- marker=dict(
- size=np.clip(df["peak_ram_gb"].fillna(1.0) * 4, 8, 26),
- color=df["hardware"].map({"CPU": "#E69F00", "GPU": TASK_COLOR}).fillna("#637381"),
- opacity=0.82,
- line=dict(color="#ffffff", width=1),
- ),
- customdata=np.stack(
- [
- df["method"].astype(str),
- df["hardware"].astype(str),
- df["peak_ram_gb"].round(3).astype(str),
- df["peak_vram_gb"].round(3).astype(str),
- ],
- axis=-1,
- ),
+ columns = ["method", "latent_consistency_r2", "n_recordings", "latent_dim", "n_pairwise"]
+ return empty_figure(message), consistency_heatmap(models), pd.DataFrame(columns=columns)
+ bar = frame.sort_values(["latent_consistency_r2", "model_order"], ascending=[True, False])
+ bar_fig = go.Figure(
+ go.Bar(
+ x=bar["latent_consistency_r2"],
+ y=bar["method"],
+ orientation="h",
+ marker=dict(color=CONSISTENCY_COLOR),
+ customdata=np.stack([bar["n_recordings"], bar["latent_dim"], bar["n_pairwise"]], axis=-1),
hovertemplate=(
- "Method=%{customdata[0]}
"
- "Hardware=%{customdata[1]}
"
- "Training time=%{x:.3f} s
"
- "Score=%{y:.4f}
"
- "Peak memory=%{customdata[2]} GB
"
- "Peak GPU memory=%{customdata[3]} GB"
- ""
+ "Method=%{y}
Latent-consistency R²=%{x:.4f}
"
+ "Recordings=%{customdata[0]:.0f}
Latent dimensions=%{customdata[1]:.0f}
"
+ "Directional pairs=%{customdata[2]:.0f}"
),
)
)
- scatter.update_layout(title=f"{DATASET_LABELS.get(dataset, dataset)} performance and training time")
- scatter.update_xaxes(title="Training time (s, log scale)", type="log")
- scatter.update_yaxes(title=metric_text(df["metric"].dropna().iloc[0]) if df["metric"].notna().any() else "score")
- fig_layout(scatter, height=500)
+ bar_fig.update_layout(title=f"{DATASET_SHORT_LABELS.get(dataset, dataset)} latent consistency")
+ bar_fig.update_xaxes(title="Symmetric linear-alignment R²", range=[0, 1.02])
+ bar_fig.update_yaxes(title="", showgrid=False)
+ figure_layout(bar_fig, height=max(400, 27 * len(bar) + 145))
+ columns = ["method", "latent_consistency_r2", "n_recordings", "latent_dim", "n_pairwise"]
+ return bar_fig, consistency_heatmap(models), frame[columns].sort_values("latent_consistency_r2", ascending=False)
- mem_df = df.sort_values("peak_ram_gb", ascending=True)
- memory = go.Figure()
- memory.add_trace(go.Bar(x=mem_df["peak_ram_gb"], y=mem_df["method"], orientation="h", name="RAM"))
- memory.add_trace(go.Bar(x=mem_df["peak_vram_gb"], y=mem_df["method"], orientation="h", name="GPU memory"))
- memory.update_layout(title=f"{DATASET_LABELS.get(dataset, dataset)} peak memory", barmode="group")
- memory.update_xaxes(title="GB")
- memory.update_yaxes(title="")
- fig_layout(memory, height=max(420, 26 * len(mem_df) + 150))
-
- table = df[["method", "hardware", "score", "training_time_sec", "inference_time_sec", "peak_ram_gb", "peak_vram_gb"]]
- return scatter, memory, round_numeric(table.sort_values("training_time_sec"), NUMERIC_COLUMNS)
-
-
-def influence_figures(dataset: str, models: list[str] | None) -> tuple[go.Figure, go.Figure, pd.DataFrame]:
- nshap = filter_models(active_rows(neuron_shap), models)
- nshap = nshap[nshap["dataset"].astype(str) == str(dataset)].copy() if not nshap.empty else nshap
- if nshap.empty:
- neuron_fig = empty_figure("No neuron-influence results are available for this dataset.")
- table = pd.DataFrame(columns=["method", "baseline_score", "full_model_score", "neuron_influence_auc", "shap_mean_value", "shap_fraction_positive"])
- else:
- nshap = add_method_columns(nshap)
- nshap["auc"] = pd.to_numeric(nshap["auc"], errors="coerce")
- bar_df = nshap.dropna(subset=["auc"]).sort_values("auc", ascending=True)
- neuron_fig = go.Figure(
- go.Bar(
- x=bar_df["auc"],
- y=bar_df["method"],
- orientation="h",
- marker=dict(color=INFLUENCE_COLOR),
- hovertemplate="Method=%{y}
AUC=%{x:.4f}",
- )
- )
- neuron_fig.update_layout(title=f"{DATASET_LABELS.get(dataset, dataset)} neuron influence")
- neuron_fig.update_xaxes(title="Neuron influence AUC")
- neuron_fig.update_yaxes(title="")
- fig_layout(neuron_fig, height=max(420, 25 * len(bar_df) + 150))
- table = nshap.rename(columns={"auc": "neuron_influence_auc"})
- table = table[
- ["method", "baseline_score", "full_model_score", "neuron_influence_auc", "shap_mean_value", "shap_fraction_positive"]
- ]
- tshap = filter_models(active_rows(trial_shapley), models)
- tshap = tshap[tshap["dataset"].astype(str) == str(dataset)].copy() if not tshap.empty else tshap
- if tshap.empty:
- trial_fig = empty_figure("No trial-influence results are available for this dataset.")
- else:
- tshap = add_method_columns(tshap)
- tshap["perturbation_auc"] = pd.to_numeric(tshap["perturbation_auc"], errors="coerce")
- trial_df = tshap.dropna(subset=["perturbation_auc"]).sort_values("perturbation_auc", ascending=True)
- trial_fig = go.Figure(
- go.Bar(
- x=trial_df["perturbation_auc"],
- y=trial_df["method"],
- orientation="h",
- marker=dict(color="#009E73"),
- hovertemplate="Method=%{y}
AUC=%{x:.4f}",
+def consistency_heatmap(models: Sequence[str] | None) -> go.Figure:
+ chosen = selected_models(models)
+ frame = filter_models(active_rows(consistency), chosen)
+ if frame.empty:
+ return empty_figure("No cross-recording latent-consistency results are available.")
+ frame["mean_r2"] = pd.to_numeric(frame["mean_r2"], errors="coerce")
+ row_order = [model for model in FIGURE_MODEL_ORDER if model in CONSISTENCY_ELIGIBLE and model in chosen]
+ datasets = [dataset for dataset in DATASETS if dataset != "mc_pacman"]
+ pivot = frame.pivot_table(index="model", columns="dataset", values="mean_r2", aggfunc="first").reindex(
+ index=row_order, columns=datasets
+ )
+ text = np.empty(pivot.shape, dtype=object)
+ for row in range(pivot.shape[0]):
+ for column in range(pivot.shape[1]):
+ value = pivot.iloc[row, column]
+ text[row, column] = "×" if pd.isna(value) else f"{value:.2f}"
+ fig = go.Figure(
+ go.Heatmap(
+ z=pivot.to_numpy(dtype=float),
+ x=[DATASET_SHORT_LABELS[dataset] for dataset in pivot.columns],
+ y=[model_label(model) for model in pivot.index],
+ text=text,
+ texttemplate="%{text}",
+ colorscale=CONSISTENCY_SCALE,
+ zmin=0,
+ zmax=1,
+ colorbar=dict(title="R²", thickness=13),
+ hovertemplate="Method=%{y}
Dataset=%{x}
Latent-consistency R²=%{z:.4f}",
+ hoverongaps=False,
+ )
+ )
+ missing_rows, missing_columns = np.where(pivot.isna().to_numpy())
+ if len(missing_rows):
+ fig.add_trace(
+ go.Scatter(
+ x=[DATASET_SHORT_LABELS[pivot.columns[index]] for index in missing_columns],
+ y=[model_label(pivot.index[index]) for index in missing_rows],
+ mode="markers",
+ marker=dict(symbol="x", size=9, color="#7A858E", line=dict(width=1)),
+ showlegend=False,
+ hovertemplate="Method=%{y}
Dataset=%{x}
Status=Unavailable",
)
)
- trial_fig.update_layout(title=f"{DATASET_LABELS.get(dataset, dataset)} trial influence")
- trial_fig.update_xaxes(title="Trial influence AUC")
- trial_fig.update_yaxes(title="")
- fig_layout(trial_fig, height=max(420, 25 * len(trial_df) + 150))
-
- return neuron_fig, trial_fig, round_numeric(table, NUMERIC_COLUMNS)
-
-
-def methods_frame(models: list[str] | None) -> pd.DataFrame:
- chosen = selected_models(models)
+ fig.update_layout(title="Cross-recording latent consistency across tasks")
+ fig.update_xaxes(title="", side="top", showgrid=False)
+ fig.update_yaxes(title="", showgrid=False)
+ return heatmap_layout(fig, height=max(470, 27 * len(pivot) + 180))
+
+
+def analysis_status(model: str, dataset: str, analysis: str) -> tuple[str, str]:
+ if analysis == "prediction":
+ available = row_exists(present_rows(prediction), model, dataset)
+ return ("Available", "") if available else ("Unavailable", missing_reason(analysis, model, dataset))
+ if analysis == "robustness":
+ available = row_exists(present_rows(robustness), model, dataset)
+ return ("Available", "") if available else ("Unavailable", missing_reason("prediction", model, dataset))
+ if analysis == "compute":
+ available = row_exists(present_rows(scalability), model, dataset)
+ return ("Available", "") if available else ("Unavailable", missing_reason("prediction", model, dataset))
+ if analysis == "consistency":
+ if dataset == "mc_pacman":
+ return "Not defined", "MC PacMan has no cross-recording consistency cohort."
+ if model not in CONSISTENCY_ELIGIBLE:
+ return "Not supported", "No predefined common three-dimensional representation for this analysis."
+ available = row_exists(active_rows(consistency), model, dataset)
+ return ("Available", "") if available else ("Unavailable", missing_reason(analysis, model, dataset))
+ if analysis == "feature":
+ if model == "marble":
+ if dataset == "allen_neuropixels":
+ return (
+ "Unavailable",
+ "MARBLE exceeded the host-memory allocation before feature attribution.",
+ )
+ return (
+ "Not supported",
+ "Masked-input evaluation changes MARBLE's transductive graph construction.",
+ )
+ if model not in FEATURE_ELIGIBLE:
+ return "Not supported", "No predefined feature-attribution interface."
+ available = row_exists(active_rows(neuron_shap), model, dataset)
+ return ("Available", "") if available else ("Unavailable", missing_reason(analysis, model, dataset))
+ if analysis == "trial":
+ if model not in TRIAL_ELIGIBLE:
+ return "Not supported", "The fixed-output/lightweight-rescoring Data Shapley rule is unavailable for this method."
+ available = row_exists(active_rows(trial_shapley), model, dataset)
+ return ("Available", "") if available else ("Unavailable", missing_reason(analysis, model, dataset))
+ raise ValueError(f"Unknown analysis: {analysis}")
+
+
+def methods_frame(dataset: str, models: Sequence[str] | None) -> pd.DataFrame:
rows = []
- for model in chosen:
+ for model in selected_models(models):
+ pred_rows = prediction[
+ (prediction["model"].astype(str) == model)
+ & (prediction["dataset"].astype(str) == str(dataset))
+ ]
+ pred = pred_rows.iloc[0] if not pred_rows.empty else pd.Series(dtype=object)
+ statuses = {}
+ notes = []
+ for analysis in ["prediction", "robustness", "compute", "consistency", "feature", "trial"]:
+ status, note = analysis_status(model, dataset, analysis)
+ statuses[analysis] = status
+ if note and status != "Available":
+ notes.append(f"{TABLE_LABELS.get(analysis + '_status', analysis.title())}: {note}")
rows.append(
{
- "id": model,
"method": model_label(model),
- "family": METHOD_FAMILY.get(model, "Model"),
- "hardware": hardware_label(model),
+ "workflow": prediction_workflow(model, pred.get("decoder"), pred.get("status")),
+ "readout": readout_label(pred.get("decoder")),
+ "hardware": "CPU" if model in CPU_ONLY_MODELS else "GPU",
+ "prediction_status": statuses["prediction"],
+ "robustness_status": statuses["robustness"],
+ "compute_status": statuses["compute"],
+ "consistency_status": statuses["consistency"],
+ "feature_status": statuses["feature"],
+ "trial_status": statuses["trial"],
+ "coverage_notes": " ".join(notes) if notes else "Complete for configured analyses.",
+ "model_order": MODEL_INDEX[model],
}
)
- return pd.DataFrame(rows).sort_values("method")
+ return pd.DataFrame(rows).sort_values("model_order").drop(columns="model_order")
+
+
+def coverage_cards(dataset: str, models: Sequence[str] | None) -> list[html.Div]:
+ chosen = selected_models(models)
+ cards = []
+ specs = [
+ ("Prediction", "prediction", set(MODELS), "prediction"),
+ ("Robustness", "robustness", set(MODELS), "robustness"),
+ ("Computational cost", "compute", set(MODELS), "compute"),
+ ("Latent consistency", "consistency", CONSISTENCY_ELIGIBLE, "consistency"),
+ ("Feature attribution", "feature", FEATURE_ELIGIBLE, "feature"),
+ ("Trial valuation", "trial", TRIAL_ELIGIBLE, "trial"),
+ ]
+ accents = {
+ "prediction": "prediction",
+ "robustness": "robustness",
+ "compute": "compute",
+ "consistency": "consistency",
+ "feature": "feature",
+ "trial": "trial",
+ }
+ for label, analysis, eligible, accent in specs:
+ if analysis == "consistency" and dataset == "mc_pacman":
+ cards.append(metric_card(label, "Not defined", "No cross-recording cohort.", accents[accent]))
+ continue
+ configured = [model for model in chosen if model in eligible]
+ available = sum(analysis_status(model, dataset, analysis)[0] == "Available" for model in configured)
+ unsupported = len(chosen) - len(configured)
+ detail = f"{len(configured)} configured"
+ if unsupported:
+ detail += f"; {unsupported} outside analysis"
+ cards.append(metric_card(label, f"{available}/{len(configured)}", detail, accents[accent]))
+ return cards
-app = Dash(__name__, title="Neural Model Benchmark")
+def release_detail() -> str:
+ stable_parts = []
+ manuscript_version = release_manifest.get("manuscript_working_version")
+ figure_set = release_manifest.get("figure_set")
+ if manuscript_version:
+ version_label = str(manuscript_version).replace("_", " ")
+ stable_parts.append(
+ version_label
+ if version_label.lower().startswith("manuscript ")
+ else f"manuscript {version_label}"
+ )
+ if figure_set:
+ stable_parts.append(str(figure_set))
+ if stable_parts:
+ return " Release alignment: " + "; ".join(stable_parts) + "."
+ for key in ["generated_at", "release_date", "created_at", "timestamp"]:
+ value = release_manifest.get(key)
+ if value:
+ return f" Release manifest: {value}."
+ return ""
+
+
+app = Dash(__name__, title="BEND-BCI Interactive Benchmark")
server = app.server
+
+@server.route("/download/")
+def download_data(filename: str):
+ if filename not in DOWNLOADABLE_FILES:
+ abort(404)
+ return send_from_directory(DATA_DIR, filename, as_attachment=True)
+
+
app.layout = html.Div(
[
- html.Div(
+ html.Header(
[
html.Div(
[
- html.Div("Tang Lab", className="eyebrow"),
- html.H1("Neural Model Benchmark"),
+ html.Div("Tang Lab · Interactive companion to Figures 2–5", className="eyebrow"),
+ html.H1("BEND-BCI Interactive Benchmark"),
html.P(
- "Explore decoding performance, robustness, latent alignment, neuron and trial influence, and compute cost across 23 benchmarked methods.",
+ "Compare held-out task prediction, robustness to noisy neural inputs, computational cost and cross-recording latent consistency across 23 neural decoding methods. Feature attribution and trial valuation are reported as separate validation-checked diagnostics.",
className="lede",
),
+ html.Nav(
+ [
+ html.A(
+ "Benchmark repository",
+ href="https://github.com/TangLab-UBC/behavior_benchmarking",
+ target="_blank",
+ rel="noopener noreferrer",
+ ),
+ html.A(
+ "Hugging Face Space",
+ href="https://huggingface.co/spaces/Tang-Lab/benchdash",
+ target="_blank",
+ rel="noopener noreferrer",
+ ),
+ ],
+ className="hero-links",
+ **{"aria-label": "Project links"},
+ ),
],
className="hero-copy",
),
+ html.Div(
+ [
+ html.Span("Six reported views", className="hero-stat-value"),
+ html.Span("Four selection measurements + two diagnostics", className="hero-stat-label"),
+ ],
+ className="hero-stat",
+ ),
],
className="hero",
),
@@ -1322,241 +2456,363 @@ app.layout = html.Div(
[
html.Div(
[
- html.Label("Dataset"),
+ html.Label("Dataset", htmlFor="dataset-filter"),
dcc.Dropdown(
id="dataset-filter",
- options=[{"label": DATASET_LABELS.get(ds, ds), "value": ds} for ds in DATASETS],
- value=DATASETS[0] if DATASETS else None,
+ options=[{"label": DATASET_LABELS[dataset], "value": dataset} for dataset in DATASETS],
+ value=DATASETS[0],
clearable=False,
+ searchable=False,
),
],
className="control",
),
+ html.Div(
+ [
+ html.Label("Compare methods", htmlFor="method-filter"),
+ dcc.Dropdown(
+ id="method-filter",
+ options=[{"label": model_label(model), "value": model} for model in FIGURE_MODEL_ORDER],
+ value=[],
+ multi=True,
+ placeholder="All 23 methods",
+ ),
+ html.Div("Leave empty to show all methods.", className="control-help"),
+ ],
+ className="control method-control",
+ ),
],
className="toolbar",
),
dcc.Tabs(
id="tabs",
- value="leaderboard",
+ value="overview",
className="tabs",
children=[
dcc.Tab(
- label="Leaderboard",
- value="leaderboard",
+ label="Overview",
+ value="overview",
className="tab",
selected_className="tab tab-selected",
children=[
panel(
- "Leaderboard",
- html.Div(id="leaderboard-top", className="top-strip"),
+ "Held-out task prediction",
+ html.Div(id="overview-cards", className="metric-strip"),
+ html.Div(id="overview-coverage", className="coverage-note"),
html.Div(
[
- html.Div(
- [
- html.Div(id="leaderboard-sort-state", className="sort-state"),
- dataframe_table("leaderboard-table", page_size=23, max_height="680px", sort_action="custom"),
- ],
- className="leaderboard-table-wrap",
- ),
- dcc.Graph(id="dataset-ranking", config={"displayModeBar": False}),
+ graph_box("prediction-ranking", "Raw held-out prediction scores for the selected dataset."),
+ graph_box("prediction-heatmap", "Within-dataset prediction percentiles across five tasks."),
],
- className="leaderboard-grid",
+ className="chart-grid two",
),
- dcc.Graph(id="performance-heatmap", config={"displayModeBar": False}),
- subtitle="Sorted by the selected dataset. Regression datasets use decoding R2; classification datasets use decoding accuracy. Higher is better.",
- className="leaderboard-panel",
- )
+ details_table("View and filter prediction rows", dataframe_table("overview-table", page_size=23)),
+ source_link("clean_prediction_summary.csv"),
+ eyebrow="Figure 2 · Prediction",
+ subtitle="Raw accuracy and R² remain visible within each task. The cross-task matrix uses within-dataset percentiles, with the raw task-specific value and metric on hover.",
+ class_name="axis-prediction",
+ ),
+ panel(
+ "Robustness to noisy neural inputs",
+ html.Div(id="robustness-coverage", className="coverage-note"),
+ graph_box("robustness-curve", "Task score as additive Poisson count-noise level increases."),
+ details_table("View and filter robustness rows", dataframe_table("robustness-table", page_size=23)),
+ source_link("robustness_summary.csv"),
+ eyebrow="Figure 2 · Robustness",
+ subtitle="The trained model and targets are held fixed while additive Poisson count noise is applied to test inputs at λ = 0, 0.2, 0.4, 0.6 and 0.8. The reported summary is the raw task-score-versus-noise area; it is unbounded, may be negative and is not normalized to λ = 0.",
+ class_name="axis-robustness",
+ ),
+ panel(
+ "Computational cost",
+ html.Div(id="compute-coverage", className="coverage-note"),
+ html.Div(
+ [
+ graph_box("runtime-bars", "Training and complete-held-out-split inference times."),
+ graph_box("memory-bars", "Peak RAM and GPU memory."),
+ ],
+ className="chart-grid two",
+ ),
+ details_table("View and filter computational-cost rows", dataframe_table("compute-table", page_size=23)),
+ source_link("scalability_summary.csv"),
+ eyebrow="Figure 2 · Supplementary Figure 11",
+ subtitle="Figure 2 reports the macaque-reaching resource block; the other selected datasets are per-dataset extensions reported in Supplementary Figure 11. Training time includes required training-side representation extraction and readout fitting. Inference time is one complete pass over the held-out split, including readout prediction and excluding metric calculation. CPU-only methods have no GPU-memory value.",
+ class_name="axis-compute",
+ ),
],
),
dcc.Tab(
- label="Consistency",
+ label="Latent consistency",
value="consistency",
- id="consistency-tab",
className="tab",
selected_className="tab tab-selected",
children=[
panel(
- "Cross-session alignment",
+ "Cross-recording latent consistency",
+ html.Div(id="consistency-coverage", className="coverage-note"),
html.Div(
[
html.Div(
[
- html.Div(id="consistency-sort-state", className="sort-state"),
- dataframe_table(
- "consistency-table",
- page_size=23,
- max_height="520px",
- sort_action="custom",
- sort_by=[{"column_id": "alignment_score", "direction": "desc"}],
- ),
+ html.Label("Representation", htmlFor="consistency-method"),
+ dcc.Dropdown(id="consistency-method", clearable=False),
],
- className="consistency-table-wrap",
+ className="control",
),
html.Div(
[
- html.Div(
- [
- html.Label("Color by"),
- dcc.Dropdown(
- id="latent-color-mode",
- clearable=False,
- ),
- ],
- id="latent-color-control",
- className="control latent-color-control",
- ),
- dcc.Graph(
- id="latent-space",
- config={
- "displayModeBar": "hover",
- "toImageButtonOptions": {
- "format": "png",
- "filename": "benchdash_latent_space",
- "height": 900,
- "width": 1200,
- "scale": 2,
- },
- },
- ),
+ html.Label("Color by", htmlFor="latent-color-mode"),
+ dcc.Dropdown(id="latent-color-mode", clearable=False),
],
- className="latent-panel",
+ id="latent-color-control",
+ className="control",
),
],
- className="latent-grid",
+ className="inline-controls",
+ ),
+ html.Div(
+ [
+ html.Strong("Display-coordinate note: "),
+ "Representations are centered and whitened within recording and transformed into the Figure 3 display frame. The displayed coordinates illustrate matched structure; the reported symmetric R² is computed from bidirectional intercept-free linear alignment of task landmarks.",
+ ],
+ className="method-note",
),
+ graph_box("latent-space", "Figure 3-aligned latent representations for each recording.", class_name="latent-graph"),
html.Div(
[
- dcc.Graph(id="consistency-bars", config={"displayModeBar": False}),
- dcc.Graph(id="consistency-heatmap", config={"displayModeBar": False}),
+ graph_box("consistency-bars", "Latent-consistency R-squared for the selected dataset."),
+ graph_box("consistency-heatmap", "Latent-consistency R-squared across four tasks."),
],
className="chart-grid two",
),
- subtitle="Each latent panel shows one recording session. Colors indicate task condition, stimulus, cue, or spatial bin.",
+ details_table("View and filter latent-consistency rows", dataframe_table("consistency-table", page_size=12)),
+ source_link("consistency_summary.csv"),
+ eyebrow="Figure 3",
+ subtitle="Latent consistency asks whether matched task-defined landmarks are linearly alignable across sessions, participants or independent simulations. It does not measure decoder transfer or identify a unique latent coordinate system.",
+ class_name="axis-consistency",
)
],
),
dcc.Tab(
- label="Robustness",
- value="robustness",
+ label="Feature attribution",
+ value="feature",
className="tab",
selected_className="tab tab-selected",
children=[
panel(
- "Robustness",
- dcc.Graph(id="robustness-curve", clear_on_unhover=True, config={"displayModeBar": False}),
- details_table("View robustness rows", dataframe_table("robustness-table")),
- subtitle="Hover a method to highlight its curve. Curves show how decoding performance changes as neural count noise increases.",
+ "Feature-attribution validation",
+ html.Div(id="feature-coverage", className="coverage-note"),
+ html.Div(
+ [
+ graph_box("feature-validation-bars", "Feature-attribution validation metric for the selected dataset."),
+ graph_box("feature-signed-bars", "Mean signed global Kernel SHAP values for the selected dataset."),
+ ],
+ className="chart-grid two",
+ ),
+ graph_box("feature-heatmap", "Within-dataset feature-attribution validation percentiles across tasks."),
+ html.Div(
+ [
+ html.Strong("Metric interpretation: "),
+ html.Span(id="feature-definition"),
+ " Validation scores and signed contribution summaries are distinct quantities. Signed values are preserved throughout; no absolute-value ranking is applied.",
+ ],
+ className="method-note",
+ ),
+ details_table("View validation and signed Kernel SHAP summaries", dataframe_table("feature-table", page_size=23)),
+ source_link("neuron_shap_summary.csv"),
+ eyebrow="Figure 4",
+ subtitle="Global Kernel SHAP estimates each neural feature’s signed contribution to the task score. Dataset-specific validation tests agreement with a constructed control, known simulated feature class or independently measured biological tuning proxy.",
+ class_name="axis-feature",
)
],
),
dcc.Tab(
- label="Influence",
- value="influence",
+ label="Trial valuation",
+ value="trial",
className="tab",
selected_className="tab tab-selected",
children=[
panel(
- "Neuron and trial influence",
+ "Controlled corrupted-trial detection",
+ html.Div(id="trial-coverage", className="coverage-note"),
+ html.Div(id="trial-convergence", className="convergence-note"),
html.Div(
[
- dcc.Graph(id="neuron-influence-bars", config={"displayModeBar": False}),
- dcc.Graph(id="trial-influence-bars", config={"displayModeBar": False}),
+ graph_box("trial-detection-bars", "Corrupted-trial detection ROC-AUC for the selected dataset."),
+ graph_box("trial-heatmap", "Within-dataset corrupted-trial detection percentiles across tasks."),
],
className="chart-grid two",
),
- details_table("View neuron-influence rows", dataframe_table("influence-table")),
- subtitle="Signed contribution summaries preserve whether neurons or trials helped or hurt prediction.",
- )
- ],
- ),
- dcc.Tab(
- label="Compute",
- value="compute",
- className="tab",
- selected_className="tab tab-selected",
- children=[
+ html.Div(
+ [
+ html.Strong("Controlled assay: "),
+ "Approximately one third of training trials were rotated by 75° in the full population-activity space while targets were unchanged. ROC-AUC uses negative trial value as the corruption-detection score. Hatched bars did not meet the TMC convergence threshold; they remain visible and are not counted as missing.",
+ ],
+ className="method-note",
+ ),
+ details_table("View detection, convergence and signed Data Shapley summaries", dataframe_table("trial-table", page_size=23)),
+ source_link("trial_shapley_summary.csv"),
+ eyebrow="Figure 5a",
+ subtitle="Data Shapley assigns each candidate training trial a signed marginal contribution to a specified held-out decoding utility. Values are model-, readout-, split- and metric-specific.",
+ class_name="axis-trial",
+ ),
panel(
- "Performance and compute cost",
+ "Macaque intervention case studies",
+ html.Div(intervention_coverage_note(), className="method-note caveat-note"),
html.Div(
[
- dcc.Graph(id="compute-scatter", config={"displayModeBar": False}),
- dcc.Graph(id="memory-bars", config={"displayModeBar": False}),
+ graph_box("trial-removal", "Held-out R-squared before and after trial-value-guided corrupted-trial removal."),
+ graph_box("trial-recovery", "Relationship between corrupted-trial detection and recovery after removal."),
],
className="chart-grid two",
),
- details_table("View compute rows", dataframe_table("compute-table")),
- subtitle="Shows training time and peak memory for each method on the selected dataset.",
- )
+ graph_box("trial-historical", "Same-subject historical-trial selection compared with current-session training."),
+ graph_box(
+ "trial-historical-trajectories",
+ "Held-out RNN target-session trajectories for ground truth, current-session training, and nonnegative-valued historical-trial selection.",
+ class_name="historical-trajectory-graph",
+ ),
+ details_table("View within-session removal and historical-selection rows", dataframe_table("trial-retrain-table", page_size=17)),
+ html.Div(
+ [
+ source_link("trial_shapley_retrain_summary.csv", "Intervention summary CSV"),
+ source_link("trial_historical_trajectories.csv", "Figure 5e trajectory CSV"),
+ ],
+ className="download-grid panel-downloads",
+ ),
+ eyebrow="Figure 5b–e",
+ subtitle="The within-session case study removes negative-valued corrupted trials and retrains each decoder. The historical-selection case study retains current-session trials and adds nonnegative-valued trials from earlier sessions in a shared M1 feature space.",
+ class_name="axis-trial",
+ ),
],
),
dcc.Tab(
- label="Methods",
+ label="Methods & coverage",
value="methods",
className="tab",
selected_className="tab tab-selected",
children=[
panel(
- "Benchmarked methods",
- dataframe_table("methods-table", page_size=23, max_height="600px"),
- subtitle="Paper-facing method names and broad method families.",
+ "Prediction workflows and analysis coverage",
+ html.Div(id="coverage-cards", className="metric-strip coverage-cards"),
+ html.Div(
+ [
+ html.Strong("Prediction workflows: "),
+ "Each method–dataset cell follows one of three paper-defined paths: predictions generated directly by the trained model, a method-specific mapping from learned outputs to benchmark targets, or a shared ridge/logistic readout on fixed model outputs. The exact path and implementation can vary by dataset.",
+ ],
+ className="method-note",
+ ),
+ dataframe_table("methods-table", page_size=23, max_height="760px"),
+ html.Div(
+ [
+ html.Strong("Coverage terminology: "),
+ "Available means a completed summary is bundled. Unavailable means a configured analysis did not produce a completed result. Not supported means the method was outside a predefined analysis interface. Not defined means the dataset has no corresponding analysis cohort.",
+ ],
+ className="method-note",
+ ),
+ html.Div(
+ [
+ source_link("clean_prediction_summary.csv", "Prediction CSV"),
+ source_link("robustness_summary.csv", "Robustness CSV"),
+ source_link("consistency_summary.csv", "Consistency CSV"),
+ source_link("scalability_summary.csv", "Cost CSV"),
+ source_link("neuron_shap_summary.csv", "Feature-attribution CSV"),
+ source_link("trial_shapley_summary.csv", "Trial-valuation CSV"),
+ source_link("trial_shapley_retrain_summary.csv", "Intervention CSV"),
+ ],
+ className="download-grid",
+ ),
+ eyebrow="Methods and provenance",
+ subtitle="Coverage is shown per method and selected dataset. Unsupported and failed entries remain explicit so chart denominators can be interpreted.",
+ class_name="axis-methods",
)
],
),
],
),
+ html.Footer(
+ [
+ html.Strong("Data provenance. "),
+ "Dashboard tables are synchronized from the paper/results exports used to generate the current Figures 2–5 and Supplementary Figures. Latent coordinates are Figure-3-aligned display coordinates exported from the same analysis artifacts; display transforms do not change the reported consistency scores.",
+ release_detail(),
+ ],
+ className="provenance-footer",
+ ),
],
className="app-shell",
)
@app.callback(
- Output("consistency-tab", "style"),
- Output("consistency-tab", "disabled"),
- Output("tabs", "value"),
- Input("dataset-filter", "value"),
- State("tabs", "value"),
-)
-def update_available_tabs(dataset: str, current_tab: str | None):
- if str(dataset) in CONSISTENCY_DATASETS:
- return {}, False, current_tab or "leaderboard"
- next_tab = "leaderboard" if current_tab == "consistency" else (current_tab or "leaderboard")
- return {"display": "none"}, True, next_tab
-
-
-@app.callback(
- Output("leaderboard-top", "children"),
- Output("leaderboard-sort-state", "children"),
- Output("leaderboard-table", "columns"),
- Output("leaderboard-table", "data"),
- Output("dataset-ranking", "figure"),
- Output("performance-heatmap", "figure"),
+ Output("overview-cards", "children"),
+ Output("overview-coverage", "children"),
+ Output("overview-table", "columns"),
+ Output("overview-table", "data"),
+ Output("prediction-ranking", "figure"),
+ Output("prediction-heatmap", "figure"),
+ Output("robustness-coverage", "children"),
+ Output("robustness-curve", "figure"),
+ Output("robustness-table", "columns"),
+ Output("robustness-table", "data"),
+ Output("compute-coverage", "children"),
+ Output("runtime-bars", "figure"),
+ Output("memory-bars", "figure"),
+ Output("compute-table", "columns"),
+ Output("compute-table", "data"),
Input("dataset-filter", "value"),
- Input("leaderboard-table", "sort_by"),
+ Input("method-filter", "value"),
)
-def update_leaderboard(dataset: str, sort_by: list[dict] | None):
- df = leaderboard_frame(dataset, None)
- visible_cols = [
+def update_overview(dataset: str, models: list[str] | None):
+ dataset = dataset or DATASETS[0]
+ frame = overview_frame(dataset, models)
+ overview_columns = [
"method",
+ "workflow",
"task_score",
+ "prediction_percentile",
"robustness_auc",
"training_time_sec",
"inference_time_sec",
"peak_ram_gb",
"peak_vram_gb",
+ "prediction_status",
+ ]
+ overview_table = frame.sort_values(
+ ["task_score", "model_order"],
+ ascending=[False, True],
+ na_position="last",
+ )[overview_columns]
+ robustness_table = robustness_frame(dataset, models)
+ robustness_columns = [
+ "method",
+ "unperturbed_score",
+ "highest_noise_score",
+ "robustness_auc",
+ "average_noisy_score",
]
- if str(dataset) in CONSISTENCY_DATASETS:
- visible_cols.insert(3, "alignment_score")
- sort_by = supported_sort(sort_by, visible_cols)
- sorted_df = sort_table(df, sort_by, [("task_score", False), ("model_order", True)])
- table_df = sorted_df[[c for c in visible_cols + ["id", "model"] if c in sorted_df.columns]]
- metric = df["metric"].dropna().iloc[0] if df["metric"].notna().any() else "score"
+ robustness_table = robustness_table[robustness_columns].dropna(subset=["robustness_auc"]).sort_values(
+ "robustness_auc", ascending=False
+ )
+ runtime, memory, compute_table = compute_figures(dataset, models)
+ chosen = selected_models(models)
+ prediction_available = int(frame["task_score"].notna().sum())
+ robustness_available = len(robustness_table)
+ compute_available = len(compute_table)
return (
- leaderboard_summary(dataset, df),
- sort_status(sort_by, ("task_score", False), {"task_score": decoding_label(metric)}),
- leaderboard_columns([c for c in visible_cols if c in table_df.columns], metric),
- records(table_df),
- ranking_figure(dataset, df),
- performance_heatmap(dataset, None),
+ overview_cards(dataset, models),
+ availability_note(prediction_available, len(chosen)),
+ column_defs(overview_columns),
+ records(round_numeric(overview_table)),
+ prediction_ranking_figure(dataset, models),
+ prediction_heatmap(models),
+ availability_note(robustness_available, len(chosen)),
+ robustness_figure(dataset, models),
+ column_defs(robustness_columns),
+ records(round_numeric(robustness_table)),
+ availability_note(compute_available, len(chosen)),
+ runtime,
+ memory,
+ column_defs(compute_table.columns),
+ records(compute_table),
)
@@ -1567,115 +2823,190 @@ def update_leaderboard(dataset: str, sort_by: list[dict] | None):
Input("dataset-filter", "value"),
)
def update_latent_color_control(dataset: str):
- if str(dataset) == "ratinabox":
+ if dataset == "ratinabox":
return (
[
- {"label": "Position bin", "value": "condition"},
- {"label": "X position", "value": "x"},
- {"label": "Y position", "value": "y"},
+ {"label": "Spatial bin", "value": "condition"},
+ {"label": "X-position bin", "value": "x"},
+ {"label": "Y-position bin", "value": "y"},
],
- "condition",
+ "x",
{},
)
- return ([{"label": condition_axis_label(str(dataset)), "value": "condition"}], "condition", {"display": "none"})
+ return ([{"label": condition_axis_label(dataset), "value": "condition"}], "condition", {"display": "none"})
+
+
+@app.callback(
+ Output("consistency-method", "options"),
+ Output("consistency-method", "value"),
+ Output("consistency-method", "disabled"),
+ Output("consistency-coverage", "children"),
+ Input("dataset-filter", "value"),
+ Input("method-filter", "value"),
+ State("consistency-method", "value"),
+)
+def update_consistency_selector(dataset: str, models: list[str] | None, current: str | None):
+ dataset = dataset or DATASETS[0]
+ chosen = selected_models(models)
+ eligible = [model for model in chosen if model in CONSISTENCY_ELIGIBLE]
+ if dataset == "mc_pacman":
+ return [], None, True, "Not defined for MC PacMan: no cross-recording consistency cohort is configured."
+ frame = consistency_frame(dataset, models)
+ latent_pairs = set(zip(latent_samples["model"].astype(str), latent_samples["dataset"].astype(str)))
+ available = []
+ if not frame.empty:
+ available = [
+ model
+ for model in frame.sort_values("latent_consistency_r2", ascending=False)["model"].astype(str)
+ if (model, dataset) in latent_pairs
+ ]
+ options = [{"label": model_label(model), "value": model} for model in available]
+ value = current if current in available else (available[0] if available else None)
+ note = availability_note(len(available), len(eligible), unsupported=len(chosen) - len(eligible))
+ return options, value, not bool(options), note
@app.callback(
- Output("consistency-table", "columns"),
- Output("consistency-table", "data"),
- Output("consistency-sort-state", "children"),
- Output("consistency-sort-state", "style"),
Output("latent-space", "figure"),
Output("consistency-bars", "figure"),
Output("consistency-heatmap", "figure"),
+ Output("consistency-table", "columns"),
+ Output("consistency-table", "data"),
Input("dataset-filter", "value"),
- Input("consistency-table", "active_cell"),
- Input("consistency-table", "sort_by"),
+ Input("method-filter", "value"),
+ Input("consistency-method", "value"),
Input("latent-color-mode", "value"),
)
def update_consistency(
dataset: str,
- active_cell: dict | None,
- sort_by: list[dict] | None,
+ models: list[str] | None,
+ method: str | None,
color_mode: str | None,
):
- if str(dataset) not in CONSISTENCY_DATASETS:
- label = DATASET_LABELS.get(dataset, dataset)
- message = f"Cross-session alignment is not defined for {label}."
- return (
- [],
- [],
- "",
- {"display": "none"},
- empty_figure(message),
- empty_figure(message),
- consistency_heatmap(None),
- )
- df = consistency_frame(dataset, None)
- model = selected_consistency_model(df, active_cell)
- visible_cols = ["method", "alignment_score", "n_sessions", "latent_dim", "n_pairwise"]
- sort_by = supported_sort(sort_by, visible_cols)
- sorted_df = sort_table(df, sort_by, [("alignment_score", False), ("model_order", True)])
- table_df = sorted_df[[c for c in visible_cols + ["id", "model"] if c in sorted_df.columns]]
+ dataset = dataset or DATASETS[0]
+ bars, heatmap, table = consistency_figures(dataset, models)
return (
- column_defs([c for c in visible_cols if c in table_df.columns]),
- records(table_df),
- sort_status(sort_by, ("alignment_score", False)),
- {},
- latent_space_figure(dataset, model, color_mode or "condition"),
- consistency_bar_figure(dataset, df),
- consistency_heatmap(None),
+ latent_space_figure(dataset, method, color_mode or "condition"),
+ bars,
+ heatmap,
+ column_defs(table.columns),
+ records(round_numeric(table)),
)
@app.callback(
- Output("robustness-curve", "figure"),
- Output("robustness-table", "columns"),
- Output("robustness-table", "data"),
+ Output("feature-coverage", "children"),
+ Output("feature-definition", "children"),
+ Output("feature-validation-bars", "figure"),
+ Output("feature-signed-bars", "figure"),
+ Output("feature-heatmap", "figure"),
+ Output("feature-table", "columns"),
+ Output("feature-table", "data"),
Input("dataset-filter", "value"),
- Input("robustness-curve", "hoverData"),
+ Input("method-filter", "value"),
)
-def update_robustness(dataset: str, hover_data: dict | None):
- table = robustness_table_frame(dataset, None)
+def update_feature(dataset: str, models: list[str] | None):
+ dataset = dataset or DATASETS[0]
+ chosen = selected_models(models)
+ eligible = [model for model in chosen if model in FEATURE_ELIGIBLE]
+ frame = feature_frame(dataset, models)
+ validation_fig, signed_fig, table = feature_figures(dataset, models)
+ _column, target, metric, reference = feature_spec(dataset)
+ definition = f"{metric} quantifies agreement with {target.lower()}."
+ if reference == 0.5:
+ definition += " Chance ROC-AUC is 0.5."
+ else:
+ definition += " Larger positive values indicate closer agreement; this proxy is not causal ground truth."
return (
- robustness_figure(dataset, None, hover_data),
+ feature_coverage_note(dataset, models),
+ definition,
+ validation_fig,
+ signed_fig,
+ feature_heatmap(models),
column_defs(table.columns),
records(table),
)
@app.callback(
- Output("compute-scatter", "figure"),
- Output("memory-bars", "figure"),
- Output("compute-table", "columns"),
- Output("compute-table", "data"),
- Input("dataset-filter", "value"),
-)
-def update_compute(dataset: str):
- scatter, memory, table = compute_figures(dataset, None)
- return scatter, memory, column_defs(table.columns), records(table)
-
-
-@app.callback(
- Output("neuron-influence-bars", "figure"),
- Output("trial-influence-bars", "figure"),
- Output("influence-table", "columns"),
- Output("influence-table", "data"),
+ Output("trial-coverage", "children"),
+ Output("trial-convergence", "children"),
+ Output("trial-detection-bars", "figure"),
+ Output("trial-heatmap", "figure"),
+ Output("trial-table", "columns"),
+ Output("trial-table", "data"),
+ Output("trial-removal", "figure"),
+ Output("trial-recovery", "figure"),
+ Output("trial-historical", "figure"),
+ Output("trial-historical-trajectories", "figure"),
+ Output("trial-retrain-table", "columns"),
+ Output("trial-retrain-table", "data"),
Input("dataset-filter", "value"),
+ Input("method-filter", "value"),
)
-def update_influence(dataset: str):
- neuron_fig, trial_fig, table = influence_figures(dataset, None)
- return neuron_fig, trial_fig, column_defs(table.columns), records(table)
+def update_trial(dataset: str, models: list[str] | None):
+ dataset = dataset or DATASETS[0]
+ chosen = selected_models(models)
+ eligible = [model for model in chosen if model in TRIAL_ELIGIBLE]
+ frame = trial_frame(dataset, models)
+ table_columns = [
+ "method",
+ "corrupted_trial_auc",
+ "converged",
+ "iterations",
+ "final_error",
+ "perturbation_fraction",
+ "rotation_angle_deg",
+ "rotation_subspace_dim_spec",
+ "shapley_mean_value",
+ "shapley_median_value",
+ "shapley_min_value",
+ "shapley_max_value",
+ "shapley_fraction_positive",
+ "shapley_fraction_negative",
+ ]
+ table = frame[[column for column in table_columns if column in frame.columns]].sort_values(
+ "corrupted_trial_auc", ascending=False
+ ) if not frame.empty else pd.DataFrame(columns=table_columns)
+ all_trial = active_rows(trial_shapley)
+ converged_mask = all_trial["converged"].fillna(False).astype(str).str.lower().isin({"true", "1", "yes"})
+ converged = int(converged_mask.sum())
+ nonconverged = int(len(all_trial) - converged)
+ selected_converged = int((frame.get("converged", pd.Series(dtype=str)) == "Yes").sum())
+ selected_nonconverged = int(len(frame) - selected_converged)
+ removal, relation, historical, retrain_table = trial_retrain_figures(models)
+ return (
+ availability_note(len(frame), len(eligible), unsupported=len(chosen) - len(eligible)),
+ (
+ f"Selected view: {selected_converged}/{len(frame)} summaries converged; "
+ f"{selected_nonconverged}/{len(frame)} did not. Bundled release overall: "
+ f"{converged}/{len(all_trial)} converged and {nonconverged}/{len(all_trial)} did not. "
+ "A non-converged summary remains visible when selected and is not treated as missing."
+ ),
+ trial_detection_figure(dataset, models),
+ trial_heatmap(models),
+ column_defs(table.columns),
+ records(round_numeric(table)),
+ removal,
+ relation,
+ historical,
+ historical_trajectory_figure(models),
+ column_defs(retrain_table.columns),
+ records(retrain_table),
+ )
@app.callback(
+ Output("coverage-cards", "children"),
Output("methods-table", "columns"),
Output("methods-table", "data"),
Input("dataset-filter", "value"),
+ Input("method-filter", "value"),
)
-def update_methods(_dataset: str):
- table = methods_frame(None)
- return column_defs(table.columns), records(table)
+def update_methods(dataset: str, models: list[str] | None):
+ dataset = dataset or DATASETS[0]
+ table = methods_frame(dataset, models)
+ return coverage_cards(dataset, models), column_defs(table.columns), records(table)
if __name__ == "__main__":