"""Shared leaderboard renderer for the Calibration Benchmark dashboard. Extracts the scored-table pipeline from the home page so both the Optimization leaderboard (home) and the UQ leaderboard (pages/UQLeaderboard.py) can reuse it without duplicating code. Usage:: from common.leaderboard import render_leaderboard render_leaderboard( metric_store, target_col="rmse_target", target_label="RMSE Target Level", title="Optimization Leaderboard", state_prefix="opt", raw_page="pages/RawData.py", ) """ from __future__ import annotations import altair as alt import pandas as pd import streamlit as st # Stable categorical palette (Vega tableau10). Methods are assigned colors by their # sorted position in the full available-method list, so toggling visibility never # reassigns colors to other methods. _METHOD_PALETTE = [ "#4c78a8", "#f58518", "#e45756", "#72b7b2", "#54a24b", "#eeca3b", "#b279a2", "#ff9da6", "#9d755d", "#bab0ac", ] def render_leaderboard( metric_store: pd.DataFrame, *, target_col: str, target_label: str, title: str, state_prefix: str, raw_page: str | None = None, show_failure_panel: bool = False, show_scoring_modes: bool = True, canonical_target_levels: list[float] | None = None, budget_store: pd.DataFrame | None = None, ) -> None: """Render a scored leaderboard backed by *metric_store*. Parameters ---------- metric_store: DataFrame produced by ``load_metric_store()`` or ``load_uq_store()``. Must contain at least the columns ``benchmark``, ``algorithm_type``, ``abbreviation``, ``Method``, ``family``, ``ensemble_size``, ``metric``, ``failure_rate``, and *target_col*. target_col: Name of the target-coordinate column, e.g. ``"rmse_target"`` or ``"uq_target"``. target_label: Human-readable label for the target-level radio control, e.g. ``"RMSE Target Level"`` or ``"UQ Target Level"``. title: Leaderboard section header text. state_prefix: Short string used to namespace ``st.session_state`` keys so multiple leaderboard pages keep independent control state. Use ``"opt"`` for the Optimization leaderboard and ``"uq"`` for the UQ leaderboard. raw_page: Optional Streamlit page path for an "Open Raw Data" link shown at the bottom. Pass ``None`` to suppress the link. show_failure_panel: If ``True``, render a grouped-bar failure-rate chart below the main performance chart. canonical_target_levels: If provided, the target-level selector always offers exactly these values (as strings) regardless of what is present in the data. Use this to pin the UQ leaderboard to its fixed set of target-scaling levels even when some have 100 % failure. budget_store: Optional DataFrame produced by ``load_uq_budget_store()``. When provided, an additional "Budget and Iterations for Coverage" section is rendered below the main performance chart, showing mean budget (N_ensΒ·k_iter, solid lines) and mean iterations (k_iter, dashed lines) vs ensemble size, followed by a coverage-failure-rate bar chart. """ st.header(title) if metric_store.empty: st.warning("No metric data found. Expected NetCDF files in `data/` with a `metric` variable.") return # Derived column name for the string version of the target coordinate target_str_col = f"{target_col}_str" benchmark_values = sorted(metric_store["benchmark"].unique().tolist()) benchmark_options = ["All"] + benchmark_values selected_benchmark = st.selectbox("Benchmark", options=benchmark_options, index=0) filtered = ( metric_store.copy() if selected_benchmark == "All" else metric_store[metric_store["benchmark"] == selected_benchmark].copy() ) filtered[target_str_col] = filtered[target_col].astype(str) if canonical_target_levels is not None: target_options = ["All targets"] + [str(float(t)) for t in canonical_target_levels] else: target_options = ["All targets"] + sorted( metric_store[target_col].astype(str).unique().tolist() ) scoring_options = [ "Mean Forward Model Runs", "Minimum Forward Model Runs", "Smallest Optimal Ensemble Size", "Custom Blend", ] # Session-state keys namespaced by state_prefix so two leaderboard pages # don't share control state. k_target = f"{state_prefix}_selected_target" k_scoring = f"{state_prefix}_scoring_mode" k_weight = f"{state_prefix}_fwdruns_weight_percent" k_methods = f"{state_prefix}_selected_methods" current_target = st.session_state.get(k_target, "All targets") if current_target not in target_options: current_target = target_options[0] current_scoring_mode = st.session_state.get(k_scoring, "Mean Forward Model Runs") if current_scoring_mode not in scoring_options: current_scoring_mode = scoring_options[0] current_fwdruns_weight_percent = int(st.session_state.get(k_weight, 80)) current_fwdruns_weight_percent = max(0, min(100, current_fwdruns_weight_percent)) selected_target = current_target scoring_mode = current_scoring_mode fwdruns_weight = current_fwdruns_weight_percent / 100.0 ensemble_weight = 1.0 - fwdruns_weight # Available methods for the current benchmark selection; used to populate the # multiselect and to prune any stale saved selections when the benchmark changes. available_methods = sorted(filtered["abbreviation"].dropna().unique().tolist()) saved_methods = st.session_state.get(k_methods, available_methods) valid_saved = [m for m in saved_methods if m in available_methods] st.session_state[k_methods] = valid_saved if valid_saved else available_methods # Stable color scale: domain covers ALL methods so colors don't shift when a # subset is displayed. color_domain = available_methods color_range = [_METHOD_PALETTE[i % len(_METHOD_PALETTE)] for i in range(len(available_methods))] method_color = alt.Color( "abbreviation:N", title="Method", scale=alt.Scale(domain=color_domain, range=color_range), ) def build_scored_table(input_df: pd.DataFrame, add_rank: bool = True) -> pd.DataFrame: ranking_source = ( input_df if selected_target == "All targets" else input_df[input_df[target_str_col] == selected_target] ) if ranking_source.empty: return ranking_source # Failure rate from every row (NaN metric rows carry failure_rate=100) failure_agg = ranking_source.groupby( ["algorithm_type", "abbreviation", "Method", "family"], as_index=False ).agg(**{"Mean Failure Rate (%)": ("failure_rate", "mean")}) # Metric stats only from runs that reached the target (non-NaN metric) valid_rows = ranking_source.dropna(subset=["metric"]) if valid_rows.empty: return pd.DataFrame() scored_df = valid_rows.groupby( ["algorithm_type", "abbreviation", "Method", "family"], as_index=False ).agg( **{"Mean Forward Model Runs": ("metric", "mean")}, **{"Minimum Forward Model Runs": ("metric", "min")}, **{"Targets Used": (target_str_col, "nunique")}, **{"Ensemble Sizes Used": ("ensemble_size", "nunique")}, ) best_per_target = ( valid_rows.sort_values(["algorithm_type", target_str_col, "metric", "ensemble_size"]) .groupby(["algorithm_type", "abbreviation", "Method", "family", target_str_col], as_index=False) .first()[ [ "algorithm_type", "abbreviation", "Method", "family", target_str_col, "ensemble_size", ] ] ) optimal_ensemble = best_per_target.groupby( ["algorithm_type", "abbreviation", "Method", "family"], as_index=False ).agg(**{"Optimal Ensemble Size": ("ensemble_size", "mean")}) scored_df = scored_df.merge( optimal_ensemble, on=["algorithm_type", "abbreviation", "Method", "family"], how="left", ) scored_df = scored_df.merge( failure_agg, on=["algorithm_type", "abbreviation", "Method", "family"], how="left", ) scored_df["Optimal Ensemble Size"] = scored_df["Optimal Ensemble Size"].round(2) scored_df["Mean Forward Model Runs"] = scored_df["Mean Forward Model Runs"].round(4) scored_df["Minimum Forward Model Runs"] = scored_df["Minimum Forward Model Runs"].round(4) scored_df["Mean Failure Rate (%)"] = scored_df["Mean Failure Rate (%)"].round(1) mean_runs_min = scored_df["Mean Forward Model Runs"].min() mean_runs_max = scored_df["Mean Forward Model Runs"].max() if mean_runs_max > mean_runs_min: scored_df["mean_runs_score"] = ( 100.0 * (mean_runs_max - scored_df["Mean Forward Model Runs"]) / (mean_runs_max - mean_runs_min) ) else: scored_df["mean_runs_score"] = 100.0 minimum_runs_min = scored_df["Minimum Forward Model Runs"].min() minimum_runs_max = scored_df["Minimum Forward Model Runs"].max() if minimum_runs_max > minimum_runs_min: scored_df["minimum_runs_score"] = ( 100.0 * (minimum_runs_max - scored_df["Minimum Forward Model Runs"]) / (minimum_runs_max - minimum_runs_min) ) else: scored_df["minimum_runs_score"] = 100.0 ens_min = scored_df["Optimal Ensemble Size"].min() ens_max = scored_df["Optimal Ensemble Size"].max() if ens_max > ens_min: scored_df["ensemble_score"] = ( 100.0 * (ens_max - scored_df["Optimal Ensemble Size"]) / (ens_max - ens_min) ) else: scored_df["ensemble_score"] = 100.0 if scoring_mode == "Mean Forward Model Runs": scored_df["Score"] = scored_df["mean_runs_score"] sort_columns = ["Mean Forward Model Runs", "Optimal Ensemble Size", "abbreviation"] ascending = [True, True, True] elif scoring_mode == "Minimum Forward Model Runs": scored_df["Score"] = scored_df["minimum_runs_score"] sort_columns = ["Minimum Forward Model Runs", "Optimal Ensemble Size", "abbreviation"] ascending = [True, True, True] elif scoring_mode == "Smallest Optimal Ensemble Size": scored_df["Score"] = scored_df["ensemble_score"] sort_columns = ["Optimal Ensemble Size", "Mean Forward Model Runs", "abbreviation"] ascending = [True, True, True] else: scored_df["Score"] = ( fwdruns_weight * scored_df["mean_runs_score"] + ensemble_weight * scored_df["ensemble_score"] ) sort_columns = ["Score", "Mean Forward Model Runs", "Optimal Ensemble Size", "abbreviation"] ascending = [False, True, True, True] scored_df = scored_df.sort_values(sort_columns, ascending=ascending).reset_index(drop=True) if add_rank: scored_df["Rank"] = scored_df.index + 1 scored_df["Placement"] = scored_df["Rank"].apply( lambda rank: f"{ {1: 'πŸ₯‡', 2: 'πŸ₯ˆ', 3: 'πŸ₯‰'}.get(rank, '')} #{rank}".strip() ) return scored_df if selected_benchmark == "All": benchmark_scores = [] for benchmark_name in benchmark_values: benchmark_df = metric_store[metric_store["benchmark"] == benchmark_name].copy() benchmark_df[target_str_col] = benchmark_df[target_col].astype(str) scored = build_scored_table(benchmark_df, add_rank=False) if scored.empty: continue scored["benchmark"] = benchmark_name benchmark_scores.append(scored) if benchmark_scores: combined_scores = ( benchmark_scores[0].copy() if len(benchmark_scores) == 1 else pd.concat(benchmark_scores, ignore_index=True) ) leaderboard_df = combined_scores.groupby( ["algorithm_type", "abbreviation", "Method", "family"], as_index=False ).agg( Score=("Score", "mean"), **{"Mean Forward Model Runs": ("Mean Forward Model Runs", "mean")}, **{"Minimum Forward Model Runs": ("Minimum Forward Model Runs", "mean")}, **{"Optimal Ensemble Size": ("Optimal Ensemble Size", "mean")}, **{"Targets Used": ("Targets Used", "mean")}, **{"Ensemble Sizes Used": ("Ensemble Sizes Used", "mean")}, **{"Benchmarks Used": ("benchmark", "nunique")}, **{"Mean Failure Rate (%)": ("Mean Failure Rate (%)", "mean")}, ) leaderboard_df["Mean Forward Model Runs"] = leaderboard_df["Mean Forward Model Runs"].round(4) leaderboard_df["Minimum Forward Model Runs"] = leaderboard_df["Minimum Forward Model Runs"].round(4) leaderboard_df["Optimal Ensemble Size"] = leaderboard_df["Optimal Ensemble Size"].round(2) leaderboard_df["Targets Used"] = leaderboard_df["Targets Used"].round().astype(int) leaderboard_df["Ensemble Sizes Used"] = leaderboard_df["Ensemble Sizes Used"].round().astype(int) leaderboard_df["Mean Failure Rate (%)"] = leaderboard_df["Mean Failure Rate (%)"].round(1) leaderboard_df = leaderboard_df.sort_values( ["Score", "Mean Forward Model Runs", "abbreviation"], ascending=[False, True, True] ).reset_index(drop=True) leaderboard_df["Rank"] = leaderboard_df.index + 1 leaderboard_df["Placement"] = leaderboard_df["Rank"].apply( lambda rank: f"{ {1: 'πŸ₯‡', 2: 'πŸ₯ˆ', 3: 'πŸ₯‰'}.get(rank, '')} #{rank}".strip() ) else: leaderboard_df = pd.DataFrame( columns=[ "Placement", "abbreviation", "Method", "family", "Mean Forward Model Runs", "Minimum Forward Model Runs", "Score", "Optimal Ensemble Size", "Targets Used", "Ensemble Sizes Used", "Benchmarks Used", "Mean Failure Rate (%)", ] ) else: leaderboard_df = build_scored_table(filtered, add_rank=True) leaderboard_df["Benchmarks Used"] = 1 if scoring_mode == "Mean Forward Model Runs": score_basis = "normalized mean of best forward model runs over selected target levels (lower is better)" elif scoring_mode == "Minimum Forward Model Runs": score_basis = "normalized minimum of forward model runs over selected targets and ensemble sizes (lower is better)" elif scoring_mode == "Smallest Optimal Ensemble Size": score_basis = "normalized mean optimal ensemble size over selected target levels (lower is better)" else: score_basis = ( "weighted blend of normalized forward-model-runs score and normalized ensemble-size score " f"(forward-runs weight {fwdruns_weight:.0%}, ensemble-size weight {ensemble_weight:.0%})" ) if selected_benchmark == "All": score_basis = f"{score_basis}; in All mode, each method's final score is the mean of its per-benchmark scores" # Controls expander β€” always shown so users can change target even when the # current selection yields all failures. with st.expander("Scoring & Target Controls", expanded=False): st.radio( target_label, options=target_options, horizontal=True, key=k_target, ) if show_scoring_modes: st.radio( "Scoring Method", options=scoring_options, horizontal=True, key=k_scoring, ) if st.session_state.get(k_scoring, "Mean Forward Model Runs") == "Custom Blend": st.slider( "Blend Weight: Forward Runs vs Ensemble Size", min_value=0, max_value=100, step=5, key=k_weight, help=( "Higher forward-runs weight prioritizes fewer model evaluations; " "higher ensemble-size weight prioritizes smaller ensembles." ), ) st.multiselect( "Methods to display in charts", options=available_methods, key=k_methods, ) selected_methods = st.session_state.get(k_methods, available_methods) if not selected_methods: selected_methods = available_methods if leaderboard_df.empty: st.warning( "All runs failed to reach the target at this selection. " "See the failure rate chart below." if show_failure_panel else "No rows available for the current benchmark/target selection." ) else: if selected_benchmark == "All": table_column_order = [ "Placement", "abbreviation", "Method", "family", "Score", "Mean Failure Rate (%)", "Targets Used", "Ensemble Sizes Used", "Benchmarks Used", ] else: table_column_order = [ "Placement", "abbreviation", "Method", "family", "Score", "Mean Forward Model Runs", "Minimum Forward Model Runs", "Mean Failure Rate (%)", "Optimal Ensemble Size", "Targets Used", "Ensemble Sizes Used", "Benchmarks Used", ] st.subheader(f"Ranked Leaderboard β€” {selected_benchmark}") st.dataframe( leaderboard_df, hide_index=True, use_container_width=True, column_config={ "Placement": st.column_config.TextColumn("Placement"), "family": st.column_config.TextColumn("Family"), "Method": st.column_config.TextColumn("Method"), "abbreviation": st.column_config.TextColumn("Abbrev."), "Mean Forward Model Runs": st.column_config.NumberColumn("Mean Forward Model Runs", format="%.4f"), "Minimum Forward Model Runs": st.column_config.NumberColumn("Minimum Forward Model Runs", format="%.4f"), "Score": st.column_config.ProgressColumn("Score (0-100)", min_value=0.0, max_value=100.0, format="%.1f"), "Optimal Ensemble Size": st.column_config.NumberColumn("Mean Optimal Ensemble Size", format="%.2f"), "Mean Failure Rate (%)": st.column_config.NumberColumn("Mean Failure Rate (%)", format="%.1f"), "Targets Used": st.column_config.NumberColumn("Targets Used", format="%d"), "Ensemble Sizes Used": st.column_config.NumberColumn("Ensemble Sizes Used", format="%d"), "Benchmarks Used": st.column_config.NumberColumn("Benchmarks Used", format="%d"), }, column_order=table_column_order, ) st.info( f"Score is a normalized 0–100 ranking based on **{score_basis}**. " "For Mean/Minimum forward-runs scoring, values are computed from all selected metric target levels " "and all ensemble sizes after averaging over random seeds." ) if selected_benchmark != "All": st.subheader("Mean Forward Model Runs vs Ensemble Size") chart_source = ( filtered if selected_target == "All targets" else filtered[filtered[target_str_col] == selected_target] ) chart_source = chart_source[chart_source["abbreviation"].isin(selected_methods)] chart_df = chart_source.dropna(subset=["metric"]).groupby( ["abbreviation", "ensemble_size"], as_index=False ).agg(mean_forward_runs=("metric", "mean")) all_ens_combos = chart_source[["abbreviation", "ensemble_size"]].drop_duplicates() ens_ticks = sorted(all_ens_combos["ensemble_size"].unique().tolist()) if not all_ens_combos.empty else [] if not chart_df.empty: _ok = chart_df[["abbreviation", "ensemble_size"]].assign(_ok=True) fail_df = all_ens_combos.merge(_ok, on=["abbreviation", "ensemble_size"], how="left") fail_df = fail_df[fail_df["_ok"].isna()].drop(columns="_ok").assign(mean_forward_runs=0.0) else: fail_df = all_ens_combos.assign(mean_forward_runs=0.0) all_failed = chart_df.empty chart_layers = [] if not chart_df.empty: chart_layers.append( alt.Chart(chart_df) .mark_line(point=True) .encode( x=alt.X( "ensemble_size:Q", title="Ensemble Size", axis=alt.Axis(values=ens_ticks, format="d"), ), y=alt.Y("mean_forward_runs:Q", title="Mean Forward Model Runs"), color=method_color, tooltip=["abbreviation", "ensemble_size", alt.Tooltip("mean_forward_runs:Q", format=".4f")], ) ) if not fail_df.empty: y_fwd = ( alt.Y("mean_forward_runs:Q", title="Mean Forward Model Runs", scale=alt.Scale(domain=[0, 1])) if all_failed else alt.Y("mean_forward_runs:Q", title="Mean Forward Model Runs") ) chart_layers.append( alt.Chart(fail_df) .mark_point(shape="cross", angle=45, size=200, filled=True, opacity=1.0) .encode( x=alt.X( "ensemble_size:Q", title="Ensemble Size", axis=alt.Axis(values=ens_ticks, format="d"), ), y=y_fwd, color=method_color, tooltip=[ alt.Tooltip("abbreviation:N", title="Method"), alt.Tooltip("ensemble_size:Q", title="Ensemble Size"), alt.Tooltip("mean_forward_runs:Q", title="Value (all failed)"), ], ) ) if chart_layers: st.altair_chart(alt.layer(*chart_layers), use_container_width=True) # Mean-iterations-for-coverage section (UQ only, when budget_store provided) if budget_store is not None and not budget_store.empty and selected_benchmark != "All": bf = budget_store[budget_store["benchmark"] == selected_benchmark].copy() bf[target_str_col] = bf[target_col].astype(str) if selected_target != "All targets": bf = bf[bf[target_str_col] == selected_target] bf = bf[bf["abbreviation"].isin(selected_methods)] iters_df = ( bf[["abbreviation", "ensemble_size", "mean_iters"]] .dropna(subset=["mean_iters"]) .groupby(["abbreviation", "ensemble_size"], as_index=False) .agg(mean_iters=("mean_iters", "mean")) ) all_ens_combos_iters = bf[["abbreviation", "ensemble_size"]].drop_duplicates() if not all_ens_combos_iters.empty: iters_ticks = sorted(all_ens_combos_iters["ensemble_size"].unique().tolist()) if not iters_df.empty: _ok_iters = iters_df[["abbreviation", "ensemble_size"]].assign(_ok=True) fail_df_iters = all_ens_combos_iters.merge(_ok_iters, on=["abbreviation", "ensemble_size"], how="left") fail_df_iters = fail_df_iters[fail_df_iters["_ok"].isna()].drop(columns="_ok").assign(mean_iters=0.0) else: fail_df_iters = all_ens_combos_iters.assign(mean_iters=0.0) all_failed_iters = iters_df.empty st.subheader("Mean Iterations for Coverage vs Ensemble Size") iters_layers = [] if not iters_df.empty: iters_layers.append( alt.Chart(iters_df) .mark_line(point=True) .encode( x=alt.X( "ensemble_size:Q", title="Ensemble Size", axis=alt.Axis(values=iters_ticks, format="d"), ), y=alt.Y("mean_iters:Q", title="Mean Iterations"), color=method_color, tooltip=[ alt.Tooltip("abbreviation:N", title="Method"), alt.Tooltip("ensemble_size:Q", title="Ensemble Size"), alt.Tooltip("mean_iters:Q", format=".2f", title="Mean Iterations"), ], ) ) if not fail_df_iters.empty: y_iters = ( alt.Y("mean_iters:Q", title="Mean Iterations", scale=alt.Scale(domain=[0, 1])) if all_failed_iters else alt.Y("mean_iters:Q", title="Mean Iterations") ) iters_layers.append( alt.Chart(fail_df_iters) .mark_point(shape="cross", angle=45, size=200, filled=True, opacity=1.0) .encode( x=alt.X( "ensemble_size:Q", title="Ensemble Size", axis=alt.Axis(values=iters_ticks, format="d"), ), y=y_iters, color=method_color, tooltip=[ alt.Tooltip("abbreviation:N", title="Method"), alt.Tooltip("ensemble_size:Q", title="Ensemble Size"), alt.Tooltip("mean_iters:Q", title="Value (all failed)"), ], ) ) if iters_layers: st.altair_chart(alt.layer(*iters_layers), use_container_width=True) # Failure panel β€” rendered regardless of whether the scored table has rows if show_failure_panel and selected_benchmark != "All": failure_source = ( filtered if selected_target == "All targets" else filtered[filtered[target_str_col] == selected_target] ) failure_source = failure_source[failure_source["abbreviation"].isin(selected_methods)] if not failure_source.empty: failure_df = failure_source.groupby( ["abbreviation", "ensemble_size"], as_index=False ).agg(mean_failure_rate=("failure_rate", "mean")) failure_df = failure_df.sort_values("ensemble_size") target_str = ( "All Targets" if selected_target == "All targets" else f"Target {selected_target}" ) st.subheader(f"Failure Rate of Hitting {target_str}") ens_ticks_fail = sorted(failure_df["ensemble_size"].unique().tolist()) failure_chart = ( alt.Chart(failure_df) .mark_bar() .encode( x=alt.X( "ensemble_size:O", title="Ensemble Size", sort=[str(e) for e in ens_ticks_fail], axis=alt.Axis(labelAngle=0), ), xOffset=alt.XOffset("abbreviation:N"), y=alt.Y( "mean_failure_rate:Q", title="Failure Rate (%)", scale=alt.Scale(domain=[0, 100]), ), color=method_color, tooltip=[ alt.Tooltip("abbreviation:N", title="Method"), alt.Tooltip("ensemble_size:O", title="Ensemble Size"), alt.Tooltip("mean_failure_rate:Q", format=".1f", title="Failure Rate (%)"), ], ) ) ceiling_line = ( alt.Chart(alt.Data(values=[{}])) .mark_rule(color="grey", strokeDash=[4, 4]) .encode(y=alt.datum(100)) ) st.altair_chart(failure_chart + ceiling_line, use_container_width=True) st.caption("Top 3 are shown as podium spots; remaining methods are directly comparable via normalized score.") if raw_page is not None: st.page_link(raw_page, label="Open Raw Data & CSV Export", icon="🧾")