| """Shared leaderboard renderer for the Calibration Benchmark dashboard. |
| |
| Extracts the scored-table pipeline from the home page so both the Optimization |
| leaderboard (home) and the UQ leaderboard (pages/UQLeaderboard.py) can reuse |
| it without duplicating code. |
| |
| Usage:: |
| |
| from common.leaderboard import render_leaderboard |
| |
| render_leaderboard( |
| metric_store, |
| target_col="rmse_target", |
| target_label="RMSE Target Level", |
| title="Optimization Leaderboard", |
| state_prefix="opt", |
| raw_page="pages/RawData.py", |
| ) |
| """ |
| from __future__ import annotations |
|
|
| import altair as alt |
| import pandas as pd |
| import streamlit as st |
|
|
| |
| |
| |
| _METHOD_PALETTE = [ |
| "#4c78a8", "#f58518", "#e45756", "#72b7b2", "#54a24b", |
| "#eeca3b", "#b279a2", "#ff9da6", "#9d755d", "#bab0ac", |
| ] |
|
|
|
|
| def render_leaderboard( |
| metric_store: pd.DataFrame, |
| *, |
| target_col: str, |
| target_label: str, |
| title: str, |
| state_prefix: str, |
| raw_page: str | None = None, |
| show_failure_panel: bool = False, |
| show_scoring_modes: bool = True, |
| canonical_target_levels: list[float] | None = None, |
| budget_store: pd.DataFrame | None = None, |
| ) -> None: |
| """Render a scored leaderboard backed by *metric_store*. |
| |
| Parameters |
| ---------- |
| metric_store: |
| DataFrame produced by ``load_metric_store()`` or ``load_uq_store()``. |
| Must contain at least the columns ``benchmark``, ``algorithm_type``, |
| ``abbreviation``, ``Method``, ``family``, ``ensemble_size``, ``metric``, |
| ``failure_rate``, and *target_col*. |
| target_col: |
| Name of the target-coordinate column, e.g. ``"rmse_target"`` or |
| ``"uq_target"``. |
| target_label: |
| Human-readable label for the target-level radio control, |
| e.g. ``"RMSE Target Level"`` or ``"UQ Target Level"``. |
| title: |
| Leaderboard section header text. |
| state_prefix: |
| Short string used to namespace ``st.session_state`` keys so multiple |
| leaderboard pages keep independent control state. Use ``"opt"`` for |
| the Optimization leaderboard and ``"uq"`` for the UQ leaderboard. |
| raw_page: |
| Optional Streamlit page path for an "Open Raw Data" link shown at the |
| bottom. Pass ``None`` to suppress the link. |
| show_failure_panel: |
| If ``True``, render a grouped-bar failure-rate chart below the main |
| performance chart. |
| canonical_target_levels: |
| If provided, the target-level selector always offers exactly these |
| values (as strings) regardless of what is present in the data. Use |
| this to pin the UQ leaderboard to its fixed set of target-scaling |
| levels even when some have 100 % failure. |
| budget_store: |
| Optional DataFrame produced by ``load_uq_budget_store()``. When |
| provided, an additional "Budget and Iterations for Coverage" section is |
| rendered below the main performance chart, showing mean budget |
| (N_ens·k_iter, solid lines) and mean iterations (k_iter, dashed lines) |
| vs ensemble size, followed by a coverage-failure-rate bar chart. |
| """ |
| st.header(title) |
|
|
| if metric_store.empty: |
| st.warning("No metric data found. Expected NetCDF files in `data/` with a `metric` variable.") |
| return |
|
|
| |
| target_str_col = f"{target_col}_str" |
|
|
| benchmark_values = sorted(metric_store["benchmark"].unique().tolist()) |
| benchmark_options = ["All"] + benchmark_values |
| selected_benchmark = st.selectbox("Benchmark", options=benchmark_options, index=0) |
|
|
| filtered = ( |
| metric_store.copy() |
| if selected_benchmark == "All" |
| else metric_store[metric_store["benchmark"] == selected_benchmark].copy() |
| ) |
| filtered[target_str_col] = filtered[target_col].astype(str) |
|
|
| if canonical_target_levels is not None: |
| target_options = ["All targets"] + [str(float(t)) for t in canonical_target_levels] |
| else: |
| target_options = ["All targets"] + sorted( |
| metric_store[target_col].astype(str).unique().tolist() |
| ) |
|
|
| scoring_options = [ |
| "Mean Forward Model Runs", |
| "Minimum Forward Model Runs", |
| "Smallest Optimal Ensemble Size", |
| "Custom Blend", |
| ] |
|
|
| |
| |
| k_target = f"{state_prefix}_selected_target" |
| k_scoring = f"{state_prefix}_scoring_mode" |
| k_weight = f"{state_prefix}_fwdruns_weight_percent" |
| k_methods = f"{state_prefix}_selected_methods" |
|
|
| current_target = st.session_state.get(k_target, "All targets") |
| if current_target not in target_options: |
| current_target = target_options[0] |
|
|
| current_scoring_mode = st.session_state.get(k_scoring, "Mean Forward Model Runs") |
| if current_scoring_mode not in scoring_options: |
| current_scoring_mode = scoring_options[0] |
|
|
| current_fwdruns_weight_percent = int(st.session_state.get(k_weight, 80)) |
| current_fwdruns_weight_percent = max(0, min(100, current_fwdruns_weight_percent)) |
|
|
| selected_target = current_target |
| scoring_mode = current_scoring_mode |
| fwdruns_weight = current_fwdruns_weight_percent / 100.0 |
| ensemble_weight = 1.0 - fwdruns_weight |
|
|
| |
| |
| available_methods = sorted(filtered["abbreviation"].dropna().unique().tolist()) |
| saved_methods = st.session_state.get(k_methods, available_methods) |
| valid_saved = [m for m in saved_methods if m in available_methods] |
| st.session_state[k_methods] = valid_saved if valid_saved else available_methods |
|
|
| |
| |
| color_domain = available_methods |
| color_range = [_METHOD_PALETTE[i % len(_METHOD_PALETTE)] for i in range(len(available_methods))] |
| method_color = alt.Color( |
| "abbreviation:N", |
| title="Method", |
| scale=alt.Scale(domain=color_domain, range=color_range), |
| ) |
|
|
| def build_scored_table(input_df: pd.DataFrame, add_rank: bool = True) -> pd.DataFrame: |
| ranking_source = ( |
| input_df |
| if selected_target == "All targets" |
| else input_df[input_df[target_str_col] == selected_target] |
| ) |
| if ranking_source.empty: |
| return ranking_source |
|
|
| |
| failure_agg = ranking_source.groupby( |
| ["algorithm_type", "abbreviation", "Method", "family"], as_index=False |
| ).agg(**{"Mean Failure Rate (%)": ("failure_rate", "mean")}) |
|
|
| |
| valid_rows = ranking_source.dropna(subset=["metric"]) |
| if valid_rows.empty: |
| return pd.DataFrame() |
|
|
| scored_df = valid_rows.groupby( |
| ["algorithm_type", "abbreviation", "Method", "family"], as_index=False |
| ).agg( |
| **{"Mean Forward Model Runs": ("metric", "mean")}, |
| **{"Minimum Forward Model Runs": ("metric", "min")}, |
| **{"Targets Used": (target_str_col, "nunique")}, |
| **{"Ensemble Sizes Used": ("ensemble_size", "nunique")}, |
| ) |
|
|
| best_per_target = ( |
| valid_rows.sort_values(["algorithm_type", target_str_col, "metric", "ensemble_size"]) |
| .groupby(["algorithm_type", "abbreviation", "Method", "family", target_str_col], as_index=False) |
| .first()[ |
| [ |
| "algorithm_type", |
| "abbreviation", |
| "Method", |
| "family", |
| target_str_col, |
| "ensemble_size", |
| ] |
| ] |
| ) |
|
|
| optimal_ensemble = best_per_target.groupby( |
| ["algorithm_type", "abbreviation", "Method", "family"], as_index=False |
| ).agg(**{"Optimal Ensemble Size": ("ensemble_size", "mean")}) |
|
|
| scored_df = scored_df.merge( |
| optimal_ensemble, |
| on=["algorithm_type", "abbreviation", "Method", "family"], |
| how="left", |
| ) |
| scored_df = scored_df.merge( |
| failure_agg, |
| on=["algorithm_type", "abbreviation", "Method", "family"], |
| how="left", |
| ) |
|
|
| scored_df["Optimal Ensemble Size"] = scored_df["Optimal Ensemble Size"].round(2) |
| scored_df["Mean Forward Model Runs"] = scored_df["Mean Forward Model Runs"].round(4) |
| scored_df["Minimum Forward Model Runs"] = scored_df["Minimum Forward Model Runs"].round(4) |
| scored_df["Mean Failure Rate (%)"] = scored_df["Mean Failure Rate (%)"].round(1) |
|
|
| mean_runs_min = scored_df["Mean Forward Model Runs"].min() |
| mean_runs_max = scored_df["Mean Forward Model Runs"].max() |
| if mean_runs_max > mean_runs_min: |
| scored_df["mean_runs_score"] = ( |
| 100.0 * (mean_runs_max - scored_df["Mean Forward Model Runs"]) / (mean_runs_max - mean_runs_min) |
| ) |
| else: |
| scored_df["mean_runs_score"] = 100.0 |
|
|
| minimum_runs_min = scored_df["Minimum Forward Model Runs"].min() |
| minimum_runs_max = scored_df["Minimum Forward Model Runs"].max() |
| if minimum_runs_max > minimum_runs_min: |
| scored_df["minimum_runs_score"] = ( |
| 100.0 * (minimum_runs_max - scored_df["Minimum Forward Model Runs"]) / (minimum_runs_max - minimum_runs_min) |
| ) |
| else: |
| scored_df["minimum_runs_score"] = 100.0 |
|
|
| ens_min = scored_df["Optimal Ensemble Size"].min() |
| ens_max = scored_df["Optimal Ensemble Size"].max() |
| if ens_max > ens_min: |
| scored_df["ensemble_score"] = ( |
| 100.0 * (ens_max - scored_df["Optimal Ensemble Size"]) / (ens_max - ens_min) |
| ) |
| else: |
| scored_df["ensemble_score"] = 100.0 |
|
|
| if scoring_mode == "Mean Forward Model Runs": |
| scored_df["Score"] = scored_df["mean_runs_score"] |
| sort_columns = ["Mean Forward Model Runs", "Optimal Ensemble Size", "abbreviation"] |
| ascending = [True, True, True] |
| elif scoring_mode == "Minimum Forward Model Runs": |
| scored_df["Score"] = scored_df["minimum_runs_score"] |
| sort_columns = ["Minimum Forward Model Runs", "Optimal Ensemble Size", "abbreviation"] |
| ascending = [True, True, True] |
| elif scoring_mode == "Smallest Optimal Ensemble Size": |
| scored_df["Score"] = scored_df["ensemble_score"] |
| sort_columns = ["Optimal Ensemble Size", "Mean Forward Model Runs", "abbreviation"] |
| ascending = [True, True, True] |
| else: |
| scored_df["Score"] = ( |
| fwdruns_weight * scored_df["mean_runs_score"] |
| + ensemble_weight * scored_df["ensemble_score"] |
| ) |
| sort_columns = ["Score", "Mean Forward Model Runs", "Optimal Ensemble Size", "abbreviation"] |
| ascending = [False, True, True, True] |
|
|
| scored_df = scored_df.sort_values(sort_columns, ascending=ascending).reset_index(drop=True) |
|
|
| if add_rank: |
| scored_df["Rank"] = scored_df.index + 1 |
| scored_df["Placement"] = scored_df["Rank"].apply( |
| lambda rank: f"{ {1: '🥇', 2: '🥈', 3: '🥉'}.get(rank, '')} #{rank}".strip() |
| ) |
|
|
| return scored_df |
|
|
| if selected_benchmark == "All": |
| benchmark_scores = [] |
| for benchmark_name in benchmark_values: |
| benchmark_df = metric_store[metric_store["benchmark"] == benchmark_name].copy() |
| benchmark_df[target_str_col] = benchmark_df[target_col].astype(str) |
| scored = build_scored_table(benchmark_df, add_rank=False) |
| if scored.empty: |
| continue |
| scored["benchmark"] = benchmark_name |
| benchmark_scores.append(scored) |
|
|
| if benchmark_scores: |
| combined_scores = ( |
| benchmark_scores[0].copy() |
| if len(benchmark_scores) == 1 |
| else pd.concat(benchmark_scores, ignore_index=True) |
| ) |
| leaderboard_df = combined_scores.groupby( |
| ["algorithm_type", "abbreviation", "Method", "family"], as_index=False |
| ).agg( |
| Score=("Score", "mean"), |
| **{"Mean Forward Model Runs": ("Mean Forward Model Runs", "mean")}, |
| **{"Minimum Forward Model Runs": ("Minimum Forward Model Runs", "mean")}, |
| **{"Optimal Ensemble Size": ("Optimal Ensemble Size", "mean")}, |
| **{"Targets Used": ("Targets Used", "mean")}, |
| **{"Ensemble Sizes Used": ("Ensemble Sizes Used", "mean")}, |
| **{"Benchmarks Used": ("benchmark", "nunique")}, |
| **{"Mean Failure Rate (%)": ("Mean Failure Rate (%)", "mean")}, |
| ) |
|
|
| leaderboard_df["Mean Forward Model Runs"] = leaderboard_df["Mean Forward Model Runs"].round(4) |
| leaderboard_df["Minimum Forward Model Runs"] = leaderboard_df["Minimum Forward Model Runs"].round(4) |
| leaderboard_df["Optimal Ensemble Size"] = leaderboard_df["Optimal Ensemble Size"].round(2) |
| leaderboard_df["Targets Used"] = leaderboard_df["Targets Used"].round().astype(int) |
| leaderboard_df["Ensemble Sizes Used"] = leaderboard_df["Ensemble Sizes Used"].round().astype(int) |
| leaderboard_df["Mean Failure Rate (%)"] = leaderboard_df["Mean Failure Rate (%)"].round(1) |
| leaderboard_df = leaderboard_df.sort_values( |
| ["Score", "Mean Forward Model Runs", "abbreviation"], ascending=[False, True, True] |
| ).reset_index(drop=True) |
| leaderboard_df["Rank"] = leaderboard_df.index + 1 |
| leaderboard_df["Placement"] = leaderboard_df["Rank"].apply( |
| lambda rank: f"{ {1: '🥇', 2: '🥈', 3: '🥉'}.get(rank, '')} #{rank}".strip() |
| ) |
| else: |
| leaderboard_df = pd.DataFrame( |
| columns=[ |
| "Placement", |
| "abbreviation", |
| "Method", |
| "family", |
| "Mean Forward Model Runs", |
| "Minimum Forward Model Runs", |
| "Score", |
| "Optimal Ensemble Size", |
| "Targets Used", |
| "Ensemble Sizes Used", |
| "Benchmarks Used", |
| "Mean Failure Rate (%)", |
| ] |
| ) |
| else: |
| leaderboard_df = build_scored_table(filtered, add_rank=True) |
| leaderboard_df["Benchmarks Used"] = 1 |
|
|
| if scoring_mode == "Mean Forward Model Runs": |
| score_basis = "normalized mean of best forward model runs over selected target levels (lower is better)" |
| elif scoring_mode == "Minimum Forward Model Runs": |
| score_basis = "normalized minimum of forward model runs over selected targets and ensemble sizes (lower is better)" |
| elif scoring_mode == "Smallest Optimal Ensemble Size": |
| score_basis = "normalized mean optimal ensemble size over selected target levels (lower is better)" |
| else: |
| score_basis = ( |
| "weighted blend of normalized forward-model-runs score and normalized ensemble-size score " |
| f"(forward-runs weight {fwdruns_weight:.0%}, ensemble-size weight {ensemble_weight:.0%})" |
| ) |
|
|
| if selected_benchmark == "All": |
| score_basis = f"{score_basis}; in All mode, each method's final score is the mean of its per-benchmark scores" |
|
|
| |
| |
| with st.expander("Scoring & Target Controls", expanded=False): |
| st.radio( |
| target_label, |
| options=target_options, |
| horizontal=True, |
| key=k_target, |
| ) |
|
|
| if show_scoring_modes: |
| st.radio( |
| "Scoring Method", |
| options=scoring_options, |
| horizontal=True, |
| key=k_scoring, |
| ) |
|
|
| if st.session_state.get(k_scoring, "Mean Forward Model Runs") == "Custom Blend": |
| st.slider( |
| "Blend Weight: Forward Runs vs Ensemble Size", |
| min_value=0, |
| max_value=100, |
| step=5, |
| key=k_weight, |
| help=( |
| "Higher forward-runs weight prioritizes fewer model evaluations; " |
| "higher ensemble-size weight prioritizes smaller ensembles." |
| ), |
| ) |
|
|
| st.multiselect( |
| "Methods to display in charts", |
| options=available_methods, |
| key=k_methods, |
| ) |
|
|
| selected_methods = st.session_state.get(k_methods, available_methods) |
| if not selected_methods: |
| selected_methods = available_methods |
|
|
| if leaderboard_df.empty: |
| st.warning( |
| "All runs failed to reach the target at this selection. " |
| "See the failure rate chart below." |
| if show_failure_panel |
| else "No rows available for the current benchmark/target selection." |
| ) |
| else: |
| if selected_benchmark == "All": |
| table_column_order = [ |
| "Placement", |
| "abbreviation", |
| "Method", |
| "family", |
| "Score", |
| "Mean Failure Rate (%)", |
| "Targets Used", |
| "Ensemble Sizes Used", |
| "Benchmarks Used", |
| ] |
| else: |
| table_column_order = [ |
| "Placement", |
| "abbreviation", |
| "Method", |
| "family", |
| "Score", |
| "Mean Forward Model Runs", |
| "Minimum Forward Model Runs", |
| "Mean Failure Rate (%)", |
| "Optimal Ensemble Size", |
| "Targets Used", |
| "Ensemble Sizes Used", |
| "Benchmarks Used", |
| ] |
|
|
| st.subheader(f"Ranked Leaderboard — {selected_benchmark}") |
| st.dataframe( |
| leaderboard_df, |
| hide_index=True, |
| use_container_width=True, |
| column_config={ |
| "Placement": st.column_config.TextColumn("Placement"), |
| "family": st.column_config.TextColumn("Family"), |
| "Method": st.column_config.TextColumn("Method"), |
| "abbreviation": st.column_config.TextColumn("Abbrev."), |
| "Mean Forward Model Runs": st.column_config.NumberColumn("Mean Forward Model Runs", format="%.4f"), |
| "Minimum Forward Model Runs": st.column_config.NumberColumn("Minimum Forward Model Runs", format="%.4f"), |
| "Score": st.column_config.ProgressColumn("Score (0-100)", min_value=0.0, max_value=100.0, format="%.1f"), |
| "Optimal Ensemble Size": st.column_config.NumberColumn("Mean Optimal Ensemble Size", format="%.2f"), |
| "Mean Failure Rate (%)": st.column_config.NumberColumn("Mean Failure Rate (%)", format="%.1f"), |
| "Targets Used": st.column_config.NumberColumn("Targets Used", format="%d"), |
| "Ensemble Sizes Used": st.column_config.NumberColumn("Ensemble Sizes Used", format="%d"), |
| "Benchmarks Used": st.column_config.NumberColumn("Benchmarks Used", format="%d"), |
| }, |
| column_order=table_column_order, |
| ) |
|
|
| st.info( |
| f"Score is a normalized 0–100 ranking based on **{score_basis}**. " |
| "For Mean/Minimum forward-runs scoring, values are computed from all selected metric target levels " |
| "and all ensemble sizes after averaging over random seeds." |
| ) |
|
|
| if selected_benchmark != "All": |
| st.subheader("Mean Forward Model Runs vs Ensemble Size") |
| chart_source = ( |
| filtered |
| if selected_target == "All targets" |
| else filtered[filtered[target_str_col] == selected_target] |
| ) |
| chart_source = chart_source[chart_source["abbreviation"].isin(selected_methods)] |
| chart_df = chart_source.dropna(subset=["metric"]).groupby( |
| ["abbreviation", "ensemble_size"], as_index=False |
| ).agg(mean_forward_runs=("metric", "mean")) |
|
|
| all_ens_combos = chart_source[["abbreviation", "ensemble_size"]].drop_duplicates() |
| ens_ticks = sorted(all_ens_combos["ensemble_size"].unique().tolist()) if not all_ens_combos.empty else [] |
|
|
| if not chart_df.empty: |
| _ok = chart_df[["abbreviation", "ensemble_size"]].assign(_ok=True) |
| fail_df = all_ens_combos.merge(_ok, on=["abbreviation", "ensemble_size"], how="left") |
| fail_df = fail_df[fail_df["_ok"].isna()].drop(columns="_ok").assign(mean_forward_runs=0.0) |
| else: |
| fail_df = all_ens_combos.assign(mean_forward_runs=0.0) |
|
|
| all_failed = chart_df.empty |
| chart_layers = [] |
| if not chart_df.empty: |
| chart_layers.append( |
| alt.Chart(chart_df) |
| .mark_line(point=True) |
| .encode( |
| x=alt.X( |
| "ensemble_size:Q", |
| title="Ensemble Size", |
| axis=alt.Axis(values=ens_ticks, format="d"), |
| ), |
| y=alt.Y("mean_forward_runs:Q", title="Mean Forward Model Runs"), |
| color=method_color, |
| tooltip=["abbreviation", "ensemble_size", alt.Tooltip("mean_forward_runs:Q", format=".4f")], |
| ) |
| ) |
| if not fail_df.empty: |
| y_fwd = ( |
| alt.Y("mean_forward_runs:Q", title="Mean Forward Model Runs", scale=alt.Scale(domain=[0, 1])) |
| if all_failed |
| else alt.Y("mean_forward_runs:Q", title="Mean Forward Model Runs") |
| ) |
| chart_layers.append( |
| alt.Chart(fail_df) |
| .mark_point(shape="cross", angle=45, size=200, filled=True, opacity=1.0) |
| .encode( |
| x=alt.X( |
| "ensemble_size:Q", |
| title="Ensemble Size", |
| axis=alt.Axis(values=ens_ticks, format="d"), |
| ), |
| y=y_fwd, |
| color=method_color, |
| tooltip=[ |
| alt.Tooltip("abbreviation:N", title="Method"), |
| alt.Tooltip("ensemble_size:Q", title="Ensemble Size"), |
| alt.Tooltip("mean_forward_runs:Q", title="Value (all failed)"), |
| ], |
| ) |
| ) |
| if chart_layers: |
| st.altair_chart(alt.layer(*chart_layers), use_container_width=True) |
|
|
| |
| if budget_store is not None and not budget_store.empty and selected_benchmark != "All": |
| bf = budget_store[budget_store["benchmark"] == selected_benchmark].copy() |
| bf[target_str_col] = bf[target_col].astype(str) |
| if selected_target != "All targets": |
| bf = bf[bf[target_str_col] == selected_target] |
| bf = bf[bf["abbreviation"].isin(selected_methods)] |
|
|
| iters_df = ( |
| bf[["abbreviation", "ensemble_size", "mean_iters"]] |
| .dropna(subset=["mean_iters"]) |
| .groupby(["abbreviation", "ensemble_size"], as_index=False) |
| .agg(mean_iters=("mean_iters", "mean")) |
| ) |
| all_ens_combos_iters = bf[["abbreviation", "ensemble_size"]].drop_duplicates() |
| if not all_ens_combos_iters.empty: |
| iters_ticks = sorted(all_ens_combos_iters["ensemble_size"].unique().tolist()) |
|
|
| if not iters_df.empty: |
| _ok_iters = iters_df[["abbreviation", "ensemble_size"]].assign(_ok=True) |
| fail_df_iters = all_ens_combos_iters.merge(_ok_iters, on=["abbreviation", "ensemble_size"], how="left") |
| fail_df_iters = fail_df_iters[fail_df_iters["_ok"].isna()].drop(columns="_ok").assign(mean_iters=0.0) |
| else: |
| fail_df_iters = all_ens_combos_iters.assign(mean_iters=0.0) |
|
|
| all_failed_iters = iters_df.empty |
| st.subheader("Mean Iterations for Coverage vs Ensemble Size") |
| iters_layers = [] |
| if not iters_df.empty: |
| iters_layers.append( |
| alt.Chart(iters_df) |
| .mark_line(point=True) |
| .encode( |
| x=alt.X( |
| "ensemble_size:Q", |
| title="Ensemble Size", |
| axis=alt.Axis(values=iters_ticks, format="d"), |
| ), |
| y=alt.Y("mean_iters:Q", title="Mean Iterations"), |
| color=method_color, |
| tooltip=[ |
| alt.Tooltip("abbreviation:N", title="Method"), |
| alt.Tooltip("ensemble_size:Q", title="Ensemble Size"), |
| alt.Tooltip("mean_iters:Q", format=".2f", title="Mean Iterations"), |
| ], |
| ) |
| ) |
| if not fail_df_iters.empty: |
| y_iters = ( |
| alt.Y("mean_iters:Q", title="Mean Iterations", scale=alt.Scale(domain=[0, 1])) |
| if all_failed_iters |
| else alt.Y("mean_iters:Q", title="Mean Iterations") |
| ) |
| iters_layers.append( |
| alt.Chart(fail_df_iters) |
| .mark_point(shape="cross", angle=45, size=200, filled=True, opacity=1.0) |
| .encode( |
| x=alt.X( |
| "ensemble_size:Q", |
| title="Ensemble Size", |
| axis=alt.Axis(values=iters_ticks, format="d"), |
| ), |
| y=y_iters, |
| color=method_color, |
| tooltip=[ |
| alt.Tooltip("abbreviation:N", title="Method"), |
| alt.Tooltip("ensemble_size:Q", title="Ensemble Size"), |
| alt.Tooltip("mean_iters:Q", title="Value (all failed)"), |
| ], |
| ) |
| ) |
| if iters_layers: |
| st.altair_chart(alt.layer(*iters_layers), use_container_width=True) |
|
|
| |
| if show_failure_panel and selected_benchmark != "All": |
| failure_source = ( |
| filtered |
| if selected_target == "All targets" |
| else filtered[filtered[target_str_col] == selected_target] |
| ) |
| failure_source = failure_source[failure_source["abbreviation"].isin(selected_methods)] |
| if not failure_source.empty: |
| failure_df = failure_source.groupby( |
| ["abbreviation", "ensemble_size"], as_index=False |
| ).agg(mean_failure_rate=("failure_rate", "mean")) |
| failure_df = failure_df.sort_values("ensemble_size") |
|
|
| target_str = ( |
| "All Targets" |
| if selected_target == "All targets" |
| else f"Target {selected_target}" |
| ) |
| st.subheader(f"Failure Rate of Hitting {target_str}") |
| ens_ticks_fail = sorted(failure_df["ensemble_size"].unique().tolist()) |
| failure_chart = ( |
| alt.Chart(failure_df) |
| .mark_bar() |
| .encode( |
| x=alt.X( |
| "ensemble_size:O", |
| title="Ensemble Size", |
| sort=[str(e) for e in ens_ticks_fail], |
| axis=alt.Axis(labelAngle=0), |
| ), |
| xOffset=alt.XOffset("abbreviation:N"), |
| y=alt.Y( |
| "mean_failure_rate:Q", |
| title="Failure Rate (%)", |
| scale=alt.Scale(domain=[0, 100]), |
| ), |
| color=method_color, |
| tooltip=[ |
| alt.Tooltip("abbreviation:N", title="Method"), |
| alt.Tooltip("ensemble_size:O", title="Ensemble Size"), |
| alt.Tooltip("mean_failure_rate:Q", format=".1f", title="Failure Rate (%)"), |
| ], |
| ) |
| ) |
| ceiling_line = ( |
| alt.Chart(alt.Data(values=[{}])) |
| .mark_rule(color="grey", strokeDash=[4, 4]) |
| .encode(y=alt.datum(100)) |
| ) |
| st.altair_chart(failure_chart + ceiling_line, use_container_width=True) |
|
|
| st.caption("Top 3 are shown as podium spots; remaining methods are directly comparable via normalized score.") |
|
|
| if raw_page is not None: |
| st.page_link(raw_page, label="Open Raw Data & CSV Export", icon="🧾") |
|
|