File size: 14,949 Bytes
53f9f3e fce6c09 53f9f3e fce6c09 53f9f3e ad36a58 fce6c09 64f4a12 fce6c09 64f4a12 fce6c09 64f4a12 fce6c09 64f4a12 fce6c09 ad36a58 fce6c09 ad36a58 fce6c09 ad36a58 fce6c09 ad36a58 fce6c09 ad36a58 fce6c09 ad36a58 fce6c09 ad36a58 fce6c09 64f4a12 fce6c09 ad36a58 fce6c09 8d4280b fce6c09 8d4280b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 | import altair as alt
import importlib
import pandas as pd
import streamlit as st
st.set_page_config(layout="wide")
if __package__:
data_store = importlib.import_module(f"{__package__}.data_store")
else:
data_store = importlib.import_module("data_store")
load_metric_store = data_store.load_metric_store
st.title("Calibration Benchmark")
# Sidebar navigation
st.sidebar.title("Navigation")
st.sidebar.page_link("streamlit_app.py", label="Home", icon="🏠")
st.sidebar.page_link("pages/MethodDetails.py", label="Methods", icon="📘")
st.sidebar.page_link("pages/RawData.py", label="Get Data", icon="🧾")
show_home = st.session_state.get("show_home", True)
if show_home:
st.header("Calibration Leaderboard")
metric_store = load_metric_store()
if metric_store.empty:
st.warning("No metric data found. Expected NetCDF files in `data/` with a `metric` variable.")
st.stop()
benchmark_values = sorted(metric_store["benchmark"].unique().tolist())
benchmark_options = ["All"] + benchmark_values
default_benchmark_index = 0
selected_benchmark = st.selectbox("Benchmark", options=benchmark_options, index=default_benchmark_index)
filtered = metric_store.copy() if selected_benchmark == "All" else metric_store[metric_store["benchmark"] == selected_benchmark].copy()
filtered["rmse_target_str"] = filtered["rmse_target"].astype(str)
target_options = ["All targets"] + sorted(metric_store["rmse_target"].astype(str).unique().tolist())
scoring_options = [
"Mean Forward Model Runs",
"Minimum Forward Model Runs",
"Smallest Optimal Ensemble Size",
"Custom Blend",
]
current_target = st.session_state.get("selected_target", "All targets")
if current_target not in target_options:
current_target = target_options[0]
current_scoring_mode = st.session_state.get("scoring_mode", "Mean Forward Model Runs")
if current_scoring_mode not in scoring_options:
current_scoring_mode = scoring_options[0]
current_fwdruns_weight_percent = int(st.session_state.get("fwdruns_weight_percent", 80))
current_fwdruns_weight_percent = max(0, min(100, current_fwdruns_weight_percent))
selected_target = current_target
scoring_mode = current_scoring_mode
fwdruns_weight = current_fwdruns_weight_percent / 100.0
ensemble_weight = 1.0 - fwdruns_weight
def build_scored_table(input_df, add_rank=True):
ranking_source = input_df if selected_target == "All targets" else input_df[input_df["rmse_target_str"] == selected_target]
if ranking_source.empty:
return ranking_source
scored_df = ranking_source.groupby(["algorithm_type", "abbreviation", "Method", "family"], as_index=False).agg(
**{"Mean Forward Model Runs": ("metric", "mean")},
**{"Minimum Forward Model Runs": ("metric", "min")},
**{"Targets Used": ("rmse_target_str", "nunique")},
**{"Ensemble Sizes Used": ("ensemble_size", "nunique")},
)
best_per_target = (
ranking_source.sort_values(["algorithm_type", "rmse_target_str", "metric", "ensemble_size"])
.groupby(["algorithm_type", "abbreviation", "Method", "family", "rmse_target_str"], as_index=False)
.first()[
[
"algorithm_type",
"abbreviation",
"Method",
"family",
"rmse_target_str",
"ensemble_size",
]
]
)
optimal_ensemble = best_per_target.groupby(
["algorithm_type", "abbreviation", "Method", "family"], as_index=False
).agg(**{"Optimal Ensemble Size": ("ensemble_size", "mean")})
scored_df = scored_df.merge(
optimal_ensemble,
on=["algorithm_type", "abbreviation", "Method", "family"],
how="left",
)
scored_df["Optimal Ensemble Size"] = scored_df["Optimal Ensemble Size"].round(2)
scored_df["Mean Forward Model Runs"] = scored_df["Mean Forward Model Runs"].round(4)
scored_df["Minimum Forward Model Runs"] = scored_df["Minimum Forward Model Runs"].round(4)
mean_runs_min = scored_df["Mean Forward Model Runs"].min()
mean_runs_max = scored_df["Mean Forward Model Runs"].max()
if mean_runs_max > mean_runs_min:
scored_df["mean_runs_score"] = 100.0 * (mean_runs_max - scored_df["Mean Forward Model Runs"]) / (mean_runs_max - mean_runs_min)
else:
scored_df["mean_runs_score"] = 100.0
minimum_runs_min = scored_df["Minimum Forward Model Runs"].min()
minimum_runs_max = scored_df["Minimum Forward Model Runs"].max()
if minimum_runs_max > minimum_runs_min:
scored_df["minimum_runs_score"] = 100.0 * (minimum_runs_max - scored_df["Minimum Forward Model Runs"]) / (minimum_runs_max - minimum_runs_min)
else:
scored_df["minimum_runs_score"] = 100.0
ens_min = scored_df["Optimal Ensemble Size"].min()
ens_max = scored_df["Optimal Ensemble Size"].max()
if ens_max > ens_min:
scored_df["ensemble_score"] = 100.0 * (ens_max - scored_df["Optimal Ensemble Size"]) / (ens_max - ens_min)
else:
scored_df["ensemble_score"] = 100.0
if scoring_mode == "Mean Forward Model Runs":
scored_df["Score"] = scored_df["mean_runs_score"]
sort_columns = ["Mean Forward Model Runs", "Optimal Ensemble Size", "abbreviation"]
ascending = [True, True, True]
elif scoring_mode == "Minimum Forward Model Runs":
scored_df["Score"] = scored_df["minimum_runs_score"]
sort_columns = ["Minimum Forward Model Runs", "Optimal Ensemble Size", "abbreviation"]
ascending = [True, True, True]
elif scoring_mode == "Smallest Optimal Ensemble Size":
scored_df["Score"] = scored_df["ensemble_score"]
sort_columns = ["Optimal Ensemble Size", "Mean Forward Model Runs", "abbreviation"]
ascending = [True, True, True]
else:
scored_df["Score"] = (
fwdruns_weight * scored_df["mean_runs_score"]
+ ensemble_weight * scored_df["ensemble_score"]
)
sort_columns = ["Score", "Mean Forward Model Runs", "Optimal Ensemble Size", "abbreviation"]
ascending = [False, True, True, True]
scored_df = scored_df.sort_values(sort_columns, ascending=ascending).reset_index(drop=True)
if add_rank:
scored_df["Rank"] = scored_df.index + 1
scored_df["Placement"] = scored_df["Rank"].apply(
lambda rank: f"{ {1: '🥇', 2: '🥈', 3: '🥉'}.get(rank, '')} #{rank}".strip()
)
return scored_df
if selected_benchmark == "All":
benchmark_scores = []
for benchmark_name in benchmark_values:
benchmark_df = metric_store[metric_store["benchmark"] == benchmark_name].copy()
benchmark_df["rmse_target_str"] = benchmark_df["rmse_target"].astype(str)
scored = build_scored_table(benchmark_df, add_rank=False)
if scored.empty:
continue
scored["benchmark"] = benchmark_name
benchmark_scores.append(scored)
if benchmark_scores:
combined_scores = benchmark_scores[0].copy() if len(benchmark_scores) == 1 else pd.concat(benchmark_scores, ignore_index=True)
leaderboard_df = combined_scores.groupby(["algorithm_type", "abbreviation", "Method", "family"], as_index=False).agg(
Score=("Score", "mean"),
**{"Mean Forward Model Runs": ("Mean Forward Model Runs", "mean")},
**{"Minimum Forward Model Runs": ("Minimum Forward Model Runs", "mean")},
**{"Optimal Ensemble Size": ("Optimal Ensemble Size", "mean")},
**{"Targets Used": ("Targets Used", "mean")},
**{"Ensemble Sizes Used": ("Ensemble Sizes Used", "mean")},
**{"Benchmarks Used": ("benchmark", "nunique")},
)
leaderboard_df["Mean Forward Model Runs"] = leaderboard_df["Mean Forward Model Runs"].round(4)
leaderboard_df["Minimum Forward Model Runs"] = leaderboard_df["Minimum Forward Model Runs"].round(4)
leaderboard_df["Optimal Ensemble Size"] = leaderboard_df["Optimal Ensemble Size"].round(2)
leaderboard_df["Targets Used"] = leaderboard_df["Targets Used"].round().astype(int)
leaderboard_df["Ensemble Sizes Used"] = leaderboard_df["Ensemble Sizes Used"].round().astype(int)
leaderboard_df = leaderboard_df.sort_values(["Score", "Mean Forward Model Runs", "abbreviation"], ascending=[False, True, True]).reset_index(drop=True)
leaderboard_df["Rank"] = leaderboard_df.index + 1
leaderboard_df["Placement"] = leaderboard_df["Rank"].apply(
lambda rank: f"{ {1: '🥇', 2: '🥈', 3: '🥉'}.get(rank, '')} #{rank}".strip()
)
else:
leaderboard_df = pd.DataFrame(
columns=[
"Placement",
"abbreviation",
"Method",
"family",
"Mean Forward Model Runs",
"Minimum Forward Model Runs",
"Score",
"Optimal Ensemble Size",
"Targets Used",
"Ensemble Sizes Used",
"Benchmarks Used",
]
)
else:
leaderboard_df = build_scored_table(filtered, add_rank=True)
leaderboard_df["Benchmarks Used"] = 1
if scoring_mode == "Mean Forward Model Runs":
score_basis = "normalized mean of best forward model runs over selected target levels (lower is better)"
elif scoring_mode == "Minimum Forward Model Runs":
score_basis = "normalized minimum of forward model runs over selected targets and ensemble sizes (lower is better)"
elif scoring_mode == "Smallest Optimal Ensemble Size":
score_basis = "normalized mean optimal ensemble size over selected target levels (lower is better)"
else:
score_basis = (
"weighted blend of normalized forward-model-runs score and normalized ensemble-size score "
f"(forward-runs weight {fwdruns_weight:.0%}, ensemble-size weight {ensemble_weight:.0%})"
)
if selected_benchmark == "All":
score_basis = f"{score_basis}; in All mode, each method's final score is the mean of its per-benchmark scores"
if leaderboard_df.empty:
st.warning("No rows available for the current benchmark/target selection.")
st.stop()
if selected_benchmark == "All":
table_column_order = [
"Placement",
"abbreviation",
"Method",
"family",
"Score",
"Targets Used",
"Ensemble Sizes Used",
"Benchmarks Used",
]
else:
table_column_order = [
"Placement",
"abbreviation",
"Method",
"family",
"Score",
"Mean Forward Model Runs",
"Minimum Forward Model Runs",
"Optimal Ensemble Size",
"Targets Used",
"Ensemble Sizes Used",
"Benchmarks Used",
]
st.subheader(f"Ranked Leaderboard — {selected_benchmark}")
st.dataframe(
leaderboard_df,
hide_index=True,
width="stretch",
column_config={
"Placement": st.column_config.TextColumn("Placement"),
"family": st.column_config.TextColumn("Family"),
"Method": st.column_config.TextColumn("Method"),
"abbreviation": st.column_config.TextColumn("Abbrev."),
"Mean Forward Model Runs": st.column_config.NumberColumn("Mean Forward Model Runs", format="%.4f"),
"Minimum Forward Model Runs": st.column_config.NumberColumn("Minimum Forward Model Runs", format="%.4f"),
"Score": st.column_config.ProgressColumn("Score (0-100)", min_value=0.0, max_value=100.0, format="%.1f"),
"Optimal Ensemble Size": st.column_config.NumberColumn("Mean Optimal Ensemble Size", format="%.2f"),
"Targets Used": st.column_config.NumberColumn("Targets Used", format="%d"),
"Ensemble Sizes Used": st.column_config.NumberColumn("Ensemble Sizes Used", format="%d"),
"Benchmarks Used": st.column_config.NumberColumn("Benchmarks Used", format="%d"),
},
column_order=table_column_order,
)
with st.expander("Scoring & Target Controls", expanded=False):
st.radio(
"RMSE Target Level",
options=target_options,
horizontal=True,
key="selected_target",
)
st.radio(
"Scoring Method",
options=scoring_options,
horizontal=True,
key="scoring_mode",
)
if st.session_state.get("scoring_mode", "Mean Forward Model Runs") == "Custom Blend":
st.slider(
"Blend Weight: Forward Runs vs Ensemble Size",
min_value=0,
max_value=100,
step=5,
key="fwdruns_weight_percent",
help="Higher forward-runs weight prioritizes fewer model evaluations; higher ensemble-size weight prioritizes smaller ensembles.",
)
st.info(
f"Score is a normalized 0–100 ranking based on **{score_basis}**. "
"For Mean/Minimum forward-runs scoring, values are computed from all selected metric target levels and all ensemble sizes "
"after averaging over random seeds."
)
if selected_benchmark != "All":
st.subheader("Mean Forward Model Runs vs Ensemble Size")
chart_source = filtered if selected_target == "All targets" else filtered[filtered["rmse_target_str"] == selected_target]
chart_df = chart_source.groupby(["abbreviation", "ensemble_size"], as_index=False).agg(mean_forward_runs=("metric", "mean"))
chart = (
alt.Chart(chart_df)
.mark_line(point=True)
.encode(
x=alt.X("ensemble_size:Q", title="Ensemble Size"),
y=alt.Y("mean_forward_runs:Q", title="Mean Forward Model Runs"),
color=alt.Color("abbreviation:N", title="Method"),
tooltip=["abbreviation", "ensemble_size", alt.Tooltip("mean_forward_runs:Q", format=".4f")],
)
)
st.altair_chart(chart, width="stretch")
st.caption("Top 3 are shown as podium spots; remaining methods are directly comparable via normalized score.")
st.page_link("pages/RawData.py", label="Open Raw Data & CSV Export", icon="🧾")
|