File size: 8,114 Bytes
fce6c09
 
 
 
ad36a58
 
fce6c09
 
 
 
 
 
 
 
 
 
 
8e1be25
 
fce6c09
 
ad36a58
 
fce6c09
ad36a58
 
 
 
 
 
fce6c09
 
 
 
 
 
ad36a58
 
 
 
 
 
8e1be25
ad36a58
8e1be25
ad36a58
fce6c09
8e1be25
 
 
fce6c09
 
8e1be25
 
 
fce6c09
ad36a58
8e1be25
 
 
ad36a58
baf1286
 
 
 
 
 
8e1be25
baf1286
 
 
8e1be25
 
 
 
 
ad36a58
 
 
 
 
 
fce6c09
ad36a58
 
 
 
 
 
 
 
 
fce6c09
 
ad36a58
fce6c09
 
 
ad36a58
fce6c09
 
 
 
 
 
 
ad36a58
fce6c09
 
 
 
8e1be25
fce6c09
 
 
 
 
a288069
 
baf1286
8e1be25
baf1286
 
 
a288069
baf1286
a288069
 
 
 
 
 
 
 
baf1286
a288069
 
 
baf1286
fce6c09
 
 
 
 
 
 
 
 
 
 
 
ad36a58
fce6c09
ad36a58
8e1be25
ad36a58
fce6c09
 
ad36a58
 
 
 
fce6c09
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8e1be25
fce6c09
 
 
 
 
8e1be25
fce6c09
 
 
 
 
 
 
 
 
8e1be25
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
from pathlib import Path
import sys

import altair as alt
import streamlit as st

try:
    from data_store import load_metric_store
except ModuleNotFoundError:
    sys.path.append(str(Path(__file__).resolve().parents[1]))
    from data_store import load_metric_store

st.set_page_config(page_title="Method Details", page_icon="πŸ“˜", layout="wide")

# Sidebar navigation
st.sidebar.title("Navigation")
st.sidebar.page_link("streamlit_app.py", label="Home", icon="🏠")
st.sidebar.page_link("pages/OptimizationLeaderboard.py", label="Optimization Leaderboard", icon="πŸ“Š")
st.sidebar.page_link("pages/UQLeaderboard.py", label="UQ Leaderboard", icon="🎯")
st.sidebar.page_link("pages/MethodDetails.py", label="Methods", icon="πŸ“˜")
st.sidebar.page_link("pages/RawData.py", label="Get Data", icon="🧾")

st.title("Method Details")
st.caption("Method-level view from NetCDF forward-model-run metrics averaged over random seeds")

# Read selected method from query params
abbr = st.query_params.get("method")
if not isinstance(abbr, str):
    abbr = None

metric_store = load_metric_store()
if metric_store.empty:
    st.warning("No metric data found. Expected NetCDF files in `data/` with a `metric` variable.")
    st.stop()

methods_df = metric_store[["family", "Method", "abbreviation"]].drop_duplicates().sort_values("abbreviation")

abbrs = methods_df["abbreviation"].tolist()

# Metadata dictionary (can be moved to a JSON/YAML later)
method_meta = {
    "TEKI": {
        "citation": "Chada et al., SIAM/ASA J. UQ, 2020",
        "url": "https://doi.org/10.1137/17M114402X",
        "summary": "EKI variant with Tikhonov regularization that adds a penalty term to prevent ensemble collapse and improve stability on nonlinear problems.",
    },
    "ETKI": {
        "citation": "Schillings & Stuart, Numer. Math., 2017",
        "url": "https://clima.github.io/EnsembleKalmanProcesses.jl/dev/",
        "summary": "Ensemble Transform Kalman Inversion β€” applies an ensemble-space transform update that preserves the ensemble mean while reducing variance inflation.",
    },
    "IEKF": {
        "citation": "Iglesias, Inverse Problems, 2016",
        "url": "https://doi.org/10.1088/0266-5611/32/2/025002",
        "summary": "Regularizing iterative ensemble Kalman method; repeatedly refines the ensemble around a regularized Gauss-Newton step for nonlinear inverse problems.",
    },
    "UKI": {
        "citation": "Huang, Huang & Stuart, Physica D, 2022",
        "url": "https://clima.github.io/EnsembleKalmanProcesses.jl/dev/",
        "summary": "Unscented Kalman Inversion β€” propagates a deterministic set of sigma points through the forward model to estimate mean and covariance without linearization.",
    },
    "ABC": {
        "citation": "Approximate Bayesian Calibration",
        "url": "https://example.com/abc",
        "summary": "Sample without exact likelihoods until error falls below a target convergence.",
    },
    "HM": {
        "citation": "Williamson et al. 2013; King et al. 2025",
        "url": "https://example.com/hm",
        "summary": "Iterative constraint of parameter space using wave reductions.",
    },
    "CES-EKI-DMC": {
        "citation": "Cleary et al., J. Comput. Phys., 2021",
        "url": "https://doi.org/10.1016/j.jcp.2020.109716",
        "summary": "Calibrate-Emulate-Sample: uses EKI with a DataMisfitController to select training points, builds a GP emulator of the forward model, then samples the posterior via MCMC.",
    },
}

# Selection UI (defaults to query param if valid)
default_idx = 0
if isinstance(abbr, str) and abbr in abbrs:
    default_idx = abbrs.index(abbr)
sel = str(st.selectbox("Choose a method", options=abbrs, index=default_idx))

# Persist selection to URL
st.query_params["method"] = sel

# Display details
row = methods_df.loc[methods_df["abbreviation"] == sel].iloc[0]
meta = method_meta.get(sel, {})

st.subheader(f"{sel} β€” {row['Method']}")

col1, col2, col3 = st.columns(3)
with col1:
    with st.container(border=True):
        st.markdown("**Citation**")
        st.write(meta.get("citation", "Citation pending"))
with col2:
    with st.container(border=True):
        st.markdown("**Link**")
        st.link_button("Open reference", meta.get("url", "https://example.com"))
with col3:
    with st.container(border=True):
        st.markdown("**Summary**")
        st.write(meta.get("summary", "Summary pending"))

st.markdown("**Benchmark Slice**")
slice_df = metric_store[metric_store["abbreviation"] == sel].copy()
slice_df = slice_df.sort_values(["benchmark", "rmse_target", "ensemble_size"])

target_options = sorted(slice_df["rmse_target"].astype(str).unique().tolist())
selected_target = st.radio("RMSE Target Level", options=target_options, horizontal=True)

best_table_view = slice_df[slice_df["rmse_target"].astype(str) == selected_target]

best_idx = best_table_view.groupby("benchmark")["metric"].idxmin()
best_ensemble_df = best_table_view.loc[best_idx, ["benchmark", "rmse_target", "ensemble_size", "metric", "failure_rate"]].rename(
    columns={"metric": "Mean Forward Model Runs", "failure_rate": "Failure Rate (%)", "ensemble_size": "Optimal Ensemble Size", "rmse_target": "RMSE Target"}
)
st.dataframe(best_ensemble_df, hide_index=True, use_container_width=True)

if sel == "HM":
    st.markdown("### Failure Analysis")
    st.write(f"Failed calibration rate for History Matching at RMSE Target {selected_target} (as % of random seeds).")
    
    chart = (
        alt.Chart(best_table_view)
        .mark_bar()
        .encode(
            x=alt.X("ensemble_size:O", title="Ensemble Size"),
            y=alt.Y("mean(failure_rate):Q", title="Failure Rate (%)", scale=alt.Scale(domain=[0, 100])),
            color="benchmark:N",
            tooltip=["benchmark", "ensemble_size", alt.Tooltip("mean(failure_rate):Q", format=".1f", title="Failure Rate (%)")]
        )
        .properties(height=350)
        .interactive()
    )
    st.altair_chart(chart, use_container_width=True)
best_by_benchmark = (
    best_table_view.loc[best_idx, ["benchmark", "metric", "ensemble_size"]]
    .rename(
        columns={
            "metric": "Best Mean Forward Model Runs",
            "ensemble_size": "Optimal Ensemble Size",
        }
    )
    .sort_values("benchmark")
)

st.markdown("**Best by Benchmark**")
st.dataframe(
    best_by_benchmark,
    hide_index=True,
    use_container_width=True,
    column_config={
        "benchmark": st.column_config.TextColumn("Benchmark"),
        "Best Mean Forward Model Runs": st.column_config.NumberColumn("Best Mean Forward Model Runs", format="%.4f"),
        "Optimal Ensemble Size": st.column_config.NumberColumn("Optimal Ensemble Size"),
    },
)

st.markdown("**Scaling by Benchmark**")
chart_view = slice_df[slice_df["rmse_target"].astype(str) == selected_target]
chart_df = (
    chart_view.groupby(["benchmark", "ensemble_size"], as_index=False)
    .agg(mean_forward_runs=("metric", "mean"))
)
chart = (
    alt.Chart(chart_df)
    .mark_line(point=True)
    .encode(
        x=alt.X("ensemble_size:Q", title="Ensemble Size"),
        y=alt.Y("mean_forward_runs:Q", title="Mean Forward Model Runs"),
        color=alt.Color("benchmark:N", title="Benchmark"),
        tooltip=["benchmark", "ensemble_size", alt.Tooltip("mean_forward_runs:Q", format=".4f")],
    )
)
st.altair_chart(chart, use_container_width=True)

st.markdown("**All Averaged Configurations for Method**")
st.dataframe(
    slice_df[["benchmark", "algorithm_alias", "rmse_target", "ensemble_size", "metric"]],
    hide_index=True,
    use_container_width=True,
    column_config={
        "benchmark": st.column_config.TextColumn("Benchmark"),
        "algorithm_alias": st.column_config.TextColumn("Source Alias"),
        "rmse_target": st.column_config.TextColumn("RMSE Target"),
        "ensemble_size": st.column_config.NumberColumn("Ensemble Size"),
        "metric": st.column_config.NumberColumn("Mean Forward Model Runs", format="%.4f"),
    },
)

st.page_link("pages/OptimizationLeaderboard.py", label="β¬… Back to Optimization Leaderboard", icon="↩️")