Improve the taxonomy of the methods

#8
.cache/known_methods_snapshot.json CHANGED
@@ -3,7 +3,10 @@
3
  "teki": {
4
  "abbreviation": "TEKI",
5
  "Method": "Tikhonov Regularized Ensemble Kalman Inversion",
6
- "family": "Kalman",
 
 
 
7
  "aliases": [
8
  "teki"
9
  ]
@@ -11,7 +14,10 @@
11
  "etki": {
12
  "abbreviation": "ETKI",
13
  "Method": "Ensemble Transform Kalman Inversion",
14
- "family": "Kalman",
 
 
 
15
  "aliases": [
16
  "etki"
17
  ]
@@ -19,7 +25,10 @@
19
  "iekf": {
20
  "abbreviation": "IEKF",
21
  "Method": "Iterative Ensemble Kalman Filter",
22
- "family": "Kalman",
 
 
 
23
  "aliases": [
24
  "iekf",
25
  "gnsl",
@@ -29,7 +38,10 @@
29
  "uki": {
30
  "abbreviation": "UKI",
31
  "Method": "Unscented Kalman Inversion",
32
- "family": "Kalman",
 
 
 
33
  "aliases": [
34
  "uki"
35
  ]
@@ -37,7 +49,10 @@
37
  "abc": {
38
  "abbreviation": "ABC",
39
  "Method": "Approximate Bayesian Calibration",
40
- "family": "Bayesian",
 
 
 
41
  "aliases": [
42
  "abc"
43
  ]
@@ -45,7 +60,10 @@
45
  "hm": {
46
  "abbreviation": "HM",
47
  "Method": "History Matching",
48
- "family": "Bayesian",
 
 
 
49
  "aliases": [
50
  "hm"
51
  ]
@@ -53,7 +71,10 @@
53
  "ces-eki-dmc": {
54
  "abbreviation": "CES-EKI-DMC",
55
  "Method": "Calibrate Emulate Sample (EKI-DataMisfitController)",
56
- "family": "calibrate_then_emulate",
 
 
 
57
  "aliases": [
58
  "ces-eki-dmc"
59
  ]
@@ -61,7 +82,10 @@
61
  "adam": {
62
  "abbreviation": "ADAM",
63
  "Method": "Adaptive Moment Estimation",
64
- "family": "gradient",
 
 
 
65
  "aliases": [
66
  "adam"
67
  ]
@@ -69,7 +93,10 @@
69
  "lm": {
70
  "abbreviation": "LM",
71
  "Method": "Levenberg-Marquardt",
72
- "family": "gradient",
 
 
 
73
  "aliases": [
74
  "lm",
75
  "levenberg_marquardt",
 
3
  "teki": {
4
  "abbreviation": "TEKI",
5
  "Method": "Tikhonov Regularized Ensemble Kalman Inversion",
6
+ "parallelism": "parallel-interacting",
7
+ "update_type": "kalman",
8
+ "method_goal": "optimization",
9
+ "emulator_use": "none",
10
  "aliases": [
11
  "teki"
12
  ]
 
14
  "etki": {
15
  "abbreviation": "ETKI",
16
  "Method": "Ensemble Transform Kalman Inversion",
17
+ "parallelism": "parallel-interacting",
18
+ "update_type": "kalman",
19
+ "method_goal": "optimization",
20
+ "emulator_use": "none",
21
  "aliases": [
22
  "etki"
23
  ]
 
25
  "iekf": {
26
  "abbreviation": "IEKF",
27
  "Method": "Iterative Ensemble Kalman Filter",
28
+ "parallelism": "parallel-interacting",
29
+ "update_type": "kalman",
30
+ "method_goal": "uq",
31
+ "emulator_use": "none",
32
  "aliases": [
33
  "iekf",
34
  "gnsl",
 
38
  "uki": {
39
  "abbreviation": "UKI",
40
  "Method": "Unscented Kalman Inversion",
41
+ "parallelism": "parallel-interacting",
42
+ "update_type": "kalman",
43
+ "method_goal": "optimization",
44
+ "emulator_use": "none",
45
  "aliases": [
46
  "uki"
47
  ]
 
49
  "abc": {
50
  "abbreviation": "ABC",
51
  "Method": "Approximate Bayesian Calibration",
52
+ "parallelism": "parallel-independent",
53
+ "update_type": "general",
54
+ "method_goal": "uq",
55
+ "emulator_use": "none",
56
  "aliases": [
57
  "abc"
58
  ]
 
60
  "hm": {
61
  "abbreviation": "HM",
62
  "Method": "History Matching",
63
+ "parallelism": "parallel-independent",
64
+ "update_type": "general",
65
+ "method_goal": "uq",
66
+ "emulator_use": "within-optimize",
67
  "aliases": [
68
  "hm"
69
  ]
 
71
  "ces-eki-dmc": {
72
  "abbreviation": "CES-EKI-DMC",
73
  "Method": "Calibrate Emulate Sample (EKI-DataMisfitController)",
74
+ "parallelism": "parallel-interacting",
75
+ "update_type": "kalman",
76
+ "method_goal": "uq",
77
+ "emulator_use": "after-optimize",
78
  "aliases": [
79
  "ces-eki-dmc"
80
  ]
 
82
  "adam": {
83
  "abbreviation": "ADAM",
84
  "Method": "Adaptive Moment Estimation",
85
+ "parallelism": "serial",
86
+ "update_type": "gradient",
87
+ "method_goal": "optimization",
88
+ "emulator_use": "none",
89
  "aliases": [
90
  "adam"
91
  ]
 
93
  "lm": {
94
  "abbreviation": "LM",
95
  "Method": "Levenberg-Marquardt",
96
+ "parallelism": "serial",
97
+ "update_type": "gradient",
98
+ "method_goal": "optimization",
99
+ "emulator_use": "none",
100
  "aliases": [
101
  "lm",
102
  "levenberg_marquardt",
.claude/skills/leaderboard-manager/SKILL.md CHANGED
@@ -75,7 +75,7 @@ NetCDF file
75
  └─ algorithm_type dim (raw string, e.g. "gnki", "teki")
76
  └─ normalize_method_name() β†’ lowercase alias
77
  └─ canonicalize_method_name() β†’ canonical key (e.g. "iekf", "teki")
78
- └─ KNOWN_METHODS[canonical] β†’ { abbreviation, Method, family, aliases }
79
  └─ method_meta[abbreviation] β†’ { citation, url, summary }
80
  ```
81
 
@@ -114,7 +114,7 @@ These traps apply to every task. Read them before touching any file.
114
 
115
  **1. Method metadata is split across three files.**
116
  Adding or editing a method requires changes in all three:
117
- - `src/common/method_registry.py` β€” canonical key, abbreviation, family, aliases
118
  - `src/pages/MethodDetails.py` β€” citation, URL, summary (in `method_meta` dict, keyed by abbreviation)
119
  - `src/data_store.py` β€” `DATASET_FILES` (optimization) or `UQ_BUDGET_FILES` (UQ) data path registration
120
 
 
75
  └─ algorithm_type dim (raw string, e.g. "gnki", "teki")
76
  └─ normalize_method_name() β†’ lowercase alias
77
  └─ canonicalize_method_name() β†’ canonical key (e.g. "iekf", "teki")
78
+ └─ KNOWN_METHODS[canonical] β†’ { abbreviation, Method, parallelism, update_type, method_goal, emulator_use, aliases }
79
  └─ method_meta[abbreviation] β†’ { citation, url, summary }
80
  ```
81
 
 
114
 
115
  **1. Method metadata is split across three files.**
116
  Adding or editing a method requires changes in all three:
117
+ - `src/common/method_registry.py` β€” canonical key, abbreviation, taxonomy tags (parallelism, update_type, method_goal, emulator_use), aliases
118
  - `src/pages/MethodDetails.py` β€” citation, URL, summary (in `method_meta` dict, keyed by abbreviation)
119
  - `src/data_store.py` β€” `DATASET_FILES` (optimization) or `UQ_BUDGET_FILES` (UQ) data path registration
120
 
.claude/skills/leaderboard-manager/references/add-method.md CHANGED
@@ -42,10 +42,13 @@ at line 7). Follow the existing pattern exactly:
42
 
43
  ```python
44
  "new-key": {
45
- "abbreviation": "NEWABBR", # Short uppercase display name
46
- "Method": "Full Method Name", # Human-readable name for tables
47
- "family": "Kalman", # One of: Kalman | Bayesian | calibrate_then_emulate
48
- "aliases": ["new-key", "alias2"], # All raw strings that may appear in NetCDF algorithm_type
 
 
 
49
  },
50
  ```
51
 
 
42
 
43
  ```python
44
  "new-key": {
45
+ "abbreviation": "NEWABBR", # Short uppercase display name
46
+ "Method": "Full Method Name", # Human-readable name for tables
47
+ "parallelism": "parallel-interacting", # serial | parallel-independent | parallel-interacting
48
+ "update_type": "kalman", # gradient | kalman | general
49
+ "method_goal": "optimization", # optimization | uq
50
+ "emulator_use": "none", # none | within-optimize | after-optimize
51
+ "aliases": ["new-key", "alias2"], # All raw strings that may appear in NetCDF algorithm_type
52
  },
53
  ```
54
 
.claude/skills/leaderboard-manager/references/assess.md CHANGED
@@ -37,7 +37,8 @@ Read these in a single parallel batch before starting the checklist:
37
  - [ ] **`.cache/known_methods_snapshot.json` β†’ `unmapped_observed_methods` not empty** β€”
38
  means a NetCDF carries an `algorithm_type` string that has no entry in `KNOWN_METHODS`
39
  (or its aliases list). Those rows silently fall through with a raw uppercase fallback
40
- abbreviation and `family = "Kalman"` default.
 
41
  - [ ] **`DATASET_FILES` references a file path that doesn't exist on disk** β€” `_load_store`
42
  emits a `warnings.warn` and skips it silently; run
43
  `python -c "from src.data_store import load_metric_store; load_metric_store()"` and
 
37
  - [ ] **`.cache/known_methods_snapshot.json` β†’ `unmapped_observed_methods` not empty** β€”
38
  means a NetCDF carries an `algorithm_type` string that has no entry in `KNOWN_METHODS`
39
  (or its aliases list). Those rows silently fall through with a raw uppercase fallback
40
+ abbreviation and `"unmapped"` for all four taxonomy tags (parallelism, update_type,
41
+ method_goal, emulator_use).
42
  - [ ] **`DATASET_FILES` references a file path that doesn't exist on disk** β€” `_load_store`
43
  emits a `warnings.warn` and skips it silently; run
44
  `python -c "from src.data_store import load_metric_store; load_metric_store()"` and
.claude/skills/leaderboard-manager/references/manage-display.md CHANGED
@@ -64,7 +64,8 @@ Search for `table_column_order = [` in `src/common/leaderboard.py` β€” there is
64
  list (not one-per-view):
65
  ```python
66
  table_column_order = [
67
- "Placement", "abbreviation", "Method", "family", "Score",
 
68
  "Mean Forward Model Runs", "Minimum Forward Model Runs",
69
  "Mean Failure Rate (%)", "Optimal Ensemble Size", "Ensemble Sizes Used",
70
  ]
@@ -239,8 +240,8 @@ table:
239
 
240
  To **add or modify a UQ chart**, edit the `if budget_store is not None` block. The
241
  `budget_store` DataFrame has columns: `benchmark`, `algorithm_type`, `abbreviation`,
242
- `family`, `uq_target`, `ensemble_size`, `mean_budget`, `mean_iters`, `failure_count`,
243
- `failure_rate`, `n_seeds`.
244
 
245
  **Bar chart integer x-axis:** Use `ensemble_size:O` (ordinal) with an explicit sort list
246
  and `axis=alt.Axis(labelAngle=0)` to suppress diagonal labels:
 
64
  list (not one-per-view):
65
  ```python
66
  table_column_order = [
67
+ "Placement", "abbreviation", "Method",
68
+ "update_type", "parallelism", "method_goal", "emulator_use", "Score",
69
  "Mean Forward Model Runs", "Minimum Forward Model Runs",
70
  "Mean Failure Rate (%)", "Optimal Ensemble Size", "Ensemble Sizes Used",
71
  ]
 
240
 
241
  To **add or modify a UQ chart**, edit the `if budget_store is not None` block. The
242
  `budget_store` DataFrame has columns: `benchmark`, `algorithm_type`, `abbreviation`,
243
+ `parallelism`, `update_type`, `method_goal`, `emulator_use`, `uq_target`, `ensemble_size`,
244
+ `mean_budget`, `mean_iters`, `failure_count`, `failure_rate`, `n_seeds`.
245
 
246
  **Bar chart integer x-axis:** Use `ensemble_size:O` (ordinal) with an explicit sort list
247
  and `axis=alt.Axis(labelAngle=0)` to suppress diagonal labels:
.claude/skills/leaderboard-manager/references/populate-metadata.md CHANGED
@@ -38,7 +38,7 @@ Method information lives in **two separate places** that serve different purpose
38
 
39
  | File | Key used | What it stores |
40
  |---|---|---|
41
- | `src/common/method_registry.py` | canonical key (e.g. `"uki"`) | Abbreviation, full method name, family, aliases β€” used for **data processing and identity** |
42
  | `src/pages/MethodDetails.py` `method_meta` dict | abbreviation (e.g. `"UKI"`) | Citation, URL, summary β€” used for **display copy only** |
43
 
44
  Both must be kept in sync. When you edit one, check the other. The most common mistake is
@@ -95,7 +95,7 @@ variant is used here).
95
 
96
  ## Editing method identity / naming
97
 
98
- If you need to correct the full method name, family classification, or abbreviation (the
99
  short label that appears in table columns), edit `KNOWN_METHODS` in
100
  `src/common/method_registry.py`:
101
 
@@ -103,7 +103,10 @@ short label that appears in table columns), edit `KNOWN_METHODS` in
103
  "uki": {
104
  "abbreviation": "UKI", # Short label in table columns
105
  "Method": "Unscented Kalman Inversion", # Full name in Method column
106
- "family": "Kalman", # Kalman | Bayesian | calibrate_then_emulate
 
 
 
107
  "aliases": ["uki"], # Raw algorithm_type strings from NetCDF
108
  },
109
  ```
@@ -111,8 +114,9 @@ short label that appears in table columns), edit `KNOWN_METHODS` in
111
  **If you change `abbreviation`**: also update the corresponding key in `method_meta` in
112
  `MethodDetails.py` (the keys must match). The app looks up `method_meta[abbreviation]`.
113
 
114
- **If you change `family`**: the "Family" column in the leaderboard table updates
115
- automatically β€” no other change needed.
 
116
 
117
  **If you add an alias**: new alias strings start matching immediately after reload. Use
118
  this when a collaborator's NetCDF uses a variant spelling not currently in the alias list.
 
38
 
39
  | File | Key used | What it stores |
40
  |---|---|---|
41
+ | `src/common/method_registry.py` | canonical key (e.g. `"uki"`) | Abbreviation, full method name, taxonomy tags (parallelism, update_type, method_goal, emulator_use), aliases β€” used for **data processing and identity** |
42
  | `src/pages/MethodDetails.py` `method_meta` dict | abbreviation (e.g. `"UKI"`) | Citation, URL, summary β€” used for **display copy only** |
43
 
44
  Both must be kept in sync. When you edit one, check the other. The most common mistake is
 
95
 
96
  ## Editing method identity / naming
97
 
98
+ If you need to correct the full method name, taxonomy tags, or abbreviation (the
99
  short label that appears in table columns), edit `KNOWN_METHODS` in
100
  `src/common/method_registry.py`:
101
 
 
103
  "uki": {
104
  "abbreviation": "UKI", # Short label in table columns
105
  "Method": "Unscented Kalman Inversion", # Full name in Method column
106
+ "parallelism": "parallel-interacting", # serial | parallel-independent | parallel-interacting
107
+ "update_type": "kalman", # gradient | kalman | general
108
+ "method_goal": "optimization", # optimization | uq
109
+ "emulator_use": "none", # none | within-optimize | after-optimize
110
  "aliases": ["uki"], # Raw algorithm_type strings from NetCDF
111
  },
112
  ```
 
114
  **If you change `abbreviation`**: also update the corresponding key in `method_meta` in
115
  `MethodDetails.py` (the keys must match). The app looks up `method_meta[abbreviation]`.
116
 
117
+ **If you change any taxonomy tag**: the corresponding column in the leaderboard table
118
+ updates automatically β€” no other change needed. Changing `update_type` also shifts that
119
+ method's group in the sort order (see `_UPDATE_TYPE_ORDER` in `manage-display.md`).
120
 
121
  **If you add an alias**: new alias strings start matching immediately after reload. Use
122
  this when a collaborator's NetCDF uses a variant spelling not currently in the alias list.
CLAUDE.md CHANGED
@@ -96,7 +96,7 @@ calibration_benchmark/
96
  | `algorithm_type` | Canonical method key (e.g. `teki`) |
97
  | `algorithm_alias` | Normalized raw name from NetCDF |
98
  | `abbreviation` | Display name (e.g. `TEKI`) |
99
- | `family` | `Kalman` Β· `Bayesian` Β· `calibrate_then_emulate` Β· `gradient` |
100
  | `rmse_target` | Target RMSE level |
101
  | `ensemble_size` | Ensemble / particle count |
102
  | `metric` | Mean forward-model runs over `random_seed` (lower = better; βˆ’1 = failed) |
@@ -116,7 +116,7 @@ rate = % of seeds that never satisfied the coverage condition (NaN metric, not
116
  |---|---|
117
  | `benchmark` | `L63`, `L96`, `L96_NN_FORCING`, `L96_SPATIAL_FORCING` |
118
  | `algorithm_type` | Canonical method key |
119
- | `abbreviation` / `Method` / `family` | From method registry |
120
  | `uq_target` | Target-scaling level c |
121
  | `ensemble_size` | Ensemble size N |
122
  | `mean_budget` | Mean NΒ·k_iter over converged seeds (NaN if all failed) |
@@ -149,25 +149,55 @@ Physical dimensions of each benchmark exported from `src/data_store.py` as
149
  Passed as `benchmark_dims=BENCHMARK_DIMS` to `render_leaderboard()` for column-header
150
  annotations in the suitability table. Update if a new benchmark is added.
151
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
152
  ## Method registry (`src/common/method_registry.py`)
153
 
154
- | Canonical | Abbreviation | Family | Aliases |
155
- |---|---|---|---|
156
- | `teki` | TEKI | Kalman | β€” |
157
- | `etki` | ETKI | Kalman | β€” |
158
- | `iekf` | IEKF | Kalman | gnsl, gnki |
159
- | `uki` | UKI | Kalman | β€” |
160
- | `abc` | ABC | Bayesian | β€” |
161
- | `hm` | HM | Bayesian | β€” |
162
- | `ces-eki-dmc` | CES-EKI-DMC | calibrate_then_emulate | β€” |
163
- | `adam` | ADAM | gradient | β€” |
164
- | `lm` | LM | gradient | gradient_descent |
165
 
166
  ## How to extend
167
 
168
  ### Add a method
169
  1. Add entry to `KNOWN_METHODS` in `src/common/method_registry.py`
170
- (keys: `abbreviation`, `Method`, `family`, `aliases`).
 
171
  2. Add detail-page metadata to the inline `method_meta` dict in `src/pages/MethodDetails.py`
172
  keyed by **abbreviation** (e.g. `"TEKI"`). ⚠️ Metadata is split across both files.
173
  3. Place result NetCDF(s) in `data/` and add paths to `DATASET_FILES` in `src/data_store.py`.
 
96
  | `algorithm_type` | Canonical method key (e.g. `teki`) |
97
  | `algorithm_alias` | Normalized raw name from NetCDF |
98
  | `abbreviation` | Display name (e.g. `TEKI`) |
99
+ | `parallelism` / `update_type` / `method_goal` / `emulator_use` | Taxonomy tags β€” see **Method taxonomy** below |
100
  | `rmse_target` | Target RMSE level |
101
  | `ensemble_size` | Ensemble / particle count |
102
  | `metric` | Mean forward-model runs over `random_seed` (lower = better; βˆ’1 = failed) |
 
116
  |---|---|
117
  | `benchmark` | `L63`, `L96`, `L96_NN_FORCING`, `L96_SPATIAL_FORCING` |
118
  | `algorithm_type` | Canonical method key |
119
+ | `abbreviation` / `Method` / `parallelism` / `update_type` / `method_goal` / `emulator_use` | From method registry |
120
  | `uq_target` | Target-scaling level c |
121
  | `ensemble_size` | Ensemble size N |
122
  | `mean_budget` | Mean NΒ·k_iter over converged seeds (NaN if all failed) |
 
149
  Passed as `benchmark_dims=BENCHMARK_DIMS` to `render_leaderboard()` for column-header
150
  annotations in the suitability table. Update if a new benchmark is added.
151
 
152
+ ## Method taxonomy
153
+
154
+ Four independent tags on every `KNOWN_METHODS` entry, each scoped to its own axis:
155
+
156
+ - **`parallelism`** β€” how the search explores parameter space:
157
+ `serial` (single point estimate advanced step by step, e.g. `ADAM`/`LM`) Β·
158
+ `parallel-independent` (population updated with no coupling between members,
159
+ e.g. `ABC`'s accepted samples / `HM`'s per-wave resampling) Β·
160
+ `parallel-interacting` (ensemble members coupled through a shared update each
161
+ iteration β€” any Kalman-based method).
162
+ - **`update_type`** β€” the mechanism driving each update step:
163
+ `gradient` (follows the loss gradient or a Gauss-Newton approximation) Β·
164
+ `kalman` (a possibly linearized/unscented Kalman-style ensemble update) Β·
165
+ `general` (anything else, e.g. `ABC`'s rejection sampling, `HM`'s implausibility cuts).
166
+ - **`method_goal`** β€” what the method is built to report:
167
+ `optimization` (single best-fit parameter estimate) or `uq` (full posterior /
168
+ parameter uncertainty). UQ methods *can* still be scored on the Optimization
169
+ leaderboard β€” they're just usually less competitive there.
170
+ - **`emulator_use`** β€” when/whether a surrogate model of the forward model is used:
171
+ `none` (samples/evaluates the true forward model throughout) Β·
172
+ `within-optimize` (refits a surrogate at each search iteration, e.g. `HM` waves) Β·
173
+ `after-optimize` (fits a surrogate once, after calibration finishes, e.g. `CES-EKI-DMC`).
174
+
175
+ `update_type` deliberately avoids a "Bayesian" label β€” Kalman methods are approximate
176
+ Bayesian updates too, so the split is about mechanism (gradient vs. Kalman vs. general),
177
+ not philosophy. `_UPDATE_TYPE_ORDER` in `src/common/leaderboard.py` controls
178
+ table/suitability sort order and must stay in sync with `update_type` values.
179
+ A quick taxonomy tree is rendered on the home page (`src/streamlit_app.py`).
180
+
181
  ## Method registry (`src/common/method_registry.py`)
182
 
183
+ | Canonical | Abbreviation | parallelism | update_type | method_goal | emulator_use | Aliases |
184
+ |---|---|---|---|---|---|---|
185
+ | `teki` | TEKI | parallel-interacting | kalman | optimization | none | β€” |
186
+ | `etki` | ETKI | parallel-interacting | kalman | optimization | none | β€” |
187
+ | `iekf` | IEKF | parallel-interacting | kalman | uq | none | gnsl, gnki |
188
+ | `uki` | UKI | parallel-interacting | kalman | optimization | none | β€” |
189
+ | `abc` | ABC | parallel-independent | general | uq | none | β€” |
190
+ | `hm` | HM | parallel-independent | general | uq | within-optimize | β€” |
191
+ | `ces-eki-dmc` | CES-EKI-DMC | parallel-interacting | kalman | uq | after-optimize | β€” |
192
+ | `adam` | ADAM | serial | gradient | optimization | none | β€” |
193
+ | `lm` | LM | serial | gradient | optimization | none | gradient_descent |
194
 
195
  ## How to extend
196
 
197
  ### Add a method
198
  1. Add entry to `KNOWN_METHODS` in `src/common/method_registry.py`
199
+ (keys: `abbreviation`, `Method`, `parallelism`, `update_type`, `method_goal`,
200
+ `emulator_use`, `aliases` β€” see **Method taxonomy** above for valid tag values).
201
  2. Add detail-page metadata to the inline `method_meta` dict in `src/pages/MethodDetails.py`
202
  keyed by **abbreviation** (e.g. `"TEKI"`). ⚠️ Metadata is split across both files.
203
  3. Place result NetCDF(s) in `data/` and add paths to `DATASET_FILES` in `src/data_store.py`.
src/common/leaderboard.py CHANGED
@@ -49,13 +49,13 @@ def _method_color(present_abbrevs: list[str]) -> alt.Color:
49
  legend=alt.Legend(values=sorted(present_abbrevs)),
50
  )
51
 
52
- # Family display order in the leaderboard table (lower = earlier).
53
- # Unknown families fall back to 99 and appear at the end.
54
- _FAMILY_ORDER: dict[str, int] = {
55
- "Kalman": 0,
56
- "Bayesian": 1,
57
- "gradient": 2,
58
- "calibrate_then_emulate": 3,
59
  }
60
 
61
 
@@ -72,15 +72,15 @@ def _render_suitability_table(
72
  store["benchmark"].unique().tolist(),
73
  key=lambda bm: benchmark_dims[bm][0] if (benchmark_dims and bm in benchmark_dims) else bm,
74
  )
75
- _abbr_family = (
76
- store[["abbreviation", "family"]].dropna()
77
  .drop_duplicates("abbreviation")
78
- .set_index("abbreviation")["family"]
79
  .to_dict()
80
  )
81
  methods = sorted(
82
  store["abbreviation"].dropna().unique().tolist(),
83
- key=lambda a: (_FAMILY_ORDER.get(_abbr_family.get(a, ""), 99), a),
84
  )
85
 
86
  target_str = str(float(suitability_target))
@@ -178,8 +178,9 @@ def render_leaderboard(
178
  metric_store:
179
  DataFrame produced by ``load_metric_store()`` or ``load_uq_store()``.
180
  Must contain at least the columns ``benchmark``, ``algorithm_type``,
181
- ``abbreviation``, ``Method``, ``family``, ``ensemble_size``, ``metric``,
182
- ``failure_rate``, and *target_col*.
 
183
  target_col:
184
  Name of the target-coordinate column, e.g. ``"rmse_target"`` or
185
  ``"uq_target"``.
@@ -292,7 +293,7 @@ def render_leaderboard(
292
 
293
  # Failure rate from every row (NaN metric rows carry failure_rate=100)
294
  failure_agg = ranking_source.groupby(
295
- ["algorithm_type", "abbreviation", "Method", "family"], as_index=False
296
  ).agg(**{"Mean Failure Rate (%)": ("failure_rate", "mean")})
297
 
298
  # Metric stats only from runs that reached the target (non-NaN metric)
@@ -301,7 +302,7 @@ def render_leaderboard(
301
  return pd.DataFrame()
302
 
303
  scored_df = valid_rows.groupby(
304
- ["algorithm_type", "abbreviation", "Method", "family"], as_index=False
305
  ).agg(
306
  **{"Mean Forward Model Runs": ("metric", "mean")},
307
  **{"Minimum Forward Model Runs": ("metric", "min")},
@@ -310,13 +311,12 @@ def render_leaderboard(
310
 
311
  best_per_target = (
312
  valid_rows.sort_values(["algorithm_type", target_str_col, "metric", "ensemble_size"])
313
- .groupby(["algorithm_type", "abbreviation", "Method", "family", target_str_col], as_index=False)
314
  .first()[
315
  [
316
  "algorithm_type",
317
  "abbreviation",
318
  "Method",
319
- "family",
320
  target_str_col,
321
  "ensemble_size",
322
  ]
@@ -324,17 +324,17 @@ def render_leaderboard(
324
  )
325
 
326
  optimal_ensemble = best_per_target.groupby(
327
- ["algorithm_type", "abbreviation", "Method", "family"], as_index=False
328
  ).agg(**{"Optimal Ensemble Size": ("ensemble_size", "mean")})
329
 
330
  scored_df = scored_df.merge(
331
  optimal_ensemble,
332
- on=["algorithm_type", "abbreviation", "Method", "family"],
333
  how="left",
334
  )
335
  scored_df = scored_df.merge(
336
  failure_agg,
337
- on=["algorithm_type", "abbreviation", "Method", "family"],
338
  how="left",
339
  )
340
 
@@ -421,17 +421,25 @@ def render_leaderboard(
421
 
422
  leaderboard_df = build_scored_table(filtered, add_rank=True)
423
 
424
- # Re-sort by family group, then by performance within each group.
 
 
 
 
 
 
 
 
425
  # Placement numbers still reflect overall performance rank.
426
  if not leaderboard_df.empty:
427
  leaderboard_df = leaderboard_df.assign(
428
- _family_sort=leaderboard_df["family"].map(_FAMILY_ORDER).fillna(99),
429
  _is_dnf=leaderboard_df["Score"].isna(),
430
  ).sort_values(
431
- ["_family_sort", "_is_dnf", "Score"],
432
  ascending=[True, True, False],
433
  na_position="last",
434
- ).drop(columns=["_family_sort", "_is_dnf"]).reset_index(drop=True)
435
 
436
  if scoring_mode == "Mean Forward Model Runs":
437
  score_basis = "mean forward-model runs at the selected target level (lower is better)"
@@ -706,7 +714,10 @@ def render_leaderboard(
706
  "Placement",
707
  "abbreviation",
708
  "Method",
709
- "family",
 
 
 
710
  "Score",
711
  "Mean Forward Model Runs",
712
  "Minimum Forward Model Runs",
@@ -730,7 +741,10 @@ def render_leaderboard(
730
  use_container_width=True,
731
  column_config={
732
  "Placement": st.column_config.TextColumn("Placement"),
733
- "family": st.column_config.TextColumn("Family"),
 
 
 
734
  "Method": st.column_config.TextColumn("Method"),
735
  "abbreviation": st.column_config.TextColumn("Abbrev."),
736
  "Mean Forward Model Runs": st.column_config.NumberColumn("Mean Forward Model Runs", format="%.4f"),
 
49
  legend=alt.Legend(values=sorted(present_abbrevs)),
50
  )
51
 
52
+ # update_type display order in the leaderboard table (lower = earlier).
53
+ # Unknown/unmapped update types fall back to 99 and appear at the end.
54
+ # See method_registry.py for the full four-axis taxonomy.
55
+ _UPDATE_TYPE_ORDER: dict[str, int] = {
56
+ "kalman": 0,
57
+ "gradient": 1,
58
+ "general": 2,
59
  }
60
 
61
 
 
72
  store["benchmark"].unique().tolist(),
73
  key=lambda bm: benchmark_dims[bm][0] if (benchmark_dims and bm in benchmark_dims) else bm,
74
  )
75
+ _abbr_update_type = (
76
+ store[["abbreviation", "update_type"]].dropna()
77
  .drop_duplicates("abbreviation")
78
+ .set_index("abbreviation")["update_type"]
79
  .to_dict()
80
  )
81
  methods = sorted(
82
  store["abbreviation"].dropna().unique().tolist(),
83
+ key=lambda a: (_UPDATE_TYPE_ORDER.get(_abbr_update_type.get(a, ""), 99), a),
84
  )
85
 
86
  target_str = str(float(suitability_target))
 
178
  metric_store:
179
  DataFrame produced by ``load_metric_store()`` or ``load_uq_store()``.
180
  Must contain at least the columns ``benchmark``, ``algorithm_type``,
181
+ ``abbreviation``, ``Method``, the four taxonomy tag columns
182
+ (``parallelism``, ``update_type``, ``method_goal``, ``emulator_use``),
183
+ ``ensemble_size``, ``metric``, ``failure_rate``, and *target_col*.
184
  target_col:
185
  Name of the target-coordinate column, e.g. ``"rmse_target"`` or
186
  ``"uq_target"``.
 
293
 
294
  # Failure rate from every row (NaN metric rows carry failure_rate=100)
295
  failure_agg = ranking_source.groupby(
296
+ ["algorithm_type", "abbreviation", "Method"], as_index=False
297
  ).agg(**{"Mean Failure Rate (%)": ("failure_rate", "mean")})
298
 
299
  # Metric stats only from runs that reached the target (non-NaN metric)
 
302
  return pd.DataFrame()
303
 
304
  scored_df = valid_rows.groupby(
305
+ ["algorithm_type", "abbreviation", "Method"], as_index=False
306
  ).agg(
307
  **{"Mean Forward Model Runs": ("metric", "mean")},
308
  **{"Minimum Forward Model Runs": ("metric", "min")},
 
311
 
312
  best_per_target = (
313
  valid_rows.sort_values(["algorithm_type", target_str_col, "metric", "ensemble_size"])
314
+ .groupby(["algorithm_type", "abbreviation", "Method", target_str_col], as_index=False)
315
  .first()[
316
  [
317
  "algorithm_type",
318
  "abbreviation",
319
  "Method",
 
320
  target_str_col,
321
  "ensemble_size",
322
  ]
 
324
  )
325
 
326
  optimal_ensemble = best_per_target.groupby(
327
+ ["algorithm_type", "abbreviation", "Method"], as_index=False
328
  ).agg(**{"Optimal Ensemble Size": ("ensemble_size", "mean")})
329
 
330
  scored_df = scored_df.merge(
331
  optimal_ensemble,
332
+ on=["algorithm_type", "abbreviation", "Method"],
333
  how="left",
334
  )
335
  scored_df = scored_df.merge(
336
  failure_agg,
337
+ on=["algorithm_type", "abbreviation", "Method"],
338
  how="left",
339
  )
340
 
 
421
 
422
  leaderboard_df = build_scored_table(filtered, add_rank=True)
423
 
424
+ # Attach the four taxonomy tags (constant per abbreviation, so this is a
425
+ # plain lookup merge rather than a groupby key threaded through every
426
+ # aggregation above).
427
+ if not leaderboard_df.empty:
428
+ tag_cols = ["abbreviation", "parallelism", "update_type", "method_goal", "emulator_use"]
429
+ tag_lookup = filtered[tag_cols].drop_duplicates("abbreviation")
430
+ leaderboard_df = leaderboard_df.merge(tag_lookup, on="abbreviation", how="left")
431
+
432
+ # Re-sort by update_type group, then by performance within each group.
433
  # Placement numbers still reflect overall performance rank.
434
  if not leaderboard_df.empty:
435
  leaderboard_df = leaderboard_df.assign(
436
+ _update_type_sort=leaderboard_df["update_type"].map(_UPDATE_TYPE_ORDER).fillna(99),
437
  _is_dnf=leaderboard_df["Score"].isna(),
438
  ).sort_values(
439
+ ["_update_type_sort", "_is_dnf", "Score"],
440
  ascending=[True, True, False],
441
  na_position="last",
442
+ ).drop(columns=["_update_type_sort", "_is_dnf"]).reset_index(drop=True)
443
 
444
  if scoring_mode == "Mean Forward Model Runs":
445
  score_basis = "mean forward-model runs at the selected target level (lower is better)"
 
714
  "Placement",
715
  "abbreviation",
716
  "Method",
717
+ "update_type",
718
+ "parallelism",
719
+ "method_goal",
720
+ "emulator_use",
721
  "Score",
722
  "Mean Forward Model Runs",
723
  "Minimum Forward Model Runs",
 
741
  use_container_width=True,
742
  column_config={
743
  "Placement": st.column_config.TextColumn("Placement"),
744
+ "update_type": st.column_config.TextColumn("Update Type"),
745
+ "parallelism": st.column_config.TextColumn("Parallelism"),
746
+ "method_goal": st.column_config.TextColumn("Method Goal"),
747
+ "emulator_use": st.column_config.TextColumn("Emulator Use"),
748
  "Method": st.column_config.TextColumn("Method"),
749
  "abbreviation": st.column_config.TextColumn("Abbrev."),
750
  "Mean Forward Model Runs": st.column_config.NumberColumn("Mean Forward Model Runs", format="%.4f"),
src/common/method_registry.py CHANGED
@@ -4,59 +4,116 @@ import json
4
  from pathlib import Path
5
 
6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7
  KNOWN_METHODS = {
8
  "teki": {
9
  "abbreviation": "TEKI",
10
  "Method": "Tikhonov Regularized Ensemble Kalman Inversion",
11
- "family": "Kalman",
 
 
 
12
  "aliases": ["teki"],
13
  },
14
  "etki": {
15
  "abbreviation": "ETKI",
16
  "Method": "Ensemble Transform Kalman Inversion",
17
- "family": "Kalman",
 
 
 
18
  "aliases": ["etki"],
19
  },
20
  "iekf": {
21
  "abbreviation": "IEKF",
22
  "Method": "Iterative Ensemble Kalman Filter",
23
- "family": "Kalman",
 
 
 
24
  "aliases": ["iekf", "gnsl", "gnki"],
25
  },
26
  "uki": {
27
  "abbreviation": "UKI",
28
  "Method": "Unscented Kalman Inversion",
29
- "family": "Kalman",
 
 
 
30
  "aliases": ["uki"],
31
  },
32
  "abc": {
33
  "abbreviation": "ABC",
34
  "Method": "Approximate Bayesian Calibration",
35
- "family": "Bayesian",
 
 
 
36
  "aliases": ["abc"],
37
  },
38
  "hm": {
39
  "abbreviation": "HM",
40
  "Method": "History Matching",
41
- "family": "Bayesian",
 
 
 
42
  "aliases": ["hm"],
43
  },
44
  "ces-eki-dmc": {
45
  "abbreviation": "CES-EKI-DMC",
46
  "Method": "Calibrate Emulate Sample (EKI-DataMisfitController)",
47
- "family": "calibrate_then_emulate",
 
 
 
48
  "aliases": ["ces-eki-dmc"]
49
  },
50
  "adam": {
51
  "abbreviation": "ADAM",
52
  "Method": "Adaptive Moment Estimation",
53
- "family": "gradient",
 
 
 
54
  "aliases": ["adam"],
55
  },
56
  "lm": {
57
  "abbreviation": "LM",
58
  "Method": "Levenberg-Marquardt",
59
- "family": "gradient",
 
 
 
60
  "aliases": ["lm", "levenberg_marquardt", "levenberg-marquardt", "gradient_descent"],
61
  },
62
  }
 
4
  from pathlib import Path
5
 
6
 
7
+ # Taxonomy
8
+ # --------
9
+ # Every method carries four independent tags, each scoped to its own axis:
10
+ #
11
+ # ``parallelism`` β€” how the search explores parameter space:
12
+ # "serial" β€” a single point estimate advanced step by step (ADAM, LM).
13
+ # "parallel-independent" β€” a population of candidates updated with no coupling
14
+ # between members (ABC's accepted samples, HM's per-wave
15
+ # resampling from the non-implausible region).
16
+ # "parallel-interacting" β€” an ensemble whose members are coupled through a shared
17
+ # update each iteration (any Kalman-based method).
18
+ #
19
+ # ``update_type`` β€” the mechanism driving each update step:
20
+ # "gradient" β€” follows the loss gradient or a Gauss-Newton approximation of it.
21
+ # "kalman" β€” a (possibly linearized/unscented) Kalman-style ensemble update.
22
+ # "general" β€” anything else (e.g. ABC's rejection sampling, HM's implausibility cuts).
23
+ #
24
+ # ``method_goal`` β€” what the method is built to report:
25
+ # "optimization" β€” a single best-fit parameter estimate.
26
+ # "uq" β€” the full posterior / parameter uncertainty. UQ methods can still
27
+ # be scored on the Optimization leaderboard (usually less
28
+ # competitive there, since they're not optimizing for speed-to-target).
29
+ #
30
+ # ``emulator_use`` β€” when/whether a surrogate model of the forward model is used:
31
+ # "none" β€” samples/evaluates the true forward model throughout.
32
+ # "within-optimize" β€” refits a surrogate at each iteration of the search (HM waves).
33
+ # "after-optimize" β€” fits a surrogate once, after calibration finishes (CES).
34
+ #
35
+ # Note: Kalman methods are Bayesian in spirit too (approximate Gaussian posterior
36
+ # updates) β€” ``update_type`` is about mechanism, not a "Bayesian vs not" philosophy.
37
  KNOWN_METHODS = {
38
  "teki": {
39
  "abbreviation": "TEKI",
40
  "Method": "Tikhonov Regularized Ensemble Kalman Inversion",
41
+ "parallelism": "parallel-interacting",
42
+ "update_type": "kalman",
43
+ "method_goal": "optimization",
44
+ "emulator_use": "none",
45
  "aliases": ["teki"],
46
  },
47
  "etki": {
48
  "abbreviation": "ETKI",
49
  "Method": "Ensemble Transform Kalman Inversion",
50
+ "parallelism": "parallel-interacting",
51
+ "update_type": "kalman",
52
+ "method_goal": "optimization",
53
+ "emulator_use": "none",
54
  "aliases": ["etki"],
55
  },
56
  "iekf": {
57
  "abbreviation": "IEKF",
58
  "Method": "Iterative Ensemble Kalman Filter",
59
+ "parallelism": "parallel-interacting",
60
+ "update_type": "kalman",
61
+ "method_goal": "uq",
62
+ "emulator_use": "none",
63
  "aliases": ["iekf", "gnsl", "gnki"],
64
  },
65
  "uki": {
66
  "abbreviation": "UKI",
67
  "Method": "Unscented Kalman Inversion",
68
+ "parallelism": "parallel-interacting",
69
+ "update_type": "kalman",
70
+ "method_goal": "optimization",
71
+ "emulator_use": "none",
72
  "aliases": ["uki"],
73
  },
74
  "abc": {
75
  "abbreviation": "ABC",
76
  "Method": "Approximate Bayesian Calibration",
77
+ "parallelism": "parallel-independent",
78
+ "update_type": "general",
79
+ "method_goal": "uq",
80
+ "emulator_use": "none",
81
  "aliases": ["abc"],
82
  },
83
  "hm": {
84
  "abbreviation": "HM",
85
  "Method": "History Matching",
86
+ "parallelism": "parallel-independent",
87
+ "update_type": "general",
88
+ "method_goal": "uq",
89
+ "emulator_use": "within-optimize",
90
  "aliases": ["hm"],
91
  },
92
  "ces-eki-dmc": {
93
  "abbreviation": "CES-EKI-DMC",
94
  "Method": "Calibrate Emulate Sample (EKI-DataMisfitController)",
95
+ "parallelism": "parallel-interacting",
96
+ "update_type": "kalman",
97
+ "method_goal": "uq",
98
+ "emulator_use": "after-optimize",
99
  "aliases": ["ces-eki-dmc"]
100
  },
101
  "adam": {
102
  "abbreviation": "ADAM",
103
  "Method": "Adaptive Moment Estimation",
104
+ "parallelism": "serial",
105
+ "update_type": "gradient",
106
+ "method_goal": "optimization",
107
+ "emulator_use": "none",
108
  "aliases": ["adam"],
109
  },
110
  "lm": {
111
  "abbreviation": "LM",
112
  "Method": "Levenberg-Marquardt",
113
+ "parallelism": "serial",
114
+ "update_type": "gradient",
115
+ "method_goal": "optimization",
116
+ "emulator_use": "none",
117
  "aliases": ["lm", "levenberg_marquardt", "levenberg-marquardt", "gradient_descent"],
118
  },
119
  }
src/data_store.py CHANGED
@@ -59,6 +59,10 @@ DATASET_FILES = {
59
  EXPECTED_DIMS_OPT = ("algorithm_type", "rmse_target", "ensemble_size", "random_seed")
60
  EXPECTED_DIMS_UQ = ("random_seed", "ensemble_size", "k_iter", "coverage_quantile")
61
 
 
 
 
 
62
 
63
  # ---------------------------------------------------------------------------
64
  # Uncertainty Quantification datasets
@@ -229,10 +233,11 @@ def _load_store(dataset_files: dict[str, list[str]], target_col: str, *, drop_na
229
  merged["ensemble_size"] = merged["ensemble_size"].astype(int)
230
  merged["abbreviation"] = merged["algorithm_type"].map(lambda method: get_method_meta(method).get("abbreviation"))
231
  merged["Method"] = merged["algorithm_type"].map(lambda method: get_method_meta(method).get("Method"))
232
- merged["family"] = merged["algorithm_type"].map(lambda method: get_method_meta(method).get("family"))
 
 
233
  merged["abbreviation"] = merged["abbreviation"].fillna(merged["algorithm_type"].str.upper())
234
  merged["Method"] = merged["Method"].fillna(merged["abbreviation"])
235
- merged["family"] = merged["family"].fillna("Kalman")
236
 
237
  try:
238
  dump_method_registry_snapshot(project_root=project_root, observed_methods=set(merged["algorithm_type"].unique()))
@@ -267,7 +272,7 @@ def load_uq_store() -> pd.DataFrame:
267
  return pd.DataFrame(
268
  columns=[
269
  "benchmark", "algorithm_type", "algorithm_alias", "abbreviation",
270
- "Method", "family", "uq_target", "ensemble_size", "metric", "failure_rate",
271
  ]
272
  )
273
  df = budget_df.copy()
@@ -281,7 +286,7 @@ def load_uq_store() -> pd.DataFrame:
281
  return df[
282
  [
283
  "benchmark", "algorithm_type", "algorithm_alias", "abbreviation",
284
- "Method", "family", "uq_target", "ensemble_size", "metric", "failure_rate",
285
  ]
286
  ]
287
 
@@ -295,8 +300,9 @@ def load_uq_budget_store() -> pd.DataFrame:
295
  UQ_COVERAGE_QUANTILES. Budget = N Β· k_iter; NaN when target never reached.
296
 
297
  Returns a DataFrame with columns:
298
- benchmark, algorithm_type, abbreviation, family, uq_target,
299
- ensemble_size, mean_budget, mean_iters, failure_count, failure_rate, n_seeds
 
300
  """
301
  xr = importlib.import_module("xarray")
302
 
@@ -383,7 +389,7 @@ def load_uq_budget_store() -> pd.DataFrame:
383
  if not records:
384
  return pd.DataFrame(
385
  columns=[
386
- "benchmark", "algorithm_type", "abbreviation", "family",
387
  "uq_target", "ensemble_size", "mean_budget", "mean_iters",
388
  "failure_count", "failure_rate", "n_seeds",
389
  ]
@@ -395,6 +401,7 @@ def load_uq_budget_store() -> pd.DataFrame:
395
  return get_method_meta(canonicalize_method_name(normalize_method_name(m))).get(key, fallback)
396
 
397
  df["abbreviation"] = df["algorithm_type"].map(lambda m: _meta(m, "abbreviation", m.upper()))
398
- df["family"] = df["algorithm_type"].map(lambda m: _meta(m, "family", "Kalman"))
 
399
  df["Method"] = df["algorithm_type"].map(lambda m: _meta(m, "Method", m.upper()))
400
  return df
 
59
  EXPECTED_DIMS_OPT = ("algorithm_type", "rmse_target", "ensemble_size", "random_seed")
60
  EXPECTED_DIMS_UQ = ("random_seed", "ensemble_size", "k_iter", "coverage_quantile")
61
 
62
+ # Method-taxonomy tag columns β€” one value per method, sourced from KNOWN_METHODS.
63
+ # See src/common/method_registry.py for the axis definitions.
64
+ TAXONOMY_TAG_COLUMNS = ["parallelism", "update_type", "method_goal", "emulator_use"]
65
+
66
 
67
  # ---------------------------------------------------------------------------
68
  # Uncertainty Quantification datasets
 
233
  merged["ensemble_size"] = merged["ensemble_size"].astype(int)
234
  merged["abbreviation"] = merged["algorithm_type"].map(lambda method: get_method_meta(method).get("abbreviation"))
235
  merged["Method"] = merged["algorithm_type"].map(lambda method: get_method_meta(method).get("Method"))
236
+ for tag_col in TAXONOMY_TAG_COLUMNS:
237
+ merged[tag_col] = merged["algorithm_type"].map(lambda method: get_method_meta(method).get(tag_col))
238
+ merged[tag_col] = merged[tag_col].fillna("unmapped")
239
  merged["abbreviation"] = merged["abbreviation"].fillna(merged["algorithm_type"].str.upper())
240
  merged["Method"] = merged["Method"].fillna(merged["abbreviation"])
 
241
 
242
  try:
243
  dump_method_registry_snapshot(project_root=project_root, observed_methods=set(merged["algorithm_type"].unique()))
 
272
  return pd.DataFrame(
273
  columns=[
274
  "benchmark", "algorithm_type", "algorithm_alias", "abbreviation",
275
+ "Method", *TAXONOMY_TAG_COLUMNS, "uq_target", "ensemble_size", "metric", "failure_rate",
276
  ]
277
  )
278
  df = budget_df.copy()
 
286
  return df[
287
  [
288
  "benchmark", "algorithm_type", "algorithm_alias", "abbreviation",
289
+ "Method", *TAXONOMY_TAG_COLUMNS, "uq_target", "ensemble_size", "metric", "failure_rate",
290
  ]
291
  ]
292
 
 
300
  UQ_COVERAGE_QUANTILES. Budget = N Β· k_iter; NaN when target never reached.
301
 
302
  Returns a DataFrame with columns:
303
+ benchmark, algorithm_type, abbreviation, parallelism, update_type,
304
+ method_goal, emulator_use, uq_target, ensemble_size, mean_budget,
305
+ mean_iters, failure_count, failure_rate, n_seeds
306
  """
307
  xr = importlib.import_module("xarray")
308
 
 
389
  if not records:
390
  return pd.DataFrame(
391
  columns=[
392
+ "benchmark", "algorithm_type", "abbreviation", *TAXONOMY_TAG_COLUMNS,
393
  "uq_target", "ensemble_size", "mean_budget", "mean_iters",
394
  "failure_count", "failure_rate", "n_seeds",
395
  ]
 
401
  return get_method_meta(canonicalize_method_name(normalize_method_name(m))).get(key, fallback)
402
 
403
  df["abbreviation"] = df["algorithm_type"].map(lambda m: _meta(m, "abbreviation", m.upper()))
404
+ for tag_col in TAXONOMY_TAG_COLUMNS:
405
+ df[tag_col] = df["algorithm_type"].map(lambda m, col=tag_col: _meta(m, col, "unmapped"))
406
  df["Method"] = df["algorithm_type"].map(lambda m: _meta(m, "Method", m.upper()))
407
  return df
src/pages/MethodDetails.py CHANGED
@@ -33,7 +33,7 @@ if metric_store.empty:
33
  st.warning("No metric data found. Expected NetCDF files in `data/` with a `metric` variable.")
34
  st.stop()
35
 
36
- methods_df = metric_store[["family", "Method", "abbreviation"]].drop_duplicates().sort_values("abbreviation")
37
 
38
  abbrs = methods_df["abbreviation"].tolist()
39
 
 
33
  st.warning("No metric data found. Expected NetCDF files in `data/` with a `metric` variable.")
34
  st.stop()
35
 
36
+ methods_df = metric_store[["Method", "abbreviation"]].drop_duplicates().sort_values("abbreviation")
37
 
38
  abbrs = methods_df["abbreviation"].tolist()
39
 
src/pages/RawData.py CHANGED
@@ -40,7 +40,10 @@ st.dataframe(
40
  "benchmark",
41
  "abbreviation",
42
  "Method",
43
- "family",
 
 
 
44
  "algorithm_type",
45
  "algorithm_alias",
46
  "rmse_target",
@@ -54,7 +57,10 @@ st.dataframe(
54
  "benchmark": st.column_config.TextColumn("Benchmark"),
55
  "abbreviation": st.column_config.TextColumn("Abbrev."),
56
  "Method": st.column_config.TextColumn("Method"),
57
- "family": st.column_config.TextColumn("Family"),
 
 
 
58
  "algorithm_type": st.column_config.TextColumn("Canonical Type"),
59
  "algorithm_alias": st.column_config.TextColumn("Source Alias"),
60
  "rmse_target": st.column_config.TextColumn("RMSE Target"),
 
40
  "benchmark",
41
  "abbreviation",
42
  "Method",
43
+ "parallelism",
44
+ "update_type",
45
+ "method_goal",
46
+ "emulator_use",
47
  "algorithm_type",
48
  "algorithm_alias",
49
  "rmse_target",
 
57
  "benchmark": st.column_config.TextColumn("Benchmark"),
58
  "abbreviation": st.column_config.TextColumn("Abbrev."),
59
  "Method": st.column_config.TextColumn("Method"),
60
+ "parallelism": st.column_config.TextColumn("Parallelism"),
61
+ "update_type": st.column_config.TextColumn("Update Type"),
62
+ "method_goal": st.column_config.TextColumn("Method Goal"),
63
+ "emulator_use": st.column_config.TextColumn("Emulator Use"),
64
  "algorithm_type": st.column_config.TextColumn("Canonical Type"),
65
  "algorithm_alias": st.column_config.TextColumn("Source Alias"),
66
  "rmse_target": st.column_config.TextColumn("RMSE Target"),
src/streamlit_app.py CHANGED
@@ -81,18 +81,48 @@ st.image(
81
  width=1200,
82
  )
83
 
84
- st.subheader("Method families")
85
  st.markdown(
86
- "Methods are grouped into three families. See the **πŸ“˜ Methods** page for "
87
- "citations and per-method performance charts."
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
88
  )
89
- families = {
90
- "Kalman": "Ensemble Kalman variants (TEKI, ETKI, IEKF, UKI) β€” update an ensemble of parameter guesses via a linearized observation operator.",
91
- "Bayesian": "Sampling-based approaches (ABC, HM) β€” explore parameter space without requiring gradient information.",
92
- "Calibrate-then-emulate": "Two-stage pipelines (CES-EKI-DMC) β€” use an initial calibration phase to build a cheap emulator, then sample the posterior via MCMC.",
93
- }
94
- for family, desc in families.items():
95
- st.markdown(f"- **{family}** β€” {desc}")
96
 
97
  st.subheader("Key metric")
98
  st.markdown(
 
81
  width=1200,
82
  )
83
 
84
+ st.subheader("Method taxonomy")
85
  st.markdown(
86
+ "Every method carries four independent tags β€” how it searches, what update "
87
+ "mechanism drives each step, what it's built to report, and whether/when it uses "
88
+ "a surrogate model. See the **πŸ“˜ Methods** page for citations and per-method "
89
+ "performance charts."
90
+ )
91
+ st.markdown(
92
+ """
93
+ - **Parallelism** β€” how the search explores parameter space
94
+ - **Serial** β€” `ADAM`, `LM` β€” a single point estimate advanced step by step.
95
+ - **Parallel-independent** β€” `ABC`, `HM` β€” a population of candidates updated
96
+ with no coupling between members (accepted samples / per-wave resampling).
97
+ - **Parallel-interacting** β€” `TEKI`, `ETKI`, `IEKF`, `UKI`, `CES-EKI-DMC` β€” an
98
+ ensemble whose members are coupled through a shared update each iteration.
99
+ - **Update type** β€” the mechanism driving each update step
100
+ - **Gradient** β€” `ADAM`, `LM` β€” follow the loss gradient (or a Gauss-Newton
101
+ approximation of it) directly.
102
+ - **Kalman** β€” `TEKI`, `ETKI`, `IEKF`, `UKI`, `CES-EKI-DMC` β€” a (possibly
103
+ linearized or unscented) Kalman-style ensemble update.
104
+ - **General** β€” `ABC`, `HM` β€” neither gradient- nor Kalman-based (rejection
105
+ sampling, implausibility cuts).
106
+ - **Method goal** β€” what the method is built to report
107
+ - **Optimization** β€” `TEKI`, `ETKI`, `UKI`, `ADAM`, `LM` β€” a single best-fit
108
+ parameter estimate.
109
+ - **UQ** β€” `IEKF`, `ABC`, `HM`, `CES-EKI-DMC` β€” the full posterior / parameter
110
+ uncertainty. Can still be scored on the Optimization leaderboard, but tends to
111
+ be less competitive there since it's not optimizing for speed-to-target.
112
+ - **Emulator use** β€” when/whether a surrogate model of the forward model is used
113
+ - **None** β€” `TEKI`, `ETKI`, `IEKF`, `UKI`, `ADAM`, `LM`, `ABC` β€” samples/evaluates
114
+ the true forward model throughout.
115
+ - **Within-optimize** β€” `HM` β€” refits a surrogate at each iteration (wave) of the
116
+ search itself.
117
+ - **After-optimize** β€” `CES-EKI-DMC` β€” fits a surrogate (e.g. a GP) once, after
118
+ calibration finishes, and samples the posterior through it.
119
+ """
120
+ )
121
+ st.caption(
122
+ "Note: Kalman methods are Bayesian in spirit too (they're approximate Gaussian "
123
+ "posterior updates) β€” update type is about mechanism (gradient vs. Kalman vs. "
124
+ "general), not whether a method is 'Bayesian'."
125
  )
 
 
 
 
 
 
 
126
 
127
  st.subheader("Key metric")
128
  st.markdown(