Improve the taxonomy of the methods
#8
by odunbar - opened
- .cache/known_methods_snapshot.json +36 -9
- .claude/skills/leaderboard-manager/SKILL.md +2 -2
- .claude/skills/leaderboard-manager/references/add-method.md +7 -4
- .claude/skills/leaderboard-manager/references/assess.md +2 -1
- .claude/skills/leaderboard-manager/references/manage-display.md +4 -3
- .claude/skills/leaderboard-manager/references/populate-metadata.md +9 -5
- CLAUDE.md +44 -14
- src/common/leaderboard.py +40 -26
- src/common/method_registry.py +66 -9
- src/data_store.py +15 -8
- src/pages/MethodDetails.py +1 -1
- src/pages/RawData.py +8 -2
- src/streamlit_app.py +40 -10
.cache/known_methods_snapshot.json
CHANGED
|
@@ -3,7 +3,10 @@
|
|
| 3 |
"teki": {
|
| 4 |
"abbreviation": "TEKI",
|
| 5 |
"Method": "Tikhonov Regularized Ensemble Kalman Inversion",
|
| 6 |
-
"
|
|
|
|
|
|
|
|
|
|
| 7 |
"aliases": [
|
| 8 |
"teki"
|
| 9 |
]
|
|
@@ -11,7 +14,10 @@
|
|
| 11 |
"etki": {
|
| 12 |
"abbreviation": "ETKI",
|
| 13 |
"Method": "Ensemble Transform Kalman Inversion",
|
| 14 |
-
"
|
|
|
|
|
|
|
|
|
|
| 15 |
"aliases": [
|
| 16 |
"etki"
|
| 17 |
]
|
|
@@ -19,7 +25,10 @@
|
|
| 19 |
"iekf": {
|
| 20 |
"abbreviation": "IEKF",
|
| 21 |
"Method": "Iterative Ensemble Kalman Filter",
|
| 22 |
-
"
|
|
|
|
|
|
|
|
|
|
| 23 |
"aliases": [
|
| 24 |
"iekf",
|
| 25 |
"gnsl",
|
|
@@ -29,7 +38,10 @@
|
|
| 29 |
"uki": {
|
| 30 |
"abbreviation": "UKI",
|
| 31 |
"Method": "Unscented Kalman Inversion",
|
| 32 |
-
"
|
|
|
|
|
|
|
|
|
|
| 33 |
"aliases": [
|
| 34 |
"uki"
|
| 35 |
]
|
|
@@ -37,7 +49,10 @@
|
|
| 37 |
"abc": {
|
| 38 |
"abbreviation": "ABC",
|
| 39 |
"Method": "Approximate Bayesian Calibration",
|
| 40 |
-
"
|
|
|
|
|
|
|
|
|
|
| 41 |
"aliases": [
|
| 42 |
"abc"
|
| 43 |
]
|
|
@@ -45,7 +60,10 @@
|
|
| 45 |
"hm": {
|
| 46 |
"abbreviation": "HM",
|
| 47 |
"Method": "History Matching",
|
| 48 |
-
"
|
|
|
|
|
|
|
|
|
|
| 49 |
"aliases": [
|
| 50 |
"hm"
|
| 51 |
]
|
|
@@ -53,7 +71,10 @@
|
|
| 53 |
"ces-eki-dmc": {
|
| 54 |
"abbreviation": "CES-EKI-DMC",
|
| 55 |
"Method": "Calibrate Emulate Sample (EKI-DataMisfitController)",
|
| 56 |
-
"
|
|
|
|
|
|
|
|
|
|
| 57 |
"aliases": [
|
| 58 |
"ces-eki-dmc"
|
| 59 |
]
|
|
@@ -61,7 +82,10 @@
|
|
| 61 |
"adam": {
|
| 62 |
"abbreviation": "ADAM",
|
| 63 |
"Method": "Adaptive Moment Estimation",
|
| 64 |
-
"
|
|
|
|
|
|
|
|
|
|
| 65 |
"aliases": [
|
| 66 |
"adam"
|
| 67 |
]
|
|
@@ -69,7 +93,10 @@
|
|
| 69 |
"lm": {
|
| 70 |
"abbreviation": "LM",
|
| 71 |
"Method": "Levenberg-Marquardt",
|
| 72 |
-
"
|
|
|
|
|
|
|
|
|
|
| 73 |
"aliases": [
|
| 74 |
"lm",
|
| 75 |
"levenberg_marquardt",
|
|
|
|
| 3 |
"teki": {
|
| 4 |
"abbreviation": "TEKI",
|
| 5 |
"Method": "Tikhonov Regularized Ensemble Kalman Inversion",
|
| 6 |
+
"parallelism": "parallel-interacting",
|
| 7 |
+
"update_type": "kalman",
|
| 8 |
+
"method_goal": "optimization",
|
| 9 |
+
"emulator_use": "none",
|
| 10 |
"aliases": [
|
| 11 |
"teki"
|
| 12 |
]
|
|
|
|
| 14 |
"etki": {
|
| 15 |
"abbreviation": "ETKI",
|
| 16 |
"Method": "Ensemble Transform Kalman Inversion",
|
| 17 |
+
"parallelism": "parallel-interacting",
|
| 18 |
+
"update_type": "kalman",
|
| 19 |
+
"method_goal": "optimization",
|
| 20 |
+
"emulator_use": "none",
|
| 21 |
"aliases": [
|
| 22 |
"etki"
|
| 23 |
]
|
|
|
|
| 25 |
"iekf": {
|
| 26 |
"abbreviation": "IEKF",
|
| 27 |
"Method": "Iterative Ensemble Kalman Filter",
|
| 28 |
+
"parallelism": "parallel-interacting",
|
| 29 |
+
"update_type": "kalman",
|
| 30 |
+
"method_goal": "uq",
|
| 31 |
+
"emulator_use": "none",
|
| 32 |
"aliases": [
|
| 33 |
"iekf",
|
| 34 |
"gnsl",
|
|
|
|
| 38 |
"uki": {
|
| 39 |
"abbreviation": "UKI",
|
| 40 |
"Method": "Unscented Kalman Inversion",
|
| 41 |
+
"parallelism": "parallel-interacting",
|
| 42 |
+
"update_type": "kalman",
|
| 43 |
+
"method_goal": "optimization",
|
| 44 |
+
"emulator_use": "none",
|
| 45 |
"aliases": [
|
| 46 |
"uki"
|
| 47 |
]
|
|
|
|
| 49 |
"abc": {
|
| 50 |
"abbreviation": "ABC",
|
| 51 |
"Method": "Approximate Bayesian Calibration",
|
| 52 |
+
"parallelism": "parallel-independent",
|
| 53 |
+
"update_type": "general",
|
| 54 |
+
"method_goal": "uq",
|
| 55 |
+
"emulator_use": "none",
|
| 56 |
"aliases": [
|
| 57 |
"abc"
|
| 58 |
]
|
|
|
|
| 60 |
"hm": {
|
| 61 |
"abbreviation": "HM",
|
| 62 |
"Method": "History Matching",
|
| 63 |
+
"parallelism": "parallel-independent",
|
| 64 |
+
"update_type": "general",
|
| 65 |
+
"method_goal": "uq",
|
| 66 |
+
"emulator_use": "within-optimize",
|
| 67 |
"aliases": [
|
| 68 |
"hm"
|
| 69 |
]
|
|
|
|
| 71 |
"ces-eki-dmc": {
|
| 72 |
"abbreviation": "CES-EKI-DMC",
|
| 73 |
"Method": "Calibrate Emulate Sample (EKI-DataMisfitController)",
|
| 74 |
+
"parallelism": "parallel-interacting",
|
| 75 |
+
"update_type": "kalman",
|
| 76 |
+
"method_goal": "uq",
|
| 77 |
+
"emulator_use": "after-optimize",
|
| 78 |
"aliases": [
|
| 79 |
"ces-eki-dmc"
|
| 80 |
]
|
|
|
|
| 82 |
"adam": {
|
| 83 |
"abbreviation": "ADAM",
|
| 84 |
"Method": "Adaptive Moment Estimation",
|
| 85 |
+
"parallelism": "serial",
|
| 86 |
+
"update_type": "gradient",
|
| 87 |
+
"method_goal": "optimization",
|
| 88 |
+
"emulator_use": "none",
|
| 89 |
"aliases": [
|
| 90 |
"adam"
|
| 91 |
]
|
|
|
|
| 93 |
"lm": {
|
| 94 |
"abbreviation": "LM",
|
| 95 |
"Method": "Levenberg-Marquardt",
|
| 96 |
+
"parallelism": "serial",
|
| 97 |
+
"update_type": "gradient",
|
| 98 |
+
"method_goal": "optimization",
|
| 99 |
+
"emulator_use": "none",
|
| 100 |
"aliases": [
|
| 101 |
"lm",
|
| 102 |
"levenberg_marquardt",
|
.claude/skills/leaderboard-manager/SKILL.md
CHANGED
|
@@ -75,7 +75,7 @@ NetCDF file
|
|
| 75 |
ββ algorithm_type dim (raw string, e.g. "gnki", "teki")
|
| 76 |
ββ normalize_method_name() β lowercase alias
|
| 77 |
ββ canonicalize_method_name() β canonical key (e.g. "iekf", "teki")
|
| 78 |
-
ββ KNOWN_METHODS[canonical] β { abbreviation, Method,
|
| 79 |
ββ method_meta[abbreviation] β { citation, url, summary }
|
| 80 |
```
|
| 81 |
|
|
@@ -114,7 +114,7 @@ These traps apply to every task. Read them before touching any file.
|
|
| 114 |
|
| 115 |
**1. Method metadata is split across three files.**
|
| 116 |
Adding or editing a method requires changes in all three:
|
| 117 |
-
- `src/common/method_registry.py` β canonical key, abbreviation,
|
| 118 |
- `src/pages/MethodDetails.py` β citation, URL, summary (in `method_meta` dict, keyed by abbreviation)
|
| 119 |
- `src/data_store.py` β `DATASET_FILES` (optimization) or `UQ_BUDGET_FILES` (UQ) data path registration
|
| 120 |
|
|
|
|
| 75 |
ββ algorithm_type dim (raw string, e.g. "gnki", "teki")
|
| 76 |
ββ normalize_method_name() β lowercase alias
|
| 77 |
ββ canonicalize_method_name() β canonical key (e.g. "iekf", "teki")
|
| 78 |
+
ββ KNOWN_METHODS[canonical] β { abbreviation, Method, parallelism, update_type, method_goal, emulator_use, aliases }
|
| 79 |
ββ method_meta[abbreviation] β { citation, url, summary }
|
| 80 |
```
|
| 81 |
|
|
|
|
| 114 |
|
| 115 |
**1. Method metadata is split across three files.**
|
| 116 |
Adding or editing a method requires changes in all three:
|
| 117 |
+
- `src/common/method_registry.py` β canonical key, abbreviation, taxonomy tags (parallelism, update_type, method_goal, emulator_use), aliases
|
| 118 |
- `src/pages/MethodDetails.py` β citation, URL, summary (in `method_meta` dict, keyed by abbreviation)
|
| 119 |
- `src/data_store.py` β `DATASET_FILES` (optimization) or `UQ_BUDGET_FILES` (UQ) data path registration
|
| 120 |
|
.claude/skills/leaderboard-manager/references/add-method.md
CHANGED
|
@@ -42,10 +42,13 @@ at line 7). Follow the existing pattern exactly:
|
|
| 42 |
|
| 43 |
```python
|
| 44 |
"new-key": {
|
| 45 |
-
"abbreviation": "NEWABBR",
|
| 46 |
-
"Method": "Full Method Name",
|
| 47 |
-
"
|
| 48 |
-
"
|
|
|
|
|
|
|
|
|
|
| 49 |
},
|
| 50 |
```
|
| 51 |
|
|
|
|
| 42 |
|
| 43 |
```python
|
| 44 |
"new-key": {
|
| 45 |
+
"abbreviation": "NEWABBR", # Short uppercase display name
|
| 46 |
+
"Method": "Full Method Name", # Human-readable name for tables
|
| 47 |
+
"parallelism": "parallel-interacting", # serial | parallel-independent | parallel-interacting
|
| 48 |
+
"update_type": "kalman", # gradient | kalman | general
|
| 49 |
+
"method_goal": "optimization", # optimization | uq
|
| 50 |
+
"emulator_use": "none", # none | within-optimize | after-optimize
|
| 51 |
+
"aliases": ["new-key", "alias2"], # All raw strings that may appear in NetCDF algorithm_type
|
| 52 |
},
|
| 53 |
```
|
| 54 |
|
.claude/skills/leaderboard-manager/references/assess.md
CHANGED
|
@@ -37,7 +37,8 @@ Read these in a single parallel batch before starting the checklist:
|
|
| 37 |
- [ ] **`.cache/known_methods_snapshot.json` β `unmapped_observed_methods` not empty** β
|
| 38 |
means a NetCDF carries an `algorithm_type` string that has no entry in `KNOWN_METHODS`
|
| 39 |
(or its aliases list). Those rows silently fall through with a raw uppercase fallback
|
| 40 |
-
abbreviation and `
|
|
|
|
| 41 |
- [ ] **`DATASET_FILES` references a file path that doesn't exist on disk** β `_load_store`
|
| 42 |
emits a `warnings.warn` and skips it silently; run
|
| 43 |
`python -c "from src.data_store import load_metric_store; load_metric_store()"` and
|
|
|
|
| 37 |
- [ ] **`.cache/known_methods_snapshot.json` β `unmapped_observed_methods` not empty** β
|
| 38 |
means a NetCDF carries an `algorithm_type` string that has no entry in `KNOWN_METHODS`
|
| 39 |
(or its aliases list). Those rows silently fall through with a raw uppercase fallback
|
| 40 |
+
abbreviation and `"unmapped"` for all four taxonomy tags (parallelism, update_type,
|
| 41 |
+
method_goal, emulator_use).
|
| 42 |
- [ ] **`DATASET_FILES` references a file path that doesn't exist on disk** β `_load_store`
|
| 43 |
emits a `warnings.warn` and skips it silently; run
|
| 44 |
`python -c "from src.data_store import load_metric_store; load_metric_store()"` and
|
.claude/skills/leaderboard-manager/references/manage-display.md
CHANGED
|
@@ -64,7 +64,8 @@ Search for `table_column_order = [` in `src/common/leaderboard.py` β there is
|
|
| 64 |
list (not one-per-view):
|
| 65 |
```python
|
| 66 |
table_column_order = [
|
| 67 |
-
"Placement", "abbreviation", "Method",
|
|
|
|
| 68 |
"Mean Forward Model Runs", "Minimum Forward Model Runs",
|
| 69 |
"Mean Failure Rate (%)", "Optimal Ensemble Size", "Ensemble Sizes Used",
|
| 70 |
]
|
|
@@ -239,8 +240,8 @@ table:
|
|
| 239 |
|
| 240 |
To **add or modify a UQ chart**, edit the `if budget_store is not None` block. The
|
| 241 |
`budget_store` DataFrame has columns: `benchmark`, `algorithm_type`, `abbreviation`,
|
| 242 |
-
`
|
| 243 |
-
`failure_rate`, `n_seeds`.
|
| 244 |
|
| 245 |
**Bar chart integer x-axis:** Use `ensemble_size:O` (ordinal) with an explicit sort list
|
| 246 |
and `axis=alt.Axis(labelAngle=0)` to suppress diagonal labels:
|
|
|
|
| 64 |
list (not one-per-view):
|
| 65 |
```python
|
| 66 |
table_column_order = [
|
| 67 |
+
"Placement", "abbreviation", "Method",
|
| 68 |
+
"update_type", "parallelism", "method_goal", "emulator_use", "Score",
|
| 69 |
"Mean Forward Model Runs", "Minimum Forward Model Runs",
|
| 70 |
"Mean Failure Rate (%)", "Optimal Ensemble Size", "Ensemble Sizes Used",
|
| 71 |
]
|
|
|
|
| 240 |
|
| 241 |
To **add or modify a UQ chart**, edit the `if budget_store is not None` block. The
|
| 242 |
`budget_store` DataFrame has columns: `benchmark`, `algorithm_type`, `abbreviation`,
|
| 243 |
+
`parallelism`, `update_type`, `method_goal`, `emulator_use`, `uq_target`, `ensemble_size`,
|
| 244 |
+
`mean_budget`, `mean_iters`, `failure_count`, `failure_rate`, `n_seeds`.
|
| 245 |
|
| 246 |
**Bar chart integer x-axis:** Use `ensemble_size:O` (ordinal) with an explicit sort list
|
| 247 |
and `axis=alt.Axis(labelAngle=0)` to suppress diagonal labels:
|
.claude/skills/leaderboard-manager/references/populate-metadata.md
CHANGED
|
@@ -38,7 +38,7 @@ Method information lives in **two separate places** that serve different purpose
|
|
| 38 |
|
| 39 |
| File | Key used | What it stores |
|
| 40 |
|---|---|---|
|
| 41 |
-
| `src/common/method_registry.py` | canonical key (e.g. `"uki"`) | Abbreviation, full method name,
|
| 42 |
| `src/pages/MethodDetails.py` `method_meta` dict | abbreviation (e.g. `"UKI"`) | Citation, URL, summary β used for **display copy only** |
|
| 43 |
|
| 44 |
Both must be kept in sync. When you edit one, check the other. The most common mistake is
|
|
@@ -95,7 +95,7 @@ variant is used here).
|
|
| 95 |
|
| 96 |
## Editing method identity / naming
|
| 97 |
|
| 98 |
-
If you need to correct the full method name,
|
| 99 |
short label that appears in table columns), edit `KNOWN_METHODS` in
|
| 100 |
`src/common/method_registry.py`:
|
| 101 |
|
|
@@ -103,7 +103,10 @@ short label that appears in table columns), edit `KNOWN_METHODS` in
|
|
| 103 |
"uki": {
|
| 104 |
"abbreviation": "UKI", # Short label in table columns
|
| 105 |
"Method": "Unscented Kalman Inversion", # Full name in Method column
|
| 106 |
-
"
|
|
|
|
|
|
|
|
|
|
| 107 |
"aliases": ["uki"], # Raw algorithm_type strings from NetCDF
|
| 108 |
},
|
| 109 |
```
|
|
@@ -111,8 +114,9 @@ short label that appears in table columns), edit `KNOWN_METHODS` in
|
|
| 111 |
**If you change `abbreviation`**: also update the corresponding key in `method_meta` in
|
| 112 |
`MethodDetails.py` (the keys must match). The app looks up `method_meta[abbreviation]`.
|
| 113 |
|
| 114 |
-
**If you change
|
| 115 |
-
automatically β no other change needed.
|
|
|
|
| 116 |
|
| 117 |
**If you add an alias**: new alias strings start matching immediately after reload. Use
|
| 118 |
this when a collaborator's NetCDF uses a variant spelling not currently in the alias list.
|
|
|
|
| 38 |
|
| 39 |
| File | Key used | What it stores |
|
| 40 |
|---|---|---|
|
| 41 |
+
| `src/common/method_registry.py` | canonical key (e.g. `"uki"`) | Abbreviation, full method name, taxonomy tags (parallelism, update_type, method_goal, emulator_use), aliases β used for **data processing and identity** |
|
| 42 |
| `src/pages/MethodDetails.py` `method_meta` dict | abbreviation (e.g. `"UKI"`) | Citation, URL, summary β used for **display copy only** |
|
| 43 |
|
| 44 |
Both must be kept in sync. When you edit one, check the other. The most common mistake is
|
|
|
|
| 95 |
|
| 96 |
## Editing method identity / naming
|
| 97 |
|
| 98 |
+
If you need to correct the full method name, taxonomy tags, or abbreviation (the
|
| 99 |
short label that appears in table columns), edit `KNOWN_METHODS` in
|
| 100 |
`src/common/method_registry.py`:
|
| 101 |
|
|
|
|
| 103 |
"uki": {
|
| 104 |
"abbreviation": "UKI", # Short label in table columns
|
| 105 |
"Method": "Unscented Kalman Inversion", # Full name in Method column
|
| 106 |
+
"parallelism": "parallel-interacting", # serial | parallel-independent | parallel-interacting
|
| 107 |
+
"update_type": "kalman", # gradient | kalman | general
|
| 108 |
+
"method_goal": "optimization", # optimization | uq
|
| 109 |
+
"emulator_use": "none", # none | within-optimize | after-optimize
|
| 110 |
"aliases": ["uki"], # Raw algorithm_type strings from NetCDF
|
| 111 |
},
|
| 112 |
```
|
|
|
|
| 114 |
**If you change `abbreviation`**: also update the corresponding key in `method_meta` in
|
| 115 |
`MethodDetails.py` (the keys must match). The app looks up `method_meta[abbreviation]`.
|
| 116 |
|
| 117 |
+
**If you change any taxonomy tag**: the corresponding column in the leaderboard table
|
| 118 |
+
updates automatically β no other change needed. Changing `update_type` also shifts that
|
| 119 |
+
method's group in the sort order (see `_UPDATE_TYPE_ORDER` in `manage-display.md`).
|
| 120 |
|
| 121 |
**If you add an alias**: new alias strings start matching immediately after reload. Use
|
| 122 |
this when a collaborator's NetCDF uses a variant spelling not currently in the alias list.
|
CLAUDE.md
CHANGED
|
@@ -96,7 +96,7 @@ calibration_benchmark/
|
|
| 96 |
| `algorithm_type` | Canonical method key (e.g. `teki`) |
|
| 97 |
| `algorithm_alias` | Normalized raw name from NetCDF |
|
| 98 |
| `abbreviation` | Display name (e.g. `TEKI`) |
|
| 99 |
-
| `
|
| 100 |
| `rmse_target` | Target RMSE level |
|
| 101 |
| `ensemble_size` | Ensemble / particle count |
|
| 102 |
| `metric` | Mean forward-model runs over `random_seed` (lower = better; β1 = failed) |
|
|
@@ -116,7 +116,7 @@ rate = % of seeds that never satisfied the coverage condition (NaN metric, not
|
|
| 116 |
|---|---|
|
| 117 |
| `benchmark` | `L63`, `L96`, `L96_NN_FORCING`, `L96_SPATIAL_FORCING` |
|
| 118 |
| `algorithm_type` | Canonical method key |
|
| 119 |
-
| `abbreviation` / `Method` / `
|
| 120 |
| `uq_target` | Target-scaling level c |
|
| 121 |
| `ensemble_size` | Ensemble size N |
|
| 122 |
| `mean_budget` | Mean NΒ·k_iter over converged seeds (NaN if all failed) |
|
|
@@ -149,25 +149,55 @@ Physical dimensions of each benchmark exported from `src/data_store.py` as
|
|
| 149 |
Passed as `benchmark_dims=BENCHMARK_DIMS` to `render_leaderboard()` for column-header
|
| 150 |
annotations in the suitability table. Update if a new benchmark is added.
|
| 151 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 152 |
## Method registry (`src/common/method_registry.py`)
|
| 153 |
|
| 154 |
-
| Canonical | Abbreviation |
|
| 155 |
-
|---|---|---|---|
|
| 156 |
-
| `teki` | TEKI |
|
| 157 |
-
| `etki` | ETKI |
|
| 158 |
-
| `iekf` | IEKF |
|
| 159 |
-
| `uki` | UKI |
|
| 160 |
-
| `abc` | ABC |
|
| 161 |
-
| `hm` | HM |
|
| 162 |
-
| `ces-eki-dmc` | CES-EKI-DMC |
|
| 163 |
-
| `adam` | ADAM | gradient | β |
|
| 164 |
-
| `lm` | LM | gradient | gradient_descent |
|
| 165 |
|
| 166 |
## How to extend
|
| 167 |
|
| 168 |
### Add a method
|
| 169 |
1. Add entry to `KNOWN_METHODS` in `src/common/method_registry.py`
|
| 170 |
-
(keys: `abbreviation`, `Method`, `
|
|
|
|
| 171 |
2. Add detail-page metadata to the inline `method_meta` dict in `src/pages/MethodDetails.py`
|
| 172 |
keyed by **abbreviation** (e.g. `"TEKI"`). β οΈ Metadata is split across both files.
|
| 173 |
3. Place result NetCDF(s) in `data/` and add paths to `DATASET_FILES` in `src/data_store.py`.
|
|
|
|
| 96 |
| `algorithm_type` | Canonical method key (e.g. `teki`) |
|
| 97 |
| `algorithm_alias` | Normalized raw name from NetCDF |
|
| 98 |
| `abbreviation` | Display name (e.g. `TEKI`) |
|
| 99 |
+
| `parallelism` / `update_type` / `method_goal` / `emulator_use` | Taxonomy tags β see **Method taxonomy** below |
|
| 100 |
| `rmse_target` | Target RMSE level |
|
| 101 |
| `ensemble_size` | Ensemble / particle count |
|
| 102 |
| `metric` | Mean forward-model runs over `random_seed` (lower = better; β1 = failed) |
|
|
|
|
| 116 |
|---|---|
|
| 117 |
| `benchmark` | `L63`, `L96`, `L96_NN_FORCING`, `L96_SPATIAL_FORCING` |
|
| 118 |
| `algorithm_type` | Canonical method key |
|
| 119 |
+
| `abbreviation` / `Method` / `parallelism` / `update_type` / `method_goal` / `emulator_use` | From method registry |
|
| 120 |
| `uq_target` | Target-scaling level c |
|
| 121 |
| `ensemble_size` | Ensemble size N |
|
| 122 |
| `mean_budget` | Mean NΒ·k_iter over converged seeds (NaN if all failed) |
|
|
|
|
| 149 |
Passed as `benchmark_dims=BENCHMARK_DIMS` to `render_leaderboard()` for column-header
|
| 150 |
annotations in the suitability table. Update if a new benchmark is added.
|
| 151 |
|
| 152 |
+
## Method taxonomy
|
| 153 |
+
|
| 154 |
+
Four independent tags on every `KNOWN_METHODS` entry, each scoped to its own axis:
|
| 155 |
+
|
| 156 |
+
- **`parallelism`** β how the search explores parameter space:
|
| 157 |
+
`serial` (single point estimate advanced step by step, e.g. `ADAM`/`LM`) Β·
|
| 158 |
+
`parallel-independent` (population updated with no coupling between members,
|
| 159 |
+
e.g. `ABC`'s accepted samples / `HM`'s per-wave resampling) Β·
|
| 160 |
+
`parallel-interacting` (ensemble members coupled through a shared update each
|
| 161 |
+
iteration β any Kalman-based method).
|
| 162 |
+
- **`update_type`** β the mechanism driving each update step:
|
| 163 |
+
`gradient` (follows the loss gradient or a Gauss-Newton approximation) Β·
|
| 164 |
+
`kalman` (a possibly linearized/unscented Kalman-style ensemble update) Β·
|
| 165 |
+
`general` (anything else, e.g. `ABC`'s rejection sampling, `HM`'s implausibility cuts).
|
| 166 |
+
- **`method_goal`** β what the method is built to report:
|
| 167 |
+
`optimization` (single best-fit parameter estimate) or `uq` (full posterior /
|
| 168 |
+
parameter uncertainty). UQ methods *can* still be scored on the Optimization
|
| 169 |
+
leaderboard β they're just usually less competitive there.
|
| 170 |
+
- **`emulator_use`** β when/whether a surrogate model of the forward model is used:
|
| 171 |
+
`none` (samples/evaluates the true forward model throughout) Β·
|
| 172 |
+
`within-optimize` (refits a surrogate at each search iteration, e.g. `HM` waves) Β·
|
| 173 |
+
`after-optimize` (fits a surrogate once, after calibration finishes, e.g. `CES-EKI-DMC`).
|
| 174 |
+
|
| 175 |
+
`update_type` deliberately avoids a "Bayesian" label β Kalman methods are approximate
|
| 176 |
+
Bayesian updates too, so the split is about mechanism (gradient vs. Kalman vs. general),
|
| 177 |
+
not philosophy. `_UPDATE_TYPE_ORDER` in `src/common/leaderboard.py` controls
|
| 178 |
+
table/suitability sort order and must stay in sync with `update_type` values.
|
| 179 |
+
A quick taxonomy tree is rendered on the home page (`src/streamlit_app.py`).
|
| 180 |
+
|
| 181 |
## Method registry (`src/common/method_registry.py`)
|
| 182 |
|
| 183 |
+
| Canonical | Abbreviation | parallelism | update_type | method_goal | emulator_use | Aliases |
|
| 184 |
+
|---|---|---|---|---|---|---|
|
| 185 |
+
| `teki` | TEKI | parallel-interacting | kalman | optimization | none | β |
|
| 186 |
+
| `etki` | ETKI | parallel-interacting | kalman | optimization | none | β |
|
| 187 |
+
| `iekf` | IEKF | parallel-interacting | kalman | uq | none | gnsl, gnki |
|
| 188 |
+
| `uki` | UKI | parallel-interacting | kalman | optimization | none | β |
|
| 189 |
+
| `abc` | ABC | parallel-independent | general | uq | none | β |
|
| 190 |
+
| `hm` | HM | parallel-independent | general | uq | within-optimize | β |
|
| 191 |
+
| `ces-eki-dmc` | CES-EKI-DMC | parallel-interacting | kalman | uq | after-optimize | β |
|
| 192 |
+
| `adam` | ADAM | serial | gradient | optimization | none | β |
|
| 193 |
+
| `lm` | LM | serial | gradient | optimization | none | gradient_descent |
|
| 194 |
|
| 195 |
## How to extend
|
| 196 |
|
| 197 |
### Add a method
|
| 198 |
1. Add entry to `KNOWN_METHODS` in `src/common/method_registry.py`
|
| 199 |
+
(keys: `abbreviation`, `Method`, `parallelism`, `update_type`, `method_goal`,
|
| 200 |
+
`emulator_use`, `aliases` β see **Method taxonomy** above for valid tag values).
|
| 201 |
2. Add detail-page metadata to the inline `method_meta` dict in `src/pages/MethodDetails.py`
|
| 202 |
keyed by **abbreviation** (e.g. `"TEKI"`). β οΈ Metadata is split across both files.
|
| 203 |
3. Place result NetCDF(s) in `data/` and add paths to `DATASET_FILES` in `src/data_store.py`.
|
src/common/leaderboard.py
CHANGED
|
@@ -49,13 +49,13 @@ def _method_color(present_abbrevs: list[str]) -> alt.Color:
|
|
| 49 |
legend=alt.Legend(values=sorted(present_abbrevs)),
|
| 50 |
)
|
| 51 |
|
| 52 |
-
#
|
| 53 |
-
# Unknown
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
"
|
| 57 |
-
"gradient":
|
| 58 |
-
"
|
| 59 |
}
|
| 60 |
|
| 61 |
|
|
@@ -72,15 +72,15 @@ def _render_suitability_table(
|
|
| 72 |
store["benchmark"].unique().tolist(),
|
| 73 |
key=lambda bm: benchmark_dims[bm][0] if (benchmark_dims and bm in benchmark_dims) else bm,
|
| 74 |
)
|
| 75 |
-
|
| 76 |
-
store[["abbreviation", "
|
| 77 |
.drop_duplicates("abbreviation")
|
| 78 |
-
.set_index("abbreviation")["
|
| 79 |
.to_dict()
|
| 80 |
)
|
| 81 |
methods = sorted(
|
| 82 |
store["abbreviation"].dropna().unique().tolist(),
|
| 83 |
-
key=lambda a: (
|
| 84 |
)
|
| 85 |
|
| 86 |
target_str = str(float(suitability_target))
|
|
@@ -178,8 +178,9 @@ def render_leaderboard(
|
|
| 178 |
metric_store:
|
| 179 |
DataFrame produced by ``load_metric_store()`` or ``load_uq_store()``.
|
| 180 |
Must contain at least the columns ``benchmark``, ``algorithm_type``,
|
| 181 |
-
``abbreviation``, ``Method``,
|
| 182 |
-
``
|
|
|
|
| 183 |
target_col:
|
| 184 |
Name of the target-coordinate column, e.g. ``"rmse_target"`` or
|
| 185 |
``"uq_target"``.
|
|
@@ -292,7 +293,7 @@ def render_leaderboard(
|
|
| 292 |
|
| 293 |
# Failure rate from every row (NaN metric rows carry failure_rate=100)
|
| 294 |
failure_agg = ranking_source.groupby(
|
| 295 |
-
["algorithm_type", "abbreviation", "Method"
|
| 296 |
).agg(**{"Mean Failure Rate (%)": ("failure_rate", "mean")})
|
| 297 |
|
| 298 |
# Metric stats only from runs that reached the target (non-NaN metric)
|
|
@@ -301,7 +302,7 @@ def render_leaderboard(
|
|
| 301 |
return pd.DataFrame()
|
| 302 |
|
| 303 |
scored_df = valid_rows.groupby(
|
| 304 |
-
["algorithm_type", "abbreviation", "Method"
|
| 305 |
).agg(
|
| 306 |
**{"Mean Forward Model Runs": ("metric", "mean")},
|
| 307 |
**{"Minimum Forward Model Runs": ("metric", "min")},
|
|
@@ -310,13 +311,12 @@ def render_leaderboard(
|
|
| 310 |
|
| 311 |
best_per_target = (
|
| 312 |
valid_rows.sort_values(["algorithm_type", target_str_col, "metric", "ensemble_size"])
|
| 313 |
-
.groupby(["algorithm_type", "abbreviation", "Method",
|
| 314 |
.first()[
|
| 315 |
[
|
| 316 |
"algorithm_type",
|
| 317 |
"abbreviation",
|
| 318 |
"Method",
|
| 319 |
-
"family",
|
| 320 |
target_str_col,
|
| 321 |
"ensemble_size",
|
| 322 |
]
|
|
@@ -324,17 +324,17 @@ def render_leaderboard(
|
|
| 324 |
)
|
| 325 |
|
| 326 |
optimal_ensemble = best_per_target.groupby(
|
| 327 |
-
["algorithm_type", "abbreviation", "Method"
|
| 328 |
).agg(**{"Optimal Ensemble Size": ("ensemble_size", "mean")})
|
| 329 |
|
| 330 |
scored_df = scored_df.merge(
|
| 331 |
optimal_ensemble,
|
| 332 |
-
on=["algorithm_type", "abbreviation", "Method"
|
| 333 |
how="left",
|
| 334 |
)
|
| 335 |
scored_df = scored_df.merge(
|
| 336 |
failure_agg,
|
| 337 |
-
on=["algorithm_type", "abbreviation", "Method"
|
| 338 |
how="left",
|
| 339 |
)
|
| 340 |
|
|
@@ -421,17 +421,25 @@ def render_leaderboard(
|
|
| 421 |
|
| 422 |
leaderboard_df = build_scored_table(filtered, add_rank=True)
|
| 423 |
|
| 424 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 425 |
# Placement numbers still reflect overall performance rank.
|
| 426 |
if not leaderboard_df.empty:
|
| 427 |
leaderboard_df = leaderboard_df.assign(
|
| 428 |
-
|
| 429 |
_is_dnf=leaderboard_df["Score"].isna(),
|
| 430 |
).sort_values(
|
| 431 |
-
["
|
| 432 |
ascending=[True, True, False],
|
| 433 |
na_position="last",
|
| 434 |
-
).drop(columns=["
|
| 435 |
|
| 436 |
if scoring_mode == "Mean Forward Model Runs":
|
| 437 |
score_basis = "mean forward-model runs at the selected target level (lower is better)"
|
|
@@ -706,7 +714,10 @@ def render_leaderboard(
|
|
| 706 |
"Placement",
|
| 707 |
"abbreviation",
|
| 708 |
"Method",
|
| 709 |
-
"
|
|
|
|
|
|
|
|
|
|
| 710 |
"Score",
|
| 711 |
"Mean Forward Model Runs",
|
| 712 |
"Minimum Forward Model Runs",
|
|
@@ -730,7 +741,10 @@ def render_leaderboard(
|
|
| 730 |
use_container_width=True,
|
| 731 |
column_config={
|
| 732 |
"Placement": st.column_config.TextColumn("Placement"),
|
| 733 |
-
"
|
|
|
|
|
|
|
|
|
|
| 734 |
"Method": st.column_config.TextColumn("Method"),
|
| 735 |
"abbreviation": st.column_config.TextColumn("Abbrev."),
|
| 736 |
"Mean Forward Model Runs": st.column_config.NumberColumn("Mean Forward Model Runs", format="%.4f"),
|
|
|
|
| 49 |
legend=alt.Legend(values=sorted(present_abbrevs)),
|
| 50 |
)
|
| 51 |
|
| 52 |
+
# update_type display order in the leaderboard table (lower = earlier).
|
| 53 |
+
# Unknown/unmapped update types fall back to 99 and appear at the end.
|
| 54 |
+
# See method_registry.py for the full four-axis taxonomy.
|
| 55 |
+
_UPDATE_TYPE_ORDER: dict[str, int] = {
|
| 56 |
+
"kalman": 0,
|
| 57 |
+
"gradient": 1,
|
| 58 |
+
"general": 2,
|
| 59 |
}
|
| 60 |
|
| 61 |
|
|
|
|
| 72 |
store["benchmark"].unique().tolist(),
|
| 73 |
key=lambda bm: benchmark_dims[bm][0] if (benchmark_dims and bm in benchmark_dims) else bm,
|
| 74 |
)
|
| 75 |
+
_abbr_update_type = (
|
| 76 |
+
store[["abbreviation", "update_type"]].dropna()
|
| 77 |
.drop_duplicates("abbreviation")
|
| 78 |
+
.set_index("abbreviation")["update_type"]
|
| 79 |
.to_dict()
|
| 80 |
)
|
| 81 |
methods = sorted(
|
| 82 |
store["abbreviation"].dropna().unique().tolist(),
|
| 83 |
+
key=lambda a: (_UPDATE_TYPE_ORDER.get(_abbr_update_type.get(a, ""), 99), a),
|
| 84 |
)
|
| 85 |
|
| 86 |
target_str = str(float(suitability_target))
|
|
|
|
| 178 |
metric_store:
|
| 179 |
DataFrame produced by ``load_metric_store()`` or ``load_uq_store()``.
|
| 180 |
Must contain at least the columns ``benchmark``, ``algorithm_type``,
|
| 181 |
+
``abbreviation``, ``Method``, the four taxonomy tag columns
|
| 182 |
+
(``parallelism``, ``update_type``, ``method_goal``, ``emulator_use``),
|
| 183 |
+
``ensemble_size``, ``metric``, ``failure_rate``, and *target_col*.
|
| 184 |
target_col:
|
| 185 |
Name of the target-coordinate column, e.g. ``"rmse_target"`` or
|
| 186 |
``"uq_target"``.
|
|
|
|
| 293 |
|
| 294 |
# Failure rate from every row (NaN metric rows carry failure_rate=100)
|
| 295 |
failure_agg = ranking_source.groupby(
|
| 296 |
+
["algorithm_type", "abbreviation", "Method"], as_index=False
|
| 297 |
).agg(**{"Mean Failure Rate (%)": ("failure_rate", "mean")})
|
| 298 |
|
| 299 |
# Metric stats only from runs that reached the target (non-NaN metric)
|
|
|
|
| 302 |
return pd.DataFrame()
|
| 303 |
|
| 304 |
scored_df = valid_rows.groupby(
|
| 305 |
+
["algorithm_type", "abbreviation", "Method"], as_index=False
|
| 306 |
).agg(
|
| 307 |
**{"Mean Forward Model Runs": ("metric", "mean")},
|
| 308 |
**{"Minimum Forward Model Runs": ("metric", "min")},
|
|
|
|
| 311 |
|
| 312 |
best_per_target = (
|
| 313 |
valid_rows.sort_values(["algorithm_type", target_str_col, "metric", "ensemble_size"])
|
| 314 |
+
.groupby(["algorithm_type", "abbreviation", "Method", target_str_col], as_index=False)
|
| 315 |
.first()[
|
| 316 |
[
|
| 317 |
"algorithm_type",
|
| 318 |
"abbreviation",
|
| 319 |
"Method",
|
|
|
|
| 320 |
target_str_col,
|
| 321 |
"ensemble_size",
|
| 322 |
]
|
|
|
|
| 324 |
)
|
| 325 |
|
| 326 |
optimal_ensemble = best_per_target.groupby(
|
| 327 |
+
["algorithm_type", "abbreviation", "Method"], as_index=False
|
| 328 |
).agg(**{"Optimal Ensemble Size": ("ensemble_size", "mean")})
|
| 329 |
|
| 330 |
scored_df = scored_df.merge(
|
| 331 |
optimal_ensemble,
|
| 332 |
+
on=["algorithm_type", "abbreviation", "Method"],
|
| 333 |
how="left",
|
| 334 |
)
|
| 335 |
scored_df = scored_df.merge(
|
| 336 |
failure_agg,
|
| 337 |
+
on=["algorithm_type", "abbreviation", "Method"],
|
| 338 |
how="left",
|
| 339 |
)
|
| 340 |
|
|
|
|
| 421 |
|
| 422 |
leaderboard_df = build_scored_table(filtered, add_rank=True)
|
| 423 |
|
| 424 |
+
# Attach the four taxonomy tags (constant per abbreviation, so this is a
|
| 425 |
+
# plain lookup merge rather than a groupby key threaded through every
|
| 426 |
+
# aggregation above).
|
| 427 |
+
if not leaderboard_df.empty:
|
| 428 |
+
tag_cols = ["abbreviation", "parallelism", "update_type", "method_goal", "emulator_use"]
|
| 429 |
+
tag_lookup = filtered[tag_cols].drop_duplicates("abbreviation")
|
| 430 |
+
leaderboard_df = leaderboard_df.merge(tag_lookup, on="abbreviation", how="left")
|
| 431 |
+
|
| 432 |
+
# Re-sort by update_type group, then by performance within each group.
|
| 433 |
# Placement numbers still reflect overall performance rank.
|
| 434 |
if not leaderboard_df.empty:
|
| 435 |
leaderboard_df = leaderboard_df.assign(
|
| 436 |
+
_update_type_sort=leaderboard_df["update_type"].map(_UPDATE_TYPE_ORDER).fillna(99),
|
| 437 |
_is_dnf=leaderboard_df["Score"].isna(),
|
| 438 |
).sort_values(
|
| 439 |
+
["_update_type_sort", "_is_dnf", "Score"],
|
| 440 |
ascending=[True, True, False],
|
| 441 |
na_position="last",
|
| 442 |
+
).drop(columns=["_update_type_sort", "_is_dnf"]).reset_index(drop=True)
|
| 443 |
|
| 444 |
if scoring_mode == "Mean Forward Model Runs":
|
| 445 |
score_basis = "mean forward-model runs at the selected target level (lower is better)"
|
|
|
|
| 714 |
"Placement",
|
| 715 |
"abbreviation",
|
| 716 |
"Method",
|
| 717 |
+
"update_type",
|
| 718 |
+
"parallelism",
|
| 719 |
+
"method_goal",
|
| 720 |
+
"emulator_use",
|
| 721 |
"Score",
|
| 722 |
"Mean Forward Model Runs",
|
| 723 |
"Minimum Forward Model Runs",
|
|
|
|
| 741 |
use_container_width=True,
|
| 742 |
column_config={
|
| 743 |
"Placement": st.column_config.TextColumn("Placement"),
|
| 744 |
+
"update_type": st.column_config.TextColumn("Update Type"),
|
| 745 |
+
"parallelism": st.column_config.TextColumn("Parallelism"),
|
| 746 |
+
"method_goal": st.column_config.TextColumn("Method Goal"),
|
| 747 |
+
"emulator_use": st.column_config.TextColumn("Emulator Use"),
|
| 748 |
"Method": st.column_config.TextColumn("Method"),
|
| 749 |
"abbreviation": st.column_config.TextColumn("Abbrev."),
|
| 750 |
"Mean Forward Model Runs": st.column_config.NumberColumn("Mean Forward Model Runs", format="%.4f"),
|
src/common/method_registry.py
CHANGED
|
@@ -4,59 +4,116 @@ import json
|
|
| 4 |
from pathlib import Path
|
| 5 |
|
| 6 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
KNOWN_METHODS = {
|
| 8 |
"teki": {
|
| 9 |
"abbreviation": "TEKI",
|
| 10 |
"Method": "Tikhonov Regularized Ensemble Kalman Inversion",
|
| 11 |
-
"
|
|
|
|
|
|
|
|
|
|
| 12 |
"aliases": ["teki"],
|
| 13 |
},
|
| 14 |
"etki": {
|
| 15 |
"abbreviation": "ETKI",
|
| 16 |
"Method": "Ensemble Transform Kalman Inversion",
|
| 17 |
-
"
|
|
|
|
|
|
|
|
|
|
| 18 |
"aliases": ["etki"],
|
| 19 |
},
|
| 20 |
"iekf": {
|
| 21 |
"abbreviation": "IEKF",
|
| 22 |
"Method": "Iterative Ensemble Kalman Filter",
|
| 23 |
-
"
|
|
|
|
|
|
|
|
|
|
| 24 |
"aliases": ["iekf", "gnsl", "gnki"],
|
| 25 |
},
|
| 26 |
"uki": {
|
| 27 |
"abbreviation": "UKI",
|
| 28 |
"Method": "Unscented Kalman Inversion",
|
| 29 |
-
"
|
|
|
|
|
|
|
|
|
|
| 30 |
"aliases": ["uki"],
|
| 31 |
},
|
| 32 |
"abc": {
|
| 33 |
"abbreviation": "ABC",
|
| 34 |
"Method": "Approximate Bayesian Calibration",
|
| 35 |
-
"
|
|
|
|
|
|
|
|
|
|
| 36 |
"aliases": ["abc"],
|
| 37 |
},
|
| 38 |
"hm": {
|
| 39 |
"abbreviation": "HM",
|
| 40 |
"Method": "History Matching",
|
| 41 |
-
"
|
|
|
|
|
|
|
|
|
|
| 42 |
"aliases": ["hm"],
|
| 43 |
},
|
| 44 |
"ces-eki-dmc": {
|
| 45 |
"abbreviation": "CES-EKI-DMC",
|
| 46 |
"Method": "Calibrate Emulate Sample (EKI-DataMisfitController)",
|
| 47 |
-
"
|
|
|
|
|
|
|
|
|
|
| 48 |
"aliases": ["ces-eki-dmc"]
|
| 49 |
},
|
| 50 |
"adam": {
|
| 51 |
"abbreviation": "ADAM",
|
| 52 |
"Method": "Adaptive Moment Estimation",
|
| 53 |
-
"
|
|
|
|
|
|
|
|
|
|
| 54 |
"aliases": ["adam"],
|
| 55 |
},
|
| 56 |
"lm": {
|
| 57 |
"abbreviation": "LM",
|
| 58 |
"Method": "Levenberg-Marquardt",
|
| 59 |
-
"
|
|
|
|
|
|
|
|
|
|
| 60 |
"aliases": ["lm", "levenberg_marquardt", "levenberg-marquardt", "gradient_descent"],
|
| 61 |
},
|
| 62 |
}
|
|
|
|
| 4 |
from pathlib import Path
|
| 5 |
|
| 6 |
|
| 7 |
+
# Taxonomy
|
| 8 |
+
# --------
|
| 9 |
+
# Every method carries four independent tags, each scoped to its own axis:
|
| 10 |
+
#
|
| 11 |
+
# ``parallelism`` β how the search explores parameter space:
|
| 12 |
+
# "serial" β a single point estimate advanced step by step (ADAM, LM).
|
| 13 |
+
# "parallel-independent" β a population of candidates updated with no coupling
|
| 14 |
+
# between members (ABC's accepted samples, HM's per-wave
|
| 15 |
+
# resampling from the non-implausible region).
|
| 16 |
+
# "parallel-interacting" β an ensemble whose members are coupled through a shared
|
| 17 |
+
# update each iteration (any Kalman-based method).
|
| 18 |
+
#
|
| 19 |
+
# ``update_type`` β the mechanism driving each update step:
|
| 20 |
+
# "gradient" β follows the loss gradient or a Gauss-Newton approximation of it.
|
| 21 |
+
# "kalman" β a (possibly linearized/unscented) Kalman-style ensemble update.
|
| 22 |
+
# "general" β anything else (e.g. ABC's rejection sampling, HM's implausibility cuts).
|
| 23 |
+
#
|
| 24 |
+
# ``method_goal`` β what the method is built to report:
|
| 25 |
+
# "optimization" β a single best-fit parameter estimate.
|
| 26 |
+
# "uq" β the full posterior / parameter uncertainty. UQ methods can still
|
| 27 |
+
# be scored on the Optimization leaderboard (usually less
|
| 28 |
+
# competitive there, since they're not optimizing for speed-to-target).
|
| 29 |
+
#
|
| 30 |
+
# ``emulator_use`` β when/whether a surrogate model of the forward model is used:
|
| 31 |
+
# "none" β samples/evaluates the true forward model throughout.
|
| 32 |
+
# "within-optimize" β refits a surrogate at each iteration of the search (HM waves).
|
| 33 |
+
# "after-optimize" β fits a surrogate once, after calibration finishes (CES).
|
| 34 |
+
#
|
| 35 |
+
# Note: Kalman methods are Bayesian in spirit too (approximate Gaussian posterior
|
| 36 |
+
# updates) β ``update_type`` is about mechanism, not a "Bayesian vs not" philosophy.
|
| 37 |
KNOWN_METHODS = {
|
| 38 |
"teki": {
|
| 39 |
"abbreviation": "TEKI",
|
| 40 |
"Method": "Tikhonov Regularized Ensemble Kalman Inversion",
|
| 41 |
+
"parallelism": "parallel-interacting",
|
| 42 |
+
"update_type": "kalman",
|
| 43 |
+
"method_goal": "optimization",
|
| 44 |
+
"emulator_use": "none",
|
| 45 |
"aliases": ["teki"],
|
| 46 |
},
|
| 47 |
"etki": {
|
| 48 |
"abbreviation": "ETKI",
|
| 49 |
"Method": "Ensemble Transform Kalman Inversion",
|
| 50 |
+
"parallelism": "parallel-interacting",
|
| 51 |
+
"update_type": "kalman",
|
| 52 |
+
"method_goal": "optimization",
|
| 53 |
+
"emulator_use": "none",
|
| 54 |
"aliases": ["etki"],
|
| 55 |
},
|
| 56 |
"iekf": {
|
| 57 |
"abbreviation": "IEKF",
|
| 58 |
"Method": "Iterative Ensemble Kalman Filter",
|
| 59 |
+
"parallelism": "parallel-interacting",
|
| 60 |
+
"update_type": "kalman",
|
| 61 |
+
"method_goal": "uq",
|
| 62 |
+
"emulator_use": "none",
|
| 63 |
"aliases": ["iekf", "gnsl", "gnki"],
|
| 64 |
},
|
| 65 |
"uki": {
|
| 66 |
"abbreviation": "UKI",
|
| 67 |
"Method": "Unscented Kalman Inversion",
|
| 68 |
+
"parallelism": "parallel-interacting",
|
| 69 |
+
"update_type": "kalman",
|
| 70 |
+
"method_goal": "optimization",
|
| 71 |
+
"emulator_use": "none",
|
| 72 |
"aliases": ["uki"],
|
| 73 |
},
|
| 74 |
"abc": {
|
| 75 |
"abbreviation": "ABC",
|
| 76 |
"Method": "Approximate Bayesian Calibration",
|
| 77 |
+
"parallelism": "parallel-independent",
|
| 78 |
+
"update_type": "general",
|
| 79 |
+
"method_goal": "uq",
|
| 80 |
+
"emulator_use": "none",
|
| 81 |
"aliases": ["abc"],
|
| 82 |
},
|
| 83 |
"hm": {
|
| 84 |
"abbreviation": "HM",
|
| 85 |
"Method": "History Matching",
|
| 86 |
+
"parallelism": "parallel-independent",
|
| 87 |
+
"update_type": "general",
|
| 88 |
+
"method_goal": "uq",
|
| 89 |
+
"emulator_use": "within-optimize",
|
| 90 |
"aliases": ["hm"],
|
| 91 |
},
|
| 92 |
"ces-eki-dmc": {
|
| 93 |
"abbreviation": "CES-EKI-DMC",
|
| 94 |
"Method": "Calibrate Emulate Sample (EKI-DataMisfitController)",
|
| 95 |
+
"parallelism": "parallel-interacting",
|
| 96 |
+
"update_type": "kalman",
|
| 97 |
+
"method_goal": "uq",
|
| 98 |
+
"emulator_use": "after-optimize",
|
| 99 |
"aliases": ["ces-eki-dmc"]
|
| 100 |
},
|
| 101 |
"adam": {
|
| 102 |
"abbreviation": "ADAM",
|
| 103 |
"Method": "Adaptive Moment Estimation",
|
| 104 |
+
"parallelism": "serial",
|
| 105 |
+
"update_type": "gradient",
|
| 106 |
+
"method_goal": "optimization",
|
| 107 |
+
"emulator_use": "none",
|
| 108 |
"aliases": ["adam"],
|
| 109 |
},
|
| 110 |
"lm": {
|
| 111 |
"abbreviation": "LM",
|
| 112 |
"Method": "Levenberg-Marquardt",
|
| 113 |
+
"parallelism": "serial",
|
| 114 |
+
"update_type": "gradient",
|
| 115 |
+
"method_goal": "optimization",
|
| 116 |
+
"emulator_use": "none",
|
| 117 |
"aliases": ["lm", "levenberg_marquardt", "levenberg-marquardt", "gradient_descent"],
|
| 118 |
},
|
| 119 |
}
|
src/data_store.py
CHANGED
|
@@ -59,6 +59,10 @@ DATASET_FILES = {
|
|
| 59 |
EXPECTED_DIMS_OPT = ("algorithm_type", "rmse_target", "ensemble_size", "random_seed")
|
| 60 |
EXPECTED_DIMS_UQ = ("random_seed", "ensemble_size", "k_iter", "coverage_quantile")
|
| 61 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
|
| 63 |
# ---------------------------------------------------------------------------
|
| 64 |
# Uncertainty Quantification datasets
|
|
@@ -229,10 +233,11 @@ def _load_store(dataset_files: dict[str, list[str]], target_col: str, *, drop_na
|
|
| 229 |
merged["ensemble_size"] = merged["ensemble_size"].astype(int)
|
| 230 |
merged["abbreviation"] = merged["algorithm_type"].map(lambda method: get_method_meta(method).get("abbreviation"))
|
| 231 |
merged["Method"] = merged["algorithm_type"].map(lambda method: get_method_meta(method).get("Method"))
|
| 232 |
-
|
|
|
|
|
|
|
| 233 |
merged["abbreviation"] = merged["abbreviation"].fillna(merged["algorithm_type"].str.upper())
|
| 234 |
merged["Method"] = merged["Method"].fillna(merged["abbreviation"])
|
| 235 |
-
merged["family"] = merged["family"].fillna("Kalman")
|
| 236 |
|
| 237 |
try:
|
| 238 |
dump_method_registry_snapshot(project_root=project_root, observed_methods=set(merged["algorithm_type"].unique()))
|
|
@@ -267,7 +272,7 @@ def load_uq_store() -> pd.DataFrame:
|
|
| 267 |
return pd.DataFrame(
|
| 268 |
columns=[
|
| 269 |
"benchmark", "algorithm_type", "algorithm_alias", "abbreviation",
|
| 270 |
-
"Method",
|
| 271 |
]
|
| 272 |
)
|
| 273 |
df = budget_df.copy()
|
|
@@ -281,7 +286,7 @@ def load_uq_store() -> pd.DataFrame:
|
|
| 281 |
return df[
|
| 282 |
[
|
| 283 |
"benchmark", "algorithm_type", "algorithm_alias", "abbreviation",
|
| 284 |
-
"Method",
|
| 285 |
]
|
| 286 |
]
|
| 287 |
|
|
@@ -295,8 +300,9 @@ def load_uq_budget_store() -> pd.DataFrame:
|
|
| 295 |
UQ_COVERAGE_QUANTILES. Budget = N Β· k_iter; NaN when target never reached.
|
| 296 |
|
| 297 |
Returns a DataFrame with columns:
|
| 298 |
-
benchmark, algorithm_type, abbreviation,
|
| 299 |
-
|
|
|
|
| 300 |
"""
|
| 301 |
xr = importlib.import_module("xarray")
|
| 302 |
|
|
@@ -383,7 +389,7 @@ def load_uq_budget_store() -> pd.DataFrame:
|
|
| 383 |
if not records:
|
| 384 |
return pd.DataFrame(
|
| 385 |
columns=[
|
| 386 |
-
"benchmark", "algorithm_type", "abbreviation",
|
| 387 |
"uq_target", "ensemble_size", "mean_budget", "mean_iters",
|
| 388 |
"failure_count", "failure_rate", "n_seeds",
|
| 389 |
]
|
|
@@ -395,6 +401,7 @@ def load_uq_budget_store() -> pd.DataFrame:
|
|
| 395 |
return get_method_meta(canonicalize_method_name(normalize_method_name(m))).get(key, fallback)
|
| 396 |
|
| 397 |
df["abbreviation"] = df["algorithm_type"].map(lambda m: _meta(m, "abbreviation", m.upper()))
|
| 398 |
-
|
|
|
|
| 399 |
df["Method"] = df["algorithm_type"].map(lambda m: _meta(m, "Method", m.upper()))
|
| 400 |
return df
|
|
|
|
| 59 |
EXPECTED_DIMS_OPT = ("algorithm_type", "rmse_target", "ensemble_size", "random_seed")
|
| 60 |
EXPECTED_DIMS_UQ = ("random_seed", "ensemble_size", "k_iter", "coverage_quantile")
|
| 61 |
|
| 62 |
+
# Method-taxonomy tag columns β one value per method, sourced from KNOWN_METHODS.
|
| 63 |
+
# See src/common/method_registry.py for the axis definitions.
|
| 64 |
+
TAXONOMY_TAG_COLUMNS = ["parallelism", "update_type", "method_goal", "emulator_use"]
|
| 65 |
+
|
| 66 |
|
| 67 |
# ---------------------------------------------------------------------------
|
| 68 |
# Uncertainty Quantification datasets
|
|
|
|
| 233 |
merged["ensemble_size"] = merged["ensemble_size"].astype(int)
|
| 234 |
merged["abbreviation"] = merged["algorithm_type"].map(lambda method: get_method_meta(method).get("abbreviation"))
|
| 235 |
merged["Method"] = merged["algorithm_type"].map(lambda method: get_method_meta(method).get("Method"))
|
| 236 |
+
for tag_col in TAXONOMY_TAG_COLUMNS:
|
| 237 |
+
merged[tag_col] = merged["algorithm_type"].map(lambda method: get_method_meta(method).get(tag_col))
|
| 238 |
+
merged[tag_col] = merged[tag_col].fillna("unmapped")
|
| 239 |
merged["abbreviation"] = merged["abbreviation"].fillna(merged["algorithm_type"].str.upper())
|
| 240 |
merged["Method"] = merged["Method"].fillna(merged["abbreviation"])
|
|
|
|
| 241 |
|
| 242 |
try:
|
| 243 |
dump_method_registry_snapshot(project_root=project_root, observed_methods=set(merged["algorithm_type"].unique()))
|
|
|
|
| 272 |
return pd.DataFrame(
|
| 273 |
columns=[
|
| 274 |
"benchmark", "algorithm_type", "algorithm_alias", "abbreviation",
|
| 275 |
+
"Method", *TAXONOMY_TAG_COLUMNS, "uq_target", "ensemble_size", "metric", "failure_rate",
|
| 276 |
]
|
| 277 |
)
|
| 278 |
df = budget_df.copy()
|
|
|
|
| 286 |
return df[
|
| 287 |
[
|
| 288 |
"benchmark", "algorithm_type", "algorithm_alias", "abbreviation",
|
| 289 |
+
"Method", *TAXONOMY_TAG_COLUMNS, "uq_target", "ensemble_size", "metric", "failure_rate",
|
| 290 |
]
|
| 291 |
]
|
| 292 |
|
|
|
|
| 300 |
UQ_COVERAGE_QUANTILES. Budget = N Β· k_iter; NaN when target never reached.
|
| 301 |
|
| 302 |
Returns a DataFrame with columns:
|
| 303 |
+
benchmark, algorithm_type, abbreviation, parallelism, update_type,
|
| 304 |
+
method_goal, emulator_use, uq_target, ensemble_size, mean_budget,
|
| 305 |
+
mean_iters, failure_count, failure_rate, n_seeds
|
| 306 |
"""
|
| 307 |
xr = importlib.import_module("xarray")
|
| 308 |
|
|
|
|
| 389 |
if not records:
|
| 390 |
return pd.DataFrame(
|
| 391 |
columns=[
|
| 392 |
+
"benchmark", "algorithm_type", "abbreviation", *TAXONOMY_TAG_COLUMNS,
|
| 393 |
"uq_target", "ensemble_size", "mean_budget", "mean_iters",
|
| 394 |
"failure_count", "failure_rate", "n_seeds",
|
| 395 |
]
|
|
|
|
| 401 |
return get_method_meta(canonicalize_method_name(normalize_method_name(m))).get(key, fallback)
|
| 402 |
|
| 403 |
df["abbreviation"] = df["algorithm_type"].map(lambda m: _meta(m, "abbreviation", m.upper()))
|
| 404 |
+
for tag_col in TAXONOMY_TAG_COLUMNS:
|
| 405 |
+
df[tag_col] = df["algorithm_type"].map(lambda m, col=tag_col: _meta(m, col, "unmapped"))
|
| 406 |
df["Method"] = df["algorithm_type"].map(lambda m: _meta(m, "Method", m.upper()))
|
| 407 |
return df
|
src/pages/MethodDetails.py
CHANGED
|
@@ -33,7 +33,7 @@ if metric_store.empty:
|
|
| 33 |
st.warning("No metric data found. Expected NetCDF files in `data/` with a `metric` variable.")
|
| 34 |
st.stop()
|
| 35 |
|
| 36 |
-
methods_df = metric_store[["
|
| 37 |
|
| 38 |
abbrs = methods_df["abbreviation"].tolist()
|
| 39 |
|
|
|
|
| 33 |
st.warning("No metric data found. Expected NetCDF files in `data/` with a `metric` variable.")
|
| 34 |
st.stop()
|
| 35 |
|
| 36 |
+
methods_df = metric_store[["Method", "abbreviation"]].drop_duplicates().sort_values("abbreviation")
|
| 37 |
|
| 38 |
abbrs = methods_df["abbreviation"].tolist()
|
| 39 |
|
src/pages/RawData.py
CHANGED
|
@@ -40,7 +40,10 @@ st.dataframe(
|
|
| 40 |
"benchmark",
|
| 41 |
"abbreviation",
|
| 42 |
"Method",
|
| 43 |
-
"
|
|
|
|
|
|
|
|
|
|
| 44 |
"algorithm_type",
|
| 45 |
"algorithm_alias",
|
| 46 |
"rmse_target",
|
|
@@ -54,7 +57,10 @@ st.dataframe(
|
|
| 54 |
"benchmark": st.column_config.TextColumn("Benchmark"),
|
| 55 |
"abbreviation": st.column_config.TextColumn("Abbrev."),
|
| 56 |
"Method": st.column_config.TextColumn("Method"),
|
| 57 |
-
"
|
|
|
|
|
|
|
|
|
|
| 58 |
"algorithm_type": st.column_config.TextColumn("Canonical Type"),
|
| 59 |
"algorithm_alias": st.column_config.TextColumn("Source Alias"),
|
| 60 |
"rmse_target": st.column_config.TextColumn("RMSE Target"),
|
|
|
|
| 40 |
"benchmark",
|
| 41 |
"abbreviation",
|
| 42 |
"Method",
|
| 43 |
+
"parallelism",
|
| 44 |
+
"update_type",
|
| 45 |
+
"method_goal",
|
| 46 |
+
"emulator_use",
|
| 47 |
"algorithm_type",
|
| 48 |
"algorithm_alias",
|
| 49 |
"rmse_target",
|
|
|
|
| 57 |
"benchmark": st.column_config.TextColumn("Benchmark"),
|
| 58 |
"abbreviation": st.column_config.TextColumn("Abbrev."),
|
| 59 |
"Method": st.column_config.TextColumn("Method"),
|
| 60 |
+
"parallelism": st.column_config.TextColumn("Parallelism"),
|
| 61 |
+
"update_type": st.column_config.TextColumn("Update Type"),
|
| 62 |
+
"method_goal": st.column_config.TextColumn("Method Goal"),
|
| 63 |
+
"emulator_use": st.column_config.TextColumn("Emulator Use"),
|
| 64 |
"algorithm_type": st.column_config.TextColumn("Canonical Type"),
|
| 65 |
"algorithm_alias": st.column_config.TextColumn("Source Alias"),
|
| 66 |
"rmse_target": st.column_config.TextColumn("RMSE Target"),
|
src/streamlit_app.py
CHANGED
|
@@ -81,18 +81,48 @@ st.image(
|
|
| 81 |
width=1200,
|
| 82 |
)
|
| 83 |
|
| 84 |
-
st.subheader("Method
|
| 85 |
st.markdown(
|
| 86 |
-
"
|
| 87 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
)
|
| 89 |
-
families = {
|
| 90 |
-
"Kalman": "Ensemble Kalman variants (TEKI, ETKI, IEKF, UKI) β update an ensemble of parameter guesses via a linearized observation operator.",
|
| 91 |
-
"Bayesian": "Sampling-based approaches (ABC, HM) β explore parameter space without requiring gradient information.",
|
| 92 |
-
"Calibrate-then-emulate": "Two-stage pipelines (CES-EKI-DMC) β use an initial calibration phase to build a cheap emulator, then sample the posterior via MCMC.",
|
| 93 |
-
}
|
| 94 |
-
for family, desc in families.items():
|
| 95 |
-
st.markdown(f"- **{family}** β {desc}")
|
| 96 |
|
| 97 |
st.subheader("Key metric")
|
| 98 |
st.markdown(
|
|
|
|
| 81 |
width=1200,
|
| 82 |
)
|
| 83 |
|
| 84 |
+
st.subheader("Method taxonomy")
|
| 85 |
st.markdown(
|
| 86 |
+
"Every method carries four independent tags β how it searches, what update "
|
| 87 |
+
"mechanism drives each step, what it's built to report, and whether/when it uses "
|
| 88 |
+
"a surrogate model. See the **π Methods** page for citations and per-method "
|
| 89 |
+
"performance charts."
|
| 90 |
+
)
|
| 91 |
+
st.markdown(
|
| 92 |
+
"""
|
| 93 |
+
- **Parallelism** β how the search explores parameter space
|
| 94 |
+
- **Serial** β `ADAM`, `LM` β a single point estimate advanced step by step.
|
| 95 |
+
- **Parallel-independent** β `ABC`, `HM` β a population of candidates updated
|
| 96 |
+
with no coupling between members (accepted samples / per-wave resampling).
|
| 97 |
+
- **Parallel-interacting** β `TEKI`, `ETKI`, `IEKF`, `UKI`, `CES-EKI-DMC` β an
|
| 98 |
+
ensemble whose members are coupled through a shared update each iteration.
|
| 99 |
+
- **Update type** β the mechanism driving each update step
|
| 100 |
+
- **Gradient** β `ADAM`, `LM` β follow the loss gradient (or a Gauss-Newton
|
| 101 |
+
approximation of it) directly.
|
| 102 |
+
- **Kalman** β `TEKI`, `ETKI`, `IEKF`, `UKI`, `CES-EKI-DMC` β a (possibly
|
| 103 |
+
linearized or unscented) Kalman-style ensemble update.
|
| 104 |
+
- **General** β `ABC`, `HM` β neither gradient- nor Kalman-based (rejection
|
| 105 |
+
sampling, implausibility cuts).
|
| 106 |
+
- **Method goal** β what the method is built to report
|
| 107 |
+
- **Optimization** β `TEKI`, `ETKI`, `UKI`, `ADAM`, `LM` β a single best-fit
|
| 108 |
+
parameter estimate.
|
| 109 |
+
- **UQ** β `IEKF`, `ABC`, `HM`, `CES-EKI-DMC` β the full posterior / parameter
|
| 110 |
+
uncertainty. Can still be scored on the Optimization leaderboard, but tends to
|
| 111 |
+
be less competitive there since it's not optimizing for speed-to-target.
|
| 112 |
+
- **Emulator use** β when/whether a surrogate model of the forward model is used
|
| 113 |
+
- **None** β `TEKI`, `ETKI`, `IEKF`, `UKI`, `ADAM`, `LM`, `ABC` β samples/evaluates
|
| 114 |
+
the true forward model throughout.
|
| 115 |
+
- **Within-optimize** β `HM` β refits a surrogate at each iteration (wave) of the
|
| 116 |
+
search itself.
|
| 117 |
+
- **After-optimize** β `CES-EKI-DMC` β fits a surrogate (e.g. a GP) once, after
|
| 118 |
+
calibration finishes, and samples the posterior through it.
|
| 119 |
+
"""
|
| 120 |
+
)
|
| 121 |
+
st.caption(
|
| 122 |
+
"Note: Kalman methods are Bayesian in spirit too (they're approximate Gaussian "
|
| 123 |
+
"posterior updates) β update type is about mechanism (gradient vs. Kalman vs. "
|
| 124 |
+
"general), not whether a method is 'Bayesian'."
|
| 125 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 126 |
|
| 127 |
st.subheader("Key metric")
|
| 128 |
st.markdown(
|