Add gradient-based results with a typical configuration of hyperparameters

.cache/known_methods_snapshot.json

@@ -57,13 +57,34 @@
       "aliases": [
         "ces-eki-dmc"
       ]
+    },
+    "adam": {
+      "abbreviation": "ADAM",
+      "Method": "Adaptive Moment Estimation",
+      "family": "gradient",
+      "aliases": [
+        "adam"
+      ]
+    },
+    "lm": {
+      "abbreviation": "LM",
+      "Method": "Levenberg-Marquardt",
+      "family": "gradient",
+      "aliases": [
+        "lm",
+        "levenberg_marquardt",
+        "levenberg-marquardt",
+        "gradient_descent"
+      ]
     }
   },
   "observed_methods": [
     "abc",
+    "adam",
     "etki",
     "hm",
     "iekf",
+    "lm",
     "teki",
     "uki"
   ],

.claude/skills/leaderboard-manager/references/manage-display.md

@@ -206,6 +206,60 @@ x=alt.X(
 
 ---
 
+## Plotting conventions and shared display constants
+
+### Where shared constants live
+
+`src/common/method_registry.py` is the single source of truth for any display property
+that must be consistent across multiple pages or charts. Currently it exports:
+
+- **`METHOD_COLORS: dict[str, str]`** — abbreviation → hex color for every known method,
+  derived from `KNOWN_METHODS` in declaration order.
+- **`_METHOD_PALETTE`** — the 10-slot Vega tableau10 palette that backs `METHOD_COLORS`.
+
+**Rule:** if a constant is used in more than one file, it belongs in `method_registry.py`,
+not defined locally in `leaderboard.py` or a page file.
+
+When **adding a new method**, its color is assigned automatically — no manual update
+needed, as long as it is appended to the end of `KNOWN_METHODS`. The palette cycles every
+10 methods.
+
+### How to import shared colors in a new chart
+
+```python
+try:
+    from common.method_registry import METHOD_COLORS
+except ModuleNotFoundError:
+    from src.common.method_registry import METHOD_COLORS
+
+method_color = alt.Color(
+    "abbreviation:N",
+    title="Method",
+    scale=alt.Scale(domain=list(METHOD_COLORS.keys()), range=list(METHOD_COLORS.values())),
+)
+```
+
+Always use the full registry domain/range (not a filtered per-benchmark subset) so colors
+are stable when the user switches benchmarks or navigates between pages.
+
+### General plotting conventions
+
+Unless the task explicitly asks for something different, follow these defaults for every
+new Altair chart in the app:
+
+| Concern | Convention |
+|---|---|
+| Method color | `METHOD_COLORS` via the import above — never a local palette |
+| Benchmark color | `alt.Color("benchmark:N")` — Altair assigns automatically; no custom palette needed |
+| Full-width charts | `st.altair_chart(chart, use_container_width=True)` — never `width="stretch"` |
+| Ensemble-size x-axis (quantitative) | `alt.Axis(values=ens_ticks, format="d")` — derive ticks from data, integer format |
+| Ensemble-size x-axis (ordinal bar) | `ensemble_size:O` with `sort=[str(e) for e in ens_ticks]` and `axis=alt.Axis(labelAngle=0)` |
+| Y-axis scale | Linear by default; add `scale=alt.Scale(type="log")` only when explicitly requested |
+| Tooltips | Always include `abbreviation`, `ensemble_size`, and the primary metric with `format=".4f"` |
+| Failed runs | Mark with a cross point (`mark_point(shape="cross", angle=45, size=200, filled=True)`) at y=0, same color as the method line |
+
+---
+
 ## Display change checklist
 
 1. Edit the relevant file (`leaderboard.py` for both leaderboards; `MethodDetails.py` for

CLAUDE.md

@@ -61,6 +61,8 @@ calibration_benchmark/
 │   ├── *.nc                     Kalman results (TEKI, ETKI, IEKF)
 │   ├── bayesian/                ABC + HM results
 │   ├── UKI_results/             UKI results
+│   ├── adam_results/            ADAM gradient optimizer results
+│   ├── levenberg_marquardt_results/  LM gradient optimizer results
 │   └── ces-eki-dmc_results/     CES-EKI-DMC ensemble-results files (UQ source)
 │       ├── ces-eki-dmc_l63_ensemble_results_2026-06-15.nc
 │       ├── ces-eki-dmc_l96_ensemble_results_2026-06-15.nc
@@ -86,7 +88,7 @@ calibration_benchmark/
 | `algorithm_type` | Canonical method key (e.g. `teki`) |
 | `algorithm_alias` | Normalized raw name from NetCDF |
 | `abbreviation` | Display name (e.g. `TEKI`) |
-| `family` | `Kalman` · `Bayesian` · `calibrate_then_emulate` |
+| `family` | `Kalman` · `Bayesian` · `calibrate_then_emulate` · `gradient` |
 | `rmse_target` | Target RMSE level |
 | `ensemble_size` | Ensemble / particle count |
 | `metric` | Mean forward-model runs over `random_seed` (lower = better; −1 = failed) |
@@ -150,6 +152,8 @@ annotations in the suitability table. Update if a new benchmark is added.
 | `abc` | ABC | Bayesian | — |
 | `hm` | HM | Bayesian | — |
 | `ces-eki-dmc` | CES-EKI-DMC | calibrate_then_emulate | — |
+| `adam` | ADAM | gradient | — |
+| `lm` | LM | gradient | gradient_descent |
 
 ## How to extend
 

data/UKI_results/uki_l96_spatial_forcing_ensemble_results.nc

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dad0b7dfa06ba4e56bedd0173a4c9312674e9ab97845244578b27abb0cef997a
-size 12408
+oid sha256:1c8999ffa5c70d8e891443a24929622e203e0e55bb263be52add1b0585b8a6c7
+size 12343

data/adam_results/leaderboard_adam_l63_2026-06-26.nc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecd403e20e439541dd463e8776df75b7f07f129b728499a1b0c856513ee6dd07
+size 11440

data/adam_results/leaderboard_adam_l96_const-force_2026-06-26.nc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b8487b9895121629d875a23dccdbd3fcc32ea138044e87ef6590d531b4d85d2
+size 11440

data/adam_results/leaderboard_adam_l96_vec-force_2026-06-26.nc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:539eb03a55d4ffe0a51b3b45db3b5dc0d0e159c8132e71639ba2eb5c0f72af95
+size 11440

data/levenberg_marquardt_results/leaderboard_lm_l63_2026-06-29.nc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3fd9735597718164ef37f3f24bd010d6984b6b1ad9cec7d7e8d9f851715f791
+size 11440

data/levenberg_marquardt_results/leaderboard_lm_l96_const-force_2026-06-29.nc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5de3b6ad488344217323ac4be8b96ad213c2cba03b075ace5fb57ef09d9e67b
+size 11440

data/levenberg_marquardt_results/leaderboard_lm_l96_vec-force_2026-06-29.nc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a647aeac2cc740665d27164aef3295bf7f07f8d461013dcb396ec843624c47e
+size 11440

src/common/leaderboard.py

@@ -24,18 +24,24 @@ import altair as alt
 import pandas as pd
 import streamlit as st
 
-# Stable categorical palette (Vega tableau10). Methods are assigned colors by their
-# sorted position in the full available-method list, so toggling visibility never
-# reassigns colors to other methods.
-_METHOD_PALETTE = [
-    "#4c78a8", "#f58518", "#e45756", "#72b7b2", "#54a24b",
-    "#eeca3b", "#b279a2", "#ff9da6", "#9d755d", "#bab0ac",
-]
+try:
+    from common.method_registry import METHOD_COLORS
+except ModuleNotFoundError:
+    from src.common.method_registry import METHOD_COLORS
 
 _SUITABLE   = "#009E73"  # Okabe-Ito teal-green (colorblind-safe)
 _UNSUITABLE = "#C0392B"  # dark red
 _UNTESTED   = "#BDBDBD"  # gray
 
+# Family display order in the leaderboard table (lower = earlier).
+# Unknown families fall back to 99 and appear at the end.
+_FAMILY_ORDER: dict[str, int] = {
+    "Kalman": 0,
+    "Bayesian": 1,
+    "gradient": 2,
+    "calibrate_then_emulate": 3,
+}
+
 
 def _render_suitability_table(
     store: pd.DataFrame,
@@ -50,7 +56,16 @@ def _render_suitability_table(
         store["benchmark"].unique().tolist(),
         key=lambda bm: benchmark_dims[bm][0] if (benchmark_dims and bm in benchmark_dims) else bm,
     )
-    methods    = sorted(store["abbreviation"].dropna().unique().tolist())
+    _abbr_family = (
+        store[["abbreviation", "family"]].dropna()
+        .drop_duplicates("abbreviation")
+        .set_index("abbreviation")["family"]
+        .to_dict()
+    )
+    methods = sorted(
+        store["abbreviation"].dropna().unique().tolist(),
+        key=lambda a: (_FAMILY_ORDER.get(_abbr_family.get(a, ""), 99), a),
+    )
 
     target_str     = str(float(suitability_target))
     target_str_col = f"{target_col}_str"
@@ -119,8 +134,8 @@ def _render_suitability_table(
         f"Evaluated at target = {suitability_target}. "
         f"**Green**: at some ensemble size, failure rate < {failure_threshold:.0f}% "
         f"and mean budget ≤ {ratio_threshold:.0f}× the best method (ratio shown). "
-        "**Red**: does not meet criteria. "
-        "**Gray**: no data for this experiment."
+        "**Red** (failed): data present but all runs failed to reach the target. "
+        "**Gray (—)**: no data for this benchmark."
     )
     st.dataframe(display_df.style.apply(_style, axis=None), use_container_width=True)
 
@@ -254,14 +269,12 @@ def render_leaderboard(
     valid_saved       = [m for m in saved_methods if m in available_methods]
     st.session_state[k_methods] = valid_saved if valid_saved else available_methods
 
-    # Stable color scale: domain covers ALL methods so colors don't shift when a
-    # subset is displayed.
-    color_domain = available_methods
-    color_range  = [_METHOD_PALETTE[i % len(_METHOD_PALETTE)] for i in range(len(available_methods))]
+    # Color scale fixed to the full registry so every chart in the app uses the same
+    # color per method regardless of which benchmark or page is shown.
     method_color = alt.Color(
         "abbreviation:N",
         title="Method",
-        scale=alt.Scale(domain=color_domain, range=color_range),
+        scale=alt.Scale(domain=list(METHOD_COLORS.keys()), range=list(METHOD_COLORS.values())),
     )
 
     def build_scored_table(input_df: pd.DataFrame, add_rank: bool = True) -> pd.DataFrame:
@@ -377,10 +390,41 @@ def render_leaderboard(
                 lambda rank: f"{ {1: '🥇', 2: '🥈', 3: '🥉'}.get(rank, '')} #{rank}".strip()
             )
 
+        # Append methods that had data for this target but every seed failed (all-NaN metric).
+        # They appear at the bottom of the table as "DNF" so users can distinguish
+        # "tried and failed" from "not tested on this benchmark".
+        tried_abbrevs  = set(ranking_source["abbreviation"].dropna().unique())
+        ranked_abbrevs = set(scored_df["abbreviation"].dropna().unique())
+        dnf_abbrevs    = tried_abbrevs - ranked_abbrevs
+        if dnf_abbrevs:
+            dnf_rows = failure_agg[failure_agg["abbreviation"].isin(dnf_abbrevs)].copy()
+            for col in [
+                "Score", "mean_runs_score", "minimum_runs_score", "ensemble_score",
+                "Mean Forward Model Runs", "Minimum Forward Model Runs",
+                "Optimal Ensemble Size", "Ensemble Sizes Used",
+            ]:
+                dnf_rows[col] = float("nan")
+            if add_rank:
+                dnf_rows["Rank"] = float("nan")
+                dnf_rows["Placement"] = "DNF"
+            scored_df = pd.concat([scored_df, dnf_rows], ignore_index=True)
+
         return scored_df
 
     leaderboard_df = build_scored_table(filtered, add_rank=True)
 
+    # Re-sort by family group, then by performance within each group.
+    # Placement numbers still reflect overall performance rank.
+    if not leaderboard_df.empty:
+        leaderboard_df = leaderboard_df.assign(
+            _family_sort=leaderboard_df["family"].map(_FAMILY_ORDER).fillna(99),
+            _is_dnf=leaderboard_df["Score"].isna(),
+        ).sort_values(
+            ["_family_sort", "_is_dnf", "Score"],
+            ascending=[True, True, False],
+            na_position="last",
+        ).drop(columns=["_family_sort", "_is_dnf"]).reset_index(drop=True)
+
     if scoring_mode == "Mean Forward Model Runs":
         score_basis = "mean forward-model runs at the selected target level (lower is better)"
     elif scoring_mode == "Minimum Forward Model Runs":
@@ -538,6 +582,35 @@ def render_leaderboard(
                     ],
                 )
             )
+        # Dashed vertical rule for methods with only one ensemble size so they
+        # remain easy to spot when multi-ensemble methods dominate the x-axis.
+        if not all_ens_combos.empty:
+            single_ens_abbrevs = (
+                all_ens_combos.groupby("abbreviation")["ensemble_size"]
+                .nunique()
+                .pipe(lambda s: s[s == 1].index.tolist())
+            )
+            if single_ens_abbrevs:
+                rule_df = (
+                    all_ens_combos[all_ens_combos["abbreviation"].isin(single_ens_abbrevs)]
+                    .drop_duplicates()
+                )
+                chart_layers.append(
+                    alt.Chart(rule_df)
+                    .mark_rule(strokeDash=[4, 4], opacity=0.5)
+                    .encode(
+                        x=alt.X(
+                            "ensemble_size:Q",
+                            axis=alt.Axis(values=ens_ticks, format="d"),
+                        ),
+                        color=method_color,
+                        tooltip=[
+                            alt.Tooltip("abbreviation:N", title="Method"),
+                            alt.Tooltip("ensemble_size:Q", title="Ensemble Size"),
+                        ],
+                    )
+                )
+
         if chart_layers:
             st.altair_chart(alt.layer(*chart_layers), use_container_width=True)
 

src/common/method_registry.py

@@ -46,7 +46,34 @@ KNOWN_METHODS = {
         "Method": "Calibrate Emulate Sample (EKI-DataMisfitController)",
         "family": "calibrate_then_emulate",
         "aliases": ["ces-eki-dmc"]
-    }
+    },
+    "adam": {
+        "abbreviation": "ADAM",
+        "Method": "Adaptive Moment Estimation",
+        "family": "gradient",
+        "aliases": ["adam"],
+    },
+    "lm": {
+        "abbreviation": "LM",
+        "Method": "Levenberg-Marquardt",
+        "family": "gradient",
+        "aliases": ["lm", "levenberg_marquardt", "levenberg-marquardt", "gradient_descent"],
+    },
+}
+
+
+# Vega tableau10 palette — one slot per method in KNOWN_METHODS declaration order.
+# New methods appended to KNOWN_METHODS get the next slot; existing colors never shift.
+_METHOD_PALETTE = [
+    "#4c78a8", "#f58518", "#e45756", "#72b7b2", "#54a24b",
+    "#eeca3b", "#b279a2", "#ff9da6", "#9d755d", "#bab0ac",
+]
+
+# Stable abbreviation → hex color mapping.  Import this wherever Altair charts are built
+# so every plot in the app assigns the same color to each method.
+METHOD_COLORS: dict[str, str] = {
+    meta["abbreviation"]: _METHOD_PALETTE[i % len(_METHOD_PALETTE)]
+    for i, meta in enumerate(KNOWN_METHODS.values())
 }
 
 

src/data_store.py

@@ -32,21 +32,27 @@ DATASET_FILES = {
         "l63_ensemble_results.nc",
         "UKI_results/uki_l63_ensemble_results.nc",
         "bayesian/l63_abc.nc",
-        "bayesian/l63_hm.nc"
+        "bayesian/l63_hm.nc",
+        "adam_results/leaderboard_adam_l63_2026-06-26.nc",
+        "levenberg_marquardt_results/leaderboard_lm_l63_2026-06-29.nc",
     ],
     "L96": [
         "l96_ensemble_results.nc",
         "UKI_results/uki_l96_ensemble_results.nc",
-        "bayesian/l96_abc.nc"
+        "bayesian/l96_abc.nc",
+        "adam_results/leaderboard_adam_l96_const-force_2026-06-26.nc",
+        "levenberg_marquardt_results/leaderboard_lm_l96_const-force_2026-06-29.nc",
     ],
     "L96_NN_FORCING": [
         "l96_nn_forcing_ensemble_results.nc",
-        "UKI_results/uki_l96_nn_forcing_ensemble_results.nc"
+        "UKI_results/uki_l96_nn_forcing_ensemble_results.nc",
     ],
     "L96_SPATIAL_FORCING": [
         "l96_spatial_forcing_ensemble_results.nc",
         "UKI_results/uki_l96_spatial_forcing_ensemble_results.nc",
-        "bayesian/l96_varying_abc.nc"
+        "bayesian/l96_varying_abc.nc",
+        "adam_results/leaderboard_adam_l96_vec-force_2026-06-26.nc",
+        "levenberg_marquardt_results/leaderboard_lm_l96_vec-force_2026-06-29.nc",
     ],
 }
 
@@ -146,8 +152,8 @@ def _load_store(dataset_files: dict[str, list[str]], target_col: str, *, drop_na
                     )
                     continue
 
-                # Track failures (metric == -1) as a percentage
-                failures_count = (metric == -1).sum(dim="random_seed")
+                # Track failures: -1 sentinel OR NaN (some methods use NaN instead of -1)
+                failures_count = ((metric == -1) | metric.isnull()).sum(dim="random_seed")
 
                 failure_rate = (failures_count / dataset.sizes["random_seed"]) * 100
 
@@ -180,7 +186,12 @@ def _load_store(dataset_files: dict[str, list[str]], target_col: str, *, drop_na
                     )
                     continue
 
-                records.append(df[required_cols])
+                # Tag rows from files with exactly one ensemble size.  Only those rows
+                # are retained when metric is NaN — they represent a method that was
+                # genuinely attempted at that size but every seed failed.  NaN rows from
+                # multi-size files are placeholder entries for sizes never actually run.
+                df["_single_ens"] = dataset.sizes.get("ensemble_size", 1) == 1
+                records.append(df[required_cols + ["_single_ens"]])
 
     if not records:
         return pd.DataFrame(
@@ -193,10 +204,16 @@ def _load_store(dataset_files: dict[str, list[str]], target_col: str, *, drop_na
     merged["ensemble_size"] = pd.to_numeric(merged["ensemble_size"], errors="coerce")
     merged["metric"] = pd.to_numeric(merged["metric"], errors="coerce")
     merged["failure_rate"] = pd.to_numeric(merged["failure_rate"], errors="coerce").fillna(0.0)
+    merged = merged.dropna(subset=["ensemble_size"])
     if drop_nan_metric:
-        merged = merged.dropna(subset=["ensemble_size", "metric"])
+        # Keep NaN-metric rows only when they came from a single-ensemble-size file
+        # AND every seed failed (failure_rate ≈ 100 %).  Those represent a method that
+        # was genuinely attempted but never converged.  NaN rows from multi-size files
+        # are placeholders for ensemble sizes that were never actually run.
+        single_ens = merged.pop("_single_ens").fillna(False)
+        merged = merged[merged["metric"].notna() | (single_ens & (merged["failure_rate"] >= 99.9))]
     else:
-        merged = merged.dropna(subset=["ensemble_size"])
+        merged = merged.drop(columns=["_single_ens"], errors="ignore")
     merged["ensemble_size"] = merged["ensemble_size"].astype(int)
     merged["abbreviation"] = merged["algorithm_type"].map(lambda method: get_method_meta(method).get("abbreviation"))
     merged["Method"] = merged["algorithm_type"].map(lambda method: get_method_meta(method).get("Method"))

src/pages/MethodDetails.py

@@ -74,6 +74,16 @@ method_meta = {
         "url": "https://doi.org/10.1016/j.jcp.2020.109716",
         "summary": "Calibrate-Emulate-Sample: uses EKI with a DataMisfitController to select training points, builds a GP emulator of the forward model, then samples the posterior via MCMC.",
     },
+    "ADAM": {
+        "citation": "Kingma & Ba, ICLR, 2015",
+        "url": "https://doi.org/10.48550/arXiv.1412.6980",
+        "summary": "Adaptive Moment Estimation — gradient-based optimizer that adapts per-parameter learning rates using first and second moment estimates of the gradient.",
+    },
+    "LM": {
+        "citation": "Levenberg, 1944; Marquardt, 1963; Fletcher, 1971",
+        "url": "https://doi.org/10.1090/qam/10666",
+        "summary": "Levenberg-Marquardt — damped least-squares algorithm that interpolates between gradient descent and Gauss-Newton steps for efficient nonlinear least-squares minimization.",
+    },
 }
 
 # Selection UI (defaults to query param if valid)

src/pages/OptimizationLeaderboard.py

@@ -34,4 +34,5 @@ render_leaderboard(
     default_target=1.1,
     raw_page="pages/RawData.py",
     benchmark_dims=BENCHMARK_DIMS,
+    show_failure_panel=True,
 )