Surface per-assay failures in admin endpoints instead of silently dropping them

Both /api/admin/run-benchmarks and /api/admin/seed-commons caught (or in
seed-commons' case, didn't even catch) per-assay errors with no way to see
why a dataset went missing from the result short of server log access. A
real run against the 3 bundled fixtures came back with only 1 dataset and
no indication why the other 2 were dropped. Now both endpoints return a
"failed": [{name, error}] list alongside any partial success, and
seed-commons' CSV-parsing loop is wrapped so one bad fixture degrades
instead of 500ing the whole request.

dee/server.py

@@ -2782,22 +2782,26 @@ def create_app() -> Flask:
 
         scorer = _scoring.get_scorer(model)
         results = []
+        failed = []
         for a in assays:
+            name = a.get("name", a.get("protein", "?"))
             try:
                 csv_path = fixtures_dir / a["csv"]
                 recs = _parse_dms(csv_path.read_text(encoding="utf-8"))
                 if not recs:
+                    failed.append({"name": name, "error": "no parseable records in csv"})
                     continue
                 labels = [lab for (lab, _v) in recs]
                 measured = [v for (_lab, v) in recs]
                 scores_df = _scoring.score_guarded(scorer, a["sequence"])
                 predicted = _bm.predict_additive(scores_df, labels)
                 results.append(_bm.evaluate_dataset(
-                    a.get("name", a.get("protein", "?")), a.get("protein", ""),
+                    name, a.get("protein", ""),
                     predicted, measured, source=a.get("source", ""),
                 ))
-            except Exception:  # noqa: BLE001 — one bad assay must not kill the run
-                logger.exception("benchmark assay failed: %s", a.get("name"))
+            except Exception as exc:  # noqa: BLE001 — one bad assay must not kill the run
+                logger.exception("benchmark assay failed: %s", name)
+                failed.append({"name": name, "error": f"{type(exc).__name__}: {exc}"})
 
         out = {
             "generated_at": _dt3.datetime.now(_dt3.timezone.utc).isoformat(),
@@ -2807,6 +2811,8 @@ def create_app() -> Flask:
             "summary": _bm.summarize(results),
             "datasets": [r.as_dict() for r in results],
         }
+        if failed:
+            out["failed"] = failed
         out_path = Path(__file__).resolve().parent / "data" / "benchmarks.json"
         try:
             out_path.write_text(json.dumps(out, indent=2), encoding="utf-8")
@@ -2840,11 +2846,19 @@ def create_app() -> Flask:
             return jsonify({"ok": False, "error": "manifest unreadable"}), 500
 
         assays = []
+        failed = []
         for a in manifest:
-            csv_path = fixtures_dir / a["csv"]
-            recs = _parse_dms(csv_path.read_text(encoding="utf-8"))
-            if recs:
-                assays.append((a["name"], recs))
+            name = a.get("name", a.get("protein", "?"))
+            try:
+                csv_path = fixtures_dir / a["csv"]
+                recs = _parse_dms(csv_path.read_text(encoding="utf-8"))
+                if recs:
+                    assays.append((name, recs))
+                else:
+                    failed.append({"name": name, "error": "no parseable records in csv"})
+            except Exception as exc:  # noqa: BLE001 — one bad assay must not kill the seed
+                logger.exception("seed-commons assay failed: %s", name)
+                failed.append({"name": name, "error": f"{type(exc).__name__}: {exc}"})
 
         try:
             rows = _seed_rows(assays)   # enforces the effective-date gate
@@ -2854,8 +2868,11 @@ def create_app() -> Flask:
 
         result = _auth.replace_mutation_priors(rows)
         _GLOBAL_PRIOR_CACHE["data"] = None   # bust so the next design sees it immediately
-        return jsonify({"ok": bool(result.get("ok")), "substitutions": len(rows),
-                        "contributing_assays": len(assays)})
+        out = {"ok": bool(result.get("ok")), "substitutions": len(rows),
+               "contributing_assays": len(assays)}
+        if failed:
+            out["failed"] = failed
+        return jsonify(out)
 
     @app.post("/api/admin/rebuild-priors")
     def admin_rebuild_priors() -> Response:

tests/test_admin_benchmarks_seed.py

@@ -117,6 +117,44 @@ def test_run_benchmarks_missing_fixtures_404(client, monkeypatch, tmp_path):
     assert r.status_code == 404
 
 
+def test_run_benchmarks_surfaces_per_assay_failure(client, monkeypatch, tmp_path):
+    """One assay's scoring blows up (e.g. a real ESM error on the Space) — the
+    other two must still complete, and the failure must be visible in the
+    response itself (name + error), not just swallowed into a shorter
+    'datasets' list with no explanation. This is what a real diagnosis of
+    'why did only 1 of 3 datasets come back' should read directly off the
+    curl output instead of requiring server-log access."""
+    _with_admin(monkeypatch)
+    real_fixtures = _fixtures_dir()
+    manifest = json.loads((real_fixtures / "manifest.json").read_text(encoding="utf-8"))
+    assert len(manifest) >= 3
+
+    def flaky_score_guarded(scorer, seq):
+        if len(seq) > 500:   # the two longer real fixtures (PABP 577aa, DLG4 724aa)
+            raise RuntimeError("boom: simulated real scoring failure")
+        return pd.DataFrame([{"position": 0, "wt_aa": seq[0], "mut_aa": "Z", "delta_ll": 0.1}])
+
+    monkeypatch.setattr(server._scoring, "get_scorer", lambda *a, **kw: "the-scorer")
+    monkeypatch.setattr(server._scoring, "score_guarded", flaky_score_guarded)
+    monkeypatch.setattr(server, "__file__", str(tmp_path / "server.py"))
+    (tmp_path / "data").mkdir()
+    import shutil
+    shutil.copytree(real_fixtures, tmp_path / "data" / "dms_fixtures")
+
+    r = client.post("/api/admin/run-benchmarks",
+                    headers={"X-Admin-Token": ADMIN_TOKEN}, json={"model": "small"})
+    assert r.status_code == 200
+    body = r.get_json()
+    long_assays = [a["name"] for a in manifest if len(a["sequence"]) > 500]
+    assert len(long_assays) == 2, "expected exactly PABP + DLG4 to be >500aa"
+    assert body["summary"]["n_datasets"] == len(manifest) - len(long_assays)
+    assert "failed" in body and len(body["failed"]) == len(long_assays)
+    failed_names = {f["name"] for f in body["failed"]}
+    assert failed_names == set(long_assays)
+    for f in body["failed"]:
+        assert "boom: simulated real scoring failure" in f["error"]
+
+
 # --------------------------------------------------------------------------- #
 # /api/admin/seed-commons — real bundled fixture, mocked Supabase write
 # --------------------------------------------------------------------------- #
@@ -147,6 +185,30 @@ def test_seed_commons_against_real_fixture(client, monkeypatch):
         assert ">" in row["substitution"]
 
 
+def test_seed_commons_survives_one_bad_csv(client, monkeypatch, tmp_path):
+    """A single unreadable/corrupt fixture CSV must not 500 the whole request
+    (there was previously no try/except around this loop at all) — it should
+    degrade to the other assays and report the failure by name."""
+    _with_admin(monkeypatch)
+    real_fixtures = _fixtures_dir()
+    monkeypatch.setattr(server._auth, "replace_mutation_priors", lambda rows: {"ok": True})
+    monkeypatch.setattr(server, "__file__", str(tmp_path / "server.py"))
+    (tmp_path / "data").mkdir()
+    import shutil
+    dest = tmp_path / "data" / "dms_fixtures"
+    shutil.copytree(real_fixtures, dest)
+    manifest = json.loads((dest / "manifest.json").read_text(encoding="utf-8"))
+    broken_name = manifest[0]["name"]
+    (dest / manifest[0]["csv"]).unlink()   # simulate a missing/corrupt fixture file
+
+    r = client.post("/api/admin/seed-commons", headers={"X-Admin-Token": ADMIN_TOKEN})
+    assert r.status_code == 200
+    body = r.get_json()
+    assert body["contributing_assays"] == len(manifest) - 1
+    assert "failed" in body and len(body["failed"]) == 1
+    assert body["failed"][0]["name"] == broken_name
+
+
 def test_seed_commons_gated_before_effective_date(client, monkeypatch):
     _with_admin(monkeypatch)
     import datetime as _dt