Receipts + seed: validation harness, DMS→commons seeding, evidence UI

Turns the glass-box thesis from claims into proof — and gives the empty
commons a path to real data. Honest by construction: no benchmark number is
ever hand-entered; the app serves only what a real run produced, and ships an
explicit "pending" state until then (this sandbox has no torch/ESM, so the
numbers get made on a box with the weights).

- dee/core/benchmark.py — validation harness: average-rank Spearman ρ +
top-decile precision ("of your top picks, how many are genuinely high-
fitness?") + cross-dataset medians. Pure numpy, predictions injected →
unit-testable without ESM.
- dee/core/dms_seed.py — turns published DMS assays into de-identified
substitution-prior rows via the SAME k-anonymized aggregate machinery user
data uses (each assay = one contributor; a substitution needs ≥ MIN_USERS
independent studies to survive). ESM-free, so it can seed the commons with
real, citable data on day one; user results refine it after.
- scripts/run_benchmarks.py + scripts/seed_commons_from_dms.py — the
reproducible runners (the honesty artifacts: exactly how numbers/seed get
made) against ProteinGym-format DMS.
- /api/benchmarks — public read of dee/data/benchmarks.json (ships empty).
- Field Atlas modal gains a "Validated on public DMS" evidence strip
(median ρ + top-pick precision + per-dataset rows), honest pending state —
the receipts paired with the commons grid.

17 new tests (Spearman/ties/top-decile math, seed k-anonymity + ProteinGym
CSV parsing, /api/benchmarks public+empty). 451 green. Evidence strip verified
in the preview: honest pending state on the live empty endpoint, and the
populated render (headline ρ + per-dataset rows above the atlas grid). No
console errors; classic UI untouched.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

dee/core/benchmark.py

@@ -0,0 +1,128 @@
+"""Validation harness — does the engine actually predict? Receipts, not claims.
+
+The glass-box thesis lives or dies on this: every score we surface is framed
+as a prediction to verify, so we owe the user proof the predictions track
+reality. This harness correlates the engine's zero-shot ESM-2 ΔLL ranking
+against measured deep-mutational-scanning (DMS) fitness from published studies
+(e.g. ProteinGym) and reports the two numbers a protein engineer actually
+cares about:
+
+  * Spearman ρ — does the ranking order match measured fitness order?
+  * top-decile precision — of the variants we rank in the top 10%, what
+    fraction are genuinely high-fitness (top quartile measured)? i.e. "if I
+    only make the picks you put on top, how often are they real?"
+
+Pure numpy, predictions injected — so it's unit-testable without ESM. The
+reproducible CLI (scripts/run_benchmarks.py) runs it against real DMS with the
+live model where the weights exist; the app only ever serves results a real
+run produced. No number is fabricated here.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Sequence
+
+import numpy as np
+
+
+def _average_ranks(x: np.ndarray) -> np.ndarray:
+    """Ranks with ties resolved to the average rank (proper Spearman ties)."""
+    order = np.argsort(x, kind="mergesort")
+    ranks = np.empty(len(x), dtype=np.float64)
+    sx = x[order]
+    i = 0
+    n = len(x)
+    while i < n:
+        j = i
+        while j + 1 < n and sx[j + 1] == sx[i]:
+            j += 1
+        avg = (i + j) / 2.0 + 1.0          # 1-indexed average rank over the tie block
+        ranks[order[i:j + 1]] = avg
+        i = j + 1
+    return ranks
+
+
+def spearman(a: Sequence[float], b: Sequence[float]) -> Optional[float]:
+    """Spearman rank correlation. None if <3 points or no variance."""
+    a = np.asarray(a, dtype=np.float64)
+    b = np.asarray(b, dtype=np.float64)
+    mask = np.isfinite(a) & np.isfinite(b)
+    a, b = a[mask], b[mask]
+    if len(a) < 3:
+        return None
+    ra, rb = _average_ranks(a), _average_ranks(b)
+    if ra.std() < 1e-9 or rb.std() < 1e-9:
+        return None
+    return float(np.corrcoef(ra, rb)[0, 1])
+
+
+def top_decile_precision(
+    predicted: Sequence[float], measured: Sequence[float],
+    *, pred_frac: float = 0.10, true_frac: float = 0.25,
+) -> Optional[float]:
+    """Of the top ``pred_frac`` by prediction, the fraction that land in the
+    top ``true_frac`` by measured fitness. The 'are your top picks real?' number.
+    None if too few points to form a meaningful top set."""
+    p = np.asarray(predicted, dtype=np.float64)
+    m = np.asarray(measured, dtype=np.float64)
+    mask = np.isfinite(p) & np.isfinite(m)
+    p, m = p[mask], m[mask]
+    n = len(p)
+    k = int(round(n * pred_frac))
+    if n < 10 or k < 1:
+        return None
+    top_pred_idx = np.argsort(-p)[:k]
+    true_cut = np.quantile(m, 1.0 - true_frac)
+    hits = int(np.sum(m[top_pred_idx] >= true_cut))
+    return float(hits / k)
+
+
+@dataclass
+class DatasetResult:
+    """One DMS assay's validation result."""
+    name: str
+    protein: str            # e.g. UniProt/DMS id
+    n: int                  # variants scored
+    spearman: Optional[float]
+    top_decile_precision: Optional[float]
+    source: str = ""        # citation / DOI / dataset id (provenance)
+
+    def as_dict(self) -> dict:
+        return {
+            "name": self.name, "protein": self.protein, "n": self.n,
+            "spearman": None if self.spearman is None else round(self.spearman, 4),
+            "top_decile_precision": None if self.top_decile_precision is None
+                else round(self.top_decile_precision, 4),
+            "source": self.source,
+        }
+
+
+def evaluate_dataset(
+    name: str, protein: str,
+    predicted: Sequence[float], measured: Sequence[float],
+    *, source: str = "",
+) -> DatasetResult:
+    """Score one aligned (predicted, measured) DMS assay."""
+    p = np.asarray(predicted, dtype=np.float64)
+    m = np.asarray(measured, dtype=np.float64)
+    mask = np.isfinite(p) & np.isfinite(m)
+    return DatasetResult(
+        name=name, protein=protein, n=int(mask.sum()),
+        spearman=spearman(p, m),
+        top_decile_precision=top_decile_precision(p, m),
+        source=source,
+    )
+
+
+def summarize(results: Sequence[DatasetResult]) -> dict:
+    """Headline across datasets: median Spearman, median top-decile precision,
+    dataset + variant counts. Medians (robust to a couple of hard assays)."""
+    rhos = [r.spearman for r in results if r.spearman is not None]
+    precs = [r.top_decile_precision for r in results if r.top_decile_precision is not None]
+    return {
+        "n_datasets": len(results),
+        "n_variants": int(sum(r.n for r in results)),
+        "median_spearman": None if not rhos else round(float(np.median(rhos)), 4),
+        "median_top_decile_precision": None if not precs
+            else round(float(np.median(precs)), 4),
+    }

dee/core/dms_seed.py

@@ -0,0 +1,86 @@
+"""Seed the commons (public.mutation_priors) from published DMS studies.
+
+The Field Atlas / cross-user prior pools MEASURED substitution effects across
+labs (dee.core.aggregate). Public deep-mutational-scanning (DMS) datasets are
+exactly that — real, measured effects from independent published assays. So we
+can treat each assay as one contributor and pre-populate the commons with real,
+citable data: the atlas has value on day one, and every user result afterward
+refines it.
+
+ESM-free — this only aggregates measured values, no model. It reuses the SAME
+de-identification as user data (dee.core.aggregate.build_priors): z-score
+within each assay, pool by substitution TYPE only, and keep a substitution
+only when ≥ MIN_USERS *independent assays* measured it. So the seed carries no
+per-variant, per-position, or per-study raw value — only 'the field finds W>L
+favorable', backed by ≥3 studies. That k-anonymity bar makes the seed both
+private-by-construction and defensibly strong.
+"""
+from __future__ import annotations
+
+import datetime as _dt
+from typing import List, Optional, Sequence, Tuple
+
+from dee.core import aggregate as _agg
+
+
+def _normalize_label(label: str) -> str:
+    """ProteinGym multi-mutants join with ':' ('A1C:D5E'); aggregate splits on
+    ',/;/whitespace'. Normalize so multi-mutant assays still contribute (the
+    aggregate ridge decomposes them into single-site effects)."""
+    return str(label).strip().replace(":", ",")
+
+
+def seed_rows(
+    assays: Sequence[Tuple[str, Sequence[Tuple[str, object]]]],
+    *,
+    now: Optional[_dt.date] = None,
+    enforce_gate: bool = True,
+) -> List[dict]:
+    """Turn published DMS assays into de-identified substitution-prior rows.
+
+    ``assays``: list of ``(assay_id, [(mutant_label, score), ...])`` — one entry
+    per independent published assay. Returns rows shaped for
+    ``auth.replace_mutation_priors`` ({substitution, n_users, n_obs,
+    mean_effect}); ``n_users`` here is the number of independent assays, so the
+    ≥ MIN_USERS k-anonymity floor means ≥ that many studies agree.
+    """
+    grouped: List[Tuple[str, List[Tuple[str, float]]]] = []
+    for assay_id, records in assays:
+        meas: List[Tuple[str, float]] = []
+        for label, score in records:
+            try:
+                v = float(score)
+            except (TypeError, ValueError):
+                continue
+            lab = _normalize_label(label)
+            if lab:
+                meas.append((lab, v))
+        if meas:
+            grouped.append((str(assay_id), meas))
+    prior = _agg.build_priors(grouped, now=now, enforce_gate=enforce_gate)
+    return prior.to_rows()
+
+
+def parse_proteingym_csv(text: str) -> List[Tuple[str, float]]:
+    """Parse a ProteinGym-style substitution CSV (header with a 'mutant' /
+    'mutation' column and a 'DMS_score' / 'score' column) into
+    [(mutant_label, score)]. Tolerant of column-name variants; skips rows it
+    can't read rather than guessing."""
+    import csv
+    import io
+
+    out: List[Tuple[str, float]] = []
+    reader = csv.DictReader(io.StringIO(text))
+    if not reader.fieldnames:
+        return out
+    lower = {f.lower(): f for f in reader.fieldnames}
+    mut_col = next((lower[k] for k in ("mutant", "mutation", "variant", "mutations") if k in lower), None)
+    score_col = next((lower[k] for k in ("dms_score", "score", "fitness", "value", "measured") if k in lower), None)
+    if not mut_col or not score_col:
+        return out
+    for row in reader:
+        try:
+            out.append((str(row[mut_col]).strip(), float(row[score_col])))
+        except (TypeError, ValueError, KeyError):
+            continue
+    return out

dee/data/benchmarks.json

@@ -0,0 +1,12 @@
+{
+  "generated_at": null,
+  "model": null,
+  "note": "Validation results are produced by scripts/run_benchmarks.py against published DMS data with the live ESM-2 model (where the weights exist), never hand-entered. Until a real run populates this file, the app shows an honest 'pending' state — no fabricated numbers.",
+  "summary": {
+    "n_datasets": 0,
+    "n_variants": 0,
+    "median_spearman": null,
+    "median_top_decile_precision": null
+  },
+  "datasets": []
+}

dee/server.py

@@ -2719,6 +2719,26 @@ def create_app() -> Flask:
             "effective_date": _agg.EFFECTIVE_DATE.isoformat(),
         })
 
+    @app.get("/api/benchmarks")
+    def benchmarks() -> Response:
+        """The receipts — how well the engine's zero-shot ranking predicts
+        measured DMS fitness (Spearman ρ + top-decile precision), per published
+        dataset. PUBLIC read. Served from dee/data/benchmarks.json, which is
+        produced ONLY by scripts/run_benchmarks.py against real data with the
+        live model — never hand-entered. Ships empty (honest 'pending' state)
+        until a real run populates it; no fabricated numbers ever reach here."""
+        import json as _json
+        path = Path(__file__).resolve().parent / "data" / "benchmarks.json"
+        try:
+            data = _json.loads(path.read_text(encoding="utf-8"))
+        except Exception:  # noqa: BLE001
+            data = {"generated_at": None, "model": None,
+                    "summary": {"n_datasets": 0, "n_variants": 0,
+                                "median_spearman": None, "median_top_decile_precision": None},
+                    "datasets": []}
+        data["ok"] = True
+        return jsonify(data)
+
     @app.get("/api/admin/mutation-priors")
     def admin_mutation_priors() -> Response:
         """View the stored cross-user aggregate (de-identified). Admin-token gated."""

dee/static/app.css

@@ -6498,6 +6498,27 @@ h3, h4 {
 .atlas-empty { padding: 20px; border: 1px dashed var(--line-strong); text-align: center; }
 .atlas-empty h3 { margin: 0 0 8px; font-size: 16px; font-weight: 600; color: var(--ink-strong); }
 .atlas-empty .muted { max-width: 52ch; margin: 0 auto 16px; line-height: 1.55; }
+
+/* Evidence strip — the receipts (Spearman ρ vs published DMS). */
+.atlas-evidence:empty { display: none; }
+.atlas-evidence-pending { padding: 12px 14px; border: 1px solid var(--line);
+    color: var(--ink-faint); line-height: 1.55; }
+.atlas-evidence-pending b { color: var(--ink); }
+.atlas-ev-head { display: flex; align-items: baseline; flex-wrap: wrap; gap: 6px 14px;
+    padding: 14px; border: 1px solid var(--line-strong); }
+.atlas-ev-k { width: 100%; color: var(--ink-faint); text-transform: uppercase; letter-spacing: 0.14em; }
+.atlas-ev-big { font-family: var(--font-display); font-size: 26px; font-weight: 600;
+    color: var(--ink-strong); line-height: 1; }
+.atlas-ev-lbl { color: var(--ink-faint); margin-right: 8px; }
+.atlas-ev-meta { width: 100%; color: var(--ink-disabled); }
+.atlas-ev-list { display: flex; flex-wrap: wrap; gap: 1px; margin-top: 1px;
+    background: var(--line); border: 1px solid var(--line); }
+.atlas-ev-row { flex: 1 1 180px; display: flex; align-items: baseline; gap: 8px;
+    padding: 8px 12px; background: var(--bg-card); }
+.aer-n { font-size: 12.5px; color: var(--ink); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+.aer-rho { font-family: var(--font-mono); font-size: 12px; color: var(--ink-strong); margin-left: auto; }
+.aer-var { color: var(--ink-faint); }
+
 @media (max-width: 560px) {
     .ag-head { font-size: 7px; }
     .atlas-legend .al-note { margin-left: 0; width: 100%; }

dee/static/app.js

@@ -9077,11 +9077,47 @@ function runOracle(opts){
         fetch('/api/atlas').then(function (r) { return r.json(); }).then(function (j) {
             if (!j || !j.ok) { body.innerHTML = '<p class="muted">The atlas is unavailable right now — try again shortly.</p>'; return; }
             render(j);
+            // Second half of the "receipts" pair: how well the engine predicts.
+            fetch('/api/benchmarks').then(function (r) { return r.json(); })
+                .then(renderEvidence).catch(function () {});
         }).catch(function () {
             body.innerHTML = '<p class="muted">Couldn’t reach the atlas — check your connection.</p>';
         });
     }
 
+    // Validation strip — Spearman ρ + top-decile precision vs published DMS.
+    // Fills the placeholder render() leaves at the top of the modal. Honest
+    // 'pending' state until a real benchmark run populates the numbers.
+    function renderEvidence(b) {
+        var host = el('atlasEvidence'); if (!host) return;
+        var s = (b && b.summary) || {};
+        if (!b || !b.ok || !s.n_datasets) {
+            host.innerHTML = '<div class="atlas-evidence-pending micro">'
+                + '<b>Independent validation.</b> How well the engine&rsquo;s ranking predicts measured '
+                + 'fitness is benchmarked against published deep-mutational-scanning datasets, computed on '
+                + 'the full model — results appear here once a run completes. We never hand-enter numbers.'
+                + '</div>';
+            return;
+        }
+        var rho = s.median_spearman == null ? '—' : s.median_spearman.toFixed(2);
+        var prec = s.median_top_decile_precision == null ? '—'
+            : Math.round(s.median_top_decile_precision * 100) + '%';
+        var head = '<div class="atlas-ev-head">'
+            + '<span class="atlas-ev-k micro">Validated on public DMS</span>'
+            + '<span class="atlas-ev-big">' + rho + '</span><span class="atlas-ev-lbl micro">median Spearman &rho;</span>'
+            + '<span class="atlas-ev-big">' + prec + '</span><span class="atlas-ev-lbl micro">top-pick precision</span>'
+            + '<span class="atlas-ev-meta micro">' + (s.n_datasets) + ' datasets · '
+            + (s.n_variants || 0).toLocaleString() + ' variants'
+            + (b.model ? ' · ESM-2 ' + esc(b.model) : '') + '</span>'
+            + '</div>';
+        var rows = (b.datasets || []).slice(0, 8).map(function (d) {
+            return '<div class="atlas-ev-row"><span class="aer-n">' + esc(d.name) + '</span>'
+                + '<span class="aer-rho">&rho; ' + (d.spearman == null ? '—' : d.spearman.toFixed(2)) + '</span>'
+                + '<span class="aer-var micro">' + (d.n || 0).toLocaleString() + ' var</span></div>';
+        }).join('');
+        host.innerHTML = head + '<div class="atlas-ev-list">' + rows + '</div>';
+    }
+
     // Collapse ESM-bin-split rows ("W>L@hi") back to the base substitution,
     // pooling by observation count so the grid shows one cell per (from,to).
     function collapse(rows) {
@@ -9120,6 +9156,9 @@ function runOracle(opts){
         var cells = collapse(j.substitutions);
         var count = j.count || 0;
 
+        // Evidence strip (filled by renderEvidence once /api/benchmarks returns)
+        // sits at the very top — the receipts before the atlas grid.
+        var ev = '<div class="atlas-evidence" id="atlasEvidence"></div>';
         var stats = '<div class="atlas-stats micro">'
             + '<span><b>' + count + '</b> substitution type' + (count === 1 ? '' : 's') + '</span>'
             + '<span><b>' + (j.total_observations || 0).toLocaleString() + '</b> pooled measurements</span>'
@@ -9127,7 +9166,7 @@ function runOracle(opts){
             + '</div>';
 
         if (!count) {
-            body.innerHTML = stats
+            body.innerHTML = ev + stats
                 + '<div class="atlas-empty">'
                 + '<h3>The commons is still filling in.</h3>'
                 + '<p class="muted">A substitution appears here only once at least <b>' + (j.min_users || 3)
@@ -9143,7 +9182,7 @@ function runOracle(opts){
         }
 
         // Grid: corner + 20 "to" headers, then 20 rows (from-header + 20 cells).
-        var html = stats
+        var html = ev + stats
             + '<div class="atlas-legend micro"><span class="al-swatch al-pos"></span> field found favorable'
             + '<span class="al-swatch al-neg"></span> unfavorable'
             + '<span class="al-note">rows = original residue · columns = substituted residue</span></div>';

dee/static/index.html

@@ -112,7 +112,7 @@
     <!-- ?v= query bumps invalidate browser + iframe asset caches when app.css /
          app.js change. Bump these numbers whenever you ship a frontend update —
          without them, users keep getting the stale file for up to a week. -->
-    <link rel="stylesheet" href="/static/app.css?v=20260714-glassbox" />
+    <link rel="stylesheet" href="/static/app.css?v=20260715-evidence" />
     <link rel="icon" type="image/svg+xml" href="/static/favicon.svg?v=2" />
     <link rel="apple-touch-icon" href="/static/favicon.svg?v=2" />
     <!-- Mol* (PDBe) 3-D viewer is ~4.9 MB. We do NOT eager-load it on every
@@ -2259,7 +2259,7 @@
     <!-- Cloning reference data must load before app.js so the Designer
          can read VECTORS / ENZYMES / CLONING_METHODS / TAGS / LINKERS. -->
     <script src="/static/cloning_db.js?v=20260530-ui-polish" defer></script>
-    <script src="/static/app.js?v=20260714-glassbox" defer></script>
+    <script src="/static/app.js?v=20260715-evidence" defer></script>
     <!-- Dwell-time heartbeat. Loads after auth.js so its /api/ping calls go
          through the JWT-attaching fetch wrapper (signed-in attribution). -->
     <script src="/static/telemetry.js?v=20260622-analytics" defer></script>

scripts/run_benchmarks.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+"""Produce the REAL validation numbers — the receipts behind the glass-box claim.
+
+Runs the engine's zero-shot ESM-2 ranking against published deep-mutational-
+scanning (DMS) datasets and writes dee/data/benchmarks.json (Spearman ρ +
+top-decile precision per dataset). Run this where torch + the ESM weights
+exist (the HF Space, or any box with the model); it is the ONLY thing that
+should ever populate benchmarks.json — the app never hand-enters numbers.
+
+Input: a manifest JSON, a list of assays:
+    [
+      {"name": "...", "protein": "GENE_ORG", "sequence": "MSK...",
+       "csv": "path/to/dms.csv", "source": "doi:..."},
+      ...
+    ]
+The CSV is ProteinGym-style (a 'mutant'/'mutation' column, a 'DMS_score'/
+'score' column). Multi-mutants ('A1C:D5E') are scored as the sum of their
+single-site ΔLLs (the same additive assumption the design engine makes — so
+this validates exactly what we ship).
+
+Usage:
+    python scripts/run_benchmarks.py manifest.json [--model small] [--out dee/data/benchmarks.json]
+"""
+import argparse
+import datetime as dt
+import json
+import sys
+from pathlib import Path
+
+# Make `dee` importable when run from the repo root.
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from dee.core import benchmark as bm          # noqa: E402
+from dee.core.dms_seed import parse_proteingym_csv  # noqa: E402
+
+
+def _predict(sequence, labels, scorer):
+    """Sum-of-single-site ESM-2 ΔLL for each (possibly multi-site) label."""
+    df = scorer.score_all_substitutions(sequence)
+    lut = {(int(r.position), str(r.mut_aa)): float(r.delta_ll)
+           for r in df.itertuples(index=False)}
+    import re
+    rx = re.compile(r"^([A-Za-z])(\d+)([A-Za-z*])$")
+    preds = []
+    for lab in labels:
+        total, ok = 0.0, True
+        for tok in str(lab).replace(":", ",").split(","):
+            m = rx.match(tok.strip())
+            if not m:
+                ok = False
+                break
+            pos, mut = int(m.group(2)) - 1, m.group(3).upper()
+            if (pos, mut) not in lut:
+                ok = False
+                break
+            total += lut[(pos, mut)]
+        preds.append(total if ok else float("nan"))
+    return preds
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("manifest")
+    ap.add_argument("--model", default="small")
+    ap.add_argument("--out", default=str(Path(__file__).resolve().parent.parent / "dee" / "data" / "benchmarks.json"))
+    args = ap.parse_args()
+
+    from dee.core import scoring
+    scorer = scoring.get_scorer(args.model)
+
+    assays = json.loads(Path(args.manifest).read_text(encoding="utf-8"))
+    results = []
+    for a in assays:
+        recs = parse_proteingym_csv(Path(a["csv"]).read_text(encoding="utf-8"))
+        if not recs:
+            print(f"  skip {a.get('name')}: no parseable records")
+            continue
+        labels = [lab for (lab, _v) in recs]
+        measured = [v for (_lab, v) in recs]
+        predicted = _predict(a["sequence"], labels, scorer)
+        r = bm.evaluate_dataset(a.get("name", a.get("protein", "?")),
+                                a.get("protein", ""), predicted, measured,
+                                source=a.get("source", ""))
+        results.append(r)
+        print(f"  {r.name:24s} n={r.n:6d}  rho={r.spearman}  top10p={r.top_decile_precision}")
+
+    out = {
+        "generated_at": dt.datetime.utcnow().isoformat() + "Z",
+        "model": args.model,
+        "summary": bm.summarize(results),
+        "datasets": [r.as_dict() for r in results],
+    }
+    Path(args.out).write_text(json.dumps(out, indent=2), encoding="utf-8")
+    print(f"\nWrote {len(results)} dataset result(s) -> {args.out}")
+    print(f"Summary: {out['summary']}")
+
+
+if __name__ == "__main__":
+    main()

scripts/seed_commons_from_dms.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""Seed the Field Atlas / commons from published DMS studies (ESM-free).
+
+Turns a set of public DMS assays into de-identified substitution-effect rows
+(dee.core.dms_seed → the same k-anonymized aggregation user data goes through)
+so the commons has real, citable value on day one. No model needed — this only
+pools measured values by substitution TYPE, keeping a substitution only when
+≥ MIN_USERS independent studies measured it.
+
+Input: a manifest JSON, a list of assays:
+    [ {"name": "...", "csv": "path/to/dms.csv"}, ... ]
+(ProteinGym-style CSVs: a 'mutant' column + a 'DMS_score' column.)
+
+By default it WRITES the rows to a JSON file for review. Pass --push to upload
+them to public.mutation_priors via dee.auth (requires SUPABASE creds in the
+environment — run it where the service key lives, e.g. the deploy box).
+
+Usage:
+    python scripts/seed_commons_from_dms.py manifest.json [--out seed_rows.json] [--push]
+"""
+import argparse
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from dee.core.dms_seed import parse_proteingym_csv, seed_rows  # noqa: E402
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("manifest")
+    ap.add_argument("--out", default="seed_rows.json")
+    ap.add_argument("--push", action="store_true",
+                    help="upload to public.mutation_priors (needs SUPABASE env)")
+    args = ap.parse_args()
+
+    assays = []
+    for a in json.loads(Path(args.manifest).read_text(encoding="utf-8")):
+        recs = parse_proteingym_csv(Path(a["csv"]).read_text(encoding="utf-8"))
+        if recs:
+            assays.append((a.get("name", a["csv"]), recs))
+        print(f"  {a.get('name', a['csv']):28s} {len(recs):7d} records")
+
+    rows = seed_rows(assays)   # enforces the effective-date gate + k-anonymity
+    Path(args.out).write_text(json.dumps(rows, indent=2), encoding="utf-8")
+    print(f"\n{len(rows)} de-identified substitution row(s) (>= MIN_USERS studies each) -> {args.out}")
+
+    if args.push:
+        from dee import auth
+        result = auth.replace_mutation_priors(rows)
+        print(f"push -> {result}")
+    else:
+        print("(dry run — pass --push to upload to the live commons)")
+
+
+if __name__ == "__main__":
+    main()

tests/test_benchmark.py

@@ -0,0 +1,93 @@
+"""Tests for the validation harness (dee.core.benchmark) — the receipts engine.
+
+Predictions are injected, so we can assert the exact correlation / precision
+behaviour without ESM. These pin the numbers we'd publish as our benchmark.
+"""
+import numpy as np
+import pytest
+
+from dee.core.benchmark import (
+    DatasetResult,
+    evaluate_dataset,
+    spearman,
+    summarize,
+    top_decile_precision,
+)
+
+
+def test_spearman_perfect_and_anti():
+    a = list(range(20))
+    assert spearman(a, a) == pytest.approx(1.0)
+    assert spearman(a, list(reversed(a))) == pytest.approx(-1.0)
+
+
+def test_spearman_monotonic_nonlinear_is_one():
+    a = list(range(1, 11))
+    b = [x ** 3 for x in a]            # monotonic → rank corr = 1 even if nonlinear
+    assert spearman(a, b) == pytest.approx(1.0)
+
+
+def test_spearman_handles_ties():
+    a = [1, 1, 2, 2, 3, 3]
+    b = [1, 1, 2, 2, 3, 3]
+    assert spearman(a, b) == pytest.approx(1.0)
+
+
+def test_spearman_none_when_degenerate():
+    assert spearman([1, 2], [3, 4]) is None          # < 3 points
+    assert spearman([5, 5, 5, 5], [1, 2, 3, 4]) is None  # no variance in a
+
+
+def test_top_decile_precision_perfect_alignment():
+    rng = np.random.default_rng(0)
+    measured = rng.normal(size=100)
+    predicted = measured.copy()                       # perfect ranking
+    # top 10% predicted are exactly the top 10% measured, all within top 25%.
+    assert top_decile_precision(predicted, measured) == pytest.approx(1.0)
+
+
+def test_top_decile_precision_anti_alignment_is_low():
+    measured = np.linspace(0, 1, 100)
+    predicted = -measured                             # worst possible ranking
+    assert top_decile_precision(predicted, measured) == pytest.approx(0.0)
+
+
+def test_top_decile_precision_none_when_too_small():
+    assert top_decile_precision([1, 2, 3], [3, 2, 1]) is None
+
+
+def test_evaluate_dataset_and_summarize():
+    rng = np.random.default_rng(1)
+    m1 = rng.normal(size=60)
+    p1 = m1 + rng.normal(scale=0.3, size=60)          # good but noisy predictor
+    m2 = rng.normal(size=40)
+    p2 = -m2                                            # a hard/anti assay
+    r1 = evaluate_dataset("assayA", "P1", p1, m1, source="doi:1")
+    r2 = evaluate_dataset("assayB", "P2", p2, m2, source="doi:2")
+    assert r1.spearman > 0.6
+    assert r2.spearman < 0
+    s = summarize([r1, r2])
+    assert s["n_datasets"] == 2
+    assert s["n_variants"] == 100
+    assert s["median_spearman"] is not None
+    # as_dict rounds + is JSON-safe
+    d = r1.as_dict()
+    assert set(d) == {"name", "protein", "n", "spearman", "top_decile_precision", "source"}
+
+
+def test_summarize_empty():
+    s = summarize([])
+    assert s == {"n_datasets": 0, "n_variants": 0,
+                 "median_spearman": None, "median_top_decile_precision": None}
+
+
+def test_benchmarks_route_public_and_honest_empty():
+    from dee import server
+    app = server.create_app()
+    app.config.update(TESTING=True)
+    body = app.test_client().get("/api/benchmarks").get_json()   # no auth — public
+    assert body["ok"] is True
+    assert "summary" in body and "datasets" in body
+    # Ships empty (no fabricated numbers) until a real run populates it.
+    assert body["summary"]["n_datasets"] == 0
+    assert body["datasets"] == []

tests/test_dms_seed.py

@@ -0,0 +1,75 @@
+"""Tests for seeding the commons from public DMS (dee.core.dms_seed).
+
+ESM-free: pure aggregation. We construct synthetic 'assays' where a
+substitution's effect is controlled across studies, and check the k-anonymity
+floor (≥ MIN_USERS independent assays) and the de-identified output shape.
+"""
+import pytest
+
+from dee.core import aggregate as _agg
+from dee.core.dms_seed import parse_proteingym_csv, seed_rows
+
+
+def _assay(aid, effects):
+    """Build one assay's [(label, score)] where each single-site 'X{i}Y' gets a
+    base score = its intended effect (plus a couple of WT-ish low rows so the
+    within-assay z-score has spread)."""
+    recs = [(lab, val) for lab, val in effects]
+    return (aid, recs)
+
+
+def test_seed_keeps_substitution_seen_in_enough_assays():
+    # 'W>L' favorable, measured in 3 independent assays → survives k-anon (=3).
+    assays = []
+    for i in range(_agg.MIN_USERS):
+        assays.append(_assay(f"study{i}", [
+            (f"W{10 + i}L", 2.0),     # the favorable W>L
+            (f"K{20 + i}D", -1.0),    # spread so z-score is defined
+            (f"A{30 + i}G", 0.0),
+        ]))
+    rows = seed_rows(assays, enforce_gate=False)
+    subs = {r["substitution"] for r in rows}
+    assert "W>L" in subs
+    wl = next(r for r in rows if r["substitution"] == "W>L")
+    assert wl["n_users"] >= _agg.MIN_USERS     # backed by ≥ 3 studies
+    assert wl["mean_effect"] > 0               # favorable, correct sign
+
+
+def test_seed_drops_substitution_below_k_anon():
+    # 'C>Y' appears in only ONE assay → dropped (privacy floor).
+    assays = [
+        _assay("only_study", [("C5Y", 3.0), ("K6D", -1.0), ("A7G", 0.0)]),
+        _assay("study2", [("K6D", -1.0), ("A7G", 0.5), ("M8I", 0.2)]),
+        _assay("study3", [("K6D", -0.8), ("A7G", 0.3), ("M8I", 0.1)]),
+    ]
+    rows = seed_rows(assays, enforce_gate=False)
+    assert "C>Y" not in {r["substitution"] for r in rows}
+
+
+def test_seed_handles_multi_mutant_colon_labels():
+    # ProteinGym multi-mutants ('A1C:D5E') must not crash and should contribute.
+    assays = [
+        _assay(f"s{i}", [("A1C:D5E", 1.5), ("K2D", -1.0), ("M3I", 0.0)])
+        for i in range(_agg.MIN_USERS)
+    ]
+    rows = seed_rows(assays, enforce_gate=False)
+    assert isinstance(rows, list)   # decomposed by the aggregate ridge, no error
+
+
+def test_seed_empty_when_no_assays():
+    assert seed_rows([], enforce_gate=False) == []
+
+
+def test_parse_proteingym_csv():
+    csv_text = "mutant,DMS_score\nA1C,1.2\nD5E,-0.4\nbad,notanumber\n"
+    recs = parse_proteingym_csv(csv_text)
+    assert recs == [("A1C", 1.2), ("D5E", -0.4)]     # bad row skipped
+
+
+def test_parse_proteingym_csv_alt_columns():
+    csv_text = "mutation,fitness\nW10L,0.9\n"
+    assert parse_proteingym_csv(csv_text) == [("W10L", 0.9)]
+
+
+def test_parse_proteingym_csv_missing_columns():
+    assert parse_proteingym_csv("foo,bar\n1,2\n") == []