Add eval report generator + training table persistence + group-col override

Adds the morning-readable end product:
- src/microbe_model/eval.py — render_report() generates artifacts/eval_report.md
with per-target metrics vs always-predict-mean baseline, fold-by-fold detail,
top features, and a limitations + next-steps section
- scripts/04_eval.py — thin wrapper to render the report from a finished training run
- scripts/03_train_baseline.py — saves the merged training_table.parquet for eval
to read, prefers BacDive's LPSN family over derived genus for GroupKFold
- src/microbe_model/train/baseline.py — train_all gains group_col_override

Each metric in the report is paired with a dumb-baseline (always-predict-mean
or always-predict-majority) so the reader can interpret it without context.

Tests still 12/12 passing, lint clean.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

scripts/03_train_baseline.py

@@ -1,38 +1,54 @@
 """Train the multi-task XGBoost baseline.
 
-Joins phenotypes + features, derives a `family` column from `species` for group K-fold,
-and writes per-target metrics to artifacts/baseline_results.json.
+Joins phenotypes + features, derives a stable group column for GroupKFold, trains, saves
+the merged training table for the eval renderer, and writes per-target metrics.
 """
 from __future__ import annotations
 
+import time
+
 import pandas as pd
 
 from microbe_model import config
 from microbe_model.train.baseline import save_results, train_all
 
 
-def derive_family(species: str | None) -> str:
-    """Crude family proxy: first word of binomial. Replace with GTDB lookup later."""
-    if not species:
-        return "__unknown__"
-    return str(species).split()[0]
+def derive_group(row: pd.Series) -> str:
+    """Group-K-fold key. Prefer LPSN family (from BacDive); fall back to genus then species."""
+    for col in ("family", "genus"):
+        val = row.get(col)
+        if isinstance(val, str) and val:
+            return val
+    species = row.get("species")
+    if isinstance(species, str) and species:
+        return species.split()[0]
+    return "__unknown__"
 
 
 def main() -> None:
+    t0 = time.time()
     pheno = pd.read_parquet(config.DATA / "bacdive_phenotypes.parquet")
     feats = pd.read_parquet(config.DATA / "features.parquet")
     df = pheno.merge(feats, on=["bacdive_id", "genome_accession"], how="inner")
-    df["family"] = df["species"].apply(derive_family)
+    df["group"] = df.apply(derive_group, axis=1)
 
     feature_cols = [c for c in feats.columns if c not in {"bacdive_id", "genome_accession"}]
-    print(f"Training on {len(df)} strains × {len(feature_cols)} features.")
-    print(f"Group counts (top 10): {df['family'].value_counts().head(10).to_dict()}")
 
-    results = train_all(df, feature_cols)
+    print(f"Training table: {len(df):,} strains × {len(feature_cols)} features")
+    print(f"Distinct groups: {df['group'].nunique():,}")
+    print(f"Group sizes (top 10): {df['group'].value_counts().head(10).to_dict()}")
+    print()
+
+    training_table = config.DATA / "training_table.parquet"
+    df.to_parquet(training_table, index=False)
+    print(f"Wrote training table to {training_table}")
+
+    results = train_all(df, feature_cols, group_col_override="group")
 
     out = config.ARTIFACTS / "baseline_results.json"
     save_results(results, out)
-    print(f"\nWrote results to {out}\n")
+
+    print(f"\nResults summary ({time.time() - t0:.1f}s):\n")
     for target, r in results.items():
         if r.folds:
             metric = r.folds[0].metric_name

scripts/04_eval.py

@@ -0,0 +1,23 @@
+"""Render the v0 eval report from the trained-results JSON + training table."""
+from __future__ import annotations
+
+from microbe_model import config
+from microbe_model.eval import render_report
+
+
+def main() -> None:
+    results_path = config.ARTIFACTS / "baseline_results.json"
+    dataset_path = config.DATA / "training_table.parquet"
+    out_path = config.ARTIFACTS / "eval_report.md"
+
+    if not results_path.exists():
+        raise SystemExit(f"Missing {results_path}. Run scripts/03_train_baseline.py first.")
+    if not dataset_path.exists():
+        raise SystemExit(f"Missing {dataset_path}. Run scripts/03_train_baseline.py first.")
+
+    render_report(results_path, dataset_path, out_path)
+    print(f"Wrote {out_path}")
+
+
+if __name__ == "__main__":
+    main()

src/microbe_model/eval.py

@@ -0,0 +1,156 @@
+"""Evaluation report generation.
+
+Renders a markdown report from a trained-results JSON (the output of train/baseline.py)
+joined with the source dataset. Designed to be readable cold — every number includes
+a comparison baseline so the reader can interpret it without context.
+"""
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+from microbe_model import config
+
+
+def _baseline_mae(y: np.ndarray) -> float:
+    """MAE of the always-predict-mean baseline (sanity floor)."""
+    if len(y) == 0:
+        return float("nan")
+    return float(np.mean(np.abs(y - np.mean(y))))
+
+
+def _baseline_f1(y: np.ndarray) -> float:
+    """Macro-F1 of the always-predict-majority baseline."""
+    from sklearn.metrics import f1_score
+    if len(y) == 0:
+        return float("nan")
+    values, counts = np.unique(y, return_counts=True)
+    majority = values[np.argmax(counts)]
+    pred = np.full_like(y, majority)
+    return float(f1_score(y, pred, average="macro"))
+
+
+def render_report(
+    results_path: Path,
+    dataset_path: Path,
+    out_path: Path,
+    *,
+    n_strains: int | None = None,
+    runtime_seconds: float | None = None,
+) -> None:
+    results: dict[str, Any] = json.loads(results_path.read_text())
+    df = pd.read_parquet(dataset_path)
+
+    lines: list[str] = []
+    lines.append("# microbe-model — v0 baseline eval report")
+    lines.append("")
+    lines.append(f"_Generated: {datetime.now(UTC).isoformat(timespec='seconds')}_")
+    lines.append("")
+
+    # Section: corpus
+    lines.append("## Corpus")
+    lines.append("")
+    lines.append(f"- Total strains in feature table: **{len(df):,}**")
+    if n_strains is not None:
+        lines.append(f"- Total strains attempted (had genome accession + label): {n_strains:,}")
+        lines.append(f"- Feature-extraction success rate: {100 * len(df) / max(1, n_strains):.1f}%")
+    if runtime_seconds is not None:
+        lines.append(f"- Featurize wall time: {runtime_seconds / 60:.1f} min")
+    lines.append("")
+
+    # Section: per-target results
+    lines.append("## Per-target results (5-fold GroupKFold by family)")
+    lines.append("")
+    lines.append("Metrics: regression = MAE (lower is better), classification = macro-F1 (higher is better).")
+    lines.append("Each is shown alongside the dumb-baseline (always-predict-mean / always-predict-majority).")
+    lines.append("")
+    lines.append("| Target | Task | n labeled | Model metric | Baseline | Improvement |")
+    lines.append("|---|---|---|---|---|---|")
+    for target, r in results.items():
+        if not r["folds"]:
+            lines.append(f"| {target} | {r['task']} | — | _skipped (insufficient data)_ | — | — |")
+            continue
+        y = df[target].dropna().to_numpy()
+        n_labeled = len(y)
+        if r["task"] == "regression":
+            baseline = _baseline_mae(y.astype(float))
+            mean = r["mean_metric"]
+            improvement = f"{(baseline - mean) / baseline * 100:+.1f}%"
+            lines.append(f"| `{target}` | regression | {n_labeled:,} | "
+                         f"MAE={mean:.3f} | MAE={baseline:.3f} | {improvement} |")
+        else:
+            baseline = _baseline_f1(y)
+            mean = r["mean_metric"]
+            improvement = f"{(mean - baseline) / max(0.01, baseline) * 100:+.1f}%"
+            lines.append(f"| `{target}` | classification | {n_labeled:,} | "
+                         f"F1={mean:.3f} | F1={baseline:.3f} | {improvement} |")
+    lines.append("")
+
+    # Section: per-fold detail
+    for target, r in results.items():
+        if not r["folds"]:
+            continue
+        lines.append(f"### `{target}` — fold-by-fold")
+        lines.append("")
+        lines.append("| Fold | Metric | Train | Test |")
+        lines.append("|---|---|---|---|")
+        for i, f in enumerate(r["folds"]):
+            lines.append(f"| {i+1} | {f['metric_name']} = {f['value']:.3f} | "
+                         f"n={f['n_train']:,} | n={f['n_test']:,} |")
+        lines.append("")
+
+        top = r.get("top_features", {})
+        if top:
+            lines.append(f"**Top 10 features for `{target}`:**")
+            lines.append("")
+            for name, importance in list(top.items())[:10]:
+                lines.append(f"- `{name}` — {importance:.4f}")
+            lines.append("")
+
+    # Section: limitations
+    lines.append("## Known limitations")
+    lines.append("")
+    lines.append("- **Survivorship bias.** BacDive only contains organisms that have been cultured "
+                 "successfully at least once. The model cannot generalize to truly uncultured strains "
+                 "without explicit out-of-distribution evaluation.")
+    lines.append("- **Optimum derivation is heuristic.** Most BacDive temperature entries are tagged "
+                 "as `growth` (positive growth at this temperature), not `optimum`. We approximate "
+                 "the optimum as the median of positive-growth temperatures when no explicit "
+                 "optimum is recorded — this can be off by 5°C or more for some strains.")
+    lines.append("- **Family grouping is naive.** The current `family` column is derived from the "
+                 "genus (first word of binomial name). A proper LPSN/GTDB family assignment would "
+                 "give tighter taxonomic grouping.")
+    lines.append("- **Feature set is shallow.** No HMM/KEGG annotations, no codon usage indices, no "
+                 "tRNA counts. These are interpretable next steps before moving to genome LMs.")
+    lines.append("- **Pyrodigal accuracy.** Gene prediction quality drops on highly-fragmented "
+                 "assemblies and atypical genetic codes. Not currently flagged in the feature set.")
+    lines.append("")
+
+    # Section: next steps
+    lines.append("## Next steps")
+    lines.append("")
+    lines.append("1. **Add tetranucleotide / codon-usage features.** ~50 extra columns, "
+                 "well-known signal for thermophily.")
+    lines.append("2. **Replace naive family lookup with LPSN/GTDB join.** Reduces leakage in CV.")
+    lines.append("3. **Integrate KOMODO media DB** as a richer label source than BacDive alone.")
+    lines.append("4. **Move to genome embeddings** (Nucleotide Transformer / Evo-1 / DNABERT-2) "
+                 "once the tabular ceiling is established.")
+    lines.append("5. **Active learning loop**: select novel-family strains where the model is "
+                 "uncertain, prioritize these for wet-lab cultivation testing.")
+    lines.append("")
+
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    out_path.write_text("\n".join(lines))
+
+
+if __name__ == "__main__":
+    render_report(
+        results_path=config.ARTIFACTS / "baseline_results.json",
+        dataset_path=config.DATA / "training_table.parquet",
+        out_path=config.ARTIFACTS / "eval_report.md",
+    )

src/microbe_model/train/baseline.py

@@ -120,12 +120,18 @@ def train_target(
     return result
 
 
-def train_all(df: pd.DataFrame, feature_cols: list[str]) -> dict[str, TargetResult]:
+def train_all(
+    df: pd.DataFrame,
+    feature_cols: list[str],
+    *,
+    group_col_override: str | None = None,
+) -> dict[str, TargetResult]:
     results: dict[str, TargetResult] = {}
+    group_col = group_col_override or "family"
     for target, task in config.PHENOTYPE_TARGETS.items():
         if target not in df.columns:
             continue
-        results[target] = train_target(df, target, task, feature_cols)
+        results[target] = train_target(df, target, task, feature_cols, group_col=group_col)
     return results