scripts/build_summary.py

#!/usr/bin/env python3
"""Write reports/SUMMARY.md from a cross-validation run and reports/final/final_fit.json.

    python scripts/build_summary.py --cv-dir reports/cv_v3 --compare reports/cv_v2/cv_results.json --names v2,v3
"""
from __future__ import annotations

import argparse
import json
import os
import sys

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, ROOT)
from cellquant.labels import ALL_LABELS, CELL_LABELS, NOSTRIL_LABELS  # noqa: E402


def pct(x: float) -> str:
    return f"{100 * x:.1f}%"


def peak_thr(agg: dict) -> float:
    """Shipped min_peak_dist: the final model's threshold from rule_study.json next to cv_results.json, else PostConfig's default."""
    from cellquant.postprocess import PostConfig
    return float(agg.get("final_min_peak_dist", PostConfig().min_peak_dist))


def main(argv=None):
    ap = argparse.ArgumentParser()
    ap.add_argument("--cv-dir", default="runs/cv", help="folder holding cv_results.json of the shipped recipe")
    ap.add_argument("--compare", default="reports/cv/cv_results.json",
                    help="cv_results.json of the previous recipe for the change table ('' = none)")
    ap.add_argument("--names", default="v1,v2", help="names of the previous and the shipped recipe")
    a = ap.parse_args(argv)
    prev_name, cur_name = a.names.split(",")
    cv = json.load(open(os.path.join(ROOT, a.cv_dir, "cv_results.json")))
    prev = json.load(open(os.path.join(ROOT, a.compare)))["aggregate"] if a.compare and os.path.exists(os.path.join(ROOT, a.compare)) else None
    agg = cv["aggregate"]
    per = agg["per_image"]
    rs = os.path.join(ROOT, a.cv_dir, "rule_study.json")
    if os.path.exists(rs):
        fin = json.load(open(rs)).get("final")
        if fin:
            agg["final_min_peak_dist"] = fin["chosen"]["thr"]
    fit = json.load(open(os.path.join(ROOT, "reports", "final", "final_fit.json")))
    tot = agg["dataset_totals"]
    n_nuc = sum(tot[c] for c in CELL_LABELS)
    n_pun = sum(tot[c] for c in NOSTRIL_LABELS)
    det = agg["detection_base"]
    det8 = agg["detection_tta8"]
    L = ["# CellQuant: validation summary", "",
         f"Data: {len(per)} annotated 1024×1024 images, {n_nuc} nucleus outlines and {n_pun} punctum outlines.",
         "Protocol: leave-one-image-out cross-validation (one model per image, 3000 iterations each, held-out image never seen in training), "
         "then a final model trained on all images with the same recipe. Each held-out image was tested 23 times: 3 identical runs "
         "(determinism), no-TTA / 4× / 8× test-time augmentation, 8 rotations and flips, 6 photometric perturbations, and 6 post-processing rule variants. "
         "The only choice made on held-out data is the rule that gives a nucleus the Nostril attribute: of three candidate rules, "
         "'contains an in-nucleus punctum' (the annotation protocol) scored best and is the default; see the rule-variant table in cv_report.md. "
         "The peak-distance threshold of v3 was chosen, for every fold, on that fold's eight training images only (scripts/rule_study.py), "
         "and for the shipped model on its own nine training images; the held-out images never entered that choice.", "",
         "Recipe (v3, shipped): the v2 model (ResNet34-UNet with a full-resolution stem; targets are a 3-px inner border ring (class weight 3), "
         "a per-nucleus normalised distance-to-outline map that the watershed uses as its landscape (markers = distance > 0.5), "
         "an antibody head whose loss up-weights antibody-negative pixels by sqrt(positive / negative pixels) (4.5 on the full set), "
         "and 25 % of training crops centred on an antibody-negative nucleus) plus one post-processing rule: a nucleus whose peak "
         f"predicted distance is below {peak_thr(agg):.2f} is dropped (PostConfig.min_peak_dist). The distance head is trained to reach 1 inside "
         "every annotated nucleus and to stay at 0 everywhere else, so a low peak marks a blob the annotator would have skipped. "
         "The weights are unchanged from v2 (reports/cv_v2/); v1 (reports/cv/) used a 2-px ring, the interior map as watershed landscape "
         "and an unweighted antibody loss. The change table below compares the shipped recipe with the previous one under the same protocol.", "",
         "## Headline numbers (micro-averaged over all held-out images, no TTA)", "",
         "| Measure | Precision | Recall | F1 |", "|---|---:|---:|---:|",
         f"| Nucleus detection, IoU ≥ 0.5 | {pct(det['cells']['precision'])} | {pct(det['cells']['recall'])} | {pct(det['cells']['f1'])} |",
         f"| Nucleus detection, IoU ≥ 0.3 | {pct(det['cells_iou30']['precision'])} | {pct(det['cells_iou30']['recall'])} | {pct(det['cells_iou30']['f1'])} |",
         f"| Nucleus detection, centroid inside GT nucleus | {pct(det['cells_centroid']['precision'])} | {pct(det['cells_centroid']['recall'])} | {pct(det['cells_centroid']['f1'])} |",
         f"| Punctum detection (centroid ≤ 6 px) | {pct(det['nostrils']['precision'])} | {pct(det['nostrils']['recall'])} | {pct(det['nostrils']['f1'])} |",
         f"| Nucleus detection, IoU ≥ 0.5, 8× TTA | {pct(det8['cells']['precision'])} | {pct(det8['cells']['recall'])} | {pct(det8['cells']['f1'])} |",
         "", "Per-class F1 (a true positive needs a matched object **and** the right label):", "",
         "| Label | GT | Pred | F1 |", "|---|---:|---:|---:|"]
    for lab in ALL_LABELS:
        r = agg["micro_base"][lab]
        L.append(f"| {lab} | {r['gt_count']} | {r['pred_count']} | {pct(r['f1'])} |")
    L += ["", "## Per held-out image", "",
          "| Image | Nuclei F1 | Antibody acc | Nostril-attr acc | Puncta F1 | Nuclei F1 over 14 perturbed runs (mean ± sd) | Deterministic |",
          "|---|---:|---:|---:|---:|---|---|"]
    for k, s in per.items():
        b = s["base"]; r = s["robustness"]["cell_det_f1"]
        L.append(f"| {k} | {pct(b['cell_det_f1'])} | {pct(b['cell_attr_antibody'])} | {pct(b['cell_attr_nostril'])} | {pct(b['nos_det_f1'])} | "
                 f"{100*r['mean']:.1f} ± {100*r['std']:.1f} | {'yes' if s['deterministic'] else 'no'} |")
    L += ["", "## Final model (trained on all images) – fit on its own training images, NOT a generalisation estimate", "",
          "| Image | Nuclei F1 | Attribute acc | Puncta F1 |", "|---|---:|---:|---:|"]
    for k, v in fit.items():
        if "summary" in v:
            s = v["summary"]
            L.append(f"| {k} | {pct(s['cell_det_f1'])} | {pct(s['cell_attr_both'])} | {pct(s['nos_det_f1'])} |")
    if prev:
        pd, pd8 = prev["detection_base"], prev["detection_tta8"]
        L += ["", f"## Change from {prev_name} (same protocol, same nine held-out images; `{a.cv_dir}/comparison.md` has the full table)", "",
              f"| Measure | {prev_name} | {cur_name} | Δ (points) |", "|---|---:|---:|---:|"]
        def row(name, x, y):
            L.append(f"| {name} | {pct(x)} | {pct(y)} | {100 * (y - x):+.1f} |")
        row("Nucleus F1, IoU ≥ 0.5", pd["cells"]["f1"], det["cells"]["f1"])
        row("Nucleus precision, IoU ≥ 0.5", pd["cells"]["precision"], det["cells"]["precision"])
        row("Nucleus recall, IoU ≥ 0.5", pd["cells"]["recall"], det["cells"]["recall"])
        row("Nucleus F1, centroid rule", pd["cells_centroid"]["f1"], det["cells_centroid"]["f1"])
        row("Nucleus F1, IoU ≥ 0.5, 8× TTA", pd8["cells"]["f1"], det8["cells"]["f1"])
        row("Puncta F1", pd["nostrils"]["f1"], det["nostrils"]["f1"])
        for lab in ALL_LABELS:
            row(f"F1 [{lab}]", prev["micro_base"][lab]["f1"], agg["micro_base"][lab]["f1"])
        pr, cr = prev["rule_variants"]["rule_nos=head_cell=contains"], agg["rule_variants"]["rule_nos=head_cell=contains"]
        row("Attribute accuracy, both (each model's own matched nuclei)", pr["pooled_cell_attr_both"], cr["pooled_cell_attr_both"])
        row("Punctum type accuracy (matched puncta)", pr["pooled_nos_type_acc"], cr["pooled_nos_type_acc"])
        for k in sorted(per):
            if k in prev["per_image"]:
                row(f"Nuclei F1, {k} held out", prev["per_image"][k]["base"]["cell_det_f1"], per[k]["base"]["cell_det_f1"])
        n_pred_prev = pd["cells"]["tp"] + pd["cells"]["fp"]
        n_pred_cur = det["cells"]["tp"] + det["cells"]["fp"]
        n_gt = det["cells"]["tp"] + det["cells"]["fn"]
        nov_prev = sum(s["fp_analysis"]["n_fp_no_gt_overlap"] for s in prev["per_image"].values())
        nov_cur = sum(s["fp_analysis"]["n_fp_no_gt_overlap"] for s in per.values())
        L += ["", f"{cur_name} keeps the {prev_name} weights and drops {n_pred_prev - n_pred_cur} of the {n_pred_prev} nuclei {prev_name} predicted on the "
              f"nine held-out images ({n_gt} annotated): unmatched predictions fall from {pd['cells']['fp']} to {det['cells']['fp']} "
              f"(those overlapping no annotated nucleus from {nov_prev} to {nov_cur}) while matched ones fall from {pd['cells']['tp']} to {det['cells']['tp']}, "
              f"so precision rises by {100 * (det['cells']['precision'] - pd['cells']['precision']):.1f} points and recall drops by "
              f"{100 * (pd['cells']['recall'] - det['cells']['recall']):.1f}. Puncta are untouched. The over-count of nuclei on the annotated images "
              f"shrinks from {100 * (n_pred_prev / n_gt - 1):.0f} % to {100 * (n_pred_cur / n_gt - 1):.0f} %. The threshold was never tuned on a held-out image: "
              "each fold used the value chosen on its own training images (0.60-0.67, see rule_study.md), the shipped model the value chosen on all nine "
              f"({peak_thr(agg):.2f})."]
    sw = agg.get("antibody_threshold_sweep")
    if sw:
        best = max(sw["sweep"], key=lambda r: r["neg_f1"])
        L += ["", f"## Antibody − threshold (held-out matched nuclei: {sw['n_matched']}, of which {sw['n_negative']} negative)", "",
              f"Default threshold 0.5. Best Antibody − F1 in the sweep: {pct(best['neg_f1'])} at threshold {best['thr']:.2f} "
              f"(precision {pct(best['neg_precision'])}, recall {pct(best['neg_recall'])}); the full curve is in cv_report.md. "
              f"The default was kept because the class is dominated by a single image (F99 holds 62 of the {tot['Cell, Antibody -'] + tot['Cell, Nostril, Antibody -']} negatives)."]
    L += ["", "## What limits the scores", "",
          "* **Touching nuclei** are still the main source of IoU-0.5 misses (worst on F113, the densest image). The distance-map watershed of v2 recovers many clusters that v1 merged (v1 marker/threshold sweeps in `reports/cv/postprocess_sweep.json` had shown post-processing alone could not), but some clusters are still merged and a few nuclei are now split in two.",
          "* **Annotation completeness**: most unmatched predictions overlap an annotated nucleus (outline disagreement); the rest are dimmer, smaller objects the annotator skipped, which the peak-distance rule of v3 halves (143 → 72 on the held-out images) at the price of 21 matched nuclei. Many faint red dots were likewise not annotated, which caps external-punctum precision.",
          f"* **Antibody − is rare** ({tot['Cell, Antibody -'] + tot['Cell, Nostril, Antibody -']} of {n_nuc} nuclei, 62 in one image), and **Cell, Nostril, Antibody −** has five examples; their scores are not reliable. The class-balanced loss of v2 trades antibody-negative precision for recall.",
          "* Scores are from nine images of one experiment (35-day spinal cord, Iba1/TMEM119). New staining batches or magnifications need a check against a few annotated images.",
          "", "## Reproduce", "",
          "```bash", "python evaluate.py cv --data-dir <folder with TIFs and CVAT zips> --runs-dir runs/cv_v2 --iters 3000",
          "python scripts/rule_study.py --data-dir <folder> --runs-dir runs/cv_v2 --out runs/cv_v3/rule_study --weights weights/cellquant.pt",
          "python evaluate.py cv --data-dir <folder> --runs-dir runs/cv_v3 --post-config runs/cv_v3/rule_study_post_config.json   # fold_*.pt linked from runs/cv_v2",
          "python evaluate.py final --data-dir <folder> --weights weights/cellquant.pt",
          "python scripts/compare_cv.py reports/cv_v2/cv_results.json runs/cv_v3/cv_results.json --names v2,v3 --out reports/cv_v3/comparison.md",
          "python scripts/build_summary.py --cv-dir reports/cv_v3 --compare reports/cv_v2/cv_results.json --names v2,v3", "```", ""]
    out = os.path.join(ROOT, "reports", "SUMMARY.md")
    with open(out, "w") as f:
        f.write("\n".join(L))
    print("wrote", out)


if __name__ == "__main__":
    main()