Spaces:
Running on Zero
Running on Zero
| #!/usr/bin/env python3 | |
| """Write reports/SUMMARY.md from a cross-validation run and reports/final/final_fit.json. | |
| python scripts/build_summary.py --cv-dir reports/cv_v3 --compare reports/cv_v2/cv_results.json --names v2,v3 | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import sys | |
| ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| sys.path.insert(0, ROOT) | |
| from cellquant.labels import ALL_LABELS, CELL_LABELS, NOSTRIL_LABELS # noqa: E402 | |
| def pct(x: float) -> str: | |
| return f"{100 * x:.1f}%" | |
| def peak_thr(agg: dict) -> float: | |
| """Shipped min_peak_dist: the final model's threshold from rule_study.json next to cv_results.json, else PostConfig's default.""" | |
| from cellquant.postprocess import PostConfig | |
| return float(agg.get("final_min_peak_dist", PostConfig().min_peak_dist)) | |
| def main(argv=None): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--cv-dir", default="runs/cv", help="folder holding cv_results.json of the shipped recipe") | |
| ap.add_argument("--compare", default="reports/cv/cv_results.json", | |
| help="cv_results.json of the previous recipe for the change table ('' = none)") | |
| ap.add_argument("--names", default="v1,v2", help="names of the previous and the shipped recipe") | |
| a = ap.parse_args(argv) | |
| prev_name, cur_name = a.names.split(",") | |
| cv = json.load(open(os.path.join(ROOT, a.cv_dir, "cv_results.json"))) | |
| prev = json.load(open(os.path.join(ROOT, a.compare)))["aggregate"] if a.compare and os.path.exists(os.path.join(ROOT, a.compare)) else None | |
| agg = cv["aggregate"] | |
| per = agg["per_image"] | |
| rs = os.path.join(ROOT, a.cv_dir, "rule_study.json") | |
| if os.path.exists(rs): | |
| fin = json.load(open(rs)).get("final") | |
| if fin: | |
| agg["final_min_peak_dist"] = fin["chosen"]["thr"] | |
| fit = json.load(open(os.path.join(ROOT, "reports", "final", "final_fit.json"))) | |
| tot = agg["dataset_totals"] | |
| n_nuc = sum(tot[c] for c in CELL_LABELS) | |
| n_pun = sum(tot[c] for c in NOSTRIL_LABELS) | |
| det = agg["detection_base"] | |
| det8 = agg["detection_tta8"] | |
| L = ["# CellQuant: validation summary", "", | |
| f"Data: {len(per)} annotated 1024×1024 images, {n_nuc} nucleus outlines and {n_pun} punctum outlines.", | |
| "Protocol: leave-one-image-out cross-validation (one model per image, 3000 iterations each, held-out image never seen in training), " | |
| "then a final model trained on all images with the same recipe. Each held-out image was tested 23 times: 3 identical runs " | |
| "(determinism), no-TTA / 4× / 8× test-time augmentation, 8 rotations and flips, 6 photometric perturbations, and 6 post-processing rule variants. " | |
| "The only choice made on held-out data is the rule that gives a nucleus the Nostril attribute: of three candidate rules, " | |
| "'contains an in-nucleus punctum' (the annotation protocol) scored best and is the default; see the rule-variant table in cv_report.md. " | |
| "The peak-distance threshold of v3 was chosen, for every fold, on that fold's eight training images only (scripts/rule_study.py), " | |
| "and for the shipped model on its own nine training images; the held-out images never entered that choice.", "", | |
| "Recipe (v3, shipped): the v2 model (ResNet34-UNet with a full-resolution stem; targets are a 3-px inner border ring (class weight 3), " | |
| "a per-nucleus normalised distance-to-outline map that the watershed uses as its landscape (markers = distance > 0.5), " | |
| "an antibody head whose loss up-weights antibody-negative pixels by sqrt(positive / negative pixels) (4.5 on the full set), " | |
| "and 25 % of training crops centred on an antibody-negative nucleus) plus one post-processing rule: a nucleus whose peak " | |
| f"predicted distance is below {peak_thr(agg):.2f} is dropped (PostConfig.min_peak_dist). The distance head is trained to reach 1 inside " | |
| "every annotated nucleus and to stay at 0 everywhere else, so a low peak marks a blob the annotator would have skipped. " | |
| "The weights are unchanged from v2 (reports/cv_v2/); v1 (reports/cv/) used a 2-px ring, the interior map as watershed landscape " | |
| "and an unweighted antibody loss. The change table below compares the shipped recipe with the previous one under the same protocol.", "", | |
| "## Headline numbers (micro-averaged over all held-out images, no TTA)", "", | |
| "| Measure | Precision | Recall | F1 |", "|---|---:|---:|---:|", | |
| f"| Nucleus detection, IoU ≥ 0.5 | {pct(det['cells']['precision'])} | {pct(det['cells']['recall'])} | {pct(det['cells']['f1'])} |", | |
| f"| Nucleus detection, IoU ≥ 0.3 | {pct(det['cells_iou30']['precision'])} | {pct(det['cells_iou30']['recall'])} | {pct(det['cells_iou30']['f1'])} |", | |
| f"| Nucleus detection, centroid inside GT nucleus | {pct(det['cells_centroid']['precision'])} | {pct(det['cells_centroid']['recall'])} | {pct(det['cells_centroid']['f1'])} |", | |
| f"| Punctum detection (centroid ≤ 6 px) | {pct(det['nostrils']['precision'])} | {pct(det['nostrils']['recall'])} | {pct(det['nostrils']['f1'])} |", | |
| f"| Nucleus detection, IoU ≥ 0.5, 8× TTA | {pct(det8['cells']['precision'])} | {pct(det8['cells']['recall'])} | {pct(det8['cells']['f1'])} |", | |
| "", "Per-class F1 (a true positive needs a matched object **and** the right label):", "", | |
| "| Label | GT | Pred | F1 |", "|---|---:|---:|---:|"] | |
| for lab in ALL_LABELS: | |
| r = agg["micro_base"][lab] | |
| L.append(f"| {lab} | {r['gt_count']} | {r['pred_count']} | {pct(r['f1'])} |") | |
| L += ["", "## Per held-out image", "", | |
| "| Image | Nuclei F1 | Antibody acc | Nostril-attr acc | Puncta F1 | Nuclei F1 over 14 perturbed runs (mean ± sd) | Deterministic |", | |
| "|---|---:|---:|---:|---:|---|---|"] | |
| for k, s in per.items(): | |
| b = s["base"]; r = s["robustness"]["cell_det_f1"] | |
| L.append(f"| {k} | {pct(b['cell_det_f1'])} | {pct(b['cell_attr_antibody'])} | {pct(b['cell_attr_nostril'])} | {pct(b['nos_det_f1'])} | " | |
| f"{100*r['mean']:.1f} ± {100*r['std']:.1f} | {'yes' if s['deterministic'] else 'no'} |") | |
| L += ["", "## Final model (trained on all images) – fit on its own training images, NOT a generalisation estimate", "", | |
| "| Image | Nuclei F1 | Attribute acc | Puncta F1 |", "|---|---:|---:|---:|"] | |
| for k, v in fit.items(): | |
| if "summary" in v: | |
| s = v["summary"] | |
| L.append(f"| {k} | {pct(s['cell_det_f1'])} | {pct(s['cell_attr_both'])} | {pct(s['nos_det_f1'])} |") | |
| if prev: | |
| pd, pd8 = prev["detection_base"], prev["detection_tta8"] | |
| L += ["", f"## Change from {prev_name} (same protocol, same nine held-out images; `{a.cv_dir}/comparison.md` has the full table)", "", | |
| f"| Measure | {prev_name} | {cur_name} | Δ (points) |", "|---|---:|---:|---:|"] | |
| def row(name, x, y): | |
| L.append(f"| {name} | {pct(x)} | {pct(y)} | {100 * (y - x):+.1f} |") | |
| row("Nucleus F1, IoU ≥ 0.5", pd["cells"]["f1"], det["cells"]["f1"]) | |
| row("Nucleus precision, IoU ≥ 0.5", pd["cells"]["precision"], det["cells"]["precision"]) | |
| row("Nucleus recall, IoU ≥ 0.5", pd["cells"]["recall"], det["cells"]["recall"]) | |
| row("Nucleus F1, centroid rule", pd["cells_centroid"]["f1"], det["cells_centroid"]["f1"]) | |
| row("Nucleus F1, IoU ≥ 0.5, 8× TTA", pd8["cells"]["f1"], det8["cells"]["f1"]) | |
| row("Puncta F1", pd["nostrils"]["f1"], det["nostrils"]["f1"]) | |
| for lab in ALL_LABELS: | |
| row(f"F1 [{lab}]", prev["micro_base"][lab]["f1"], agg["micro_base"][lab]["f1"]) | |
| pr, cr = prev["rule_variants"]["rule_nos=head_cell=contains"], agg["rule_variants"]["rule_nos=head_cell=contains"] | |
| row("Attribute accuracy, both (each model's own matched nuclei)", pr["pooled_cell_attr_both"], cr["pooled_cell_attr_both"]) | |
| row("Punctum type accuracy (matched puncta)", pr["pooled_nos_type_acc"], cr["pooled_nos_type_acc"]) | |
| for k in sorted(per): | |
| if k in prev["per_image"]: | |
| row(f"Nuclei F1, {k} held out", prev["per_image"][k]["base"]["cell_det_f1"], per[k]["base"]["cell_det_f1"]) | |
| n_pred_prev = pd["cells"]["tp"] + pd["cells"]["fp"] | |
| n_pred_cur = det["cells"]["tp"] + det["cells"]["fp"] | |
| n_gt = det["cells"]["tp"] + det["cells"]["fn"] | |
| nov_prev = sum(s["fp_analysis"]["n_fp_no_gt_overlap"] for s in prev["per_image"].values()) | |
| nov_cur = sum(s["fp_analysis"]["n_fp_no_gt_overlap"] for s in per.values()) | |
| L += ["", f"{cur_name} keeps the {prev_name} weights and drops {n_pred_prev - n_pred_cur} of the {n_pred_prev} nuclei {prev_name} predicted on the " | |
| f"nine held-out images ({n_gt} annotated): unmatched predictions fall from {pd['cells']['fp']} to {det['cells']['fp']} " | |
| f"(those overlapping no annotated nucleus from {nov_prev} to {nov_cur}) while matched ones fall from {pd['cells']['tp']} to {det['cells']['tp']}, " | |
| f"so precision rises by {100 * (det['cells']['precision'] - pd['cells']['precision']):.1f} points and recall drops by " | |
| f"{100 * (pd['cells']['recall'] - det['cells']['recall']):.1f}. Puncta are untouched. The over-count of nuclei on the annotated images " | |
| f"shrinks from {100 * (n_pred_prev / n_gt - 1):.0f} % to {100 * (n_pred_cur / n_gt - 1):.0f} %. The threshold was never tuned on a held-out image: " | |
| "each fold used the value chosen on its own training images (0.60-0.67, see rule_study.md), the shipped model the value chosen on all nine " | |
| f"({peak_thr(agg):.2f})."] | |
| sw = agg.get("antibody_threshold_sweep") | |
| if sw: | |
| best = max(sw["sweep"], key=lambda r: r["neg_f1"]) | |
| L += ["", f"## Antibody − threshold (held-out matched nuclei: {sw['n_matched']}, of which {sw['n_negative']} negative)", "", | |
| f"Default threshold 0.5. Best Antibody − F1 in the sweep: {pct(best['neg_f1'])} at threshold {best['thr']:.2f} " | |
| f"(precision {pct(best['neg_precision'])}, recall {pct(best['neg_recall'])}); the full curve is in cv_report.md. " | |
| f"The default was kept because the class is dominated by a single image (F99 holds 62 of the {tot['Cell, Antibody -'] + tot['Cell, Nostril, Antibody -']} negatives)."] | |
| L += ["", "## What limits the scores", "", | |
| "* **Touching nuclei** are still the main source of IoU-0.5 misses (worst on F113, the densest image). The distance-map watershed of v2 recovers many clusters that v1 merged (v1 marker/threshold sweeps in `reports/cv/postprocess_sweep.json` had shown post-processing alone could not), but some clusters are still merged and a few nuclei are now split in two.", | |
| "* **Annotation completeness**: most unmatched predictions overlap an annotated nucleus (outline disagreement); the rest are dimmer, smaller objects the annotator skipped, which the peak-distance rule of v3 halves (143 → 72 on the held-out images) at the price of 21 matched nuclei. Many faint red dots were likewise not annotated, which caps external-punctum precision.", | |
| f"* **Antibody − is rare** ({tot['Cell, Antibody -'] + tot['Cell, Nostril, Antibody -']} of {n_nuc} nuclei, 62 in one image), and **Cell, Nostril, Antibody −** has five examples; their scores are not reliable. The class-balanced loss of v2 trades antibody-negative precision for recall.", | |
| "* Scores are from nine images of one experiment (35-day spinal cord, Iba1/TMEM119). New staining batches or magnifications need a check against a few annotated images.", | |
| "", "## Reproduce", "", | |
| "```bash", "python evaluate.py cv --data-dir <folder with TIFs and CVAT zips> --runs-dir runs/cv_v2 --iters 3000", | |
| "python scripts/rule_study.py --data-dir <folder> --runs-dir runs/cv_v2 --out runs/cv_v3/rule_study --weights weights/cellquant.pt", | |
| "python evaluate.py cv --data-dir <folder> --runs-dir runs/cv_v3 --post-config runs/cv_v3/rule_study_post_config.json # fold_*.pt linked from runs/cv_v2", | |
| "python evaluate.py final --data-dir <folder> --weights weights/cellquant.pt", | |
| "python scripts/compare_cv.py reports/cv_v2/cv_results.json runs/cv_v3/cv_results.json --names v2,v3 --out reports/cv_v3/comparison.md", | |
| "python scripts/build_summary.py --cv-dir reports/cv_v3 --compare reports/cv_v2/cv_results.json --names v2,v3", "```", ""] | |
| out = os.path.join(ROOT, "reports", "SUMMARY.md") | |
| with open(out, "w") as f: | |
| f.write("\n".join(L)) | |
| print("wrote", out) | |
| if __name__ == "__main__": | |
| main() | |