| """Assemble figures and the data appendix that the response letter cites.
|
|
|
| Reads the CSVs written by ``05_evaluate.py`` and emits:
|
|
|
| * ``figures/Figure2_FROC.{pdf,png,tiff}`` official-protocol FROC curves
|
| * ``figures/Figure4_Foldwise.*`` per-fold CPM on the official subsets
|
| * ``figures/Figure5_PairedCI.*`` paired bootstrap CPM differences
|
| * ``figures/Figure6_RSweep.*`` supervision-extent sweep
|
| * ``figures/Figure7_WminSweep.*`` minimum-box-size sweep
|
| * ``figures/Figure8_YOLO26.*`` YOLO11n vs YOLO26n FROC
|
| * ``results/RESULTS_SUMMARY.md`` every headline number, in one place
|
|
|
| Usage
|
| -----
|
| python scripts/07_report.py
|
| """
|
|
|
| from __future__ import annotations
|
|
|
| import json
|
| import sys
|
| from pathlib import Path
|
|
|
| import pandas as pd
|
|
|
| sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
| from luna_rev import config as cfg
|
| from luna_rev import figures as fx
|
|
|
| R = cfg.RESULTS_DIR
|
| MAIN_ORDER = ["Exp1_2D_Loose", "Exp2_MIP_Loose", "Exp3_2p5D_Loose", "Exp4_2p5D_Strict"]
|
|
|
|
|
| def read(name: str) -> pd.DataFrame | None:
|
| p = R / name
|
| return pd.read_csv(p) if p.exists() else None
|
|
|
|
|
| def md_table(df: pd.DataFrame | None, cols=None, floatfmt=4) -> str:
|
| if df is None or df.empty:
|
| return "_(not available)_"
|
| d = df[[c for c in cols if c in df.columns]] if cols else df
|
| return d.round(floatfmt).to_markdown(index=False)
|
|
|
|
|
| def build_figures() -> list[str]:
|
| made = []
|
| curves, main = read("froc_curves.csv"), read("table_main.csv")
|
| if curves is not None and main is not None:
|
| order = [n for n in MAIN_ORDER if n in set(curves.experiment)]
|
| fx.froc_figure(curves[curves.experiment.isin(order)], main, "Figure2_FROC", order=order)
|
| made.append("Figure2_FROC")
|
|
|
| fold = read("table_foldwise.csv")
|
| if fold is not None:
|
| fx.foldwise_figure(fold, "Figure4_Foldwise",
|
| order=[n for n in MAIN_ORDER if n in set(fold.experiment)])
|
| made.append("Figure4_Foldwise")
|
|
|
| paired = read("table_paired.csv")
|
| if paired is not None:
|
| fx.forest_figure(paired, "Figure5_PairedCI")
|
| made.append("Figure5_PairedCI")
|
|
|
| rs = read("table_r_sweep.csv")
|
| if rs is not None:
|
| d = rs[rs.representation == "naive"].drop_duplicates("r_sample")
|
| fx.sweep_figure(d, "r_sample", "Figure6_RSweep",
|
| "Supervision-extent ratio $r_{\\mathrm{sample}}$",
|
| symbol="$r$", value_fmt="{:.1f}")
|
| made.append("Figure6_RSweep")
|
|
|
| ws = read("table_wmin_sweep.csv")
|
| if ws is not None:
|
| d = ws.drop_duplicates("w_min_px")
|
| fx.sweep_figure(d, "w_min_px", "Figure7_WminSweep",
|
| "Minimum box side $w_{\\min}$ (native pixels)",
|
| symbol="$w_{\\min}$", value_fmt="{:g} px")
|
| made.append("Figure7_WminSweep")
|
|
|
|
|
|
|
| y_curves, summ = read("froc_curves_yolo26.csv"), read("table_yolo26.csv")
|
| if y_curves is not None:
|
| y26 = [n for n in y_curves.experiment.unique() if n.startswith("Y26_")]
|
| if y26:
|
| order = y26 + [n for n in ("Exp3_2p5D_Loose", "Exp4_2p5D_Strict")
|
| if n in set(y_curves.experiment)]
|
| fx.froc_figure(y_curves[y_curves.experiment.isin(order)],
|
| summ if summ is not None else pd.DataFrame(columns=["experiment", "cpm"]),
|
| "Figure8_YOLO26", order=order)
|
| made.append("Figure8_YOLO26")
|
|
|
| pe = read("table_protocol_effect.csv")
|
| if pe is not None:
|
| fx.protocol_figure(pe, "Figure9_Protocol")
|
| made.append("Figure9_Protocol")
|
| return made
|
|
|
|
|
| def build_summary() -> Path:
|
| stats = json.loads((R / "dataset_stats.json").read_text(encoding="utf-8")) \
|
| if (R / "dataset_stats.json").exists() else {}
|
| prep = json.loads((R / "prepare_report.json").read_text(encoding="utf-8")) \
|
| if (R / "prepare_report.json").exists() else {}
|
|
|
| L: list[str] = ["# Revision R1 - results summary", ""]
|
| L += ["## Protocol verification", ""]
|
| ver = prep.get("evaluator_verification", {})
|
| L += [f"- Evaluator reproduces the official `CADAnalysis.txt` reference exactly: "
|
| f"**{ver.get('matches_reference')}**",
|
| f"- Cohort: {prep.get('n_scans')} scans, {prep.get('n_nodules')} reference nodules, "
|
| f"{prep.get('n_excluded_findings')} irrelevant findings",
|
| "- Folds: official subset0-9; validation subset disjoint from both train and test", ""]
|
|
|
| L += ["## Training data", ""]
|
| if stats:
|
| L += [f"- Slices per representation: **{stats['n_positive_slices'] + stats['n_negative_slices']}** "
|
| f"({stats['n_positive_slices']} positive, {stats['n_negative_slices']} background)",
|
| f"- Scans contributing training slices: **{stats['n_scans']}** "
|
| f"(including {stats['n_nodule_free_scans']} nodule-free scans)",
|
| f"- Detector input: {stats['img_size']} x {stats['img_size']} (native resolution)", ""]
|
| v = pd.DataFrame(stats["variants"])
|
| L += ["### Fraction of training boxes clamped at `w_min`", "",
|
| md_table(v, ["variant", "r_sample", "w_min_px", "n_boxes",
|
| "n_boxes_clamped_at_w_min", "frac_clamped"]), ""]
|
|
|
| sections = [
|
| ("Main comparison (official protocol, 888 scans)", "table_main.csv",
|
| ["label", "cpm", "ci_low", "ci_high", "ci_width", "candidates_per_scan",
|
| "false_positives", "max_recall"]),
|
| ("Seven-point FROC sensitivity", "table_froc_points.csv",
|
| ["label", "fp_per_scan", "sensitivity"]),
|
| ("Paired bootstrap CPM differences", "table_paired.csv",
|
| ["contrast", "delta_cpm", "ci_low", "ci_high", "p_bootstrap", "significant"]),
|
| ("Fold-wise paired tests (official subsets)", "table_foldwise_tests.csv",
|
| ["contrast", "n_folds", "mean_diff", "median_diff", "wins_a", "wilcoxon_p",
|
| "wilcoxon_p_holm", "cohen_dz"]),
|
| ("Effect of applying annotations_excluded.csv", "table_excluded_effect.csv",
|
| ["label", "cpm_without_excluded", "cpm_with_excluded_official", "delta_cpm",
|
| "fp_without_excluded", "fp_with_excluded", "candidates_ignored", "pct_fp_removed"]),
|
| ("Supervision-extent sweep", "table_r_sweep.csv",
|
| ["label", "r_sample", "w_min_px", "cpm", "ci_low", "ci_high"]),
|
| ("Minimum-box-size sweep", "table_wmin_sweep.csv",
|
| ["label", "r_sample", "w_min_px", "cpm", "ci_low", "ci_high"]),
|
| ("Negative mining", "table_negatives.csv",
|
| ["label", "negatives", "cpm", "ci_low", "ci_high", "false_positives",
|
| "candidates_per_scan"]),
|
| ("Multi-seed repetition", "table_seeds.csv",
|
| ["configuration", "n_seeds", "cpm_mean", "cpm_sd", "cpm_range", "cpm_by_seed"]),
|
| ("YOLO11n vs YOLO26n", "table_yolo26.csv",
|
| ["label", "model", "cpm", "ci_low", "ci_high"]),
|
| ("Size-stratified recall", "table_size_recall.csv",
|
| ["label", "size_group", "n_nodules", "n_detected", "recall",
|
| "operating_point_fp_per_scan"]),
|
| ]
|
| for title, fname, cols in sections:
|
| L += [f"## {title}", "", md_table(read(fname), cols), ""]
|
|
|
| sota = read("sota_reference.csv")
|
| if sota is not None:
|
| L += ["## Published LUNA16 results, with the protocol each used", "",
|
| md_table(sota, ["method", "year", "architecture", "protocol", "metric",
|
| "value", "comparable_to_official_cpm"]), ""]
|
|
|
| prof = R / "model_profile.md"
|
| if prof.exists():
|
| L += ["## Model and hardware profile", "", prof.read_text(encoding="utf-8"), ""]
|
|
|
| out = R / "RESULTS_SUMMARY.md"
|
| out.write_text("\n".join(L), encoding="utf-8")
|
| return out
|
|
|
|
|
| def main() -> int:
|
| made = build_figures()
|
| print("figures:", ", ".join(made) if made else "(none - run 05_evaluate.py first)")
|
| out = build_summary()
|
| print(f"summary: {out}")
|
| return 0
|
|
|
|
|
| if __name__ == "__main__":
|
| raise SystemExit(main())
|
|
|
