"""Assemble figures and the data appendix that the response letter cites. Reads the CSVs written by ``05_evaluate.py`` and emits: * ``figures/Figure2_FROC.{pdf,png,tiff}`` official-protocol FROC curves * ``figures/Figure4_Foldwise.*`` per-fold CPM on the official subsets * ``figures/Figure5_PairedCI.*`` paired bootstrap CPM differences * ``figures/Figure6_RSweep.*`` supervision-extent sweep * ``figures/Figure7_WminSweep.*`` minimum-box-size sweep * ``figures/Figure8_YOLO26.*`` YOLO11n vs YOLO26n FROC * ``results/RESULTS_SUMMARY.md`` every headline number, in one place Usage ----- python scripts/07_report.py """ from __future__ import annotations import json import sys from pathlib import Path import pandas as pd sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from luna_rev import config as cfg from luna_rev import figures as fx R = cfg.RESULTS_DIR MAIN_ORDER = ["Exp1_2D_Loose", "Exp2_MIP_Loose", "Exp3_2p5D_Loose", "Exp4_2p5D_Strict"] def read(name: str) -> pd.DataFrame | None: p = R / name return pd.read_csv(p) if p.exists() else None def md_table(df: pd.DataFrame | None, cols=None, floatfmt=4) -> str: if df is None or df.empty: return "_(not available)_" d = df[[c for c in cols if c in df.columns]] if cols else df return d.round(floatfmt).to_markdown(index=False) def build_figures() -> list[str]: made = [] curves, main = read("froc_curves.csv"), read("table_main.csv") if curves is not None and main is not None: order = [n for n in MAIN_ORDER if n in set(curves.experiment)] fx.froc_figure(curves[curves.experiment.isin(order)], main, "Figure2_FROC", order=order) made.append("Figure2_FROC") fold = read("table_foldwise.csv") if fold is not None: fx.foldwise_figure(fold, "Figure4_Foldwise", order=[n for n in MAIN_ORDER if n in set(fold.experiment)]) made.append("Figure4_Foldwise") paired = read("table_paired.csv") if paired is not None: fx.forest_figure(paired, "Figure5_PairedCI") made.append("Figure5_PairedCI") rs = read("table_r_sweep.csv") if rs is not None: d = rs[rs.representation == "naive"].drop_duplicates("r_sample") fx.sweep_figure(d, "r_sample", "Figure6_RSweep", "Supervision-extent ratio $r_{\\mathrm{sample}}$", symbol="$r$", value_fmt="{:.1f}") made.append("Figure6_RSweep") ws = read("table_wmin_sweep.csv") if ws is not None: d = ws.drop_duplicates("w_min_px") fx.sweep_figure(d, "w_min_px", "Figure7_WminSweep", "Minimum box side $w_{\\min}$ (native pixels)", symbol="$w_{\\min}$", value_fmt="{:g} px") made.append("Figure7_WminSweep") # Drawn from curves scored on the backbone comparison's own cohort, so the # curves and the CPM values in the legend describe the same scans. y_curves, summ = read("froc_curves_yolo26.csv"), read("table_yolo26.csv") if y_curves is not None: y26 = [n for n in y_curves.experiment.unique() if n.startswith("Y26_")] if y26: order = y26 + [n for n in ("Exp3_2p5D_Loose", "Exp4_2p5D_Strict") if n in set(y_curves.experiment)] fx.froc_figure(y_curves[y_curves.experiment.isin(order)], summ if summ is not None else pd.DataFrame(columns=["experiment", "cpm"]), "Figure8_YOLO26", order=order) made.append("Figure8_YOLO26") pe = read("table_protocol_effect.csv") if pe is not None: fx.protocol_figure(pe, "Figure9_Protocol") made.append("Figure9_Protocol") return made def build_summary() -> Path: stats = json.loads((R / "dataset_stats.json").read_text(encoding="utf-8")) \ if (R / "dataset_stats.json").exists() else {} prep = json.loads((R / "prepare_report.json").read_text(encoding="utf-8")) \ if (R / "prepare_report.json").exists() else {} L: list[str] = ["# Revision R1 - results summary", ""] L += ["## Protocol verification", ""] ver = prep.get("evaluator_verification", {}) L += [f"- Evaluator reproduces the official `CADAnalysis.txt` reference exactly: " f"**{ver.get('matches_reference')}**", f"- Cohort: {prep.get('n_scans')} scans, {prep.get('n_nodules')} reference nodules, " f"{prep.get('n_excluded_findings')} irrelevant findings", "- Folds: official subset0-9; validation subset disjoint from both train and test", ""] L += ["## Training data", ""] if stats: L += [f"- Slices per representation: **{stats['n_positive_slices'] + stats['n_negative_slices']}** " f"({stats['n_positive_slices']} positive, {stats['n_negative_slices']} background)", f"- Scans contributing training slices: **{stats['n_scans']}** " f"(including {stats['n_nodule_free_scans']} nodule-free scans)", f"- Detector input: {stats['img_size']} x {stats['img_size']} (native resolution)", ""] v = pd.DataFrame(stats["variants"]) L += ["### Fraction of training boxes clamped at `w_min`", "", md_table(v, ["variant", "r_sample", "w_min_px", "n_boxes", "n_boxes_clamped_at_w_min", "frac_clamped"]), ""] sections = [ ("Main comparison (official protocol, 888 scans)", "table_main.csv", ["label", "cpm", "ci_low", "ci_high", "ci_width", "candidates_per_scan", "false_positives", "max_recall"]), ("Seven-point FROC sensitivity", "table_froc_points.csv", ["label", "fp_per_scan", "sensitivity"]), ("Paired bootstrap CPM differences", "table_paired.csv", ["contrast", "delta_cpm", "ci_low", "ci_high", "p_bootstrap", "significant"]), ("Fold-wise paired tests (official subsets)", "table_foldwise_tests.csv", ["contrast", "n_folds", "mean_diff", "median_diff", "wins_a", "wilcoxon_p", "wilcoxon_p_holm", "cohen_dz"]), ("Effect of applying annotations_excluded.csv", "table_excluded_effect.csv", ["label", "cpm_without_excluded", "cpm_with_excluded_official", "delta_cpm", "fp_without_excluded", "fp_with_excluded", "candidates_ignored", "pct_fp_removed"]), ("Supervision-extent sweep", "table_r_sweep.csv", ["label", "r_sample", "w_min_px", "cpm", "ci_low", "ci_high"]), ("Minimum-box-size sweep", "table_wmin_sweep.csv", ["label", "r_sample", "w_min_px", "cpm", "ci_low", "ci_high"]), ("Negative mining", "table_negatives.csv", ["label", "negatives", "cpm", "ci_low", "ci_high", "false_positives", "candidates_per_scan"]), ("Multi-seed repetition", "table_seeds.csv", ["configuration", "n_seeds", "cpm_mean", "cpm_sd", "cpm_range", "cpm_by_seed"]), ("YOLO11n vs YOLO26n", "table_yolo26.csv", ["label", "model", "cpm", "ci_low", "ci_high"]), ("Size-stratified recall", "table_size_recall.csv", ["label", "size_group", "n_nodules", "n_detected", "recall", "operating_point_fp_per_scan"]), ] for title, fname, cols in sections: L += [f"## {title}", "", md_table(read(fname), cols), ""] sota = read("sota_reference.csv") if sota is not None: L += ["## Published LUNA16 results, with the protocol each used", "", md_table(sota, ["method", "year", "architecture", "protocol", "metric", "value", "comparable_to_official_cpm"]), ""] prof = R / "model_profile.md" if prof.exists(): L += ["## Model and hardware profile", "", prof.read_text(encoding="utf-8"), ""] out = R / "RESULTS_SUMMARY.md" out.write_text("\n".join(L), encoding="utf-8") return out def main() -> int: made = build_figures() print("figures:", ", ".join(made) if made else "(none - run 05_evaluate.py first)") out = build_summary() print(f"summary: {out}") return 0 if __name__ == "__main__": raise SystemExit(main())