Spaces:
Running
Running
Download pkg/src/strokecopilot/analysis/figures.py from parsa-hke/stroke-copilot: direct link, hf CLI and curl.
- Browser
- Download file 28.9 kB
-
https://huggingface.co/spaces/parsa-hke/stroke-copilot/resolve/main/pkg/src/strokecopilot/analysis/figures.py
- Command line
-
hf download hf://spaces/parsa-hke/stroke-copilot/pkg/src/strokecopilot/analysis/figures.py
-
curl -L -o figures.py https://huggingface.co/spaces/parsa-hke/stroke-copilot/resolve/main/pkg/src/strokecopilot/analysis/figures.py
28.9 kB
| """Phase 5 figures (DEC-34), in the style of `pipeline/figures.py`, written to Drive `artifacts/figures/` next to the | |
| existing ones. The figures README gains or replaces one line per figure; the others are left as they are. | |
| """ | |
| import json | |
| from pathlib import Path | |
| import numpy as np | |
| from strokecopilot.pipeline.figures import GRID, INK, INK_2, SLOTS, _plt, _save # noqa: F401 | |
| VISION_COLOR, NOTE_COLOR = SLOTS[0], SLOTS[1] # validated as a pair (dataviz validator, light surface) | |
| LABELS = {"tmax6": "Tmax > 6 s volume", "core": "core volume"} | |
| CAPTIONS = { | |
| "fig_rule_engine_shap": ( | |
| "EXP-68 (DEC-35, SHAP row 8): exact Shapley values of the rule engine's 13 input groups on the 150 synthetic " | |
| "scenarios; an absent input is unknown. Bars: mean |Shapley value| for reaching each scenario's decision. Text: " | |
| "the share carried by vision inputs where the rules say yes. Describes our scenario set, not real patients."), | |
| "fig_ttests_forest": ( | |
| "EXP-69 (DEC-36): the paired NLP comparisons, mean difference of per-case rates with its 95% t interval (Student's " | |
| "paired t-test, dot) beside the reported paired case bootstrap of the pooled rate (open circle). The case is the " | |
| "unit. Holm-adjusted p over all 35 tests is in the run's report; the data had been seen before the tests " | |
| "were fixed."), | |
| "fig_confidence_v2_shap": ( | |
| "EXP-70 (DEC-35, SHAP row 3): share of mean |SHAP value| (exact, LinearExplainer) of confidence.core v2's four " | |
| "signals over the 129 development cases, in log-odds of a large error; in-sample (v2 was fitted on them). The " | |
| "contract uses the distance alone (DEC-27)."), | |
| "fig_retrieval_token_shap": ( | |
| "EXP-71 (DEC-35, SHAP row 10): for each of the 12 key fixture passages, the share of its bge similarity to a " | |
| "gold claim that falls on number tokens (filled) against the share of its tokens that are numbers (open); " | |
| "Partition SHAP over passage tokens. The rank is the passage's place among all 47 for that claim."), | |
| "fig_surrogate_flags_shap": ( | |
| "EXP-73 (DEC-35, SHAP row 11): a SURROGATE gradient-boosted model of which claims the verifier flags, from claim " | |
| "features known before verification (27,472 claims, six models, rulebook arms); out-of-fold AUC 0.92 [0.92, 0.93], " | |
| "grouped by case. Mean SHAP value (log-odds of being flagged) when a claim has each kind or comes from each model, " | |
| "with the flag rate. It explains the surrogate, not the verifier."), | |
| "fig_visionB_pixel_shap": ( | |
| "EXP-72 (DEC-35, SHAP row 6): four of the 68 ISLES'24 development scans Vision B flags at DEC-28 (top slab, no " | |
| "case ID): brain window, Grad-CAM (Item 2) and |SHAP| by expected gradients, background 50 unflagged top slabs. " | |
| "Over all 136 top-2 slabs, median share in brain tissue: SHAP 50%, Grad-CAM 76%; bone 26% and 9%. ISLES'24, " | |
| "CC BY-NC-SA 4.0."), | |
| "fig_subgroups_vision": ( | |
| "EXP-80 (DEC-37, task E): Vision A's Dice and wrong-side rate and Vision B's bleed-flag rate on the 129 development " | |
| "cases, by age band, sex, centre and onset-information pattern; case-bootstrap (Dice) and Wilson intervals. " | |
| "Unadjusted: infarct size differs between groups. Groups under 10 cases are left out."), | |
| "fig_visionA_channel_shap": ( | |
| "EXP-75 (DEC-35, SHAP row 1): exact Shapley values of nnU-Net's five inputs (NCCT, CBF, CBV, MTT, Tmax with its " | |
| "mask) on the 129 development cases, each predicted by its validating fold model for all 32 combinations; mean " | |
| "|value| for the predicted core volume and for Dice. Reliance, not ablation: no model was trained without an input."), | |
| "fig_reasoner_shap": ( | |
| "EXP-74 (DEC-35, SHAP row 9, amended): exact Shapley values of six input groups on 24 synthetic scenarios (the " | |
| "bleed fixed), for the rule engine and for Gemma 3 4B prompted and LoRA-tuned on the same 64 versions of each " | |
| "scenario; mean |value| for reaching the full-input decision."), | |
| "fig_visionB_window_shap": ( | |
| "EXP-76 and EXP-77 (DEC-35, SHAP row 5): exact Shapley values of the three CT windows on the calibrated scan " | |
| "probability; mean value by group on RSNA validation and on ISLES'24's 129 development scans. An absent window is " | |
| "set to its RSNA training mean. Reliance, not ablation."), | |
| "fig_judge_agreement": ( | |
| "EXP-81 (DEC-38, task F): agreement with the human consensus on EXP-41's 100 claims, per reasoner model, for the " | |
| "Claude Haiku 5.5 judge and for GPT-6 Luna (no family shared with any reasoner). Metric points 19 and 20 were " | |
| "adjusted on these labels."), | |
| "fig_biomedclip": ( | |
| "EXP-82 and EXP-83 (DEC-33, task D): on RSNA validation, scan AUC and specificity at the threshold reaching 96% " | |
| "sensitivity (in-sample for every model alike) for EXP-6 and for BiomedCLIP as a linear probe and zero-shot, " | |
| "each scan scored by EXP-6's top-2 aggregation; ISLES'24 flags are of the 129 presumed bleed-free development " | |
| "scans. Right: per-subtype probes on EXP-6's frozen features beside EXP-6's own 'any' score. Report only; the " | |
| "RSNA test split was not read."), | |
| } | |
| SHORT = {"claude-sonnet-5-5": "Sonnet 5.5", "qwen3.5-27b": "Qwen3.5-27B", "gemma-3-27b": "Gemma 3 27B", | |
| "medgemma-27b-text": "MedGemma 27B", "gemma-3-4b": "Gemma 3 4B", "gemma-3-4b-tuned": "Gemma 3 4B tuned"} | |
| GROUP = {"T3": "With minus without the rulebook (content rate)", "T4": "Predicted minus ground-truth vision", | |
| "T5": "Minus Sonnet 5.5", "T6": "Tuned minus base Gemma 3 4B"} | |
| def _forest_label(r): | |
| head = r["label"].split(":")[0].split(" minus ")[0] | |
| if r["id"].startswith("T6"): | |
| return {"T6a": "all 175 cases", "T6b": "held-out routes (51)", "T6c": "critical rate, all"}[r["id"]] | |
| return SHORT.get(head, head) | |
| def ttest_forest_figure(rows, out): | |
| """T3-T6 on one axis in percentage points: t interval (dot) and the existing bootstrap interval (open circle).""" | |
| plt = _plt() | |
| rows = [r for r in rows if r["id"][:2] in GROUP] | |
| ys, labels, y = [], [], 0 | |
| heads = [] | |
| for g in GROUP: | |
| heads.append((y, GROUP[g])) | |
| y += 1 | |
| for r in [r for r in rows if r["id"].startswith(g)]: | |
| ys.append((y, r)) | |
| labels.append((y, _forest_label(r))) | |
| y += 1 | |
| y += 0.4 | |
| fig, ax = plt.subplots(figsize=(7.2, 0.26 * y + 1.0)) | |
| for yy, r in ys: | |
| lo, hi = (100 * v for v in r["ci"]) | |
| ax.plot([lo, hi], [yy - 0.13] * 2, color=VISION_COLOR, lw=2, solid_capstyle="round") | |
| ax.plot([100 * r["mean_difference"]], [yy - 0.13], "o", ms=6, color=VISION_COLOR, mec="white", mew=1.2) | |
| ex = r.get("existing") or {} | |
| if ex.get("ci95"): | |
| blo, bhi = (100 * v for v in ex["ci95"]) | |
| ax.plot([blo, bhi], [yy + 0.13] * 2, color=NOTE_COLOR, lw=2, solid_capstyle="round") | |
| ax.plot([100 * ex["pooled_difference"]], [yy + 0.13], "o", ms=6, mfc="white", mec=NOTE_COLOR, mew=1.5) | |
| if r["agrees_with_existing"] is False: | |
| ax.text(max(hi, bhi) + 0.8, yy, "intervals disagree", va="center", fontsize=7, color=INK_2) | |
| ax.axvline(0, color=INK_2, lw=0.8) | |
| ax.set_yticks([l[0] for l in labels] + [h[0] for h in heads]) | |
| ax.set_yticklabels([l[1] for l in labels] + [h[1] for h in heads]) | |
| for tick, (_, _) in zip(ax.get_yticklabels()[len(labels):], heads): | |
| tick.set_fontweight("bold") | |
| tick.set_color(INK) | |
| ax.tick_params(axis="y", length=0) | |
| ax.set_ylim(y - 0.2, -0.8) | |
| ax.grid(axis="y", visible=False) | |
| ax.set_xlabel("difference, percentage points") | |
| handles = [plt.Line2D([], [], color=VISION_COLOR, marker="o", lw=2, mec="white"), | |
| plt.Line2D([], [], color=NOTE_COLOR, marker="o", lw=2, mfc="white", mec=NOTE_COLOR)] | |
| ax.legend(handles, ["paired t-test: mean of per-case rates, 95% t interval", | |
| "reported: pooled rate, 95% paired case bootstrap"], loc="upper right", frameon=False, fontsize=7.5) | |
| written = [] | |
| for target in out if isinstance(out, (list, tuple)) else (out,): | |
| Path(target).mkdir(parents=True, exist_ok=True) | |
| _save(fig, target, "fig_ttests_forest", written) | |
| plt.close(fig) | |
| return written[:1] | |
| def update_readme(out, name): | |
| """Add or replace one figure's line in the figures README.""" | |
| path = Path(out) / "README.md" | |
| lines = path.read_text().splitlines() if path.exists() else ["# Figures", ""] | |
| entry = f"- **{name}**: {CAPTIONS[name]}" | |
| lines = [entry if line.startswith(f"- **{name}**") else line for line in lines] | |
| if entry not in lines: | |
| lines.append(entry) | |
| path.write_text("\n".join(lines) + "\n") | |
| def rule_shap_figure(metrics, out, players_order=None): | |
| """Two panels (thrombolysis, thrombectomy): mean |phi| per input group, coloured vision or note.""" | |
| from strokecopilot.analysis import rule_shap | |
| plt = _plt() | |
| s = metrics["summary"] | |
| rows = json.loads((Path(metrics["_folder"]) / "per_scenario.json").read_text()) if "_folder" in metrics else None | |
| mag = {d: {p: float(np.mean([abs(r["players"][d][p]) for r in rows])) for p in rule_shap.PLAYERS} | |
| for d in rule_shap.DECISIONS} | |
| order = players_order or sorted(rule_shap.PLAYERS, key=lambda p: mag["thrombolysis"][p] + mag["thrombectomy"][p]) | |
| fig, axes = plt.subplots(1, 2, figsize=(8.6, 4.2), sharey=True) | |
| for ax, d in zip(axes, rule_shap.DECISIONS): | |
| vals = [mag[d][p] for p in order] | |
| colors = [VISION_COLOR if p in rule_shap.VISION else NOTE_COLOR for p in order] | |
| ax.barh(range(len(order)), vals, height=0.62, color=colors) | |
| for i in np.argsort(vals)[-3:]: | |
| ax.text(vals[i] + 0.012, i, f"{vals[i]:.2f}", va="center", fontsize=8, color=INK_2) | |
| yes = s[d].get("decision yes") | |
| share = yes["vision_vs_note"]["vision"] / sum(yes["vision_vs_note"].values()) if yes else float("nan") | |
| ax.set_title(f"{d.capitalize()}\nvision inputs carry {100 * share:.0f}% where the rules\nsay yes " | |
| f"({yes['cases'] if yes else 0} scenarios)", fontsize=9, color=INK, loc="left") | |
| ax.tick_params(axis="y", length=0) | |
| ax.set_xlim(0, 0.68) | |
| ax.set_xlabel("mean |Shapley value|") | |
| ax.grid(axis="y", visible=False) | |
| axes[0].set_yticks(range(len(order))) | |
| axes[0].set_yticklabels([LABELS.get(p, p) for p in order]) | |
| handles = [plt.Rectangle((0, 0), 1, 1, color=c) for c in (VISION_COLOR, NOTE_COLOR)] | |
| fig.legend(handles, ["vision input (contract)", "note input"], loc="lower center", ncol=2, frameon=False, | |
| bbox_to_anchor=(0.55, -0.04), fontsize=8, labelcolor=INK_2) | |
| fig.subplots_adjust(wspace=0.12, bottom=0.18) | |
| written = [] | |
| for target in (out,) if not isinstance(out, (list, tuple)) else out: | |
| Path(target).mkdir(parents=True, exist_ok=True) | |
| _save(fig, target, "fig_rule_engine_shap", written) | |
| plt.close(fig) | |
| return written[:1] | |
| def main(argv=None): | |
| import argparse | |
| from strokecopilot.config import local_drive_root | |
| ap = argparse.ArgumentParser(description="Phase 5 figures") | |
| ap.add_argument("--rule-shap", default="runs/EXP-68_rule_shap") | |
| ap.add_argument("--ttests", default="runs/EXP-69_ttests") | |
| ap.add_argument("--out", default=None, help="default: <Drive>/StrokeCopilot/artifacts/figures") | |
| args = ap.parse_args(argv) | |
| out = Path(args.out) if args.out else local_drive_root() / "artifacts" / "figures" | |
| m = json.loads((Path(args.rule_shap) / "metrics.json").read_text()) | |
| m["_folder"] = args.rule_shap | |
| names = rule_shap_figure(m, [out, args.rule_shap]) | |
| for run, draw, name in (("runs/EXP-70_confidence_shap", confidence_shap_figure, "metrics.json"), | |
| ("runs/EXP-71_retrieval_shap", retrieval_shap_figure, "per_pair.json"), | |
| ("runs/EXP-73_surrogate_flags", surrogate_flags_figure, "metrics.json")): | |
| if (Path(run) / name).exists(): | |
| names += draw(json.loads((Path(run) / name).read_text()), [out, run]) | |
| sub = Path("runs/EXP-80_subgroups/metrics.json") | |
| if sub.exists(): | |
| names += subgroups_figure(json.loads(sub.read_text()), [out, sub.parent]) | |
| for run, draw in (("runs/EXP-75_visionA_channel_shap", visiona_channel_figure), ("runs/EXP-74_reasoner_shap", reasoner_shap_figure), | |
| ("runs/EXP-81_rejudge", judge_figure)): | |
| if (Path(run) / "metrics.json").exists(): | |
| names += draw(json.loads((Path(run) / "metrics.json").read_text()), [out, run]) | |
| rsna, isles = Path("runs/EXP-76_visionB_window_shap/metrics.json"), Path("runs/EXP-77_visionB_window_shap_isles/metrics.json") | |
| if rsna.exists() or isles.exists(): | |
| names += visionb_window_figure(json.loads(rsna.read_text()) if rsna.exists() else None, | |
| json.loads(isles.read_text()) if isles.exists() else None, [out, isles.parent]) | |
| clip, clip_isles = Path("runs/EXP-82_biomedclip/metrics.json"), Path("runs/EXP-83_biomedclip_isles/metrics.json") | |
| if clip.exists(): | |
| names += biomedclip_figure(json.loads(clip.read_text()), | |
| json.loads(clip_isles.read_text()) if clip_isles.exists() else None, [out, clip.parent]) | |
| pixel = Path("runs/EXP-72_visionB_pixel_shap/pixel_shap_examples.png") | |
| if pixel.exists(): | |
| import shutil | |
| shutil.copyfile(pixel, out / "fig_visionB_pixel_shap.png") | |
| names.append("fig_visionB_pixel_shap") | |
| tt = Path(args.ttests) / "metrics.json" | |
| if tt.exists(): | |
| names += ttest_forest_figure(json.loads(tt.read_text())["rows"], [out, args.ttests]) | |
| for n in names: | |
| update_readme(out, n) | |
| print(json.dumps({"written": names, "out": str(out)})) | |
| def confidence_shap_figure(metrics, out): | |
| plt = _plt() | |
| sig = metrics["signals"] | |
| order = sorted(sig, key=lambda k: sig[k]["share_of_attribution"]) | |
| fig, ax = plt.subplots(figsize=(6.4, 2.2)) | |
| vals = [100 * sig[k]["share_of_attribution"] for k in order] | |
| ax.barh(range(len(order)), vals, height=0.6, color=VISION_COLOR) | |
| for i, v in enumerate(vals): | |
| ax.text(v + 1, i, f"{v:.0f}%", va="center", fontsize=8, color=INK_2) | |
| ax.set_yticks(range(len(order))) | |
| ax.set_yticklabels([sig[k]["label"] for k in order]) | |
| ax.tick_params(axis="y", length=0) | |
| ax.set_xlim(0, 100) | |
| ax.set_xlabel("share of mean |SHAP value| (log-odds of a large error)") | |
| ax.grid(axis="y", visible=False) | |
| ax.set_title("confidence.core v2: what moves it (129 development cases)", fontsize=9, color=INK, loc="left") | |
| return _write(fig, out, "fig_confidence_v2_shap") | |
| def retrieval_shap_figure(rows, out): | |
| plt = _plt() | |
| rows = sorted(rows, key=lambda r: r["digit_attribution_share"]) | |
| fig, ax = plt.subplots(figsize=(7.2, 0.32 * len(rows) + 1.1)) | |
| for i, r in enumerate(rows): | |
| a, t = 100 * r["digit_attribution_share"], 100 * r["digit_token_share"] | |
| ax.plot([t, a], [i, i], color=GRID, lw=2, zorder=1) | |
| ax.plot([t], [i], "o", ms=7, mfc="white", mec=NOTE_COLOR, mew=1.5, zorder=2) | |
| ax.plot([a], [i], "o", ms=7, color=VISION_COLOR, mec="white", mew=1.2, zorder=3) | |
| ax.set_yticks(range(len(rows))) | |
| ax.set_yticklabels([f"{r['passage']} (rank {r['rank']})" for r in rows], fontsize=7.5) | |
| ax.tick_params(axis="y", length=0) | |
| ax.grid(axis="y", visible=False) | |
| ax.set_xlabel("percent") | |
| handles = [plt.Line2D([], [], ls="", marker="o", color=VISION_COLOR, mec="white"), | |
| plt.Line2D([], [], ls="", marker="o", mfc="white", mec=NOTE_COLOR, mew=1.5)] | |
| ax.legend(handles, ["share of the similarity's |SHAP| on number tokens", "share of tokens that are numbers"], | |
| loc="lower right", frameon=False, fontsize=7.5) | |
| return _write(fig, out, "fig_retrieval_token_shap") | |
| def _write(fig, out, name): | |
| written = [] | |
| for target in out if isinstance(out, (list, tuple)) else (out,): | |
| Path(target).mkdir(parents=True, exist_ok=True) | |
| _save(fig, target, name, written) | |
| _plt().close(fig) | |
| return written[:1] | |
| KIND_WORDS = {"rule": "rule claim", "comparison": "comparison or reason", "patient": "patient fact", "number": "number", | |
| "citation": "decision citation (rule_ids)", "absence": "something not recorded", "vision": "imaging value"} | |
| def surrogate_flags_figure(metrics, out): | |
| plt = _plt() | |
| lv = metrics["explained"]["levels"] | |
| panels = [("kind", "Claim kind"), ("model", "Model")] | |
| fig, axes = plt.subplots(1, 2, figsize=(8.6, 3.2), gridspec_kw={"width_ratios": [1, 1]}) | |
| for ax, (feat, title) in zip(axes, panels): | |
| items = sorted([(k.split("=", 1)[1], v) for k, v in lv.items() if k.startswith(feat + "=")], | |
| key=lambda kv: kv[1]["mean_shap_when_present"]) | |
| names = [KIND_WORDS.get(n, n) if feat == "kind" else SHORT.get(n, n) for n, _ in items] | |
| vals = [v["mean_shap_when_present"] for _, v in items] | |
| ax.barh(range(len(items)), vals, height=0.6, color=[NOTE_COLOR if x > 0 else VISION_COLOR for x in vals]) | |
| for i, (_, v) in enumerate(items): | |
| x = v["mean_shap_when_present"] | |
| ax.text(x + (0.06 if x >= 0 else -0.06), i, f"{100 * v['flag_rate']:.0f}% flagged", va="center", | |
| ha="left" if x >= 0 else "right", fontsize=7, color=INK_2) | |
| ax.axvline(0, color=INK_2, lw=0.8) | |
| ax.set_yticks(range(len(items))) | |
| ax.set_yticklabels(names) | |
| ax.tick_params(axis="y", length=0) | |
| ax.grid(axis="y", visible=False) | |
| ax.set_xlim(-2.2, 2.6) | |
| ax.set_xlabel("mean SHAP value, log-odds of a flag") | |
| ax.set_title(title, fontsize=9, color=INK, loc="left") | |
| fig.suptitle("Surrogate model of the verifier's flags (not the verifier itself)", fontsize=9, color=INK, x=0.01, | |
| ha="left") | |
| fig.tight_layout() | |
| return _write(fig, out, "fig_surrogate_flags_shap") | |
| def subgroups_figure(metrics, out): | |
| plt = _plt() | |
| v = metrics["vision"] | |
| rows = [(g, lab, d) for g in ("age band", "sex", "centre", "onset") for lab, d in v.get(g, {}).items() if not d["too_small"]] | |
| panels = [("Vision A Dice", lambda d: (d["dice_mean"], d["dice_ci"])), | |
| ("Vision A: core on the wrong side of a cut-off", lambda d: (d["wrong_side"] / d["cases"], d["wrong_side_ci"])), | |
| ("Vision B: bleed flag (all presumed false)", lambda d: (d["bleed_flags"] / d["cases"], d["bleed_flags_ci"]))] | |
| fig, axes = plt.subplots(1, 3, figsize=(10, 0.32 * len(rows) + 1.3), sharey=True) | |
| for ax, (title, get) in zip(axes, panels): | |
| for i, (g, lab, d) in enumerate(rows): | |
| val, ci = get(d) | |
| y = len(rows) - 1 - i | |
| ax.plot(ci, [y, y], color=VISION_COLOR, lw=2, alpha=0.55, solid_capstyle="round") | |
| ax.plot([val], [y], "o", ms=6, color=VISION_COLOR, mec="white", mew=1.2) | |
| ax.set_title(title, fontsize=8.5, color=INK, loc="left") | |
| ax.set_xlim(0, 1) | |
| ax.grid(axis="y", visible=False) | |
| ax.tick_params(axis="y", length=0) | |
| axes[0].set_yticks(range(len(rows))) | |
| axes[0].set_yticklabels([f"{lab} (n={d['cases']})" for g, lab, d in reversed(rows)], fontsize=8) | |
| fig.tight_layout() | |
| return _write(fig, out, "fig_subgroups_vision") | |
| THREE = (SLOTS[0], SLOTS[1], SLOTS[2]) # validated as a set (dataviz validator, light; labels carry the contrast) | |
| def _hbars(ax, names, series, labels, xlabel, title, fmt="{:.2f}"): | |
| """Grouped horizontal bars: names top to bottom, one bar per series, a value label at each bar's end.""" | |
| n = len(series) | |
| h = 0.8 / n | |
| for k, (vals, lab) in enumerate(zip(series, labels)): | |
| ys = [len(names) - 1 - i + (n - 1 - 2 * k) * h / 2 for i in range(len(names))] | |
| ax.barh(ys, vals, height=h * 0.92, color=THREE[k % 3], label=lab) | |
| for y, v in zip(ys, vals): | |
| if abs(v) < 0.005: | |
| continue # no label on an empty bar | |
| ax.text(v + (0.01 * max(max(s) for s in series) if v >= 0 else -0.01), y, fmt.format(v), va="center", | |
| ha="left" if v >= 0 else "right", fontsize=6.5, color=INK_2) | |
| ax.set_yticks(range(len(names))) | |
| ax.set_yticklabels(list(reversed(names))) | |
| ax.tick_params(axis="y", length=0) | |
| ax.grid(axis="y", visible=False) | |
| ax.set_xlabel(xlabel) | |
| ax.set_title(title, fontsize=9, color=INK, loc="left") | |
| def visiona_channel_figure(metrics, out): | |
| plt = _plt() | |
| players = metrics["players"] | |
| fig, axes = plt.subplots(1, 2, figsize=(8.6, 2.8), sharey=True) | |
| for ax, key, xlabel, fmt in ((axes[0], "pred_ml", "mean |Shapley value|, mL", "{:.1f}"), | |
| (axes[1], "dice", "mean |Shapley value|, Dice", "{:.3f}")): | |
| vals = [metrics[key]["mean_abs"][p] for p in players] | |
| unit, f = (" mL", "{:.1f}") if key == "pred_ml" else ("", "{:.3f}") | |
| _hbars(ax, players, [vals], [None], xlabel, | |
| f"{'Predicted core volume' if key == 'pred_ml' else 'Dice'}\nmean over cases: all inputs " | |
| f"{f.format(metrics[key]['full_mean'])}{unit}, none {f.format(metrics[key]['none_mean'])}{unit}", fmt) | |
| ax.set_xlim(0, max(vals) * 1.25 or 1) | |
| fig.suptitle(f"Vision A: which inputs nnU-Net relies on (development cases: {metrics['complete']})", fontsize=9, | |
| color=INK, x=0.01, ha="left") | |
| fig.tight_layout() | |
| return _write(fig, out, "fig_visionA_channel_shap") | |
| def reasoner_shap_figure(metrics, out): | |
| plt = _plt() | |
| s = metrics["summary"] | |
| groups = metrics["groups"] | |
| base, tuned = "gemma-3-4b", "gemma-3-4b-tuned" | |
| fig, axes = plt.subplots(1, 2, figsize=(9, 3.4), sharey=True) | |
| for ax, d in zip(axes, ("thrombolysis", "thrombectomy")): | |
| series = [[s[base][d]["rules_mean_abs"][g] for g in groups], [s[base][d]["model_mean_abs"][g] for g in groups], | |
| [s[tuned][d]["model_mean_abs"][g] for g in groups]] | |
| _hbars(ax, groups, series, ["rule engine", "Gemma 3 4B", "Gemma 3 4B tuned"], "mean |Shapley value|", | |
| d.capitalize()) | |
| ax.set_xlim(0, max(max(v) for v in series) * 1.3 or 1) | |
| for name, vals in (("Gemma 3 4B", series[1]), ("Gemma 3 4B tuned", series[2])): | |
| if not any(vals): # an empty bar would look like missing data | |
| ax.text(0.98, 0.02, f"{name}: 0 for every input\n(no input changes its answer)", transform=ax.transAxes, | |
| ha="right", va="bottom", fontsize=7, color=INK_2) | |
| axes[1].legend(loc="lower right", frameon=False, fontsize=7.5) | |
| fig.suptitle(f"What the rules and the small reasoners rely on ({s[base]['scenarios_complete']} and " | |
| f"{s[tuned]['scenarios_complete']} complete scenarios)", fontsize=9, color=INK, x=0.01, ha="left") | |
| fig.tight_layout() | |
| return _write(fig, out, "fig_reasoner_shap") | |
| def visionb_window_figure(rsna, isles, out): | |
| plt = _plt() | |
| windows = ("brain", "subdural", "bone") | |
| panels = [] | |
| if rsna: | |
| g = rsna["groups"] | |
| panels.append(("RSNA validation", [k for k in ("bleed", "no bleed", "false positive") if k in g], g)) | |
| if isles: | |
| g = isles["groups"] | |
| panels.append(("ISLES'24 development, all presumed bleed-free", list(g), g)) | |
| fig, axes = plt.subplots(1, len(panels), figsize=(4.6 * len(panels), 2.9), sharey=False) | |
| axes = np.atleast_1d(axes) | |
| for ax, (title, names, g) in zip(axes, panels): | |
| series = [[g[n]["mean_phi"][w] for n in names] for w in windows] | |
| labels = [f"{n.replace(' (presumed bleed-free)', '')} ({g[n]['scans']})" for n in names] | |
| _hbars(ax, labels, series, list(windows), "mean Shapley value, calibrated probability", title) | |
| ax.axvline(0, color=INK_2, lw=0.8) | |
| ax.set_xlim(-0.25, 1.0) | |
| axes[-1].legend(loc="lower right", frameon=False, fontsize=7.5, title="window", title_fontsize=7.5) | |
| fig.tight_layout() | |
| return _write(fig, out, "fig_visionB_window_shap") | |
| def judge_figure(metrics, out): | |
| plt = _plt() | |
| names = {"claude-sonnet-5-5": "Sonnet 5.5", "qwen/qwen3.5-27b": "Qwen3.5-27B", "gemma-3-27b": "Gemma 3 27B", | |
| "medgemma-27b-text": "MedGemma 27B"} | |
| h, n = metrics["haiku_vs_humans"], metrics["new_judge_vs_humans"] | |
| models = sorted(h["per_model"]) | |
| rows = [("all claims", h["all"], n["all"])] + [(names.get(m, m), h["per_model"][m], n["per_model"][m]) for m in models] | |
| fig, ax = plt.subplots(figsize=(6.4, 0.42 * len(rows) + 1.0)) | |
| for i, (lab, a, b) in enumerate(rows): | |
| y = len(rows) - 1 - i | |
| ax.plot([a["agreement"], b["agreement"]], [y + 0.12, y - 0.12], color=GRID, lw=2, zorder=1) | |
| ax.plot([a["agreement"]], [y + 0.12], "o", ms=7, color=THREE[0], mec="white", mew=1.2, zorder=2, | |
| label="Claude Haiku 5.5" if i == 0 else None) | |
| ax.plot([b["agreement"]], [y - 0.12], "o", ms=7, color=THREE[1], mec="white", mew=1.2, zorder=3, | |
| label="GPT-6 Luna" if i == 0 else None) | |
| ax.text(1.01, y, f"n={a['n']}", va="center", fontsize=7, color=INK_2) | |
| ax.set_yticks(range(len(rows))) | |
| ax.set_yticklabels([r[0] for r in reversed(rows)]) | |
| ax.set_xlim(0, 1) | |
| ax.tick_params(axis="y", length=0) | |
| ax.grid(axis="y", visible=False) | |
| ax.set_xlabel("agreement with the human consensus (flag = error)") | |
| jj = metrics["judge_vs_judge"] | |
| ax.set_title(f"Two judges on the same {metrics['claims']} claims: judge-judge agreement {jj['agreement']:.2f}, κ {jj['kappa']:.2f}", | |
| fontsize=9, color=INK, loc="left") | |
| ax.legend(loc="lower left", frameon=False, fontsize=7.5) | |
| fig.tight_layout() | |
| return _write(fig, out, "fig_judge_agreement") | |
| MODEL_WORDS = {"exp6": "EXP-6 ResNet-50", "probe": "BiomedCLIP, linear probe", "zero_shot": "BiomedCLIP, zero-shot"} | |
| def _dots(ax, names, series, labels, xlabel, title, xlim): | |
| """Dot rows (a truncated axis needs dots, not bars): names top to bottom, one offset dot per series.""" | |
| n = len(series) | |
| for k, (vals, lab) in enumerate(zip(series, labels)): | |
| off = (n - 1 - 2 * k) * 0.12 | |
| ys = [len(names) - 1 - i + off for i in range(len(names))] | |
| keep = [(y, v) for y, v in zip(ys, vals) if v is not None] | |
| ax.plot([v for _, v in keep], [y for y, _ in keep], "o", ms=6.5, color=THREE[k % 3], mec="white", mew=1.1, | |
| label=lab, zorder=2 + k, ls="none") | |
| for y, v in keep: | |
| ax.text(v + 0.02 * (xlim[1] - xlim[0]), y, f"{v:.3f}", va="center", ha="left", fontsize=6.5, color=INK_2) | |
| ax.set_yticks(range(len(names))) | |
| ax.set_yticklabels(list(reversed(names)), fontsize=7.5) | |
| ax.tick_params(axis="y", length=0) | |
| ax.grid(axis="y", visible=False) | |
| ax.set_xlim(*xlim) | |
| ax.set_xlabel(xlabel) | |
| ax.set_title(title, fontsize=9, color=INK, loc="left") | |
| def biomedclip_figure(metrics, isles, out): | |
| """Task D (EXP-82, and EXP-83 when present): scan AUC and specificity at 96% sensitivity for the three models on RSNA | |
| validation; per-subtype scan AUC of the subtype probes beside EXP-6's 'any' score.""" | |
| plt = _plt() | |
| models = [m for m in ("exp6", "probe", "zero_shot") if m in metrics["models"]] | |
| subtypes = sorted(metrics.get("subtypes", {})) | |
| fig, axes = plt.subplots(1, 2 if subtypes else 1, figsize=(9.6 if subtypes else 5.2, 3.3)) | |
| axes = np.atleast_1d(axes) | |
| names = [MODEL_WORDS[m] for m in models] | |
| if isles: | |
| names = [f"{w}\nISLES'24 flags {isles['flagged'][m]}/{isles['scans']}" if m in isles["flagged"] else w | |
| for w, m in zip(names, models)] | |
| vals = [[metrics["models"][m][k] for m in models] for k in ("scan_auc", "specificity")] | |
| lo = min(v for s in vals for v in s if v is not None) | |
| _dots(axes[0], names, vals, ["scan AUC", "specificity at 96% sensitivity"], "", | |
| f"Haemorrhage, RSNA validation ({metrics['val_scans']:,} scans)", (min(0.5, np.floor(lo * 20) / 20), 1.04)) | |
| axes[0].legend(loc="upper center", bbox_to_anchor=(0.5, -0.24), ncol=2, frameon=False, fontsize=7) | |
| if subtypes: | |
| s = metrics["subtypes"] | |
| sv = [[s[t]["probe_scan_auc"] for t in subtypes], [s[t]["exp6_any_scan_auc"] for t in subtypes]] | |
| lo = min(v for row in sv for v in row if v is not None) | |
| _dots(axes[1], [f"{t} ({s[t]['scans_positive']})" for t in subtypes], sv, | |
| ["subtype probe on EXP-6's features", "EXP-6's 'any' score"], "scan AUC for that subtype", | |
| "Per subtype (positive scans)", (min(0.5, np.floor(lo * 20) / 20), 1.04)) | |
| axes[1].legend(loc="upper center", bbox_to_anchor=(0.5, -0.24), ncol=2, frameon=False, fontsize=7) | |
| fig.tight_layout() | |
| return _write(fig, out, "fig_biomedclip") | |
| if __name__ == "__main__": | |
| main() | |