parsa-hke's picture
Redesign: the story (Two hours, thirty minutes), a live-run page and the record page
6025853 verified
Raw History Blame Contribute Delete
28.9 kB
"""Phase 5 figures (DEC-34), in the style of `pipeline/figures.py`, written to Drive `artifacts/figures/` next to the
existing ones. The figures README gains or replaces one line per figure; the others are left as they are.
"""
import json
from pathlib import Path
import numpy as np
from strokecopilot.pipeline.figures import GRID, INK, INK_2, SLOTS, _plt, _save # noqa: F401
VISION_COLOR, NOTE_COLOR = SLOTS[0], SLOTS[1] # validated as a pair (dataviz validator, light surface)
LABELS = {"tmax6": "Tmax > 6 s volume", "core": "core volume"}
CAPTIONS = {
"fig_rule_engine_shap": (
"EXP-68 (DEC-35, SHAP row 8): exact Shapley values of the rule engine's 13 input groups on the 150 synthetic "
"scenarios; an absent input is unknown. Bars: mean |Shapley value| for reaching each scenario's decision. Text: "
"the share carried by vision inputs where the rules say yes. Describes our scenario set, not real patients."),
"fig_ttests_forest": (
"EXP-69 (DEC-36): the paired NLP comparisons, mean difference of per-case rates with its 95% t interval (Student's "
"paired t-test, dot) beside the reported paired case bootstrap of the pooled rate (open circle). The case is the "
"unit. Holm-adjusted p over all 35 tests is in the run's report; the data had been seen before the tests "
"were fixed."),
"fig_confidence_v2_shap": (
"EXP-70 (DEC-35, SHAP row 3): share of mean |SHAP value| (exact, LinearExplainer) of confidence.core v2's four "
"signals over the 129 development cases, in log-odds of a large error; in-sample (v2 was fitted on them). The "
"contract uses the distance alone (DEC-27)."),
"fig_retrieval_token_shap": (
"EXP-71 (DEC-35, SHAP row 10): for each of the 12 key fixture passages, the share of its bge similarity to a "
"gold claim that falls on number tokens (filled) against the share of its tokens that are numbers (open); "
"Partition SHAP over passage tokens. The rank is the passage's place among all 47 for that claim."),
"fig_surrogate_flags_shap": (
"EXP-73 (DEC-35, SHAP row 11): a SURROGATE gradient-boosted model of which claims the verifier flags, from claim "
"features known before verification (27,472 claims, six models, rulebook arms); out-of-fold AUC 0.92 [0.92, 0.93], "
"grouped by case. Mean SHAP value (log-odds of being flagged) when a claim has each kind or comes from each model, "
"with the flag rate. It explains the surrogate, not the verifier."),
"fig_visionB_pixel_shap": (
"EXP-72 (DEC-35, SHAP row 6): four of the 68 ISLES'24 development scans Vision B flags at DEC-28 (top slab, no "
"case ID): brain window, Grad-CAM (Item 2) and |SHAP| by expected gradients, background 50 unflagged top slabs. "
"Over all 136 top-2 slabs, median share in brain tissue: SHAP 50%, Grad-CAM 76%; bone 26% and 9%. ISLES'24, "
"CC BY-NC-SA 4.0."),
"fig_subgroups_vision": (
"EXP-80 (DEC-37, task E): Vision A's Dice and wrong-side rate and Vision B's bleed-flag rate on the 129 development "
"cases, by age band, sex, centre and onset-information pattern; case-bootstrap (Dice) and Wilson intervals. "
"Unadjusted: infarct size differs between groups. Groups under 10 cases are left out."),
"fig_visionA_channel_shap": (
"EXP-75 (DEC-35, SHAP row 1): exact Shapley values of nnU-Net's five inputs (NCCT, CBF, CBV, MTT, Tmax with its "
"mask) on the 129 development cases, each predicted by its validating fold model for all 32 combinations; mean "
"|value| for the predicted core volume and for Dice. Reliance, not ablation: no model was trained without an input."),
"fig_reasoner_shap": (
"EXP-74 (DEC-35, SHAP row 9, amended): exact Shapley values of six input groups on 24 synthetic scenarios (the "
"bleed fixed), for the rule engine and for Gemma 3 4B prompted and LoRA-tuned on the same 64 versions of each "
"scenario; mean |value| for reaching the full-input decision."),
"fig_visionB_window_shap": (
"EXP-76 and EXP-77 (DEC-35, SHAP row 5): exact Shapley values of the three CT windows on the calibrated scan "
"probability; mean value by group on RSNA validation and on ISLES'24's 129 development scans. An absent window is "
"set to its RSNA training mean. Reliance, not ablation."),
"fig_judge_agreement": (
"EXP-81 (DEC-38, task F): agreement with the human consensus on EXP-41's 100 claims, per reasoner model, for the "
"Claude Haiku 5.5 judge and for GPT-6 Luna (no family shared with any reasoner). Metric points 19 and 20 were "
"adjusted on these labels."),
"fig_biomedclip": (
"EXP-82 and EXP-83 (DEC-33, task D): on RSNA validation, scan AUC and specificity at the threshold reaching 96% "
"sensitivity (in-sample for every model alike) for EXP-6 and for BiomedCLIP as a linear probe and zero-shot, "
"each scan scored by EXP-6's top-2 aggregation; ISLES'24 flags are of the 129 presumed bleed-free development "
"scans. Right: per-subtype probes on EXP-6's frozen features beside EXP-6's own 'any' score. Report only; the "
"RSNA test split was not read."),
}
SHORT = {"claude-sonnet-5-5": "Sonnet 5.5", "qwen3.5-27b": "Qwen3.5-27B", "gemma-3-27b": "Gemma 3 27B",
"medgemma-27b-text": "MedGemma 27B", "gemma-3-4b": "Gemma 3 4B", "gemma-3-4b-tuned": "Gemma 3 4B tuned"}
GROUP = {"T3": "With minus without the rulebook (content rate)", "T4": "Predicted minus ground-truth vision",
"T5": "Minus Sonnet 5.5", "T6": "Tuned minus base Gemma 3 4B"}
def _forest_label(r):
head = r["label"].split(":")[0].split(" minus ")[0]
if r["id"].startswith("T6"):
return {"T6a": "all 175 cases", "T6b": "held-out routes (51)", "T6c": "critical rate, all"}[r["id"]]
return SHORT.get(head, head)
def ttest_forest_figure(rows, out):
"""T3-T6 on one axis in percentage points: t interval (dot) and the existing bootstrap interval (open circle)."""
plt = _plt()
rows = [r for r in rows if r["id"][:2] in GROUP]
ys, labels, y = [], [], 0
heads = []
for g in GROUP:
heads.append((y, GROUP[g]))
y += 1
for r in [r for r in rows if r["id"].startswith(g)]:
ys.append((y, r))
labels.append((y, _forest_label(r)))
y += 1
y += 0.4
fig, ax = plt.subplots(figsize=(7.2, 0.26 * y + 1.0))
for yy, r in ys:
lo, hi = (100 * v for v in r["ci"])
ax.plot([lo, hi], [yy - 0.13] * 2, color=VISION_COLOR, lw=2, solid_capstyle="round")
ax.plot([100 * r["mean_difference"]], [yy - 0.13], "o", ms=6, color=VISION_COLOR, mec="white", mew=1.2)
ex = r.get("existing") or {}
if ex.get("ci95"):
blo, bhi = (100 * v for v in ex["ci95"])
ax.plot([blo, bhi], [yy + 0.13] * 2, color=NOTE_COLOR, lw=2, solid_capstyle="round")
ax.plot([100 * ex["pooled_difference"]], [yy + 0.13], "o", ms=6, mfc="white", mec=NOTE_COLOR, mew=1.5)
if r["agrees_with_existing"] is False:
ax.text(max(hi, bhi) + 0.8, yy, "intervals disagree", va="center", fontsize=7, color=INK_2)
ax.axvline(0, color=INK_2, lw=0.8)
ax.set_yticks([l[0] for l in labels] + [h[0] for h in heads])
ax.set_yticklabels([l[1] for l in labels] + [h[1] for h in heads])
for tick, (_, _) in zip(ax.get_yticklabels()[len(labels):], heads):
tick.set_fontweight("bold")
tick.set_color(INK)
ax.tick_params(axis="y", length=0)
ax.set_ylim(y - 0.2, -0.8)
ax.grid(axis="y", visible=False)
ax.set_xlabel("difference, percentage points")
handles = [plt.Line2D([], [], color=VISION_COLOR, marker="o", lw=2, mec="white"),
plt.Line2D([], [], color=NOTE_COLOR, marker="o", lw=2, mfc="white", mec=NOTE_COLOR)]
ax.legend(handles, ["paired t-test: mean of per-case rates, 95% t interval",
"reported: pooled rate, 95% paired case bootstrap"], loc="upper right", frameon=False, fontsize=7.5)
written = []
for target in out if isinstance(out, (list, tuple)) else (out,):
Path(target).mkdir(parents=True, exist_ok=True)
_save(fig, target, "fig_ttests_forest", written)
plt.close(fig)
return written[:1]
def update_readme(out, name):
"""Add or replace one figure's line in the figures README."""
path = Path(out) / "README.md"
lines = path.read_text().splitlines() if path.exists() else ["# Figures", ""]
entry = f"- **{name}**: {CAPTIONS[name]}"
lines = [entry if line.startswith(f"- **{name}**") else line for line in lines]
if entry not in lines:
lines.append(entry)
path.write_text("\n".join(lines) + "\n")
def rule_shap_figure(metrics, out, players_order=None):
"""Two panels (thrombolysis, thrombectomy): mean |phi| per input group, coloured vision or note."""
from strokecopilot.analysis import rule_shap
plt = _plt()
s = metrics["summary"]
rows = json.loads((Path(metrics["_folder"]) / "per_scenario.json").read_text()) if "_folder" in metrics else None
mag = {d: {p: float(np.mean([abs(r["players"][d][p]) for r in rows])) for p in rule_shap.PLAYERS}
for d in rule_shap.DECISIONS}
order = players_order or sorted(rule_shap.PLAYERS, key=lambda p: mag["thrombolysis"][p] + mag["thrombectomy"][p])
fig, axes = plt.subplots(1, 2, figsize=(8.6, 4.2), sharey=True)
for ax, d in zip(axes, rule_shap.DECISIONS):
vals = [mag[d][p] for p in order]
colors = [VISION_COLOR if p in rule_shap.VISION else NOTE_COLOR for p in order]
ax.barh(range(len(order)), vals, height=0.62, color=colors)
for i in np.argsort(vals)[-3:]:
ax.text(vals[i] + 0.012, i, f"{vals[i]:.2f}", va="center", fontsize=8, color=INK_2)
yes = s[d].get("decision yes")
share = yes["vision_vs_note"]["vision"] / sum(yes["vision_vs_note"].values()) if yes else float("nan")
ax.set_title(f"{d.capitalize()}\nvision inputs carry {100 * share:.0f}% where the rules\nsay yes "
f"({yes['cases'] if yes else 0} scenarios)", fontsize=9, color=INK, loc="left")
ax.tick_params(axis="y", length=0)
ax.set_xlim(0, 0.68)
ax.set_xlabel("mean |Shapley value|")
ax.grid(axis="y", visible=False)
axes[0].set_yticks(range(len(order)))
axes[0].set_yticklabels([LABELS.get(p, p) for p in order])
handles = [plt.Rectangle((0, 0), 1, 1, color=c) for c in (VISION_COLOR, NOTE_COLOR)]
fig.legend(handles, ["vision input (contract)", "note input"], loc="lower center", ncol=2, frameon=False,
bbox_to_anchor=(0.55, -0.04), fontsize=8, labelcolor=INK_2)
fig.subplots_adjust(wspace=0.12, bottom=0.18)
written = []
for target in (out,) if not isinstance(out, (list, tuple)) else out:
Path(target).mkdir(parents=True, exist_ok=True)
_save(fig, target, "fig_rule_engine_shap", written)
plt.close(fig)
return written[:1]
def main(argv=None):
import argparse
from strokecopilot.config import local_drive_root
ap = argparse.ArgumentParser(description="Phase 5 figures")
ap.add_argument("--rule-shap", default="runs/EXP-68_rule_shap")
ap.add_argument("--ttests", default="runs/EXP-69_ttests")
ap.add_argument("--out", default=None, help="default: <Drive>/StrokeCopilot/artifacts/figures")
args = ap.parse_args(argv)
out = Path(args.out) if args.out else local_drive_root() / "artifacts" / "figures"
m = json.loads((Path(args.rule_shap) / "metrics.json").read_text())
m["_folder"] = args.rule_shap
names = rule_shap_figure(m, [out, args.rule_shap])
for run, draw, name in (("runs/EXP-70_confidence_shap", confidence_shap_figure, "metrics.json"),
("runs/EXP-71_retrieval_shap", retrieval_shap_figure, "per_pair.json"),
("runs/EXP-73_surrogate_flags", surrogate_flags_figure, "metrics.json")):
if (Path(run) / name).exists():
names += draw(json.loads((Path(run) / name).read_text()), [out, run])
sub = Path("runs/EXP-80_subgroups/metrics.json")
if sub.exists():
names += subgroups_figure(json.loads(sub.read_text()), [out, sub.parent])
for run, draw in (("runs/EXP-75_visionA_channel_shap", visiona_channel_figure), ("runs/EXP-74_reasoner_shap", reasoner_shap_figure),
("runs/EXP-81_rejudge", judge_figure)):
if (Path(run) / "metrics.json").exists():
names += draw(json.loads((Path(run) / "metrics.json").read_text()), [out, run])
rsna, isles = Path("runs/EXP-76_visionB_window_shap/metrics.json"), Path("runs/EXP-77_visionB_window_shap_isles/metrics.json")
if rsna.exists() or isles.exists():
names += visionb_window_figure(json.loads(rsna.read_text()) if rsna.exists() else None,
json.loads(isles.read_text()) if isles.exists() else None, [out, isles.parent])
clip, clip_isles = Path("runs/EXP-82_biomedclip/metrics.json"), Path("runs/EXP-83_biomedclip_isles/metrics.json")
if clip.exists():
names += biomedclip_figure(json.loads(clip.read_text()),
json.loads(clip_isles.read_text()) if clip_isles.exists() else None, [out, clip.parent])
pixel = Path("runs/EXP-72_visionB_pixel_shap/pixel_shap_examples.png")
if pixel.exists():
import shutil
shutil.copyfile(pixel, out / "fig_visionB_pixel_shap.png")
names.append("fig_visionB_pixel_shap")
tt = Path(args.ttests) / "metrics.json"
if tt.exists():
names += ttest_forest_figure(json.loads(tt.read_text())["rows"], [out, args.ttests])
for n in names:
update_readme(out, n)
print(json.dumps({"written": names, "out": str(out)}))
def confidence_shap_figure(metrics, out):
plt = _plt()
sig = metrics["signals"]
order = sorted(sig, key=lambda k: sig[k]["share_of_attribution"])
fig, ax = plt.subplots(figsize=(6.4, 2.2))
vals = [100 * sig[k]["share_of_attribution"] for k in order]
ax.barh(range(len(order)), vals, height=0.6, color=VISION_COLOR)
for i, v in enumerate(vals):
ax.text(v + 1, i, f"{v:.0f}%", va="center", fontsize=8, color=INK_2)
ax.set_yticks(range(len(order)))
ax.set_yticklabels([sig[k]["label"] for k in order])
ax.tick_params(axis="y", length=0)
ax.set_xlim(0, 100)
ax.set_xlabel("share of mean |SHAP value| (log-odds of a large error)")
ax.grid(axis="y", visible=False)
ax.set_title("confidence.core v2: what moves it (129 development cases)", fontsize=9, color=INK, loc="left")
return _write(fig, out, "fig_confidence_v2_shap")
def retrieval_shap_figure(rows, out):
plt = _plt()
rows = sorted(rows, key=lambda r: r["digit_attribution_share"])
fig, ax = plt.subplots(figsize=(7.2, 0.32 * len(rows) + 1.1))
for i, r in enumerate(rows):
a, t = 100 * r["digit_attribution_share"], 100 * r["digit_token_share"]
ax.plot([t, a], [i, i], color=GRID, lw=2, zorder=1)
ax.plot([t], [i], "o", ms=7, mfc="white", mec=NOTE_COLOR, mew=1.5, zorder=2)
ax.plot([a], [i], "o", ms=7, color=VISION_COLOR, mec="white", mew=1.2, zorder=3)
ax.set_yticks(range(len(rows)))
ax.set_yticklabels([f"{r['passage']} (rank {r['rank']})" for r in rows], fontsize=7.5)
ax.tick_params(axis="y", length=0)
ax.grid(axis="y", visible=False)
ax.set_xlabel("percent")
handles = [plt.Line2D([], [], ls="", marker="o", color=VISION_COLOR, mec="white"),
plt.Line2D([], [], ls="", marker="o", mfc="white", mec=NOTE_COLOR, mew=1.5)]
ax.legend(handles, ["share of the similarity's |SHAP| on number tokens", "share of tokens that are numbers"],
loc="lower right", frameon=False, fontsize=7.5)
return _write(fig, out, "fig_retrieval_token_shap")
def _write(fig, out, name):
written = []
for target in out if isinstance(out, (list, tuple)) else (out,):
Path(target).mkdir(parents=True, exist_ok=True)
_save(fig, target, name, written)
_plt().close(fig)
return written[:1]
KIND_WORDS = {"rule": "rule claim", "comparison": "comparison or reason", "patient": "patient fact", "number": "number",
"citation": "decision citation (rule_ids)", "absence": "something not recorded", "vision": "imaging value"}
def surrogate_flags_figure(metrics, out):
plt = _plt()
lv = metrics["explained"]["levels"]
panels = [("kind", "Claim kind"), ("model", "Model")]
fig, axes = plt.subplots(1, 2, figsize=(8.6, 3.2), gridspec_kw={"width_ratios": [1, 1]})
for ax, (feat, title) in zip(axes, panels):
items = sorted([(k.split("=", 1)[1], v) for k, v in lv.items() if k.startswith(feat + "=")],
key=lambda kv: kv[1]["mean_shap_when_present"])
names = [KIND_WORDS.get(n, n) if feat == "kind" else SHORT.get(n, n) for n, _ in items]
vals = [v["mean_shap_when_present"] for _, v in items]
ax.barh(range(len(items)), vals, height=0.6, color=[NOTE_COLOR if x > 0 else VISION_COLOR for x in vals])
for i, (_, v) in enumerate(items):
x = v["mean_shap_when_present"]
ax.text(x + (0.06 if x >= 0 else -0.06), i, f"{100 * v['flag_rate']:.0f}% flagged", va="center",
ha="left" if x >= 0 else "right", fontsize=7, color=INK_2)
ax.axvline(0, color=INK_2, lw=0.8)
ax.set_yticks(range(len(items)))
ax.set_yticklabels(names)
ax.tick_params(axis="y", length=0)
ax.grid(axis="y", visible=False)
ax.set_xlim(-2.2, 2.6)
ax.set_xlabel("mean SHAP value, log-odds of a flag")
ax.set_title(title, fontsize=9, color=INK, loc="left")
fig.suptitle("Surrogate model of the verifier's flags (not the verifier itself)", fontsize=9, color=INK, x=0.01,
ha="left")
fig.tight_layout()
return _write(fig, out, "fig_surrogate_flags_shap")
def subgroups_figure(metrics, out):
plt = _plt()
v = metrics["vision"]
rows = [(g, lab, d) for g in ("age band", "sex", "centre", "onset") for lab, d in v.get(g, {}).items() if not d["too_small"]]
panels = [("Vision A Dice", lambda d: (d["dice_mean"], d["dice_ci"])),
("Vision A: core on the wrong side of a cut-off", lambda d: (d["wrong_side"] / d["cases"], d["wrong_side_ci"])),
("Vision B: bleed flag (all presumed false)", lambda d: (d["bleed_flags"] / d["cases"], d["bleed_flags_ci"]))]
fig, axes = plt.subplots(1, 3, figsize=(10, 0.32 * len(rows) + 1.3), sharey=True)
for ax, (title, get) in zip(axes, panels):
for i, (g, lab, d) in enumerate(rows):
val, ci = get(d)
y = len(rows) - 1 - i
ax.plot(ci, [y, y], color=VISION_COLOR, lw=2, alpha=0.55, solid_capstyle="round")
ax.plot([val], [y], "o", ms=6, color=VISION_COLOR, mec="white", mew=1.2)
ax.set_title(title, fontsize=8.5, color=INK, loc="left")
ax.set_xlim(0, 1)
ax.grid(axis="y", visible=False)
ax.tick_params(axis="y", length=0)
axes[0].set_yticks(range(len(rows)))
axes[0].set_yticklabels([f"{lab} (n={d['cases']})" for g, lab, d in reversed(rows)], fontsize=8)
fig.tight_layout()
return _write(fig, out, "fig_subgroups_vision")
THREE = (SLOTS[0], SLOTS[1], SLOTS[2]) # validated as a set (dataviz validator, light; labels carry the contrast)
def _hbars(ax, names, series, labels, xlabel, title, fmt="{:.2f}"):
"""Grouped horizontal bars: names top to bottom, one bar per series, a value label at each bar's end."""
n = len(series)
h = 0.8 / n
for k, (vals, lab) in enumerate(zip(series, labels)):
ys = [len(names) - 1 - i + (n - 1 - 2 * k) * h / 2 for i in range(len(names))]
ax.barh(ys, vals, height=h * 0.92, color=THREE[k % 3], label=lab)
for y, v in zip(ys, vals):
if abs(v) < 0.005:
continue # no label on an empty bar
ax.text(v + (0.01 * max(max(s) for s in series) if v >= 0 else -0.01), y, fmt.format(v), va="center",
ha="left" if v >= 0 else "right", fontsize=6.5, color=INK_2)
ax.set_yticks(range(len(names)))
ax.set_yticklabels(list(reversed(names)))
ax.tick_params(axis="y", length=0)
ax.grid(axis="y", visible=False)
ax.set_xlabel(xlabel)
ax.set_title(title, fontsize=9, color=INK, loc="left")
def visiona_channel_figure(metrics, out):
plt = _plt()
players = metrics["players"]
fig, axes = plt.subplots(1, 2, figsize=(8.6, 2.8), sharey=True)
for ax, key, xlabel, fmt in ((axes[0], "pred_ml", "mean |Shapley value|, mL", "{:.1f}"),
(axes[1], "dice", "mean |Shapley value|, Dice", "{:.3f}")):
vals = [metrics[key]["mean_abs"][p] for p in players]
unit, f = (" mL", "{:.1f}") if key == "pred_ml" else ("", "{:.3f}")
_hbars(ax, players, [vals], [None], xlabel,
f"{'Predicted core volume' if key == 'pred_ml' else 'Dice'}\nmean over cases: all inputs "
f"{f.format(metrics[key]['full_mean'])}{unit}, none {f.format(metrics[key]['none_mean'])}{unit}", fmt)
ax.set_xlim(0, max(vals) * 1.25 or 1)
fig.suptitle(f"Vision A: which inputs nnU-Net relies on (development cases: {metrics['complete']})", fontsize=9,
color=INK, x=0.01, ha="left")
fig.tight_layout()
return _write(fig, out, "fig_visionA_channel_shap")
def reasoner_shap_figure(metrics, out):
plt = _plt()
s = metrics["summary"]
groups = metrics["groups"]
base, tuned = "gemma-3-4b", "gemma-3-4b-tuned"
fig, axes = plt.subplots(1, 2, figsize=(9, 3.4), sharey=True)
for ax, d in zip(axes, ("thrombolysis", "thrombectomy")):
series = [[s[base][d]["rules_mean_abs"][g] for g in groups], [s[base][d]["model_mean_abs"][g] for g in groups],
[s[tuned][d]["model_mean_abs"][g] for g in groups]]
_hbars(ax, groups, series, ["rule engine", "Gemma 3 4B", "Gemma 3 4B tuned"], "mean |Shapley value|",
d.capitalize())
ax.set_xlim(0, max(max(v) for v in series) * 1.3 or 1)
for name, vals in (("Gemma 3 4B", series[1]), ("Gemma 3 4B tuned", series[2])):
if not any(vals): # an empty bar would look like missing data
ax.text(0.98, 0.02, f"{name}: 0 for every input\n(no input changes its answer)", transform=ax.transAxes,
ha="right", va="bottom", fontsize=7, color=INK_2)
axes[1].legend(loc="lower right", frameon=False, fontsize=7.5)
fig.suptitle(f"What the rules and the small reasoners rely on ({s[base]['scenarios_complete']} and "
f"{s[tuned]['scenarios_complete']} complete scenarios)", fontsize=9, color=INK, x=0.01, ha="left")
fig.tight_layout()
return _write(fig, out, "fig_reasoner_shap")
def visionb_window_figure(rsna, isles, out):
plt = _plt()
windows = ("brain", "subdural", "bone")
panels = []
if rsna:
g = rsna["groups"]
panels.append(("RSNA validation", [k for k in ("bleed", "no bleed", "false positive") if k in g], g))
if isles:
g = isles["groups"]
panels.append(("ISLES'24 development, all presumed bleed-free", list(g), g))
fig, axes = plt.subplots(1, len(panels), figsize=(4.6 * len(panels), 2.9), sharey=False)
axes = np.atleast_1d(axes)
for ax, (title, names, g) in zip(axes, panels):
series = [[g[n]["mean_phi"][w] for n in names] for w in windows]
labels = [f"{n.replace(' (presumed bleed-free)', '')} ({g[n]['scans']})" for n in names]
_hbars(ax, labels, series, list(windows), "mean Shapley value, calibrated probability", title)
ax.axvline(0, color=INK_2, lw=0.8)
ax.set_xlim(-0.25, 1.0)
axes[-1].legend(loc="lower right", frameon=False, fontsize=7.5, title="window", title_fontsize=7.5)
fig.tight_layout()
return _write(fig, out, "fig_visionB_window_shap")
def judge_figure(metrics, out):
plt = _plt()
names = {"claude-sonnet-5-5": "Sonnet 5.5", "qwen/qwen3.5-27b": "Qwen3.5-27B", "gemma-3-27b": "Gemma 3 27B",
"medgemma-27b-text": "MedGemma 27B"}
h, n = metrics["haiku_vs_humans"], metrics["new_judge_vs_humans"]
models = sorted(h["per_model"])
rows = [("all claims", h["all"], n["all"])] + [(names.get(m, m), h["per_model"][m], n["per_model"][m]) for m in models]
fig, ax = plt.subplots(figsize=(6.4, 0.42 * len(rows) + 1.0))
for i, (lab, a, b) in enumerate(rows):
y = len(rows) - 1 - i
ax.plot([a["agreement"], b["agreement"]], [y + 0.12, y - 0.12], color=GRID, lw=2, zorder=1)
ax.plot([a["agreement"]], [y + 0.12], "o", ms=7, color=THREE[0], mec="white", mew=1.2, zorder=2,
label="Claude Haiku 5.5" if i == 0 else None)
ax.plot([b["agreement"]], [y - 0.12], "o", ms=7, color=THREE[1], mec="white", mew=1.2, zorder=3,
label="GPT-6 Luna" if i == 0 else None)
ax.text(1.01, y, f"n={a['n']}", va="center", fontsize=7, color=INK_2)
ax.set_yticks(range(len(rows)))
ax.set_yticklabels([r[0] for r in reversed(rows)])
ax.set_xlim(0, 1)
ax.tick_params(axis="y", length=0)
ax.grid(axis="y", visible=False)
ax.set_xlabel("agreement with the human consensus (flag = error)")
jj = metrics["judge_vs_judge"]
ax.set_title(f"Two judges on the same {metrics['claims']} claims: judge-judge agreement {jj['agreement']:.2f}, κ {jj['kappa']:.2f}",
fontsize=9, color=INK, loc="left")
ax.legend(loc="lower left", frameon=False, fontsize=7.5)
fig.tight_layout()
return _write(fig, out, "fig_judge_agreement")
MODEL_WORDS = {"exp6": "EXP-6 ResNet-50", "probe": "BiomedCLIP, linear probe", "zero_shot": "BiomedCLIP, zero-shot"}
def _dots(ax, names, series, labels, xlabel, title, xlim):
"""Dot rows (a truncated axis needs dots, not bars): names top to bottom, one offset dot per series."""
n = len(series)
for k, (vals, lab) in enumerate(zip(series, labels)):
off = (n - 1 - 2 * k) * 0.12
ys = [len(names) - 1 - i + off for i in range(len(names))]
keep = [(y, v) for y, v in zip(ys, vals) if v is not None]
ax.plot([v for _, v in keep], [y for y, _ in keep], "o", ms=6.5, color=THREE[k % 3], mec="white", mew=1.1,
label=lab, zorder=2 + k, ls="none")
for y, v in keep:
ax.text(v + 0.02 * (xlim[1] - xlim[0]), y, f"{v:.3f}", va="center", ha="left", fontsize=6.5, color=INK_2)
ax.set_yticks(range(len(names)))
ax.set_yticklabels(list(reversed(names)), fontsize=7.5)
ax.tick_params(axis="y", length=0)
ax.grid(axis="y", visible=False)
ax.set_xlim(*xlim)
ax.set_xlabel(xlabel)
ax.set_title(title, fontsize=9, color=INK, loc="left")
def biomedclip_figure(metrics, isles, out):
"""Task D (EXP-82, and EXP-83 when present): scan AUC and specificity at 96% sensitivity for the three models on RSNA
validation; per-subtype scan AUC of the subtype probes beside EXP-6's 'any' score."""
plt = _plt()
models = [m for m in ("exp6", "probe", "zero_shot") if m in metrics["models"]]
subtypes = sorted(metrics.get("subtypes", {}))
fig, axes = plt.subplots(1, 2 if subtypes else 1, figsize=(9.6 if subtypes else 5.2, 3.3))
axes = np.atleast_1d(axes)
names = [MODEL_WORDS[m] for m in models]
if isles:
names = [f"{w}\nISLES'24 flags {isles['flagged'][m]}/{isles['scans']}" if m in isles["flagged"] else w
for w, m in zip(names, models)]
vals = [[metrics["models"][m][k] for m in models] for k in ("scan_auc", "specificity")]
lo = min(v for s in vals for v in s if v is not None)
_dots(axes[0], names, vals, ["scan AUC", "specificity at 96% sensitivity"], "",
f"Haemorrhage, RSNA validation ({metrics['val_scans']:,} scans)", (min(0.5, np.floor(lo * 20) / 20), 1.04))
axes[0].legend(loc="upper center", bbox_to_anchor=(0.5, -0.24), ncol=2, frameon=False, fontsize=7)
if subtypes:
s = metrics["subtypes"]
sv = [[s[t]["probe_scan_auc"] for t in subtypes], [s[t]["exp6_any_scan_auc"] for t in subtypes]]
lo = min(v for row in sv for v in row if v is not None)
_dots(axes[1], [f"{t} ({s[t]['scans_positive']})" for t in subtypes], sv,
["subtype probe on EXP-6's features", "EXP-6's 'any' score"], "scan AUC for that subtype",
"Per subtype (positive scans)", (min(0.5, np.floor(lo * 20) / 20), 1.04))
axes[1].legend(loc="upper center", bbox_to_anchor=(0.5, -0.24), ncol=2, frameon=False, fontsize=7)
fig.tight_layout()
return _write(fig, out, "fig_biomedclip")
if __name__ == "__main__":
main()