Spaces:
Running
Running
File size: 10,782 Bytes
98b213b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 | """Phase 5 task E (DEC-37): subgroup analysis from saved results, and SHAP rows 12 and 13 (surrogates).
Groupings (fixed before any count): age band (under 80, 80 and over), sex, centre, and onset-information pattern
(known: an onset-to-imaging time; wake-up; none: no onset time, last known well at most). A value the note gives as not
recorded is its own group. A group with fewer than MIN_CASES cases is reported with its counts only and marked too
small to conclude anything.
- **Vision, 129 development cases:** Vision A (EXP-27: Dice, absolute volume error, core on the wrong side of any of
21/31/51/70 mL) and Vision B (EXP-33's contracts: the bleed flag at DEC-28, all presumed false alarms).
- **Reasoners, saved verified answers (rulebook arms, predicted vision; EXP-37, EXP-47):** hallucination rate and
recommendation correctness per model, on the 150 synthetic scenarios (age band, sex, onset pattern) and on the 20
development real cases (all four groupings; every group small).
- **Rows 12 and 13:** a gradient-boosted SURROGATE of "core on the wrong side of a cut-off" (12) and of "bleed flag" (13)
from case features known at presentation (age, sex, centre, onset pattern, NIHSS, Tmax > 6 s volume, predicted core);
out-of-fold AUC first, SHAP (`TreeExplainer`) only if its interval excludes 0.5.
Intervals: Wilson for counts, 95% case bootstrap (10,000, seed 20261104) for means and claim rates. No case ID is
returned; IDs stay in memory.
"""
import csv
import json
from pathlib import Path
import numpy as np
MIN_CASES = 10
SEED = 20261120
GROUPINGS = ("age band", "sex", "centre", "onset")
MODELS = ("claude-sonnet-5-5", "qwen3.5-27b", "gemma-3-27b", "medgemma-27b-text", "gemma-3-4b", "gemma-3-4b-tuned")
def _val(facts, key):
f = facts.get(key)
return None if f is None or f.type == "not_recorded" else f.value
def groups_from_facts(facts, centre=None):
"""{grouping: label} for one case's fact sheet (and centre, for real cases)."""
age = _val(facts, "age")
sex = _val(facts, "sex")
if _val(facts, "wake_up_stroke"):
onset = "wake-up"
elif _val(facts, "onset_to_imaging") is not None:
onset = "known"
else:
onset = "none"
out = {"age band": "not recorded" if age is None else ("80 and over" if float(age) >= 80 else "under 80"),
"sex": "not recorded" if sex is None else str(sex).lower(), "onset": onset}
if centre is not None:
out["centre"] = f"centre {centre}"
return out
def wilson(k, n):
from strokecopilot.vision.haemorrhage.stats import wilson as w
return list(w(k, n)) if n else None
def boot_mean(x, n_boot=10000, seed=20261104):
x = np.asarray(x, dtype=float)
if len(x) < 2:
return None
idx = np.random.default_rng(seed).integers(0, len(x), size=(n_boot, len(x)))
m = x[idx].mean(axis=1)
return [float(np.percentile(m, 2.5)), float(np.percentile(m, 97.5))]
def vision_table(rows, groups):
"""rows: [{case, dice, abs_error, wrong_side, flagged}]; groups: {case: {grouping: label}}."""
out = {}
for g in GROUPINGS:
labels = sorted({groups[r["case"]].get(g) for r in rows if groups[r["case"]].get(g) is not None})
if not labels:
continue
out[g] = {}
for lab in labels:
sel = [r for r in rows if groups[r["case"]].get(g) == lab]
n = len(sel)
small = n < MIN_CASES
wrong, flags = sum(r["wrong_side"] for r in sel), sum(r["flagged"] for r in sel)
out[g][lab] = {"cases": n, "too_small": small,
"dice_mean": float(np.mean([r["dice"] for r in sel])),
"dice_ci": None if small else boot_mean([r["dice"] for r in sel]),
"abs_error_ml_mean": float(np.mean([r["abs_error"] for r in sel])),
"abs_error_ml_median": float(np.median([r["abs_error"] for r in sel])),
"abs_error_ci": None if small else boot_mean([r["abs_error"] for r in sel]),
"wrong_side": wrong, "wrong_side_ci": None if small else wilson(wrong, n),
"bleed_flags": flags, "bleed_flags_ci": None if small else wilson(flags, n)}
return out
def reasoner_table(records_by_model, groups):
"""records_by_model: {model: [case records]}; groups: {case: {grouping: label}}. Pooled rates per group."""
from strokecopilot.nlp.verifier import metrics as vm
out = {}
for model, records in records_by_model.items():
out[model] = {}
for g in GROUPINGS:
labels = sorted({groups[r["case_id"]].get(g) for r in records if groups[r["case_id"]].get(g) is not None})
if not labels:
continue
out[model][g] = {}
for lab in labels:
sel = [r for r in records if groups[r["case_id"]].get(g) == lab]
s = vm.summarise(sel)
out[model][g][lab] = {"cases": len(sel), "too_small": len(sel) < MIN_CASES,
**{k: {"value": s[k]["value"], "ci95": s[k]["ci95"]}
for k in ("hallucination_rate", "recommendation_correctness")}}
return out
def surrogate(X, y, names, n_boot=2000):
"""Out-of-fold AUC (5 stratified folds) with a case-bootstrap interval; SHAP only if the interval excludes 0.5."""
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import StratifiedKFold
make = lambda: GradientBoostingClassifier(n_estimators=100, max_depth=2, learning_rate=0.05, subsample=0.8,
random_state=SEED)
oof = np.zeros(len(y))
for train, test in StratifiedKFold(5, shuffle=True, random_state=SEED).split(X, y):
oof[test] = make().fit(X[train], y[train]).predict_proba(X[test])[:, 1]
rng = np.random.default_rng(SEED)
boots = []
for _ in range(n_boot):
idx = rng.integers(0, len(y), len(y))
if len(np.unique(y[idx])) == 2:
boots.append(roc_auc_score(y[idx], oof[idx]))
out = {"cases": int(len(y)), "positives": int(y.sum()), "auc": float(roc_auc_score(y, oof)),
"ci95": [float(np.percentile(boots, 2.5)), float(np.percentile(boots, 97.5))], "explained": None}
if out["ci95"][0] > 0.5:
import shap
sv = np.asarray(shap.TreeExplainer(make().fit(X, y)).shap_values(X))
mag = np.abs(sv).mean(0)
out["explained"] = {"share_of_attribution": dict(sorted(zip(names, (mag / mag.sum()).tolist()), key=lambda kv: -kv[1])),
"scale": "log-odds"}
return out
def features(cases, groups, contracts, facts):
"""X for the surrogates: age, female, onset pattern, centre one-hot, NIHSS, Tmax > 6 s and predicted core volumes."""
centres = sorted({groups[c].get("centre") for c in cases})
names = ["age", "female", "onset known", "wake-up", "NIHSS", "Tmax > 6 s volume", "predicted core"] + centres
X = []
for c in cases:
age = _val(facts[c], "age")
nihss = _val(facts[c], "nihss")
X.append([np.nan if age is None else float(age), float(groups[c]["sex"] == "female"),
float(groups[c]["onset"] == "known"), float(groups[c]["onset"] == "wake-up"),
np.nan if nihss is None else float(nihss), float(contracts[c]["penumbra_ml"] or 0),
float(contracts[c]["core_ml"] or 0)] + [float(groups[c].get("centre") == k) for k in centres])
X = np.array(X)
col_mean = np.nanmean(X, axis=0)
return np.where(np.isnan(X), col_mean, X), names
def main(argv=None):
import argparse
from strokecopilot.config import local_drive_root
from strokecopilot.data import isles24, splits
from strokecopilot.nlp.verifier import scenarios as sc
from strokecopilot.pipeline import compare as cp
from strokecopilot.pipeline import heldout_analysis as ha
from strokecopilot.vision.haemorrhage import isles_check as ic
ap = argparse.ArgumentParser(description="Task E: subgroups, and SHAP rows 12-13")
ap.add_argument("--out", default="runs/EXP-80_subgroups")
args = ap.parse_args(argv)
out = Path(args.out)
if not (out / "config.json").exists():
raise SystemExit("write the run's config.json first (plan §11)")
drive = local_drive_root()
folds = splits.load_folds()
dev = list(splits.development_ids(folds))
root = Path(__file__).resolve().parents[3] / "data" / "local" / isles24.SLIM_NAME
facts = ha.load_facts(drive / "artifacts" / "isles24_notes_v1", dev)
centre = {c: isles24.load_case(c, root, names=(), folds=folds).clinical.get("Center") for c in dev}
groups = {c: groups_from_facts(facts[c], centre[c]) for c in dev}
contracts = ha.load_contracts(drive / "artifacts" / "vision_outputs_v1", dev)
per_case = {r["case"]: r for r in csv.DictReader(open(drive / "runs" / "EXP-27_visionA_error_followups" / "per_case.csv"))}
crosses = lambda r: int(any(r[f"nnunet_crosses_{c}"] in ("True", "true", "1") for c in (21, 31, 51, 70)))
rows = [{"case": c, "dice": float(per_case[c]["nnunet_dice"]), "abs_error": float(per_case[c]["nnunet_abs_error_ml"]),
"wrong_side": crosses(per_case[c]), "flagged": int(contracts[c]["haemorrhage"]["present"])} for c in dev]
result = {"experiment": "EXP-80", "min_cases": MIN_CASES, "vision": vision_table(rows, groups)}
scen = sc.generate()
s_groups = {s.id: groups_from_facts(s.facts) for s in scen}
root_of = lambda m: Path("runs/EXP-47_small_reasoner" if m.startswith("gemma-3-4b") else "runs/EXP-37_paid_runs")
scen_recs = {m: cp.case_records(root_of(m) / m / "scenarios_rulebook") for m in MODELS}
real_recs = {m: cp.case_records(root_of(m) / m / "real_predicted") for m in MODELS}
result["reasoners_scenarios"] = reasoner_table(scen_recs, s_groups)
result["reasoners_real"] = reasoner_table(real_recs, {c: groups[c] for c in dev})
X, names = features(dev, groups, contracts, facts)
result["row12_surrogate_vision_a_wrong_side"] = surrogate(X, np.array([r["wrong_side"] for r in rows]), names)
result["row13_surrogate_vision_b_flag"] = surrogate(X, np.array([r["flagged"] for r in rows]), names)
text = json.dumps(result, indent=1, default=float)
if ic.ids_in(text, dev + list(folds["heldout"])):
raise ValueError("A case ID reached metrics.json. Nothing was saved.")
(out / "metrics.json").write_text(text)
print(json.dumps({k: result[k] for k in ("row12_surrogate_vision_a_wrong_side", "row13_surrogate_vision_b_flag")},
indent=1))
if __name__ == "__main__":
main()
|