File size: 10,782 Bytes
98b213b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
"""Phase 5 task E (DEC-37): subgroup analysis from saved results, and SHAP rows 12 and 13 (surrogates).

Groupings (fixed before any count): age band (under 80, 80 and over), sex, centre, and onset-information pattern
(known: an onset-to-imaging time; wake-up; none: no onset time, last known well at most). A value the note gives as not
recorded is its own group. A group with fewer than MIN_CASES cases is reported with its counts only and marked too
small to conclude anything.

- **Vision, 129 development cases:** Vision A (EXP-27: Dice, absolute volume error, core on the wrong side of any of
  21/31/51/70 mL) and Vision B (EXP-33's contracts: the bleed flag at DEC-28, all presumed false alarms).
- **Reasoners, saved verified answers (rulebook arms, predicted vision; EXP-37, EXP-47):** hallucination rate and
  recommendation correctness per model, on the 150 synthetic scenarios (age band, sex, onset pattern) and on the 20
  development real cases (all four groupings; every group small).
- **Rows 12 and 13:** a gradient-boosted SURROGATE of "core on the wrong side of a cut-off" (12) and of "bleed flag" (13)
  from case features known at presentation (age, sex, centre, onset pattern, NIHSS, Tmax > 6 s volume, predicted core);
  out-of-fold AUC first, SHAP (`TreeExplainer`) only if its interval excludes 0.5.

Intervals: Wilson for counts, 95% case bootstrap (10,000, seed 20261104) for means and claim rates. No case ID is
returned; IDs stay in memory.
"""

import csv
import json
from pathlib import Path

import numpy as np

MIN_CASES = 10
SEED = 20261120
GROUPINGS = ("age band", "sex", "centre", "onset")
MODELS = ("claude-sonnet-5-5", "qwen3.5-27b", "gemma-3-27b", "medgemma-27b-text", "gemma-3-4b", "gemma-3-4b-tuned")


def _val(facts, key):
    f = facts.get(key)
    return None if f is None or f.type == "not_recorded" else f.value


def groups_from_facts(facts, centre=None):
    """{grouping: label} for one case's fact sheet (and centre, for real cases)."""
    age = _val(facts, "age")
    sex = _val(facts, "sex")
    if _val(facts, "wake_up_stroke"):
        onset = "wake-up"
    elif _val(facts, "onset_to_imaging") is not None:
        onset = "known"
    else:
        onset = "none"
    out = {"age band": "not recorded" if age is None else ("80 and over" if float(age) >= 80 else "under 80"),
           "sex": "not recorded" if sex is None else str(sex).lower(), "onset": onset}
    if centre is not None:
        out["centre"] = f"centre {centre}"
    return out


def wilson(k, n):
    from strokecopilot.vision.haemorrhage.stats import wilson as w
    return list(w(k, n)) if n else None


def boot_mean(x, n_boot=10000, seed=20261104):
    x = np.asarray(x, dtype=float)
    if len(x) < 2:
        return None
    idx = np.random.default_rng(seed).integers(0, len(x), size=(n_boot, len(x)))
    m = x[idx].mean(axis=1)
    return [float(np.percentile(m, 2.5)), float(np.percentile(m, 97.5))]


def vision_table(rows, groups):
    """rows: [{case, dice, abs_error, wrong_side, flagged}]; groups: {case: {grouping: label}}."""
    out = {}
    for g in GROUPINGS:
        labels = sorted({groups[r["case"]].get(g) for r in rows if groups[r["case"]].get(g) is not None})
        if not labels:
            continue
        out[g] = {}
        for lab in labels:
            sel = [r for r in rows if groups[r["case"]].get(g) == lab]
            n = len(sel)
            small = n < MIN_CASES
            wrong, flags = sum(r["wrong_side"] for r in sel), sum(r["flagged"] for r in sel)
            out[g][lab] = {"cases": n, "too_small": small,
                           "dice_mean": float(np.mean([r["dice"] for r in sel])),
                           "dice_ci": None if small else boot_mean([r["dice"] for r in sel]),
                           "abs_error_ml_mean": float(np.mean([r["abs_error"] for r in sel])),
                           "abs_error_ml_median": float(np.median([r["abs_error"] for r in sel])),
                           "abs_error_ci": None if small else boot_mean([r["abs_error"] for r in sel]),
                           "wrong_side": wrong, "wrong_side_ci": None if small else wilson(wrong, n),
                           "bleed_flags": flags, "bleed_flags_ci": None if small else wilson(flags, n)}
    return out


def reasoner_table(records_by_model, groups):
    """records_by_model: {model: [case records]}; groups: {case: {grouping: label}}. Pooled rates per group."""
    from strokecopilot.nlp.verifier import metrics as vm
    out = {}
    for model, records in records_by_model.items():
        out[model] = {}
        for g in GROUPINGS:
            labels = sorted({groups[r["case_id"]].get(g) for r in records if groups[r["case_id"]].get(g) is not None})
            if not labels:
                continue
            out[model][g] = {}
            for lab in labels:
                sel = [r for r in records if groups[r["case_id"]].get(g) == lab]
                s = vm.summarise(sel)
                out[model][g][lab] = {"cases": len(sel), "too_small": len(sel) < MIN_CASES,
                                      **{k: {"value": s[k]["value"], "ci95": s[k]["ci95"]}
                                         for k in ("hallucination_rate", "recommendation_correctness")}}
    return out


def surrogate(X, y, names, n_boot=2000):
    """Out-of-fold AUC (5 stratified folds) with a case-bootstrap interval; SHAP only if the interval excludes 0.5."""
    from sklearn.ensemble import GradientBoostingClassifier
    from sklearn.metrics import roc_auc_score
    from sklearn.model_selection import StratifiedKFold
    make = lambda: GradientBoostingClassifier(n_estimators=100, max_depth=2, learning_rate=0.05, subsample=0.8,
                                              random_state=SEED)
    oof = np.zeros(len(y))
    for train, test in StratifiedKFold(5, shuffle=True, random_state=SEED).split(X, y):
        oof[test] = make().fit(X[train], y[train]).predict_proba(X[test])[:, 1]
    rng = np.random.default_rng(SEED)
    boots = []
    for _ in range(n_boot):
        idx = rng.integers(0, len(y), len(y))
        if len(np.unique(y[idx])) == 2:
            boots.append(roc_auc_score(y[idx], oof[idx]))
    out = {"cases": int(len(y)), "positives": int(y.sum()), "auc": float(roc_auc_score(y, oof)),
           "ci95": [float(np.percentile(boots, 2.5)), float(np.percentile(boots, 97.5))], "explained": None}
    if out["ci95"][0] > 0.5:
        import shap
        sv = np.asarray(shap.TreeExplainer(make().fit(X, y)).shap_values(X))
        mag = np.abs(sv).mean(0)
        out["explained"] = {"share_of_attribution": dict(sorted(zip(names, (mag / mag.sum()).tolist()), key=lambda kv: -kv[1])),
                            "scale": "log-odds"}
    return out


def features(cases, groups, contracts, facts):
    """X for the surrogates: age, female, onset pattern, centre one-hot, NIHSS, Tmax > 6 s and predicted core volumes."""
    centres = sorted({groups[c].get("centre") for c in cases})
    names = ["age", "female", "onset known", "wake-up", "NIHSS", "Tmax > 6 s volume", "predicted core"] + centres
    X = []
    for c in cases:
        age = _val(facts[c], "age")
        nihss = _val(facts[c], "nihss")
        X.append([np.nan if age is None else float(age), float(groups[c]["sex"] == "female"),
                  float(groups[c]["onset"] == "known"), float(groups[c]["onset"] == "wake-up"),
                  np.nan if nihss is None else float(nihss), float(contracts[c]["penumbra_ml"] or 0),
                  float(contracts[c]["core_ml"] or 0)] + [float(groups[c].get("centre") == k) for k in centres])
    X = np.array(X)
    col_mean = np.nanmean(X, axis=0)
    return np.where(np.isnan(X), col_mean, X), names


def main(argv=None):
    import argparse

    from strokecopilot.config import local_drive_root
    from strokecopilot.data import isles24, splits
    from strokecopilot.nlp.verifier import scenarios as sc
    from strokecopilot.pipeline import compare as cp
    from strokecopilot.pipeline import heldout_analysis as ha
    from strokecopilot.vision.haemorrhage import isles_check as ic

    ap = argparse.ArgumentParser(description="Task E: subgroups, and SHAP rows 12-13")
    ap.add_argument("--out", default="runs/EXP-80_subgroups")
    args = ap.parse_args(argv)
    out = Path(args.out)
    if not (out / "config.json").exists():
        raise SystemExit("write the run's config.json first (plan §11)")
    drive = local_drive_root()
    folds = splits.load_folds()
    dev = list(splits.development_ids(folds))
    root = Path(__file__).resolve().parents[3] / "data" / "local" / isles24.SLIM_NAME
    facts = ha.load_facts(drive / "artifacts" / "isles24_notes_v1", dev)
    centre = {c: isles24.load_case(c, root, names=(), folds=folds).clinical.get("Center") for c in dev}
    groups = {c: groups_from_facts(facts[c], centre[c]) for c in dev}
    contracts = ha.load_contracts(drive / "artifacts" / "vision_outputs_v1", dev)
    per_case = {r["case"]: r for r in csv.DictReader(open(drive / "runs" / "EXP-27_visionA_error_followups" / "per_case.csv"))}
    crosses = lambda r: int(any(r[f"nnunet_crosses_{c}"] in ("True", "true", "1") for c in (21, 31, 51, 70)))
    rows = [{"case": c, "dice": float(per_case[c]["nnunet_dice"]), "abs_error": float(per_case[c]["nnunet_abs_error_ml"]),
             "wrong_side": crosses(per_case[c]), "flagged": int(contracts[c]["haemorrhage"]["present"])} for c in dev]
    result = {"experiment": "EXP-80", "min_cases": MIN_CASES, "vision": vision_table(rows, groups)}

    scen = sc.generate()
    s_groups = {s.id: groups_from_facts(s.facts) for s in scen}
    root_of = lambda m: Path("runs/EXP-47_small_reasoner" if m.startswith("gemma-3-4b") else "runs/EXP-37_paid_runs")
    scen_recs = {m: cp.case_records(root_of(m) / m / "scenarios_rulebook") for m in MODELS}
    real_recs = {m: cp.case_records(root_of(m) / m / "real_predicted") for m in MODELS}
    result["reasoners_scenarios"] = reasoner_table(scen_recs, s_groups)
    result["reasoners_real"] = reasoner_table(real_recs, {c: groups[c] for c in dev})

    X, names = features(dev, groups, contracts, facts)
    result["row12_surrogate_vision_a_wrong_side"] = surrogate(X, np.array([r["wrong_side"] for r in rows]), names)
    result["row13_surrogate_vision_b_flag"] = surrogate(X, np.array([r["flagged"] for r in rows]), names)
    text = json.dumps(result, indent=1, default=float)
    if ic.ids_in(text, dev + list(folds["heldout"])):
        raise ValueError("A case ID reached metrics.json. Nothing was saved.")
    (out / "metrics.json").write_text(text)
    print(json.dumps({k: result[k] for k in ("row12_surrogate_vision_a_wrong_side", "row13_surrogate_vision_b_flag")},
                     indent=1))


if __name__ == "__main__":
    main()