File size: 10,754 Bytes
2d82128 c816150 2d82128 c816150 2d82128 c816150 2d82128 c816150 2d82128 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 | #!/usr/bin/env python3
"""
Build the frozen drift reference profile for each disease.
WHY A PROFILE AND NOT THE CSV AT RUNTIME
----------------------------------------
1. PSI needs bin edges that belong to the reference. Re-binning against each
incoming sample lets the reference move with the present, and a PSI computed
that way is not a number that means anything.
2. A reference read from a CSV changes whenever the CSV changes, silently. The
profile carries the source's sha256, so a changed source is an explicit
failure rather than drift that appears out of nowhere.
3. Reading a training CSV on every drift run to recompute a fixed distribution
is waste.
The profile is committed, and rebuilt with --verify at image build time: if the
regenerated profile differs from the committed one, the build fails. Same shape
as scripts/regen_heart_importance.py (the F0-1 pattern).
HEART: THE 920 AS TRAINED, IN THE SPACE THE MODEL READS
-------------------------------------------------------
The profile is built from `encode_for_training(...)`, not from the raw CSV
columns, and this is not a detail. The chest-pain coding is INVERTED between the
two entry points:
training CP_MAP_UCI_RAW = {ASY: 3, NAP: 2, ATA: 1, TA: 0}
inference CP_MAP_CLINICAL = {TA: 3, ATA: 2, NAP: 1, ASY: 0}
So a reference built on the raw string `ChestPainType` and compared against API
rows would be comparing opposite meanings, and would report confident drift on
a population that had not moved at all. In encoded space both sides are
`cp_anginal`, and the comparison is between like and like.
Only the seven inputs the model reads are monitored. The other four are profiled
and reported separately: blanking any of them changes the decision for exactly
zero of the 920 patients (Gate 8.3), so letting them raise the drift share would
raise alarms nothing can act on.
OTHER MODULES: NOT BUILT HERE
-----------------------------
This script builds the heart profile only. The retired BRFSS diabetes profile
is archived under archive/post_expo_2026-10/models/diabetes/ and is not rebuilt
or verified at image build. The NHANES dysglycaemia
module's profile is built in the research repository from raw survey files that
are not committed (F9-17), and says so in its own `source` block; adding that
module here would make every image build depend on files it does not have.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import sys
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional
import numpy as np
import pandas as pd
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from backend.monitoring.drift_stats import ( # noqa: E402
DEFAULT_ALPHA, MIN_CURRENT_ROWS, PSI_MAJOR, PSI_MINOR,
bin_counts, quantile_edges,
)
HEART_CSV = ROOT / "data/heart_disease/processed/uci_heart_by_site.csv"
N_BINS = 10
def sha256(path: Path) -> str:
h = hashlib.sha256()
with open(path, "rb") as fh:
for chunk in iter(lambda: fh.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def numeric_profile(values: pd.Series) -> Dict[str, Any]:
edges = quantile_edges(values.tolist(), N_BINS)
counts = bin_counts(values.tolist(), edges)
# The reference SAMPLE, run-length encoded. PSI needs the bins; KS needs the
# sample, and every numeric feature here has at most 216 distinct values, so
# this is exact rather than an approximated CDF.
finite = values.dropna()
uniq, cnt = np.unique(finite.to_numpy(dtype=float), return_counts=True)
return {"kind": "numeric", "edges": [None if not np.isfinite(e) else float(e) for e in edges],
"counts": counts, "n": int(len(values)),
"n_missing": int(values.isna().sum()),
"ecdf": {"values": [float(x) for x in uniq], "counts": [int(x) for x in cnt]},
"note": "the last bin counts missing values; `ecdf` is the exact reference sample, "
"run-length encoded, for the KS test"}
def categorical_profile(values: pd.Series) -> Dict[str, Any]:
cats = Counter("__missing__" if pd.isna(v) else str(v) for v in values)
order = sorted(cats)
return {"kind": "categorical", "categories": order,
"counts": [cats[c] for c in order], "n": int(len(values)),
"n_missing": int(cats.get("__missing__", 0))}
def heart_profile(csv_path: Optional[Path] = None) -> Dict[str, Any]:
from backend.heart_glm.stack import (
CP_MAP_CLINICAL, CP_MAP_UCI_RAW, MODEL_FEATURES, encode_for_training,
)
csv_path = csv_path or HEART_CSV
frame = pd.read_csv(csv_path)
encoded = encode_for_training(frame)
# `cp_anginal` is a 0-3 count and `Sex_m` is 0/1: both are categorical here.
# A KS test on four ordered levels answers a question nobody asked.
kinds = {"Age": "numeric", "RestingBP": "numeric", "Cholesterol": "numeric",
"Sex_m": "categorical", "cp_anginal": "categorical",
"FastingBS_cat": "categorical", "RestingECG": "categorical"}
assert set(kinds) == set(MODEL_FEATURES), "monitored set must be exactly the model's features"
monitored = {
f: (numeric_profile(encoded[f]) if kinds[f] == "numeric"
else categorical_profile(encoded[f]))
for f in MODEL_FEATURES
}
# Profiled but deliberately outside drift_share (Gate 8.3: zero decisions move)
unread = {}
for f in ("MaxHR", "Oldpeak", "ExerciseAngina", "ST_Slope"):
col = frame[f]
unread[f] = (numeric_profile(pd.to_numeric(col, errors="coerce"))
if f in ("MaxHR", "Oldpeak") else categorical_profile(col))
return {
"disease": "heart_disease",
"space": "encoded — backend.heart_glm.stack.encode_for_training / encode_for_inference",
"space_reason": ("the chest-pain coding is inverted between training and the API "
f"(training {CP_MAP_UCI_RAW}, inference {CP_MAP_CLINICAL}), so a "
"comparison on the raw string would compare opposite meanings"),
"source": {"path": str(csv_path.relative_to(ROOT)) if ROOT in csv_path.parents
else str(csv_path),
"sha256": sha256(csv_path), "rows": int(len(frame)),
"note": "the 920 rows the shipped model is fitted on, same file and same "
"sha256 as the build run records"},
"reweighting": None,
"monitored": monitored,
"profiled_but_not_monitored": unread,
"not_monitored_reason": ("the model does not read these four; blanking any of them "
"changes the decision for 0 of 920 patients (Gate 8.3), so "
"they cannot move a decision and must not move drift_share"),
}
def canonical(profile: Dict[str, Any]) -> str:
"""The profile without its timestamp, for comparing two builds."""
return json.dumps({k: v for k, v in profile.items() if k != "built_at"},
sort_keys=True, separators=(",", ":"))
def _rule() -> Dict[str, Any]:
return {"alpha": DEFAULT_ALPHA, "psi_minor": PSI_MINOR, "psi_major": PSI_MAJOR,
"min_current_rows": MIN_CURRENT_ROWS,
"flag": "Holm-corrected p < alpha AND PSI >= psi_major",
"not_measured": "output/label drift — no follow-up catheterisation result exists "
"for any patient in this system"}
def build_profiles(sources: Optional[Dict[str, Path]] = None) -> Dict[str, Dict[str, Any]]:
"""Build the heart profile. `sources` lets a test point at a tampered CSV
without touching the real one -- see verify_profiles() and its test."""
src = sources or {}
built = {"heart_disease": heart_profile(src.get("heart_disease"))}
rule = _rule()
for profile in built.values():
profile["rule"] = rule
profile["built_at"] = datetime.now(timezone.utc).isoformat(timespec="seconds")
return built
def verify_profiles(built: Dict[str, Dict[str, Any]],
profile_paths: Optional[Dict[str, Path]] = None) -> List[str]:
"""Compare freshly built profiles against the committed ones.
Returns the list of failure messages -- empty means everything matched. This
is what --verify calls, and what its own test calls directly against a
tampered source: the falsifying case for "the reference is anchored to this
exact training data" (reviewer protocol rule 7) is that a changed CSV without
a rebuilt profile must fail here, not just that a matching one passes.
"""
failures = []
for disease, profile in built.items():
out = (profile_paths or {}).get(
disease, ROOT / "models" / disease / "drift_reference.json")
if not out.exists():
failures.append(f"{disease}: {out} is missing")
continue
committed = json.loads(out.read_text())
if canonical(committed) != canonical(profile):
failures.append(f"{disease}: the rebuilt profile differs from the committed one ({out})")
return failures
def write_profiles(built: Dict[str, Dict[str, Any]],
profile_paths: Optional[Dict[str, Path]] = None) -> None:
for disease, profile in built.items():
out = (profile_paths or {}).get(
disease, ROOT / "models" / disease / "drift_reference.json")
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(profile, indent=1) + "\n")
print(f"{disease:14s}: wrote {out.relative_to(ROOT) if ROOT in out.parents else out} "
f"({len(profile['monitored'])} monitored features)")
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--verify", action="store_true",
help="rebuild and compare against the committed profiles; "
"exit 1 on any difference")
args = ap.parse_args()
built = build_profiles()
if args.verify:
failures = verify_profiles(built)
for disease, profile in built.items():
if not any(disease in f for f in failures):
print(f"{disease:14s}: profile matches "
f"({len(profile['monitored'])} monitored features, "
f"source sha256 {profile['source']['sha256'][:12]}…)")
if failures:
print("\nFAILED:", file=sys.stderr)
for f in failures:
print(" -", f, file=sys.stderr)
return 1
return 0
write_profiles(built)
return 0
if __name__ == "__main__":
raise SystemExit(main())
|