omnidiag / scripts /verify_live.py
yahyoha's picture
Claude Opus 5.5
Release: retire BRFSS diabetes (source deploy/v2-platform 8bbbfe9)
c816150
Raw History Blame Contribute Delete
6.15 kB
"""
Diff a deployed OmniDiag backend against the local environment, for every
demo patient in frontend/src/mockPatients.js.
backend/.venv/bin/python scripts/verify_live.py # live HF Space
backend/.venv/bin/python scripts/verify_live.py --url http://127.0.0.1:7860
backend/.venv/bin/python scripts/verify_live.py --save verify_live_result.json
The local side is the REAL FastAPI app called in-process (no server needed),
so both sides go through identical validation, caching and serialisation.
Compared per patient:
/predict prediction, diagnosis, confidence, inference_threshold
/explain every chart_data SHAP value, by feature name
/counterfactuals status, scenario count, best_achievable probability
Exit code 0 = identical within tolerance, 1 = differences.
Why this exists: the Space once resolved scikit-learn 1.7.2 (models saved
with 1.8/1.9) and served different probabilities from the same pickles (the
since-retired BRFSS diabetes module: Case C 63.7% local vs 49.4% live).
requirements.txt pins the numeric stack; this script is the check that the
pin holds. The demo patients are heart and NHANES since gate B6.
"""
import os
import sys
os.environ.setdefault("DATABASE_URL", "sqlite+aiosqlite:///:memory:")
import argparse
import json
import time
import urllib.error
import urllib.request
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
os.chdir(ROOT)
sys.path.insert(0, ROOT)
sys.path.insert(0, os.path.join(ROOT, "scripts"))
from golden_master import load_demo_patients # noqa: E402
LIVE = "https://yahyoha-omnidiag.hf.space"
TOL = 1e-6
ENDPOINTS = ("predict", "explain", "counterfactuals")
def remote_call(base, path, body, timeout=300):
req = urllib.request.Request(
base.rstrip("/") + path, data=json.dumps(body).encode(), method="POST",
headers={"Content-Type": "application/json"},
)
for attempt in range(4):
try:
with urllib.request.urlopen(req, timeout=timeout) as r:
return r.status, json.loads(r.read())
except urllib.error.HTTPError as e:
if e.code == 429 and attempt < 3: # rate limit: back off
time.sleep(20)
continue
try:
return e.code, json.loads(e.read())
except Exception:
return e.code, None
def summarise(endpoint, body):
"""The fields compared for one endpoint response."""
if body is None:
return None
if endpoint == "predict":
return {k: body.get(k) for k in ("prediction", "diagnosis", "confidence", "inference_threshold")}
if endpoint == "explain":
return {row["feature"]: row["shap_value"] for row in body.get("chart_data", [])}
best = body.get("best_achievable") or {}
return {
"status": body.get("status"),
"n_scenarios": len(body.get("counterfactuals") or []),
"best_probability": next(
(best[k] for k in ("new_probability_corrected", "new_probability", "probability")
if isinstance(best.get(k), (int, float))), None),
"baseline_probability": body.get("baseline_probability"),
}
def diff(a, b, path=""):
out = []
if isinstance(a, dict) and isinstance(b, dict):
for k in sorted(set(a) | set(b)):
if k not in a or k not in b:
out.append(f"{path}.{k}: only in {'local' if k in a else 'remote'}")
else:
out += diff(a[k], b[k], f"{path}.{k}")
elif isinstance(a, (int, float)) and isinstance(b, (int, float)) and not isinstance(a, bool):
if abs(a - b) > TOL:
out.append(f"{path}: local {a:.6g} vs remote {b:.6g} (|d|={abs(a - b):.3g})")
elif a != b:
out.append(f"{path}: local {a!r} vs remote {b!r}")
return out
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--url", default=LIVE)
ap.add_argument("--pause", type=float, default=2.2, help="seconds between remote calls (rate limit 30/min)")
ap.add_argument("--save")
args = ap.parse_args()
from fastapi.testclient import TestClient
from backend.main import app
import logging
logging.disable(logging.WARNING) # the app logs at DEBUG; keep the report readable
import warnings
warnings.filterwarnings("ignore")
patients = load_demo_patients()
report, n_diff = {}, 0
with TestClient(app) as local:
for disease, cases in patients.items():
for pid, data in cases:
row = {}
for ep in ENDPOINTS:
path = f"/api/v4/{disease}/{ep}"
lr = local.post(path, json=data)
rs, rb = remote_call(args.url, path, data)
time.sleep(args.pause)
l_sum = summarise(ep, lr.json()) if lr.status_code == 200 else {"http": lr.status_code}
r_sum = summarise(ep, rb) if rs == 200 else {"http": rs}
d = diff(l_sum, r_sum)
n_diff += len(d)
row[ep] = {"local": l_sum, "remote": r_sum, "diff": d}
report[f"{disease}/{pid}"] = row
p = row["predict"]
conf = lambda s: f"{s['confidence']*100:.2f}%" if s and "confidence" in s else str(s)
status = "OK " if not any(row[e]["diff"] for e in ENDPOINTS) else "DIFF"
print(f"{status} {disease:14s} {pid} predict local {conf(p['local'])} remote {conf(p['remote'])}"
f" | explain diffs {len(row['explain']['diff'])} | cf diffs {len(row['counterfactuals']['diff'])}")
for ep in ENDPOINTS:
for line in row[ep]["diff"][:6]:
print(f" {ep}{line}")
print(f"\n{len(report)} patients x {len(ENDPOINTS)} endpoints against {args.url}: "
f"{'IDENTICAL' if n_diff == 0 else f'{n_diff} difference(s)'} (tol {TOL})")
if args.save:
with open(args.save, "w") as f:
json.dump(report, f, indent=1)
sys.exit(0 if n_diff == 0 else 1)
if __name__ == "__main__":
main()