Download scripts/verify_live.py from yahyoha/omnidiag: direct link, hf CLI and curl.
- Browser
- Download file 6.15 kB
-
https://huggingface.co/spaces/yahyoha/omnidiag/resolve/main/scripts/verify_live.py
- Command line
-
hf download hf://spaces/yahyoha/omnidiag/scripts/verify_live.py
-
curl -L -o verify_live.py https://huggingface.co/spaces/yahyoha/omnidiag/resolve/main/scripts/verify_live.py
6.15 kB
| """ | |
| Diff a deployed OmniDiag backend against the local environment, for every | |
| demo patient in frontend/src/mockPatients.js. | |
| backend/.venv/bin/python scripts/verify_live.py # live HF Space | |
| backend/.venv/bin/python scripts/verify_live.py --url http://127.0.0.1:7860 | |
| backend/.venv/bin/python scripts/verify_live.py --save verify_live_result.json | |
| The local side is the REAL FastAPI app called in-process (no server needed), | |
| so both sides go through identical validation, caching and serialisation. | |
| Compared per patient: | |
| /predict prediction, diagnosis, confidence, inference_threshold | |
| /explain every chart_data SHAP value, by feature name | |
| /counterfactuals status, scenario count, best_achievable probability | |
| Exit code 0 = identical within tolerance, 1 = differences. | |
| Why this exists: the Space once resolved scikit-learn 1.7.2 (models saved | |
| with 1.8/1.9) and served different probabilities from the same pickles (the | |
| since-retired BRFSS diabetes module: Case C 63.7% local vs 49.4% live). | |
| requirements.txt pins the numeric stack; this script is the check that the | |
| pin holds. The demo patients are heart and NHANES since gate B6. | |
| """ | |
| import os | |
| import sys | |
| os.environ.setdefault("DATABASE_URL", "sqlite+aiosqlite:///:memory:") | |
| import argparse | |
| import json | |
| import time | |
| import urllib.error | |
| import urllib.request | |
| ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| os.chdir(ROOT) | |
| sys.path.insert(0, ROOT) | |
| sys.path.insert(0, os.path.join(ROOT, "scripts")) | |
| from golden_master import load_demo_patients # noqa: E402 | |
| LIVE = "https://yahyoha-omnidiag.hf.space" | |
| TOL = 1e-6 | |
| ENDPOINTS = ("predict", "explain", "counterfactuals") | |
| def remote_call(base, path, body, timeout=300): | |
| req = urllib.request.Request( | |
| base.rstrip("/") + path, data=json.dumps(body).encode(), method="POST", | |
| headers={"Content-Type": "application/json"}, | |
| ) | |
| for attempt in range(4): | |
| try: | |
| with urllib.request.urlopen(req, timeout=timeout) as r: | |
| return r.status, json.loads(r.read()) | |
| except urllib.error.HTTPError as e: | |
| if e.code == 429 and attempt < 3: # rate limit: back off | |
| time.sleep(20) | |
| continue | |
| try: | |
| return e.code, json.loads(e.read()) | |
| except Exception: | |
| return e.code, None | |
| def summarise(endpoint, body): | |
| """The fields compared for one endpoint response.""" | |
| if body is None: | |
| return None | |
| if endpoint == "predict": | |
| return {k: body.get(k) for k in ("prediction", "diagnosis", "confidence", "inference_threshold")} | |
| if endpoint == "explain": | |
| return {row["feature"]: row["shap_value"] for row in body.get("chart_data", [])} | |
| best = body.get("best_achievable") or {} | |
| return { | |
| "status": body.get("status"), | |
| "n_scenarios": len(body.get("counterfactuals") or []), | |
| "best_probability": next( | |
| (best[k] for k in ("new_probability_corrected", "new_probability", "probability") | |
| if isinstance(best.get(k), (int, float))), None), | |
| "baseline_probability": body.get("baseline_probability"), | |
| } | |
| def diff(a, b, path=""): | |
| out = [] | |
| if isinstance(a, dict) and isinstance(b, dict): | |
| for k in sorted(set(a) | set(b)): | |
| if k not in a or k not in b: | |
| out.append(f"{path}.{k}: only in {'local' if k in a else 'remote'}") | |
| else: | |
| out += diff(a[k], b[k], f"{path}.{k}") | |
| elif isinstance(a, (int, float)) and isinstance(b, (int, float)) and not isinstance(a, bool): | |
| if abs(a - b) > TOL: | |
| out.append(f"{path}: local {a:.6g} vs remote {b:.6g} (|d|={abs(a - b):.3g})") | |
| elif a != b: | |
| out.append(f"{path}: local {a!r} vs remote {b!r}") | |
| return out | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--url", default=LIVE) | |
| ap.add_argument("--pause", type=float, default=2.2, help="seconds between remote calls (rate limit 30/min)") | |
| ap.add_argument("--save") | |
| args = ap.parse_args() | |
| from fastapi.testclient import TestClient | |
| from backend.main import app | |
| import logging | |
| logging.disable(logging.WARNING) # the app logs at DEBUG; keep the report readable | |
| import warnings | |
| warnings.filterwarnings("ignore") | |
| patients = load_demo_patients() | |
| report, n_diff = {}, 0 | |
| with TestClient(app) as local: | |
| for disease, cases in patients.items(): | |
| for pid, data in cases: | |
| row = {} | |
| for ep in ENDPOINTS: | |
| path = f"/api/v4/{disease}/{ep}" | |
| lr = local.post(path, json=data) | |
| rs, rb = remote_call(args.url, path, data) | |
| time.sleep(args.pause) | |
| l_sum = summarise(ep, lr.json()) if lr.status_code == 200 else {"http": lr.status_code} | |
| r_sum = summarise(ep, rb) if rs == 200 else {"http": rs} | |
| d = diff(l_sum, r_sum) | |
| n_diff += len(d) | |
| row[ep] = {"local": l_sum, "remote": r_sum, "diff": d} | |
| report[f"{disease}/{pid}"] = row | |
| p = row["predict"] | |
| conf = lambda s: f"{s['confidence']*100:.2f}%" if s and "confidence" in s else str(s) | |
| status = "OK " if not any(row[e]["diff"] for e in ENDPOINTS) else "DIFF" | |
| print(f"{status} {disease:14s} {pid} predict local {conf(p['local'])} remote {conf(p['remote'])}" | |
| f" | explain diffs {len(row['explain']['diff'])} | cf diffs {len(row['counterfactuals']['diff'])}") | |
| for ep in ENDPOINTS: | |
| for line in row[ep]["diff"][:6]: | |
| print(f" {ep}{line}") | |
| print(f"\n{len(report)} patients x {len(ENDPOINTS)} endpoints against {args.url}: " | |
| f"{'IDENTICAL' if n_diff == 0 else f'{n_diff} difference(s)'} (tol {TOL})") | |
| if args.save: | |
| with open(args.save, "w") as f: | |
| json.dump(report, f, indent=1) | |
| sys.exit(0 if n_diff == 0 else 1) | |
| if __name__ == "__main__": | |
| main() | |