"""Live smoke test against a running instance of the API (TEST-04). Three phases, all against the real, auth-gated, two-role system: 1. Logs in as the seeded officer, creates a client, logs in as that client, and submits transactions across different simulated dates -- proving the full officer-creates-client / client-transacts loop works against the live server (not just TestClient). 2. Confirms role separation live: the officer sees the resulting alert in the cross-client queue; the client sees only their own alert; a second, unrelated client sees neither. 3. Runs the full TEST-03 hand-labeled fixture set (tests/fixtures/ labeled_transactions.json) through the officer-only `POST /predict` endpoint -- fixtures carry exact PaySim-shaped balances, so `/predict` (which accepts them directly) is the faithful way to replay them. Prints predicted vs. expected risk tier per transaction and a final accuracy summary. Exits non-zero if accuracy falls below the same bar enforced by tests/unit/test_labeled_fixtures.py, or if any role-separation check fails -- so this can double as a post-deploy CI gate. Usage: python -m scripts.test_live --base-url https://bank-fraud.hdev.rw python -m scripts.test_live --base-url http://127.0.0.1:8811 """ from __future__ import annotations import argparse import json import secrets import sys from datetime import datetime, timedelta, timezone from pathlib import Path import httpx from app.config import settings FIXTURES_PATH = Path("tests/fixtures/labeled_transactions.json") MIN_ACCURACY = 0.90 DEMO_EPOCH = datetime(2026, 1, 1, tzinfo=timezone.utc) def _label_for_tier(risk_tier: str) -> str: return "legit" if risk_tier == "low" else "fraud" def login(client: httpx.Client, email: str, password: str) -> dict: resp = client.post("/api/auth/login", json={"email": email, "password": password}) resp.raise_for_status() return {"cookies": {"session_token": resp.cookies["session_token"]}, "user": resp.json()} def create_client(client: httpx.Client, officer: dict, *, name: str, email: str, balance: float) -> dict: resp = client.post( "/api/officer/clients", json={ "name": name, "email": email, "temp_password": "SmokeTestPass123!", "starting_balance": balance, "account_type": "checking", }, cookies=officer["cookies"], ) resp.raise_for_status() body = resp.json() identity = login(client, email, "SmokeTestPass123!") identity["client_id"] = body["client"]["id"] return identity def run_role_flow(client: httpx.Client) -> bool: print("\n=== Phase 1+2: officer/client flow and role separation ===") ok = True run_id = secrets.token_hex(4) officer = login(client, settings.SEED_OFFICER_EMAIL, settings.SEED_OFFICER_PASSWORD) print(f" Logged in as officer: {officer['user']['email']}") client_a = create_client( client, officer, name="Smoke Test A", email=f"smoke.a.{run_id}@fakebankmail.com", balance=10000 ) client_b = create_client( client, officer, name="Smoke Test B", email=f"smoke.b.{run_id}@fakebankmail.com", balance=2000 ) print(f" Created client A (#{client_a['client_id']}) and client B (#{client_b['client_id']})") scenarios = [ ("legit small payment", "PAYMENT", 45.0, "MSMOKE001", DEMO_EPOCH + timedelta(days=1), "legit"), ("legit partial transfer", "TRANSFER", 2000.0, "C_SMOKE_DEST_1", DEMO_EPOCH + timedelta(days=5), "legit"), ("full-balance drain", "TRANSFER", 7955.0, "C_SMOKE_DEST_2", DEMO_EPOCH + timedelta(days=10), "fraud"), ] for label, type_, amount, dest, simulated_at, expected in scenarios: resp = client.post( "/api/client/transactions", json={"type": type_, "amount": amount, "name_dest": dest, "simulated_at": simulated_at.isoformat()}, cookies=client_a["cookies"], ) if resp.status_code != 200: print(f" FAIL [{label}]: {resp.status_code} {resp.text}") ok = False continue body = resp.json() predicted = _label_for_tier(body["risk_tier"]) match = "OK" if predicted == expected else "MISMATCH" if predicted != expected: ok = False print(f" [{match}] {label} (client A): predicted={body['risk_tier']} (p={body['probability']:.4f}) expected={expected}") # Role separation checks officer_alerts = client.get("/api/officer/alerts", cookies=officer["cookies"]).json() officer_sees_a = any( item["client_id"] == client_a["client_id"] for item in officer_alerts["items"] ) check = "OK" if officer_sees_a else "FAIL" if not officer_sees_a: ok = False print(f" [{check}] officer sees client A's alert in the cross-client queue") a_alerts = client.get("/api/client/alerts", cookies=client_a["cookies"]).json() check = "OK" if len(a_alerts) == 1 else "FAIL" if len(a_alerts) != 1: ok = False print(f" [{check}] client A sees exactly their own alert ({len(a_alerts)} found)") b_alerts = client.get("/api/client/alerts", cookies=client_b["cookies"]).json() check = "OK" if len(b_alerts) == 0 else "FAIL" if len(b_alerts) != 0: ok = False print(f" [{check}] client B sees none of client A's alerts ({len(b_alerts)} found)") b_sees_a_profile = client.get(f"/api/officer/clients/{client_a['client_id']}", cookies=client_b["cookies"]) check = "OK" if b_sees_a_profile.status_code == 403 else "FAIL" if b_sees_a_profile.status_code != 403: ok = False print(f" [{check}] client B is rejected (403) from officer-only client-detail route") return ok def run_fixture_replay(client: httpx.Client, officer: dict) -> bool: print("\n=== Phase 3: hand-labeled fixture replay via /predict (officer) ===") fixtures = json.loads(FIXTURES_PATH.read_text()) correct = 0 for fixture in fixtures: raw = {k: fixture[k] for k in [ "step", "type", "amount", "nameOrig", "oldbalanceOrg", "newbalanceOrig", "nameDest", "oldbalanceDest", "newbalanceDest", ]} resp = client.post("/api/predict", json=raw, cookies=officer["cookies"]) if resp.status_code != 200: print(f" FAIL [{fixture['id']}]: {resp.status_code} {resp.text}") continue body = resp.json() predicted_label = _label_for_tier(body["risk_tier"]) expected_label = fixture["expected_label"] is_match = predicted_label == expected_label correct += int(is_match) marker = "OK" if is_match else "MISMATCH" print( f" [{marker}] {fixture['id']:<30} predicted={body['risk_tier']:<7} " f"(p={body['probability']:.4f}) expected={expected_label}" ) accuracy = correct / len(fixtures) print(f"\nAccuracy: {correct}/{len(fixtures)} = {accuracy:.2%} (bar: {MIN_ACCURACY:.0%})") return accuracy >= MIN_ACCURACY def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--base-url", default="http://127.0.0.1:8811", help="Base URL of the running API (default: local dev server)", ) parser.add_argument("--timeout", type=float, default=30.0) args = parser.parse_args() with httpx.Client(base_url=args.base_url, timeout=args.timeout) as client: health_resp = client.get("/health") if health_resp.status_code != 200: print(f"FAIL: /health returned {health_resp.status_code}") sys.exit(1) print(f"Connected: {args.base_url} -- {health_resp.json()}") role_ok = run_role_flow(client) officer = login(client, settings.SEED_OFFICER_EMAIL, settings.SEED_OFFICER_PASSWORD) fixtures_ok = run_fixture_replay(client, officer) if role_ok and fixtures_ok: print("\nLive smoke test PASSED") sys.exit(0) else: print("\nLive smoke test FAILED") sys.exit(1) if __name__ == "__main__": main()