scripts/test_live.py

"""Live smoke test against a running instance of the API (TEST-04).

Three phases, all against the real, auth-gated, two-role system:

1. Logs in as the seeded officer, creates a client, logs in as that client,
   and submits transactions across different simulated dates -- proving the
   full officer-creates-client / client-transacts loop works against the
   live server (not just TestClient).
2. Confirms role separation live: the officer sees the resulting alert in
   the cross-client queue; the client sees only their own alert; a second,
   unrelated client sees neither.
3. Runs the full TEST-03 hand-labeled fixture set (tests/fixtures/
   labeled_transactions.json) through the officer-only `POST /predict`
   endpoint -- fixtures carry exact PaySim-shaped balances, so `/predict`
   (which accepts them directly) is the faithful way to replay them.

Prints predicted vs. expected risk tier per transaction and a final
accuracy summary. Exits non-zero if accuracy falls below the same bar
enforced by tests/unit/test_labeled_fixtures.py, or if any role-separation
check fails -- so this can double as a post-deploy CI gate.

Usage:
    python -m scripts.test_live --base-url https://bank-fraud.hdev.rw
    python -m scripts.test_live --base-url http://127.0.0.1:8811
"""

from __future__ import annotations

import argparse
import json
import secrets
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path

import httpx

from app.config import settings

FIXTURES_PATH = Path("tests/fixtures/labeled_transactions.json")
MIN_ACCURACY = 0.90

DEMO_EPOCH = datetime(2026, 1, 1, tzinfo=timezone.utc)


def _label_for_tier(risk_tier: str) -> str:
    return "legit" if risk_tier == "low" else "fraud"


def login(client: httpx.Client, email: str, password: str) -> dict:
    resp = client.post("/api/auth/login", json={"email": email, "password": password})
    resp.raise_for_status()
    return {"cookies": {"session_token": resp.cookies["session_token"]}, "user": resp.json()}


def create_client(client: httpx.Client, officer: dict, *, name: str, email: str, balance: float) -> dict:
    resp = client.post(
        "/api/officer/clients",
        json={
            "name": name,
            "email": email,
            "temp_password": "SmokeTestPass123!",
            "starting_balance": balance,
            "account_type": "checking",
        },
        cookies=officer["cookies"],
    )
    resp.raise_for_status()
    body = resp.json()
    identity = login(client, email, "SmokeTestPass123!")
    identity["client_id"] = body["client"]["id"]
    return identity


def run_role_flow(client: httpx.Client) -> bool:
    print("\n=== Phase 1+2: officer/client flow and role separation ===")
    ok = True
    run_id = secrets.token_hex(4)

    officer = login(client, settings.SEED_OFFICER_EMAIL, settings.SEED_OFFICER_PASSWORD)
    print(f"  Logged in as officer: {officer['user']['email']}")

    client_a = create_client(
        client, officer, name="Smoke Test A", email=f"smoke.a.{run_id}@fakebankmail.com", balance=10000
    )
    client_b = create_client(
        client, officer, name="Smoke Test B", email=f"smoke.b.{run_id}@fakebankmail.com", balance=2000
    )
    print(f"  Created client A (#{client_a['client_id']}) and client B (#{client_b['client_id']})")

    scenarios = [
        ("legit small payment", "PAYMENT", 45.0, "MSMOKE001", DEMO_EPOCH + timedelta(days=1), "legit"),
        ("legit partial transfer", "TRANSFER", 2000.0, "C_SMOKE_DEST_1", DEMO_EPOCH + timedelta(days=5), "legit"),
        ("full-balance drain", "TRANSFER", 7955.0, "C_SMOKE_DEST_2", DEMO_EPOCH + timedelta(days=10), "fraud"),
    ]
    for label, type_, amount, dest, simulated_at, expected in scenarios:
        resp = client.post(
            "/api/client/transactions",
            json={"type": type_, "amount": amount, "name_dest": dest, "simulated_at": simulated_at.isoformat()},
            cookies=client_a["cookies"],
        )
        if resp.status_code != 200:
            print(f"  FAIL [{label}]: {resp.status_code} {resp.text}")
            ok = False
            continue
        body = resp.json()
        predicted = _label_for_tier(body["risk_tier"])
        match = "OK" if predicted == expected else "MISMATCH"
        if predicted != expected:
            ok = False
        print(f"  [{match}] {label} (client A): predicted={body['risk_tier']} (p={body['probability']:.4f}) expected={expected}")

    # Role separation checks
    officer_alerts = client.get("/api/officer/alerts", cookies=officer["cookies"]).json()
    officer_sees_a = any(
        item["client_id"] == client_a["client_id"] for item in officer_alerts["items"]
    )
    check = "OK" if officer_sees_a else "FAIL"
    if not officer_sees_a:
        ok = False
    print(f"  [{check}] officer sees client A's alert in the cross-client queue")

    a_alerts = client.get("/api/client/alerts", cookies=client_a["cookies"]).json()
    check = "OK" if len(a_alerts) == 1 else "FAIL"
    if len(a_alerts) != 1:
        ok = False
    print(f"  [{check}] client A sees exactly their own alert ({len(a_alerts)} found)")

    b_alerts = client.get("/api/client/alerts", cookies=client_b["cookies"]).json()
    check = "OK" if len(b_alerts) == 0 else "FAIL"
    if len(b_alerts) != 0:
        ok = False
    print(f"  [{check}] client B sees none of client A's alerts ({len(b_alerts)} found)")

    b_sees_a_profile = client.get(f"/api/officer/clients/{client_a['client_id']}", cookies=client_b["cookies"])
    check = "OK" if b_sees_a_profile.status_code == 403 else "FAIL"
    if b_sees_a_profile.status_code != 403:
        ok = False
    print(f"  [{check}] client B is rejected (403) from officer-only client-detail route")

    return ok


def run_fixture_replay(client: httpx.Client, officer: dict) -> bool:
    print("\n=== Phase 3: hand-labeled fixture replay via /predict (officer) ===")
    fixtures = json.loads(FIXTURES_PATH.read_text())
    correct = 0
    for fixture in fixtures:
        raw = {k: fixture[k] for k in [
            "step", "type", "amount", "nameOrig", "oldbalanceOrg", "newbalanceOrig",
            "nameDest", "oldbalanceDest", "newbalanceDest",
        ]}
        resp = client.post("/api/predict", json=raw, cookies=officer["cookies"])
        if resp.status_code != 200:
            print(f"  FAIL [{fixture['id']}]: {resp.status_code} {resp.text}")
            continue
        body = resp.json()
        predicted_label = _label_for_tier(body["risk_tier"])
        expected_label = fixture["expected_label"]
        is_match = predicted_label == expected_label
        correct += int(is_match)
        marker = "OK" if is_match else "MISMATCH"
        print(
            f"  [{marker}] {fixture['id']:<30} predicted={body['risk_tier']:<7} "
            f"(p={body['probability']:.4f}) expected={expected_label}"
        )

    accuracy = correct / len(fixtures)
    print(f"\nAccuracy: {correct}/{len(fixtures)} = {accuracy:.2%} (bar: {MIN_ACCURACY:.0%})")
    return accuracy >= MIN_ACCURACY


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--base-url",
        default="http://127.0.0.1:8811",
        help="Base URL of the running API (default: local dev server)",
    )
    parser.add_argument("--timeout", type=float, default=30.0)
    args = parser.parse_args()

    with httpx.Client(base_url=args.base_url, timeout=args.timeout) as client:
        health_resp = client.get("/health")
        if health_resp.status_code != 200:
            print(f"FAIL: /health returned {health_resp.status_code}")
            sys.exit(1)
        print(f"Connected: {args.base_url} -- {health_resp.json()}")

        role_ok = run_role_flow(client)
        officer = login(client, settings.SEED_OFFICER_EMAIL, settings.SEED_OFFICER_PASSWORD)
        fixtures_ok = run_fixture_replay(client, officer)

    if role_ok and fixtures_ok:
        print("\nLive smoke test PASSED")
        sys.exit(0)
    else:
        print("\nLive smoke test FAILED")
        sys.exit(1)


if __name__ == "__main__":
    main()