app/analytics.py

"""Phase 2 — Clinic analytics from anonymized ohip_feedback approvals.

No MOH remittance required. Underbilling proxy = optimized − selected when
Phase 1 revenue fields were logged at approve time.
"""
from __future__ import annotations

from collections import Counter, defaultdict
from typing import Any

from . import feedback
from .opensearch_client import get_client


def _num(value: Any) -> float | None:
    if value is None:
        return None
    try:
        return float(value)
    except (TypeError, ValueError):
        return None


def clinic_analytics(*, size: int = 500) -> dict:
    """Aggregate clinic KPIs from recent approval events."""
    client = get_client()
    if not client.indices.exists(index=feedback.FEEDBACK_INDEX):
        return _empty()

    resp = client.search(
        index=feedback.FEEDBACK_INDEX,
        body={
            "size": size,
            "sort": [{"created_at": {"order": "desc"}}],
            "_source": {
                "excludes": ["case_vector"],
            },
            "query": {"match_all": {}},
        },
    )
    hits = [h.get("_source") or {} for h in resp.get("hits", {}).get("hits", [])]
    return _aggregate(hits)


def _empty() -> dict:
    return {
        "total_cases": 0,
        "agreed": 0,
        "disagreed": 0,
        "agreement_rate": None,
        "override_rate": None,
        "avg_selected_cad": None,
        "avg_optimized_cad": None,
        "avg_difference_cad": None,
        "total_underbilling_cad": 0.0,
        "cases_with_revenue": 0,
        "avg_revenue_per_encounter_cad": None,
        "risk_levels": {},
        "top_approved_codes": [],
        "top_override_codes": [],
        "top_missed_opportunity_proxy": [],
        "by_encounter_type": [],
        "recent_cases": [],
    }


def _aggregate(hits: list[dict]) -> dict:
    if not hits:
        return _empty()

    total = len(hits)
    agreed_n = sum(1 for h in hits if h.get("agreed") is True)
    disagreed_n = total - agreed_n

    approved_counter: Counter[str] = Counter()
    override_counter: Counter[str] = Counter()
    risk_counter: Counter[str] = Counter()
    encounter_stats: dict[str, dict[str, float | int]] = defaultdict(
        lambda: {"cases": 0, "agreed": 0, "selected_sum": 0.0, "diff_sum": 0.0, "diff_n": 0}
    )

    selected_vals: list[float] = []
    optimized_vals: list[float] = []
    diff_vals: list[float] = []
    # Proxy for "missed": codes in ai_top that were not approved when disagreed,
    # or positive difference cases where optimized > selected.
    missed_proxy: Counter[str] = Counter()

    recent: list[dict] = []

    for h in hits:
        for code in h.get("approved_codes") or []:
            approved_counter[code] += 1
        for code in h.get("override_codes") or []:
            override_counter[code] += 1

        risk = h.get("risk_level")
        if risk:
            risk_counter[str(risk)] += 1

        enc = h.get("encounter_type") or "unspecified"
        est = encounter_stats[enc]
        est["cases"] = int(est["cases"]) + 1
        if h.get("agreed") is True:
            est["agreed"] = int(est["agreed"]) + 1

        sel = _num(h.get("selected_claim_cad"))
        opt = _num(h.get("optimized_claim_cad"))
        diff = _num(h.get("difference_cad"))
        if diff is None and sel is not None and opt is not None:
            diff = round(opt - sel, 2)

        if sel is not None:
            selected_vals.append(sel)
            est["selected_sum"] = float(est["selected_sum"]) + sel
        if opt is not None:
            optimized_vals.append(opt)
        if diff is not None:
            diff_vals.append(diff)
            est["diff_sum"] = float(est["diff_sum"]) + diff
            est["diff_n"] = int(est["diff_n"]) + 1
            if diff > 0:
                # Rank-1 AI code not taken, or first unused AI suggestion as proxy.
                ai = h.get("ai_top_codes") or []
                approved = set(h.get("approved_codes") or [])
                for code in ai:
                    if code not in approved:
                        missed_proxy[code] += 1
                        break

        recent.append(
            {
                "case_id": h.get("case_id"),
                "created_at": h.get("created_at"),
                "encounter_type": h.get("encounter_type"),
                "agreed": h.get("agreed"),
                "approved_codes": h.get("approved_codes") or [],
                "selected_claim_cad": sel,
                "optimized_claim_cad": opt,
                "difference_cad": diff,
                "risk_level": risk,
            }
        )

    def avg(vals: list[float]) -> float | None:
        return round(sum(vals) / len(vals), 2) if vals else None

    underbilling = round(sum(d for d in diff_vals if d > 0), 2)

    by_encounter = []
    for enc, est in sorted(
        encounter_stats.items(), key=lambda kv: int(kv[1]["cases"]), reverse=True
    ):
        cases = int(est["cases"])
        by_encounter.append(
            {
                "encounter_type": enc,
                "cases": cases,
                "agreement_rate": round(int(est["agreed"]) / cases, 3) if cases else None,
                "avg_selected_cad": (
                    round(float(est["selected_sum"]) / cases, 2)
                    if float(est["selected_sum"]) and cases
                    else None
                ),
                "avg_difference_cad": (
                    round(float(est["diff_sum"]) / int(est["diff_n"]), 2)
                    if int(est["diff_n"])
                    else None
                ),
            }
        )

    return {
        "total_cases": total,
        "agreed": agreed_n,
        "disagreed": disagreed_n,
        "agreement_rate": round(agreed_n / total, 3) if total else None,
        "override_rate": round(disagreed_n / total, 3) if total else None,
        "avg_selected_cad": avg(selected_vals),
        "avg_optimized_cad": avg(optimized_vals),
        "avg_difference_cad": avg(diff_vals),
        "total_underbilling_cad": underbilling,
        "cases_with_revenue": len(selected_vals),
        "avg_revenue_per_encounter_cad": avg(selected_vals),
        "risk_levels": dict(risk_counter),
        "top_approved_codes": [
            {"billing_code": c, "count": n} for c, n in approved_counter.most_common(10)
        ],
        "top_override_codes": [
            {"billing_code": c, "count": n} for c, n in override_counter.most_common(10)
        ],
        "top_missed_opportunity_proxy": [
            {"billing_code": c, "count": n} for c, n in missed_proxy.most_common(10)
        ],
        "by_encounter_type": by_encounter,
        "recent_cases": recent[:25],
    }