"""Evaluation: strict top-k, bootstrap confidence intervals, exact McNemar test, simulated doctor.""" from __future__ import annotations import math import random from collections import defaultdict import numpy as np import pandas as pd from .agent import QuestionAgent from .knowledge_base import KnowledgeBase from .llm import QuotaExceeded from .system import DiagnosticSystem def rank_of(ranking, targets: set[str]) -> int | None: """1-based rank of the first correct answer (items can be codes or sets of codes).""" for i, x in enumerate(ranking, 1): if (x & targets) if isinstance(x, set) else (x in targets): return i return None def hits(ranks, k: int) -> np.ndarray: return np.array([r is not None and not (isinstance(r, float) and math.isnan(r)) and r <= k for r in ranks]) def bootstrap_ci(x: np.ndarray, n: int = 1000, seed: int = 0) -> tuple[float, float]: g = np.random.default_rng(seed) means = [x[g.integers(0, len(x), len(x))].mean() for _ in range(n)] return float(np.percentile(means, 2.5)), float(np.percentile(means, 97.5)) def mcnemar_exact(a: np.ndarray, b: np.ndarray) -> tuple[int, int, float]: """Paired comparison: (patients gained by a, patients lost by a, two-sided exact p-value).""" gained, lost = int((a & ~b).sum()), int((~a & b).sum()) m = gained + lost p = min(1.0, 2 * sum(math.comb(m, i) for i in range(min(gained, lost) + 1)) / 2 ** m) if m else 1.0 return gained, lost, p def simulated_doctor(kb: KnowledgeBase, questions: list[str], targets: set[str], rng: random.Random, p_unknown: float = 0.5, p_error: float = 0.2, max_precise: int = 1): """Answers generated from the TRUE disease profile, with noise ("oracle": a favourable scenario).""" signs: dict[str, float] = {} for g in targets: for d in kb.diseases_by_group.get(g, []): for s in d["symptoms"]: signs[s["hpo"]] = max(signs.get(s["hpo"], 0), s["weight"]) confirmed, excluded = [], [] for q in questions: if not signs or rng.random() < p_unknown: continue precise = sorted([h for h, w in signs.items() if w >= 0.17 and q in kb.ancestors(h)], key=lambda h: -signs[h])[:max_precise] present = bool(precise) if rng.random() < p_error: present = not present precise = [q] if present else [] if present: confirmed += [h for h in precise if h not in confirmed] else: excluded.append(q) return confirmed, excluded def evaluate_cases(system: DiagnosticSystem, cases: list[dict], agent: QuestionAgent | None = None) -> dict[str, list]: """Ranks of the right disease for RAG alone, LLM alone, the system and (optionally) the agent.""" ranks: dict[str, list] = defaultdict(list) rng = random.Random(1) for c in cases: targets = {system.kb.group(x) for x in c["targets"]} try: d = system.diagnose(c["codes"]) except QuotaExceeded as e: print(e) break ranks["RAG alone"].append(rank_of(d.rag, targets)) ranks["LLM alone"].append(rank_of(d.llm_sets, targets)) ranks["System"].append(rank_of(d.ranking, targets)) if agent: conf, excl = simulated_doctor(system.kb, agent.choose_questions(d.ranking, c["codes"]), targets, rng) ranks["Agent v2 (simulated doctor)"].append(rank_of(agent.update(c["codes"], d.llm_sets, conf, excl), targets)) return dict(ranks) def summarize(ranks: dict[str, list], with_ci: bool = True) -> pd.DataFrame: rows = [] for method, r in ranks.items(): row = {"method": method, "patients": len(r)} for k in (1, 5, 10): x = hits(r, k) row[f"top {k}"] = f"{x.mean():.0%}" if with_ci and len(x) and k in (1, 5): lo, hi = bootstrap_ci(x) row[f"top {k}"] += f" [{lo:.0%}-{hi:.0%}]" rows.append(row) return pd.DataFrame(rows)