File size: 4,065 Bytes
613af80
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
"""Evaluation: strict top-k, bootstrap confidence intervals, exact McNemar test, simulated doctor."""
from __future__ import annotations

import math
import random
from collections import defaultdict

import numpy as np
import pandas as pd

from .agent import QuestionAgent
from .knowledge_base import KnowledgeBase
from .llm import QuotaExceeded
from .system import DiagnosticSystem


def rank_of(ranking, targets: set[str]) -> int | None:
    """1-based rank of the first correct answer (items can be codes or sets of codes)."""
    for i, x in enumerate(ranking, 1):
        if (x & targets) if isinstance(x, set) else (x in targets):
            return i
    return None


def hits(ranks, k: int) -> np.ndarray:
    return np.array([r is not None and not (isinstance(r, float) and math.isnan(r)) and r <= k for r in ranks])


def bootstrap_ci(x: np.ndarray, n: int = 1000, seed: int = 0) -> tuple[float, float]:
    g = np.random.default_rng(seed)
    means = [x[g.integers(0, len(x), len(x))].mean() for _ in range(n)]
    return float(np.percentile(means, 2.5)), float(np.percentile(means, 97.5))


def mcnemar_exact(a: np.ndarray, b: np.ndarray) -> tuple[int, int, float]:
    """Paired comparison: (patients gained by a, patients lost by a, two-sided exact p-value)."""
    gained, lost = int((a & ~b).sum()), int((~a & b).sum())
    m = gained + lost
    p = min(1.0, 2 * sum(math.comb(m, i) for i in range(min(gained, lost) + 1)) / 2 ** m) if m else 1.0
    return gained, lost, p


def simulated_doctor(kb: KnowledgeBase, questions: list[str], targets: set[str], rng: random.Random,
                     p_unknown: float = 0.5, p_error: float = 0.2, max_precise: int = 1):
    """Answers generated from the TRUE disease profile, with noise ("oracle": a favourable scenario)."""
    signs: dict[str, float] = {}
    for g in targets:
        for d in kb.diseases_by_group.get(g, []):
            for s in d["symptoms"]:
                signs[s["hpo"]] = max(signs.get(s["hpo"], 0), s["weight"])
    confirmed, excluded = [], []
    for q in questions:
        if not signs or rng.random() < p_unknown:
            continue
        precise = sorted([h for h, w in signs.items() if w >= 0.17 and q in kb.ancestors(h)],
                         key=lambda h: -signs[h])[:max_precise]
        present = bool(precise)
        if rng.random() < p_error:
            present = not present
            precise = [q] if present else []
        if present:
            confirmed += [h for h in precise if h not in confirmed]
        else:
            excluded.append(q)
    return confirmed, excluded


def evaluate_cases(system: DiagnosticSystem, cases: list[dict], agent: QuestionAgent | None = None) -> dict[str, list]:
    """Ranks of the right disease for RAG alone, LLM alone, the system and (optionally) the agent."""
    ranks: dict[str, list] = defaultdict(list)
    rng = random.Random(1)
    for c in cases:
        targets = {system.kb.group(x) for x in c["targets"]}
        try:
            d = system.diagnose(c["codes"])
        except QuotaExceeded as e:
            print(e)
            break
        ranks["RAG alone"].append(rank_of(d.rag, targets))
        ranks["LLM alone"].append(rank_of(d.llm_sets, targets))
        ranks["System"].append(rank_of(d.ranking, targets))
        if agent:
            conf, excl = simulated_doctor(system.kb, agent.choose_questions(d.ranking, c["codes"]), targets, rng)
            ranks["Agent v2 (simulated doctor)"].append(rank_of(agent.update(c["codes"], d.llm_sets, conf, excl), targets))
    return dict(ranks)


def summarize(ranks: dict[str, list], with_ci: bool = True) -> pd.DataFrame:
    rows = []
    for method, r in ranks.items():
        row = {"method": method, "patients": len(r)}
        for k in (1, 5, 10):
            x = hits(r, k)
            row[f"top {k}"] = f"{x.mean():.0%}"
            if with_ci and len(x) and k in (1, 5):
                lo, hi = bootstrap_ci(x)
                row[f"top {k}"] += f" [{lo:.0%}-{hi:.0%}]"
        rows.append(row)
    return pd.DataFrame(rows)