RareDx / src /evaluation.py
sarra2002's picture
Démo RareDx
613af80 verified
Raw History Blame Contribute Delete
4.07 kB
"""Evaluation: strict top-k, bootstrap confidence intervals, exact McNemar test, simulated doctor."""
from __future__ import annotations
import math
import random
from collections import defaultdict
import numpy as np
import pandas as pd
from .agent import QuestionAgent
from .knowledge_base import KnowledgeBase
from .llm import QuotaExceeded
from .system import DiagnosticSystem
def rank_of(ranking, targets: set[str]) -> int | None:
"""1-based rank of the first correct answer (items can be codes or sets of codes)."""
for i, x in enumerate(ranking, 1):
if (x & targets) if isinstance(x, set) else (x in targets):
return i
return None
def hits(ranks, k: int) -> np.ndarray:
return np.array([r is not None and not (isinstance(r, float) and math.isnan(r)) and r <= k for r in ranks])
def bootstrap_ci(x: np.ndarray, n: int = 1000, seed: int = 0) -> tuple[float, float]:
g = np.random.default_rng(seed)
means = [x[g.integers(0, len(x), len(x))].mean() for _ in range(n)]
return float(np.percentile(means, 2.5)), float(np.percentile(means, 97.5))
def mcnemar_exact(a: np.ndarray, b: np.ndarray) -> tuple[int, int, float]:
"""Paired comparison: (patients gained by a, patients lost by a, two-sided exact p-value)."""
gained, lost = int((a & ~b).sum()), int((~a & b).sum())
m = gained + lost
p = min(1.0, 2 * sum(math.comb(m, i) for i in range(min(gained, lost) + 1)) / 2 ** m) if m else 1.0
return gained, lost, p
def simulated_doctor(kb: KnowledgeBase, questions: list[str], targets: set[str], rng: random.Random,
p_unknown: float = 0.5, p_error: float = 0.2, max_precise: int = 1):
"""Answers generated from the TRUE disease profile, with noise ("oracle": a favourable scenario)."""
signs: dict[str, float] = {}
for g in targets:
for d in kb.diseases_by_group.get(g, []):
for s in d["symptoms"]:
signs[s["hpo"]] = max(signs.get(s["hpo"], 0), s["weight"])
confirmed, excluded = [], []
for q in questions:
if not signs or rng.random() < p_unknown:
continue
precise = sorted([h for h, w in signs.items() if w >= 0.17 and q in kb.ancestors(h)],
key=lambda h: -signs[h])[:max_precise]
present = bool(precise)
if rng.random() < p_error:
present = not present
precise = [q] if present else []
if present:
confirmed += [h for h in precise if h not in confirmed]
else:
excluded.append(q)
return confirmed, excluded
def evaluate_cases(system: DiagnosticSystem, cases: list[dict], agent: QuestionAgent | None = None) -> dict[str, list]:
"""Ranks of the right disease for RAG alone, LLM alone, the system and (optionally) the agent."""
ranks: dict[str, list] = defaultdict(list)
rng = random.Random(1)
for c in cases:
targets = {system.kb.group(x) for x in c["targets"]}
try:
d = system.diagnose(c["codes"])
except QuotaExceeded as e:
print(e)
break
ranks["RAG alone"].append(rank_of(d.rag, targets))
ranks["LLM alone"].append(rank_of(d.llm_sets, targets))
ranks["System"].append(rank_of(d.ranking, targets))
if agent:
conf, excl = simulated_doctor(system.kb, agent.choose_questions(d.ranking, c["codes"]), targets, rng)
ranks["Agent v2 (simulated doctor)"].append(rank_of(agent.update(c["codes"], d.llm_sets, conf, excl), targets))
return dict(ranks)
def summarize(ranks: dict[str, list], with_ci: bool = True) -> pd.DataFrame:
rows = []
for method, r in ranks.items():
row = {"method": method, "patients": len(r)}
for k in (1, 5, 10):
x = hits(r, k)
row[f"top {k}"] = f"{x.mean():.0%}"
if with_ci and len(x) and k in (1, 5):
lo, hi = bootstrap_ci(x)
row[f"top {k}"] += f" [{lo:.0%}-{hi:.0%}]"
rows.append(row)
return pd.DataFrame(rows)