Download src/evaluation.py from sarra2002/RareDx: direct link, hf CLI and curl.
- Browser
- Download file 4.07 kB
-
https://huggingface.co/spaces/sarra2002/RareDx/resolve/main/src/evaluation.py
- Command line
-
hf download hf://spaces/sarra2002/RareDx/src/evaluation.py
-
curl -L -o evaluation.py https://huggingface.co/spaces/sarra2002/RareDx/resolve/main/src/evaluation.py
4.07 kB
| """Evaluation: strict top-k, bootstrap confidence intervals, exact McNemar test, simulated doctor.""" | |
| from __future__ import annotations | |
| import math | |
| import random | |
| from collections import defaultdict | |
| import numpy as np | |
| import pandas as pd | |
| from .agent import QuestionAgent | |
| from .knowledge_base import KnowledgeBase | |
| from .llm import QuotaExceeded | |
| from .system import DiagnosticSystem | |
| def rank_of(ranking, targets: set[str]) -> int | None: | |
| """1-based rank of the first correct answer (items can be codes or sets of codes).""" | |
| for i, x in enumerate(ranking, 1): | |
| if (x & targets) if isinstance(x, set) else (x in targets): | |
| return i | |
| return None | |
| def hits(ranks, k: int) -> np.ndarray: | |
| return np.array([r is not None and not (isinstance(r, float) and math.isnan(r)) and r <= k for r in ranks]) | |
| def bootstrap_ci(x: np.ndarray, n: int = 1000, seed: int = 0) -> tuple[float, float]: | |
| g = np.random.default_rng(seed) | |
| means = [x[g.integers(0, len(x), len(x))].mean() for _ in range(n)] | |
| return float(np.percentile(means, 2.5)), float(np.percentile(means, 97.5)) | |
| def mcnemar_exact(a: np.ndarray, b: np.ndarray) -> tuple[int, int, float]: | |
| """Paired comparison: (patients gained by a, patients lost by a, two-sided exact p-value).""" | |
| gained, lost = int((a & ~b).sum()), int((~a & b).sum()) | |
| m = gained + lost | |
| p = min(1.0, 2 * sum(math.comb(m, i) for i in range(min(gained, lost) + 1)) / 2 ** m) if m else 1.0 | |
| return gained, lost, p | |
| def simulated_doctor(kb: KnowledgeBase, questions: list[str], targets: set[str], rng: random.Random, | |
| p_unknown: float = 0.5, p_error: float = 0.2, max_precise: int = 1): | |
| """Answers generated from the TRUE disease profile, with noise ("oracle": a favourable scenario).""" | |
| signs: dict[str, float] = {} | |
| for g in targets: | |
| for d in kb.diseases_by_group.get(g, []): | |
| for s in d["symptoms"]: | |
| signs[s["hpo"]] = max(signs.get(s["hpo"], 0), s["weight"]) | |
| confirmed, excluded = [], [] | |
| for q in questions: | |
| if not signs or rng.random() < p_unknown: | |
| continue | |
| precise = sorted([h for h, w in signs.items() if w >= 0.17 and q in kb.ancestors(h)], | |
| key=lambda h: -signs[h])[:max_precise] | |
| present = bool(precise) | |
| if rng.random() < p_error: | |
| present = not present | |
| precise = [q] if present else [] | |
| if present: | |
| confirmed += [h for h in precise if h not in confirmed] | |
| else: | |
| excluded.append(q) | |
| return confirmed, excluded | |
| def evaluate_cases(system: DiagnosticSystem, cases: list[dict], agent: QuestionAgent | None = None) -> dict[str, list]: | |
| """Ranks of the right disease for RAG alone, LLM alone, the system and (optionally) the agent.""" | |
| ranks: dict[str, list] = defaultdict(list) | |
| rng = random.Random(1) | |
| for c in cases: | |
| targets = {system.kb.group(x) for x in c["targets"]} | |
| try: | |
| d = system.diagnose(c["codes"]) | |
| except QuotaExceeded as e: | |
| print(e) | |
| break | |
| ranks["RAG alone"].append(rank_of(d.rag, targets)) | |
| ranks["LLM alone"].append(rank_of(d.llm_sets, targets)) | |
| ranks["System"].append(rank_of(d.ranking, targets)) | |
| if agent: | |
| conf, excl = simulated_doctor(system.kb, agent.choose_questions(d.ranking, c["codes"]), targets, rng) | |
| ranks["Agent v2 (simulated doctor)"].append(rank_of(agent.update(c["codes"], d.llm_sets, conf, excl), targets)) | |
| return dict(ranks) | |
| def summarize(ranks: dict[str, list], with_ci: bool = True) -> pd.DataFrame: | |
| rows = [] | |
| for method, r in ranks.items(): | |
| row = {"method": method, "patients": len(r)} | |
| for k in (1, 5, 10): | |
| x = hits(r, k) | |
| row[f"top {k}"] = f"{x.mean():.0%}" | |
| if with_ci and len(x) and k in (1, 5): | |
| lo, hi = bootstrap_ci(x) | |
| row[f"top {k}"] += f" [{lo:.0%}-{hi:.0%}]" | |
| rows.append(row) | |
| return pd.DataFrame(rows) | |