Download src/system.py from sarra2002/RareDx: direct link, hf CLI and curl.
- Browser
- Download file 7.55 kB
-
https://huggingface.co/spaces/sarra2002/RareDx/resolve/main/src/system.py
- Command line
-
hf download hf://spaces/sarra2002/RareDx/src/system.py
-
curl -L -o system.py https://huggingface.co/spaces/sarra2002/RareDx/resolve/main/src/system.py
7.55 kB
| """Final system: routing, two opinions (RAG and LLM) and rank fusion.""" | |
| from __future__ import annotations | |
| from dataclasses import dataclass, field | |
| import numpy as np | |
| from .config import Settings, get_secret | |
| from .embeddings import EntityLinker | |
| from .knowledge_base import KnowledgeBase | |
| from .llm import LLMClient | |
| from .observability import Tracer | |
| from .rag import HierarchicalRAG | |
| DISCLAIMER = "Research tool to support reasoning, not a medical device and not a diagnosis." | |
| def rrf_fusion(rag: list[str], llm_sets: list[set[str]], llm_weight: float, c: int = 1) -> list[str]: | |
| """Reciprocal Rank Fusion: score = 1 / (c + rank RAG) + w / (c + rank LLM).""" | |
| score: dict[str, float] = {} | |
| for r, x in enumerate(rag, 1): | |
| score[x] = score.get(x, 0) + 1 / (c + r) | |
| for r, s in enumerate(llm_sets, 1): | |
| for x in s: | |
| score[x] = score.get(x, 0) + llm_weight / (c + r) | |
| return sorted(score, key=score.get, reverse=True) | |
| class Diagnosis: | |
| ranking: list[str] # disease groups, most likely first | |
| rag: list[str] | |
| llm_names: list[str] | |
| llm_sets: list[set[str]] | |
| metabolic_share: float | |
| llm_weight: float | |
| extra: dict = field(default_factory=dict) | |
| class DiagnosticSystem: | |
| def __init__(self, kb: KnowledgeBase, rag: HierarchicalRAG, linker: EntityLinker, llm: LLMClient, | |
| settings: Settings, tracer: Tracer | None = None): | |
| self.kb, self.rag, self.linker, self.llm, self.settings = kb, rag, linker, llm, settings | |
| self.tracer = tracer or Tracer(enabled=False) | |
| # Each step becomes a Langfuse span when observability is enabled | |
| self.search_rag = self.tracer.wrap(self.search_rag, "RAG : recherche hiérarchique") | |
| self.llm_diagnoses = self.tracer.wrap(self.llm_diagnoses, "LLM : 10 diagnostics") | |
| self.diagnose = self.tracer.wrap(self.diagnose, "diagnostic") | |
| self.understand_note = self.tracer.wrap(self.understand_note, "note -> symptômes HPO") | |
| # ---- the two opinions ---- | |
| def search_rag(self, codes: list[str]) -> list[str]: | |
| return self.rag.search(codes, k=self.settings.n_candidates) | |
| def llm_diagnoses(self, codes: list[str]) -> list[str]: | |
| return self.llm.diagnoses([self.kb.term_name(h) for h in codes]) | |
| def llm_sets(self, names: list[str], by_family: bool | None = None) -> list[set[str]]: | |
| """LLM disease names -> ordered list of sets of disease groups (no repetition).""" | |
| by_family = self.settings.by_family if by_family is None else by_family | |
| result, seen = [], set() | |
| for name in names: | |
| s = set() | |
| code = self.linker.disease_to_code(name) | |
| if code: | |
| s.add(self.kb.group(code)) | |
| if by_family: | |
| s |= self.linker.family(name) | |
| s -= seen | |
| if s: | |
| result.append(s) | |
| seen |= s | |
| return result | |
| # ---- routing and fusion ---- | |
| def metabolic_share(self, codes: list[str]) -> float: | |
| return float(np.mean([self.kb.is_metabolic(h) for h in codes])) if codes else 0.0 | |
| def llm_weight(self, codes: list[str]) -> float: | |
| s = self.settings | |
| return s.llm_weight_metabolic if self.metabolic_share(codes) >= s.metabolic_threshold else s.llm_weight_clinical | |
| def fuse(self, rag: list[str], llm_sets: list[set[str]], codes: list[str]) -> list[str]: | |
| return rrf_fusion(rag, llm_sets, self.llm_weight(codes), self.settings.rrf_constant) | |
| def diagnose(self, codes: list[str]) -> Diagnosis: | |
| """HPO codes -> final ranking, with the RAG and LLM opinions.""" | |
| rag = self.search_rag(codes) | |
| names = self.llm_diagnoses(codes) | |
| sets = self.llm_sets(names) | |
| ranking = self.fuse(rag, sets, codes) | |
| share = self.metabolic_share(codes) | |
| top5_llm = set().union(*sets[:5]) if sets else set() | |
| self.tracer.update_span(input={"symptomes": [self.kb.term_name(h) for h in codes]}, | |
| output={"top5": [self.kb.disease_name(g) for g in ranking[:5]]}) | |
| self.tracer.score("part_metabolique", share) | |
| self.tracer.score("noms_llm_relies", | |
| sum(self.linker.disease_to_code(n) is not None for n in names) / len(names) if names else 0.0) | |
| self.tracer.score("accord_rag_llm_top5", len(set(rag[:5]) & top5_llm) / 5) | |
| return Diagnosis(ranking, rag, names, sets, share, self.llm_weight(codes)) | |
| # ---- free text ---- | |
| def understand_note(self, note: str) -> list[tuple[str, str, float]]: | |
| """Free-text note -> [(HPO code, name, similarity)]; low similarity = "to verify".""" | |
| found: list[tuple[str, str, float]] = [] | |
| phrases = self.llm.extract_symptoms(note) | |
| for p in phrases: | |
| r = self.linker.to_hpo(p) | |
| if r and r[0] not in [c for c, _, _ in found]: | |
| found.append(r) | |
| self.tracer.score("symptomes_reconnus", len(found) / len(phrases) if phrases else 0.0) | |
| self.tracer.score("symptomes_a_verifier", | |
| sum(s < self.settings.to_verify_threshold for _, _, s in found) / len(found) if found else 0.0) | |
| return found | |
| # ---- explanation ---- | |
| def shared_symptoms(self, group: str, codes: list[str]) -> list[str]: | |
| known = set() | |
| for d in self.kb.diseases_by_group.get(group, []): | |
| for s in d["symptoms"]: | |
| known |= self.kb.ancestors(s["hpo"]) | |
| return [self.kb.term_name(h) for h in codes if h in known] | |
| def explain(self, ranking: list[str], codes: list[str], k: int = 10) -> list[dict]: | |
| return [{"rank": r, "code": g, "name": self.kb.disease_name(g), | |
| "shared_symptoms": self.shared_symptoms(g, codes)[:6]} | |
| for r, g in enumerate(ranking[:k], 1)] | |
| def format_ranking(self, ranking: list[str], codes: list[str], k: int = 10) -> str: | |
| lines = [f"Metabolic share: {self.metabolic_share(codes):.0%} -> LLM weight = {self.llm_weight(codes)}"] | |
| for item in self.explain(ranking, codes, k): | |
| lines.append(f"{item['rank']:2d}. {item['name']} ({item['code']})") | |
| if item["shared_symptoms"]: | |
| lines.append(" shared symptoms: " + ", ".join(item["shared_symptoms"])) | |
| lines.append("\n" + DISCLAIMER) | |
| return "\n".join(lines) | |
| def build_system(settings: Settings | None = None, download: bool = True, tracer: Tracer | None = None, | |
| encoder=None, completion_fn=None) -> DiagnosticSystem: | |
| """Download the data if needed and assemble the full system.""" | |
| from .data_loader import download_public_data | |
| from .embeddings import load_encoder | |
| settings = settings or Settings() | |
| tracer = tracer or Tracer() | |
| if download: | |
| download_public_data(settings.data_dir) | |
| kb = KnowledgeBase.from_dir(settings.data_dir, decontaminate=settings.decontaminate) | |
| rag = HierarchicalRAG(kb, alpha=settings.alpha) | |
| linker = EntityLinker(kb, encoder or load_encoder(settings.embedding_model), cache_dir=settings.data_dir, | |
| hpo_threshold=settings.hpo_threshold, name_threshold=settings.name_threshold, | |
| decontaminated=settings.decontaminate) | |
| llm = LLMClient(settings.llm_model, api_key=get_secret("GROQ_API_KEY"), | |
| cache_path=settings.data_dir / "cache_llm.json", completion_fn=completion_fn, tracer=tracer) | |
| return DiagnosticSystem(kb, rag, linker, llm, settings, tracer) | |