"""Embeddings: link a text to a code (symptom -> HPO, disease name -> ORPHA / OMIM).""" from __future__ import annotations import re from pathlib import Path from typing import Protocol import numpy as np from .knowledge_base import KnowledgeBase ROMAN = {"i": "1", "ii": "2", "iii": "3", "iv": "4", "v": "5", "vi": "6", "vii": "7", "viii": "8", "ix": "9", "x": "10", "xi": "11", "xii": "12"} class Encoder(Protocol): def encode(self, texts, batch_size: int = 32, normalize_embeddings: bool = True, show_progress_bar: bool = False) -> np.ndarray: ... def load_encoder(model_name: str) -> Encoder: from sentence_transformers import SentenceTransformer return SentenceTransformer(model_name) def normalize(name: str) -> str: """Lower case, no parentheses, roman numerals -> digits, aciduria -> acidemia.""" t = re.sub(r"\([^)]*\)", " ", name.lower()).replace("aemia", "emia") t = re.sub(r"aciduria\b", "acidemia", t) words = re.findall(r"[a-z0-9]+", t) out = [] for k, w in enumerate(words): nxt = words[k + 1] if k + 1 < len(words) else "" out.append(ROMAN[w] if w in ROMAN and not (w == "x" and nxt == "linked") else w) return " ".join(out) def numbers(t: str) -> set[str]: return set(re.findall(r"\b\d+[a-z]?\b", t)) def keywords(t: str) -> set[str]: return {w for w in t.split() if len(w) > 3} class EntityLinker: def __init__(self, kb: KnowledgeBase, encoder: Encoder, cache_dir: Path | None = None, hpo_threshold: float = 0.75, name_threshold: float = 0.85, decontaminated: bool = False): self.kb, self.encoder = kb, encoder self.cache_dir = Path(cache_dir) if cache_dir else None self.hpo_threshold, self.name_threshold = hpo_threshold, name_threshold # ---- symptom -> HPO code (names and synonyms) ---- self.labels, self.label_codes = [], [] for code, t in kb.terms.items(): for txt in [t["name"]] + t["synonyms"]: self.labels.append(txt) self.label_codes.append(code) self.label_vectors = self._encode_cached(self.labels, "hpo_libelles_vecteurs.npy") # ---- disease name -> ORPHA / OMIM code ---- pairs = [(d["name"], d["code"]) for d in kb.diseases] for code, name in kb.mapping.items(): if code in kb.by_code: for n in (name if isinstance(name, list) else re.split(r"\s/\s|;", str(name))): if n.strip(): pairs.append((n.strip(), code)) pairs = list(dict.fromkeys(pairs)) self.names = [p[0] for p in pairs] self.name_codes = [p[1] for p in pairs] self.normalized = [normalize(n) for n in self.names] self.exact: dict[str, str] = {} for n, c in zip(self.normalized, self.name_codes): self.exact.setdefault(n, c) vec_file = "noms_maladies_vecteurs_decont.npy" if decontaminated else "noms_maladies_vecteurs.npy" self.name_vectors = self._encode_cached(self.names, vec_file) # ---- disease families ("Bardet-Biedl syndrome" -> all numbered subtypes) ---- self.families: dict[str, set[str]] = {} for n, c in zip(self.normalized, self.name_codes): m = re.match(r"^(.*?)(?:\s+type)?\s+\d+[a-z]?$", n) if m and len(m.group(1)) > 3: self.families.setdefault(m.group(1), set()).add(kb.group(c)) def _encode_cached(self, texts: list[str], filename: str) -> np.ndarray: path = self.cache_dir / filename if self.cache_dir else None if path and path.exists(): v = np.load(path) if v.shape[0] == len(texts): return v v = self.encoder.encode(texts, batch_size=256, normalize_embeddings=True, show_progress_bar=True) if path: np.save(path, v) return v def _vector(self, text: str) -> np.ndarray: return self.encoder.encode([text], normalize_embeddings=True)[0] def to_hpo(self, phrase: str, threshold: float | None = None): """English symptom -> (HPO code, official name, similarity), or None below the threshold.""" threshold = self.hpo_threshold if threshold is None else threshold sims = self.label_vectors @ self._vector(phrase) i = int(sims.argmax()) if sims[i] < threshold: return None code = self.label_codes[i] return code, self.kb.term_name(code), float(sims[i]) def disease_to_code(self, name: str, threshold: float | None = None, k: int = 20) -> str | None: """Disease name proposed by the LLM -> ORPHA / OMIM code, with guards on numbers and keywords.""" threshold = self.name_threshold if threshold is None else threshold n = normalize(name) if not n: return None if n in self.exact: return self.exact[n] sims = self.name_vectors @ self._vector(name) nums, keys = numbers(n), keywords(n) for i in np.argsort(-sims)[:k]: if sims[i] < threshold: break cand = self.normalized[i] if nums and not nums <= numbers(cand): continue if keys and len(keys & keywords(cand)) / len(keys) < 0.5 and sims[i] < 0.93: continue return self.name_codes[i] return None def family(self, name: str) -> set[str]: """All subtypes of a general disease name without a number ("Bardet-Biedl syndrome").""" n = re.sub(r"\s+type$", "", normalize(name)) return set() if numbers(n) else self.families.get(n, set())