RareDx / src /embeddings.py
sarra2002's picture
Démo RareDx
613af80 verified
Raw History Blame Contribute Delete
5.67 kB
"""Embeddings: link a text to a code (symptom -> HPO, disease name -> ORPHA / OMIM)."""
from __future__ import annotations
import re
from pathlib import Path
from typing import Protocol
import numpy as np
from .knowledge_base import KnowledgeBase
ROMAN = {"i": "1", "ii": "2", "iii": "3", "iv": "4", "v": "5", "vi": "6", "vii": "7",
"viii": "8", "ix": "9", "x": "10", "xi": "11", "xii": "12"}
class Encoder(Protocol):
def encode(self, texts, batch_size: int = 32, normalize_embeddings: bool = True,
show_progress_bar: bool = False) -> np.ndarray: ...
def load_encoder(model_name: str) -> Encoder:
from sentence_transformers import SentenceTransformer
return SentenceTransformer(model_name)
def normalize(name: str) -> str:
"""Lower case, no parentheses, roman numerals -> digits, aciduria -> acidemia."""
t = re.sub(r"\([^)]*\)", " ", name.lower()).replace("aemia", "emia")
t = re.sub(r"aciduria\b", "acidemia", t)
words = re.findall(r"[a-z0-9]+", t)
out = []
for k, w in enumerate(words):
nxt = words[k + 1] if k + 1 < len(words) else ""
out.append(ROMAN[w] if w in ROMAN and not (w == "x" and nxt == "linked") else w)
return " ".join(out)
def numbers(t: str) -> set[str]:
return set(re.findall(r"\b\d+[a-z]?\b", t))
def keywords(t: str) -> set[str]:
return {w for w in t.split() if len(w) > 3}
class EntityLinker:
def __init__(self, kb: KnowledgeBase, encoder: Encoder, cache_dir: Path | None = None,
hpo_threshold: float = 0.75, name_threshold: float = 0.85, decontaminated: bool = False):
self.kb, self.encoder = kb, encoder
self.cache_dir = Path(cache_dir) if cache_dir else None
self.hpo_threshold, self.name_threshold = hpo_threshold, name_threshold
# ---- symptom -> HPO code (names and synonyms) ----
self.labels, self.label_codes = [], []
for code, t in kb.terms.items():
for txt in [t["name"]] + t["synonyms"]:
self.labels.append(txt)
self.label_codes.append(code)
self.label_vectors = self._encode_cached(self.labels, "hpo_libelles_vecteurs.npy")
# ---- disease name -> ORPHA / OMIM code ----
pairs = [(d["name"], d["code"]) for d in kb.diseases]
for code, name in kb.mapping.items():
if code in kb.by_code:
for n in (name if isinstance(name, list) else re.split(r"\s/\s|;", str(name))):
if n.strip():
pairs.append((n.strip(), code))
pairs = list(dict.fromkeys(pairs))
self.names = [p[0] for p in pairs]
self.name_codes = [p[1] for p in pairs]
self.normalized = [normalize(n) for n in self.names]
self.exact: dict[str, str] = {}
for n, c in zip(self.normalized, self.name_codes):
self.exact.setdefault(n, c)
vec_file = "noms_maladies_vecteurs_decont.npy" if decontaminated else "noms_maladies_vecteurs.npy"
self.name_vectors = self._encode_cached(self.names, vec_file)
# ---- disease families ("Bardet-Biedl syndrome" -> all numbered subtypes) ----
self.families: dict[str, set[str]] = {}
for n, c in zip(self.normalized, self.name_codes):
m = re.match(r"^(.*?)(?:\s+type)?\s+\d+[a-z]?$", n)
if m and len(m.group(1)) > 3:
self.families.setdefault(m.group(1), set()).add(kb.group(c))
def _encode_cached(self, texts: list[str], filename: str) -> np.ndarray:
path = self.cache_dir / filename if self.cache_dir else None
if path and path.exists():
v = np.load(path)
if v.shape[0] == len(texts):
return v
v = self.encoder.encode(texts, batch_size=256, normalize_embeddings=True, show_progress_bar=True)
if path:
np.save(path, v)
return v
def _vector(self, text: str) -> np.ndarray:
return self.encoder.encode([text], normalize_embeddings=True)[0]
def to_hpo(self, phrase: str, threshold: float | None = None):
"""English symptom -> (HPO code, official name, similarity), or None below the threshold."""
threshold = self.hpo_threshold if threshold is None else threshold
sims = self.label_vectors @ self._vector(phrase)
i = int(sims.argmax())
if sims[i] < threshold:
return None
code = self.label_codes[i]
return code, self.kb.term_name(code), float(sims[i])
def disease_to_code(self, name: str, threshold: float | None = None, k: int = 20) -> str | None:
"""Disease name proposed by the LLM -> ORPHA / OMIM code, with guards on numbers and keywords."""
threshold = self.name_threshold if threshold is None else threshold
n = normalize(name)
if not n:
return None
if n in self.exact:
return self.exact[n]
sims = self.name_vectors @ self._vector(name)
nums, keys = numbers(n), keywords(n)
for i in np.argsort(-sims)[:k]:
if sims[i] < threshold:
break
cand = self.normalized[i]
if nums and not nums <= numbers(cand):
continue
if keys and len(keys & keywords(cand)) / len(keys) < 0.5 and sims[i] < 0.93:
continue
return self.name_codes[i]
return None
def family(self, name: str) -> set[str]:
"""All subtypes of a general disease name without a number ("Bardet-Biedl syndrome")."""
n = re.sub(r"\s+type$", "", normalize(name))
return set() if numbers(n) else self.families.get(n, set())