Download src/embeddings.py from sarra2002/RareDx: direct link, hf CLI and curl.
- Browser
- Download file 5.67 kB
-
https://huggingface.co/spaces/sarra2002/RareDx/resolve/main/src/embeddings.py
- Command line
-
hf download hf://spaces/sarra2002/RareDx/src/embeddings.py
-
curl -L -o embeddings.py https://huggingface.co/spaces/sarra2002/RareDx/resolve/main/src/embeddings.py
5.67 kB
| """Embeddings: link a text to a code (symptom -> HPO, disease name -> ORPHA / OMIM).""" | |
| from __future__ import annotations | |
| import re | |
| from pathlib import Path | |
| from typing import Protocol | |
| import numpy as np | |
| from .knowledge_base import KnowledgeBase | |
| ROMAN = {"i": "1", "ii": "2", "iii": "3", "iv": "4", "v": "5", "vi": "6", "vii": "7", | |
| "viii": "8", "ix": "9", "x": "10", "xi": "11", "xii": "12"} | |
| class Encoder(Protocol): | |
| def encode(self, texts, batch_size: int = 32, normalize_embeddings: bool = True, | |
| show_progress_bar: bool = False) -> np.ndarray: ... | |
| def load_encoder(model_name: str) -> Encoder: | |
| from sentence_transformers import SentenceTransformer | |
| return SentenceTransformer(model_name) | |
| def normalize(name: str) -> str: | |
| """Lower case, no parentheses, roman numerals -> digits, aciduria -> acidemia.""" | |
| t = re.sub(r"\([^)]*\)", " ", name.lower()).replace("aemia", "emia") | |
| t = re.sub(r"aciduria\b", "acidemia", t) | |
| words = re.findall(r"[a-z0-9]+", t) | |
| out = [] | |
| for k, w in enumerate(words): | |
| nxt = words[k + 1] if k + 1 < len(words) else "" | |
| out.append(ROMAN[w] if w in ROMAN and not (w == "x" and nxt == "linked") else w) | |
| return " ".join(out) | |
| def numbers(t: str) -> set[str]: | |
| return set(re.findall(r"\b\d+[a-z]?\b", t)) | |
| def keywords(t: str) -> set[str]: | |
| return {w for w in t.split() if len(w) > 3} | |
| class EntityLinker: | |
| def __init__(self, kb: KnowledgeBase, encoder: Encoder, cache_dir: Path | None = None, | |
| hpo_threshold: float = 0.75, name_threshold: float = 0.85, decontaminated: bool = False): | |
| self.kb, self.encoder = kb, encoder | |
| self.cache_dir = Path(cache_dir) if cache_dir else None | |
| self.hpo_threshold, self.name_threshold = hpo_threshold, name_threshold | |
| # ---- symptom -> HPO code (names and synonyms) ---- | |
| self.labels, self.label_codes = [], [] | |
| for code, t in kb.terms.items(): | |
| for txt in [t["name"]] + t["synonyms"]: | |
| self.labels.append(txt) | |
| self.label_codes.append(code) | |
| self.label_vectors = self._encode_cached(self.labels, "hpo_libelles_vecteurs.npy") | |
| # ---- disease name -> ORPHA / OMIM code ---- | |
| pairs = [(d["name"], d["code"]) for d in kb.diseases] | |
| for code, name in kb.mapping.items(): | |
| if code in kb.by_code: | |
| for n in (name if isinstance(name, list) else re.split(r"\s/\s|;", str(name))): | |
| if n.strip(): | |
| pairs.append((n.strip(), code)) | |
| pairs = list(dict.fromkeys(pairs)) | |
| self.names = [p[0] for p in pairs] | |
| self.name_codes = [p[1] for p in pairs] | |
| self.normalized = [normalize(n) for n in self.names] | |
| self.exact: dict[str, str] = {} | |
| for n, c in zip(self.normalized, self.name_codes): | |
| self.exact.setdefault(n, c) | |
| vec_file = "noms_maladies_vecteurs_decont.npy" if decontaminated else "noms_maladies_vecteurs.npy" | |
| self.name_vectors = self._encode_cached(self.names, vec_file) | |
| # ---- disease families ("Bardet-Biedl syndrome" -> all numbered subtypes) ---- | |
| self.families: dict[str, set[str]] = {} | |
| for n, c in zip(self.normalized, self.name_codes): | |
| m = re.match(r"^(.*?)(?:\s+type)?\s+\d+[a-z]?$", n) | |
| if m and len(m.group(1)) > 3: | |
| self.families.setdefault(m.group(1), set()).add(kb.group(c)) | |
| def _encode_cached(self, texts: list[str], filename: str) -> np.ndarray: | |
| path = self.cache_dir / filename if self.cache_dir else None | |
| if path and path.exists(): | |
| v = np.load(path) | |
| if v.shape[0] == len(texts): | |
| return v | |
| v = self.encoder.encode(texts, batch_size=256, normalize_embeddings=True, show_progress_bar=True) | |
| if path: | |
| np.save(path, v) | |
| return v | |
| def _vector(self, text: str) -> np.ndarray: | |
| return self.encoder.encode([text], normalize_embeddings=True)[0] | |
| def to_hpo(self, phrase: str, threshold: float | None = None): | |
| """English symptom -> (HPO code, official name, similarity), or None below the threshold.""" | |
| threshold = self.hpo_threshold if threshold is None else threshold | |
| sims = self.label_vectors @ self._vector(phrase) | |
| i = int(sims.argmax()) | |
| if sims[i] < threshold: | |
| return None | |
| code = self.label_codes[i] | |
| return code, self.kb.term_name(code), float(sims[i]) | |
| def disease_to_code(self, name: str, threshold: float | None = None, k: int = 20) -> str | None: | |
| """Disease name proposed by the LLM -> ORPHA / OMIM code, with guards on numbers and keywords.""" | |
| threshold = self.name_threshold if threshold is None else threshold | |
| n = normalize(name) | |
| if not n: | |
| return None | |
| if n in self.exact: | |
| return self.exact[n] | |
| sims = self.name_vectors @ self._vector(name) | |
| nums, keys = numbers(n), keywords(n) | |
| for i in np.argsort(-sims)[:k]: | |
| if sims[i] < threshold: | |
| break | |
| cand = self.normalized[i] | |
| if nums and not nums <= numbers(cand): | |
| continue | |
| if keys and len(keys & keywords(cand)) / len(keys) < 0.5 and sims[i] < 0.93: | |
| continue | |
| return self.name_codes[i] | |
| return None | |
| def family(self, name: str) -> set[str]: | |
| """All subtypes of a general disease name without a number ("Bardet-Biedl syndrome").""" | |
| n = re.sub(r"\s+type$", "", normalize(name)) | |
| return set() if numbers(n) else self.families.get(n, set()) | |