Download src/knowledge_base.py from sarra2002/RareDx: direct link, hf CLI and curl.
- Browser
- Download file 6.05 kB
-
https://huggingface.co/spaces/sarra2002/RareDx/resolve/main/src/knowledge_base.py
- Command line
-
hf download hf://spaces/sarra2002/RareDx/src/knowledge_base.py
-
curl -L -o knowledge_base.py https://huggingface.co/spaces/sarra2002/RareDx/resolve/main/src/knowledge_base.py
6.05 kB
| """Knowledge base: HPO terms and hierarchy, disease profiles (Orphanet + OMIM), disease groups.""" | |
| from __future__ import annotations | |
| import json | |
| from collections import defaultdict | |
| from pathlib import Path | |
| import pandas as pd | |
| METABOLIC_ROOT = "HP:0001939" # Abnormality of metabolism / homeostasis | |
| # HPO frequency terms -> weight | |
| FREQUENCY_WEIGHTS = {"HP:0040280": 1.0, "HP:0040281": 0.9, "HP:0040282": 0.55, | |
| "HP:0040283": 0.17, "HP:0040284": 0.025} | |
| def frequency_weight(f) -> float: | |
| """HPO frequency (term, "n/m" or "x%") -> weight between 0 and 1 (0.5 if unknown).""" | |
| if not isinstance(f, str) or not f: | |
| return 0.5 | |
| if f in FREQUENCY_WEIGHTS: | |
| return FREQUENCY_WEIGHTS[f] | |
| if "/" in f: | |
| n, d = f.split("/") | |
| return int(n) / int(d) if int(d) else 0.5 | |
| if f.endswith("%"): | |
| return float(f[:-1]) / 100 | |
| return 0.5 | |
| class KnowledgeBase: | |
| """HPO ontology + disease profiles, with optional leave-publication-out decontamination.""" | |
| def __init__(self, hpo_json: Path, hpoa: Path, mapping_json: Path, removed_pmids: set[str] | None = None): | |
| graph = json.load(open(hpo_json, encoding="utf-8"))["graphs"][0] | |
| self.version = graph.get("meta", {}).get("version", "unknown") | |
| # ---- HPO terms and hierarchy ---- | |
| self.terms: dict[str, dict] = {} | |
| for n in graph["nodes"]: | |
| if "HP_" in n.get("id", "") and "lbl" in n: | |
| meta = n.get("meta", {}) | |
| self.terms["HP:" + n["id"].split("HP_")[-1]] = { | |
| "name": n["lbl"], "synonyms": [s["val"] for s in meta.get("synonyms", [])]} | |
| self.parents: dict[str, set[str]] = defaultdict(set) | |
| for e in graph["edges"]: | |
| if e.get("pred") == "is_a" and "HP_" in e["sub"] and "HP_" in e["obj"]: | |
| self.parents["HP:" + e["sub"].split("HP_")[-1]].add("HP:" + e["obj"].split("HP_")[-1]) | |
| self._ancestors: dict[str, set[str]] = {} | |
| # ---- Disease profiles with frequencies ---- | |
| table = pd.read_csv(hpoa, sep="\t", comment="#", dtype=str) | |
| table = table[(table["aspect"] == "P") & (table["qualifier"] != "NOT") & table["hpo_id"].isin(self.terms.keys())] | |
| self.removed_annotations = 0 | |
| if removed_pmids: | |
| mask = table["reference"].fillna("").str.split(";").apply( | |
| lambda refs: any(r.strip() in removed_pmids for r in refs)) | |
| self.removed_annotations = int(mask.sum()) | |
| table = table[~mask] | |
| self.diseases: list[dict] = [] | |
| for prefix in ("ORPHA", "OMIM"): | |
| rows = table[table["database_id"].str.startswith(prefix)] | |
| for (code, name), grp in rows.groupby(["database_id", "disease_name"]): | |
| symptoms: dict[str, float] = {} | |
| for h, f in zip(grp["hpo_id"], grp["frequency"]): | |
| symptoms[h] = max(symptoms.get(h, 0), frequency_weight(f)) | |
| self.diseases.append({"code": code, "name": name, | |
| "symptoms": [{"hpo": h, "weight": w} for h, w in symptoms.items()]}) | |
| self.by_code = {d["code"]: d for d in self.diseases} | |
| # ---- ORPHA / OMIM / CCRD duplicates -> one group per disease ---- | |
| self.mapping: dict = json.load(open(mapping_json, encoding="utf-8")) | |
| codes_by_name: dict[str, set[str]] = defaultdict(set) | |
| for code, name in self.mapping.items(): | |
| codes_by_name[str(name)].add(code) | |
| self._group = {c: self._representative(cs) for cs in codes_by_name.values() for c in cs} | |
| self.diseases_by_group: dict[str, list[dict]] = defaultdict(list) | |
| for d in self.diseases: | |
| self.diseases_by_group[self.group(d["code"])].append(d) | |
| self._extended: dict[str, dict[str, float]] = {} | |
| def from_dir(cls, data_dir: Path, decontaminate: bool = False) -> "KnowledgeBase": | |
| data_dir = Path(data_dir) | |
| removed = None | |
| if decontaminate: | |
| from .data_loader import phenopacket_test_publications | |
| removed = phenopacket_test_publications(data_dir) | |
| return cls(data_dir / "hp.json", data_dir / "phenotype.hpoa", | |
| data_dir / "rarebench" / "mapping" / "disease_mapping.json", removed) | |
| # ---- hierarchy ---- | |
| def ancestors(self, code: str) -> set[str]: | |
| """The code and all its ancestors in the HPO hierarchy.""" | |
| if code not in self._ancestors: | |
| res = {code} | |
| for p in self.parents.get(code, ()): | |
| res |= self.ancestors(p) | |
| self._ancestors[code] = res | |
| return self._ancestors[code] | |
| def expand(self, codes) -> set[str]: | |
| return set().union(*(self.ancestors(c) for c in codes)) if codes else set() | |
| def is_metabolic(self, code: str) -> bool: | |
| return METABOLIC_ROOT in self.ancestors(code) | |
| # ---- diseases ---- | |
| def _representative(self, codes: set[str]) -> str: | |
| """Prefer an ORPHA / OMIM code that has a profile (readable name).""" | |
| return min(codes, key=lambda c: (c not in self.by_code, not c.startswith(("ORPHA", "OMIM")), c)) | |
| def group(self, code: str) -> str: | |
| return self._group.get(code, code) | |
| def disease_name(self, code: str) -> str: | |
| d = self.by_code.get(code) | |
| return str(d["name"] if d else self.mapping.get(code, code)).replace("@", "") | |
| def term_name(self, code: str) -> str: | |
| return self.terms[code]["name"] | |
| def extended_weights(self, group: str) -> dict[str, float]: | |
| """HPO term -> frequency in the disease, counting more specific terms too.""" | |
| if group not in self._extended: | |
| weights: dict[str, float] = {} | |
| for d in self.diseases_by_group.get(group, []): | |
| for s in d["symptoms"]: | |
| for a in self.ancestors(s["hpo"]): | |
| weights[a] = max(weights.get(a, 0), s["weight"]) | |
| self._extended[group] = weights | |
| return self._extended[group] | |