RareDx / src /knowledge_base.py
sarra2002's picture
Démo RareDx
613af80 verified
Raw History Blame Contribute Delete
6.05 kB
"""Knowledge base: HPO terms and hierarchy, disease profiles (Orphanet + OMIM), disease groups."""
from __future__ import annotations
import json
from collections import defaultdict
from pathlib import Path
import pandas as pd
METABOLIC_ROOT = "HP:0001939" # Abnormality of metabolism / homeostasis
# HPO frequency terms -> weight
FREQUENCY_WEIGHTS = {"HP:0040280": 1.0, "HP:0040281": 0.9, "HP:0040282": 0.55,
"HP:0040283": 0.17, "HP:0040284": 0.025}
def frequency_weight(f) -> float:
"""HPO frequency (term, "n/m" or "x%") -> weight between 0 and 1 (0.5 if unknown)."""
if not isinstance(f, str) or not f:
return 0.5
if f in FREQUENCY_WEIGHTS:
return FREQUENCY_WEIGHTS[f]
if "/" in f:
n, d = f.split("/")
return int(n) / int(d) if int(d) else 0.5
if f.endswith("%"):
return float(f[:-1]) / 100
return 0.5
class KnowledgeBase:
"""HPO ontology + disease profiles, with optional leave-publication-out decontamination."""
def __init__(self, hpo_json: Path, hpoa: Path, mapping_json: Path, removed_pmids: set[str] | None = None):
graph = json.load(open(hpo_json, encoding="utf-8"))["graphs"][0]
self.version = graph.get("meta", {}).get("version", "unknown")
# ---- HPO terms and hierarchy ----
self.terms: dict[str, dict] = {}
for n in graph["nodes"]:
if "HP_" in n.get("id", "") and "lbl" in n:
meta = n.get("meta", {})
self.terms["HP:" + n["id"].split("HP_")[-1]] = {
"name": n["lbl"], "synonyms": [s["val"] for s in meta.get("synonyms", [])]}
self.parents: dict[str, set[str]] = defaultdict(set)
for e in graph["edges"]:
if e.get("pred") == "is_a" and "HP_" in e["sub"] and "HP_" in e["obj"]:
self.parents["HP:" + e["sub"].split("HP_")[-1]].add("HP:" + e["obj"].split("HP_")[-1])
self._ancestors: dict[str, set[str]] = {}
# ---- Disease profiles with frequencies ----
table = pd.read_csv(hpoa, sep="\t", comment="#", dtype=str)
table = table[(table["aspect"] == "P") & (table["qualifier"] != "NOT") & table["hpo_id"].isin(self.terms.keys())]
self.removed_annotations = 0
if removed_pmids:
mask = table["reference"].fillna("").str.split(";").apply(
lambda refs: any(r.strip() in removed_pmids for r in refs))
self.removed_annotations = int(mask.sum())
table = table[~mask]
self.diseases: list[dict] = []
for prefix in ("ORPHA", "OMIM"):
rows = table[table["database_id"].str.startswith(prefix)]
for (code, name), grp in rows.groupby(["database_id", "disease_name"]):
symptoms: dict[str, float] = {}
for h, f in zip(grp["hpo_id"], grp["frequency"]):
symptoms[h] = max(symptoms.get(h, 0), frequency_weight(f))
self.diseases.append({"code": code, "name": name,
"symptoms": [{"hpo": h, "weight": w} for h, w in symptoms.items()]})
self.by_code = {d["code"]: d for d in self.diseases}
# ---- ORPHA / OMIM / CCRD duplicates -> one group per disease ----
self.mapping: dict = json.load(open(mapping_json, encoding="utf-8"))
codes_by_name: dict[str, set[str]] = defaultdict(set)
for code, name in self.mapping.items():
codes_by_name[str(name)].add(code)
self._group = {c: self._representative(cs) for cs in codes_by_name.values() for c in cs}
self.diseases_by_group: dict[str, list[dict]] = defaultdict(list)
for d in self.diseases:
self.diseases_by_group[self.group(d["code"])].append(d)
self._extended: dict[str, dict[str, float]] = {}
@classmethod
def from_dir(cls, data_dir: Path, decontaminate: bool = False) -> "KnowledgeBase":
data_dir = Path(data_dir)
removed = None
if decontaminate:
from .data_loader import phenopacket_test_publications
removed = phenopacket_test_publications(data_dir)
return cls(data_dir / "hp.json", data_dir / "phenotype.hpoa",
data_dir / "rarebench" / "mapping" / "disease_mapping.json", removed)
# ---- hierarchy ----
def ancestors(self, code: str) -> set[str]:
"""The code and all its ancestors in the HPO hierarchy."""
if code not in self._ancestors:
res = {code}
for p in self.parents.get(code, ()):
res |= self.ancestors(p)
self._ancestors[code] = res
return self._ancestors[code]
def expand(self, codes) -> set[str]:
return set().union(*(self.ancestors(c) for c in codes)) if codes else set()
def is_metabolic(self, code: str) -> bool:
return METABOLIC_ROOT in self.ancestors(code)
# ---- diseases ----
def _representative(self, codes: set[str]) -> str:
"""Prefer an ORPHA / OMIM code that has a profile (readable name)."""
return min(codes, key=lambda c: (c not in self.by_code, not c.startswith(("ORPHA", "OMIM")), c))
def group(self, code: str) -> str:
return self._group.get(code, code)
def disease_name(self, code: str) -> str:
d = self.by_code.get(code)
return str(d["name"] if d else self.mapping.get(code, code)).replace("@", "")
def term_name(self, code: str) -> str:
return self.terms[code]["name"]
def extended_weights(self, group: str) -> dict[str, float]:
"""HPO term -> frequency in the disease, counting more specific terms too."""
if group not in self._extended:
weights: dict[str, float] = {}
for d in self.diseases_by_group.get(group, []):
for s in d["symptoms"]:
for a in self.ancestors(s["hpo"]):
weights[a] = max(weights.get(a, 0), s["weight"])
self._extended[group] = weights
return self._extended[group]