File size: 6,045 Bytes
613af80 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 | """Knowledge base: HPO terms and hierarchy, disease profiles (Orphanet + OMIM), disease groups."""
from __future__ import annotations
import json
from collections import defaultdict
from pathlib import Path
import pandas as pd
METABOLIC_ROOT = "HP:0001939" # Abnormality of metabolism / homeostasis
# HPO frequency terms -> weight
FREQUENCY_WEIGHTS = {"HP:0040280": 1.0, "HP:0040281": 0.9, "HP:0040282": 0.55,
"HP:0040283": 0.17, "HP:0040284": 0.025}
def frequency_weight(f) -> float:
"""HPO frequency (term, "n/m" or "x%") -> weight between 0 and 1 (0.5 if unknown)."""
if not isinstance(f, str) or not f:
return 0.5
if f in FREQUENCY_WEIGHTS:
return FREQUENCY_WEIGHTS[f]
if "/" in f:
n, d = f.split("/")
return int(n) / int(d) if int(d) else 0.5
if f.endswith("%"):
return float(f[:-1]) / 100
return 0.5
class KnowledgeBase:
"""HPO ontology + disease profiles, with optional leave-publication-out decontamination."""
def __init__(self, hpo_json: Path, hpoa: Path, mapping_json: Path, removed_pmids: set[str] | None = None):
graph = json.load(open(hpo_json, encoding="utf-8"))["graphs"][0]
self.version = graph.get("meta", {}).get("version", "unknown")
# ---- HPO terms and hierarchy ----
self.terms: dict[str, dict] = {}
for n in graph["nodes"]:
if "HP_" in n.get("id", "") and "lbl" in n:
meta = n.get("meta", {})
self.terms["HP:" + n["id"].split("HP_")[-1]] = {
"name": n["lbl"], "synonyms": [s["val"] for s in meta.get("synonyms", [])]}
self.parents: dict[str, set[str]] = defaultdict(set)
for e in graph["edges"]:
if e.get("pred") == "is_a" and "HP_" in e["sub"] and "HP_" in e["obj"]:
self.parents["HP:" + e["sub"].split("HP_")[-1]].add("HP:" + e["obj"].split("HP_")[-1])
self._ancestors: dict[str, set[str]] = {}
# ---- Disease profiles with frequencies ----
table = pd.read_csv(hpoa, sep="\t", comment="#", dtype=str)
table = table[(table["aspect"] == "P") & (table["qualifier"] != "NOT") & table["hpo_id"].isin(self.terms.keys())]
self.removed_annotations = 0
if removed_pmids:
mask = table["reference"].fillna("").str.split(";").apply(
lambda refs: any(r.strip() in removed_pmids for r in refs))
self.removed_annotations = int(mask.sum())
table = table[~mask]
self.diseases: list[dict] = []
for prefix in ("ORPHA", "OMIM"):
rows = table[table["database_id"].str.startswith(prefix)]
for (code, name), grp in rows.groupby(["database_id", "disease_name"]):
symptoms: dict[str, float] = {}
for h, f in zip(grp["hpo_id"], grp["frequency"]):
symptoms[h] = max(symptoms.get(h, 0), frequency_weight(f))
self.diseases.append({"code": code, "name": name,
"symptoms": [{"hpo": h, "weight": w} for h, w in symptoms.items()]})
self.by_code = {d["code"]: d for d in self.diseases}
# ---- ORPHA / OMIM / CCRD duplicates -> one group per disease ----
self.mapping: dict = json.load(open(mapping_json, encoding="utf-8"))
codes_by_name: dict[str, set[str]] = defaultdict(set)
for code, name in self.mapping.items():
codes_by_name[str(name)].add(code)
self._group = {c: self._representative(cs) for cs in codes_by_name.values() for c in cs}
self.diseases_by_group: dict[str, list[dict]] = defaultdict(list)
for d in self.diseases:
self.diseases_by_group[self.group(d["code"])].append(d)
self._extended: dict[str, dict[str, float]] = {}
@classmethod
def from_dir(cls, data_dir: Path, decontaminate: bool = False) -> "KnowledgeBase":
data_dir = Path(data_dir)
removed = None
if decontaminate:
from .data_loader import phenopacket_test_publications
removed = phenopacket_test_publications(data_dir)
return cls(data_dir / "hp.json", data_dir / "phenotype.hpoa",
data_dir / "rarebench" / "mapping" / "disease_mapping.json", removed)
# ---- hierarchy ----
def ancestors(self, code: str) -> set[str]:
"""The code and all its ancestors in the HPO hierarchy."""
if code not in self._ancestors:
res = {code}
for p in self.parents.get(code, ()):
res |= self.ancestors(p)
self._ancestors[code] = res
return self._ancestors[code]
def expand(self, codes) -> set[str]:
return set().union(*(self.ancestors(c) for c in codes)) if codes else set()
def is_metabolic(self, code: str) -> bool:
return METABOLIC_ROOT in self.ancestors(code)
# ---- diseases ----
def _representative(self, codes: set[str]) -> str:
"""Prefer an ORPHA / OMIM code that has a profile (readable name)."""
return min(codes, key=lambda c: (c not in self.by_code, not c.startswith(("ORPHA", "OMIM")), c))
def group(self, code: str) -> str:
return self._group.get(code, code)
def disease_name(self, code: str) -> str:
d = self.by_code.get(code)
return str(d["name"] if d else self.mapping.get(code, code)).replace("@", "")
def term_name(self, code: str) -> str:
return self.terms[code]["name"]
def extended_weights(self, group: str) -> dict[str, float]:
"""HPO term -> frequency in the disease, counting more specific terms too."""
if group not in self._extended:
weights: dict[str, float] = {}
for d in self.diseases_by_group.get(group, []):
for s in d["symptoms"]:
for a in self.ancestors(s["hpo"]):
weights[a] = max(weights.get(a, 0), s["weight"])
self._extended[group] = weights
return self._extended[group]
|