"""Knowledge base: HPO terms and hierarchy, disease profiles (Orphanet + OMIM), disease groups.""" from __future__ import annotations import json from collections import defaultdict from pathlib import Path import pandas as pd METABOLIC_ROOT = "HP:0001939" # Abnormality of metabolism / homeostasis # HPO frequency terms -> weight FREQUENCY_WEIGHTS = {"HP:0040280": 1.0, "HP:0040281": 0.9, "HP:0040282": 0.55, "HP:0040283": 0.17, "HP:0040284": 0.025} def frequency_weight(f) -> float: """HPO frequency (term, "n/m" or "x%") -> weight between 0 and 1 (0.5 if unknown).""" if not isinstance(f, str) or not f: return 0.5 if f in FREQUENCY_WEIGHTS: return FREQUENCY_WEIGHTS[f] if "/" in f: n, d = f.split("/") return int(n) / int(d) if int(d) else 0.5 if f.endswith("%"): return float(f[:-1]) / 100 return 0.5 class KnowledgeBase: """HPO ontology + disease profiles, with optional leave-publication-out decontamination.""" def __init__(self, hpo_json: Path, hpoa: Path, mapping_json: Path, removed_pmids: set[str] | None = None): graph = json.load(open(hpo_json, encoding="utf-8"))["graphs"][0] self.version = graph.get("meta", {}).get("version", "unknown") # ---- HPO terms and hierarchy ---- self.terms: dict[str, dict] = {} for n in graph["nodes"]: if "HP_" in n.get("id", "") and "lbl" in n: meta = n.get("meta", {}) self.terms["HP:" + n["id"].split("HP_")[-1]] = { "name": n["lbl"], "synonyms": [s["val"] for s in meta.get("synonyms", [])]} self.parents: dict[str, set[str]] = defaultdict(set) for e in graph["edges"]: if e.get("pred") == "is_a" and "HP_" in e["sub"] and "HP_" in e["obj"]: self.parents["HP:" + e["sub"].split("HP_")[-1]].add("HP:" + e["obj"].split("HP_")[-1]) self._ancestors: dict[str, set[str]] = {} # ---- Disease profiles with frequencies ---- table = pd.read_csv(hpoa, sep="\t", comment="#", dtype=str) table = table[(table["aspect"] == "P") & (table["qualifier"] != "NOT") & table["hpo_id"].isin(self.terms.keys())] self.removed_annotations = 0 if removed_pmids: mask = table["reference"].fillna("").str.split(";").apply( lambda refs: any(r.strip() in removed_pmids for r in refs)) self.removed_annotations = int(mask.sum()) table = table[~mask] self.diseases: list[dict] = [] for prefix in ("ORPHA", "OMIM"): rows = table[table["database_id"].str.startswith(prefix)] for (code, name), grp in rows.groupby(["database_id", "disease_name"]): symptoms: dict[str, float] = {} for h, f in zip(grp["hpo_id"], grp["frequency"]): symptoms[h] = max(symptoms.get(h, 0), frequency_weight(f)) self.diseases.append({"code": code, "name": name, "symptoms": [{"hpo": h, "weight": w} for h, w in symptoms.items()]}) self.by_code = {d["code"]: d for d in self.diseases} # ---- ORPHA / OMIM / CCRD duplicates -> one group per disease ---- self.mapping: dict = json.load(open(mapping_json, encoding="utf-8")) codes_by_name: dict[str, set[str]] = defaultdict(set) for code, name in self.mapping.items(): codes_by_name[str(name)].add(code) self._group = {c: self._representative(cs) for cs in codes_by_name.values() for c in cs} self.diseases_by_group: dict[str, list[dict]] = defaultdict(list) for d in self.diseases: self.diseases_by_group[self.group(d["code"])].append(d) self._extended: dict[str, dict[str, float]] = {} @classmethod def from_dir(cls, data_dir: Path, decontaminate: bool = False) -> "KnowledgeBase": data_dir = Path(data_dir) removed = None if decontaminate: from .data_loader import phenopacket_test_publications removed = phenopacket_test_publications(data_dir) return cls(data_dir / "hp.json", data_dir / "phenotype.hpoa", data_dir / "rarebench" / "mapping" / "disease_mapping.json", removed) # ---- hierarchy ---- def ancestors(self, code: str) -> set[str]: """The code and all its ancestors in the HPO hierarchy.""" if code not in self._ancestors: res = {code} for p in self.parents.get(code, ()): res |= self.ancestors(p) self._ancestors[code] = res return self._ancestors[code] def expand(self, codes) -> set[str]: return set().union(*(self.ancestors(c) for c in codes)) if codes else set() def is_metabolic(self, code: str) -> bool: return METABOLIC_ROOT in self.ancestors(code) # ---- diseases ---- def _representative(self, codes: set[str]) -> str: """Prefer an ORPHA / OMIM code that has a profile (readable name).""" return min(codes, key=lambda c: (c not in self.by_code, not c.startswith(("ORPHA", "OMIM")), c)) def group(self, code: str) -> str: return self._group.get(code, code) def disease_name(self, code: str) -> str: d = self.by_code.get(code) return str(d["name"] if d else self.mapping.get(code, code)).replace("@", "") def term_name(self, code: str) -> str: return self.terms[code]["name"] def extended_weights(self, group: str) -> dict[str, float]: """HPO term -> frequency in the disease, counting more specific terms too.""" if group not in self._extended: weights: dict[str, float] = {} for d in self.diseases_by_group.get(group, []): for s in d["symptoms"]: for a in self.ancestors(s["hpo"]): weights[a] = max(weights.get(a, 0), s["weight"]) self._extended[group] = weights return self._extended[group]