File size: 6,045 Bytes
613af80
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
"""Knowledge base: HPO terms and hierarchy, disease profiles (Orphanet + OMIM), disease groups."""
from __future__ import annotations

import json
from collections import defaultdict
from pathlib import Path

import pandas as pd

METABOLIC_ROOT = "HP:0001939"   # Abnormality of metabolism / homeostasis

# HPO frequency terms -> weight
FREQUENCY_WEIGHTS = {"HP:0040280": 1.0, "HP:0040281": 0.9, "HP:0040282": 0.55,
                     "HP:0040283": 0.17, "HP:0040284": 0.025}


def frequency_weight(f) -> float:
    """HPO frequency (term, "n/m" or "x%") -> weight between 0 and 1 (0.5 if unknown)."""
    if not isinstance(f, str) or not f:
        return 0.5
    if f in FREQUENCY_WEIGHTS:
        return FREQUENCY_WEIGHTS[f]
    if "/" in f:
        n, d = f.split("/")
        return int(n) / int(d) if int(d) else 0.5
    if f.endswith("%"):
        return float(f[:-1]) / 100
    return 0.5


class KnowledgeBase:
    """HPO ontology + disease profiles, with optional leave-publication-out decontamination."""

    def __init__(self, hpo_json: Path, hpoa: Path, mapping_json: Path, removed_pmids: set[str] | None = None):
        graph = json.load(open(hpo_json, encoding="utf-8"))["graphs"][0]
        self.version = graph.get("meta", {}).get("version", "unknown")

        # ---- HPO terms and hierarchy ----
        self.terms: dict[str, dict] = {}
        for n in graph["nodes"]:
            if "HP_" in n.get("id", "") and "lbl" in n:
                meta = n.get("meta", {})
                self.terms["HP:" + n["id"].split("HP_")[-1]] = {
                    "name": n["lbl"], "synonyms": [s["val"] for s in meta.get("synonyms", [])]}
        self.parents: dict[str, set[str]] = defaultdict(set)
        for e in graph["edges"]:
            if e.get("pred") == "is_a" and "HP_" in e["sub"] and "HP_" in e["obj"]:
                self.parents["HP:" + e["sub"].split("HP_")[-1]].add("HP:" + e["obj"].split("HP_")[-1])
        self._ancestors: dict[str, set[str]] = {}

        # ---- Disease profiles with frequencies ----
        table = pd.read_csv(hpoa, sep="\t", comment="#", dtype=str)
        table = table[(table["aspect"] == "P") & (table["qualifier"] != "NOT") & table["hpo_id"].isin(self.terms.keys())]
        self.removed_annotations = 0
        if removed_pmids:
            mask = table["reference"].fillna("").str.split(";").apply(
                lambda refs: any(r.strip() in removed_pmids for r in refs))
            self.removed_annotations = int(mask.sum())
            table = table[~mask]
        self.diseases: list[dict] = []
        for prefix in ("ORPHA", "OMIM"):
            rows = table[table["database_id"].str.startswith(prefix)]
            for (code, name), grp in rows.groupby(["database_id", "disease_name"]):
                symptoms: dict[str, float] = {}
                for h, f in zip(grp["hpo_id"], grp["frequency"]):
                    symptoms[h] = max(symptoms.get(h, 0), frequency_weight(f))
                self.diseases.append({"code": code, "name": name,
                                      "symptoms": [{"hpo": h, "weight": w} for h, w in symptoms.items()]})
        self.by_code = {d["code"]: d for d in self.diseases}

        # ---- ORPHA / OMIM / CCRD duplicates -> one group per disease ----
        self.mapping: dict = json.load(open(mapping_json, encoding="utf-8"))
        codes_by_name: dict[str, set[str]] = defaultdict(set)
        for code, name in self.mapping.items():
            codes_by_name[str(name)].add(code)
        self._group = {c: self._representative(cs) for cs in codes_by_name.values() for c in cs}
        self.diseases_by_group: dict[str, list[dict]] = defaultdict(list)
        for d in self.diseases:
            self.diseases_by_group[self.group(d["code"])].append(d)
        self._extended: dict[str, dict[str, float]] = {}

    @classmethod
    def from_dir(cls, data_dir: Path, decontaminate: bool = False) -> "KnowledgeBase":
        data_dir = Path(data_dir)
        removed = None
        if decontaminate:
            from .data_loader import phenopacket_test_publications
            removed = phenopacket_test_publications(data_dir)
        return cls(data_dir / "hp.json", data_dir / "phenotype.hpoa",
                   data_dir / "rarebench" / "mapping" / "disease_mapping.json", removed)

    # ---- hierarchy ----
    def ancestors(self, code: str) -> set[str]:
        """The code and all its ancestors in the HPO hierarchy."""
        if code not in self._ancestors:
            res = {code}
            for p in self.parents.get(code, ()):
                res |= self.ancestors(p)
            self._ancestors[code] = res
        return self._ancestors[code]

    def expand(self, codes) -> set[str]:
        return set().union(*(self.ancestors(c) for c in codes)) if codes else set()

    def is_metabolic(self, code: str) -> bool:
        return METABOLIC_ROOT in self.ancestors(code)

    # ---- diseases ----
    def _representative(self, codes: set[str]) -> str:
        """Prefer an ORPHA / OMIM code that has a profile (readable name)."""
        return min(codes, key=lambda c: (c not in self.by_code, not c.startswith(("ORPHA", "OMIM")), c))

    def group(self, code: str) -> str:
        return self._group.get(code, code)

    def disease_name(self, code: str) -> str:
        d = self.by_code.get(code)
        return str(d["name"] if d else self.mapping.get(code, code)).replace("@", "")

    def term_name(self, code: str) -> str:
        return self.terms[code]["name"]

    def extended_weights(self, group: str) -> dict[str, float]:
        """HPO term -> frequency in the disease, counting more specific terms too."""
        if group not in self._extended:
            weights: dict[str, float] = {}
            for d in self.diseases_by_group.get(group, []):
                for s in d["symptoms"]:
                    for a in self.ancestors(s["hpo"]):
                        weights[a] = max(weights.get(a, 0), s["weight"])
            self._extended[group] = weights
        return self._extended[group]