File size: 4,241 Bytes
613af80
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
"""Download and load the public data: HPO, disease annotations, RareBench, Phenopacket Store.

Nothing is redistributed with the repository: every file is downloaded at run time
into the data folder (see Settings.data_dir).
"""
from __future__ import annotations

import glob
import json
import os
import random
import urllib.request
import zipfile
from pathlib import Path

# HPO_RELEASE=2026-09-01 pins a release (same version as the LLM cache); otherwise the latest one is used
_HPO_RELEASE = os.environ.get("HPO_RELEASE")
HPO_URL = (f"https://github.com/obophenotype/human-phenotype-ontology/releases/download/v{_HPO_RELEASE}/"
           if _HPO_RELEASE else "https://github.com/obophenotype/human-phenotype-ontology/releases/latest/download/")
RAREBENCH_REPO = "chenxz/RareBench"
DATASETS = ("RAMEDIS", "LIRICAL", "HMS", "MME")


def download(url: str, path: Path, attempts: int = 4) -> Path:
    """Download a file once (skipped if it already exists).

    The file is written to "<name>.part" and renamed only when complete, so an
    interrupted download never leaves a truncated file behind.
    """
    import time

    path = Path(path)
    if path.exists():
        return path
    part = path.with_name(path.name + ".part")
    for i in range(attempts):
        try:
            print(f"Downloading: {path.name}" + (f" (attempt {i + 1})" if i else ""))
            urllib.request.urlretrieve(url, part)
            part.replace(path)
            return path
        except Exception as e:  # noqa: BLE001 - network errors of any kind
            print(f"  failed: {str(e)[:100]}")
            part.unlink(missing_ok=True)
            time.sleep(5 * (i + 1))
    raise RuntimeError(f"Could not download {url}. Check your connection, or copy {path.name} into {path.parent}.")


def download_public_data(data_dir: Path) -> None:
    """HPO ontology, HPO disease annotations and the RareBench benchmark."""
    data_dir = Path(data_dir)
    download(HPO_URL + "hp.json", data_dir / "hp.json")
    download(HPO_URL + "phenotype.hpoa", data_dir / "phenotype.hpoa")

    from huggingface_hub import hf_hub_download

    rb = data_dir / "rarebench"
    if not glob.glob(f"{rb}/data/**/*.json*", recursive=True):
        archive = hf_hub_download(RAREBENCH_REPO, "data.zip", repo_type="dataset", local_dir=rb)
        zipfile.ZipFile(archive).extractall(rb / "data")
    hf_hub_download(RAREBENCH_REPO, "mapping/disease_mapping.json", repo_type="dataset", local_dir=rb)


def _read_cases(path: str) -> list[dict]:
    text = open(path, encoding="utf-8").read().strip()
    return json.loads(text) if text.startswith("[") else [json.loads(l) for l in text.splitlines() if l.strip()]


def load_rarebench(data_dir: Path, name: str, known_terms: set[str]) -> tuple[list[dict], list[dict]]:
    """One RareBench dataset, split into two fixed halves (seed 0): (tuning, test).

    Each case is {"codes": [HPO codes], "targets": [ORPHA / OMIM codes]}.
    """
    files = glob.glob(f"{Path(data_dir) / 'rarebench'}/data/**/*", recursive=True)
    path = next(f for f in files if os.path.isfile(f) and name.lower() in os.path.basename(f).lower())
    cases = []
    for c in _read_cases(path):
        codes = [h for h in c["Phenotype"] if h in known_terms]
        targets = [x for x in c["RareDisease"] if x.startswith(("ORPHA", "OMIM"))]
        if codes and targets:
            cases.append({"codes": codes, "targets": targets})
    random.Random(0).shuffle(cases)
    half = len(cases) // 2
    return cases[:half], cases[half:]


def load_phenopacket_split(data_dir: Path) -> tuple[list[dict], list[dict]]:
    """Frozen train / test split of Phenopacket Store (created once by the notebook, cell 15).

    Cases use the notebook keys: "codes", "exclus" (absent findings), "cibles" (targets), "publication".
    """
    split = json.load(open(Path(data_dir) / "phenopacket_decoupage.json", encoding="utf-8"))
    return split["apprentissage"], split["test"]


def phenopacket_test_publications(data_dir: Path) -> set[str]:
    """PMIDs of the Phenopacket test publications (removed from the base when decontaminating)."""
    _, test = load_phenopacket_split(data_dir)
    return {c["publication"] for c in test if c.get("publication")}