File size: 4,241 Bytes
613af80 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 | """Download and load the public data: HPO, disease annotations, RareBench, Phenopacket Store.
Nothing is redistributed with the repository: every file is downloaded at run time
into the data folder (see Settings.data_dir).
"""
from __future__ import annotations
import glob
import json
import os
import random
import urllib.request
import zipfile
from pathlib import Path
# HPO_RELEASE=2026-09-01 pins a release (same version as the LLM cache); otherwise the latest one is used
_HPO_RELEASE = os.environ.get("HPO_RELEASE")
HPO_URL = (f"https://github.com/obophenotype/human-phenotype-ontology/releases/download/v{_HPO_RELEASE}/"
if _HPO_RELEASE else "https://github.com/obophenotype/human-phenotype-ontology/releases/latest/download/")
RAREBENCH_REPO = "chenxz/RareBench"
DATASETS = ("RAMEDIS", "LIRICAL", "HMS", "MME")
def download(url: str, path: Path, attempts: int = 4) -> Path:
"""Download a file once (skipped if it already exists).
The file is written to "<name>.part" and renamed only when complete, so an
interrupted download never leaves a truncated file behind.
"""
import time
path = Path(path)
if path.exists():
return path
part = path.with_name(path.name + ".part")
for i in range(attempts):
try:
print(f"Downloading: {path.name}" + (f" (attempt {i + 1})" if i else ""))
urllib.request.urlretrieve(url, part)
part.replace(path)
return path
except Exception as e: # noqa: BLE001 - network errors of any kind
print(f" failed: {str(e)[:100]}")
part.unlink(missing_ok=True)
time.sleep(5 * (i + 1))
raise RuntimeError(f"Could not download {url}. Check your connection, or copy {path.name} into {path.parent}.")
def download_public_data(data_dir: Path) -> None:
"""HPO ontology, HPO disease annotations and the RareBench benchmark."""
data_dir = Path(data_dir)
download(HPO_URL + "hp.json", data_dir / "hp.json")
download(HPO_URL + "phenotype.hpoa", data_dir / "phenotype.hpoa")
from huggingface_hub import hf_hub_download
rb = data_dir / "rarebench"
if not glob.glob(f"{rb}/data/**/*.json*", recursive=True):
archive = hf_hub_download(RAREBENCH_REPO, "data.zip", repo_type="dataset", local_dir=rb)
zipfile.ZipFile(archive).extractall(rb / "data")
hf_hub_download(RAREBENCH_REPO, "mapping/disease_mapping.json", repo_type="dataset", local_dir=rb)
def _read_cases(path: str) -> list[dict]:
text = open(path, encoding="utf-8").read().strip()
return json.loads(text) if text.startswith("[") else [json.loads(l) for l in text.splitlines() if l.strip()]
def load_rarebench(data_dir: Path, name: str, known_terms: set[str]) -> tuple[list[dict], list[dict]]:
"""One RareBench dataset, split into two fixed halves (seed 0): (tuning, test).
Each case is {"codes": [HPO codes], "targets": [ORPHA / OMIM codes]}.
"""
files = glob.glob(f"{Path(data_dir) / 'rarebench'}/data/**/*", recursive=True)
path = next(f for f in files if os.path.isfile(f) and name.lower() in os.path.basename(f).lower())
cases = []
for c in _read_cases(path):
codes = [h for h in c["Phenotype"] if h in known_terms]
targets = [x for x in c["RareDisease"] if x.startswith(("ORPHA", "OMIM"))]
if codes and targets:
cases.append({"codes": codes, "targets": targets})
random.Random(0).shuffle(cases)
half = len(cases) // 2
return cases[:half], cases[half:]
def load_phenopacket_split(data_dir: Path) -> tuple[list[dict], list[dict]]:
"""Frozen train / test split of Phenopacket Store (created once by the notebook, cell 15).
Cases use the notebook keys: "codes", "exclus" (absent findings), "cibles" (targets), "publication".
"""
split = json.load(open(Path(data_dir) / "phenopacket_decoupage.json", encoding="utf-8"))
return split["apprentissage"], split["test"]
def phenopacket_test_publications(data_dir: Path) -> set[str]:
"""PMIDs of the Phenopacket test publications (removed from the base when decontaminating)."""
_, test = load_phenopacket_split(data_dir)
return {c["publication"] for c in test if c.get("publication")} |