"""Download and load the public data: HPO, disease annotations, RareBench, Phenopacket Store. Nothing is redistributed with the repository: every file is downloaded at run time into the data folder (see Settings.data_dir). """ from __future__ import annotations import glob import json import os import random import urllib.request import zipfile from pathlib import Path # HPO_RELEASE=2026-09-01 pins a release (same version as the LLM cache); otherwise the latest one is used _HPO_RELEASE = os.environ.get("HPO_RELEASE") HPO_URL = (f"https://github.com/obophenotype/human-phenotype-ontology/releases/download/v{_HPO_RELEASE}/" if _HPO_RELEASE else "https://github.com/obophenotype/human-phenotype-ontology/releases/latest/download/") RAREBENCH_REPO = "chenxz/RareBench" DATASETS = ("RAMEDIS", "LIRICAL", "HMS", "MME") def download(url: str, path: Path, attempts: int = 4) -> Path: """Download a file once (skipped if it already exists). The file is written to ".part" and renamed only when complete, so an interrupted download never leaves a truncated file behind. """ import time path = Path(path) if path.exists(): return path part = path.with_name(path.name + ".part") for i in range(attempts): try: print(f"Downloading: {path.name}" + (f" (attempt {i + 1})" if i else "")) urllib.request.urlretrieve(url, part) part.replace(path) return path except Exception as e: # noqa: BLE001 - network errors of any kind print(f" failed: {str(e)[:100]}") part.unlink(missing_ok=True) time.sleep(5 * (i + 1)) raise RuntimeError(f"Could not download {url}. Check your connection, or copy {path.name} into {path.parent}.") def download_public_data(data_dir: Path) -> None: """HPO ontology, HPO disease annotations and the RareBench benchmark.""" data_dir = Path(data_dir) download(HPO_URL + "hp.json", data_dir / "hp.json") download(HPO_URL + "phenotype.hpoa", data_dir / "phenotype.hpoa") from huggingface_hub import hf_hub_download rb = data_dir / "rarebench" if not glob.glob(f"{rb}/data/**/*.json*", recursive=True): archive = hf_hub_download(RAREBENCH_REPO, "data.zip", repo_type="dataset", local_dir=rb) zipfile.ZipFile(archive).extractall(rb / "data") hf_hub_download(RAREBENCH_REPO, "mapping/disease_mapping.json", repo_type="dataset", local_dir=rb) def _read_cases(path: str) -> list[dict]: text = open(path, encoding="utf-8").read().strip() return json.loads(text) if text.startswith("[") else [json.loads(l) for l in text.splitlines() if l.strip()] def load_rarebench(data_dir: Path, name: str, known_terms: set[str]) -> tuple[list[dict], list[dict]]: """One RareBench dataset, split into two fixed halves (seed 0): (tuning, test). Each case is {"codes": [HPO codes], "targets": [ORPHA / OMIM codes]}. """ files = glob.glob(f"{Path(data_dir) / 'rarebench'}/data/**/*", recursive=True) path = next(f for f in files if os.path.isfile(f) and name.lower() in os.path.basename(f).lower()) cases = [] for c in _read_cases(path): codes = [h for h in c["Phenotype"] if h in known_terms] targets = [x for x in c["RareDisease"] if x.startswith(("ORPHA", "OMIM"))] if codes and targets: cases.append({"codes": codes, "targets": targets}) random.Random(0).shuffle(cases) half = len(cases) // 2 return cases[:half], cases[half:] def load_phenopacket_split(data_dir: Path) -> tuple[list[dict], list[dict]]: """Frozen train / test split of Phenopacket Store (created once by the notebook, cell 15). Cases use the notebook keys: "codes", "exclus" (absent findings), "cibles" (targets), "publication". """ split = json.load(open(Path(data_dir) / "phenopacket_decoupage.json", encoding="utf-8")) return split["apprentissage"], split["test"] def phenopacket_test_publications(data_dir: Path) -> set[str]: """PMIDs of the Phenopacket test publications (removed from the base when decontaminating).""" _, test = load_phenopacket_split(data_dir) return {c["publication"] for c in test if c.get("publication")}