Download src/data_loader.py from sarra2002/RareDx: direct link, hf CLI and curl.
- Browser
- Download file 4.24 kB
-
https://huggingface.co/spaces/sarra2002/RareDx/resolve/main/src/data_loader.py
- Command line
-
hf download hf://spaces/sarra2002/RareDx/src/data_loader.py
-
curl -L -o data_loader.py https://huggingface.co/spaces/sarra2002/RareDx/resolve/main/src/data_loader.py
4.24 kB
| """Download and load the public data: HPO, disease annotations, RareBench, Phenopacket Store. | |
| Nothing is redistributed with the repository: every file is downloaded at run time | |
| into the data folder (see Settings.data_dir). | |
| """ | |
| from __future__ import annotations | |
| import glob | |
| import json | |
| import os | |
| import random | |
| import urllib.request | |
| import zipfile | |
| from pathlib import Path | |
| # HPO_RELEASE=2026-09-01 pins a release (same version as the LLM cache); otherwise the latest one is used | |
| _HPO_RELEASE = os.environ.get("HPO_RELEASE") | |
| HPO_URL = (f"https://github.com/obophenotype/human-phenotype-ontology/releases/download/v{_HPO_RELEASE}/" | |
| if _HPO_RELEASE else "https://github.com/obophenotype/human-phenotype-ontology/releases/latest/download/") | |
| RAREBENCH_REPO = "chenxz/RareBench" | |
| DATASETS = ("RAMEDIS", "LIRICAL", "HMS", "MME") | |
| def download(url: str, path: Path, attempts: int = 4) -> Path: | |
| """Download a file once (skipped if it already exists). | |
| The file is written to "<name>.part" and renamed only when complete, so an | |
| interrupted download never leaves a truncated file behind. | |
| """ | |
| import time | |
| path = Path(path) | |
| if path.exists(): | |
| return path | |
| part = path.with_name(path.name + ".part") | |
| for i in range(attempts): | |
| try: | |
| print(f"Downloading: {path.name}" + (f" (attempt {i + 1})" if i else "")) | |
| urllib.request.urlretrieve(url, part) | |
| part.replace(path) | |
| return path | |
| except Exception as e: # noqa: BLE001 - network errors of any kind | |
| print(f" failed: {str(e)[:100]}") | |
| part.unlink(missing_ok=True) | |
| time.sleep(5 * (i + 1)) | |
| raise RuntimeError(f"Could not download {url}. Check your connection, or copy {path.name} into {path.parent}.") | |
| def download_public_data(data_dir: Path) -> None: | |
| """HPO ontology, HPO disease annotations and the RareBench benchmark.""" | |
| data_dir = Path(data_dir) | |
| download(HPO_URL + "hp.json", data_dir / "hp.json") | |
| download(HPO_URL + "phenotype.hpoa", data_dir / "phenotype.hpoa") | |
| from huggingface_hub import hf_hub_download | |
| rb = data_dir / "rarebench" | |
| if not glob.glob(f"{rb}/data/**/*.json*", recursive=True): | |
| archive = hf_hub_download(RAREBENCH_REPO, "data.zip", repo_type="dataset", local_dir=rb) | |
| zipfile.ZipFile(archive).extractall(rb / "data") | |
| hf_hub_download(RAREBENCH_REPO, "mapping/disease_mapping.json", repo_type="dataset", local_dir=rb) | |
| def _read_cases(path: str) -> list[dict]: | |
| text = open(path, encoding="utf-8").read().strip() | |
| return json.loads(text) if text.startswith("[") else [json.loads(l) for l in text.splitlines() if l.strip()] | |
| def load_rarebench(data_dir: Path, name: str, known_terms: set[str]) -> tuple[list[dict], list[dict]]: | |
| """One RareBench dataset, split into two fixed halves (seed 0): (tuning, test). | |
| Each case is {"codes": [HPO codes], "targets": [ORPHA / OMIM codes]}. | |
| """ | |
| files = glob.glob(f"{Path(data_dir) / 'rarebench'}/data/**/*", recursive=True) | |
| path = next(f for f in files if os.path.isfile(f) and name.lower() in os.path.basename(f).lower()) | |
| cases = [] | |
| for c in _read_cases(path): | |
| codes = [h for h in c["Phenotype"] if h in known_terms] | |
| targets = [x for x in c["RareDisease"] if x.startswith(("ORPHA", "OMIM"))] | |
| if codes and targets: | |
| cases.append({"codes": codes, "targets": targets}) | |
| random.Random(0).shuffle(cases) | |
| half = len(cases) // 2 | |
| return cases[:half], cases[half:] | |
| def load_phenopacket_split(data_dir: Path) -> tuple[list[dict], list[dict]]: | |
| """Frozen train / test split of Phenopacket Store (created once by the notebook, cell 15). | |
| Cases use the notebook keys: "codes", "exclus" (absent findings), "cibles" (targets), "publication". | |
| """ | |
| split = json.load(open(Path(data_dir) / "phenopacket_decoupage.json", encoding="utf-8")) | |
| return split["apprentissage"], split["test"] | |
| def phenopacket_test_publications(data_dir: Path) -> set[str]: | |
| """PMIDs of the Phenopacket test publications (removed from the base when decontaminating).""" | |
| _, test = load_phenopacket_split(data_dir) | |
| return {c["publication"] for c in test if c.get("publication")} |