Spaces:
Sleeping
Sleeping
apingali Claude Sonnet 4.6 commited on
Commit ·
76e6d27
1
Parent(s): bf0e212
feat(drive-and-save): load curated slice from private HF Dataset
Browse filesImplements data_source.py (Task 7) — the single module where network and
token access lives. Adds one mocked test (no network) that verifies repo_id,
repo_type, token forwarding, and correct str dtypes for procedure_code and
h3_cell. Full suite: 14/14 green.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
- data_source.py +29 -0
- test_core.py +33 -0
data_source.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# data_source.py
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import os
|
| 5 |
+
|
| 6 |
+
import pandas as pd
|
| 7 |
+
from huggingface_hub import hf_hub_download
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def get_token() -> "str | None":
|
| 11 |
+
return os.environ.get("HF_TOKEN")
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def load_slices(cfg: dict, token: "str | None" = None):
|
| 15 |
+
"""Download medians + metros CSVs from the PRIVATE HF Dataset and return
|
| 16 |
+
them as DataFrames. Token comes from the HF_TOKEN Space secret."""
|
| 17 |
+
d = cfg["data"]
|
| 18 |
+
token = token or get_token()
|
| 19 |
+
medians_path = hf_hub_download(
|
| 20 |
+
repo_id=d["dataset_repo"], filename=d["medians_file"],
|
| 21 |
+
repo_type="dataset", token=token,
|
| 22 |
+
)
|
| 23 |
+
metros_path = hf_hub_download(
|
| 24 |
+
repo_id=d["dataset_repo"], filename=d["metros_file"],
|
| 25 |
+
repo_type="dataset", token=token,
|
| 26 |
+
)
|
| 27 |
+
medians = pd.read_csv(medians_path, dtype={"procedure_code": str})
|
| 28 |
+
metros = pd.read_csv(metros_path, dtype={"h3_cell": str})
|
| 29 |
+
return medians, metros
|
test_core.py
CHANGED
|
@@ -7,6 +7,7 @@ import pandas as pd
|
|
| 7 |
import pytest
|
| 8 |
|
| 9 |
import core
|
|
|
|
| 10 |
|
| 11 |
# ---------------------------------------------------------------------------
|
| 12 |
# Part A — config loading
|
|
@@ -150,3 +151,35 @@ def test_build_cta_url_uses_deep_link_pattern():
|
|
| 150 |
assert "/procedure/45378/" in url
|
| 151 |
assert "metro=colorado-springs" in url
|
| 152 |
assert "utm_source=mile-hi-labs" in url
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
import pytest
|
| 8 |
|
| 9 |
import core
|
| 10 |
+
import data_source
|
| 11 |
|
| 12 |
# ---------------------------------------------------------------------------
|
| 13 |
# Part A — config loading
|
|
|
|
| 151 |
assert "/procedure/45378/" in url
|
| 152 |
assert "metro=colorado-springs" in url
|
| 153 |
assert "utm_source=mile-hi-labs" in url
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
# ---------------------------------------------------------------------------
|
| 157 |
+
# Part F — data_source: load curated slice from private HF Dataset
|
| 158 |
+
# ---------------------------------------------------------------------------
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
def test_load_slices_uses_token_and_reads_csvs(monkeypatch, tmp_path):
|
| 162 |
+
medians_csv = tmp_path / "medians.csv"
|
| 163 |
+
metros_csv = tmp_path / "metros.csv"
|
| 164 |
+
medians_csv.write_text(
|
| 165 |
+
"state,procedure_code,procedure_name,metro,median_price,n_hospitals\n"
|
| 166 |
+
"CO,45378,Diagnostic colonoscopy,Denver,2267,12\n"
|
| 167 |
+
)
|
| 168 |
+
metros_csv.write_text("state,metro,h3_cell\nCO,Denver,8a2a1072b59ffff\n")
|
| 169 |
+
|
| 170 |
+
calls = {}
|
| 171 |
+
def fake_download(repo_id, filename, repo_type, token):
|
| 172 |
+
calls["repo_id"] = repo_id
|
| 173 |
+
calls["repo_type"] = repo_type
|
| 174 |
+
calls["token"] = token
|
| 175 |
+
return str(medians_csv if filename.endswith("medians.csv") else metros_csv)
|
| 176 |
+
|
| 177 |
+
monkeypatch.setattr(data_source, "hf_hub_download", fake_download)
|
| 178 |
+
cfg = core.load_config()
|
| 179 |
+
medians, metros = data_source.load_slices(cfg, token="hf_test")
|
| 180 |
+
|
| 181 |
+
assert calls["repo_type"] == "dataset"
|
| 182 |
+
assert calls["token"] == "hf_test"
|
| 183 |
+
assert calls["repo_id"] == cfg["data"]["dataset_repo"]
|
| 184 |
+
assert list(medians["metro"]) == ["Denver"]
|
| 185 |
+
assert list(metros["metro"]) == ["Denver"]
|