apingali Claude Sonnet 4.6 commited on
Commit
76e6d27
·
1 Parent(s): bf0e212

feat(drive-and-save): load curated slice from private HF Dataset

Browse files

Implements data_source.py (Task 7) — the single module where network and
token access lives. Adds one mocked test (no network) that verifies repo_id,
repo_type, token forwarding, and correct str dtypes for procedure_code and
h3_cell. Full suite: 14/14 green.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Files changed (2) hide show
  1. data_source.py +29 -0
  2. test_core.py +33 -0
data_source.py ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # data_source.py
2
+ from __future__ import annotations
3
+
4
+ import os
5
+
6
+ import pandas as pd
7
+ from huggingface_hub import hf_hub_download
8
+
9
+
10
+ def get_token() -> "str | None":
11
+ return os.environ.get("HF_TOKEN")
12
+
13
+
14
+ def load_slices(cfg: dict, token: "str | None" = None):
15
+ """Download medians + metros CSVs from the PRIVATE HF Dataset and return
16
+ them as DataFrames. Token comes from the HF_TOKEN Space secret."""
17
+ d = cfg["data"]
18
+ token = token or get_token()
19
+ medians_path = hf_hub_download(
20
+ repo_id=d["dataset_repo"], filename=d["medians_file"],
21
+ repo_type="dataset", token=token,
22
+ )
23
+ metros_path = hf_hub_download(
24
+ repo_id=d["dataset_repo"], filename=d["metros_file"],
25
+ repo_type="dataset", token=token,
26
+ )
27
+ medians = pd.read_csv(medians_path, dtype={"procedure_code": str})
28
+ metros = pd.read_csv(metros_path, dtype={"h3_cell": str})
29
+ return medians, metros
test_core.py CHANGED
@@ -7,6 +7,7 @@ import pandas as pd
7
  import pytest
8
 
9
  import core
 
10
 
11
  # ---------------------------------------------------------------------------
12
  # Part A — config loading
@@ -150,3 +151,35 @@ def test_build_cta_url_uses_deep_link_pattern():
150
  assert "/procedure/45378/" in url
151
  assert "metro=colorado-springs" in url
152
  assert "utm_source=mile-hi-labs" in url
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7
  import pytest
8
 
9
  import core
10
+ import data_source
11
 
12
  # ---------------------------------------------------------------------------
13
  # Part A — config loading
 
151
  assert "/procedure/45378/" in url
152
  assert "metro=colorado-springs" in url
153
  assert "utm_source=mile-hi-labs" in url
154
+
155
+
156
+ # ---------------------------------------------------------------------------
157
+ # Part F — data_source: load curated slice from private HF Dataset
158
+ # ---------------------------------------------------------------------------
159
+
160
+
161
+ def test_load_slices_uses_token_and_reads_csvs(monkeypatch, tmp_path):
162
+ medians_csv = tmp_path / "medians.csv"
163
+ metros_csv = tmp_path / "metros.csv"
164
+ medians_csv.write_text(
165
+ "state,procedure_code,procedure_name,metro,median_price,n_hospitals\n"
166
+ "CO,45378,Diagnostic colonoscopy,Denver,2267,12\n"
167
+ )
168
+ metros_csv.write_text("state,metro,h3_cell\nCO,Denver,8a2a1072b59ffff\n")
169
+
170
+ calls = {}
171
+ def fake_download(repo_id, filename, repo_type, token):
172
+ calls["repo_id"] = repo_id
173
+ calls["repo_type"] = repo_type
174
+ calls["token"] = token
175
+ return str(medians_csv if filename.endswith("medians.csv") else metros_csv)
176
+
177
+ monkeypatch.setattr(data_source, "hf_hub_download", fake_download)
178
+ cfg = core.load_config()
179
+ medians, metros = data_source.load_slices(cfg, token="hf_test")
180
+
181
+ assert calls["repo_type"] == "dataset"
182
+ assert calls["token"] == "hf_test"
183
+ assert calls["repo_id"] == cfg["data"]["dataset_repo"]
184
+ assert list(medians["metro"]) == ["Denver"]
185
+ assert list(metros["metro"]) == ["Denver"]