Download eval/fetch_sample.py from Traxic/perde: direct link, hf CLI and curl.
- Browser
- Download file 1.59 kB
-
https://huggingface.co/spaces/Traxic/perde/resolve/main/eval/fetch_sample.py
- Command line
-
hf download hf://spaces/Traxic/perde/eval/fetch_sample.py
-
curl -L -o fetch_sample.py https://huggingface.co/spaces/Traxic/perde/resolve/main/eval/fetch_sample.py
1.59 kB
| """Download fixed, reproducible, non-overlapping splits of LocalDoc/pii_ner_azerbaijani (CC BY 4.0). | |
| Usage: python eval/fetch_sample.py | |
| Output: | |
| data/localdoc_test.jsonl 1,000 rows, used only for reported results | |
| data/localdoc_dev.jsonl 2,000 rows, used for tuning and to build the name lexicon | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import random | |
| import urllib.request | |
| from pathlib import Path | |
| API = "https://datasets-server.huggingface.co/rows?dataset=LocalDoc/pii_ner_azerbaijani&config=default&split=train&offset={offset}&length=100" | |
| TOTAL_ROWS = 120_634 | |
| SEED = 2026 | |
| DATA = Path(__file__).resolve().parents[1] / "data" | |
| def write(offsets: list[int], path: Path) -> None: | |
| n = 0 | |
| with path.open("w", encoding="utf-8") as f: | |
| for off in sorted(offsets): | |
| with urllib.request.urlopen(API.format(offset=off), timeout=60) as r: | |
| rows = json.load(r)["rows"] | |
| for row in rows: | |
| d = row["row"] | |
| f.write(json.dumps({ | |
| "uid": d["uid"], | |
| "text": d["translated_text"], | |
| "labels": json.loads(d["privacy_mask"]), | |
| }, ensure_ascii=False) + "\n") | |
| n += 1 | |
| print(f"wrote {n} rows to {path.name}") | |
| def main() -> None: | |
| rng = random.Random(SEED) | |
| pages = rng.sample(range(0, TOTAL_ROWS - 100, 100), 30) # 30 distinct pages, no overlap | |
| DATA.mkdir(exist_ok=True) | |
| write(pages[:10], DATA / "localdoc_test.jsonl") | |
| write(pages[10:], DATA / "localdoc_dev.jsonl") | |
| if __name__ == "__main__": | |
| main() | |