perde / eval /fetch_sample.py
Traxic's picture
Claude Opus 5.5
Add Pərdə: local masking of Azerbaijani personal data for AI requests
6684cad
Raw History Blame Contribute Delete
1.59 kB
"""Download fixed, reproducible, non-overlapping splits of LocalDoc/pii_ner_azerbaijani (CC BY 4.0).
Usage: python eval/fetch_sample.py
Output:
data/localdoc_test.jsonl 1,000 rows, used only for reported results
data/localdoc_dev.jsonl 2,000 rows, used for tuning and to build the name lexicon
"""
from __future__ import annotations
import json
import random
import urllib.request
from pathlib import Path
API = "https://datasets-server.huggingface.co/rows?dataset=LocalDoc/pii_ner_azerbaijani&config=default&split=train&offset={offset}&length=100"
TOTAL_ROWS = 120_634
SEED = 2026
DATA = Path(__file__).resolve().parents[1] / "data"
def write(offsets: list[int], path: Path) -> None:
n = 0
with path.open("w", encoding="utf-8") as f:
for off in sorted(offsets):
with urllib.request.urlopen(API.format(offset=off), timeout=60) as r:
rows = json.load(r)["rows"]
for row in rows:
d = row["row"]
f.write(json.dumps({
"uid": d["uid"],
"text": d["translated_text"],
"labels": json.loads(d["privacy_mask"]),
}, ensure_ascii=False) + "\n")
n += 1
print(f"wrote {n} rows to {path.name}")
def main() -> None:
rng = random.Random(SEED)
pages = rng.sample(range(0, TOTAL_ROWS - 100, 100), 30) # 30 distinct pages, no overlap
DATA.mkdir(exist_ok=True)
write(pages[:10], DATA / "localdoc_test.jsonl")
write(pages[10:], DATA / "localdoc_dev.jsonl")
if __name__ == "__main__":
main()