File size: 1,594 Bytes
6684cad
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
"""Download fixed, reproducible, non-overlapping splits of LocalDoc/pii_ner_azerbaijani (CC BY 4.0).

Usage: python eval/fetch_sample.py
Output:
  data/localdoc_test.jsonl  1,000 rows, used only for reported results
  data/localdoc_dev.jsonl   2,000 rows, used for tuning and to build the name lexicon
"""
from __future__ import annotations

import json
import random
import urllib.request
from pathlib import Path

API = "https://datasets-server.huggingface.co/rows?dataset=LocalDoc/pii_ner_azerbaijani&config=default&split=train&offset={offset}&length=100"
TOTAL_ROWS = 120_634
SEED = 2026
DATA = Path(__file__).resolve().parents[1] / "data"


def write(offsets: list[int], path: Path) -> None:
    n = 0
    with path.open("w", encoding="utf-8") as f:
        for off in sorted(offsets):
            with urllib.request.urlopen(API.format(offset=off), timeout=60) as r:
                rows = json.load(r)["rows"]
            for row in rows:
                d = row["row"]
                f.write(json.dumps({
                    "uid": d["uid"],
                    "text": d["translated_text"],
                    "labels": json.loads(d["privacy_mask"]),
                }, ensure_ascii=False) + "\n")
                n += 1
    print(f"wrote {n} rows to {path.name}")


def main() -> None:
    rng = random.Random(SEED)
    pages = rng.sample(range(0, TOTAL_ROWS - 100, 100), 30)  # 30 distinct pages, no overlap
    DATA.mkdir(exist_ok=True)
    write(pages[:10], DATA / "localdoc_test.jsonl")
    write(pages[10:], DATA / "localdoc_dev.jsonl")


if __name__ == "__main__":
    main()