File size: 7,642 Bytes
6684cad
 
 
fb70710
6684cad
 
 
 
26b4286
 
6684cad
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
26b4286
 
 
 
6684cad
 
26b4286
 
6684cad
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fb70710
 
6684cad
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
"""Measure detection quality on the held-out LocalDoc split and on hand-written cases.

Usage: python eval/evaluate.py [--split test|dev] [--show-failures N]
Writes eval/results.json (test split) or eval/results_dev.json (dev split) and prints a Markdown table.

Metrics, per entity group:
  recall      gold values masked by a detection of the right type (>=50% of characters)
  protected   gold values masked by any detection (>=90% of characters); 1 - protected = leak rate
  precision   detections that overlap an in-scope labelled value of the same type; detections that
              touch only out-of-scope labels (city, time, age, zip) are left out of the count
"""
from __future__ import annotations

import argparse
import json
import re
import sys
from collections import defaultdict
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))

from perde.shield import detect  # noqa: E402

GOLD_GROUP = {
    "GIVENNAME": "PERSON", "SURNAME": "PERSON",
    "TELEPHONENUM": "PHONE", "EMAIL": "EMAIL",
    "IDCARDNUM": "ID_NUMBER", "PASSPORTNUM": "ID_NUMBER", "TAXNUM": "ID_NUMBER",
    "DRIVERLICENSENUM": "ID_NUMBER", "CREDITCARDNUMBER": "CARD",
    "DATE": "DATE", "STREET": "ADDRESS", "BUILDINGNUM": "ADDRESS",
}
OUT_OF_SCOPE = {"TIME", "CITY", "AGE", "ZIPCODE"}
PRED_GROUP = {
    "PERSON": "PERSON", "PHONE": "PHONE", "EMAIL": "EMAIL", "FIN": "ID_NUMBER",
    "ID_CARD": "ID_NUMBER", "PASSPORT": "ID_NUMBER", "VOEN": "ID_NUMBER",
    "IBAN": "ID_NUMBER", "DRIVER_LICENSE": "ID_NUMBER", "CAR_PLATE": "ID_NUMBER", "CARD": "CARD",
    "DATE": "DATE", "ADDRESS": "ADDRESS",
}
GROUPS = ["PERSON", "PHONE", "EMAIL", "ID_NUMBER", "CARD", "DATE", "ADDRESS"]


# --- systems under test -----------------------------------------------------

def system_perde(text):
    return [(s.start, s.end, PRED_GROUP[s.entity]) for s in detect(text)]


_GEN_EMAIL = re.compile(r"\b[\w.+-]+@[\w-]+(?:\.[\w-]+)+\b")
_GEN_PHONE = re.compile(r"(?<!\w)\+?\d[\d\s\-().]{7,}\d(?!\w)")
_GEN_DATE = re.compile(r"\b\d{4}-\d{2}-\d{2}\b|\b\d{1,2}/\d{1,2}/\d{2,4}\b")


def system_generic(text):
    """Language-agnostic regex baseline: what a default, non-localised filter catches."""
    out = [(m.start(), m.end(), "EMAIL") for m in _GEN_EMAIL.finditer(text)]
    out += [(m.start(), m.end(), "PHONE") for m in _GEN_PHONE.finditer(text)]
    out += [(m.start(), m.end(), "DATE") for m in _GEN_DATE.finditer(text)]
    return out


def system_no_names(text):
    return [p for p in system_perde(text) if p[2] != "PERSON"]


SYSTEMS = {"Pərdə": system_perde, "Pərdə without name detector": system_no_names,
           "Generic regex baseline": system_generic}


# --- scoring ----------------------------------------------------------------

def coverage(gs, ge, preds, group=None):
    covered = set()
    for ps, pe, pg in preds:
        if group is None or pg == group:
            covered.update(range(max(gs, ps), min(ge, pe)))
    span_chars = [i for i in range(gs, ge)]
    return len(covered) / max(1, len(span_chars))


def score(rows, system):
    stats = defaultdict(lambda: {"gold": 0, "typed": 0, "protected": 0, "pred": 0, "pred_ok": 0})
    failures = []
    for row in rows:
        text, labels = row["text"], row["labels"]
        preds = system(text)
        for lab in labels:
            g = GOLD_GROUP.get(lab["label"])
            if g is None:
                continue
            st = stats[g]
            st["gold"] += 1
            if coverage(lab["start"], lab["end"], preds, g) >= 0.5:
                st["typed"] += 1
            if coverage(lab["start"], lab["end"], preds) >= 0.9:
                st["protected"] += 1
            else:
                failures.append({"uid": row.get("uid"), "missed": lab["value"], "label": lab["label"], "text": text})
        for ps, pe, pg in preds:
            hits = [l for l in labels if ps < l["end"] and pe > l["start"]]
            in_scope = [l for l in hits if l["label"] in GOLD_GROUP]
            if hits and not in_scope:
                continue  # touches only out-of-scope labels (city, time, age, zip): neither right nor wrong
            st = stats[pg]
            st["pred"] += 1
            if any(GOLD_GROUP[l["label"]] == pg for l in in_scope):
                st["pred_ok"] += 1  # counted correct only when it hits an in-scope value of the same type
    return stats, failures


def summarize(stats):
    out = {}
    tot = {"gold": 0, "typed": 0, "protected": 0, "pred": 0, "pred_ok": 0}
    for g in GROUPS:
        st = stats.get(g, {"gold": 0, "typed": 0, "protected": 0, "pred": 0, "pred_ok": 0})
        for k in tot:
            tot[k] += st[k]
        out[g] = _ratios(st)
    out["ALL"] = _ratios(tot)
    return out


def _ratios(st):
    return {
        "n_gold": st["gold"],
        "recall": round(st["typed"] / st["gold"], 3) if st["gold"] else None,
        "protected": round(st["protected"] / st["gold"], 3) if st["gold"] else None,
        "precision": round(st["pred_ok"] / st["pred"], 3) if st["pred"] else None,
        "n_pred": st["pred"],
    }


def load_rows(path):
    return [json.loads(l) for l in path.read_text(encoding="utf-8").splitlines() if l.strip()]


def load_handwritten(path):
    """Hand-written cases list values, not offsets; locate each value in the text."""
    rows = []
    for case in load_rows(path):
        labels = []
        for lab, value in case["pii"]:
            i = case["text"].find(value)
            assert i >= 0, (case["id"], value)
            labels.append({"label": lab, "start": i, "end": i + len(value), "value": value})
        rows.append({"uid": case["id"], "text": case["text"], "labels": labels})
    return rows


def fmt(x):
    return "n/a" if x is None else f"{x * 100:.1f}%"


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--split", default="test", choices=["test", "dev"])
    ap.add_argument("--show-failures", type=int, default=0)
    args = ap.parse_args()

    datasets = {
        f"LocalDoc {args.split} (held-out)" if args.split == "test" else "LocalDoc dev": load_rows(ROOT / "data" / f"localdoc_{args.split}.jsonl"),
        "Hand-written AZ/RU/EN cases": load_handwritten(ROOT / "data" / "handwritten.jsonl"),
    }
    results = {}
    for dname, rows in datasets.items():
        results[dname] = {}
        print(f"\n### {dname} ({len(rows)} texts)\n")
        print("| System | Protected (all PII) | Leak rate | Precision |")
        print("|---|---|---|---|")
        for sname, fn in SYSTEMS.items():
            stats, failures = score(rows, fn)
            summ = summarize(stats)
            results[dname][sname] = summ
            a = summ["ALL"]
            print(f"| {sname} | {fmt(a['protected'])} | {fmt(1 - a['protected'])} | {fmt(a['precision'])} |")
            if sname == "Pərdə" and args.show_failures:
                for f in failures[: args.show_failures]:
                    print(f"  MISS {f['label']}: {f['missed']!r} :: {f['text'][:120]}")
        print(f"\nPer entity (Pərdə):\n")
        print("| Entity | Gold | Recall (right type) | Protected | Precision |")
        print("|---|---|---|---|---|")
        for g in GROUPS:
            s = results[dname]["Pərdə"][g]
            if s["n_gold"]:
                print(f"| {g} | {s['n_gold']} | {fmt(s['recall'])} | {fmt(s['protected'])} | {fmt(s['precision'])} |")
    out_name = "results.json" if args.split == "test" else "results_dev.json"  # the app shows results.json
    (ROOT / "eval" / out_name).write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")


if __name__ == "__main__":
    main()