File size: 7,642 Bytes
6684cad fb70710 6684cad 26b4286 6684cad 26b4286 6684cad 26b4286 6684cad fb70710 6684cad | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 | """Measure detection quality on the held-out LocalDoc split and on hand-written cases.
Usage: python eval/evaluate.py [--split test|dev] [--show-failures N]
Writes eval/results.json (test split) or eval/results_dev.json (dev split) and prints a Markdown table.
Metrics, per entity group:
recall gold values masked by a detection of the right type (>=50% of characters)
protected gold values masked by any detection (>=90% of characters); 1 - protected = leak rate
precision detections that overlap an in-scope labelled value of the same type; detections that
touch only out-of-scope labels (city, time, age, zip) are left out of the count
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from collections import defaultdict
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from perde.shield import detect # noqa: E402
GOLD_GROUP = {
"GIVENNAME": "PERSON", "SURNAME": "PERSON",
"TELEPHONENUM": "PHONE", "EMAIL": "EMAIL",
"IDCARDNUM": "ID_NUMBER", "PASSPORTNUM": "ID_NUMBER", "TAXNUM": "ID_NUMBER",
"DRIVERLICENSENUM": "ID_NUMBER", "CREDITCARDNUMBER": "CARD",
"DATE": "DATE", "STREET": "ADDRESS", "BUILDINGNUM": "ADDRESS",
}
OUT_OF_SCOPE = {"TIME", "CITY", "AGE", "ZIPCODE"}
PRED_GROUP = {
"PERSON": "PERSON", "PHONE": "PHONE", "EMAIL": "EMAIL", "FIN": "ID_NUMBER",
"ID_CARD": "ID_NUMBER", "PASSPORT": "ID_NUMBER", "VOEN": "ID_NUMBER",
"IBAN": "ID_NUMBER", "DRIVER_LICENSE": "ID_NUMBER", "CAR_PLATE": "ID_NUMBER", "CARD": "CARD",
"DATE": "DATE", "ADDRESS": "ADDRESS",
}
GROUPS = ["PERSON", "PHONE", "EMAIL", "ID_NUMBER", "CARD", "DATE", "ADDRESS"]
# --- systems under test -----------------------------------------------------
def system_perde(text):
return [(s.start, s.end, PRED_GROUP[s.entity]) for s in detect(text)]
_GEN_EMAIL = re.compile(r"\b[\w.+-]+@[\w-]+(?:\.[\w-]+)+\b")
_GEN_PHONE = re.compile(r"(?<!\w)\+?\d[\d\s\-().]{7,}\d(?!\w)")
_GEN_DATE = re.compile(r"\b\d{4}-\d{2}-\d{2}\b|\b\d{1,2}/\d{1,2}/\d{2,4}\b")
def system_generic(text):
"""Language-agnostic regex baseline: what a default, non-localised filter catches."""
out = [(m.start(), m.end(), "EMAIL") for m in _GEN_EMAIL.finditer(text)]
out += [(m.start(), m.end(), "PHONE") for m in _GEN_PHONE.finditer(text)]
out += [(m.start(), m.end(), "DATE") for m in _GEN_DATE.finditer(text)]
return out
def system_no_names(text):
return [p for p in system_perde(text) if p[2] != "PERSON"]
SYSTEMS = {"Pərdə": system_perde, "Pərdə without name detector": system_no_names,
"Generic regex baseline": system_generic}
# --- scoring ----------------------------------------------------------------
def coverage(gs, ge, preds, group=None):
covered = set()
for ps, pe, pg in preds:
if group is None or pg == group:
covered.update(range(max(gs, ps), min(ge, pe)))
span_chars = [i for i in range(gs, ge)]
return len(covered) / max(1, len(span_chars))
def score(rows, system):
stats = defaultdict(lambda: {"gold": 0, "typed": 0, "protected": 0, "pred": 0, "pred_ok": 0})
failures = []
for row in rows:
text, labels = row["text"], row["labels"]
preds = system(text)
for lab in labels:
g = GOLD_GROUP.get(lab["label"])
if g is None:
continue
st = stats[g]
st["gold"] += 1
if coverage(lab["start"], lab["end"], preds, g) >= 0.5:
st["typed"] += 1
if coverage(lab["start"], lab["end"], preds) >= 0.9:
st["protected"] += 1
else:
failures.append({"uid": row.get("uid"), "missed": lab["value"], "label": lab["label"], "text": text})
for ps, pe, pg in preds:
hits = [l for l in labels if ps < l["end"] and pe > l["start"]]
in_scope = [l for l in hits if l["label"] in GOLD_GROUP]
if hits and not in_scope:
continue # touches only out-of-scope labels (city, time, age, zip): neither right nor wrong
st = stats[pg]
st["pred"] += 1
if any(GOLD_GROUP[l["label"]] == pg for l in in_scope):
st["pred_ok"] += 1 # counted correct only when it hits an in-scope value of the same type
return stats, failures
def summarize(stats):
out = {}
tot = {"gold": 0, "typed": 0, "protected": 0, "pred": 0, "pred_ok": 0}
for g in GROUPS:
st = stats.get(g, {"gold": 0, "typed": 0, "protected": 0, "pred": 0, "pred_ok": 0})
for k in tot:
tot[k] += st[k]
out[g] = _ratios(st)
out["ALL"] = _ratios(tot)
return out
def _ratios(st):
return {
"n_gold": st["gold"],
"recall": round(st["typed"] / st["gold"], 3) if st["gold"] else None,
"protected": round(st["protected"] / st["gold"], 3) if st["gold"] else None,
"precision": round(st["pred_ok"] / st["pred"], 3) if st["pred"] else None,
"n_pred": st["pred"],
}
def load_rows(path):
return [json.loads(l) for l in path.read_text(encoding="utf-8").splitlines() if l.strip()]
def load_handwritten(path):
"""Hand-written cases list values, not offsets; locate each value in the text."""
rows = []
for case in load_rows(path):
labels = []
for lab, value in case["pii"]:
i = case["text"].find(value)
assert i >= 0, (case["id"], value)
labels.append({"label": lab, "start": i, "end": i + len(value), "value": value})
rows.append({"uid": case["id"], "text": case["text"], "labels": labels})
return rows
def fmt(x):
return "n/a" if x is None else f"{x * 100:.1f}%"
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--split", default="test", choices=["test", "dev"])
ap.add_argument("--show-failures", type=int, default=0)
args = ap.parse_args()
datasets = {
f"LocalDoc {args.split} (held-out)" if args.split == "test" else "LocalDoc dev": load_rows(ROOT / "data" / f"localdoc_{args.split}.jsonl"),
"Hand-written AZ/RU/EN cases": load_handwritten(ROOT / "data" / "handwritten.jsonl"),
}
results = {}
for dname, rows in datasets.items():
results[dname] = {}
print(f"\n### {dname} ({len(rows)} texts)\n")
print("| System | Protected (all PII) | Leak rate | Precision |")
print("|---|---|---|---|")
for sname, fn in SYSTEMS.items():
stats, failures = score(rows, fn)
summ = summarize(stats)
results[dname][sname] = summ
a = summ["ALL"]
print(f"| {sname} | {fmt(a['protected'])} | {fmt(1 - a['protected'])} | {fmt(a['precision'])} |")
if sname == "Pərdə" and args.show_failures:
for f in failures[: args.show_failures]:
print(f" MISS {f['label']}: {f['missed']!r} :: {f['text'][:120]}")
print(f"\nPer entity (Pərdə):\n")
print("| Entity | Gold | Recall (right type) | Protected | Precision |")
print("|---|---|---|---|---|")
for g in GROUPS:
s = results[dname]["Pərdə"][g]
if s["n_gold"]:
print(f"| {g} | {s['n_gold']} | {fmt(s['recall'])} | {fmt(s['protected'])} | {fmt(s['precision'])} |")
out_name = "results.json" if args.split == "test" else "results_dev.json" # the app shows results.json
(ROOT / "eval" / out_name).write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
if __name__ == "__main__":
main()
|