Download eval/evaluate.py from Traxic/perde: direct link, hf CLI and curl.
- Browser
- Download file 7.64 kB
-
https://huggingface.co/spaces/Traxic/perde/resolve/main/eval/evaluate.py
- Command line
-
hf download hf://spaces/Traxic/perde/eval/evaluate.py
-
curl -L -o evaluate.py https://huggingface.co/spaces/Traxic/perde/resolve/main/eval/evaluate.py
7.64 kB
| """Measure detection quality on the held-out LocalDoc split and on hand-written cases. | |
| Usage: python eval/evaluate.py [--split test|dev] [--show-failures N] | |
| Writes eval/results.json (test split) or eval/results_dev.json (dev split) and prints a Markdown table. | |
| Metrics, per entity group: | |
| recall gold values masked by a detection of the right type (>=50% of characters) | |
| protected gold values masked by any detection (>=90% of characters); 1 - protected = leak rate | |
| precision detections that overlap an in-scope labelled value of the same type; detections that | |
| touch only out-of-scope labels (city, time, age, zip) are left out of the count | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import re | |
| import sys | |
| from collections import defaultdict | |
| from pathlib import Path | |
| ROOT = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(ROOT)) | |
| from perde.shield import detect # noqa: E402 | |
| GOLD_GROUP = { | |
| "GIVENNAME": "PERSON", "SURNAME": "PERSON", | |
| "TELEPHONENUM": "PHONE", "EMAIL": "EMAIL", | |
| "IDCARDNUM": "ID_NUMBER", "PASSPORTNUM": "ID_NUMBER", "TAXNUM": "ID_NUMBER", | |
| "DRIVERLICENSENUM": "ID_NUMBER", "CREDITCARDNUMBER": "CARD", | |
| "DATE": "DATE", "STREET": "ADDRESS", "BUILDINGNUM": "ADDRESS", | |
| } | |
| OUT_OF_SCOPE = {"TIME", "CITY", "AGE", "ZIPCODE"} | |
| PRED_GROUP = { | |
| "PERSON": "PERSON", "PHONE": "PHONE", "EMAIL": "EMAIL", "FIN": "ID_NUMBER", | |
| "ID_CARD": "ID_NUMBER", "PASSPORT": "ID_NUMBER", "VOEN": "ID_NUMBER", | |
| "IBAN": "ID_NUMBER", "DRIVER_LICENSE": "ID_NUMBER", "CAR_PLATE": "ID_NUMBER", "CARD": "CARD", | |
| "DATE": "DATE", "ADDRESS": "ADDRESS", | |
| } | |
| GROUPS = ["PERSON", "PHONE", "EMAIL", "ID_NUMBER", "CARD", "DATE", "ADDRESS"] | |
| # --- systems under test ----------------------------------------------------- | |
| def system_perde(text): | |
| return [(s.start, s.end, PRED_GROUP[s.entity]) for s in detect(text)] | |
| _GEN_EMAIL = re.compile(r"\b[\w.+-]+@[\w-]+(?:\.[\w-]+)+\b") | |
| _GEN_PHONE = re.compile(r"(?<!\w)\+?\d[\d\s\-().]{7,}\d(?!\w)") | |
| _GEN_DATE = re.compile(r"\b\d{4}-\d{2}-\d{2}\b|\b\d{1,2}/\d{1,2}/\d{2,4}\b") | |
| def system_generic(text): | |
| """Language-agnostic regex baseline: what a default, non-localised filter catches.""" | |
| out = [(m.start(), m.end(), "EMAIL") for m in _GEN_EMAIL.finditer(text)] | |
| out += [(m.start(), m.end(), "PHONE") for m in _GEN_PHONE.finditer(text)] | |
| out += [(m.start(), m.end(), "DATE") for m in _GEN_DATE.finditer(text)] | |
| return out | |
| def system_no_names(text): | |
| return [p for p in system_perde(text) if p[2] != "PERSON"] | |
| SYSTEMS = {"Pərdə": system_perde, "Pərdə without name detector": system_no_names, | |
| "Generic regex baseline": system_generic} | |
| # --- scoring ---------------------------------------------------------------- | |
| def coverage(gs, ge, preds, group=None): | |
| covered = set() | |
| for ps, pe, pg in preds: | |
| if group is None or pg == group: | |
| covered.update(range(max(gs, ps), min(ge, pe))) | |
| span_chars = [i for i in range(gs, ge)] | |
| return len(covered) / max(1, len(span_chars)) | |
| def score(rows, system): | |
| stats = defaultdict(lambda: {"gold": 0, "typed": 0, "protected": 0, "pred": 0, "pred_ok": 0}) | |
| failures = [] | |
| for row in rows: | |
| text, labels = row["text"], row["labels"] | |
| preds = system(text) | |
| for lab in labels: | |
| g = GOLD_GROUP.get(lab["label"]) | |
| if g is None: | |
| continue | |
| st = stats[g] | |
| st["gold"] += 1 | |
| if coverage(lab["start"], lab["end"], preds, g) >= 0.5: | |
| st["typed"] += 1 | |
| if coverage(lab["start"], lab["end"], preds) >= 0.9: | |
| st["protected"] += 1 | |
| else: | |
| failures.append({"uid": row.get("uid"), "missed": lab["value"], "label": lab["label"], "text": text}) | |
| for ps, pe, pg in preds: | |
| hits = [l for l in labels if ps < l["end"] and pe > l["start"]] | |
| in_scope = [l for l in hits if l["label"] in GOLD_GROUP] | |
| if hits and not in_scope: | |
| continue # touches only out-of-scope labels (city, time, age, zip): neither right nor wrong | |
| st = stats[pg] | |
| st["pred"] += 1 | |
| if any(GOLD_GROUP[l["label"]] == pg for l in in_scope): | |
| st["pred_ok"] += 1 # counted correct only when it hits an in-scope value of the same type | |
| return stats, failures | |
| def summarize(stats): | |
| out = {} | |
| tot = {"gold": 0, "typed": 0, "protected": 0, "pred": 0, "pred_ok": 0} | |
| for g in GROUPS: | |
| st = stats.get(g, {"gold": 0, "typed": 0, "protected": 0, "pred": 0, "pred_ok": 0}) | |
| for k in tot: | |
| tot[k] += st[k] | |
| out[g] = _ratios(st) | |
| out["ALL"] = _ratios(tot) | |
| return out | |
| def _ratios(st): | |
| return { | |
| "n_gold": st["gold"], | |
| "recall": round(st["typed"] / st["gold"], 3) if st["gold"] else None, | |
| "protected": round(st["protected"] / st["gold"], 3) if st["gold"] else None, | |
| "precision": round(st["pred_ok"] / st["pred"], 3) if st["pred"] else None, | |
| "n_pred": st["pred"], | |
| } | |
| def load_rows(path): | |
| return [json.loads(l) for l in path.read_text(encoding="utf-8").splitlines() if l.strip()] | |
| def load_handwritten(path): | |
| """Hand-written cases list values, not offsets; locate each value in the text.""" | |
| rows = [] | |
| for case in load_rows(path): | |
| labels = [] | |
| for lab, value in case["pii"]: | |
| i = case["text"].find(value) | |
| assert i >= 0, (case["id"], value) | |
| labels.append({"label": lab, "start": i, "end": i + len(value), "value": value}) | |
| rows.append({"uid": case["id"], "text": case["text"], "labels": labels}) | |
| return rows | |
| def fmt(x): | |
| return "n/a" if x is None else f"{x * 100:.1f}%" | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--split", default="test", choices=["test", "dev"]) | |
| ap.add_argument("--show-failures", type=int, default=0) | |
| args = ap.parse_args() | |
| datasets = { | |
| f"LocalDoc {args.split} (held-out)" if args.split == "test" else "LocalDoc dev": load_rows(ROOT / "data" / f"localdoc_{args.split}.jsonl"), | |
| "Hand-written AZ/RU/EN cases": load_handwritten(ROOT / "data" / "handwritten.jsonl"), | |
| } | |
| results = {} | |
| for dname, rows in datasets.items(): | |
| results[dname] = {} | |
| print(f"\n### {dname} ({len(rows)} texts)\n") | |
| print("| System | Protected (all PII) | Leak rate | Precision |") | |
| print("|---|---|---|---|") | |
| for sname, fn in SYSTEMS.items(): | |
| stats, failures = score(rows, fn) | |
| summ = summarize(stats) | |
| results[dname][sname] = summ | |
| a = summ["ALL"] | |
| print(f"| {sname} | {fmt(a['protected'])} | {fmt(1 - a['protected'])} | {fmt(a['precision'])} |") | |
| if sname == "Pərdə" and args.show_failures: | |
| for f in failures[: args.show_failures]: | |
| print(f" MISS {f['label']}: {f['missed']!r} :: {f['text'][:120]}") | |
| print(f"\nPer entity (Pərdə):\n") | |
| print("| Entity | Gold | Recall (right type) | Protected | Precision |") | |
| print("|---|---|---|---|---|") | |
| for g in GROUPS: | |
| s = results[dname]["Pərdə"][g] | |
| if s["n_gold"]: | |
| print(f"| {g} | {s['n_gold']} | {fmt(s['recall'])} | {fmt(s['protected'])} | {fmt(s['precision'])} |") | |
| out_name = "results.json" if args.split == "test" else "results_dev.json" # the app shows results.json | |
| (ROOT / "eval" / out_name).write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8") | |
| if __name__ == "__main__": | |
| main() | |