Spaces:
Running
Running
File size: 4,332 Bytes
f390b04 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 | """
Evaluation scripts for the extraction system.
Provides entity-level evaluation of NER (precision / recall / F1 via seqeval)
and recall-oriented evaluation of the anonymizer. These scripts produce the
numbers used in the "Testing" chapter of the thesis.
Usage:
python evaluate.py ner # evaluate NER on a small annotated sample
python evaluate.py anonymize # evaluate PII recall (NER vs regex)
The samples here are tiny illustrative examples. For the thesis, replace them
with a real annotated dataset (e.g. a subset of CoNLL-2003 for NER, or
ai4privacy/pii-masking-200k for anonymization).
"""
from __future__ import annotations
import sys
# ---------------------------------------------------------------------------
# NER evaluation (entity level, seqeval)
# ---------------------------------------------------------------------------
def evaluate_ner():
from seqeval.metrics import classification_report, f1_score, precision_score, recall_score
from processors import ner
# Each example: (tokens, gold BIO tags). Illustrative only.
samples = [
(
["My", "name", "is", "John", "Smith", "from", "London"],
["O", "O", "O", "B-PER", "I-PER", "O", "B-LOC"],
),
(
["Sarah", "works", "at", "Google", "in", "Berlin"],
["B-PER", "O", "O", "B-ORG", "O", "B-LOC"],
),
]
y_true, y_pred = [], []
for tokens, gold in samples:
text = " ".join(tokens)
ents = ner.extract_entities(text)
# Build predicted BIO tags aligned to whitespace tokens.
pred = ["O"] * len(tokens)
# character offset of each token
offsets = []
pos = 0
for tok in tokens:
start = text.index(tok, pos)
offsets.append((start, start + len(tok)))
pos = start + len(tok)
for ent in ents:
etype = ent["type"]
first = True
for i, (s, e) in enumerate(offsets):
# token overlaps the entity span
if s >= ent["start"] and e <= ent["end"] + 1:
pred[i] = ("B-" if first else "I-") + etype
first = False
y_true.append(gold)
y_pred.append(pred)
print("=== NER evaluation (entity level) ===")
print(classification_report(y_true, y_pred))
print(f"Precision: {precision_score(y_true, y_pred):.4f}")
print(f"Recall: {recall_score(y_true, y_pred):.4f}")
print(f"F1: {f1_score(y_true, y_pred):.4f}")
# ---------------------------------------------------------------------------
# Anonymization evaluation (PII recall: NER+Presidio vs regex only)
# ---------------------------------------------------------------------------
def evaluate_anonymize():
from processors import anonymizer
import re
# (text, list of PII substrings that MUST be removed)
samples = [
("My name is John Smith, email john@example.com", ["John Smith", "john@example.com"]),
("Call Sarah at +1 202 555 0143", ["Sarah", "+1 202 555 0143"]),
("I live in Berlin and work at Google", ["Berlin", "Google"]),
]
def recall(anon_fn):
found, total = 0, 0
for text, pii_list in samples:
anon = anon_fn(text)
for pii in pii_list:
total += 1
# PII counts as removed if it no longer appears verbatim
if pii.lower() not in anon.lower():
found += 1
return found / total if total else 0.0
# regex-only baseline (the original approach)
def regex_only(text):
from processors.anonymizer import _regex_anonymize
return _regex_anonymize(text)
print("=== Anonymization evaluation (PII recall) ===")
print(f"Active backend: {anonymizer.backend_name()}")
print(f"Recall (regex only): {recall(regex_only):.2%}")
print(f"Recall (full system): {recall(anonymizer.anonymize_text):.2%}")
print("\nNote: full system requires Presidio + spaCy model for name/location recall.")
if __name__ == "__main__":
mode = sys.argv[1] if len(sys.argv) > 1 else "ner"
if mode == "ner":
evaluate_ner()
elif mode == "anonymize":
evaluate_anonymize()
else:
print("Usage: python evaluate.py [ner|anonymize]")
|