Token Classification
Transformers
ONNX
Safetensors
English
roberta
pii
redaction
logs
security
secrets
privacy
Eval Results (legacy)
Instructions to use Project-Bastion/log-pii-redactor with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use Project-Bastion/log-pii-redactor with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("token-classification", model="Project-Bastion/log-pii-redactor")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForTokenClassification tokenizer = AutoTokenizer.from_pretrained("Project-Bastion/log-pii-redactor") model = AutoModelForTokenClassification.from_pretrained("Project-Bastion/log-pii-redactor", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download postproc.py from Project-Bastion/log-pii-redactor: direct link, hf CLI and curl.
- Browser
- Download file 5.82 kB
-
https://huggingface.co/Project-Bastion/log-pii-redactor/resolve/main/postproc.py
- Command line
-
hf download hf://Project-Bastion/log-pii-redactor/postproc.py
-
curl -L -o postproc.py https://huggingface.co/Project-Bastion/log-pii-redactor/resolve/main/postproc.py
5.82 kB
| """Deterministic span clean-up applied after the token classifier (documented, optional). | |
| snap spans to token boundaries, strip ports/punctuation, validate by type. Pure stdlib.""" | |
| import ipaddress, re | |
| DELIM = set(" \t\"'=&,;[]<>|{}#") | |
| TLDS = set("com net org io edu gov mil int info biz local lan corp internal cloud dev app xyz online site tech example test localdomain home intranet".split()) | |
| _SSN = re.compile(r"^\d{3}-\d{2}-\d{4}$") | |
| _MAC = re.compile(r"^([0-9A-Fa-f]{2}[:-]){5}[0-9A-Fa-f]{2}$") | |
| def _snap(text, a, b, delim=DELIM): | |
| while a > 0 and text[a - 1] not in delim and text[a - 1] != "(" : a -= 1 | |
| while b < len(text) and text[b] not in delim and text[b] not in ")(": b += 1 | |
| return a, b | |
| def _trim(text, a, b, lead="=:(", trail=".,:;)"): | |
| while a < b and text[a] in lead: a += 1 | |
| while b > a and text[b - 1] in trail: b -= 1 | |
| return a, b | |
| def fix_span(text, a, b, lab): | |
| if lab == "PHONE": | |
| while a > 0 and text[a - 1] in "+(": a -= 1 | |
| a, b = _trim(text, a, b, lead="=:&", trail=".,:;&") | |
| while b < len(text) and text[b] == ")" and text[a:b].count("(") > text[a:b].count(")"): b += 1 | |
| return (a, b, lab) if 10 <= sum(c.isdigit() for c in text[a:b]) <= 15 else None | |
| if lab == "CARD": | |
| a, b = _trim(text, a, b, lead="=:\"' ", trail=".,:;\"' ") | |
| while a > 0 and (text[a - 1].isdigit() or (text[a - 1] in "- " and a > 1 and text[a - 2].isdigit())): a -= 1 | |
| while b < len(text) and (text[b].isdigit() or (text[b] in "- " and b + 1 < len(text) and text[b + 1].isdigit())): b += 1 | |
| a, b = _trim(text, a, b, lead=" -", trail=" -") | |
| n = sum(c.isdigit() for c in text[a:b]) | |
| return (a, b, lab) if 13 <= n <= 19 and all(c.isdigit() or c in "- " for c in text[a:b]) else None | |
| a, b = _snap(text, a, b, DELIM if lab in ("SECRET", "EMAIL") else DELIM | {"/", "@", "\\"}); a, b = _trim(text, a, b) | |
| s = text[a:b] | |
| if lab == "IP": | |
| m = re.match(r"^(\d+\.\d+\.\d+\.\d+):\d+$", s) | |
| if m: b = a + len(m.group(1)); s = m.group(1) | |
| try: ipaddress.ip_address(s) | |
| except ValueError: return None | |
| elif lab == "HOST": | |
| m = re.match(r"^(.*?):\d+$", s) | |
| if m: b = a + len(m.group(1)); s = m.group(1) | |
| if "." not in s: # short hostname: letter-start, >=3 chars, hostname charset only | |
| if not re.match(r"^[A-Za-z][A-Za-z0-9_-]{2,}$", s): return None | |
| else: | |
| tld = s.rsplit(".", 1)[-1].lower() | |
| if not tld or not re.search("[A-Za-z]", s) or re.match(r"^\d+(\.\d+)*$", s) or not (len(tld) == 2 or tld in TLDS): return None | |
| elif lab == "EMAIL": | |
| if not re.match(r"^[^@\s]+@[^@\s]+\.[^@\s]+$", s): return None | |
| elif lab == "SSN": | |
| if not _SSN.match(s): return None | |
| elif lab == "MAC": | |
| if not _MAC.match(s): return None | |
| elif lab == "SECRET": | |
| if len(s) < 16: return None | |
| elif lab == "USER": | |
| if len(s) < 2 or s.isdigit(): return None | |
| return (a, b, lab) | |
| def clean(text, spans): | |
| """Snap/validate each span; spans that collapse onto identical offsets are merged: the label of the LONGER original model span wins | |
| (v0.1.1 fix: v0.1 compared a tuple with a list, so duplicates with different labels were all kept).""" | |
| out = [] # [a, b, label, original_length] | |
| for a0, b0, l in spans: | |
| r = fix_span(text, a0, b0, l) | |
| if not r: continue | |
| ol = b0 - a0; dup = next((e for e in out if e[0] == r[0] and e[1] == r[1]), None) | |
| if dup is not None: | |
| if dup[2] != r[2] and ol > dup[3]: dup[2] = r[2]; dup[3] = ol | |
| continue | |
| out.append([r[0], r[1], r[2], ol]) | |
| return [[a, b, lab] for a, b, lab, _ in out] | |
| # ---- optional recall fallback (v0.1.1): validated IPv4/IPv6 regex for addresses the model missed ---- | |
| _IP4 = re.compile(r"(?<![\w.])(?:(?:25[0-5]|2[0-4]\d|1?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|1?\d?\d)(?![\w]|\.\d)") | |
| _IP6 = re.compile(r"(?<![\w:])(?:[0-9A-Fa-f]{0,4}:){2,7}[0-9A-Fa-f]{0,4}(?![\w:])") | |
| _VERCTX = re.compile(r"(?i)(?:ver(?:sion)?|build|release|rev|v)\W{0,3}$") | |
| def _ip_ok(v): | |
| try: a = ipaddress.ip_address(v) | |
| except ValueError: return False | |
| return not (a.is_unspecified or a.is_loopback or a.is_multicast) | |
| def ip_fallback(text, spans): | |
| """Return `spans` plus validated IP addresses that no existing span overlaps. Skips 0.0.0.0, loopback, x.0.0.0 network-style numbers and version contexts.""" | |
| out = [list(s) for s in spans] | |
| def free(a, b): return not any(a < s[1] and s[0] < b for s in out) | |
| for m in _IP4.finditer(text): | |
| v = m.group() | |
| if not _ip_ok(v) or v.endswith(".0.0.0") or _VERCTX.search(text[max(0, m.start() - 12):m.start()]): continue | |
| if free(m.start(), m.end()): out.append([m.start(), m.end(), "IP"]) | |
| for m in _IP6.finditer(text): | |
| v = m.group().rstrip(":") if m.group().endswith(":") and not m.group().endswith("::") else m.group() | |
| if v.count(":") < 2 or not _ip_ok(v): continue | |
| if free(m.start(), m.start() + len(v)): out.append([m.start(), m.start() + len(v), "IP"]) | |
| return sorted(out) | |
| if __name__ == "__main__": | |
| def at(t, sub, lab, dl=0, dr=0): | |
| a = t.index(sub); return fix_span(t, a + dl, a + len(sub) + dr, lab) | |
| t = 'GET /r?sms_to=(725) 709-7260 HTTP 4.4.26|100 db-prod13.acme.io:3306 for om' | |
| print(at(t, "=(725) 709-7260", "PHONE"), at(t, "(725) 709-7260", "PHONE", dl=2, dr=-3)) | |
| print(at(t, ".4.26", "IP"), at(t, "db-prod13.acme.io:3306", "HOST"), at(t, "prod13.acme", "HOST")) | |
| t2 = '9.127.216.33 - priya [20'; print(at(t2, ".127.216.33", "IP"), at(t2, "pri", "USER")) | |
| t3 = 'ref=4042748715986199 ok'; print(at(t3, "404", "CARD")) | |
| t4 = 'card="4111 1111 1111 1111" x'; print(at(t4, "4111 1111", "CARD")) | |
| t5 = '1.2.3.4:8080 x'; print(at(t5, "1.2.3.4:8080", "IP")) | |