log-pii-redactor / postproc.py
Project-Bastion's picture
v0.1.1: postproc.py
dca53ff verified
Raw History Blame Contribute Delete
5.82 kB
"""Deterministic span clean-up applied after the token classifier (documented, optional).
snap spans to token boundaries, strip ports/punctuation, validate by type. Pure stdlib."""
import ipaddress, re
DELIM = set(" \t\"'=&,;[]<>|{}#")
TLDS = set("com net org io edu gov mil int info biz local lan corp internal cloud dev app xyz online site tech example test localdomain home intranet".split())
_SSN = re.compile(r"^\d{3}-\d{2}-\d{4}$")
_MAC = re.compile(r"^([0-9A-Fa-f]{2}[:-]){5}[0-9A-Fa-f]{2}$")
def _snap(text, a, b, delim=DELIM):
while a > 0 and text[a - 1] not in delim and text[a - 1] != "(" : a -= 1
while b < len(text) and text[b] not in delim and text[b] not in ")(": b += 1
return a, b
def _trim(text, a, b, lead="=:(", trail=".,:;)"):
while a < b and text[a] in lead: a += 1
while b > a and text[b - 1] in trail: b -= 1
return a, b
def fix_span(text, a, b, lab):
if lab == "PHONE":
while a > 0 and text[a - 1] in "+(": a -= 1
a, b = _trim(text, a, b, lead="=:&", trail=".,:;&")
while b < len(text) and text[b] == ")" and text[a:b].count("(") > text[a:b].count(")"): b += 1
return (a, b, lab) if 10 <= sum(c.isdigit() for c in text[a:b]) <= 15 else None
if lab == "CARD":
a, b = _trim(text, a, b, lead="=:\"' ", trail=".,:;\"' ")
while a > 0 and (text[a - 1].isdigit() or (text[a - 1] in "- " and a > 1 and text[a - 2].isdigit())): a -= 1
while b < len(text) and (text[b].isdigit() or (text[b] in "- " and b + 1 < len(text) and text[b + 1].isdigit())): b += 1
a, b = _trim(text, a, b, lead=" -", trail=" -")
n = sum(c.isdigit() for c in text[a:b])
return (a, b, lab) if 13 <= n <= 19 and all(c.isdigit() or c in "- " for c in text[a:b]) else None
a, b = _snap(text, a, b, DELIM if lab in ("SECRET", "EMAIL") else DELIM | {"/", "@", "\\"}); a, b = _trim(text, a, b)
s = text[a:b]
if lab == "IP":
m = re.match(r"^(\d+\.\d+\.\d+\.\d+):\d+$", s)
if m: b = a + len(m.group(1)); s = m.group(1)
try: ipaddress.ip_address(s)
except ValueError: return None
elif lab == "HOST":
m = re.match(r"^(.*?):\d+$", s)
if m: b = a + len(m.group(1)); s = m.group(1)
if "." not in s: # short hostname: letter-start, >=3 chars, hostname charset only
if not re.match(r"^[A-Za-z][A-Za-z0-9_-]{2,}$", s): return None
else:
tld = s.rsplit(".", 1)[-1].lower()
if not tld or not re.search("[A-Za-z]", s) or re.match(r"^\d+(\.\d+)*$", s) or not (len(tld) == 2 or tld in TLDS): return None
elif lab == "EMAIL":
if not re.match(r"^[^@\s]+@[^@\s]+\.[^@\s]+$", s): return None
elif lab == "SSN":
if not _SSN.match(s): return None
elif lab == "MAC":
if not _MAC.match(s): return None
elif lab == "SECRET":
if len(s) < 16: return None
elif lab == "USER":
if len(s) < 2 or s.isdigit(): return None
return (a, b, lab)
def clean(text, spans):
"""Snap/validate each span; spans that collapse onto identical offsets are merged: the label of the LONGER original model span wins
(v0.1.1 fix: v0.1 compared a tuple with a list, so duplicates with different labels were all kept)."""
out = [] # [a, b, label, original_length]
for a0, b0, l in spans:
r = fix_span(text, a0, b0, l)
if not r: continue
ol = b0 - a0; dup = next((e for e in out if e[0] == r[0] and e[1] == r[1]), None)
if dup is not None:
if dup[2] != r[2] and ol > dup[3]: dup[2] = r[2]; dup[3] = ol
continue
out.append([r[0], r[1], r[2], ol])
return [[a, b, lab] for a, b, lab, _ in out]
# ---- optional recall fallback (v0.1.1): validated IPv4/IPv6 regex for addresses the model missed ----
_IP4 = re.compile(r"(?<![\w.])(?:(?:25[0-5]|2[0-4]\d|1?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|1?\d?\d)(?![\w]|\.\d)")
_IP6 = re.compile(r"(?<![\w:])(?:[0-9A-Fa-f]{0,4}:){2,7}[0-9A-Fa-f]{0,4}(?![\w:])")
_VERCTX = re.compile(r"(?i)(?:ver(?:sion)?|build|release|rev|v)\W{0,3}$")
def _ip_ok(v):
try: a = ipaddress.ip_address(v)
except ValueError: return False
return not (a.is_unspecified or a.is_loopback or a.is_multicast)
def ip_fallback(text, spans):
"""Return `spans` plus validated IP addresses that no existing span overlaps. Skips 0.0.0.0, loopback, x.0.0.0 network-style numbers and version contexts."""
out = [list(s) for s in spans]
def free(a, b): return not any(a < s[1] and s[0] < b for s in out)
for m in _IP4.finditer(text):
v = m.group()
if not _ip_ok(v) or v.endswith(".0.0.0") or _VERCTX.search(text[max(0, m.start() - 12):m.start()]): continue
if free(m.start(), m.end()): out.append([m.start(), m.end(), "IP"])
for m in _IP6.finditer(text):
v = m.group().rstrip(":") if m.group().endswith(":") and not m.group().endswith("::") else m.group()
if v.count(":") < 2 or not _ip_ok(v): continue
if free(m.start(), m.start() + len(v)): out.append([m.start(), m.start() + len(v), "IP"])
return sorted(out)
if __name__ == "__main__":
def at(t, sub, lab, dl=0, dr=0):
a = t.index(sub); return fix_span(t, a + dl, a + len(sub) + dr, lab)
t = 'GET /r?sms_to=(725) 709-7260 HTTP 4.4.26|100 db-prod13.acme.io:3306 for om'
print(at(t, "=(725) 709-7260", "PHONE"), at(t, "(725) 709-7260", "PHONE", dl=2, dr=-3))
print(at(t, ".4.26", "IP"), at(t, "db-prod13.acme.io:3306", "HOST"), at(t, "prod13.acme", "HOST"))
t2 = '9.127.216.33 - priya [20'; print(at(t2, ".127.216.33", "IP"), at(t2, "pri", "USER"))
t3 = 'ref=4042748715986199 ok'; print(at(t3, "404", "CARD"))
t4 = 'card="4111 1111 1111 1111" x'; print(at(t4, "4111 1111", "CARD"))
t5 = '1.2.3.4:8080 x'; print(at(t5, "1.2.3.4:8080", "IP"))