"""Deterministic span clean-up applied after the token classifier (documented, optional). snap spans to token boundaries, strip ports/punctuation, validate by type. Pure stdlib.""" import ipaddress, re DELIM = set(" \t\"'=&,;[]<>|{}#") TLDS = set("com net org io edu gov mil int info biz local lan corp internal cloud dev app xyz online site tech example test localdomain home intranet".split()) _SSN = re.compile(r"^\d{3}-\d{2}-\d{4}$") _MAC = re.compile(r"^([0-9A-Fa-f]{2}[:-]){5}[0-9A-Fa-f]{2}$") def _snap(text, a, b, delim=DELIM): while a > 0 and text[a - 1] not in delim and text[a - 1] != "(" : a -= 1 while b < len(text) and text[b] not in delim and text[b] not in ")(": b += 1 return a, b def _trim(text, a, b, lead="=:(", trail=".,:;)"): while a < b and text[a] in lead: a += 1 while b > a and text[b - 1] in trail: b -= 1 return a, b def fix_span(text, a, b, lab): if lab == "PHONE": while a > 0 and text[a - 1] in "+(": a -= 1 a, b = _trim(text, a, b, lead="=:&", trail=".,:;&") while b < len(text) and text[b] == ")" and text[a:b].count("(") > text[a:b].count(")"): b += 1 return (a, b, lab) if 10 <= sum(c.isdigit() for c in text[a:b]) <= 15 else None if lab == "CARD": a, b = _trim(text, a, b, lead="=:\"' ", trail=".,:;\"' ") while a > 0 and (text[a - 1].isdigit() or (text[a - 1] in "- " and a > 1 and text[a - 2].isdigit())): a -= 1 while b < len(text) and (text[b].isdigit() or (text[b] in "- " and b + 1 < len(text) and text[b + 1].isdigit())): b += 1 a, b = _trim(text, a, b, lead=" -", trail=" -") n = sum(c.isdigit() for c in text[a:b]) return (a, b, lab) if 13 <= n <= 19 and all(c.isdigit() or c in "- " for c in text[a:b]) else None a, b = _snap(text, a, b, DELIM if lab in ("SECRET", "EMAIL") else DELIM | {"/", "@", "\\"}); a, b = _trim(text, a, b) s = text[a:b] if lab == "IP": m = re.match(r"^(\d+\.\d+\.\d+\.\d+):\d+$", s) if m: b = a + len(m.group(1)); s = m.group(1) try: ipaddress.ip_address(s) except ValueError: return None elif lab == "HOST": m = re.match(r"^(.*?):\d+$", s) if m: b = a + len(m.group(1)); s = m.group(1) if "." not in s: # short hostname: letter-start, >=3 chars, hostname charset only if not re.match(r"^[A-Za-z][A-Za-z0-9_-]{2,}$", s): return None else: tld = s.rsplit(".", 1)[-1].lower() if not tld or not re.search("[A-Za-z]", s) or re.match(r"^\d+(\.\d+)*$", s) or not (len(tld) == 2 or tld in TLDS): return None elif lab == "EMAIL": if not re.match(r"^[^@\s]+@[^@\s]+\.[^@\s]+$", s): return None elif lab == "SSN": if not _SSN.match(s): return None elif lab == "MAC": if not _MAC.match(s): return None elif lab == "SECRET": if len(s) < 16: return None elif lab == "USER": if len(s) < 2 or s.isdigit(): return None return (a, b, lab) def clean(text, spans): """Snap/validate each span; spans that collapse onto identical offsets are merged: the label of the LONGER original model span wins (v0.1.1 fix: v0.1 compared a tuple with a list, so duplicates with different labels were all kept).""" out = [] # [a, b, label, original_length] for a0, b0, l in spans: r = fix_span(text, a0, b0, l) if not r: continue ol = b0 - a0; dup = next((e for e in out if e[0] == r[0] and e[1] == r[1]), None) if dup is not None: if dup[2] != r[2] and ol > dup[3]: dup[2] = r[2]; dup[3] = ol continue out.append([r[0], r[1], r[2], ol]) return [[a, b, lab] for a, b, lab, _ in out] # ---- optional recall fallback (v0.1.1): validated IPv4/IPv6 regex for addresses the model missed ---- _IP4 = re.compile(r"(?