Update rule_detector.py

rule_detector.py

@@ -1,8 +1,23 @@
-"""Rule-based PII detector — HYBRID single-file edition (v2, precision-tuned).
+"""Rule-based PII detector — HYBRID single-file edition (v3, CONTEXT-ONLY).
+
+v3 derives from v2 (the precision-tuned 2026-06-12 set) with ONE structural
+change requested on 2026-06-23: **every class is now detected through a positive
+keyword context only.** All bare-regex rules are removed (``_build_regex_rules``
+returns an empty list), and the three classes that used to be bare-only — EMAIL,
+IBAN, IP — gain dedicated context rules (``_EMAIL_CONTEXT`` / ``_IBAN_CONTEXT`` /
+``_IP_CONTEXT``). A span now fires only when a cue word ("email:", "IBAN", "IP
+address", "phone", "SWIFT", …) precedes the value.
+
+Trade-off (intended): precision is held at the v2 level (context-gating tends to
+raise it), while recall, F1 and the number of predictions DROP — values that
+appear with no cue (most bare EMAIL/IP/COORDINATE, the bare SSN shape, bare
+PHONE/CARD/IBAN runs) are no longer matched. This is accepted by design.
+
+The detection engine, validators, dedup and blocking map are unchanged from v2.
 
 Same organisation as ``old rule/rule_detector.py`` (PIIMatch / RegexRule /
-validators / RulebasePIIDetector with bare-regex rules, keyword-context specs, a
-single dedup pass), but it carries the **HYBRID v2** rule set:
+validators / RulebasePIIDetector with keyword-context specs, a single dedup
+pass), but it carries the **HYBRID v2** rule set:
 
     OLD6  = {PHONE, IBAN, CARD_NUMBER, SWIFT, PIN, COORDINATE}
             → the precision-first OLD rules, upgraded so they still pass the
@@ -109,7 +124,9 @@ ACTIVE_RULEBASE_BLOCKING_OVERLAPS = {
     "PIN":          frozenset({"PHONE", "CVV"}),
 }
 
-CONTEXT_NORMALIZE_AS_TEXT_ENTITY_TYPES = frozenset({"PASSWORD", "TIN"})
+# EMAIL/URL/IP added in v3: their context-captured value must not have hyphens
+# or dots stripped (those are part of the address), so keep it as collapsed text.
+CONTEXT_NORMALIZE_AS_TEXT_ENTITY_TYPES = frozenset({"PASSWORD", "TIN", "EMAIL", "URL", "IP"})
 
 ISO_COUNTRY_CODES = {
     "AD","AE","AF","AG","AI","AL","AM","AO","AR","AS","AT","AU","AW","AX","AZ",
@@ -597,6 +614,15 @@ _EMAIL_BARE_DOUBLE_AT = re.compile(
     r"(?<![\w.+*\-])[A-Z0-9._%+-]+@@[A-Z0-9.-]+\.[A-Z]{2,63}(?![\w-])",
     re.IGNORECASE,
 )
+# v3 CONTEXT-ONLY: EMAIL must now be introduced by a cue ("email:", "thư điện tử là …").
+# The value keeps the same shape as the bare rule; the '*' lookbehind still blocks a
+# masked-tail leak. Validator is_valid_email keeps precision at the v2 level.
+_EMAIL_CONTEXT = re.compile(
+    r"(?i)\b(?:e[-\s]?mail(?:\s*address)?|thư\s*điện\s*tử|địa\s*chỉ\s*(?:email|e-mail|thư)"
+    r"|hộp\s*thư(?:\s*điện\s*tử)?|gửi\s*(?:tới|đến|về)|liên\s*hệ\s*(?:qua\s*)?(?:email|mail))\b"
+    r"\s*(?:[:#\-]\s*|[^:\n@]{0,20}(?:là|is|at|tại)\s+|\s+)"
+    r"(?<![\w.+*\-])([A-Z0-9._%+\-À-ɏḀ-ỿ]+@[A-Z0-9.-]+\.[A-Z]{2,63})(?![\w-])"
+)
 
 # ---- PHONE (OLD6 hybrid) --------------------------------------------------
 _PHONE_MASK_GUARD = r"(?! ?\*{3,})(?! ?\*+\d)(?!\s*-{2,})(?!-\*)"
@@ -672,6 +698,14 @@ _IBAN_NEG_CONTEXT = re.compile(
     r"|(?:sensor|equipment|device)(?:\s+unit)?\b[^\n\d]{0,16}"
     r"|sensor\s+unit\b[^\n\d]{0,16})$"
 )
+# v3 CONTEXT-ONLY: IBAN must now be introduced by an "IBAN" cue. The value is
+# captured uppercase-only (?-i:) so a trailing lowercase word can't be absorbed;
+# both compact and single-space-grouped forms are allowed. Validator runs mod-97.
+_IBAN_CONTEXT = re.compile(
+    r"(?i)\b(?:iban(?:\s*(?:number|no\.?|code))?|số\s*iban|mã\s*iban)\b"
+    r"\s*(?:[:#\-]\s*|[^:\n0-9]{0,16}(?:là|is)\s+|\s+)"
+    r"((?-i:[A-Z]{2}\d{2}(?:\s?[A-Z0-9]){11,34}))"
+)
 
 # ---- CARD_NUMBER (OLD6) ---------------------------------------------------
 _CARD_BARE = re.compile(r"(?<![\d.])\d{4}[ -]\d{4}[ -]\d{4}[ -]\d{1,7}(?![\d])")
@@ -694,6 +728,15 @@ _IP_BARE = re.compile(
     r")",
     re.IGNORECASE,
 )
+# v3 CONTEXT-ONLY: IP must now be introduced by an "IP / IP address / IPv4 / IPv6"
+# cue. The value reuses the bare IPv4/IPv6 alternation; validator is_valid_ip uses
+# the stdlib ipaddress parser, so a malformed run is rejected (precision held).
+_IP_CONTEXT = re.compile(
+    r"(?i)\b(?:ip(?:\s*address|\s*addr)?|ipv4|ipv6|địa\s*chỉ\s*ip|server\s*ip|host\s*ip)\b"
+    r"\s*(?:[:#\-=]\s*|[^:\n0-9a-f]{0,16}(?:là|is|at)\s+|\s+)"
+    r"((?:\d{1,3}\.){3}\d{1,3}(?!\.\d)|[A-F0-9]{0,4}(?::[A-F0-9]{0,4}){2,7})",
+    re.IGNORECASE,
+)
 
 # ---- URL (TUNED) ----------------------------------------------------------
 _URL_BARE = re.compile(
@@ -868,49 +911,22 @@ class RulebasePIIDetector:
         return entity_type in self.allowed_entity_types
 
     def _build_regex_rules(self):
-        rules = [
-            # EMAIL (TUNED)
-            RegexRule(entity_type="EMAIL", pattern=_EMAIL_BARE, validator=is_valid_email),
-            RegexRule(entity_type="EMAIL", pattern=_EMAIL_BARE_DOUBLE_AT,
-                      validator=is_valid_email_double_at, normalizer=normalize_email_double_at),
-            # PHONE (OLD6)
-            RegexRule(entity_type="PHONE", pattern=_PHONE_BARE_INTL,
-                      validator=is_phone_value, normalizer=normalize_phone),
-            RegexRule(entity_type="PHONE", pattern=_PHONE_BARE_SEP,
-                      validator=is_phone_value, normalizer=normalize_phone,
-                      neg_context=_PHONE_NEG_LOCAL),
-            # NATIONAL_ID (TUNED)
-            RegexRule(entity_type="NATIONAL_ID", pattern=_NATID_BARE_SSN,
-                      validator=is_valid_ssn, neg_context=_NATID_NEG_CONTEXT),
-            # IBAN (OLD6)
-            RegexRule(entity_type="IBAN", pattern=_IBAN_BARE, validator=is_valid_iban,
-                      normalizer=lambda v: re.sub(r"[\s-]", "", v).upper(),
-                      neg_context=_IBAN_NEG_CONTEXT),
-            RegexRule(entity_type="IBAN", pattern=_IBAN_BARE_SPACED, validator=is_valid_iban,
-                      normalizer=lambda v: re.sub(r"[\s-]", "", v).upper(),
-                      neg_context=_IBAN_NEG_CONTEXT),
-            # CARD_NUMBER (OLD6)
-            RegexRule(entity_type="CARD_NUMBER", pattern=_CARD_BARE,
-                      validator=is_valid_card_number, normalizer=strip_number_separators,
-                      neg_context=_CARD_NEG_CONTEXT),
-            # IP (TUNED)
-            RegexRule(entity_type="IP", pattern=_IP_BARE, validator=is_valid_ip),
-            # URL (TUNED)
-            RegexRule(entity_type="URL", pattern=_URL_BARE, validator=is_valid_url),
-            RegexRule(entity_type="URL", pattern=_URL_BARE_PATH, validator=is_valid_url_barepath),
-            # COORDINATE (OLD6)
-            RegexRule(entity_type="COORDINATE", pattern=_COORD_BARE_DECIMAL_PAIR,
-                      validator=is_valid_coord_pair_bare),
-            RegexRule(entity_type="COORDINATE", pattern=_COORD_BARE_DMS_PAIR,
-                      validator=is_valid_coord_pair),
-        ]
-        return [r for r in rules if self._is_enabled(r.entity_type)]
+        # v3 CONTEXT-ONLY: all bare-regex rules are intentionally removed. Every
+        # class is detected through a positive keyword context (see
+        # _build_context_rules). The bare patterns are still defined above (some
+        # are reused as value sub-patterns inside the context rules) but no rule
+        # scans the text without a cue anymore.
+        return []
 
     def _build_context_rules(self):
         # (entity_type, pattern, validator[, neg_context]) — value captured in
         # group(1); if neg_context is set, the rule is suppressed when the ~28 chars
         # of left context match it (e.g. an explicit API-key cue before a PASSWORD).
         specs = [
+            # v3: EMAIL / IBAN / IP context rules (these classes were bare-only in v2).
+            ("EMAIL", _EMAIL_CONTEXT, is_valid_email),
+            ("IBAN", _IBAN_CONTEXT, is_valid_iban),
+            ("IP", _IP_CONTEXT, is_valid_ip),
             ("PHONE", _PHONE_CONTEXT, is_phone_value_ctx),
             ("BANK_ACCOUNT", _BANK_CONTEXT, is_valid_account_number),
             ("CARD_NUMBER", _CARD_CONTEXT, is_valid_card_number),