Upload pii_extension.py with huggingface_hub

pii_extension.py

@@ -18,6 +18,93 @@ class PIILabel(Enum):
     SOCIAL_MEDIA = "social_media"
     URL = "url"
 
+class UnicodeDeobfuscator:
+    """Detect and normalize unicode obfuscation attempts"""
+    
+    # Unicode ranges for suspicious characters
+    CIRCLED_LETTERS = range(0x24B6, 0x24EA)  # Ⓐ-ⓩ
+    MATHEMATICAL_CHARS = range(0x1D400, 0x1D800)  # 𝐀-𝑍, etc
+    FULLWIDTH_CHARS = range(0xFF01, 0xFF5F)  # ！-｝
+    DOUBLE_STRUCK = range(0x2100, 0x2150)  # ℂ, ℍ, etc
+    BOX_DRAWING = range(0x2500, 0x2580)  # ┌─┐ etc
+    BLOCK_ELEMENTS = range(0x2580, 0x25A0)  # ▀-▟
+    
+    # Mapping of circled letters to normal
+    CIRCLED_MAP = {
+        # Uppercase
+        'Ⓐ': 'A', 'Ⓑ': 'B', 'Ⓒ': 'C', 'Ⓓ': 'D', 'Ⓔ': 'E',
+        'Ⓕ': 'F', 'Ⓖ': 'G', 'Ⓗ': 'H', 'Ⓘ': 'I', 'Ⓙ': 'J',
+        'Ⓚ': 'K', 'Ⓛ': 'L', 'Ⓜ': 'M', 'Ⓝ': 'N', 'Ⓞ': 'O',
+        'Ⓟ': 'P', 'Ⓠ': 'Q', 'Ⓡ': 'R', 'Ⓢ': 'S', 'Ⓣ': 'T',
+        'Ⓤ': 'U', 'Ⓥ': 'V', 'Ⓦ': 'W', 'Ⓧ': 'X', 'Ⓨ': 'Y', 'Ⓩ': 'Z',
+        # Lowercase
+        'ⓐ': 'a', 'ⓑ': 'b', 'ⓒ': 'c', 'ⓓ': 'd', 'ⓔ': 'e',
+        'ⓕ': 'f', 'ⓖ': 'g', 'ⓗ': 'h', 'ⓘ': 'i', 'ⓙ': 'j',
+        'ⓚ': 'k', 'ⓛ': 'l', 'ⓜ': 'm', 'ⓝ': 'n', 'ⓞ': 'o',
+        'ⓟ': 'p', 'ⓠ': 'q', 'ⓡ': 'r', 'ⓢ': 's', 'ⓣ': 't',
+        'ⓤ': 'u', 'ⓥ': 'v', 'ⓦ': 'w', 'ⓧ': 'x', 'ⓨ': 'y', 'ⓩ': 'z',
+    }
+    
+    @classmethod
+    def detect_obfuscation(cls, text: str) -> Tuple[bool, List[Tuple[str, str]], str]:
+        """
+        Detect unicode obfuscation
+        Returns: (is_obfuscated, [(char, type)], normalized_text)
+        """
+        suspicious = []
+        normalized = []
+        
+        for char in text:
+            code = ord(char)
+            
+            # Check circled letters
+            if char in cls.CIRCLED_MAP:
+                suspicious.append((char, 'circled'))
+                normalized.append(cls.CIRCLED_MAP[char])
+            # Check double-struck
+            elif code in cls.DOUBLE_STRUCK:
+                suspicious.append((char, 'double-struck'))
+                # Map common double-struck to normal
+                if char == 'ℂ':
+                    normalized.append('C')
+                elif char == 'ℍ':
+                    normalized.append('H')
+                elif char == 'ℕ':
+                    normalized.append('N')
+                elif char == 'ℙ':
+                    normalized.append('P')
+                elif char == 'ℚ':
+                    normalized.append('Q')
+                elif char == 'ℝ':
+                    normalized.append('R')
+                elif char == 'ℤ':
+                    normalized.append('Z')
+                else:
+                    normalized.append(char)
+            # Check fullwidth
+            elif code in cls.FULLWIDTH_CHARS:
+                suspicious.append((char, 'fullwidth'))
+                # Convert to normal ASCII
+                normalized.append(chr(code - 0xFEE0))
+            # Check mathematical
+            elif code in cls.MATHEMATICAL_CHARS:
+                suspicious.append((char, 'mathematical'))
+                normalized.append(char)  # Keep as-is for now
+            else:
+                normalized.append(char)
+        
+        is_obfuscated = len(suspicious) > 0
+        normalized_text = ''.join(normalized)
+        
+        return is_obfuscated, suspicious, normalized_text
+    
+    @classmethod
+    def normalize(cls, text: str) -> str:
+        """Quick normalize without detection details"""
+        _, _, normalized = cls.detect_obfuscation(text)
+        return normalized
+
+
 class PIIDetector:
     """Detect PII in text with context awareness"""
     
@@ -168,6 +255,7 @@ class PIIDetector:
     def scan(self, text: str, age: int) -> Dict:
         """
         Full PII scan with age-appropriate rules
+        Also detects unicode obfuscation
         
         Returns:
             {
@@ -177,45 +265,53 @@ class PIIDetector:
                 "social_media_allowed": bool,
                 "grooming_risk": float,
                 "action": "allow" | "block" | "flag",
-                "reason": str
+                "reason": str,
+                "obfuscation_detected": bool,
+                "normalized_text": str
             }
         """
+        # Step 0: Detect unicode obfuscation
+        is_obfuscated, suspicious_chars, normalized_text = UnicodeDeobfuscator.detect_obfuscation(text)
+        
+        # Use normalized text for detection if obfuscated
+        detection_text = normalized_text if is_obfuscated else text
+        
         pii_found = []
         pii_types = set()
         
-        # Detect various PII types
-        emails = self.detect_emails(text)
+        # Detect various PII types (using normalized text if obfuscated)
+        emails = self.detect_emails(detection_text)
         if emails:
             pii_types.add(PIILabel.EMAIL)
             for email, start, end in emails:
                 pii_found.append({"type": "email", "value": email, "start": start, "end": end})
         
-        phones = self.detect_phones(text)
+        phones = self.detect_phones(detection_text)
         if phones:
             pii_types.add(PIILabel.PHONE)
             for phone, start, end in phones:
                 pii_found.append({"type": "phone", "value": phone, "start": start, "end": end})
         
-        addresses = self.detect_addresses(text)
+        addresses = self.detect_addresses(detection_text)
         if addresses:
             pii_types.add(PIILabel.ADDRESS)
             for addr, start, end in addresses:
                 pii_found.append({"type": "address", "value": addr, "start": start, "end": end})
         
-        credit_cards = self.detect_credit_cards(text)
+        credit_cards = self.detect_credit_cards(detection_text)
         if credit_cards:
             pii_types.add(PIILabel.CREDIT_CARD)
             for cc, start, end in credit_cards:
                 pii_found.append({"type": "credit_card", "value": cc, "start": start, "end": end})
         
-        ssns = self.detect_ssn(text)
+        ssns = self.detect_ssn(detection_text)
         if ssns:
             pii_types.add(PIILabel.SSN)
             for ssn, start, end in ssns:
                 pii_found.append({"type": "ssn", "value": ssn, "start": start, "end": end})
         
-        # Social media detection
-        social_links = self.detect_social_media(text)
+        # Social media detection (also on normalized text)
+        social_links = self.detect_social_media(detection_text)
         has_social_media = len(social_links) > 0
         
         if has_social_media:
@@ -234,8 +330,8 @@ class PIIDetector:
             action = "block"
             reason = f"PII detected: {', '.join([p.value for p in critical_pii])}"
         elif has_social_media:
-            # Social media rules
-            is_grooming, grooming_risk, grooming_keywords = self.detect_grooming_context(text)
+            # Social media rules (use normalized text for grooming detection)
+            is_grooming, grooming_risk, grooming_keywords = self.detect_grooming_context(detection_text)
             
             if age < 13:
                 # Under 13: Block ALL social media sharing
@@ -261,6 +357,10 @@ class PIIDetector:
             elif grooming_risk > 0:
                 social_media_allowed = False
         
+        # Add obfuscation info to reason if detected
+        if is_obfuscated and action == "allow":
+            reason = f"Unicode obfuscation detected and normalized. {reason}"
+        
         return {
             "has_pii": len(pii_types) > 0,
             "pii_types": [p.value for p in pii_types],
@@ -270,7 +370,10 @@ class PIIDetector:
             "grooming_keywords": grooming_keywords,
             "action": action,
             "reason": reason,
-            "age": age
+            "age": age,
+            "obfuscation_detected": is_obfuscated,
+            "obfuscation_chars": [(c, t) for c, t in suspicious_chars] if is_obfuscated else [],
+            "normalized_text": normalized_text if is_obfuscated else text
         }