utils/masker3.py

import re
import spacy
from typing import Dict, Any, List
from langdetect import detect
from deep_translator import GoogleTranslator

# Load spaCy model
nlp = spacy.load("en_core_web_sm")

def mask_pii(text: str) -> Dict[str, Any]:
    """
    Enhanced PII masking with JSON output format
    """
    lang = detect(text)
    if lang == 'en':
            #return text
            masked_text = text
    else:
            # Translate to English
            translated = GoogleTranslator(source=lang, target='en').translate(text)
            masked_text = translated
    entities = []
    
    def mask_and_record(pattern, label, group=0):
        nonlocal masked_text, entities
        for match in reversed(list(re.finditer(pattern, masked_text))):
            start, end = match.span(group)
            original = match.group(group)
            
            # Skip if already masked or overlaps
            if any(e['position'][0] <= start < e['position'][1] for e in entities):
                continue
            
            masked_text = masked_text[:start] + f"[{label}]" + masked_text[end:]
            entities.append({
                "position": [start, end],
                "classification": label,
                "entity": original
            })

    # Specific patterns first
    mask_and_record(r'\b(\d{4}[ -]?\d{4}[ -]?\d{4})\b', 'aadhar_num')
    mask_and_record(r'\b((?:\d[ -]*?){15,18}\d)\b', 'credit_debit_no')
    mask_and_record(r'(?:CVV|CVC|Security Code)[: ]*(\d{3,4})\b', 'cvv_no', 1)
    mask_and_record(r'\b((0[1-9]|1[0-2])[/-](\d{2}|\d{4}))\b', 'expiry_no', 1)

    dob_patterns = [
        r'\b(\d{1,2}[/-]\d{1,2}[/-]\d{2,4})\b',
        r'\b(\d{4}[/-]\d{1,2}[/-]\d{1,2})\b',
        r'\b((?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* \d{1,2},? \d{4})\b'
    ]
    for pattern in dob_patterns:
        mask_and_record(pattern, 'dob', 1)

    mask_and_record(r'(\+?\d{1,3}[-.\s]?)?\(?\d{1,4}\)?[-.\s]?\d{1,4}[-.\s]?\d{1,9}', 'phone_number')
    mask_and_record(r'(\b[\w.-]+@[\w.-]+\.\w+\b)', 'email')

    # spaCy for full names
    doc = nlp(masked_text)
    for ent in reversed(doc.ents):
        if ent.label_ == "PERSON":
            if any(e['position'][0] <= ent.start_char < e['position'][1] for e in entities):
                continue
            masked_text = masked_text[:ent.start_char] + "[full_name]" + masked_text[ent.end_char:]
            entities.append({
                "position": [ent.start_char, ent.end_char],
                "classification": "full_name",
                "entity": ent.text
            })

    # Optional: Set category based on simple rule or ML model
    category = "Problem"
    eng_mask = masked_text


    # if lang == 'en':
    #      masked_text = masked_text
    # else:
    #      masked_text = GoogleTranslator(source='en', target=lang).translate(masked_text)
    text2 = text
    for ent in entities:
        entity_value = ent['entity']
        classification = ent['classification']
        text = text.replace(entity_value, f"[{classification}]")
    masked_text = text
    return {
        "input_email_body": text2,
        "list_of_masked_entities": sorted(entities, key=lambda x: x["position"][0]),
        "masked_email": masked_text,
        "category_of_the_email": category,
        "English_masked": eng_mask

    }


text = "Subject: Unvorhergesehener Absturz der Datenanalyse-Plattform\n\nDie Datenanalyse-Plattform brach unerwartet ab, da die Speicheroberfläche zu gering war My name is Sophia Rossi.. Ich habe versucht, Laravel 8 und meinen MacBook Pro neu zu starten, aber das Problem behält sich bei. Ich benötige Ihre Unterstützung, um diesen Fehler zu beheben. You can reach me at janesmith@company.com."
print(mask_pii(text))