import re import html from cleantext import clean from text_unidecode import unidecode def clean_text( text: str, *, # Basic cleanup normalize_whitespace: bool = True, remove_newlines: bool = True, strip: bool = True, # Character handling to_lowercase: bool = False, remove_punctuation: bool = False, # Content removal remove_urls: bool = False, remove_emails: bool = False, remove_phone_numbers: bool = False, remove_numbers: bool = False, remove_digits: bool = False, # Escape sequences fix_escape_sequences: bool = False, remove_html_entities: bool = True, # Special characters remove_currency_symbols: bool = True, remove_emoji: bool = True, normalize_unicode: bool = True, # Custom replacements custom_replacements: dict | None = None, ) -> str: """ Robust generic string cleaner using validated cleantext API parameters and text_unidecode for superior ASCII transliteration. Parameters ---------- text : Input string to clean. normalize_whitespace : Normalize all whitespace variants to single space. remove_newlines : Remove line breaks (\\n, \\r, etc.). strip : Strip leading/trailing whitespace. to_lowercase : Convert text to lowercase. remove_punctuation : Remove punctuation characters. remove_urls : Remove URLs (http/https/www). remove_emails : Remove email addresses. remove_phone_numbers : Remove phone numbers. remove_numbers : Remove standalone numbers. remove_digits : Remove all digit characters. fix_escape_sequences : Fix literal escape sequences (\\\\n, \\\\t, etc.). remove_html_entities : Decode HTML entities (& → &). remove_currency_symbols: Remove $, €, £, ¥, etc. remove_emoji : Remove emoji characters. normalize_unicode : Apply unicode fix + transliterate to ASCII via unidecode. custom_replacements : Dict of {exact_string: replacement} applied first. Returns ------- str : Cleaned string. """ # ── Guard: handle None / non-string input safely ───────────────────────── if text is None: return "" if not isinstance(text, str): text = str(text) if not text.strip(): return "" # ──────────────────────────────────────────────────────────────────────── # STEP 1 ── Custom replacements (exact string match, applied first) # ──────────────────────────────────────────────────────────────────────── if custom_replacements: for target, replacement in custom_replacements.items(): text = text.replace(target, replacement) # ──────────────────────────────────────────────────────────────────────── # STEP 2 ── Fix literal escape sequences BEFORE any other processing # ──────────────────────────────────────────────────────────────────────── if fix_escape_sequences: LITERAL_ESCAPE_MAP = [ ("\\n", " "), ("\\t", " "), ("\\r", " "), ("\\v", " "), ("\\f", " "), ("\\a", ""), ("\\b", ""), ("\\\\", " "), ("\\/", "/"), ("\\'", "'"), ('\\"', '"'), ] for literal, replacement in LITERAL_ESCAPE_MAP: text = text.replace(literal, replacement) # ──────────────────────────────────────────────────────────────────────── # STEP 3 ── Decode HTML entities (& → &, < → <, ' → ') # ──────────────────────────────────────────────────────────────────────── if remove_html_entities: text = html.unescape(text) # ──────────────────────────────────────────────────────────────────────── # STEP 4 ── Core cleaning via cleantext (validated API parameters only) # We disable cleantext's to_ascii because text_unidecode # handles transliteration far more robustly in STEP 5. # ──────────────────────────────────────────────────────────────────────── text = clean( text, fix_unicode=True, # Fix mojibake/encoding errors to_ascii=False, # Defer to text_unidecode lower=to_lowercase, normalize_whitespace=normalize_whitespace, no_line_breaks=remove_newlines, strip_lines=strip, no_urls=remove_urls, no_emails=remove_emails, no_phone_numbers=remove_phone_numbers, no_numbers=remove_numbers, no_digits=remove_digits, no_currency_symbols=remove_currency_symbols, no_punct=remove_punctuation, no_emoji=remove_emoji, replace_with_url="", replace_with_email="", replace_with_phone_number="", replace_with_number="", replace_with_digit="", replace_with_currency_symbol="", replace_with_punct="", lang="en", ) # ──────────────────────────────────────────────────────────────────────── # STEP 5 ── Robust ASCII transliteration via text_unidecode # Converts remaining non-ASCII (accents, cyrillic, greek, etc.) # ──────────────────────────────────────────────────────────────────────── if normalize_unicode: text = unidecode(text) # ──────────────────────────────────────────────────────────────────────── # STEP 6 ── Post-clean whitespace tidy-up # Removal + transliteration may leave stray multi-spaces # ──────────────────────────────────────────────────────────────────────── text = re.sub(r" {2,}", " ", text) if strip: text = text.strip() return text class TextCleanerService: def clean(self, text: str, **kwargs) -> str: return clean_text(text, **kwargs)