app/services/text_cleaner_service.py

import re
import html
from cleantext import clean
from text_unidecode import unidecode


def clean_text(
    text: str,
    *,
    # Basic cleanup
    normalize_whitespace: bool = True,
    remove_newlines: bool = True,
    strip: bool = True,
    # Character handling
    to_lowercase: bool = False,
    remove_punctuation: bool = False,
    # Content removal
    remove_urls: bool = False,
    remove_emails: bool = False,
    remove_phone_numbers: bool = False,
    remove_numbers: bool = False,
    remove_digits: bool = False,
    # Escape sequences
    fix_escape_sequences: bool = False,
    remove_html_entities: bool = True,
    # Special characters
    remove_currency_symbols: bool = True,
    remove_emoji: bool = True,
    normalize_unicode: bool = True,
    # Custom replacements
    custom_replacements: dict | None = None,
) -> str:
    """
    Robust generic string cleaner using validated cleantext API parameters 
    and text_unidecode for superior ASCII transliteration.

    Parameters
    ----------
    text                  : Input string to clean.
    normalize_whitespace  : Normalize all whitespace variants to single space.
    remove_newlines       : Remove line breaks (\\n, \\r, etc.).
    strip                 : Strip leading/trailing whitespace.
    to_lowercase          : Convert text to lowercase.
    remove_punctuation    : Remove punctuation characters.
    remove_urls           : Remove URLs (http/https/www).
    remove_emails         : Remove email addresses.
    remove_phone_numbers  : Remove phone numbers.
    remove_numbers        : Remove standalone numbers.
    remove_digits         : Remove all digit characters.
    fix_escape_sequences  : Fix literal escape sequences (\\\\n, \\\\t, etc.).
    remove_html_entities  : Decode HTML entities (& → &).
    remove_currency_symbols: Remove $, €, £, ¥, etc.
    remove_emoji          : Remove emoji characters.
    normalize_unicode     : Apply unicode fix + transliterate to ASCII via unidecode.
    custom_replacements   : Dict of {exact_string: replacement} applied first.

    Returns
    -------
    str : Cleaned string.
    """

    # ── Guard: handle None / non-string input safely ─────────────────────────
    if text is None:
        return ""
    if not isinstance(text, str):
        text = str(text)
    if not text.strip():
        return ""

    # ────────────────────────────────────────────────────────────────────────
    # STEP 1 ── Custom replacements (exact string match, applied first)
    # ────────────────────────────────────────────────────────────────────────
    if custom_replacements:
        for target, replacement in custom_replacements.items():
            text = text.replace(target, replacement)

    # ────────────────────────────────────────────────────────────────────────
    # STEP 2 ── Fix literal escape sequences BEFORE any other processing
    # ────────────────────────────────────────────────────────────────────────
    if fix_escape_sequences:
        LITERAL_ESCAPE_MAP = [
            ("\\n",  " "),   
            ("\\t",  " "),   
            ("\\r",  " "),   
            ("\\v",  " "),   
            ("\\f",  " "),   
            ("\\a",  ""),    
            ("\\b",  ""),    
            ("\\\\", " "),   
            ("\\/",  "/"),   
            ("\\'",  "'"),   
            ('\\"',  '"'),   
        ]
        for literal, replacement in LITERAL_ESCAPE_MAP:
            text = text.replace(literal, replacement)

    # ────────────────────────────────────────────────────────────────────────
    # STEP 3 ── Decode HTML entities  (&amp; → &,  &lt; → <,  &#39; → ')
    # ────────────────────────────────────────────────────────────────────────
    if remove_html_entities:
        text = html.unescape(text)

    # ────────────────────────────────────────────────────────────────────────
    # STEP 4 ── Core cleaning via cleantext (validated API parameters only)
    #           We disable cleantext's to_ascii because text_unidecode 
    #           handles transliteration far more robustly in STEP 5.
    # ────────────────────────────────────────────────────────────────────────
    text = clean(
        text,
        fix_unicode=True,                 # Fix mojibake/encoding errors
        to_ascii=False,                   # Defer to text_unidecode
        lower=to_lowercase,
        normalize_whitespace=normalize_whitespace,
        no_line_breaks=remove_newlines,
        strip_lines=strip,
        no_urls=remove_urls,
        no_emails=remove_emails,
        no_phone_numbers=remove_phone_numbers,
        no_numbers=remove_numbers,
        no_digits=remove_digits,
        no_currency_symbols=remove_currency_symbols,
        no_punct=remove_punctuation,
        no_emoji=remove_emoji,
        replace_with_url="",
        replace_with_email="",
        replace_with_phone_number="",
        replace_with_number="",
        replace_with_digit="",
        replace_with_currency_symbol="",
        replace_with_punct="",
        lang="en",
    )

    # ────────────────────────────────────────────────────────────────────────
    # STEP 5 ── Robust ASCII transliteration via text_unidecode
    #           Converts remaining non-ASCII (accents, cyrillic, greek, etc.)
    # ────────────────────────────────────────────────────────────────────────
    if normalize_unicode:
        text = unidecode(text)

    # ────────────────────────────────────────────────────────────────────────
    # STEP 6 ── Post-clean whitespace tidy-up
    #           Removal + transliteration may leave stray multi-spaces
    # ────────────────────────────────────────────────────────────────────────
    text = re.sub(r" {2,}", " ", text)   
    if strip:
        text = text.strip()

    return text


class TextCleanerService:
    def clean(self, text: str, **kwargs) -> str:
        return clean_text(text, **kwargs)