app/preprocessing.py

"""Text preprocessing shared by training and serving.

The same cleaning must run at train time and at inference time, so it lives in
one place and is imported by both ``model/train.py`` and the serving app.
Cleaning is regex-only (no NLTK) to keep the serving image light.
"""

import re

_HTML = re.compile(r"<[^>]+>")
_URL = re.compile(r"http\S+|www\S+|https\S+", re.MULTILINE)
_EMAIL = re.compile(r"\S+@\S+")
_NON_TEXT = re.compile(r"[^a-zA-Z0-9\s']")
_DOUBLE_APOS = re.compile(r"''")
_LONE_APOS = re.compile(r"\s'\s")
_WHITESPACE = re.compile(r"\s+")


def clean_text(text: str) -> str:
    """Normalise raw review text for the sentiment model.

    Strips HTML, lowercases, drops URLs/emails, keeps alphanumerics and
    apostrophes, and collapses whitespace. Applied identically at train and
    inference time so the model sees the same text distribution.

    Args:
        text: Raw input text.

    Returns:
        The cleaned text.
    """
    text = _HTML.sub(" ", text)
    text = text.lower()
    text = _URL.sub("", text)
    text = _EMAIL.sub("", text)
    text = _NON_TEXT.sub(" ", text)
    text = _DOUBLE_APOS.sub("", text)
    text = _LONE_APOS.sub(" ", text)
    text = _WHITESPACE.sub(" ", text)
    return text.strip()