preprocessing.py

"""ScamShield NLP — inference-time text preprocessing.

This mirrors the exact preprocessing applied when the training corpus was
built in the original experiment (``src/data/clean.py``):

1. Strip HTML tags (text inside the tags is preserved).
2. Collapse runs of whitespace into a single space.

Everything else (lowercasing, tokenization, n-gram extraction) is handled by
the fitted ``TfidfVectorizer`` during vectorization — exactly as it was during
training, so inference behaviour is identical to the experiment.

Note: the ``[SUBJECT]`` / ``[BODY]`` markers used when building email samples
are *not* applied to arbitrary user messages — they are a dataset-formatting
artifact, not a text transformation.
"""

import re

_HTML_TAG_RE = re.compile(r"<[^>]+>")
_WHITESPACE_RE = re.compile(r"\s+")


def clean_text(text) -> str:
    """Normalise incoming message text for the trained TF-IDF pipeline."""
    if not isinstance(text, str):
        return ""
    text = _HTML_TAG_RE.sub(" ", text)
    text = _WHITESPACE_RE.sub(" ", text).strip()
    return text