src/preprocessing.py

"""Dual-pipeline text preprocessing.

Two functions, two audiences. Do not unify - transformer encoders rely on slang
and informal markers that traditional/static-embedding models cannot exploit.

- ``preprocess_minimal(text)`` - Pipeline A, for transformer models
  (IndoBERT, IBT, XLM-R, mDeBERTa). Steps: NFKD → lowercase → URL strip →
  whitespace collapse. **Keeps** slang, emojis, mentions, hashtags.

- ``preprocess_full(text, do_stemming=False)`` - Pipeline B, for TF-IDF
  (RF/SVM/NB) and FastText (Hybrid DL). Steps: NFKD → lowercase →
  strip URLs / @mentions / #hashtags / non-ASCII → regex tokenize →
  slang normalize → stopword removal → optional Sastrawi stemming.

Resources loaded at import time from ``data/raw/``:
- slang dicts: ``kamus-singkatan.json``, ``slang-indo.json``,
  ``custom-slang-words.json`` (later files override earlier)
- stopwords: Sastrawi's default ∪ ``custom-stopwords.json``
- stemmer: lazy singleton via ``Sastrawi.Stemmer.StemmerFactory``

NFKD always runs **before** lowercase so math-bold (U+1D400–U+1D7FF) decomposes
to ASCII first (`𝐒` → `S` → `s`). Mojibake fix happens upstream in
``data_loader.py``, not here.
"""

from __future__ import annotations

import json
import logging
import re
import unicodedata
from functools import lru_cache
from pathlib import Path

logger = logging.getLogger(__name__)

_DATA_DIR = Path(__file__).parent.parent / "data" / "raw"

_SLANG_FILES = ("kamus-singkatan.json", "slang-indo.json", "custom-slang-words.json")
_STOPWORD_FILE = "custom-stopwords.json"

_URL_RE = re.compile(r"https?://\S+|www\.\S+")
_MENTION_RE = re.compile(r"@\w+")
_HASHTAG_RE = re.compile(r"#\w+")
_NON_ASCII_RE = re.compile(r"[^\x00-\x7f]")
_WHITESPACE_RE = re.compile(r"\s+")
# Word characters only - drops any leftover punctuation (",.!?:;...) and emoticons.
# Numbers are preserved (e.g. "2024").
_TOKEN_PATTERN = re.compile(r"\w+", flags=re.UNICODE)


def _load_json(path: Path) -> dict | list | None:
    """Read a JSON file or return None if missing/unreadable."""
    if not path.is_file():
        logger.warning("missing dictionary file: %s - falling back to empty", path)
        return None
    try:
        with path.open(encoding="utf-8") as fh:
            return json.load(fh)
    except (OSError, json.JSONDecodeError) as exc:
        logger.warning("failed to read %s: %s - falling back to empty", path, exc)
        return None


def _build_slang_dict() -> dict[str, str]:
    """Merge the three slang dictionaries, with later files overriding earlier ones."""
    merged: dict[str, str] = {}
    for fname in _SLANG_FILES:
        data = _load_json(_DATA_DIR / fname)
        if isinstance(data, dict):
            merged.update({str(k).lower(): str(v).lower() for k, v in data.items()})
    logger.info("slang dictionary loaded: %d entries", len(merged))
    return merged


def _build_stopword_set() -> set[str]:
    """Union Sastrawi's stopword list with the custom JSON stopword list."""
    stopwords: set[str] = set()
    try:
        from Sastrawi.StopWordRemover.StopWordRemoverFactory import StopWordRemoverFactory
        stopwords.update(StopWordRemoverFactory().get_stop_words())
    except ImportError:
        logger.warning("Sastrawi not installed - Sastrawi stopwords skipped")

    custom = _load_json(_DATA_DIR / _STOPWORD_FILE)
    if isinstance(custom, list):
        stopwords.update(str(w).lower() for w in custom)
    elif isinstance(custom, dict):
        stopwords.update(str(k).lower() for k in custom.keys())

    logger.info("stopword set loaded: %d entries", len(stopwords))
    return stopwords


_SLANG_DICT: dict[str, str] = _build_slang_dict()
_STOPWORDS: set[str] = _build_stopword_set()


@lru_cache(maxsize=1)
def _get_stemmer():
    """Lazy singleton for the Sastrawi stemmer (slow to construct)."""
    from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
    return StemmerFactory().create_stemmer()


def preprocess_minimal(text: str) -> str:
    """Light preprocessing for IndoBERT: NFKD + lowercase + URL strip + whitespace."""
    if not isinstance(text, str) or not text:
        return ""
    text = unicodedata.normalize("NFKD", text)
    text = text.lower()
    text = _URL_RE.sub(" ", text)
    text = _WHITESPACE_RE.sub(" ", text)
    return text.strip()


def preprocess_full(text: str, *, do_stemming: bool = False) -> str:
    """Full preprocessing for TF-IDF/FastText: strip noise, normalize slang, remove stopwords."""
    if not isinstance(text, str) or not text:
        return ""

    text = unicodedata.normalize("NFKD", text)
    text = text.lower()
    text = _URL_RE.sub(" ", text)
    text = _MENTION_RE.sub(" ", text)
    text = _HASHTAG_RE.sub(" ", text)
    # Non-ASCII strip removes emojis and any residual mojibake markers.
    # Safe for Indonesian (Latin alphabet); NFKD already decomposed math-bold to ASCII.
    text = _NON_ASCII_RE.sub(" ", text)

    tokens = _TOKEN_PATTERN.findall(text)
    tokens = [_SLANG_DICT.get(tok, tok) for tok in tokens]
    tokens = [tok for tok in tokens if tok and tok not in _STOPWORDS]

    if do_stemming:
        stemmer = _get_stemmer()
        tokens = [stemmer.stem(tok) for tok in tokens]
        tokens = [tok for tok in tokens if tok]

    return " ".join(tokens)