preprocessing.py

"""Preprocessing self-contained untuk HF Space (mirror dari cloudsentimen.preprocess).

Harus IDENTIK dengan transformasi training agar tidak terjadi train/serve skew:
lowercase, hapus URL/emoji/tanda baca, normalisasi slang, hapus stopword (Sastrawi).
Stemming dimatikan (sesuai default config.preprocess.do_stemming = false).
"""
from __future__ import annotations

import csv
import re
from functools import lru_cache
from pathlib import Path

URL_RE = re.compile(r"https?://\S+|www\.\S+")
EMOJI_RE = re.compile(
    "[\U0001F000-\U0001FAFF\U00002600-\U000027BF\U0001F1E6-\U0001F1FF]", flags=re.UNICODE
)
NON_ALPHA_RE = re.compile(r"[^a-z\s]")
MULTISPACE_RE = re.compile(r"\s+")
HERE = Path(__file__).resolve().parent


@lru_cache(maxsize=1)
def _slang() -> dict:
    p = HERE / "slang.csv"
    if not p.exists():
        return {}
    with open(p, encoding="utf-8") as f:
        return {row["slang"]: row["baku"] for row in csv.DictReader(f)}


# Negasi krusial untuk sentimen — JANGAN dibuang (harus sama dgn cloudsentimen.preprocess).
NEGATION_KEEP = {"tidak", "tak", "bukan", "jangan", "belum", "kurang", "tanpa", "gagal"}


@lru_cache(maxsize=1)
def _stopwords() -> set:
    from Sastrawi.StopWordRemover.StopWordRemoverFactory import StopWordRemoverFactory

    return set(StopWordRemoverFactory().get_stop_words()) - NEGATION_KEEP


def clean_for_inference(text: str, min_len: int = 2) -> str:
    t = str(text).lower()
    t = URL_RE.sub(" ", t)
    t = EMOJI_RE.sub(" ", t)
    t = NON_ALPHA_RE.sub(" ", t)
    t = MULTISPACE_RE.sub(" ", t).strip()
    slang = _slang()
    t = " ".join(slang.get(tok, tok) for tok in t.split())
    sw = _stopwords()
    t = " ".join(tok for tok in t.split() if tok not in sw and len(tok) >= min_len)
    return t.strip()