File size: 1,159 Bytes
62d0c8c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 | """Indonesian review cleaning - the SAME pipeline used to train the topic models, so new
text is processed identically before prediction:
lowercase -> strip URLs/mentions/symbols -> cut elongation -> normalize slang -> letters only.
"""
import re
import pandas as pd
from utils.paths import LEXICON
_slang = None
def _slang_map():
"""Lazy-load the colloquial Indonesian lexicon (slang -> formal)."""
global _slang
if _slang is None:
lex = pd.read_csv(LEXICON)
_slang = dict(zip(lex["slang"].astype(str), lex["formal"].astype(str)))
return _slang
def clean_review(text: str) -> str:
t = str(text).lower()
t = re.sub(r"http\S+|www\.\S+", " ", t) # URLs
t = re.sub(r"@\w+", " ", t) # mentions
t = re.sub(r"[^a-z\s]", " ", t) # keep letters + spaces
t = re.sub(r"(.)\1{2,}", r"\1\1", t) # cut elongation: "bagusss" -> "baguss"
t = re.sub(r"\s+", " ", t).strip()
t = " ".join(_slang_map().get(w, w) for w in t.split()) # slang -> formal
t = re.sub(r"[^a-z\s]", " ", t) # formal forms may add hyphens
return re.sub(r"\s+", " ", t).strip()
|