import re import pandas as pd import regex as reg URL_RE = re.compile(r"(https?://\S+|www\.\S+)", re.IGNORECASE) HTML_RE = re.compile(r"<.*?>") MULTISPACE_RE = re.compile(r"\s+") # Keep Bangla + English + numbers + basic punctuation ALLOWED_CHARS_RE = reg.compile(r"[^\p{Bengali}A-Za-z0-9\s\.\,\!\?\:\;\-\(\)\'\"\/]+") def clean_text(s: str) -> str: if pd.isna(s): return "" s = str(s) s = s.replace("\u200d", " ").replace("\u200c", " ") s = URL_RE.sub(" ", s) s = HTML_RE.sub(" ", s) s = s.replace("’", "'").replace("“", '"').replace("”", '"') s = ALLOWED_CHARS_RE.sub(" ", s) s = MULTISPACE_RE.sub(" ", s).strip() return s def basic_stats(name: str, df: pd.DataFrame, text_col: str, label_col: str) -> None: print(f"\n[{name}] null text: {df[text_col].isna().sum()} / {len(df)}") print(f"[{name}] label counts:\n{df[label_col].value_counts(dropna=False)}")