src/text_cleaning.py

import re

import pandas as pd
import regex as reg

URL_RE = re.compile(r"(https?://\S+|www\.\S+)", re.IGNORECASE)
HTML_RE = re.compile(r"<.*?>")
MULTISPACE_RE = re.compile(r"\s+")

# Keep Bangla + English + numbers + basic punctuation
ALLOWED_CHARS_RE = reg.compile(r"[^\p{Bengali}A-Za-z0-9\s\.\,\!\?\:\;\-\(\)\'\"\/]+")


def clean_text(s: str) -> str:
    if pd.isna(s):
        return ""
    s = str(s)

    s = s.replace("\u200d", " ").replace("\u200c", " ")
    s = URL_RE.sub(" ", s)
    s = HTML_RE.sub(" ", s)

    s = s.replace("’", "'").replace("“", '"').replace("”", '"')
    s = ALLOWED_CHARS_RE.sub(" ", s)

    s = MULTISPACE_RE.sub(" ", s).strip()
    return s


def basic_stats(name: str, df: pd.DataFrame, text_col: str, label_col: str) -> None:
    print(f"\n[{name}] null text: {df[text_col].isna().sum()} / {len(df)}")
    print(f"[{name}] label counts:\n{df[label_col].value_counts(dropna=False)}")