""" Data Preprocessing Module Handles text cleaning, tokenization, lemmatization, and class balancing. """ import re import string import pandas as pd import numpy as np from collections import Counter import nltk from nltk.corpus import stopwords from nltk.stem import WordNetLemmatizer from textblob import TextBlob # Download required NLTK data def download_nltk_data(): resources = ['stopwords', 'wordnet', 'omw-1.4', 'punkt', 'averaged_perceptron_tagger'] for r in resources: try: nltk.download(r, quiet=True) except Exception: pass download_nltk_data() # ───────────────────────────────────────────── # Label Mapping # ───────────────────────────────────────────── # Map dataset labels → unified 7-class schema LABEL_MAP = { "Normal": "Normal", "Anxiety": "Anxiety", "Depression": "Depression", "Suicidal": "Suicidal", "Bipolar": "Bipolar", "Stress": "Stress", "Personality disorder": "Personality Disorder", # Aliases (in case of variations) "PTSD": "Stress", "BPD": "Personality Disorder", "Borderline": "Personality Disorder", } LABEL2ID = { "Normal": 0, "Anxiety": 1, "Depression": 2, "Suicidal": 3, "Bipolar": 4, "Stress": 5, "Personality Disorder": 6, } ID2LABEL = {v: k for k, v in LABEL2ID.items()} NUM_LABELS = len(LABEL2ID) # ───────────────────────────────────────────── # Text Cleaning # ───────────────────────────────────────────── def clean_text(text: str) -> str: """ Thorough text cleaning pipeline: - Lowercase - Remove URLs, mentions, hashtags - Remove special characters / digits - Normalize whitespace """ if not isinstance(text, str): return "" # Lowercase text = text.lower() # Remove URLs text = re.sub(r"http\S+|www\S+|https\S+", "", text, flags=re.MULTILINE) # Remove Reddit-style mentions and subreddits text = re.sub(r"@\w+|r/\w+|u/\w+", "", text) # Remove HTML entities text = re.sub(r"&[a-z]+;", " ", text) # Remove digits text = re.sub(r"\d+", "", text) # Remove punctuation (keep apostrophes for contractions) text = text.translate(str.maketrans("", "", string.punctuation.replace("'", ""))) # Remove extra whitespace text = re.sub(r"\s+", " ", text).strip() return text def lemmatize_text(text: str) -> str: """Lemmatize tokens and remove stopwords.""" lemmatizer = WordNetLemmatizer() stop_words = set(stopwords.words("english")) tokens = text.split() tokens = [lemmatizer.lemmatize(t) for t in tokens if t not in stop_words and len(t) > 2] return " ".join(tokens) def get_sentiment_features(text: str) -> dict: """ Extract TextBlob sentiment features: - polarity (-1 to 1) - subjectivity (0 to 1) - sentiment_label: Positive / Neutral / Negative """ blob = TextBlob(text) polarity = blob.sentiment.polarity subjectivity = blob.sentiment.subjectivity if polarity > 0.05: label = "Positive" elif polarity < -0.05: label = "Negative" else: label = "Neutral" return { "polarity": polarity, "subjectivity": subjectivity, "sentiment_label": label, } def full_preprocess(text: str, lemmatize: bool = True) -> str: """Full preprocessing pipeline for inference.""" text = clean_text(text) if lemmatize: text = lemmatize_text(text) return text # ───────────────────────────────────────────── # Dataset Loading # ───────────────────────────────────────────── def load_dataset(csv_path: str, max_len: int = 512) -> pd.DataFrame: """ Load and preprocess the CSV dataset. Expected columns: (index), statement, status Returns a clean DataFrame with columns: text, label, label_id, sentiment_* """ print(f"[DATA] Loading dataset from: {csv_path}") df = pd.read_csv(csv_path, index_col=0) # Rename columns df.columns = [c.strip().lower() for c in df.columns] if "statement" in df.columns: df.rename(columns={"statement": "text", "status": "label"}, inplace=True) # Drop nulls df.dropna(subset=["text", "label"], inplace=True) # Normalize labels df["label"] = df["label"].str.strip().map(LABEL_MAP) df.dropna(subset=["label"], inplace=True) # Map to numeric IDs df["label_id"] = df["label"].map(LABEL2ID) # Clean text print("[DATA] Cleaning text...") df["text_clean"] = df["text"].apply(clean_text) # Filter too-short texts df = df[df["text_clean"].str.len() > 5].reset_index(drop=True) # Sentiment features print("[DATA] Extracting sentiment features...") sentiment = df["text"].apply(get_sentiment_features).apply(pd.Series) df = pd.concat([df, sentiment], axis=1) print(f"[DATA] Dataset loaded: {len(df)} samples") print(f"[DATA] Label distribution:\n{df['label'].value_counts()}") return df def compute_class_weights(label_ids: np.ndarray) -> np.ndarray: """ Compute inverse-frequency class weights for weighted loss. Returns array of shape [num_classes]. """ counts = Counter(label_ids) total = sum(counts.values()) weights = np.zeros(NUM_LABELS, dtype=np.float32) for cls_id, count in counts.items(): weights[cls_id] = total / (NUM_LABELS * count) return weights