pipeline/preprocessing.py

import re
import nltk
import pandas as pd
from nltk.tokenize import wordpunct_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer

nltk.download('stopwords')
nltk.download('punkt')
nltk.download('wordnet')
nltk.download('omw-1.4')

# Initializing stopwords and lemmatizer once
stop_words = set(stopwords.words('english'))
lemmatizer = WordNetLemmatizer()


def clean_text(text: str) -> str:
    """
    Preprocesses input text:
    - Lowercase conversion
    - Removing non-alphanumeric characters
    - Tokenization
    - Stopword removal
    - Lemmatization

    Returns a cleaned string.
    """
    text = text.lower()
    text = re.sub(r"[^a-zA-Z0-9\s]", '', text)
    tokens = wordpunct_tokenize(text)
    tokens = [t for t in tokens if t not in stop_words]
    tokens = [lemmatizer.lemmatize(t) for t in tokens]
    return ' '.join(tokens)

def load_and_preprocess(filepath: str) -> pd.DataFrame:
    """
    Loads dataset from a CSV file, merges title + text, applies cleaning,
    and adds num_words column.

    Returns:
        Cleaned pandas DataFrame.
    """
    df = pd.read_csv(filepath)
    df.dropna(subset=['text', 'title'], inplace=True)
    df['text'] = df['title'] + ' ' + df['text']
    df.drop(columns=['title'], inplace=True)
    df['clean_text'] = df['text'].apply(clean_text)
    df['num_words'] = df['clean_text'].apply(lambda x: len(x.split()))
    return df