preprocessing.py

import re

def preprocess_text(text):
    """

    Preprocess text by removing URLs, mentions, hashtags, and special characters.

    Convert to lowercase and remove extra whitespace.

    """
    # Convert to lowercase
    text = text.lower()
    
    # Remove URLs
    text = re.sub(r'http\S+|www\S+|https\S+', '', text, flags=re.MULTILINE)
    
    # Remove mentions and hashtags
    text = re.sub(r'@\w+|#\w+', '', text)
    
    # Remove RT (retweet) indicators
    text = re.sub(r'\brt\b', '', text)
    
    # Remove punctuation except spaces
    text = re.sub(r'[^\w\s]', '', text)
    
    # Remove extra whitespace
    text = ' '.join(text.split())
    
    return text