| import re | |
| def preprocess_text(text): | |
| """ | |
| Preprocess text by removing URLs, mentions, hashtags, and special characters. | |
| Convert to lowercase and remove extra whitespace. | |
| """ | |
| # Convert to lowercase | |
| text = text.lower() | |
| # Remove URLs | |
| text = re.sub(r'http\S+|www\S+|https\S+', '', text, flags=re.MULTILINE) | |
| # Remove mentions and hashtags | |
| text = re.sub(r'@\w+|#\w+', '', text) | |
| # Remove RT (retweet) indicators | |
| text = re.sub(r'\brt\b', '', text) | |
| # Remove punctuation except spaces | |
| text = re.sub(r'[^\w\s]', '', text) | |
| # Remove extra whitespace | |
| text = ' '.join(text.split()) | |
| return text | |