Spaces:
Sleeping
Sleeping
File size: 2,872 Bytes
605fc75 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 |
import pandas as pd
import re
import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
# Download necessary NLTK data
nltk.download('punkt')
nltk.download('stopwords')
nltk.download('wordnet')
nltk.download('punkt_tab')
# Initialize SentimentIntensityAnalyzer and Lemmatizer
sia = SentimentIntensityAnalyzer()
lemmatizer = WordNetLemmatizer()
# Load your datasets (adjust path if needed)
twitter_aapl_tweets = pd.read_csv("aapl_tweets.csv") # Example CSV file for Twitter data
reddit_aapl_posts = pd.read_csv("reddit_aapl_posts.csv") # Example CSV file for Reddit data
# Print columns to verify correct data loading
print("Twitter columns:", twitter_aapl_tweets.columns)
print("Reddit columns:", reddit_aapl_posts.columns)
# Function to clean the text
def clean_text(text):
# Remove URLs, mentions, hashtags, special characters, and lower the text
text = re.sub(r'http\S+|www\S+|https\S+', '', text) # Remove URLs
text = re.sub(r'@\w+', '', text) # Remove mentions
text = re.sub(r'#\w+', '', text) # Remove hashtags
text = re.sub(r'[^A-Za-z0-9\s]', '', text) # Remove special characters
text = text.lower() # Convert text to lowercase
text = word_tokenize(text) # Tokenize the text
# Remove stopwords
stop_words = set(stopwords.words('english'))
text = [word for word in text if word not in stop_words]
# Lemmatize words
text = [lemmatizer.lemmatize(word) for word in text]
# Join the words back into a string
return ' '.join(text)
# Function to get sentiment score using Vader
def get_sentiment(text):
return sia.polarity_scores(text)['compound']
# Apply cleaning and sentiment analysis to Twitter data
twitter_aapl_tweets['cleaned_text'] = twitter_aapl_tweets['Tweet'].apply(clean_text)
twitter_aapl_tweets['sentiment'] = twitter_aapl_tweets['cleaned_text'].apply(get_sentiment)
# Print the cleaned Twitter data
print("\nCleaned Twitter Data:")
print(twitter_aapl_tweets[['Tweet', 'cleaned_text', 'sentiment']].head())
# Apply cleaning and sentiment analysis to Reddit data
# Assuming 'Title' contains the post text (adjust to 'Comments' if needed)
reddit_aapl_posts['cleaned_text'] = reddit_aapl_posts['Title'].apply(clean_text) # Or 'Comments'
reddit_aapl_posts['sentiment'] = reddit_aapl_posts['cleaned_text'].apply(get_sentiment)
# Print the cleaned Reddit data
print("\nCleaned Reddit Data:")
print(reddit_aapl_posts[['Title', 'cleaned_text', 'sentiment']].head())
# Save cleaned data to CSV
twitter_aapl_tweets.to_csv("cleaned_twitter_data.csv", index=False)
reddit_aapl_posts.to_csv("cleaned_reddit_data.csv", index=False)
print("\nData cleaned and saved to CSV.")
|