Update app.py

app.py

@@ -4,14 +4,19 @@ from bs4 import BeautifulSoup
 import pandas as pd
 import plotly.express as px
 from dateutil import parser
-import nltk
-nltk.downloader.download('vader_lexicon')
-from nltk.sentiment.vader import SentimentIntensityAnalyzer
 import datetime
 import requests
+from transformers import pipeline
 
 st.set_page_config(page_title="Stock News Sentiment Analyzer", layout="wide")
 
+# Initialize FinBERT pipeline
+@st.cache_resource
+def load_model():
+    return pipeline("text-classification", model="ProsusAI/finbert")
+
+finbert = load_model()
+
 def verify_link(url, timeout=10, retries=3):
     for _ in range(retries):
         try:
@@ -64,19 +69,40 @@ def parse_news(news_table):
     return parsed_news_df
 
 def score_news(parsed_news_df):
-    vader = SentimentIntensityAnalyzer()
+    # Get FinBERT predictions
+    predictions = finbert(parsed_news_df['headline'].tolist())
+    
+    # Convert predictions to sentiment scores
+    sentiment_scores = []
+    for pred in predictions:
+        label = pred['label']
+        score = pred['score']
+        
+        # Convert to -1 to 1 scale
+        if label == 'positive':
+            sentiment_score = score
+        elif label == 'negative':
+            sentiment_score = -score
+        else:  # neutral
+            sentiment_score = 0
+            
+        sentiment_scores.append({
+            'sentiment_score': sentiment_score,
+            'label': label,
+            'confidence': score
+        })
+    
+    # Convert to DataFrame
+    scores_df = pd.DataFrame(sentiment_scores)
+    
+    # Join with original news DataFrame
+    parsed_and_scored_news = parsed_news_df.join(scores_df)
+    parsed_and_scored_news = parsed_and_scored_news.set_index('datetime')
     
-    scores = parsed_news_df['headline'].apply(vader.polarity_scores).tolist()
-    scores_df = pd.DataFrame(scores)
-    parsed_and_scored_news = parsed_news_df.join(scores_df, rsuffix='_right')        
-    parsed_and_scored_news = parsed_and_scored_news.set_index('datetime')    
-    parsed_and_scored_news = parsed_and_scored_news.rename(columns={"compound": "sentiment_score"})
-
     return parsed_and_scored_news
 
 def plot_hourly_sentiment(parsed_and_scored_news, ticker):
-    numeric_cols = parsed_and_scored_news.select_dtypes(include=['float64', 'int64'])
-    mean_scores = numeric_cols.resample('h').mean()
+    mean_scores = parsed_and_scored_news['sentiment_score'].resample('h').mean()
     
     fig = px.bar(mean_scores, x=mean_scores.index, y='sentiment_score', 
                  title=f'{ticker} Hourly Sentiment Scores',
@@ -93,8 +119,7 @@ def plot_hourly_sentiment(parsed_and_scored_news, ticker):
     return fig
 
 def plot_daily_sentiment(parsed_and_scored_news, ticker):
-    numeric_cols = parsed_and_scored_news.select_dtypes(include=['float64', 'int64'])
-    mean_scores = numeric_cols.resample('D').mean()
+    mean_scores = parsed_and_scored_news['sentiment_score'].resample('D').mean()
     
     fig = px.bar(mean_scores, x=mean_scores.index, y='sentiment_score', 
                  title=f'{ticker} Daily Sentiment Scores',
@@ -113,14 +138,14 @@ def plot_daily_sentiment(parsed_and_scored_news, ticker):
 def get_recommendation(sentiment_scores):
     avg_sentiment = sentiment_scores['sentiment_score'].mean()
     
-    if avg_sentiment >= 0.05:
+    if avg_sentiment >= 0.3:
         return f"Positive sentiment (Score: {avg_sentiment:.2f}). The recent news suggests a favorable outlook for this stock. Consider buying or holding if you already own it."
-    elif avg_sentiment <= -0.05:
+    elif avg_sentiment <= -0.3:
         return f"Negative sentiment (Score: {avg_sentiment:.2f}). The recent news suggests caution. Consider selling or avoiding this stock for now."
     else:
         return f"Neutral sentiment (Score: {avg_sentiment:.2f}). The recent news doesn't show a strong bias. Consider holding if you own the stock, or watch for more definitive trends before making a decision."
 
-st.header("Stock News Sentiment Analyzer")
+st.header("Stock News Sentiment Analyzer (FinBERT)")
 
 ticker = st.text_input('Enter Stock Ticker', '').upper()
 
@@ -145,9 +170,9 @@ try:
 
     description = f"""
         The above charts average the sentiment scores of {ticker} stock hourly and daily.
-        The table below gives each of the most recent headlines of the stock and the negative, neutral, positive and an aggregated sentiment score.
+        The table below shows recent headlines with their sentiment scores and classifications.
         The news headlines are obtained from the FinViz website.
-        Sentiments are given by the nltk.sentiment.vader Python library.
+        Sentiments are analyzed using the ProsusAI/finbert model, which is specifically trained for financial text.
         Links have been verified for validity.
         """
         
@@ -158,7 +183,8 @@ try:
         axis=1
     )
     
-    st.write(parsed_and_scored_news.drop(columns=['is_valid']).to_html(escape=False), unsafe_allow_html=True)
+    display_df = parsed_and_scored_news.drop(columns=['is_valid'])
+    st.write(display_df.to_html(escape=False), unsafe_allow_html=True)
     
 except Exception as e:
     print(str(e))