more fix

app.py

@@ -32,11 +32,13 @@ with left_column:
 
     st.subheader("Vader-based Sentiment Analysis")
 
-    with st.spinner("Connecting with www.investing.com..."):
+    with st.spinner("Connecting with Hopsworks..."):
         df = sentiment_analysis(option_name, datetime(2023, 1, 5))
-        st.table(df)
+        df_copy = df.copy()
+        df_copy = df_copy.set_index('publish_date')
+        st.table(df_copy.drop(['body_text', 'text_w_puncts', 'text_tokenized', 'text_w_stopwords', 'text_lemmatized', 'text_stemmed', 'text_processed', 'predicted_class'], axis=1))
         daily_df = aggregate_by_date(df)
-        "Current sentiment:", daily_df["compound"], "%"
+        "Current sentiment:", daily_df.iloc[0]['compound']
     
 with right_column:
 
@@ -48,9 +50,6 @@ with right_column:
 
     st.subheader("LSTM-based 7-day stock price prediction model")
 
-    with st.spinner("Loading LSTM model.."):
-        predict = model(daily_df, stock_df)
-        if predict == 1.0:
-            "The stock price tommorow is going up!"
-        else:
-            "The stock price tommorow is going down!"
+    with st.spinner("Loading LSTM model from Hopsworks.."):
+        date, value = model(option_ticker)
+        "The predicted stock value on ", date, "is", value

data_loader_functions.py

@@ -55,16 +55,18 @@ def scrape_news(urls, df, company):
 def get_news_from_hopsworks():
   project = hopsworks.login()
   fs = project.get_feature_store() 
-  try: 
-    feature_view = fs.get_feature_view(name="iris_modal", version=1)
-  except:
-    news_fg = fs.get_feature_group(name="market_news_fg", version=1)
-    query = news_fg.select_all()
-    feature_view = fs.create_feature_view(name="market_news",
-                                          version=1,
-                                          description="Read from market_news_fg",
-                                          query=query)
-  return feature_view.get_batch_data()
+  news_fg = fs.get_feature_group(name="market_news_fg", version=1)  
+  # try: 
+  #   feature_view = fs.get_feature_view(name="market_news", version=1)
+  # except:
+  #   news_fg = fs.get_feature_group(name="market_news_fg", version=1)
+  #   query = news_fg.select_all()
+  #   feature_view = fs.create_feature_view(name="market_news",
+  #                                         version=1,
+  #                                         description="Read from market_news_fg",
+  #                                         query=query)
+  query = news_fg.select_all()
+  return query.read()
 
 ## Formalize the date column
 def remove_parentheses(s):

sentiment_analysis.py

@@ -45,7 +45,7 @@ def time_2_datetime(x):
     dt_obj = datetime.fromtimestamp(x / 1000)
     return dt_obj
 
-def fetching_news():
+def fetching_news(company):
     articles_df = get_news_from_hopsworks()
     articles_df.loc[articles_df['ticker'] == company]
     articles_df['publish_date'] = articles_df['publish_date'].apply(time_2_datetime)
@@ -196,11 +196,11 @@ def vader_sentiment(articles_processed):
     return articles_processed
 
 def sentiment_analysis(company, day):
-    articles_df = news_scraping(company)
+    articles_df = fetching_news(company)
     articles_df = select_oneday_news(articles_df, day)
-    articles_processed = nlp_processing(articles_df)
-    articles_sentimentalized = vader_sentiment(articles_processed)
-    return articles_sentimentalized
+    # articles_processed = nlp_processing(articles_df)
+    # articles_sentimentalized = vader_sentiment(articles_processed)
+    return articles_df
 
 ## Aggregate News Sentiments Each Day
 def aggregate_by_date(articles_sentiments):

stock_prediction.py

@@ -1,22 +1,59 @@
 import hopsworks
 import joblib
+import math
+from sklearn.preprocessing import MinMaxScaler 
+import numpy as np
+from datetime import timedelta, datetime
 
 
 
 
-def model(daily_sentiment, stock_df):
-    daily_sentiment = daily_sentiment.rename(columns={'publish_date': 'date', 'ticker': 'name'})
-    daily_sentiment['name'] = daily_sentiment['name'].str.upper()
-    stock_df['date'] = stock_df['date'].apply(lambda x : x.date())
 
-    X = daily_sentiment.merge(stock_df)
-    X = X.drop(['date', 'name'], axis=1)
+def model(ticker):
+    project = hopsworks.login() 
 
-    project = hopsworks.login()
+    # import data
+    fs = project.get_feature_store() 
+    feature_view = fs.get_feature_view(
+        name = 'stock_prediction_fv',
+        version = 1
+    )
 
+    data = feature_view.get_training_data(2)[0]
+    data = data.sort_values(by='date')
+
+    last_date = data['date'].values[-1]
+    last_date = datetime.fromtimestamp(int(int(last_date) / 1000))
+    date = last_date.date() + timedelta(days=1)
+
+    data = data.set_index('date')
+    data.loc[data['name'] == 'APPLE']
+    data.drop(['name', 'predicted_class'], axis=1, inplace=True)
+
+    # scaling data
+    prices = data[['close','neg','neu','pos','compound']]
+    scaler = MinMaxScaler(feature_range=(0,1))
+    scaled_data = scaler.fit_transform(prices)
+
+    prediction_list = scaled_data[-60:]
+
+    x = []
+    x.append(prediction_list[-60:])
+    x = np.array(x)
+
+    # import model
     mr = project.get_model_registry()
-    model = mr.get_model("random_forest_classifier", version=1)
-    model_dir = model.download()
-    model = joblib.load(model_dir + "/model.pkl")
-    arr = model.predict(X)
-    return arr[0]
+    if ticker == 'AAPL':
+        remote_model = mr.get_model("LSTM_Apple", version=1)
+    elif ticker == 'AMZN':
+        remote_model = mr.get_model("LSTM_Amazon", version=1)
+    else:
+        remote_model = mr.get_model("LSTM_Meta", version=1)
+    model_dir = remote_model.download()
+    remote_model = joblib.load(model_dir + "/model.pkl")
+
+    # predict
+    out = remote_model.predict(x)
+    B=np.hstack((out,scaled_data[ : 1,1:]))
+    out = scaler.inverse_transform(B)[0,0]
+    return date, out