Update app.py

app.py

@@ -1,7 +1,8 @@
 import pandas as pd
 import numpy as np
-import os
-import io
+
+import os,io
+
 from datetime import datetime
 from datetime import timedelta
 
@@ -12,6 +13,11 @@ import hvplot.pandas
 
 import yfinance as yf
 
+from dotenv import load_dotenv
+# from datasets import load_dataset
+
+from utils import upload_to_hf_dataset, download_from_hf_dataset, load_hf_dataset
+
 pn.extension('bokeh', template='bootstrap')
 
 def _extract_raw_data(ticker):
@@ -60,31 +66,69 @@ def get_text(df):
     sio.seek(0)
     return sio
 
-#Rading gurufocus from github action pipeline
+
+# Load environment variables from .env file
+load_dotenv()
+
+# Get the Hugging Face API token from the environment; either set in .env file or in the environment directly in GitHub
+HF_TOKEN = os.getenv('HF_TOKEN')
+
+# Get the name of the GuruFocus dataset for TradingView, Finviz, MarketBeat and TipRanks to read from
+dataset_name_BeatingMarket_input = os.getenv('dataset_name_BeatingMarket_input')
+dataset_name_TradingView_input = os.getenv('dataset_name_TradingView_input')
+dataset_name_GuruFocus_input = os.getenv('dataset_name_GuruFocus_input')
+dataset_name_FinViz_input = os.getenv('dataset_name_FinViz_input')
+dataset_name_TipRanks_input = os.getenv('dataset_name_TipRanks_input')
+
+
 current_datetime = datetime.now().strftime("%Y-%m-%d")    
-# daily_gurufocus_csvfile = f"https://raw.githubusercontent.com/alirezax2/GurusFocusCrawl/main/gurufocusuruFocus_merged_{current_datetime}.csv"
-daily_gurufocus_csvfile = f"https://raw.githubusercontent.com/alirezax2/GurusFocusCrawl/main/gurufocus/GuruFocus_merged_{current_datetime}.csv"
-daily_gurufocus_DF = pd.read_csv(daily_gurufocus_csvfile)
 
-#Reading tiprank from local crawling pipeline; Need to update github action to merge gurufocus daily with weekly tiprank
-monthly_tiprank_csvfile = f"https://raw.githubusercontent.com/alirezax2/GurusFocusCrawl/main/tipranks/tipranks_2024-06-01.csv"
-monthly_tiprank_DF = pd.read_csv(monthly_tiprank_csvfile)[['Ticker','SmartScore','Market Capitalization','Sector','Industry']]
 
-#Reading finviz from github action pipeline another repository(public)
-daily_finviz_csvfile = f"https://raw.githubusercontent.com/alirezax2/FinVizCrawl/main/finviz/FinViz_{current_datetime}.csv"
-daily_finviz_DF = pd.read_csv(daily_finviz_csvfile)
+@pn.cache()
+def get_tradingview(current_datetime):
+  #Load lastest TradingView DataSet from HuggingFace Dataset which is always america.csv
+  # download_from_hf_dataset("america.csv", dataset_name_TradingView_input, HF_TOKEN)
+  return load_hf_dataset("america.csv", HF_TOKEN, dataset_name_TradingView_input)
+
+@pn.cache()
+def get_marketbeat(current_datetime):
+  return load_hf_dataset("beatingmarket.csv", HF_TOKEN, dataset_name_BeatingMarket_input)
+
+@pn.cache()
+def get_gurufocus(current_datetime):
+  #Rading gurufocus from github action pipeline
+  # daily_gurufocus_csvfile = f"https://raw.githubusercontent.com/alirezax2/GurusFocusCrawl/main/gurufocusuruFocus_merged_{current_datetime}.csv"
+  # daily_gurufocus_DF = pd.read_csv(daily_gurufocus_csvfile)
+  # daily_gurufocus_DF = load_dataset(dataset_name_GuruFocus_output , data_files="GuruFocus_merged.csv", split="train" , token=HF_TOKEN ).to_pandas()
+  return load_hf_dataset("GuruFocus_merged.csv", HF_TOKEN, dataset_name_GuruFocus_input)
+  
+@pn.cache()
+def get_finviz(current_datetime):
+   return load_hf_dataset("FinViz.csv", HF_TOKEN, dataset_name_FinViz_input)
+
+@pn.cache()
+def get_tipranks(current_datetime):
+  return load_hf_dataset("tipranks.csv", HF_TOKEN, dataset_name_TipRanks_input)
+
+
+#Reading gurufocus,finviz from local crawling pipeline; Need to update github action to merge gurufocus daily with monthly tiprank
+daily_gurufocus_DF = get_gurufocus(current_datetime)
+daily_finviz_DF = get_finviz(current_datetime)
+monthly_tiprank_DF = get_tipranks(current_datetime)
+monthly_tiprank_DF = monthly_tiprank_DF[['Ticker','SmartScore','Market Capitalization','Sector','Industry']]
+
+
 daily_finviz_DF['FinVizPrice']  = pd.to_numeric(daily_finviz_DF['Price'], errors='coerce').fillna(0).astype(float)
 daily_finviz_DF['FinVizTarget']  = pd.to_numeric(daily_finviz_DF['Target Price'], errors='coerce').fillna(0).astype(float)
 daily_finviz_DF['FinVizTargetpercent'] = (100*(daily_finviz_DF['FinVizTarget']-daily_finviz_DF['FinVizPrice'])/daily_finviz_DF['FinVizPrice']).round(2)
-daily_finviz_DF = daily_finviz_DF[['Ticker','Price','FinVizTarget','FinVizTargetpercent']]
-
+daily_finviz_DF_filtered = daily_finviz_DF[['Ticker','Price','FinVizTarget','FinVizTargetpercent']]
 
 #Merging tipranks with Gurufocus
 DFgurufocus = daily_gurufocus_DF[['Ticker' , 'GFValue']] # , 'GFValuediff']]
 DFmerge_tipranks_gurufocus = DFgurufocus.merge(monthly_tiprank_DF)
 
 #Merging Finviz with Merged last one
-DFmerge_tipranks_gurufocus = DFmerge_tipranks_gurufocus.merge(daily_finviz_DF)
+DFmerge_tipranks_gurufocus = DFmerge_tipranks_gurufocus.merge(daily_finviz_DF_filtered)
 
 if 'Price' in DFmerge_tipranks_gurufocus.columns and 'GFValue' in DFmerge_tipranks_gurufocus.columns:
   DFmerge_tipranks_gurufocus['GFValuepercent'] = 100* ( DFmerge_tipranks_gurufocus['GFValue'] - DFmerge_tipranks_gurufocus['Price']) / DFmerge_tipranks_gurufocus['Price']
@@ -94,13 +138,14 @@ if 'Price' in DFmerge_tipranks_gurufocus.columns and 'GFValue' in DFmerge_tipran
   DFmerge_tipranks_gurufocus = DFmerge_tipranks_gurufocus.drop(columns=['Market Capitalization'])
   DFmerge_tipranks_gurufocus = DFmerge_tipranks_gurufocus[['Ticker', 'Sector', 'Industry' , 'MarketCap' , 'SmartScore', 'Price' , 'GFValue' , 'GFValuepercent' , 'FinVizTarget','FinVizTargetpercent']]
 
+
 #widget
 ticker = pn.widgets.AutocompleteInput(name='Ticker', options=list(DFmerge_tipranks_gurufocus.Ticker) , placeholder='Write Ticker here همین جا',value='ALL', restrict=False)
 SmartScore = pn.widgets.EditableRangeSlider(name='Smart Score', start=0, end=10, value=(9, 10), step=1)
 Industry = pn.widgets.CheckBoxGroup( name='Select Industry', value=list(set(DFmerge_tipranks_gurufocus.Industry)), options=list(set(DFmerge_tipranks_gurufocus.Industry)), inline=True)
 Sector = pn.widgets.CheckBoxGroup( name='Select Sector', value=list(set(DFmerge_tipranks_gurufocus.Sector)), options=list(set(DFmerge_tipranks_gurufocus.Sector)), inline=False)
-GFValuepercent = pn.widgets.FloatSlider(name='GF Value %', start=-100, end=1000, step=1, value=12.0)
-FinVizTargetpercent = pn.widgets.FloatSlider(name='FinViz Target %', start=-100, end=1000, step=1, value=12.0)
+GFValuepercent = pn.widgets.FloatSlider(name='GF Value %', start=-100, end=100, step=1, value=12.0)
+FinVizTargetpercent = pn.widgets.FloatSlider(name='FinViz Target %', start=-100, end=100, step=1, value=12.0)
 MarketCap = pn.widgets.FloatSlider(name='Market Capital (B$)', start=0, end=1000, step=1, value=10)
 # Create a file download link
 download_button_GuruFocus = pn.widgets.FileDownload(
@@ -146,4 +191,4 @@ download_button_watchlist = pn.widgets.FileDownload(
 pn.extension('tabulator')
 bound_plot = pn.bind(get_DF, DF=DFmerge_tipranks_gurufocus,ticker=ticker,SmartScore=SmartScore,GFValuepercent=GFValuepercent, FinVizTargetpercent=FinVizTargetpercent, Sector=Sector ,MarketCap=MarketCap)
 
-pn.Column(pn.Row(pn.Column(ticker,SmartScore,GFValuepercent, FinVizTargetpercent, MarketCap, Sector, download_button_GuruFocus,download_button_FinViz ,download_button_Filter, download_button_watchlist, png_pane),bound_plot)).servable(title="Fair Value Ranking - Merged Gurufocus & Tipranks & FinViz & MorningStars")
+pn.Column(pn.Row(pn.Column(ticker,SmartScore,GFValuepercent, FinVizTargetpercent, MarketCap, Sector, download_button_GuruFocus,download_button_FinViz ,download_button_Filter, download_button_watchlist, png_pane),bound_plot)).servable(title="Fair Value Ranking - Merged Gurufocus & Tipranks & FinViz & MorningStars")