Spaces:
Sleeping
Sleeping
Manus commited on
Commit ·
9d08131
1
Parent(s): 045d943
Fix indentation error in core/plot.py and update app.py for correct data passing.
Browse files- __pycache__/config.cpython-311.pyc +0 -0
- app.py +29 -23
- core/__pycache__/__init__.cpython-311.pyc +0 -0
- core/__pycache__/data.cpython-311.pyc +0 -0
- core/__pycache__/model_runner.cpython-311.pyc +0 -0
- core/__pycache__/models.cpython-311.pyc +0 -0
- core/__pycache__/plot.cpython-311.pyc +0 -0
- core/__pycache__/signals.cpython-311.pyc +0 -0
- core/__pycache__/train_eval.cpython-311.pyc +0 -0
- core/data.py +164 -185
- core/models.py +15 -0
- core/plot.py +100 -82
- core/signals.py +71 -58
- core/train_eval.py +13 -13
- test_app.py +164 -0
__pycache__/config.cpython-311.pyc
ADDED
|
Binary file (968 Bytes). View file
|
|
|
app.py
CHANGED
|
@@ -21,7 +21,7 @@ logging.basicConfig(
|
|
| 21 |
logging.FileHandler(log_path),
|
| 22 |
logging.StreamHandler()
|
| 23 |
],
|
| 24 |
-
format='%(asctime)s - %(levelname)s - %(message)s'
|
| 25 |
)
|
| 26 |
analyzer = SentimentIntensityAnalyzer()
|
| 27 |
|
|
@@ -33,10 +33,10 @@ def sentiment_analysis(ticker, start_date, end_date, api_key):
|
|
| 33 |
start = pd.to_datetime(start_date)
|
| 34 |
end = pd.to_datetime(end_date)
|
| 35 |
articles = newsapi.get_everything(
|
| 36 |
-
q=ticker, from_param=start.strftime(
|
| 37 |
-
language='en', sort_by='relevancy'
|
| 38 |
)
|
| 39 |
-
sentiments = [analyzer.polarity_scores(article[
|
| 40 |
avg_sentiment = np.mean(sentiments) if sentiments else 0.0
|
| 41 |
sentiment_text = f"Average sentiment for {ticker}: {avg_sentiment:.2f}"
|
| 42 |
return sentiment_text, avg_sentiment
|
|
@@ -45,9 +45,9 @@ def sentiment_analysis(ticker, start_date, end_date, api_key):
|
|
| 45 |
return f"Sentiment analysis failed: {str(e)}", None
|
| 46 |
|
| 47 |
def update_horizon_label(timeframe):
|
| 48 |
-
units = {'1m': 'minutes', '5m': 'minutes', '15m': 'minutes', '30m': 'minutes',
|
| 49 |
-
'1h': 'hours', '4h': 'hours', '1d': 'days', '1wk': 'weeks'}
|
| 50 |
-
return gr.update(label=f"Horizon ({units.get(timeframe, 'days')})")
|
| 51 |
|
| 52 |
def run_dashboard(data_src, ticker, file_upload, timeframe, start_date, end_date, horizon, indicators,
|
| 53 |
include_sentiment, news_api_key, alpha_api_key, account_size, risk_percent, model,
|
|
@@ -65,13 +65,13 @@ def run_dashboard(data_src, ticker, file_upload, timeframe, start_date, end_date
|
|
| 65 |
logging.error("Failed to load data")
|
| 66 |
return [None] * 12 + ["Failed to load data", None, None, None, None, "Failed to load data"]
|
| 67 |
|
| 68 |
-
df = add_technical_indicators(df, indicators)
|
| 69 |
if include_sentiment and news_api_key:
|
| 70 |
df = add_sentiment(df, ticker, news_api_key, start_date, end_date)
|
| 71 |
sentiment_text, sentiment_score = sentiment_analysis(ticker, start_date, end_date, news_api_key)
|
| 72 |
|
| 73 |
-
features =
|
| 74 |
-
target = 'value'
|
| 75 |
result = get_model(
|
| 76 |
df=df,
|
| 77 |
features=features,
|
|
@@ -92,23 +92,25 @@ def run_dashboard(data_src, ticker, file_upload, timeframe, start_date, end_date
|
|
| 92 |
importance_threshold=feat_threshold
|
| 93 |
)
|
| 94 |
if isinstance(result, dict) and result.get("error"):
|
| 95 |
-
logging.error(f"Model training failed: {result[
|
| 96 |
-
return [None] * 12 + [f"Model training failed: {result[
|
| 97 |
|
| 98 |
-
signals_df = generate_signals(df, result)
|
| 99 |
if signals_df.empty:
|
| 100 |
logging.error("Failed to generate signals")
|
| 101 |
return [None] * 12 + ["Failed to generate signals", None, None, None, None, "Failed to generate signals"]
|
| 102 |
|
| 103 |
chart_plot = plot_indicators(df, ticker)
|
| 104 |
signals_plot = plot_signals(signals_df, ticker)
|
| 105 |
-
backtest_plot = plot_backtest(
|
| 106 |
-
|
|
|
|
|
|
|
| 107 |
future_table = pd.DataFrame({
|
| 108 |
-
|
| 109 |
-
|
| 110 |
})
|
| 111 |
-
signals_table = signals_df.reset_index()[[
|
| 112 |
r2_plot = plot_metrics_r2(result)
|
| 113 |
error_plot = plot_metrics_errors(result)
|
| 114 |
precision_recall_plot = plot_metrics_precision_recall(result)
|
|
@@ -118,15 +120,18 @@ def run_dashboard(data_src, ticker, file_upload, timeframe, start_date, end_date
|
|
| 118 |
|
| 119 |
signals_csv = f"signals_{ticker}.csv"
|
| 120 |
signals_df.to_csv(signals_csv)
|
|
|
|
|
|
|
|
|
|
| 121 |
predictions_csv = f"predictions_{ticker}.csv"
|
| 122 |
pd.DataFrame({
|
| 123 |
-
|
| 124 |
-
|
| 125 |
}).to_csv(predictions_csv)
|
| 126 |
chart_png = f"chart_{ticker}.png"
|
| 127 |
-
pio.write(chart_plot, chart_png, format='png')
|
| 128 |
|
| 129 |
-
with open(log_path, 'r') as log_file:
|
| 130 |
log_output = log_file.read()
|
| 131 |
|
| 132 |
logging.info("Dashboard run completed successfully")
|
|
@@ -228,4 +233,5 @@ def main_interface():
|
|
| 228 |
return app
|
| 229 |
|
| 230 |
if __name__ == "__main__":
|
| 231 |
-
main_interface().launch(server_name="0.0.0.0", server_port=7860, share=False)
|
|
|
|
|
|
| 21 |
logging.FileHandler(log_path),
|
| 22 |
logging.StreamHandler()
|
| 23 |
],
|
| 24 |
+
format=\'%(asctime)s - %(levelname)s - %(message)s\'
|
| 25 |
)
|
| 26 |
analyzer = SentimentIntensityAnalyzer()
|
| 27 |
|
|
|
|
| 33 |
start = pd.to_datetime(start_date)
|
| 34 |
end = pd.to_datetime(end_date)
|
| 35 |
articles = newsapi.get_everything(
|
| 36 |
+
q=ticker, from_param=start.strftime(\"%Y-%m-%d\"), to=end.strftime(\"%Y-%m-%d\"),
|
| 37 |
+
language=\'en\', sort_by=\'relevancy\'
|
| 38 |
)
|
| 39 |
+
sentiments = [analyzer.polarity_scores(article[\"title\"])[\"compound\"] for article in articles[\"articles\"]]
|
| 40 |
avg_sentiment = np.mean(sentiments) if sentiments else 0.0
|
| 41 |
sentiment_text = f"Average sentiment for {ticker}: {avg_sentiment:.2f}"
|
| 42 |
return sentiment_text, avg_sentiment
|
|
|
|
| 45 |
return f"Sentiment analysis failed: {str(e)}", None
|
| 46 |
|
| 47 |
def update_horizon_label(timeframe):
|
| 48 |
+
units = {\'1m\': \'minutes\', \'5m\': \'minutes\', \'15m\': \'minutes\', \'30m\': \'minutes\',
|
| 49 |
+
\'1h\': \'hours\', \'4h\': \'hours\', \'1d\': \'days\', \'1wk\': \'weeks\'}
|
| 50 |
+
return gr.update(label=f"Horizon ({units.get(timeframe, \'days\')})")
|
| 51 |
|
| 52 |
def run_dashboard(data_src, ticker, file_upload, timeframe, start_date, end_date, horizon, indicators,
|
| 53 |
include_sentiment, news_api_key, alpha_api_key, account_size, risk_percent, model,
|
|
|
|
| 65 |
logging.error("Failed to load data")
|
| 66 |
return [None] * 12 + ["Failed to load data", None, None, None, None, "Failed to load data"]
|
| 67 |
|
| 68 |
+
df, valid_indicators = add_technical_indicators(df, indicators)
|
| 69 |
if include_sentiment and news_api_key:
|
| 70 |
df = add_sentiment(df, ticker, news_api_key, start_date, end_date)
|
| 71 |
sentiment_text, sentiment_score = sentiment_analysis(ticker, start_date, end_date, news_api_key)
|
| 72 |
|
| 73 |
+
features = valid_indicators # Use valid_indicators for features
|
| 74 |
+
target = \'value\'
|
| 75 |
result = get_model(
|
| 76 |
df=df,
|
| 77 |
features=features,
|
|
|
|
| 92 |
importance_threshold=feat_threshold
|
| 93 |
)
|
| 94 |
if isinstance(result, dict) and result.get("error"):
|
| 95 |
+
logging.error(f"Model training failed: {result[\"error\"]}")
|
| 96 |
+
return [None] * 12 + [f"Model training failed: {result[\"error\"]}", None, None, None, None, f"Model training failed: {result[\"error\"]}"]
|
| 97 |
|
| 98 |
+
signals_df, trades_df, equity_df = generate_signals(df, result)
|
| 99 |
if signals_df.empty:
|
| 100 |
logging.error("Failed to generate signals")
|
| 101 |
return [None] * 12 + ["Failed to generate signals", None, None, None, None, "Failed to generate signals"]
|
| 102 |
|
| 103 |
chart_plot = plot_indicators(df, ticker)
|
| 104 |
signals_plot = plot_signals(signals_df, ticker)
|
| 105 |
+
backtest_plot = plot_backtest(equity_df, trades_df, ticker)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
future_plot = plot_future_forecast(df, result, indicators)
|
| 109 |
future_table = pd.DataFrame({
|
| 110 |
+
\"Date\": [df.index[-1] + timedelta(days=i+1) for i in range(horizon)],
|
| 111 |
+
\"Prediction\": result[\"latest_prediction\"]
|
| 112 |
})
|
| 113 |
+
signals_table = signals_df.reset_index()[[\"Date\", \"Price\", \"Signal\", \"Position_Size\", \"Stop_Loss\", \"Take_Profit\", \"Equity\"]]
|
| 114 |
r2_plot = plot_metrics_r2(result)
|
| 115 |
error_plot = plot_metrics_errors(result)
|
| 116 |
precision_recall_plot = plot_metrics_precision_recall(result)
|
|
|
|
| 120 |
|
| 121 |
signals_csv = f"signals_{ticker}.csv"
|
| 122 |
signals_df.to_csv(signals_csv)
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
|
| 126 |
predictions_csv = f"predictions_{ticker}.csv"
|
| 127 |
pd.DataFrame({
|
| 128 |
+
\"Actual\": result[\"actual\"],
|
| 129 |
+
\"Forecast\": result[\"forecast\"]
|
| 130 |
}).to_csv(predictions_csv)
|
| 131 |
chart_png = f"chart_{ticker}.png"
|
| 132 |
+
pio.write(chart_plot, chart_png, format=\'png\')
|
| 133 |
|
| 134 |
+
with open(log_path, \'r\') as log_file:
|
| 135 |
log_output = log_file.read()
|
| 136 |
|
| 137 |
logging.info("Dashboard run completed successfully")
|
|
|
|
| 233 |
return app
|
| 234 |
|
| 235 |
if __name__ == "__main__":
|
| 236 |
+
main_interface().launch(server_name="0.0.0.0", server_port=7860, share=False)
|
| 237 |
+
|
core/__pycache__/__init__.cpython-311.pyc
ADDED
|
Binary file (158 Bytes). View file
|
|
|
core/__pycache__/data.cpython-311.pyc
ADDED
|
Binary file (23 kB). View file
|
|
|
core/__pycache__/model_runner.cpython-311.pyc
ADDED
|
Binary file (4.06 kB). View file
|
|
|
core/__pycache__/models.cpython-311.pyc
ADDED
|
Binary file (17.1 kB). View file
|
|
|
core/__pycache__/plot.cpython-311.pyc
ADDED
|
Binary file (24.4 kB). View file
|
|
|
core/__pycache__/signals.cpython-311.pyc
ADDED
|
Binary file (10.9 kB). View file
|
|
|
core/__pycache__/train_eval.cpython-311.pyc
ADDED
|
Binary file (23.6 kB). View file
|
|
|
core/data.py
CHANGED
|
@@ -8,45 +8,46 @@ except ImportError:
|
|
| 8 |
ta = None
|
| 9 |
from datetime import datetime, timedelta
|
| 10 |
from newsapi import NewsApiClient
|
| 11 |
-
from textblob import TextBlob
|
| 12 |
-
import pickle
|
| 13 |
-
from requests.exceptions import HTTPError, ConnectionError, Timeout
|
| 14 |
-
from alpha_vantage.timeseries import TimeSeries
|
| 15 |
-
import time
|
| 16 |
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
|
| 17 |
from sklearn.preprocessing import MinMaxScaler
|
| 18 |
-
from
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
|
| 24 |
analyzer = SentimentIntensityAnalyzer()
|
| 25 |
|
| 26 |
-
def load_data(data_src=
|
| 27 |
try:
|
| 28 |
-
# Log inputs to console for visibility in Hugging Face Spaces
|
| 29 |
print_log(f"Loading data: source={data_src}, ticker={ticker}, start={start}, end={end}, interval={interval}, file_upload={'set' if file_upload else 'unset'}, alpha_api_key={'set' if alpha_api_key else 'unset'}")
|
| 30 |
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
# No caching for Hugging Face Spaces compatibility
|
| 44 |
-
if data_src == "csv" and file_upload:
|
| 45 |
try:
|
| 46 |
file_path = getattr(file_upload, 'name', file_upload)
|
| 47 |
print_log(f"Loading CSV from {file_path}")
|
| 48 |
df = pd.read_csv(file_path)
|
| 49 |
-
print_log(f"CSV columns: {df.columns.tolist()}")
|
| 50 |
if 'Date' not in df.columns:
|
| 51 |
raise ValueError("CSV must contain a 'Date' column")
|
| 52 |
df['Date'] = pd.to_datetime(df['Date']).dt.tz_localize(None)
|
|
@@ -60,69 +61,63 @@ def load_data(data_src="yahoo", ticker="AAPL", start="2020-01-01", end="2023-01-
|
|
| 60 |
if df['value'].isna().all():
|
| 61 |
raise ValueError(f"CSV 'value' column contains only NaNs for {ticker}")
|
| 62 |
except Exception as e:
|
| 63 |
-
print_log(f"Failed to load CSV {file_path}: {str(e)}",
|
| 64 |
raise ValueError(f"Failed to load CSV: {str(e)}")
|
| 65 |
else:
|
| 66 |
print_log(f"Fetching data for {ticker} from Yahoo Finance")
|
| 67 |
try:
|
| 68 |
-
df = yf.download(ticker, start=
|
| 69 |
-
print_log(f"Yahoo Finance raw columns type: {type(df.columns)}")
|
| 70 |
-
print_log(f"Yahoo Finance raw columns: {df.columns.tolist() if not isinstance(df.columns, pd.MultiIndex) else df.columns.tolist()}")
|
| 71 |
-
|
| 72 |
-
# Fix for MultiIndex columns (common for crypto/forex like BTC-USD in recent yfinance versions)
|
| 73 |
if isinstance(df.columns, pd.MultiIndex):
|
| 74 |
-
|
| 75 |
-
df.columns = df.columns.droplevel(1) # Drop the ticker level (e.g., 'BTC-USD'), keep fields like 'Close'
|
| 76 |
-
|
| 77 |
-
print_log(f"Yahoo Finance processed columns: {df.columns.tolist()}")
|
| 78 |
if df.empty:
|
| 79 |
raise ValueError(f"No data returned from Yahoo Finance for {ticker}")
|
| 80 |
if 'Close' not in df.columns:
|
| 81 |
raise ValueError(f"Yahoo Finance data missing 'Close' column for {ticker}")
|
| 82 |
df = df.rename(columns={'Close': 'value'})
|
| 83 |
-
|
| 84 |
-
df = df.reset_index(drop=True)
|
| 85 |
if df['value'].isna().all():
|
| 86 |
raise ValueError(f"Yahoo Finance 'value' column contains only NaNs for {ticker}")
|
| 87 |
if df['value'].empty:
|
| 88 |
raise ValueError(f"Yahoo Finance 'value' column is empty for {ticker}")
|
| 89 |
except Exception as e:
|
| 90 |
-
print_log(f"Yahoo Finance failed for {ticker}: {str(e)}",
|
| 91 |
raise ValueError(f"Yahoo Finance failed for {ticker}: {str(e)}")
|
| 92 |
|
| 93 |
-
# Alpha Vantage
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
|
|
|
| 113 |
|
| 114 |
if df.empty:
|
| 115 |
raise ValueError(f"No data loaded for {ticker} from {data_src}")
|
| 116 |
|
| 117 |
-
# Ensure
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
|
|
|
|
|
|
| 124 |
|
| 125 |
-
# Validate 'value' column
|
| 126 |
if 'value' not in df.columns:
|
| 127 |
raise ValueError(f"Target column 'value' is missing for {ticker}")
|
| 128 |
if df['value'].isna().all():
|
|
@@ -130,30 +125,34 @@ def load_data(data_src="yahoo", ticker="AAPL", start="2020-01-01", end="2023-01-
|
|
| 130 |
if df['value'].empty:
|
| 131 |
raise ValueError(f"Target column 'value' is empty for {ticker}")
|
| 132 |
|
| 133 |
-
print_log(f"Data loaded for {ticker} with date range: {df
|
| 134 |
-
print_log(f"DataFrame columns: {df.columns.tolist()}")
|
| 135 |
return df
|
| 136 |
except Exception as e:
|
| 137 |
-
print_log(f"Error in load_data for {ticker}: {str(e)}",
|
| 138 |
raise ValueError(f"Failed to load data for {ticker}: {str(e)}")
|
| 139 |
|
| 140 |
def add_technical_indicators(df, selected_indicators):
|
| 141 |
try:
|
| 142 |
print_log(f"Starting add_technical_indicators with indicators: {selected_indicators}")
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 150 |
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
if missing_cols:
|
| 155 |
-
print_log(f"Missing or entirely NaN columns: {missing_cols}. Cannot compute indicators.", "ERROR")
|
| 156 |
-
return df
|
| 157 |
|
| 158 |
close = df['value'].values
|
| 159 |
high = df['High'].values
|
|
@@ -161,7 +160,6 @@ def add_technical_indicators(df, selected_indicators):
|
|
| 161 |
volume = df['Volume'].values
|
| 162 |
open_ = df['Open'].values
|
| 163 |
|
| 164 |
-
# Updated indicator_map with 'inputs' key to specify required arrays (e.g., ['close'] or ['high', 'low', 'close'])
|
| 165 |
indicator_map = {
|
| 166 |
'rsi': {'func': ta.RSI, 'inputs': ['close'], 'params': {'timeperiod': 14}, 'output': ['rsi_14']},
|
| 167 |
'macd': {'func': ta.MACD, 'inputs': ['close'], 'params': {'fastperiod': 12, 'slowperiod': 26, 'signalperiod': 9}, 'output': ['macd_12_26_9', 'macds_12_26_9', 'macdh_12_26_9']},
|
|
@@ -188,7 +186,6 @@ def add_technical_indicators(df, selected_indicators):
|
|
| 188 |
inputs = config['inputs']
|
| 189 |
params = config['params']
|
| 190 |
try:
|
| 191 |
-
# Get input arrays based on 'inputs' key
|
| 192 |
input_arrays = [input_dict[inp] for inp in inputs]
|
| 193 |
result = func(*input_arrays, **params)
|
| 194 |
if isinstance(result, tuple):
|
|
@@ -196,58 +193,30 @@ def add_technical_indicators(df, selected_indicators):
|
|
| 196 |
if isinstance(res, np.ndarray) and len(res) == len(df):
|
| 197 |
df[out_col] = res
|
| 198 |
nan_count = np.isnan(res).sum()
|
| 199 |
-
print_log(f"Added {out_col}: sample value {res[0] if len(res) > 0 else 'empty'}, NaN count: {nan_count}")
|
| 200 |
if nan_count < len(res) * 0.5:
|
| 201 |
valid_indicators.append(out_col)
|
| 202 |
else:
|
| 203 |
-
print_log(f"{out_col} has excessive NaNs: {nan_count}/{len(res)}. Excluding from valid indicators.",
|
| 204 |
else:
|
| 205 |
-
print_log(f"Invalid output for {out_col}: {type(res)}, length: {len(res) if hasattr(res, '__len__') else 'N/A'}",
|
| 206 |
else:
|
| 207 |
if isinstance(result, np.ndarray) and len(result) == len(df):
|
| 208 |
df[config['output'][0]] = result
|
| 209 |
nan_count = np.isnan(result).sum()
|
| 210 |
-
print_log(f"Added {config['output'][0]}: sample value {result[0] if len(result) > 0 else 'empty'}, NaN count: {nan_count}")
|
| 211 |
if nan_count < len(result) * 0.5:
|
| 212 |
valid_indicators.append(config['output'][0])
|
| 213 |
else:
|
| 214 |
-
print_log(f"{config['output'][0]} has excessive NaNs: {nan_count}/{len(result)}. Excluding from valid indicators.",
|
| 215 |
else:
|
| 216 |
-
print_log(f"Invalid output for {ind}: {type(result)}, length: {len(result) if hasattr(result, '__len__') else 'N/A'}",
|
| 217 |
except Exception as e:
|
| 218 |
-
print_log(f"Error computing {ind}: {str(e)}",
|
| 219 |
else:
|
| 220 |
-
print_log(f"Indicator {ind} not supported by TA-Lib",
|
| 221 |
-
|
| 222 |
-
# Ensure core indicators as fallback
|
| 223 |
-
core_indicators = ['rsi_14', 'macd_12_26_9', 'macds_12_26_9', 'macdh_12_26_9', 'adx_14', 'pdi_14', 'mdi_14', 'willr_14', 'cci_20', 'atr_14', 'stochk_14_3_3', 'stochd_14_3_3']
|
| 224 |
-
for ind, config in indicator_map.items():
|
| 225 |
-
for out_col in config['output']:
|
| 226 |
-
if out_col not in df.columns and out_col not in valid_indicators:
|
| 227 |
-
try:
|
| 228 |
-
input_arrays = [input_dict[inp] for inp in config['inputs']]
|
| 229 |
-
result = config['func'](*input_arrays, **config['params'])
|
| 230 |
-
if isinstance(result, tuple):
|
| 231 |
-
for j, (res, col) in enumerate(zip(result, config['output'])):
|
| 232 |
-
if col not in df.columns and isinstance(res, np.ndarray) and len(res) == len(df):
|
| 233 |
-
df[col] = res
|
| 234 |
-
nan_count = np.isnan(res).sum()
|
| 235 |
-
print_log(f"Added fallback {col}: sample value {res[0] if len(res) > 0 else 'empty'}, NaN count: {nan_count}")
|
| 236 |
-
if nan_count < len(res) * 0.5:
|
| 237 |
-
valid_indicators.append(col)
|
| 238 |
-
else:
|
| 239 |
-
if out_col not in df.columns and isinstance(result, np.ndarray) and len(result) == len(df):
|
| 240 |
-
df[out_col] = result
|
| 241 |
-
nan_count = np.isnan(result).sum()
|
| 242 |
-
print_log(f"Added fallback {out_col}: sample value {result[0] if len(result) > 0 else 'empty'}, NaN count: {nan_count}")
|
| 243 |
-
if nan_count < len(result) * 0.5:
|
| 244 |
-
valid_indicators.append(out_col)
|
| 245 |
-
except Exception as e:
|
| 246 |
-
print_log(f"Error adding fallback {out_col}: {str(e)}", "ERROR")
|
| 247 |
|
| 248 |
# Drop rows with NaN in 'value', preserve valid indicators
|
| 249 |
initial_rows = len(df)
|
| 250 |
-
df = df.dropna(subset=['value']).reset_index(drop=
|
| 251 |
print_log(f"Dropped {initial_rows - len(df)} rows with NaN in 'value'")
|
| 252 |
|
| 253 |
# Drop columns with excessive NaNs, but protect 'value'
|
|
@@ -255,97 +224,106 @@ def add_technical_indicators(df, selected_indicators):
|
|
| 255 |
if col not in ['Date', 'Open', 'High', 'Low', 'value', 'Volume']:
|
| 256 |
nan_ratio = df[col].isna().mean()
|
| 257 |
if nan_ratio > 0.5:
|
| 258 |
-
print_log(f"Dropping {col} due to excessive NaNs: {nan_ratio:.2%}",
|
| 259 |
df = df.drop(columns=[col])
|
| 260 |
if col in valid_indicators:
|
| 261 |
valid_indicators.remove(col)
|
| 262 |
|
| 263 |
-
# Ensure at least one valid feature
|
| 264 |
if not valid_indicators:
|
| 265 |
print_log("No valid indicators computed. Falling back to 'value'.", "WARNING")
|
| 266 |
valid_indicators.append('value')
|
| 267 |
|
| 268 |
print_log(f"Valid indicators: {valid_indicators}")
|
| 269 |
-
print_log(f"DataFrame columns after: {df.columns.tolist()}")
|
| 270 |
-
print_log(f"Sample data after: {df.head().to_dict()}")
|
| 271 |
print_log(f"Technical indicators added successfully, shape: {df.shape}")
|
| 272 |
-
|
|
|
|
| 273 |
except Exception as e:
|
| 274 |
-
print_log(f"Error in add_technical_indicators: {str(e)}",
|
| 275 |
-
return
|
|
|
|
| 276 |
|
| 277 |
def add_sentiment(df, ticker, news_api_key, start_date, end_date):
|
| 278 |
try:
|
| 279 |
-
print_log(f"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 280 |
newsapi = NewsApiClient(api_key=news_api_key)
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 297 |
return df
|
| 298 |
except Exception as e:
|
| 299 |
-
print_log(f"Error in add_sentiment: {str(e)}",
|
|
|
|
| 300 |
return df
|
| 301 |
|
| 302 |
-
def preprocess_data(df, features, target, window_size
|
| 303 |
try:
|
| 304 |
-
print_log("Starting
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
if target not in df.columns:
|
| 310 |
-
raise ValueError(f"Target column '{target}' not found in DataFrame")
|
| 311 |
|
| 312 |
-
|
| 313 |
-
if
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
raise ValueError(f"Target column '{target}' contains only NaNs")
|
| 317 |
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
print_log(f"No valid features found in {features}. Falling back to ['value']", "WARNING")
|
| 322 |
-
feature_cols = ['value'] if 'value' in available_columns else []
|
| 323 |
-
|
| 324 |
-
if not feature_cols:
|
| 325 |
-
raise ValueError("No valid features available, including fallback 'value'")
|
| 326 |
-
|
| 327 |
-
feature_data = df[feature_cols].values.astype(float)
|
| 328 |
-
print_log(f"Feature columns: {feature_cols}, feature_data shape: {feature_data.shape}")
|
| 329 |
|
| 330 |
-
if
|
| 331 |
-
raise ValueError(
|
| 332 |
|
| 333 |
feature_scaler = MinMaxScaler()
|
| 334 |
-
scaled_features = feature_scaler.fit_transform(feature_data)
|
| 335 |
target_scaler = MinMaxScaler()
|
| 336 |
-
scaled_target = target_scaler.fit_transform(target_series.values.reshape(-1, 1)).flatten()
|
| 337 |
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
if len(feature_cols) > 10:
|
| 341 |
-
print_log("Applying PCA due to high feature count")
|
| 342 |
-
pca = PCA(n_components=min(10, len(feature_cols)))
|
| 343 |
-
scaled_features = pca.fit_transform(scaled_features)
|
| 344 |
-
updated_feature_cols = [f'pca_{i}' for i in range(min(10, len(feature_cols)))]
|
| 345 |
-
print_log(f"PCA applied: new feature count: {len(updated_feature_cols)}")
|
| 346 |
|
| 347 |
-
|
| 348 |
-
full_scaled = np.column_stack((scaled_features, scaled_target))
|
| 349 |
target_idx = len(updated_feature_cols)
|
| 350 |
|
| 351 |
X, y = [], []
|
|
@@ -360,7 +338,8 @@ def preprocess_data(df, features, target, window_size=30, horizon=1):
|
|
| 360 |
raise ValueError(f"Insufficient data after preprocessing. Data length: {len(full_scaled)}, window_size: {window_size}, horizon: {horizon}")
|
| 361 |
|
| 362 |
print_log(f"Preprocessed data: X.shape={X.shape}, y.shape={y.shape}, Final features: {full_features}, Target idx: {target_idx}")
|
| 363 |
-
return X, y, feature_scaler, target_scaler, full_features, target_idx,
|
| 364 |
except Exception as e:
|
| 365 |
-
print_log(f"Preprocessing error: {str(e)}",
|
| 366 |
-
raise
|
|
|
|
|
|
| 8 |
ta = None
|
| 9 |
from datetime import datetime, timedelta
|
| 10 |
from newsapi import NewsApiClient
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
|
| 12 |
from sklearn.preprocessing import MinMaxScaler
|
| 13 |
+
from alpha_vantage.timeseries import TimeSeries
|
| 14 |
+
import time
|
| 15 |
+
import logging
|
| 16 |
+
|
| 17 |
+
# Configure logging
|
| 18 |
+
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
| 19 |
|
| 20 |
+
def print_log(message, level='INFO'):
|
| 21 |
+
if level == 'INFO':
|
| 22 |
+
logging.info(message)
|
| 23 |
+
elif level == 'WARNING':
|
| 24 |
+
logging.warning(message)
|
| 25 |
+
elif level == 'ERROR':
|
| 26 |
+
logging.error(message)
|
| 27 |
+
else:
|
| 28 |
+
logging.debug(message)
|
| 29 |
|
| 30 |
analyzer = SentimentIntensityAnalyzer()
|
| 31 |
|
| 32 |
+
def load_data(data_src='yahoo', ticker='AAPL', start='2020-01-01', end='2023-01-01', interval='1d', file_upload=None, alpha_api_key=None):
|
| 33 |
try:
|
|
|
|
| 34 |
print_log(f"Loading data: source={data_src}, ticker={ticker}, start={start}, end={end}, interval={interval}, file_upload={'set' if file_upload else 'unset'}, alpha_api_key={'set' if alpha_api_key else 'unset'}")
|
| 35 |
|
| 36 |
+
start_date = pd.to_datetime(start)
|
| 37 |
+
end_date = pd.to_datetime(end)
|
| 38 |
+
if start_date >= end_date:
|
| 39 |
+
raise ValueError(f"Start date {start} must be before end date {end}")
|
| 40 |
+
if end_date > datetime.now():
|
| 41 |
+
print_log(f"End date {end} is in the future. Using current date as end date.", 'WARNING')
|
| 42 |
+
end_date = datetime.now()
|
| 43 |
+
|
| 44 |
+
df = pd.DataFrame()
|
| 45 |
+
|
| 46 |
+
if data_src == 'csv' and file_upload:
|
|
|
|
|
|
|
|
|
|
| 47 |
try:
|
| 48 |
file_path = getattr(file_upload, 'name', file_upload)
|
| 49 |
print_log(f"Loading CSV from {file_path}")
|
| 50 |
df = pd.read_csv(file_path)
|
|
|
|
| 51 |
if 'Date' not in df.columns:
|
| 52 |
raise ValueError("CSV must contain a 'Date' column")
|
| 53 |
df['Date'] = pd.to_datetime(df['Date']).dt.tz_localize(None)
|
|
|
|
| 61 |
if df['value'].isna().all():
|
| 62 |
raise ValueError(f"CSV 'value' column contains only NaNs for {ticker}")
|
| 63 |
except Exception as e:
|
| 64 |
+
print_log(f"Failed to load CSV {file_path}: {str(e)}", 'ERROR')
|
| 65 |
raise ValueError(f"Failed to load CSV: {str(e)}")
|
| 66 |
else:
|
| 67 |
print_log(f"Fetching data for {ticker} from Yahoo Finance")
|
| 68 |
try:
|
| 69 |
+
df = yf.download(ticker, start=start_date, end=end_date, interval=interval, progress=False, auto_adjust=False)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
if isinstance(df.columns, pd.MultiIndex):
|
| 71 |
+
df.columns = df.columns.droplevel(1)
|
|
|
|
|
|
|
|
|
|
| 72 |
if df.empty:
|
| 73 |
raise ValueError(f"No data returned from Yahoo Finance for {ticker}")
|
| 74 |
if 'Close' not in df.columns:
|
| 75 |
raise ValueError(f"Yahoo Finance data missing 'Close' column for {ticker}")
|
| 76 |
df = df.rename(columns={'Close': 'value'})
|
| 77 |
+
# The index is already datetime, no need to create a 'Date' column and then reset
|
|
|
|
| 78 |
if df['value'].isna().all():
|
| 79 |
raise ValueError(f"Yahoo Finance 'value' column contains only NaNs for {ticker}")
|
| 80 |
if df['value'].empty:
|
| 81 |
raise ValueError(f"Yahoo Finance 'value' column is empty for {ticker}")
|
| 82 |
except Exception as e:
|
| 83 |
+
print_log(f"Yahoo Finance failed for {ticker}: {str(e)}", 'ERROR')
|
| 84 |
raise ValueError(f"Yahoo Finance failed for {ticker}: {str(e)}")
|
| 85 |
|
| 86 |
+
# Alpha Vantage data loading (if applicable)
|
| 87 |
+
# Note: Alpha Vantage data loading logic is commented out for now to simplify debugging
|
| 88 |
+
# if interval in ['1m', '5m', '15m', '30m', '60m'] and alpha_api_key:
|
| 89 |
+
# print_log(f"Attempting Alpha Vantage for {ticker}, interval {interval}")
|
| 90 |
+
# try:
|
| 91 |
+
# ts = TimeSeries(key=alpha_api_key, output_format='pandas')
|
| 92 |
+
# df_av, _ = ts.get_intraday(symbol=ticker, interval=interval, outputsize='full')
|
| 93 |
+
# if df_av.empty:
|
| 94 |
+
# raise ValueError(f"No data returned from Alpha Vantage for {ticker}")
|
| 95 |
+
# if '4. close' not in df_av.columns:
|
| 96 |
+
# raise ValueError(f"Alpha Vantage data missing '4. close' column for {ticker}")
|
| 97 |
+
# df_av = df_av.rename(columns={'4. close': 'value', '1. open': 'Open', '2. high': 'High', '3. low': 'Low', '5. volume': 'Volume'}) # Standardize column names
|
| 98 |
+
# df_av['Date'] = pd.to_datetime(df_av.index)
|
| 99 |
+
# df_av = df_av.reset_index(drop=True)
|
| 100 |
+
# if df_av['value'].isna().all():
|
| 101 |
+
# raise ValueError(f"Alpha Vantage 'value' column contains only NaNs for {ticker}")
|
| 102 |
+
# if df_av['value'].empty:
|
| 103 |
+
# raise ValueError(f"Alpha Vantage 'value' column is empty for {ticker}")
|
| 104 |
+
# df = df_av # Use Alpha Vantage data if successful
|
| 105 |
+
# except Exception as e:
|
| 106 |
+
# print_log(f"Alpha Vantage failed: {str(e)}, using Yahoo Finance data", 'WARNING')
|
| 107 |
|
| 108 |
if df.empty:
|
| 109 |
raise ValueError(f"No data loaded for {ticker} from {data_src}")
|
| 110 |
|
| 111 |
+
# Ensure index is DatetimeIndex and sorted
|
| 112 |
+
if not isinstance(df.index, pd.DatetimeIndex):
|
| 113 |
+
df.index = pd.to_datetime(df.index)
|
| 114 |
+
df = df.sort_index()
|
| 115 |
+
|
| 116 |
+
required_cols = ['Open', 'High', 'Low', 'value', 'Volume']
|
| 117 |
+
for col in required_cols:
|
| 118 |
+
if col not in df.columns:
|
| 119 |
+
df[col] = np.nan # Add missing columns with NaNs
|
| 120 |
|
|
|
|
| 121 |
if 'value' not in df.columns:
|
| 122 |
raise ValueError(f"Target column 'value' is missing for {ticker}")
|
| 123 |
if df['value'].isna().all():
|
|
|
|
| 125 |
if df['value'].empty:
|
| 126 |
raise ValueError(f"Target column 'value' is empty for {ticker}")
|
| 127 |
|
| 128 |
+
print_log(f"Data loaded for {ticker} with date range: {df.index.min()} to {df.index.max()}, shape: {df.shape}")
|
|
|
|
| 129 |
return df
|
| 130 |
except Exception as e:
|
| 131 |
+
print_log(f"Error in load_data for {ticker}: {str(e)}", 'ERROR')
|
| 132 |
raise ValueError(f"Failed to load data for {ticker}: {str(e)}")
|
| 133 |
|
| 134 |
def add_technical_indicators(df, selected_indicators):
|
| 135 |
try:
|
| 136 |
print_log(f"Starting add_technical_indicators with indicators: {selected_indicators}")
|
| 137 |
+
if df.empty:
|
| 138 |
+
print_log("DataFrame is empty, skipping technical indicator calculation.", "WARNING")
|
| 139 |
+
return df, []
|
| 140 |
|
| 141 |
+
# Ensure columns are numeric and handle missing ones
|
| 142 |
+
for col in ['Open', 'High', 'Low', 'value', 'Volume']:
|
| 143 |
+
if col not in df.columns:
|
| 144 |
+
df[col] = np.nan
|
| 145 |
+
df[col] = pd.to_numeric(df[col], errors='coerce')
|
| 146 |
+
|
| 147 |
+
# Drop rows with NaN in core columns after indicator calculation
|
| 148 |
+
df.dropna(subset=['Open', 'High', 'Low', 'value', 'Volume'], inplace=True)
|
| 149 |
+
if df.empty:
|
| 150 |
+
print_log("DataFrame is empty after dropping NaNs for technical indicators.", "WARNING")
|
| 151 |
+
return df, []
|
| 152 |
|
| 153 |
+
if ta is None:
|
| 154 |
+
print_log("TA-Lib not available. Cannot compute indicators. Falling back to 'value'.", 'ERROR')
|
| 155 |
+
return df, []
|
|
|
|
|
|
|
|
|
|
| 156 |
|
| 157 |
close = df['value'].values
|
| 158 |
high = df['High'].values
|
|
|
|
| 160 |
volume = df['Volume'].values
|
| 161 |
open_ = df['Open'].values
|
| 162 |
|
|
|
|
| 163 |
indicator_map = {
|
| 164 |
'rsi': {'func': ta.RSI, 'inputs': ['close'], 'params': {'timeperiod': 14}, 'output': ['rsi_14']},
|
| 165 |
'macd': {'func': ta.MACD, 'inputs': ['close'], 'params': {'fastperiod': 12, 'slowperiod': 26, 'signalperiod': 9}, 'output': ['macd_12_26_9', 'macds_12_26_9', 'macdh_12_26_9']},
|
|
|
|
| 186 |
inputs = config['inputs']
|
| 187 |
params = config['params']
|
| 188 |
try:
|
|
|
|
| 189 |
input_arrays = [input_dict[inp] for inp in inputs]
|
| 190 |
result = func(*input_arrays, **params)
|
| 191 |
if isinstance(result, tuple):
|
|
|
|
| 193 |
if isinstance(res, np.ndarray) and len(res) == len(df):
|
| 194 |
df[out_col] = res
|
| 195 |
nan_count = np.isnan(res).sum()
|
|
|
|
| 196 |
if nan_count < len(res) * 0.5:
|
| 197 |
valid_indicators.append(out_col)
|
| 198 |
else:
|
| 199 |
+
print_log(f"{out_col} has excessive NaNs: {nan_count}/{len(res)}. Excluding from valid indicators.", 'WARNING')
|
| 200 |
else:
|
| 201 |
+
print_log(f"Invalid output for {out_col}: {type(res)}, length: {len(res) if hasattr(res, '__len__') else 'N/A'}", 'WARNING')
|
| 202 |
else:
|
| 203 |
if isinstance(result, np.ndarray) and len(result) == len(df):
|
| 204 |
df[config['output'][0]] = result
|
| 205 |
nan_count = np.isnan(result).sum()
|
|
|
|
| 206 |
if nan_count < len(result) * 0.5:
|
| 207 |
valid_indicators.append(config['output'][0])
|
| 208 |
else:
|
| 209 |
+
print_log(f"{config['output'][0]} has excessive NaNs: {nan_count}/{len(result)}. Excluding from valid indicators.", 'WARNING')
|
| 210 |
else:
|
| 211 |
+
print_log(f"Invalid output for {ind}: {type(result)}, length: {len(result) if hasattr(result, '__len__') else 'N/A'}", 'WARNING')
|
| 212 |
except Exception as e:
|
| 213 |
+
print_log(f"Error computing {ind}: {str(e)}", 'ERROR')
|
| 214 |
else:
|
| 215 |
+
print_log(f"Indicator {ind} not supported by TA-Lib", 'WARNING')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 216 |
|
| 217 |
# Drop rows with NaN in 'value', preserve valid indicators
|
| 218 |
initial_rows = len(df)
|
| 219 |
+
df = df.dropna(subset=['value']).reset_index(drop=False) # Keep index as a column for now
|
| 220 |
print_log(f"Dropped {initial_rows - len(df)} rows with NaN in 'value'")
|
| 221 |
|
| 222 |
# Drop columns with excessive NaNs, but protect 'value'
|
|
|
|
| 224 |
if col not in ['Date', 'Open', 'High', 'Low', 'value', 'Volume']:
|
| 225 |
nan_ratio = df[col].isna().mean()
|
| 226 |
if nan_ratio > 0.5:
|
| 227 |
+
print_log(f"Dropping {col} due to excessive NaNs: {nan_ratio:.2%}", 'WARNING')
|
| 228 |
df = df.drop(columns=[col])
|
| 229 |
if col in valid_indicators:
|
| 230 |
valid_indicators.remove(col)
|
| 231 |
|
|
|
|
| 232 |
if not valid_indicators:
|
| 233 |
print_log("No valid indicators computed. Falling back to 'value'.", "WARNING")
|
| 234 |
valid_indicators.append('value')
|
| 235 |
|
| 236 |
print_log(f"Valid indicators: {valid_indicators}")
|
|
|
|
|
|
|
| 237 |
print_log(f"Technical indicators added successfully, shape: {df.shape}")
|
| 238 |
+
df.set_index('Date', inplace=True) # Set index back to Date after all processing
|
| 239 |
+
return df, valid_indicators
|
| 240 |
except Exception as e:
|
| 241 |
+
print_log(f"Error in add_technical_indicators: {str(e)}", 'ERROR')
|
| 242 |
+
# If an error occurs, return the original DataFrame to prevent further errors
|
| 243 |
+
return df, []
|
| 244 |
|
| 245 |
def add_sentiment(df, ticker, news_api_key, start_date, end_date):
|
| 246 |
try:
|
| 247 |
+
print_log(f"Starting add_sentiment for {ticker} from {start_date} to {end_date}")
|
| 248 |
+
if not news_api_key:
|
| 249 |
+
print_log("News API key not provided. Skipping sentiment analysis.", "WARNING")
|
| 250 |
+
df['sentiment_score'] = 0.0
|
| 251 |
+
return df
|
| 252 |
+
|
| 253 |
newsapi = NewsApiClient(api_key=news_api_key)
|
| 254 |
+
all_articles = []
|
| 255 |
+
current_date = pd.to_datetime(start_date)
|
| 256 |
+
end_date = pd.to_datetime(end_date)
|
| 257 |
+
|
| 258 |
+
while current_date <= end_date:
|
| 259 |
+
from_param = current_date.strftime("%Y-%m-%d")
|
| 260 |
+
to_param = (current_date + timedelta(days=1)).strftime("%Y-%m-%d")
|
| 261 |
+
print_log(f"Fetching news for {ticker} from {from_param} to {to_param}")
|
| 262 |
+
try:
|
| 263 |
+
articles = newsapi.get_everything(q=ticker, language='en', sort_by='relevancy', from_param=from_param, to=to_param)
|
| 264 |
+
all_articles.extend(articles["articles"])
|
| 265 |
+
except Exception as e:
|
| 266 |
+
print_log(f"Error fetching news for {ticker} on {from_param}: {str(e)}", 'ERROR')
|
| 267 |
+
current_date += timedelta(days=1)
|
| 268 |
+
time.sleep(0.1)
|
| 269 |
+
|
| 270 |
+
if not all_articles:
|
| 271 |
+
print_log(f"No articles found for {ticker}. Setting sentiment to 0.", 'WARNING')
|
| 272 |
+
df['sentiment_score'] = 0.0
|
| 273 |
+
return df
|
| 274 |
+
|
| 275 |
+
sentiment_data = []
|
| 276 |
+
for article in all_articles:
|
| 277 |
+
if article["publishedAt"] and article["description"]:
|
| 278 |
+
date = pd.to_datetime(article["publishedAt"]).tz_localize(None).date()
|
| 279 |
+
text = article["description"]
|
| 280 |
+
vs = analyzer.polarity_scores(text)
|
| 281 |
+
sentiment_data.append({"Date": date, "sentiment_score": vs["compound"]})
|
| 282 |
+
|
| 283 |
+
sentiment_df = pd.DataFrame(sentiment_data)
|
| 284 |
+
sentiment_df["Date"] = pd.to_datetime(sentiment_df["Date"])
|
| 285 |
+
sentiment_df = sentiment_df.groupby("Date")["sentiment_score"].mean().reset_index()
|
| 286 |
+
|
| 287 |
+
df.reset_index(inplace=True)
|
| 288 |
+
df['Date'] = pd.to_datetime(df['Date'])
|
| 289 |
+
df = pd.merge(df, sentiment_df, on="Date", how="left")
|
| 290 |
+
df['sentiment_score'] = df['sentiment_score'].fillna(0.0)
|
| 291 |
+
df.set_index('Date', inplace=True)
|
| 292 |
+
|
| 293 |
+
print_log(f"Sentiment analysis completed for {ticker}. Added sentiment_score column.")
|
| 294 |
return df
|
| 295 |
except Exception as e:
|
| 296 |
+
print_log(f"Error in add_sentiment for {ticker}: {str(e)}", 'ERROR')
|
| 297 |
+
df['sentiment_score'] = 0.0
|
| 298 |
return df
|
| 299 |
|
| 300 |
+
def preprocess_data(df, features, target, window_size, horizon):
|
| 301 |
try:
|
| 302 |
+
print_log(f"Starting preprocessing: features={features}, target={target}, window={window_size}, horizon={horizon}")
|
| 303 |
+
|
| 304 |
+
# Ensure the DataFrame index is a DatetimeIndex
|
| 305 |
+
if not isinstance(df.index, pd.DatetimeIndex):
|
| 306 |
+
raise ValueError("DataFrame index must be a DatetimeIndex for preprocessing.")
|
|
|
|
|
|
|
| 307 |
|
| 308 |
+
# Filter features to only include those present in the DataFrame columns
|
| 309 |
+
updated_feature_cols = [f for f in features if f in df.columns]
|
| 310 |
+
if not updated_feature_cols:
|
| 311 |
+
raise ValueError("No valid features found in DataFrame after indicator calculation.")
|
|
|
|
| 312 |
|
| 313 |
+
full_features = updated_feature_cols + [target]
|
| 314 |
+
data = df[full_features].copy()
|
| 315 |
+
data.dropna(inplace=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 316 |
|
| 317 |
+
if data.empty:
|
| 318 |
+
raise ValueError("DataFrame is empty after dropping NaNs. Cannot proceed with scaling.")
|
| 319 |
|
| 320 |
feature_scaler = MinMaxScaler()
|
|
|
|
| 321 |
target_scaler = MinMaxScaler()
|
|
|
|
| 322 |
|
| 323 |
+
data_features_scaled = feature_scaler.fit_transform(data[updated_feature_cols])
|
| 324 |
+
data_target_scaled = target_scaler.fit_transform(data[[target]])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 325 |
|
| 326 |
+
full_scaled = np.hstack((data_features_scaled, data_target_scaled))
|
|
|
|
| 327 |
target_idx = len(updated_feature_cols)
|
| 328 |
|
| 329 |
X, y = [], []
|
|
|
|
| 338 |
raise ValueError(f"Insufficient data after preprocessing. Data length: {len(full_scaled)}, window_size: {window_size}, horizon: {horizon}")
|
| 339 |
|
| 340 |
print_log(f"Preprocessed data: X.shape={X.shape}, y.shape={y.shape}, Final features: {full_features}, Target idx: {target_idx}")
|
| 341 |
+
return X, y, feature_scaler, target_scaler, full_features, target_idx, None, updated_feature_cols
|
| 342 |
except Exception as e:
|
| 343 |
+
print_log(f"Preprocessing error: {str(e)}", 'ERROR')
|
| 344 |
+
raise ValueError(f"Preprocessing failed: {str(e)}")
|
| 345 |
+
|
core/models.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
# core/models.py
|
| 2 |
import torch
|
|
|
|
| 3 |
import torch.nn as nn
|
| 4 |
import math
|
| 5 |
|
|
@@ -31,7 +32,9 @@ class LSTMModel(nn.Module):
|
|
| 31 |
self.dropout = nn.Dropout(dropout)
|
| 32 |
|
| 33 |
def forward(self, x):
|
|
|
|
| 34 |
if isinstance(x, (tuple, list)):
|
|
|
|
| 35 |
x = x[0]
|
| 36 |
if not isinstance(x, torch.Tensor):
|
| 37 |
x = torch.tensor(
|
|
@@ -62,7 +65,9 @@ class GRUModel(nn.Module):
|
|
| 62 |
self.dropout = nn.Dropout(dropout)
|
| 63 |
|
| 64 |
def forward(self, x):
|
|
|
|
| 65 |
if isinstance(x, (tuple, list)):
|
|
|
|
| 66 |
x = x[0]
|
| 67 |
if not isinstance(x, torch.Tensor):
|
| 68 |
x = torch.tensor(
|
|
@@ -85,7 +90,9 @@ class CNNModel(nn.Module):
|
|
| 85 |
self.fc = nn.Linear(hidden_size, output_size)
|
| 86 |
|
| 87 |
def forward(self, x):
|
|
|
|
| 88 |
if isinstance(x, (tuple, list)):
|
|
|
|
| 89 |
x = x[0]
|
| 90 |
if not isinstance(x, torch.Tensor):
|
| 91 |
x = torch.tensor(
|
|
@@ -114,7 +121,9 @@ class MLPModel(nn.Module):
|
|
| 114 |
self.mlp = nn.Sequential(*layers)
|
| 115 |
|
| 116 |
def forward(self, x):
|
|
|
|
| 117 |
if isinstance(x, (tuple, list)):
|
|
|
|
| 118 |
x = x[0]
|
| 119 |
if not isinstance(x, torch.Tensor):
|
| 120 |
x = torch.tensor(
|
|
@@ -133,7 +142,9 @@ class HybridCNNGRUModel(nn.Module):
|
|
| 133 |
self.dropout = nn.Dropout(dropout)
|
| 134 |
|
| 135 |
def forward(self, x):
|
|
|
|
| 136 |
if isinstance(x, (tuple, list)):
|
|
|
|
| 137 |
x = x[0]
|
| 138 |
if not isinstance(x, torch.Tensor):
|
| 139 |
x = torch.tensor(
|
|
@@ -160,7 +171,9 @@ class TransformerModel(nn.Module):
|
|
| 160 |
self.fc = nn.Linear(hidden_size, output_size)
|
| 161 |
|
| 162 |
def forward(self, x):
|
|
|
|
| 163 |
if isinstance(x, (tuple, list)):
|
|
|
|
| 164 |
x = x[0]
|
| 165 |
if not isinstance(x, torch.Tensor):
|
| 166 |
x = torch.tensor(
|
|
@@ -190,7 +203,9 @@ class BiLSTMModel(nn.Module):
|
|
| 190 |
self.dropout = nn.Dropout(dropout)
|
| 191 |
|
| 192 |
def forward(self, x):
|
|
|
|
| 193 |
if isinstance(x, (tuple, list)):
|
|
|
|
| 194 |
x = x[0]
|
| 195 |
if not isinstance(x, torch.Tensor):
|
| 196 |
x = torch.tensor(
|
|
|
|
| 1 |
# core/models.py
|
| 2 |
import torch
|
| 3 |
+
import logging
|
| 4 |
import torch.nn as nn
|
| 5 |
import math
|
| 6 |
|
|
|
|
| 32 |
self.dropout = nn.Dropout(dropout)
|
| 33 |
|
| 34 |
def forward(self, x):
|
| 35 |
+
logging.debug(f"Inside forward: initial x type={type(x)}, x={x}")
|
| 36 |
if isinstance(x, (tuple, list)):
|
| 37 |
+
logging.debug(f"Model forward received tuple/list: type={type(x)}, length={len(x)}")
|
| 38 |
x = x[0]
|
| 39 |
if not isinstance(x, torch.Tensor):
|
| 40 |
x = torch.tensor(
|
|
|
|
| 65 |
self.dropout = nn.Dropout(dropout)
|
| 66 |
|
| 67 |
def forward(self, x):
|
| 68 |
+
logging.debug(f"Inside forward: initial x type={type(x)}, x={x}")
|
| 69 |
if isinstance(x, (tuple, list)):
|
| 70 |
+
logging.debug(f"Model forward received tuple/list: type={type(x)}, length={len(x)}")
|
| 71 |
x = x[0]
|
| 72 |
if not isinstance(x, torch.Tensor):
|
| 73 |
x = torch.tensor(
|
|
|
|
| 90 |
self.fc = nn.Linear(hidden_size, output_size)
|
| 91 |
|
| 92 |
def forward(self, x):
|
| 93 |
+
logging.debug(f"Inside forward: initial x type={type(x)}, x={x}")
|
| 94 |
if isinstance(x, (tuple, list)):
|
| 95 |
+
logging.debug(f"Model forward received tuple/list: type={type(x)}, length={len(x)}")
|
| 96 |
x = x[0]
|
| 97 |
if not isinstance(x, torch.Tensor):
|
| 98 |
x = torch.tensor(
|
|
|
|
| 121 |
self.mlp = nn.Sequential(*layers)
|
| 122 |
|
| 123 |
def forward(self, x):
|
| 124 |
+
logging.debug(f"Inside forward: initial x type={type(x)}, x={x}")
|
| 125 |
if isinstance(x, (tuple, list)):
|
| 126 |
+
logging.debug(f"Model forward received tuple/list: type={type(x)}, length={len(x)}")
|
| 127 |
x = x[0]
|
| 128 |
if not isinstance(x, torch.Tensor):
|
| 129 |
x = torch.tensor(
|
|
|
|
| 142 |
self.dropout = nn.Dropout(dropout)
|
| 143 |
|
| 144 |
def forward(self, x):
|
| 145 |
+
logging.debug(f"Inside forward: initial x type={type(x)}, x={x}")
|
| 146 |
if isinstance(x, (tuple, list)):
|
| 147 |
+
logging.debug(f"Model forward received tuple/list: type={type(x)}, length={len(x)}")
|
| 148 |
x = x[0]
|
| 149 |
if not isinstance(x, torch.Tensor):
|
| 150 |
x = torch.tensor(
|
|
|
|
| 171 |
self.fc = nn.Linear(hidden_size, output_size)
|
| 172 |
|
| 173 |
def forward(self, x):
|
| 174 |
+
logging.debug(f"Inside forward: initial x type={type(x)}, x={x}")
|
| 175 |
if isinstance(x, (tuple, list)):
|
| 176 |
+
logging.debug(f"Model forward received tuple/list: type={type(x)}, length={len(x)}")
|
| 177 |
x = x[0]
|
| 178 |
if not isinstance(x, torch.Tensor):
|
| 179 |
x = torch.tensor(
|
|
|
|
| 203 |
self.dropout = nn.Dropout(dropout)
|
| 204 |
|
| 205 |
def forward(self, x):
|
| 206 |
+
logging.debug(f"Inside forward: initial x type={type(x)}, x={x}")
|
| 207 |
if isinstance(x, (tuple, list)):
|
| 208 |
+
logging.debug(f"Model forward received tuple/list: type={type(x)}, length={len(x)}")
|
| 209 |
x = x[0]
|
| 210 |
if not isinstance(x, torch.Tensor):
|
| 211 |
x = torch.tensor(
|
core/plot.py
CHANGED
|
@@ -2,14 +2,9 @@ import plotly.graph_objects as go
|
|
| 2 |
import plotly.express as px
|
| 3 |
import pandas as pd
|
| 4 |
import logging
|
| 5 |
-
|
| 6 |
-
logging.basicConfig(level=logging.DEBUG, filename="debug.log", filemode="a")
|
| 7 |
-
|
| 8 |
-
import plotly.graph_objects as go
|
| 9 |
from plotly.subplots import make_subplots
|
| 10 |
-
import pandas as pd
|
| 11 |
import numpy as np
|
| 12 |
-
import logging
|
| 13 |
|
| 14 |
logging.basicConfig(level=logging.INFO)
|
| 15 |
|
|
@@ -27,55 +22,31 @@ def plot_indicators(df, ticker):
|
|
| 27 |
# Price and Moving Averages
|
| 28 |
fig.add_trace(
|
| 29 |
go.Candlestick(
|
| 30 |
-
x=df
|
| 31 |
name='Price', increasing_line_color='#00CC96', decreasing_line_color='#EF553B'
|
| 32 |
), row=1, col=1
|
| 33 |
)
|
| 34 |
for ma in ['sma_10', 'sma_20', 'sma_50', 'ema_12', 'ema_26', 'ema_50']:
|
| 35 |
if ma in df:
|
| 36 |
fig.add_trace(
|
| 37 |
-
go.Scatter(x=df
|
| 38 |
row=1, col=1
|
| 39 |
)
|
| 40 |
if 'bbu_20_2' in df:
|
| 41 |
fig.add_trace(
|
| 42 |
-
go.Scatter(x=df
|
| 43 |
row=1, col=1
|
| 44 |
)
|
| 45 |
fig.add_trace(
|
| 46 |
-
go.Scatter(x=df
|
| 47 |
row=1, col=1
|
| 48 |
)
|
| 49 |
fig.add_trace(
|
| 50 |
-
go.Scatter(x=df
|
| 51 |
row=1, col=1
|
| 52 |
)
|
| 53 |
|
| 54 |
-
|
| 55 |
-
buy_signals = df[df['Signal'] == 'Buy']
|
| 56 |
-
sell_signals = df[df['Signal'] == 'Sell']
|
| 57 |
-
hold_signals = df[df['Signal'] == 'Hold']
|
| 58 |
-
fig.add_trace(
|
| 59 |
-
go.Scatter(
|
| 60 |
-
x=buy_signals['Date'], y=buy_signals['value'], mode='markers+text',
|
| 61 |
-
name='Buy', marker=dict(symbol='triangle-up', size=12, color='green'),
|
| 62 |
-
text=['Buy'] * len(buy_signals), textposition='top center'
|
| 63 |
-
), row=1, col=1
|
| 64 |
-
)
|
| 65 |
-
fig.add_trace(
|
| 66 |
-
go.Scatter(
|
| 67 |
-
x=sell_signals['Date'], y=sell_signals['value'], mode='markers+text',
|
| 68 |
-
name='Sell', marker=dict(symbol='triangle-down', size=12, color='red'),
|
| 69 |
-
text=['Sell'] * len(sell_signals), textposition='bottom center'
|
| 70 |
-
), row=1, col=1
|
| 71 |
-
)
|
| 72 |
-
fig.add_trace(
|
| 73 |
-
go.Scatter(
|
| 74 |
-
x=hold_signals['Date'], y=hold_signals['value'], mode='markers',
|
| 75 |
-
name='Hold', marker=dict(symbol='circle', size=8, color='gray'),
|
| 76 |
-
opacity=0.5
|
| 77 |
-
), row=1, col=1
|
| 78 |
-
)
|
| 79 |
|
| 80 |
# Position Size and Risk Annotation
|
| 81 |
if 'atr_14' in df:
|
|
@@ -90,57 +61,57 @@ def plot_indicators(df, ticker):
|
|
| 90 |
|
| 91 |
# Volume
|
| 92 |
fig.add_trace(
|
| 93 |
-
go.Bar(x=df
|
| 94 |
row=2, col=1
|
| 95 |
)
|
| 96 |
|
| 97 |
# MACD & RSI
|
| 98 |
if 'macd_12_26_9' in df:
|
| 99 |
fig.add_trace(
|
| 100 |
-
go.Scatter(x=df
|
| 101 |
row=3, col=1
|
| 102 |
)
|
| 103 |
fig.add_trace(
|
| 104 |
-
go.Scatter(x=df
|
| 105 |
row=3, col=1
|
| 106 |
)
|
| 107 |
fig.add_trace(
|
| 108 |
-
go.Bar(x=df
|
| 109 |
row=3, col=1
|
| 110 |
)
|
| 111 |
if 'rsi_14' in df:
|
| 112 |
fig.add_trace(
|
| 113 |
-
go.Scatter(x=df
|
| 114 |
row=3, col=1
|
| 115 |
)
|
| 116 |
fig.add_hline(y=70, line_dash="dash", line_color="red", row=3, col=1)
|
| 117 |
fig.add_hline(y=30, line_dash="dash", line_color="green", row=3, col=1)
|
| 118 |
if 'rsi_21' in df:
|
| 119 |
fig.add_trace(
|
| 120 |
-
go.Scatter(x=df
|
| 121 |
row=3, col=1
|
| 122 |
)
|
| 123 |
if 'rsi_50' in df:
|
| 124 |
fig.add_trace(
|
| 125 |
-
go.Scatter(x=df
|
| 126 |
row=3, col=1
|
| 127 |
)
|
| 128 |
|
| 129 |
# Stochastic & Williams %R
|
| 130 |
if 'stochk_14_3_3' in df:
|
| 131 |
fig.add_trace(
|
| 132 |
-
go.Scatter(x=df
|
| 133 |
row=4, col=1
|
| 134 |
)
|
| 135 |
fig.add_trace(
|
| 136 |
-
go.Scatter(x=df
|
| 137 |
row=4, col=1
|
| 138 |
)
|
| 139 |
fig.add_hline(y=80, line_dash="dash", line_color="red", row=4, col=1)
|
| 140 |
fig.add_hline(y=20, line_dash="dash", line_color="green", row=4, col=1)
|
| 141 |
if 'willr_14' in df:
|
| 142 |
fig.add_trace(
|
| 143 |
-
go.Scatter(x=df
|
| 144 |
row=4, col=1
|
| 145 |
)
|
| 146 |
fig.add_hline(y=-20, line_dash="dash", line_color="red", row=4, col=1)
|
|
@@ -149,15 +120,15 @@ def plot_indicators(df, ticker):
|
|
| 149 |
# ADX & DI
|
| 150 |
if 'adx_14' in df:
|
| 151 |
fig.add_trace(
|
| 152 |
-
go.Scatter(x=df
|
| 153 |
row=5, col=1
|
| 154 |
)
|
| 155 |
fig.add_trace(
|
| 156 |
-
go.Scatter(x=df
|
| 157 |
row=5, col=1
|
| 158 |
)
|
| 159 |
fig.add_trace(
|
| 160 |
-
go.Scatter(x=df
|
| 161 |
row=5, col=1
|
| 162 |
)
|
| 163 |
fig.add_hline(y=25, line_dash="dash", line_color="black", row=5, col=1)
|
|
@@ -165,12 +136,12 @@ def plot_indicators(df, ticker):
|
|
| 165 |
# ATR & CCI
|
| 166 |
if 'atr_14' in df:
|
| 167 |
fig.add_trace(
|
| 168 |
-
go.Scatter(x=df
|
| 169 |
row=6, col=1
|
| 170 |
)
|
| 171 |
if 'cci_20' in df:
|
| 172 |
fig.add_trace(
|
| 173 |
-
go.Scatter(x=df
|
| 174 |
row=6, col=1
|
| 175 |
)
|
| 176 |
fig.add_hline(y=100, line_dash="dash", line_color="red", row=6, col=1)
|
|
@@ -187,7 +158,7 @@ def plot_indicators(df, ticker):
|
|
| 187 |
)
|
| 188 |
fig.add_trace(
|
| 189 |
go.Scatter(
|
| 190 |
-
x=df
|
| 191 |
line=dict(color='teal'), fill='tozeroy'
|
| 192 |
), row=7, col=1
|
| 193 |
)
|
|
@@ -212,15 +183,15 @@ def plot_indicators(df, ticker):
|
|
| 212 |
return None
|
| 213 |
def plot_future_forecast(df, result, indicators):
|
| 214 |
fig, ax = plt.subplots(figsize=(10, 6))
|
| 215 |
-
ax.plot(df
|
| 216 |
for ind in indicators:
|
| 217 |
if ind in df.columns:
|
| 218 |
-
ax.plot(df
|
| 219 |
if "latest_prediction" in result:
|
| 220 |
-
last_date = df
|
| 221 |
horizon = len(result["latest_prediction"])
|
| 222 |
future_dates = pd.date_range(start=last_date + pd.Timedelta(days=1), periods=horizon, freq='B')
|
| 223 |
-
ax.plot(future_dates, result["latest_prediction"], label="Forecast
|
| 224 |
for i, val in enumerate(result["latest_prediction"]):
|
| 225 |
ax.text(future_dates[i], val, f"{val:.2f}", color="orange")
|
| 226 |
ax.legend()
|
|
@@ -238,7 +209,7 @@ def plot_forecast(result, df):
|
|
| 238 |
forecast = result.get("forecast", [])
|
| 239 |
if not actual or not forecast:
|
| 240 |
return None
|
| 241 |
-
dates = df
|
| 242 |
fig = go.Figure()
|
| 243 |
fig.add_trace(go.Scatter(x=dates, y=actual, name='Actual', line=dict(color='blue')))
|
| 244 |
fig.add_trace(go.Scatter(x=dates, y=forecast, name='Forecast', line=dict(color='orange')))
|
|
@@ -423,8 +394,7 @@ def plot_model_architecture(result):
|
|
| 423 |
xaxis=dict(visible=False),
|
| 424 |
yaxis=dict(visible=False),
|
| 425 |
plot_bgcolor="white",
|
| 426 |
-
paper_bgcolor="white"
|
| 427 |
-
margin=dict(l=20, r=20, t=50, b=20)
|
| 428 |
)
|
| 429 |
return fig
|
| 430 |
except Exception as e:
|
|
@@ -433,40 +403,88 @@ def plot_model_architecture(result):
|
|
| 433 |
|
| 434 |
def plot_signals(signals_df, ticker):
|
| 435 |
try:
|
| 436 |
-
logging.debug(f"Plotting signals for {ticker}")
|
| 437 |
fig = go.Figure()
|
| 438 |
-
fig.add_trace(go.Scatter(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 439 |
buy_signals = signals_df[signals_df['Signal'] == 'Buy']
|
| 440 |
sell_signals = signals_df[signals_df['Signal'] == 'Sell']
|
| 441 |
-
|
| 442 |
-
fig.add_trace(go.Scatter(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 443 |
fig.update_layout(
|
| 444 |
-
title=f
|
| 445 |
-
xaxis_title=
|
| 446 |
-
yaxis_title=
|
| 447 |
-
template="
|
| 448 |
-
|
| 449 |
)
|
| 450 |
-
logging.info(f"Signals plot generated for {ticker}")
|
| 451 |
return fig
|
| 452 |
except Exception as e:
|
| 453 |
-
logging.error(f"
|
| 454 |
-
return
|
| 455 |
|
| 456 |
-
def plot_backtest(
|
| 457 |
try:
|
| 458 |
-
logging.debug(f"Plotting backtest for {ticker}")
|
| 459 |
fig = go.Figure()
|
| 460 |
-
fig.add_trace(go.Scatter(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 461 |
fig.update_layout(
|
| 462 |
-
title=f
|
| 463 |
-
xaxis_title=
|
| 464 |
-
yaxis_title=
|
| 465 |
-
template="
|
| 466 |
-
|
| 467 |
)
|
| 468 |
-
logging.info(f"Backtest plot generated for {ticker}")
|
| 469 |
return fig
|
| 470 |
except Exception as e:
|
| 471 |
-
logging.error(f"
|
| 472 |
-
return
|
|
|
|
|
|
| 2 |
import plotly.express as px
|
| 3 |
import pandas as pd
|
| 4 |
import logging
|
| 5 |
+
import matplotlib.pyplot as plt
|
|
|
|
|
|
|
|
|
|
| 6 |
from plotly.subplots import make_subplots
|
|
|
|
| 7 |
import numpy as np
|
|
|
|
| 8 |
|
| 9 |
logging.basicConfig(level=logging.INFO)
|
| 10 |
|
|
|
|
| 22 |
# Price and Moving Averages
|
| 23 |
fig.add_trace(
|
| 24 |
go.Candlestick(
|
| 25 |
+
x=df.index, open=df['Open'], high=df['High'], low=df['Low'], close=df['value'],
|
| 26 |
name='Price', increasing_line_color='#00CC96', decreasing_line_color='#EF553B'
|
| 27 |
), row=1, col=1
|
| 28 |
)
|
| 29 |
for ma in ['sma_10', 'sma_20', 'sma_50', 'ema_12', 'ema_26', 'ema_50']:
|
| 30 |
if ma in df:
|
| 31 |
fig.add_trace(
|
| 32 |
+
go.Scatter(x=df.index, y=df[ma], name=ma.upper(), line=dict(width=1.5)),
|
| 33 |
row=1, col=1
|
| 34 |
)
|
| 35 |
if 'bbu_20_2' in df:
|
| 36 |
fig.add_trace(
|
| 37 |
+
go.Scatter(x=df.index, y=df['bbu_20_2'], name='BB Upper', line=dict(color='gray', dash='dot')),
|
| 38 |
row=1, col=1
|
| 39 |
)
|
| 40 |
fig.add_trace(
|
| 41 |
+
go.Scatter(x=df.index, y=df['bbm_20_2'], name='BB Middle', line=dict(color='gray')),
|
| 42 |
row=1, col=1
|
| 43 |
)
|
| 44 |
fig.add_trace(
|
| 45 |
+
go.Scatter(x=df.index, y=df["bbl_20_2"], name="BB Lower", line=dict(color="gray", dash="dot")),
|
| 46 |
row=1, col=1
|
| 47 |
)
|
| 48 |
|
| 49 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
|
| 51 |
# Position Size and Risk Annotation
|
| 52 |
if 'atr_14' in df:
|
|
|
|
| 61 |
|
| 62 |
# Volume
|
| 63 |
fig.add_trace(
|
| 64 |
+
go.Bar(x=df.index, y=df["Volume"], name="Volume", marker_color="blue", opacity=0.5),
|
| 65 |
row=2, col=1
|
| 66 |
)
|
| 67 |
|
| 68 |
# MACD & RSI
|
| 69 |
if 'macd_12_26_9' in df:
|
| 70 |
fig.add_trace(
|
| 71 |
+
go.Scatter(x=df.index, y=df['macd_12_26_9'], name='MACD', line=dict(color='blue')),
|
| 72 |
row=3, col=1
|
| 73 |
)
|
| 74 |
fig.add_trace(
|
| 75 |
+
go.Scatter(x=df.index, y=df["macds_12_26_9"], name="MACD Signal", line=dict(color="orange")),
|
| 76 |
row=3, col=1
|
| 77 |
)
|
| 78 |
fig.add_trace(
|
| 79 |
+
go.Bar(x=df.index, y=df["macdh_12_26_9"], name="MACD Hist", marker_color="gray"),
|
| 80 |
row=3, col=1
|
| 81 |
)
|
| 82 |
if 'rsi_14' in df:
|
| 83 |
fig.add_trace(
|
| 84 |
+
go.Scatter(x=df.index, y=df["rsi_14"], name="RSI 14", line=dict(color="purple")),
|
| 85 |
row=3, col=1
|
| 86 |
)
|
| 87 |
fig.add_hline(y=70, line_dash="dash", line_color="red", row=3, col=1)
|
| 88 |
fig.add_hline(y=30, line_dash="dash", line_color="green", row=3, col=1)
|
| 89 |
if 'rsi_21' in df:
|
| 90 |
fig.add_trace(
|
| 91 |
+
go.Scatter(x=df.index, y=df["rsi_21"], name="RSI 21", line=dict(color="magenta", dash="dash")),
|
| 92 |
row=3, col=1
|
| 93 |
)
|
| 94 |
if 'rsi_50' in df:
|
| 95 |
fig.add_trace(
|
| 96 |
+
go.Scatter(x=df.index, y=df["rsi_50"], name="RSI 50", line=dict(color="cyan", dash="dot")),
|
| 97 |
row=3, col=1
|
| 98 |
)
|
| 99 |
|
| 100 |
# Stochastic & Williams %R
|
| 101 |
if 'stochk_14_3_3' in df:
|
| 102 |
fig.add_trace(
|
| 103 |
+
go.Scatter(x=df.index, y=df["stochk_14_3_3"], name="Stoch %K", line=dict(color="blue")),
|
| 104 |
row=4, col=1
|
| 105 |
)
|
| 106 |
fig.add_trace(
|
| 107 |
+
go.Scatter(x=df.index, y=df["stochd_14_3_3"], name="Stoch %D", line=dict(color="orange")),
|
| 108 |
row=4, col=1
|
| 109 |
)
|
| 110 |
fig.add_hline(y=80, line_dash="dash", line_color="red", row=4, col=1)
|
| 111 |
fig.add_hline(y=20, line_dash="dash", line_color="green", row=4, col=1)
|
| 112 |
if 'willr_14' in df:
|
| 113 |
fig.add_trace(
|
| 114 |
+
go.Scatter(x=df.index, y=df["willr_14"], name="Williams %R", line=dict(color="green")),
|
| 115 |
row=4, col=1
|
| 116 |
)
|
| 117 |
fig.add_hline(y=-20, line_dash="dash", line_color="red", row=4, col=1)
|
|
|
|
| 120 |
# ADX & DI
|
| 121 |
if 'adx_14' in df:
|
| 122 |
fig.add_trace(
|
| 123 |
+
go.Scatter(x=df.index, y=df['adx_14'], name='ADX', line=dict(color='blue')),
|
| 124 |
row=5, col=1
|
| 125 |
)
|
| 126 |
fig.add_trace(
|
| 127 |
+
go.Scatter(x=df.index, y=df.get('pdi_14'), name='+DI', line=dict(color='green')),
|
| 128 |
row=5, col=1
|
| 129 |
)
|
| 130 |
fig.add_trace(
|
| 131 |
+
go.Scatter(x=df.index, y=df.get('mdi_14'), name='-DI', line=dict(color='red')),
|
| 132 |
row=5, col=1
|
| 133 |
)
|
| 134 |
fig.add_hline(y=25, line_dash="dash", line_color="black", row=5, col=1)
|
|
|
|
| 136 |
# ATR & CCI
|
| 137 |
if 'atr_14' in df:
|
| 138 |
fig.add_trace(
|
| 139 |
+
go.Scatter(x=df.index, y=df["atr_14"], name="ATR", line=dict(color="orange")),
|
| 140 |
row=6, col=1
|
| 141 |
)
|
| 142 |
if 'cci_20' in df:
|
| 143 |
fig.add_trace(
|
| 144 |
+
go.Scatter(x=df.index, y=df["cci_20"], name="CCI", line=dict(color="purple")),
|
| 145 |
row=6, col=1
|
| 146 |
)
|
| 147 |
fig.add_hline(y=100, line_dash="dash", line_color="red", row=6, col=1)
|
|
|
|
| 158 |
)
|
| 159 |
fig.add_trace(
|
| 160 |
go.Scatter(
|
| 161 |
+
x=df.index, y=signal_strength, name='Signal Strength',
|
| 162 |
line=dict(color='teal'), fill='tozeroy'
|
| 163 |
), row=7, col=1
|
| 164 |
)
|
|
|
|
| 183 |
return None
|
| 184 |
def plot_future_forecast(df, result, indicators):
|
| 185 |
fig, ax = plt.subplots(figsize=(10, 6))
|
| 186 |
+
ax.plot(df.index, df["value"], label="Historical Value", color="blue", linewidth=2)
|
| 187 |
for ind in indicators:
|
| 188 |
if ind in df.columns:
|
| 189 |
+
ax.plot(df.index, df[ind], label=ind, linestyle='--')
|
| 190 |
if "latest_prediction" in result:
|
| 191 |
+
last_date = df.index[-1]
|
| 192 |
horizon = len(result["latest_prediction"])
|
| 193 |
future_dates = pd.date_range(start=last_date + pd.Timedelta(days=1), periods=horizon, freq='B')
|
| 194 |
+
ax.plot(future_dates, result["latest_prediction"], label="Forecast Value", color="orange", linestyle="--", linewidth=2)
|
| 195 |
for i, val in enumerate(result["latest_prediction"]):
|
| 196 |
ax.text(future_dates[i], val, f"{val:.2f}", color="orange")
|
| 197 |
ax.legend()
|
|
|
|
| 209 |
forecast = result.get("forecast", [])
|
| 210 |
if not actual or not forecast:
|
| 211 |
return None
|
| 212 |
+
dates = df.index[-len(actual):]
|
| 213 |
fig = go.Figure()
|
| 214 |
fig.add_trace(go.Scatter(x=dates, y=actual, name='Actual', line=dict(color='blue')))
|
| 215 |
fig.add_trace(go.Scatter(x=dates, y=forecast, name='Forecast', line=dict(color='orange')))
|
|
|
|
| 394 |
xaxis=dict(visible=False),
|
| 395 |
yaxis=dict(visible=False),
|
| 396 |
plot_bgcolor="white",
|
| 397 |
+
paper_bgcolor="white"
|
|
|
|
| 398 |
)
|
| 399 |
return fig
|
| 400 |
except Exception as e:
|
|
|
|
| 403 |
|
| 404 |
def plot_signals(signals_df, ticker):
|
| 405 |
try:
|
|
|
|
| 406 |
fig = go.Figure()
|
| 407 |
+
fig.add_trace(go.Scatter(
|
| 408 |
+
x=signals_df.index,
|
| 409 |
+
y=signals_df['Price'],
|
| 410 |
+
mode='lines',
|
| 411 |
+
name='Price'
|
| 412 |
+
))
|
| 413 |
buy_signals = signals_df[signals_df['Signal'] == 'Buy']
|
| 414 |
sell_signals = signals_df[signals_df['Signal'] == 'Sell']
|
| 415 |
+
|
| 416 |
+
fig.add_trace(go.Scatter(
|
| 417 |
+
x=buy_signals.index,
|
| 418 |
+
y=buy_signals['Price'],
|
| 419 |
+
mode='markers',
|
| 420 |
+
marker=dict(symbol='triangle-up', size=10, color='green'),
|
| 421 |
+
name='Buy Signal'
|
| 422 |
+
))
|
| 423 |
+
fig.add_trace(go.Scatter(
|
| 424 |
+
x=sell_signals.index,
|
| 425 |
+
y=sell_signals['Price'],
|
| 426 |
+
mode='markers',
|
| 427 |
+
marker=dict(symbol='triangle-down', size=10, color='red'),
|
| 428 |
+
name='Sell Signal'
|
| 429 |
+
))
|
| 430 |
+
|
| 431 |
fig.update_layout(
|
| 432 |
+
title=f'{ticker} Trading Signals',
|
| 433 |
+
xaxis_title='Date',
|
| 434 |
+
yaxis_title='Price',
|
| 435 |
+
template="plotly_white",
|
| 436 |
+
xaxis_rangeslider_visible=False
|
| 437 |
)
|
|
|
|
| 438 |
return fig
|
| 439 |
except Exception as e:
|
| 440 |
+
logging.error(f"Plot signals error: {e}")
|
| 441 |
+
return None
|
| 442 |
|
| 443 |
+
def plot_backtest(equity_df, trades_df, ticker):
|
| 444 |
try:
|
|
|
|
| 445 |
fig = go.Figure()
|
| 446 |
+
fig.add_trace(go.Scatter(
|
| 447 |
+
x=equity_df.index,
|
| 448 |
+
y=equity_df['Equity'],
|
| 449 |
+
mode='lines',
|
| 450 |
+
name='Equity Curve'
|
| 451 |
+
))
|
| 452 |
+
|
| 453 |
+
buy_trades = trades_df[trades_df['Type'] == 'Buy']
|
| 454 |
+
sell_trades = trades_df[trades_df['Type'] == 'Sell']
|
| 455 |
+
exit_trades = trades_df[trades_df['Type'] == 'Exit']
|
| 456 |
+
|
| 457 |
+
fig.add_trace(go.Scatter(
|
| 458 |
+
x=buy_trades['Date'],
|
| 459 |
+
y=buy_trades['Price'],
|
| 460 |
+
mode='markers',
|
| 461 |
+
marker=dict(symbol='triangle-up', size=10, color='green'),
|
| 462 |
+
name='Buy Trades'
|
| 463 |
+
))
|
| 464 |
+
fig.add_trace(go.Scatter(
|
| 465 |
+
x=sell_trades['Date'],
|
| 466 |
+
y=sell_trades['Price'],
|
| 467 |
+
mode='markers',
|
| 468 |
+
marker=dict(symbol='triangle-down', size=10, color='red'),
|
| 469 |
+
name='Sell Trades'
|
| 470 |
+
))
|
| 471 |
+
fig.add_trace(go.Scatter(
|
| 472 |
+
x=exit_trades['Date'],
|
| 473 |
+
y=exit_trades['Price'],
|
| 474 |
+
mode='markers',
|
| 475 |
+
marker=dict(symbol='circle', size=8, color='blue'),
|
| 476 |
+
name='Exit Trades'
|
| 477 |
+
))
|
| 478 |
+
|
| 479 |
fig.update_layout(
|
| 480 |
+
title=f'{ticker} Backtest Equity Curve and Trades',
|
| 481 |
+
xaxis_title='Date',
|
| 482 |
+
yaxis_title='Equity / Price',
|
| 483 |
+
template="plotly_white",
|
| 484 |
+
xaxis_rangeslider_visible=False
|
| 485 |
)
|
|
|
|
| 486 |
return fig
|
| 487 |
except Exception as e:
|
| 488 |
+
logging.error(f"Plot backtest error: {e}")
|
| 489 |
+
return None
|
| 490 |
+
|
core/signals.py
CHANGED
|
@@ -7,19 +7,19 @@ logging.basicConfig(level=logging.DEBUG, filename="debug.log", filemode="a")
|
|
| 7 |
def generate_signals(df, result, volatility_window=14):
|
| 8 |
try:
|
| 9 |
signals_df = pd.DataFrame(index=df.index)
|
| 10 |
-
signals_df[
|
| 11 |
-
signals_df[
|
| 12 |
-
signals_df[
|
| 13 |
-
signals_df[
|
| 14 |
-
signals_df[
|
| 15 |
-
|
| 16 |
-
rsi_key =
|
| 17 |
-
macd_key =
|
| 18 |
-
adx_key =
|
| 19 |
-
pdi_key =
|
| 20 |
-
mdi_key =
|
| 21 |
-
atr_key =
|
| 22 |
-
sentiment_key =
|
| 23 |
|
| 24 |
for i in range(1, len(df)):
|
| 25 |
vote = 0
|
|
@@ -29,14 +29,14 @@ def generate_signals(df, result, volatility_window=14):
|
|
| 29 |
rsi = df[rsi_key].iloc[i]
|
| 30 |
rsi_signal = 1 if rsi < 50 else -1 if rsi > 50 else 0
|
| 31 |
vote += rsi_signal
|
| 32 |
-
logging.debug(f"RSI at {df
|
| 33 |
|
| 34 |
if macd_key in df.columns and not pd.isna(df[macd_key].iloc[i]):
|
| 35 |
macd = df[macd_key].iloc[i]
|
| 36 |
macd_prev = df[macd_key].iloc[i-1] if i > 0 else 0
|
| 37 |
macd_signal = 1 if macd > 0 and macd_prev <= 0 else -1 if macd < 0 and macd_prev >= 0 else 0
|
| 38 |
vote += macd_signal
|
| 39 |
-
logging.debug(f"MACD at {df
|
| 40 |
|
| 41 |
if adx_key in df.columns and pdi_key in df.columns and mdi_key in df.columns:
|
| 42 |
adx = df[adx_key].iloc[i]
|
|
@@ -45,43 +45,45 @@ def generate_signals(df, result, volatility_window=14):
|
|
| 45 |
if not pd.isna(adx) and adx > 20:
|
| 46 |
adx_signal = 1 if pdi > mdi else -1 if mdi > pdi else 0
|
| 47 |
vote += adx_signal
|
| 48 |
-
logging.debug(f"ADX at {df
|
| 49 |
|
| 50 |
if sentiment_key in df.columns and not pd.isna(df[sentiment_key].iloc[i]):
|
| 51 |
sentiment = df[sentiment_key].iloc[i]
|
| 52 |
sentiment_signal = 1 if sentiment > 0.1 else -1 if sentiment < -0.1 else 0
|
| 53 |
vote += sentiment_signal
|
| 54 |
-
logging.debug(f"Sentiment at {df
|
| 55 |
|
| 56 |
-
if
|
| 57 |
-
forecast = result[
|
| 58 |
-
actual = df[
|
| 59 |
model_signal = 1 if forecast > actual * 1.01 else -1 if forecast < actual * 0.99 else 0
|
| 60 |
vote += model_signal
|
| 61 |
-
logging.debug(f"Model at {df
|
| 62 |
|
| 63 |
-
signals_df.loc[df.index[i],
|
| 64 |
-
signals_df.loc[df.index[i],
|
| 65 |
|
| 66 |
if atr_key in df.columns and not pd.isna(df[atr_key].iloc[i]):
|
| 67 |
atr = df[atr_key].iloc[i]
|
| 68 |
-
signals_df.loc[df.index[i],
|
| 69 |
-
signals_df.loc[df.index[i],
|
| 70 |
|
| 71 |
-
|
|
|
|
| 72 |
|
| 73 |
trades_df, equity_df = backtest_signals(signals_df, df)
|
| 74 |
-
signals_df[
|
| 75 |
|
| 76 |
-
signal_counts = signals_df[
|
| 77 |
total = sum(signal_counts.values())
|
| 78 |
signal_dist = {k: f"{v} ({v/total*100:.2f}%)" for k, v in signal_counts.items()}
|
| 79 |
-
|
|
|
|
| 80 |
logging.info(f"Signals generated: {signal_counts}")
|
| 81 |
-
return signals_df
|
| 82 |
except Exception as e:
|
| 83 |
logging.error(f"Error in generate_signals: {e}")
|
| 84 |
-
return pd.DataFrame()
|
| 85 |
|
| 86 |
def backtest_signals(signals_df, df, initial_balance=10000):
|
| 87 |
try:
|
|
@@ -91,39 +93,50 @@ def backtest_signals(signals_df, df, initial_balance=10000):
|
|
| 91 |
equity_curve = [balance]
|
| 92 |
entry_price = 0
|
| 93 |
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
signal
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
if position > 0 and not pd.isna(stop_loss) and not pd.isna(take_profit):
|
| 115 |
-
if price <= stop_loss or price >= take_profit:
|
| 116 |
balance += position * (price - entry_price)
|
| 117 |
-
trades.append({
|
| 118 |
position = 0
|
| 119 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 120 |
|
| 121 |
-
|
|
|
|
| 122 |
|
|
|
|
|
|
|
|
|
|
| 123 |
trades_df = pd.DataFrame(trades)
|
| 124 |
-
equity_df = pd.DataFrame({
|
| 125 |
logging.info(f"Backtest completed: {len(trades)} trades, Final Balance: {balance:.2f}")
|
| 126 |
return trades_df, equity_df
|
| 127 |
except Exception as e:
|
| 128 |
logging.error(f"Backtest error: {e}")
|
| 129 |
-
return pd.DataFrame(), pd.DataFrame()
|
|
|
|
| 7 |
def generate_signals(df, result, volatility_window=14):
|
| 8 |
try:
|
| 9 |
signals_df = pd.DataFrame(index=df.index)
|
| 10 |
+
signals_df["Price"] = df["value"]
|
| 11 |
+
signals_df["Signal"] = "Hold"
|
| 12 |
+
signals_df["Position_Size"] = 0.0
|
| 13 |
+
signals_df["Stop_Loss"] = np.nan
|
| 14 |
+
signals_df["Take_Profit"] = np.nan
|
| 15 |
+
|
| 16 |
+
rsi_key = "rsi_14"
|
| 17 |
+
macd_key = "macdh_12_26_9"
|
| 18 |
+
adx_key = "adx_14"
|
| 19 |
+
pdi_key = "pdi_14"
|
| 20 |
+
mdi_key = "mdi_14"
|
| 21 |
+
atr_key = "atr_14"
|
| 22 |
+
sentiment_key = "sentiment"
|
| 23 |
|
| 24 |
for i in range(1, len(df)):
|
| 25 |
vote = 0
|
|
|
|
| 29 |
rsi = df[rsi_key].iloc[i]
|
| 30 |
rsi_signal = 1 if rsi < 50 else -1 if rsi > 50 else 0
|
| 31 |
vote += rsi_signal
|
| 32 |
+
logging.debug(f"RSI at {df.index[i]}: value={rsi:.2f}, signal={rsi_signal}")
|
| 33 |
|
| 34 |
if macd_key in df.columns and not pd.isna(df[macd_key].iloc[i]):
|
| 35 |
macd = df[macd_key].iloc[i]
|
| 36 |
macd_prev = df[macd_key].iloc[i-1] if i > 0 else 0
|
| 37 |
macd_signal = 1 if macd > 0 and macd_prev <= 0 else -1 if macd < 0 and macd_prev >= 0 else 0
|
| 38 |
vote += macd_signal
|
| 39 |
+
logging.debug(f"MACD at {df.index[i]}: value={macd:.2f}, prev={macd_prev:.2f}, signal={macd_signal}")
|
| 40 |
|
| 41 |
if adx_key in df.columns and pdi_key in df.columns and mdi_key in df.columns:
|
| 42 |
adx = df[adx_key].iloc[i]
|
|
|
|
| 45 |
if not pd.isna(adx) and adx > 20:
|
| 46 |
adx_signal = 1 if pdi > mdi else -1 if mdi > pdi else 0
|
| 47 |
vote += adx_signal
|
| 48 |
+
logging.debug(f"ADX at {df.index[i]}: adx={adx:.2f}, pdi={pdi:.2f}, mdi={mdi:.2f}, signal={adx_signal}")
|
| 49 |
|
| 50 |
if sentiment_key in df.columns and not pd.isna(df[sentiment_key].iloc[i]):
|
| 51 |
sentiment = df[sentiment_key].iloc[i]
|
| 52 |
sentiment_signal = 1 if sentiment > 0.1 else -1 if sentiment < -0.1 else 0
|
| 53 |
vote += sentiment_signal
|
| 54 |
+
logging.debug(f"Sentiment at {df.index[i]}: value={sentiment:.2f}, signal={sentiment_signal}")
|
| 55 |
|
| 56 |
+
if "forecast" in result and len(result["forecast"]) > i:
|
| 57 |
+
forecast = result["forecast"][i]
|
| 58 |
+
actual = df["value"].iloc[i]
|
| 59 |
model_signal = 1 if forecast > actual * 1.01 else -1 if forecast < actual * 0.99 else 0
|
| 60 |
vote += model_signal
|
| 61 |
+
logging.debug(f"Model at {df.index[i]}: forecast={forecast:.2f}, actual={actual:.2f}, signal={model_signal}")
|
| 62 |
|
| 63 |
+
signals_df.loc[df.index[i], "Signal"] = "Buy" if vote >= 2 else "Sell" if vote <= -2 else "Hold"
|
| 64 |
+
signals_df.loc[df.index[i], "Position_Size"] = min(0.1 * abs(vote), 1.0)
|
| 65 |
|
| 66 |
if atr_key in df.columns and not pd.isna(df[atr_key].iloc[i]):
|
| 67 |
atr = df[atr_key].iloc[i]
|
| 68 |
+
signals_df.loc[df.index[i], "Stop_Loss"] = df["value"].iloc[i] - 2 * atr if vote >= 2 else df["value"].iloc[i] + 2 * atr if vote <= -2 else np.nan
|
| 69 |
+
signals_df.loc[df.index[i], "Take_Profit"] = df["value"].iloc[i] + 3 * atr if vote >= 2 else df["value"].iloc[i] - 3 * atr if vote <= -2 else np.nan
|
| 70 |
|
| 71 |
+
current_signal = signals_df.iloc[i]["Signal"]
|
| 72 |
+
logging.debug(f"Signal at {df.index[i]}: RSI={rsi_signal}, MACD={macd_signal}, ADX={adx_signal}, Sentiment={sentiment_signal}, Model={model_signal}, Vote={vote}, Signal={current_signal}")
|
| 73 |
|
| 74 |
trades_df, equity_df = backtest_signals(signals_df, df)
|
| 75 |
+
signals_df["Equity"] = equity_df["Equity"]
|
| 76 |
|
| 77 |
+
signal_counts = signals_df["Signal"].value_counts().to_dict()
|
| 78 |
total = sum(signal_counts.values())
|
| 79 |
signal_dist = {k: f"{v} ({v/total*100:.2f}%)" for k, v in signal_counts.items()}
|
| 80 |
+
signal_dist_str = ", ".join([f'{k}={v}' for k, v in signal_dist.items()])
|
| 81 |
+
logging.info(f"Signal distribution: {signal_dist_str}")
|
| 82 |
logging.info(f"Signals generated: {signal_counts}")
|
| 83 |
+
return signals_df, trades_df, equity_df
|
| 84 |
except Exception as e:
|
| 85 |
logging.error(f"Error in generate_signals: {e}")
|
| 86 |
+
return pd.DataFrame(), pd.DataFrame(), pd.DataFrame()
|
| 87 |
|
| 88 |
def backtest_signals(signals_df, df, initial_balance=10000):
|
| 89 |
try:
|
|
|
|
| 93 |
equity_curve = [balance]
|
| 94 |
entry_price = 0
|
| 95 |
|
| 96 |
+
# Iterate through the original DataFrame's index to ensure equity_curve aligns
|
| 97 |
+
for idx, row in df.iterrows():
|
| 98 |
+
# Find the corresponding signal for this date
|
| 99 |
+
signal_row = signals_df.loc[idx] if idx in signals_df.index else None
|
| 100 |
+
|
| 101 |
+
if signal_row is not None:
|
| 102 |
+
price = signal_row["Price"]
|
| 103 |
+
signal = signal_row["Signal"]
|
| 104 |
+
position_size = signal_row["Position_Size"]
|
| 105 |
+
stop_loss = signal_row["Stop_Loss"]
|
| 106 |
+
take_profit = signal_row["Take_Profit"]
|
| 107 |
+
|
| 108 |
+
if signal == "Buy" and position == 0:
|
| 109 |
+
shares = position_size * balance / price
|
| 110 |
+
position = shares
|
| 111 |
+
entry_price = price
|
| 112 |
+
trades.append({"Date": str(idx.date()), "Type": "Buy", "Price": price, "Shares": shares})
|
| 113 |
+
logging.debug(f"Buy at {price:.2f}, Shares: {shares:.2f}")
|
| 114 |
+
|
| 115 |
+
elif signal == "Sell" and position > 0:
|
|
|
|
|
|
|
| 116 |
balance += position * (price - entry_price)
|
| 117 |
+
trades.append({"Date": str(idx.date()), "Type": "Sell", "Price": price, "Shares": position, "Profit": position * (price - entry_price)})
|
| 118 |
position = 0
|
| 119 |
+
profit_val = trades[-1]["Profit"]
|
| 120 |
+
logging.debug(f"Sell at {price:.2f}, Profit: {profit_val:.2f}")
|
| 121 |
+
|
| 122 |
+
if position > 0 and not pd.isna(stop_loss) and not pd.isna(take_profit):
|
| 123 |
+
if price <= stop_loss or price >= take_profit:
|
| 124 |
+
balance += position * (price - entry_price)
|
| 125 |
+
trades.append({"Date": str(idx.date()), "Type": "Exit", "Price": price, "Shares": position, "Profit": position * (price - entry_price)})
|
| 126 |
+
position = 0
|
| 127 |
+
profit_val = trades[-1]["Profit"]
|
| 128 |
+
logging.debug(f"Exit at {price:.2f}, Profit: {profit_val:.2f}")
|
| 129 |
|
| 130 |
+
current_equity = balance + position * (row["value"] - entry_price) if position > 0 else balance
|
| 131 |
+
equity_curve.append(current_equity)
|
| 132 |
|
| 133 |
+
# The first element of equity_curve is the initial balance, remove it to align with df.index
|
| 134 |
+
equity_curve = equity_curve[1:]
|
| 135 |
+
|
| 136 |
trades_df = pd.DataFrame(trades)
|
| 137 |
+
equity_df = pd.DataFrame({"Equity": equity_curve}, index=df.index)
|
| 138 |
logging.info(f"Backtest completed: {len(trades)} trades, Final Balance: {balance:.2f}")
|
| 139 |
return trades_df, equity_df
|
| 140 |
except Exception as e:
|
| 141 |
logging.error(f"Backtest error: {e}")
|
| 142 |
+
return pd.DataFrame(), pd.DataFrame(), pd.DataFrame(), pd.DataFrame()
|
core/train_eval.py
CHANGED
|
@@ -152,7 +152,7 @@ def train_and_evaluate(
|
|
| 152 |
logging.info(f"Selected features: {selected_features}")
|
| 153 |
|
| 154 |
X, y, feature_scaler, target_scaler, full_features, target_idx, pca, updated_feature_cols = preprocess_data(df, selected_features, target, window, horizon)
|
| 155 |
-
logging.debug(f"Preprocess: type(X)={type(X)},
|
| 156 |
|
| 157 |
if X.shape[0] < 10:
|
| 158 |
logging.error(f"Insufficient data samples: {X.shape[0]}")
|
|
@@ -184,15 +184,15 @@ def train_and_evaluate(
|
|
| 184 |
model = model_cls(input_size=input_size, hidden_size=hidden, num_layers=layers, output_size=horizon, dropout=dropout).to(device)
|
| 185 |
logging.debug(f"Model initialized: {model_cls.__name__}, input_size={input_size}, hidden={hidden}, layers={layers}")
|
| 186 |
|
| 187 |
-
if verbose and summary:
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
|
| 197 |
optimizer = optim.Adam(model.parameters(), lr=lr, betas=(beta1, beta2), weight_decay=weight_decay)
|
| 198 |
criterion = nn.MSELoss()
|
|
@@ -213,7 +213,7 @@ def train_and_evaluate(
|
|
| 213 |
logging.debug(f"Training Batch_X type: {type(batch_X)}, shape: {batch_X.shape}")
|
| 214 |
logging.debug(f"Training Batch_Y type: {type(batch_y)}, shape: {batch_y.shape}")
|
| 215 |
optimizer.zero_grad()
|
| 216 |
-
logging.debug(f"Training input to model: type={type(batch_X)},
|
| 217 |
try:
|
| 218 |
outputs = model(batch_X)
|
| 219 |
logging.debug(f"Training model output shape: {outputs.shape}")
|
|
@@ -233,7 +233,7 @@ def train_and_evaluate(
|
|
| 233 |
for batch_X, batch_y in test_loader:
|
| 234 |
logging.debug(f"Validation Batch_X type: {type(batch_X)}, shape: {batch_X.shape}")
|
| 235 |
logging.debug(f"Validation Batch_Y type: {type(batch_y)}, shape: {batch_y.shape}")
|
| 236 |
-
logging.debug(f"Validation input to model: type={type(batch_X)},
|
| 237 |
try:
|
| 238 |
outputs = model(batch_X)
|
| 239 |
logging.debug(f"Validation model output shape: {outputs.shape}")
|
|
@@ -256,7 +256,7 @@ def train_and_evaluate(
|
|
| 256 |
model.eval()
|
| 257 |
with torch.no_grad():
|
| 258 |
X_test_tensor = torch.tensor(X_test, dtype=torch.float32).to(device)
|
| 259 |
-
logging.debug(f"Eval model call: type={type(X_test_tensor)},
|
| 260 |
try:
|
| 261 |
y_pred_scaled = model(X_test_tensor).cpu().numpy()
|
| 262 |
logging.debug(f"Eval model output shape: {y_pred_scaled.shape}")
|
|
|
|
| 152 |
logging.info(f"Selected features: {selected_features}")
|
| 153 |
|
| 154 |
X, y, feature_scaler, target_scaler, full_features, target_idx, pca, updated_feature_cols = preprocess_data(df, selected_features, target, window, horizon)
|
| 155 |
+
logging.debug(f"Preprocess: type(X)={type(X)}, example={X if isinstance(X, tuple) else X.shape}, type(y)={type(y)}, example={y if isinstance(y, tuple) else y.shape}")
|
| 156 |
|
| 157 |
if X.shape[0] < 10:
|
| 158 |
logging.error(f"Insufficient data samples: {X.shape[0]}")
|
|
|
|
| 184 |
model = model_cls(input_size=input_size, hidden_size=hidden, num_layers=layers, output_size=horizon, dropout=dropout).to(device)
|
| 185 |
logging.debug(f"Model initialized: {model_cls.__name__}, input_size={input_size}, hidden={hidden}, layers={layers}")
|
| 186 |
|
| 187 |
+
# if verbose and summary:
|
| 188 |
+
# try:
|
| 189 |
+
# output = StringIO()
|
| 190 |
+
# sys.stdout = output
|
| 191 |
+
# summary(model, input_size=(window, input_size))
|
| 192 |
+
# sys.stdout = sys.__stdout__
|
| 193 |
+
# logging.debug(f"Model summary:\n{output.getvalue()}")
|
| 194 |
+
# except Exception as e:
|
| 195 |
+
# logging.warning(f"Failed to generate model summary: {str(e)}")
|
| 196 |
|
| 197 |
optimizer = optim.Adam(model.parameters(), lr=lr, betas=(beta1, beta2), weight_decay=weight_decay)
|
| 198 |
criterion = nn.MSELoss()
|
|
|
|
| 213 |
logging.debug(f"Training Batch_X type: {type(batch_X)}, shape: {batch_X.shape}")
|
| 214 |
logging.debug(f"Training Batch_Y type: {type(batch_y)}, shape: {batch_y.shape}")
|
| 215 |
optimizer.zero_grad()
|
| 216 |
+
logging.debug(f"Training input to model: type={type(batch_X)}, example={batch_X if isinstance(batch_X, tuple) else batch_X.shape}")
|
| 217 |
try:
|
| 218 |
outputs = model(batch_X)
|
| 219 |
logging.debug(f"Training model output shape: {outputs.shape}")
|
|
|
|
| 233 |
for batch_X, batch_y in test_loader:
|
| 234 |
logging.debug(f"Validation Batch_X type: {type(batch_X)}, shape: {batch_X.shape}")
|
| 235 |
logging.debug(f"Validation Batch_Y type: {type(batch_y)}, shape: {batch_y.shape}")
|
| 236 |
+
logging.debug(f"Validation input to model: type={type(batch_X)}, example={batch_X if isinstance(batch_X, tuple) else batch_X.shape}")
|
| 237 |
try:
|
| 238 |
outputs = model(batch_X)
|
| 239 |
logging.debug(f"Validation model output shape: {outputs.shape}")
|
|
|
|
| 256 |
model.eval()
|
| 257 |
with torch.no_grad():
|
| 258 |
X_test_tensor = torch.tensor(X_test, dtype=torch.float32).to(device)
|
| 259 |
+
logging.debug(f"Eval model call: type={type(X_test_tensor)}, example={X_test_tensor if isinstance(X_test_tensor, tuple) else X_test_tensor.shape}")
|
| 260 |
try:
|
| 261 |
y_pred_scaled = model(X_test_tensor).cpu().numpy()
|
| 262 |
logging.debug(f"Eval model output shape: {y_pred_scaled.shape}")
|
test_app.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
import pandas as pd
|
| 3 |
+
import numpy as np
|
| 4 |
+
import os
|
| 5 |
+
from datetime import datetime, timedelta
|
| 6 |
+
import logging
|
| 7 |
+
from core.data import load_data, add_technical_indicators, add_sentiment, preprocess_data
|
| 8 |
+
from core.model_runner import get_model
|
| 9 |
+
from core.plot import plot_forecast, plot_metrics_r2, plot_metrics_errors, plot_metrics_precision_recall, plot_metrics_risk, plot_loss_curve, plot_model_architecture, plot_future_forecast, plot_indicators, plot_signals, plot_backtest
|
| 10 |
+
import plotly.io as pio
|
| 11 |
+
from core.signals import generate_signals
|
| 12 |
+
from config import AVAILABLE_MODELS, DEFAULT_TICKERS, AVAILABLE_TIMEFRAMES, AVAILABLE_INDICATORS
|
| 13 |
+
from newsapi import NewsApiClient
|
| 14 |
+
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
|
| 15 |
+
|
| 16 |
+
log_path = "/tmp/app_log.txt"
|
| 17 |
+
os.makedirs("/tmp", exist_ok=True)
|
| 18 |
+
logging.basicConfig(
|
| 19 |
+
level=logging.DEBUG,
|
| 20 |
+
handlers=[
|
| 21 |
+
logging.FileHandler(log_path),
|
| 22 |
+
logging.StreamHandler()
|
| 23 |
+
],
|
| 24 |
+
format='%(asctime)s - %(levelname)s - %(message)s'
|
| 25 |
+
)
|
| 26 |
+
analyzer = SentimentIntensityAnalyzer()
|
| 27 |
+
|
| 28 |
+
def sentiment_analysis(ticker, start_date, end_date, api_key):
|
| 29 |
+
try:
|
| 30 |
+
if not api_key:
|
| 31 |
+
return "No API key provided", None
|
| 32 |
+
newsapi = NewsApiClient(api_key=api_key)
|
| 33 |
+
start = pd.to_datetime(start_date)
|
| 34 |
+
end = pd.to_datetime(end_date)
|
| 35 |
+
articles = newsapi.get_everything(
|
| 36 |
+
q=ticker, from_param=start.strftime("%Y-%m-%d"), to=end.strftime("%Y-%m-%d"),
|
| 37 |
+
language='en', sort_by='relevancy'
|
| 38 |
+
)
|
| 39 |
+
sentiments = [analyzer.polarity_scores(article["title"])["compound"] for article in articles["articles"]]
|
| 40 |
+
avg_sentiment = np.mean(sentiments) if sentiments else 0.0
|
| 41 |
+
sentiment_text = f"Average sentiment for {ticker}: {avg_sentiment:.2f}"
|
| 42 |
+
return sentiment_text, avg_sentiment
|
| 43 |
+
except Exception as e:
|
| 44 |
+
logging.error(f"Sentiment analysis failed: {str(e)}")
|
| 45 |
+
return f"Sentiment analysis failed: {str(e)}", None
|
| 46 |
+
|
| 47 |
+
def run_dashboard_test():
|
| 48 |
+
print("Starting run_dashboard_test...")
|
| 49 |
+
try:
|
| 50 |
+
# Sample parameters for run_dashboard
|
| 51 |
+
data_src = "yahoo"
|
| 52 |
+
ticker = "AAPL"
|
| 53 |
+
file_upload = None
|
| 54 |
+
timeframe = "1d"
|
| 55 |
+
start_date = "2020-01-01"
|
| 56 |
+
end_date = "2023-01-01"
|
| 57 |
+
horizon = 1
|
| 58 |
+
indicators = ["rsi", "macd", "bbands"]
|
| 59 |
+
include_sentiment = False
|
| 60 |
+
news_api_key = None # Replace with your News API key if testing sentiment
|
| 61 |
+
alpha_api_key = None # Replace with your Alpha Vantage API key if testing intraday
|
| 62 |
+
account_size = 10000
|
| 63 |
+
risk_percent = 0.01
|
| 64 |
+
model = "LSTM"
|
| 65 |
+
hidden_units = 64
|
| 66 |
+
n_layers = 1
|
| 67 |
+
epochs = 10 # Reduced for faster testing
|
| 68 |
+
learning_rate = 0.001
|
| 69 |
+
beta1 = 0.9
|
| 70 |
+
beta2 = 0.999
|
| 71 |
+
weight_decay = 0.01
|
| 72 |
+
dropout = 0.2
|
| 73 |
+
window_size = 30
|
| 74 |
+
test_split = 0.2
|
| 75 |
+
rsi_mid = 50
|
| 76 |
+
macd_sens = 0.0
|
| 77 |
+
adx_thr = 20
|
| 78 |
+
sent_thr = 0.1
|
| 79 |
+
vote_buy = 2
|
| 80 |
+
vote_sell = -2
|
| 81 |
+
feat_selector = "RandomForest"
|
| 82 |
+
feat_threshold = 0.0
|
| 83 |
+
|
| 84 |
+
print(f"Loading data for {ticker}...")
|
| 85 |
+
df = load_data(data_src=data_src, ticker=ticker, start=start_date, end=end_date,
|
| 86 |
+
interval=timeframe, file_upload=file_upload, alpha_api_key=alpha_api_key)
|
| 87 |
+
print(f"Data loaded. Shape: {df.shape}")
|
| 88 |
+
|
| 89 |
+
print("Adding technical indicators...")
|
| 90 |
+
df, valid_indicators = add_technical_indicators(df, indicators)
|
| 91 |
+
# Update the indicators list to use only valid ones for the model
|
| 92 |
+
indicators = valid_indicators
|
| 93 |
+
print(f"Indicators added. Shape: {df.shape}")
|
| 94 |
+
|
| 95 |
+
if include_sentiment and news_api_key:
|
| 96 |
+
print("Adding sentiment data...")
|
| 97 |
+
df = add_sentiment(df, ticker, news_api_key, start_date, end_date)
|
| 98 |
+
print(f"Sentiment added. Shape: {df.shape}")
|
| 99 |
+
sentiment_text, sentiment_score = sentiment_analysis(ticker, start_date, end_date, news_api_key)
|
| 100 |
+
print(f"Sentiment analysis result: {sentiment_text}")
|
| 101 |
+
|
| 102 |
+
print("Getting model...")
|
| 103 |
+
result = get_model(
|
| 104 |
+
df=df,
|
| 105 |
+
features=indicators,
|
| 106 |
+
target='value',
|
| 107 |
+
model_name=model,
|
| 108 |
+
horizon=horizon,
|
| 109 |
+
hidden_units=hidden_units,
|
| 110 |
+
n_layers=n_layers,
|
| 111 |
+
epochs=epochs,
|
| 112 |
+
learning_rate=learning_rate,
|
| 113 |
+
beta1=beta1,
|
| 114 |
+
beta2=beta2,
|
| 115 |
+
weight_decay=weight_decay,
|
| 116 |
+
dropout=dropout,
|
| 117 |
+
window_size=window_size,
|
| 118 |
+
test_split=test_split,
|
| 119 |
+
selector_method=feat_selector,
|
| 120 |
+
importance_threshold=feat_threshold
|
| 121 |
+
)
|
| 122 |
+
if isinstance(result, dict) and result.get("error"):
|
| 123 |
+
print(f"Model training failed: {result['error']}")
|
| 124 |
+
return
|
| 125 |
+
print("Model obtained.")
|
| 126 |
+
|
| 127 |
+
print("Generating signals...")
|
| 128 |
+
signals_df, trades_df, equity_df = generate_signals(df, result)
|
| 129 |
+
if signals_df.empty: # signals_df is the DataFrame after unpacking
|
| 130 |
+
print("Failed to generate signals")
|
| 131 |
+
return
|
| 132 |
+
print(f"Signals generated. Shape: {signals_df.shape}")
|
| 133 |
+
|
| 134 |
+
# Just print confirmation for plots, actual plot generation is not needed for local test
|
| 135 |
+
print("Generating plots...")
|
| 136 |
+
chart_plot = plot_indicators(df, ticker)
|
| 137 |
+
|
| 138 |
+
signals_plot = plot_signals(signals_df, ticker)
|
| 139 |
+
|
| 140 |
+
backtest_plot = plot_backtest(equity_df, trades_df, ticker)
|
| 141 |
+
|
| 142 |
+
future_plot = plot_future_forecast(df, result, indicators)
|
| 143 |
+
future_table = pd.DataFrame({
|
| 144 |
+
'Date': [df.index[-1] + timedelta(days=i+1) for i in range(horizon)],
|
| 145 |
+
'Prediction': result["latest_prediction"]
|
| 146 |
+
})
|
| 147 |
+
signals_table = signals_df.reset_index()[['Date', 'Price', 'Signal', 'Position_Size', 'Stop_Loss', 'Take_Profit', 'Equity']]
|
| 148 |
+
r2_plot = plot_metrics_r2(result)
|
| 149 |
+
error_plot = plot_metrics_errors(result)
|
| 150 |
+
precision_recall_plot = plot_metrics_precision_recall(result)
|
| 151 |
+
risk_plot = plot_metrics_risk(result)
|
| 152 |
+
loss_plot = plot_loss_curve(result)
|
| 153 |
+
architecture_plot = plot_model_architecture(result)
|
| 154 |
+
print("Plots generated.")
|
| 155 |
+
|
| 156 |
+
print("Dashboard run completed successfully.")
|
| 157 |
+
|
| 158 |
+
except Exception as e:
|
| 159 |
+
logging.error(f"Dashboard test error: {str(e)}")
|
| 160 |
+
print(f"Dashboard test error: {str(e)}")
|
| 161 |
+
|
| 162 |
+
if __name__ == "__main__":
|
| 163 |
+
run_dashboard_test()
|
| 164 |
+
|