Upload 6 files

core/data (1).py

@@ -0,0 +1,345 @@
+import pandas as pd
+import numpy as np
+import yfinance as yf
+import os
+try:
+    import talib as ta
+except ImportError:
+    ta = None
+from datetime import datetime, timedelta
+from newsapi import NewsApiClient
+from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
+from sklearn.preprocessing import MinMaxScaler
+from alpha_vantage.timeseries import TimeSeries
+import time
+import logging
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+def print_log(message, level='INFO'):
+    if level == 'INFO':
+        logging.info(message)
+    elif level == 'WARNING':
+        logging.warning(message)
+    elif level == 'ERROR':
+        logging.error(message)
+    else:
+        logging.debug(message)
+
+analyzer = SentimentIntensityAnalyzer()
+
+def load_data(data_src='yahoo', ticker='AAPL', start='2020-01-01', end='2023-01-01', interval='1d', file_upload=None, alpha_api_key=None):
+    try:
+        print_log(f"Loading data: source={data_src}, ticker={ticker}, start={start}, end={end}, interval={interval}, file_upload={'set' if file_upload else 'unset'}, alpha_api_key={'set' if alpha_api_key else 'unset'}")
+
+        start_date = pd.to_datetime(start)
+        end_date = pd.to_datetime(end)
+        if start_date >= end_date:
+            raise ValueError(f"Start date {start} must be before end date {end}")
+        if end_date > datetime.now():
+            print_log(f"End date {end} is in the future. Using current date as end date.", 'WARNING')
+            end_date = datetime.now()
+
+        df = pd.DataFrame()
+
+        if data_src == 'csv' and file_upload:
+            try:
+                file_path = getattr(file_upload, 'name', file_upload)
+                print_log(f"Loading CSV from {file_path}")
+                df = pd.read_csv(file_path)
+                if 'Date' not in df.columns:
+                    raise ValueError("CSV must contain a 'Date' column")
+                df['Date'] = pd.to_datetime(df['Date']).dt.tz_localize(None)
+                df = df.set_index('Date')
+                if 'Close' not in df.columns and 'value' not in df.columns:
+                    raise ValueError("CSV must contain 'Close' or 'value' column")
+                if 'Close' in df.columns:
+                    df = df.rename(columns={'Close': 'value'})
+                if df.empty:
+                    raise ValueError(f"CSV data is empty for {ticker}")
+                if df['value'].isna().all():
+                    raise ValueError(f"CSV 'value' column contains only NaNs for {ticker}")
+            except Exception as e:
+                print_log(f"Failed to load CSV {file_path}: {str(e)}", 'ERROR')
+                raise ValueError(f"Failed to load CSV: {str(e)}")
+        else:
+            print_log(f"Fetching data for {ticker} from Yahoo Finance")
+            try:
+                df = yf.download(ticker, start=start_date, end=end_date, interval=interval, progress=False, auto_adjust=False)
+                if isinstance(df.columns, pd.MultiIndex):
+                    df.columns = df.columns.droplevel(1)
+                if df.empty:
+                    raise ValueError(f"No data returned from Yahoo Finance for {ticker}")
+                if 'Close' not in df.columns:
+                    raise ValueError(f"Yahoo Finance data missing 'Close' column for {ticker}")
+                df = df.rename(columns={'Close': 'value'})
+                # The index is already datetime, no need to create a 'Date' column and then reset
+                if df['value'].isna().all():
+                    raise ValueError(f"Yahoo Finance 'value' column contains only NaNs for {ticker}")
+                if df['value'].empty:
+                    raise ValueError(f"Yahoo Finance 'value' column is empty for {ticker}")
+            except Exception as e:
+                print_log(f"Yahoo Finance failed for {ticker}: {str(e)}", 'ERROR')
+                raise ValueError(f"Yahoo Finance failed for {ticker}: {str(e)}")
+
+        # Alpha Vantage data loading (if applicable)
+        # Note: Alpha Vantage data loading logic is commented out for now to simplify debugging
+        # if interval in ['1m', '5m', '15m', '30m', '60m'] and alpha_api_key:
+        #     print_log(f"Attempting Alpha Vantage for {ticker}, interval {interval}")
+        #     try:
+        #         ts = TimeSeries(key=alpha_api_key, output_format='pandas')
+        #         df_av, _ = ts.get_intraday(symbol=ticker, interval=interval, outputsize='full')
+        #         if df_av.empty:
+        #             raise ValueError(f"No data returned from Alpha Vantage for {ticker}")
+        #         if '4. close' not in df_av.columns:
+        #             raise ValueError(f"Alpha Vantage data missing '4. close' column for {ticker}")
+        #         df_av = df_av.rename(columns={'4. close': 'value', '1. open': 'Open', '2. high': 'High', '3. low': 'Low', '5. volume': 'Volume'}) # Standardize column names
+        #         df_av['Date'] = pd.to_datetime(df_av.index)
+        #         df_av = df_av.reset_index(drop=True)
+        #         if df_av['value'].isna().all():
+        #             raise ValueError(f"Alpha Vantage 'value' column contains only NaNs for {ticker}")
+        #         if df_av['value'].empty:
+        #             raise ValueError(f"Alpha Vantage 'value' column is empty for {ticker}")
+        #         df = df_av # Use Alpha Vantage data if successful
+        #     except Exception as e:
+        #         print_log(f"Alpha Vantage failed: {str(e)}, using Yahoo Finance data", 'WARNING')
+
+        if df.empty:
+            raise ValueError(f"No data loaded for {ticker} from {data_src}")
+
+        # Ensure index is DatetimeIndex and sorted
+        if not isinstance(df.index, pd.DatetimeIndex):
+            df.index = pd.to_datetime(df.index)
+        df = df.sort_index()
+
+        required_cols = ['Open', 'High', 'Low', 'value', 'Volume']
+        for col in required_cols:
+            if col not in df.columns:
+                df[col] = np.nan # Add missing columns with NaNs
+
+        if 'value' not in df.columns:
+            raise ValueError(f"Target column 'value' is missing for {ticker}")
+        if df['value'].isna().all():
+            raise ValueError(f"Target column 'value' contains only NaNs for {ticker}")
+        if df['value'].empty:
+            raise ValueError(f"Target column 'value' is empty for {ticker}")
+
+        print_log(f"Data loaded for {ticker} with date range: {df.index.min()} to {df.index.max()}, shape: {df.shape}")
+        return df
+    except Exception as e:
+        print_log(f"Error in load_data for {ticker}: {str(e)}", 'ERROR')
+        raise ValueError(f"Failed to load data for {ticker}: {str(e)}")
+
+def add_technical_indicators(df, selected_indicators):
+    try:
+        print_log(f"Starting add_technical_indicators with indicators: {selected_indicators}")
+        if df.empty:
+            print_log("DataFrame is empty, skipping technical indicator calculation.", "WARNING")
+            return df, []
+
+        # Ensure columns are numeric and handle missing ones
+        for col in ['Open', 'High', 'Low', 'value', 'Volume']:
+            if col not in df.columns:
+                df[col] = np.nan
+            df[col] = pd.to_numeric(df[col], errors='coerce')
+        
+        # Drop rows with NaN in core columns after indicator calculation
+        df.dropna(subset=['Open', 'High', 'Low', 'value', 'Volume'], inplace=True)
+        if df.empty:
+            print_log("DataFrame is empty after dropping NaNs for technical indicators.", "WARNING")
+            return df, []
+
+        if ta is None:
+            print_log("TA-Lib not available. Cannot compute indicators. Falling back to 'value'.", 'ERROR')
+            return df, []
+
+        close = df['value'].values
+        high = df['High'].values
+        low = df['Low'].values
+        volume = df['Volume'].values
+        open_ = df['Open'].values
+
+        indicator_map = {
+            'rsi': {'func': ta.RSI, 'inputs': ['close'], 'params': {'timeperiod': 14}, 'output': ['rsi_14']},
+            'macd': {'func': ta.MACD, 'inputs': ['close'], 'params': {'fastperiod': 12, 'slowperiod': 26, 'signalperiod': 9}, 'output': ['macd_12_26_9', 'macds_12_26_9', 'macdh_12_26_9']},
+            'bbands': {'func': ta.BBANDS, 'inputs': ['close'], 'params': {'timeperiod': 20, 'nbdevup': 2, 'nbdevdn': 2}, 'output': ['bbu_20_2.0', 'bbm_20_2.0', 'bbl_20_2.0']},
+            'sma': {'func': ta.SMA, 'inputs': ['close'], 'params': {'timeperiod': 20}, 'output': ['sma_20']},
+            'ema': {'func': ta.EMA, 'inputs': ['close'], 'params': {'timeperiod': 20}, 'output': ['ema_20']},
+            'atr': {'func': ta.ATR, 'inputs': ['high', 'low', 'close'], 'params': {'timeperiod': 14}, 'output': ['atr_14']},
+            'stoch': {'func': ta.STOCH, 'inputs': ['high', 'low', 'close'], 'params': {'fastk_period': 14, 'slowk_period': 3, 'slowd_period': 3}, 'output': ['stochk_14_3_3', 'stochd_14_3_3']},
+            'adx': {'func': ta.ADX, 'inputs': ['high', 'low', 'close'], 'params': {'timeperiod': 14}, 'output': ['adx_14']},
+            'willr': {'func': ta.WILLR, 'inputs': ['high', 'low', 'close'], 'params': {'timeperiod': 14}, 'output': ['willr_14']},
+            'cci': {'func': ta.CCI, 'inputs': ['high', 'low', 'close'], 'params': {'timeperiod': 20}, 'output': ['cci_20']},
+            'pdi': {'func': ta.PLUS_DI, 'inputs': ['high', 'low', 'close'], 'params': {'timeperiod': 14}, 'output': ['pdi_14']},
+            'mdi': {'func': ta.MINUS_DI, 'inputs': ['high', 'low', 'close'], 'params': {'timeperiod': 14}, 'output': ['mdi_14']}
+        }
+
+        input_dict = {'close': close, 'high': high, 'low': low, 'open': open_, 'volume': volume}
+
+        valid_indicators = []
+        for ind in selected_indicators:
+            if ind in indicator_map:
+                print_log(f"Computing indicator: {ind}")
+                config = indicator_map[ind]
+                func = config['func']
+                inputs = config['inputs']
+                params = config['params']
+                try:
+                    input_arrays = [input_dict[inp] for inp in inputs]
+                    result = func(*input_arrays, **params)
+                    if isinstance(result, tuple):
+                        for j, (res, out_col) in enumerate(zip(result, config['output'])):
+                            if isinstance(res, np.ndarray) and len(res) == len(df):
+                                df[out_col] = res
+                                nan_count = np.isnan(res).sum()
+                                if nan_count < len(res) * 0.5:
+                                    valid_indicators.append(out_col)
+                                else:
+                                    print_log(f"{out_col} has excessive NaNs: {nan_count}/{len(res)}. Excluding from valid indicators.", 'WARNING')
+                            else:
+                                print_log(f"Invalid output for {out_col}: {type(res)}, length: {len(res) if hasattr(res, '__len__') else 'N/A'}", 'WARNING')
+                    else:
+                        if isinstance(result, np.ndarray) and len(result) == len(df):
+                            df[config['output'][0]] = result
+                            nan_count = np.isnan(result).sum()
+                            if nan_count < len(result) * 0.5:
+                                valid_indicators.append(config['output'][0])
+                            else:
+                                print_log(f"{config['output'][0]} has excessive NaNs: {nan_count}/{len(result)}. Excluding from valid indicators.", 'WARNING')
+                        else:
+                            print_log(f"Invalid output for {ind}: {type(result)}, length: {len(result) if hasattr(result, '__len__') else 'N/A'}", 'WARNING')
+                except Exception as e:
+                    print_log(f"Error computing {ind}: {str(e)}", 'ERROR')
+            else:
+                print_log(f"Indicator {ind} not supported by TA-Lib", 'WARNING')
+
+        # Drop rows with NaN in 'value', preserve valid indicators
+        initial_rows = len(df)
+        df = df.dropna(subset=['value']).reset_index(drop=False) # Keep index as a column for now
+        print_log(f"Dropped {initial_rows - len(df)} rows with NaN in 'value'")
+
+        # Drop columns with excessive NaNs, but protect 'value'
+        for col in df.columns:
+            if col not in ['Date', 'Open', 'High', 'Low', 'value', 'Volume']:
+                nan_ratio = df[col].isna().mean()
+                if nan_ratio > 0.5:
+                    print_log(f"Dropping {col} due to excessive NaNs: {nan_ratio:.2%}", 'WARNING')
+                    df = df.drop(columns=[col])
+                    if col in valid_indicators:
+                        valid_indicators.remove(col)
+
+        if not valid_indicators:
+            print_log("No valid indicators computed. Falling back to 'value'.", "WARNING")
+            valid_indicators.append('value')
+
+        print_log(f"Valid indicators: {valid_indicators}")
+        print_log(f"Technical indicators added successfully, shape: {df.shape}")
+        df.set_index('Date', inplace=True) # Set index back to Date after all processing
+        return df, valid_indicators
+    except Exception as e:
+        print_log(f"Error in add_technical_indicators: {str(e)}", 'ERROR')
+        # If an error occurs, return the original DataFrame to prevent further errors
+        return df, []
+
+def add_sentiment(df, ticker, news_api_key, start_date, end_date):
+    try:
+        print_log(f"Starting add_sentiment for {ticker} from {start_date} to {end_date}")
+        if not news_api_key:
+            print_log("News API key not provided. Skipping sentiment analysis.", "WARNING")
+            df['sentiment_score'] = 0.0
+            return df
+
+        newsapi = NewsApiClient(api_key=news_api_key)
+        all_articles = []
+        current_date = pd.to_datetime(start_date)
+        end_date = pd.to_datetime(end_date)
+
+        while current_date <= end_date:
+            from_param = current_date.strftime("%Y-%m-%d")
+            to_param = (current_date + timedelta(days=1)).strftime("%Y-%m-%d")
+            print_log(f"Fetching news for {ticker} from {from_param} to {to_param}")
+            try:
+                articles = newsapi.get_everything(q=ticker, language='en', sort_by='relevancy', from_param=from_param, to=to_param)
+                all_articles.extend(articles["articles"])
+            except Exception as e:
+                print_log(f"Error fetching news for {ticker} on {from_param}: {str(e)}", 'ERROR')
+            current_date += timedelta(days=1)
+            time.sleep(0.1)
+
+        if not all_articles:
+            print_log(f"No articles found for {ticker}. Setting sentiment to 0.", 'WARNING')
+            df['sentiment_score'] = 0.0
+            return df
+
+        sentiment_data = []
+        for article in all_articles:
+            if article["publishedAt"] and article["description"]:
+                date = pd.to_datetime(article["publishedAt"]).tz_localize(None).date()
+                text = article["description"]
+                vs = analyzer.polarity_scores(text)
+                sentiment_data.append({"Date": date, "sentiment_score": vs["compound"]})
+
+        sentiment_df = pd.DataFrame(sentiment_data)
+        sentiment_df["Date"] = pd.to_datetime(sentiment_df["Date"])
+        sentiment_df = sentiment_df.groupby("Date")["sentiment_score"].mean().reset_index()
+
+        df.reset_index(inplace=True)
+        df['Date'] = pd.to_datetime(df['Date'])
+        df = pd.merge(df, sentiment_df, on="Date", how="left")
+        df['sentiment_score'] = df['sentiment_score'].fillna(0.0)
+        df.set_index('Date', inplace=True)
+
+        print_log(f"Sentiment analysis completed for {ticker}. Added sentiment_score column.")
+        return df
+    except Exception as e:
+        print_log(f"Error in add_sentiment for {ticker}: {str(e)}", 'ERROR')
+        df['sentiment_score'] = 0.0
+        return df
+
+def preprocess_data(df, features, target, window_size, horizon):
+    try:
+        print_log(f"Starting preprocessing: features={features}, target={target}, window={window_size}, horizon={horizon}")
+        
+        # Ensure the DataFrame index is a DatetimeIndex
+        if not isinstance(df.index, pd.DatetimeIndex):
+            raise ValueError("DataFrame index must be a DatetimeIndex for preprocessing.")
+
+        # Filter features to only include those present in the DataFrame columns
+        updated_feature_cols = [f for f in features if f in df.columns]
+        if not updated_feature_cols:
+            raise ValueError("No valid features found in DataFrame after indicator calculation.")
+
+        full_features = updated_feature_cols + [target]
+        data = df[full_features].copy()
+        data.dropna(inplace=True)
+
+        if data.empty:
+            raise ValueError("DataFrame is empty after dropping NaNs. Cannot proceed with scaling.")
+
+        feature_scaler = MinMaxScaler()
+        target_scaler = MinMaxScaler()
+
+        data_features_scaled = feature_scaler.fit_transform(data[updated_feature_cols])
+        data_target_scaled = target_scaler.fit_transform(data[[target]])
+
+        full_scaled = np.hstack((data_features_scaled, data_target_scaled))
+        target_idx = len(updated_feature_cols)
+
+        X, y = [], []
+        for i in range(len(full_scaled) - window_size - horizon + 1):
+            X.append(full_scaled[i:i + window_size])
+            y.append(full_scaled[i + window_size:i + window_size + horizon, target_idx])
+
+        X = np.array(X)
+        y = np.array(y)
+
+        if X.shape[0] == 0 or y.shape[0] == 0:
+            raise ValueError(f"Insufficient data after preprocessing. Data length: {len(full_scaled)}, window_size: {window_size}, horizon: {horizon}")
+
+        print_log(f"Preprocessed data: X.shape={X.shape}, y.shape={y.shape}, Final features: {full_features}, Target idx: {target_idx}")
+        return X, y, feature_scaler, target_scaler, full_features, target_idx, None, updated_feature_cols
+    except Exception as e:
+        print_log(f"Preprocessing error: {str(e)}", 'ERROR')
+        raise ValueError(f"Preprocessing failed: {str(e)}")
+

core/model_runner (2).py

@@ -0,0 +1,144 @@
+# core/model_runner.py
+import torch
+import logging
+from core.train_eval import train_and_evaluate
+from core.models import (
+    LSTMModel,
+    GRUModel,
+    CNNModel,
+    MLPModel,
+    HybridCNNGRUModel,
+    TransformerModel,
+    BiLSTMModel,
+)
+
+logging.basicConfig(
+    level=logging.INFO,
+    filename="/tmp/app_log.txt",
+    filemode="a",
+    format="%(asctime)s - %(levelname)s - %(message)s",
+)
+
+
+def get_model(
+    df,
+    features,
+    target,
+    model_name="LSTM",
+    horizon=1,
+    # Hidden size aliases
+    hidden=None,
+    hidden_units=None,
+    # Layers aliases
+    layers=None,
+    n_layers=None,
+    # Learning rate aliases
+    lr=None,
+    learning_rate=None,
+    # Betas for optimizer
+    beta1=0.9,
+    beta2=0.999,
+    # Other hyperparams
+    epochs=50,
+    weight_decay=0.01,
+    dropout=0.2,
+    # Window aliases
+    window=None,
+    window_size=None,
+    test_split=0.2,
+    selector_method="RandomForest",
+    importance_threshold=0.0,
+    scheduler_type="None",
+    device=None,
+    verbose=True,
+):
+    """
+    Wrapper that accepts many common argument names used by the UI/calls,
+    normalizes them, and calls train_and_evaluate(...) with the canonical names.
+    """
+    try:
+        # --- Normalize aliases & defaults ---
+        # hidden size: prefer explicit hidden_units, then hidden, else default 64
+        if hidden_units is not None:
+            hidden = hidden_units
+        if hidden is None:
+            hidden = 64
+
+        # layers: prefer explicit n_layers, then layers, else default 1
+        if n_layers is not None:
+            layers = n_layers
+        if layers is None:
+            layers = 1
+
+        # learning rate: prefer learning_rate then lr, else default 0.001
+        if learning_rate is not None:
+            lr = learning_rate
+        if lr is None:
+            lr = 0.001
+
+        # window size: prefer window_size then window, else default 30
+        if window_size is not None:
+            window = window_size
+        if window is None:
+            window = 30
+
+        # device: caller may pass it; otherwise detect automatically
+        if device is None:
+            device = "cuda" if torch.cuda.is_available() else "cpu"
+
+        logging.info(
+            f"get_model called: model={model_name}, device={device}, hidden={hidden}, layers={layers}, lr={lr}, window={window}, epochs={epochs}"
+        )
+
+        # --- Select model class mapping (keys as used in UI) ---
+        model_classes = {
+            "LSTM": LSTMModel,
+            "GRU": GRUModel,
+            "CNN": CNNModel,
+            "MLP": MLPModel,
+            "Hybrid": HybridCNNGRUModel,
+            "HybridCNNGRU": HybridCNNGRUModel,
+            "Transformer": TransformerModel,
+            "BiLSTM": BiLSTMModel,
+        }
+
+        model_cls = model_classes.get(model_name, LSTMModel)
+
+        # --- Call the core training function with canonical param names ---
+        result = train_and_evaluate(
+            df=df,
+            features=features,
+            target=target,
+            model_cls=model_cls,
+            horizon=horizon,
+            hidden=hidden,
+            layers=layers,
+            epochs=epochs,
+            lr=lr,
+            beta1=beta1,
+            beta2=beta2,
+            weight_decay=weight_decay,
+            dropout=dropout,
+            window=window,
+            test_split=test_split,
+            selector_method=selector_method,
+            importance_threshold=importance_threshold,
+            scheduler_type=scheduler_type,
+            device=device,
+            verbose=verbose,
+        )
+
+        # --- Normalize return --- 
+        if not result:
+            logging.error(f"{model_name} returned empty result.")
+            return {"error": "Empty result from training"}
+        if isinstance(result, dict) and result.get("error"):
+            logging.error(f"{model_name} training error: {result['error']}")
+            return {"error": result["error"]}
+
+        logging.info(f"{model_name} training completed successfully")
+        return result
+
+    except Exception as e:
+        logging.error(f"Model runner error for {model_name}: {str(e)}", exc_info=True)
+        return {"error": str(e)}

core/models (1).py

@@ -0,0 +1,219 @@
+# core/models.py
+import torch
+import logging
+import torch.nn as nn
+import math
+
+# ---------------- Base ----------------
+class BaseTimeSeriesModel(nn.Module):
+    def __init__(self):
+        super(BaseTimeSeriesModel, self).__init__()
+
+    def reset_weights(self):
+        for layer in self.children():
+            if hasattr(layer, "reset_parameters"):
+                layer.reset_parameters()
+
+
+# ---------------- LSTM ----------------
+class LSTMModel(nn.Module):
+    def __init__(self, input_size, hidden_size, num_layers, output_size, dropout=0.2):
+        super(LSTMModel, self).__init__()
+        self.hidden_size = hidden_size
+        self.num_layers = num_layers
+        self.lstm = nn.LSTM(
+            input_size=input_size,
+            hidden_size=hidden_size,
+            num_layers=num_layers,
+            batch_first=True,
+            dropout=dropout if num_layers > 1 else 0.0,
+        )
+        self.fc = nn.Linear(hidden_size, output_size)
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, x):
+        logging.debug(f"Inside forward: initial x type={type(x)}, x={x}")
+        if isinstance(x, (tuple, list)):
+            logging.debug(f"Model forward received tuple/list: type={type(x)}, length={len(x)}")
+            x = x[0]
+        if not isinstance(x, torch.Tensor):
+            x = torch.tensor(
+                x, dtype=torch.float32, device=next(self.parameters()).device
+            )
+        batch_size = x.size(0)
+        h0 = torch.zeros(self.num_layers, batch_size, self.hidden_size).to(x.device)
+        c0 = torch.zeros(self.num_layers, batch_size, self.hidden_size).to(x.device)
+        out, _ = self.lstm(x, (h0, c0))
+        out = self.dropout(out[:, -1, :])
+        return self.fc(out)
+
+
+# ---------------- GRU ----------------
+class GRUModel(nn.Module):
+    def __init__(self, input_size, hidden_size, num_layers, output_size, dropout=0.2):
+        super(GRUModel, self).__init__()
+        self.hidden_size = hidden_size
+        self.num_layers = num_layers
+        self.gru = nn.GRU(
+            input_size=input_size,
+            hidden_size=hidden_size,
+            num_layers=num_layers,
+            batch_first=True,
+            dropout=dropout if num_layers > 1 else 0.0,
+        )
+        self.fc = nn.Linear(hidden_size, output_size)
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, x):
+        logging.debug(f"Inside forward: initial x type={type(x)}, x={x}")
+        if isinstance(x, (tuple, list)):
+            logging.debug(f"Model forward received tuple/list: type={type(x)}, length={len(x)}")
+            x = x[0]
+        if not isinstance(x, torch.Tensor):
+            x = torch.tensor(
+                x, dtype=torch.float32, device=next(self.parameters()).device
+            )
+        batch_size = x.size(0)
+        h0 = torch.zeros(self.num_layers, batch_size, self.hidden_size).to(x.device)
+        out, _ = self.gru(x, h0)
+        out = self.dropout(out[:, -1, :])
+        return self.fc(out)
+
+
+# ---------------- CNN ----------------
+class CNNModel(nn.Module):
+    def __init__(self, input_size, hidden_size, num_layers, output_size, dropout=0.2):
+        super(CNNModel, self).__init__()
+        self.conv1 = nn.Conv1d(input_size, hidden_size, kernel_size=3, padding=1)
+        self.relu = nn.ReLU()
+        self.dropout = nn.Dropout(dropout)
+        self.fc = nn.Linear(hidden_size, output_size)
+
+    def forward(self, x):
+        logging.debug(f"Inside forward: initial x type={type(x)}, x={x}")
+        if isinstance(x, (tuple, list)):
+            logging.debug(f"Model forward received tuple/list: type={type(x)}, length={len(x)}")
+            x = x[0]
+        if not isinstance(x, torch.Tensor):
+            x = torch.tensor(
+                x, dtype=torch.float32, device=next(self.parameters()).device
+            )
+        x = x.transpose(1, 2)  # [batch, features, seq_len]
+        out = self.conv1(x)
+        out = self.relu(out)
+        out = out.mean(dim=2)  # global avg pooling
+        out = self.dropout(out)
+        return self.fc(out)
+
+
+# ---------------- MLP ----------------
+class MLPModel(nn.Module):
+    def __init__(self, input_size, hidden_size, num_layers, output_size, dropout=0.2):
+        super(MLPModel, self).__init__()
+        layers = []
+        in_features = input_size
+        for _ in range(num_layers):
+            layers.append(nn.Linear(in_features, hidden_size))
+            layers.append(nn.ReLU())
+            layers.append(nn.Dropout(dropout))
+            in_features = hidden_size
+        layers.append(nn.Linear(hidden_size, output_size))
+        self.mlp = nn.Sequential(*layers)
+
+    def forward(self, x):
+        logging.debug(f"Inside forward: initial x type={type(x)}, x={x}")
+        if isinstance(x, (tuple, list)):
+            logging.debug(f"Model forward received tuple/list: type={type(x)}, length={len(x)}")
+            x = x[0]
+        if not isinstance(x, torch.Tensor):
+            x = torch.tensor(
+                x, dtype=torch.float32, device=next(self.parameters()).device
+            )
+        return self.mlp(x[:, -1, :])  # flatten last timestep
+
+
+# ---------------- Hybrid CNN-GRU ----------------
+class HybridCNNGRUModel(nn.Module):
+    def __init__(self, input_size, hidden_size, num_layers, output_size, dropout=0.2):
+        super(HybridCNNGRUModel, self).__init__()
+        self.conv1 = nn.Conv1d(input_size, hidden_size, kernel_size=3, padding=1)
+        self.gru = nn.GRU(hidden_size, hidden_size, num_layers, batch_first=True)
+        self.fc = nn.Linear(hidden_size, output_size)
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, x):
+        logging.debug(f"Inside forward: initial x type={type(x)}, x={x}")
+        if isinstance(x, (tuple, list)):
+            logging.debug(f"Model forward received tuple/list: type={type(x)}, length={len(x)}")
+            x = x[0]
+        if not isinstance(x, torch.Tensor):
+            x = torch.tensor(
+                x, dtype=torch.float32, device=next(self.parameters()).device
+            )
+        x = x.transpose(1, 2)
+        out = self.conv1(x).transpose(1, 2)
+        out, _ = self.gru(out)
+        out = self.dropout(out[:, -1, :])
+        return self.fc(out)
+
+
+# ---------------- Transformer ----------------
+class TransformerModel(nn.Module):
+    def __init__(
+        self, input_size, hidden_size, num_layers, output_size, dropout=0.2, nhead=4
+    ):
+        super(TransformerModel, self).__init__()
+        self.embedding = nn.Linear(input_size, hidden_size)
+        encoder_layer = nn.TransformerEncoderLayer(
+            d_model=hidden_size, nhead=nhead, dropout=dropout
+        )
+        self.transformer = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)
+        self.fc = nn.Linear(hidden_size, output_size)
+
+    def forward(self, x):
+        logging.debug(f"Inside forward: initial x type={type(x)}, x={x}")
+        if isinstance(x, (tuple, list)):
+            logging.debug(f"Model forward received tuple/list: type={type(x)}, length={len(x)}")
+            x = x[0]
+        if not isinstance(x, torch.Tensor):
+            x = torch.tensor(
+                x, dtype=torch.float32, device=next(self.parameters()).device
+            )
+        x = self.embedding(x)
+        out = self.transformer(x.transpose(0, 1))  # seq_first
+        out = out[-1, :, :]
+        return self.fc(out)
+
+
+# ---------------- BiLSTM ----------------
+class BiLSTMModel(nn.Module):
+    def __init__(self, input_size, hidden_size, num_layers, output_size, dropout=0.2):
+        super(BiLSTMModel, self).__init__()
+        self.hidden_size = hidden_size
+        self.num_layers = num_layers
+        self.lstm = nn.LSTM(
+            input_size=input_size,
+            hidden_size=hidden_size,
+            num_layers=num_layers,
+            batch_first=True,
+            dropout=dropout if num_layers > 1 else 0.0,
+            bidirectional=True,
+        )
+        self.fc = nn.Linear(hidden_size * 2, output_size)
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, x):
+        logging.debug(f"Inside forward: initial x type={type(x)}, x={x}")
+        if isinstance(x, (tuple, list)):
+            logging.debug(f"Model forward received tuple/list: type={type(x)}, length={len(x)}")
+            x = x[0]
+        if not isinstance(x, torch.Tensor):
+            x = torch.tensor(
+                x, dtype=torch.float32, device=next(self.parameters()).device
+            )
+        batch_size = x.size(0)
+        h0 = torch.zeros(self.num_layers * 2, batch_size, self.hidden_size).to(x.device)
+        c0 = torch.zeros(self.num_layers * 2, batch_size, self.hidden_size).to(x.device)
+        out, _ = self.lstm(x, (h0, c0))
+        out = self.dropout(out[:, -1, :])
+        return self.fc(out)

core/plot (9).py

@@ -0,0 +1,490 @@
+import plotly.graph_objects as go
+import plotly.express as px
+import pandas as pd
+import logging
+import matplotlib.pyplot as plt
+from plotly.subplots import make_subplots
+import numpy as np
+
+logging.basicConfig(level=logging.INFO)
+
+def plot_indicators(df, ticker):
+    try:
+        fig = make_subplots(
+            rows=7, cols=1, shared_xaxes=True, vertical_spacing=0.03,
+            subplot_titles=(
+                'Price & Moving Averages', 'Volume', 'MACD & RSI',
+                'Stochastic & Williams %R', 'ADX & DI', 'ATR & CCI', 'Signal Strength'
+            ),
+            row_heights=[0.4, 0.1, 0.15, 0.15, 0.15, 0.15, 0.15]
+        )
+
+        # Price and Moving Averages
+        fig.add_trace(
+            go.Candlestick(
+                x=df.index, open=df['Open'], high=df['High'], low=df['Low'], close=df['value'],
+                name='Price', increasing_line_color='#00CC96', decreasing_line_color='#EF553B'
+            ), row=1, col=1
+        )
+        for ma in ['sma_10', 'sma_20', 'sma_50', 'ema_12', 'ema_26', 'ema_50']:
+            if ma in df:
+                fig.add_trace(
+                    go.Scatter(x=df.index, y=df[ma], name=ma.upper(), line=dict(width=1.5)),
+                    row=1, col=1
+                )
+        if 'bbu_20_2' in df:
+            fig.add_trace(
+                go.Scatter(x=df.index, y=df['bbu_20_2'], name='BB Upper', line=dict(color='gray', dash='dot')),
+                row=1, col=1
+            )
+            fig.add_trace(
+                go.Scatter(x=df.index, y=df['bbm_20_2'], name='BB Middle', line=dict(color='gray')),
+                row=1, col=1
+            )
+            fig.add_trace(
+                go.Scatter(x=df.index, y=df["bbl_20_2"], name="BB Lower", line=dict(color="gray", dash="dot")),
+                row=1, col=1
+            )
+
+
+
+        # Position Size and Risk Annotation
+        if 'atr_14' in df:
+            atr = df['atr_14'].iloc[-1]
+            stop_distance = atr * 2
+            position_size = (10000 * 0.01) / stop_distance
+            fig.add_annotation(
+                text=f"Position Size: {position_size:.0f} shares (1% risk, ATR {atr:.2f})",
+                xref="paper", yref="paper", x=0.05, y=0.95, showarrow=False,
+                font=dict(color="black", size=12)
+            )
+
+        # Volume
+        fig.add_trace(
+            go.Bar(x=df.index, y=df["Volume"], name="Volume", marker_color="blue", opacity=0.5),
+            row=2, col=1
+        )
+
+        # MACD & RSI
+        if 'macd_12_26_9' in df:
+            fig.add_trace(
+                go.Scatter(x=df.index, y=df['macd_12_26_9'], name='MACD', line=dict(color='blue')),
+                row=3, col=1
+            )
+            fig.add_trace(
+                go.Scatter(x=df.index, y=df["macds_12_26_9"], name="MACD Signal", line=dict(color="orange")),
+                row=3, col=1
+            )
+            fig.add_trace(
+                go.Bar(x=df.index, y=df["macdh_12_26_9"], name="MACD Hist", marker_color="gray"),
+                row=3, col=1
+            )
+        if 'rsi_14' in df:
+            fig.add_trace(
+                go.Scatter(x=df.index, y=df["rsi_14"], name="RSI 14", line=dict(color="purple")),
+                row=3, col=1
+            )
+            fig.add_hline(y=70, line_dash="dash", line_color="red", row=3, col=1)
+            fig.add_hline(y=30, line_dash="dash", line_color="green", row=3, col=1)
+        if 'rsi_21' in df:
+            fig.add_trace(
+                go.Scatter(x=df.index, y=df["rsi_21"], name="RSI 21", line=dict(color="magenta", dash="dash")),
+                row=3, col=1
+            )
+        if 'rsi_50' in df:
+            fig.add_trace(
+                go.Scatter(x=df.index, y=df["rsi_50"], name="RSI 50", line=dict(color="cyan", dash="dot")),
+                row=3, col=1
+            )
+
+        # Stochastic & Williams %R
+        if 'stochk_14_3_3' in df:
+            fig.add_trace(
+                go.Scatter(x=df.index, y=df["stochk_14_3_3"], name="Stoch %K", line=dict(color="blue")),
+                row=4, col=1
+            )
+            fig.add_trace(
+                go.Scatter(x=df.index, y=df["stochd_14_3_3"], name="Stoch %D", line=dict(color="orange")),
+                row=4, col=1
+            )
+            fig.add_hline(y=80, line_dash="dash", line_color="red", row=4, col=1)
+            fig.add_hline(y=20, line_dash="dash", line_color="green", row=4, col=1)
+        if 'willr_14' in df:
+            fig.add_trace(
+                go.Scatter(x=df.index, y=df["willr_14"], name="Williams %R", line=dict(color="green")),
+                row=4, col=1
+            )
+            fig.add_hline(y=-20, line_dash="dash", line_color="red", row=4, col=1)
+            fig.add_hline(y=-80, line_dash="dash", line_color="green", row=4, col=1)
+
+        # ADX & DI
+        if 'adx_14' in df:
+            fig.add_trace(
+                go.Scatter(x=df.index, y=df['adx_14'], name='ADX', line=dict(color='blue')),
+                row=5, col=1
+            )
+            fig.add_trace(
+                go.Scatter(x=df.index, y=df.get('pdi_14'), name='+DI', line=dict(color='green')),
+                row=5, col=1
+            )
+            fig.add_trace(
+                go.Scatter(x=df.index, y=df.get('mdi_14'), name='-DI', line=dict(color='red')),
+                row=5, col=1
+            )
+            fig.add_hline(y=25, line_dash="dash", line_color="black", row=5, col=1)
+
+        # ATR & CCI
+        if 'atr_14' in df:
+            fig.add_trace(
+                go.Scatter(x=df.index, y=df["atr_14"], name="ATR", line=dict(color="orange")),
+                row=6, col=1
+            )
+        if 'cci_20' in df:
+            fig.add_trace(
+                go.Scatter(x=df.index, y=df["cci_20"], name="CCI", line=dict(color="purple")),
+                row=6, col=1
+            )
+            fig.add_hline(y=100, line_dash="dash", line_color="red", row=6, col=1)
+            fig.add_hline(y=-100, line_dash="dash", line_color="green", row=6, col=1)
+
+        # Signal Strength Plot
+        if all(col in df for col in ['RSI_Signal', 'MACD_Signal', 'ADX_Signal', 'Sentiment_Signal', 'Model_Signal']):
+            signal_strength = (
+                df['RSI_Signal'].abs() +
+                df['MACD_Signal'].abs() +
+                df['ADX_Signal'].abs() +
+                df['Sentiment_Signal'].abs() +
+                df['Model_Signal'].abs()
+            )
+            fig.add_trace(
+                go.Scatter(
+                    x=df.index, y=signal_strength, name='Signal Strength',
+                    line=dict(color='teal'), fill='tozeroy'
+                ), row=7, col=1
+            )
+            fig.add_hline(y=3, line_dash="dash", line_color="orange", row=7, col=1, annotation_text="Strong Signal Threshold")
+
+        fig.update_layout(
+            title=f"{ticker} Price and Technical Indicators",
+            template="plotly_white",
+            height=2400,
+            width=1400,
+            showlegend=True,
+            xaxis_rangeslider_visible=False,
+            margin=dict(l=50, r=50, t=100, b=50),
+            xaxis=dict(tickformat="%Y-%m-%d", minor=dict(ticks="inside", showgrid=True), gridcolor="lightgrey"),
+            plot_bgcolor="white",
+            paper_bgcolor="white",
+            hovermode="x unified"
+        )
+        return fig
+    except Exception as e:
+        logging.error(f"Plot indicators error: {e}")
+        return None
+def plot_future_forecast(df, result, indicators):
+    fig, ax = plt.subplots(figsize=(10, 6))
+    ax.plot(df.index, df["value"], label="Historical Value", color="blue", linewidth=2)
+    for ind in indicators:
+        if ind in df.columns:
+            ax.plot(df.index, df[ind], label=ind, linestyle='--')
+    if "latest_prediction" in result:
+        last_date = df.index[-1]
+        horizon = len(result["latest_prediction"])
+        future_dates = pd.date_range(start=last_date + pd.Timedelta(days=1), periods=horizon, freq='B')
+        ax.plot(future_dates, result["latest_prediction"], label="Forecast Value", color="orange", linestyle="--", linewidth=2)
+        for i, val in enumerate(result["latest_prediction"]):
+            ax.text(future_dates[i], val, f"{val:.2f}", color="orange")
+    ax.legend()
+    ax.set_title("Historical Data, Indicators, and Future Forecast")
+    ax.set_xlabel("Date")
+    ax.set_ylabel("Value")
+    ax.grid(True)
+    plt.tight_layout()
+    return fig
+    
+# Other plotting functions remain unchanged
+def plot_forecast(result, df):
+    try:
+        actual = result.get("actual", [])
+        forecast = result.get("forecast", [])
+        if not actual or not forecast:
+            return None
+        dates = df.index[-len(actual):]
+        fig = go.Figure()
+        fig.add_trace(go.Scatter(x=dates, y=actual, name='Actual', line=dict(color='blue')))
+        fig.add_trace(go.Scatter(x=dates, y=forecast, name='Forecast', line=dict(color='orange')))
+        fig.update_layout(
+            title="Backtest: Actual vs Forecast",
+            template="plotly_white",
+            height=600,
+            xaxis_title="Date",
+            yaxis_title="Price",
+            xaxis=dict(tickformat="%Y-%m-%d", minor=dict(ticks="inside", showgrid=True), gridcolor="lightgrey"),
+            yaxis=dict(gridcolor="lightgrey"),
+            plot_bgcolor="white",
+            paper_bgcolor="white"
+        )
+        return fig
+    except Exception as e:
+        logging.error(f"Plot forecast error: {e}")
+        return None
+
+# ... (other plotting functions like plot_future_forecast, plot_metrics_r2, etc., remain as provided)
+
+def plot_metrics_r2(result):
+    try:
+        metrics = result.get("metrics", {})
+        if not metrics:
+            return None
+        fig = go.Figure()
+        fig.add_trace(go.Bar(
+            x=['R²', 'MAPE'],
+            y=[metrics.get('R2', 0), metrics.get('MAPE', 0)],
+            marker_color=['#1E90FF', '#FF6347']
+        ))
+        fig.update_layout(
+            title="R² and MAPE Metrics",
+            template="plotly_white",
+            height=600,
+            xaxis_title="Metric",
+            yaxis_title="Value",
+            xaxis=dict(minor=dict(ticks="inside", showgrid=True), gridcolor="lightgrey"),
+            yaxis=dict(gridcolor="lightgrey"),
+            plot_bgcolor="white",
+            paper_bgcolor="white"
+        )
+        return fig
+    except Exception as e:
+        logging.error(f"Plot R2 error: {e}")
+        return None
+
+def plot_metrics_errors(result):
+    try:
+        metrics = result.get("metrics", {})
+        if not metrics:
+            return None
+        fig = go.Figure()
+        fig.add_trace(go.Bar(
+            x=['RMSE', 'MAE'],
+            y=[metrics.get('RMSE', 0), metrics.get('MAE', 0)],
+            marker_color=['#32CD32', '#9370DB']
+        ))
+        fig.update_layout(
+            title="Error Metrics",
+            template="plotly_white",
+            height=600,
+            xaxis_title="Metric",
+            yaxis_title="Value",
+            xaxis=dict(minor=dict(ticks="inside", showgrid=True), gridcolor="lightgrey"),
+            yaxis=dict(gridcolor="lightgrey"),
+            plot_bgcolor="white",
+            paper_bgcolor="white"
+        )
+        return fig
+    except Exception as e:
+        logging.error(f"Plot metrics errors: {e}")
+        return None
+
+def plot_metrics_precision_recall(result):
+    try:
+        metrics = result.get("metrics", {})
+        if not metrics:
+            return None
+        fig = go.Figure()
+        fig.add_trace(go.Bar(
+            x=['Precision', 'Recall'],
+            y=[metrics.get('Precision', 0), metrics.get('Recall', 0)],
+            marker_color=['#1E90FF', '#FF6347']
+        ))
+        fig.update_layout(
+            title="Precision and Recall Metrics",
+            template="plotly_white",
+            height=600,
+            xaxis_title="Metric",
+            yaxis_title="Value",
+            xaxis=dict(minor=dict(ticks="inside", showgrid=True), gridcolor="lightgrey"),
+            yaxis=dict(gridcolor="lightgrey"),
+            plot_bgcolor="white",
+            paper_bgcolor="white"
+        )
+        return fig
+    except Exception as e:
+        logging.error(f"Plot precision recall error: {e}")
+        return None
+
+def plot_metrics_risk(result):
+    try:
+        metrics = result.get("metrics", {})
+        if not metrics:
+            return None
+        fig = go.Figure()
+        fig.add_trace(go.Bar(
+            x=['MASE', 'Sharpe', 'Volatility'],
+            y=[metrics.get('MASE', 0), metrics.get('Sharpe', 0), metrics.get('Volatility', 0)],
+            marker_color=['#32CD32', '#9370DB', '#FFD700']
+        ))
+        fig.update_layout(
+            title="Risk Metrics",
+            template="plotly_white",
+            height=600,
+            xaxis_title="Metric",
+            yaxis_title="Value",
+            xaxis=dict(minor=dict(ticks="inside", showgrid=True), gridcolor="lightgrey"),
+            yaxis=dict(gridcolor="lightgrey"),
+            plot_bgcolor="white",
+            paper_bgcolor="white"
+        )
+        return fig
+    except Exception as e:
+        logging.error(f"Plot risk metrics error: {e}")
+        return None
+
+def plot_loss_curve(result):
+    try:
+        train_loss = result.get("train_loss", [])
+        val_loss = result.get("val_loss", [])
+        if not train_loss:
+            return None
+        epochs = list(range(1, len(train_loss) + 1))
+        fig = go.Figure()
+        fig.add_trace(go.Scatter(x=epochs, y=train_loss, mode='lines', name='Train Loss', line=dict(color='#00CC96')))
+        fig.add_trace(go.Scatter(x=epochs, y=val_loss, mode='lines', name='Validation Loss', line=dict(color='#EF553B')))
+        fig.update_layout(
+            title="Training and Validation Loss",
+            template="plotly_white",
+            height=600,
+            xaxis_title="Epoch",
+            yaxis_title="Loss",
+            xaxis=dict(minor=dict(ticks="inside", showgrid=True), gridcolor="lightgrey"),
+            yaxis=dict(gridcolor="lightgrey"),
+            plot_bgcolor="white",
+            paper_bgcolor="white"
+        )
+        return fig
+    except Exception as e:
+        logging.error(f"Plot loss curve error: {e}")
+        return None
+
+def plot_model_architecture(result):
+    try:
+        summary_text = result.get("model_summary", "No model summary available.")
+        if not summary_text or summary_text == "No model summary available.":
+            summary_text = "Model architecture summary could not be generated."
+        fig = go.Figure()
+        fig.add_annotation(
+            text=summary_text.replace('\n', '<br>'),
+            xref="paper",
+            yref="paper",
+            x=0,
+            y=1,
+            showarrow=False,
+            font=dict(size=14, family="Courier New", color="#1E90FF"),
+            align="left",
+            bgcolor="white",
+            bordercolor="black",
+            borderwidth=1,
+            width=600,
+            height=400
+        )
+        fig.update_layout(
+            title="Model Architecture",
+            template="plotly_white",
+            height=600,
+            showlegend=False,
+            xaxis=dict(visible=False),
+            yaxis=dict(visible=False),
+            plot_bgcolor="white",
+            paper_bgcolor="white"
+        )
+        return fig
+    except Exception as e:
+        logging.error(f"Plot model architecture error: {e}")
+        return None
+
+def plot_signals(signals_df, ticker):
+    try:
+        fig = go.Figure()
+        fig.add_trace(go.Scatter(
+            x=signals_df.index,
+            y=signals_df['Price'],
+            mode='lines',
+            name='Price'
+        ))
+        buy_signals = signals_df[signals_df['Signal'] == 'Buy']
+        sell_signals = signals_df[signals_df['Signal'] == 'Sell']
+
+        fig.add_trace(go.Scatter(
+            x=buy_signals.index,
+            y=buy_signals['Price'],
+            mode='markers',
+            marker=dict(symbol='triangle-up', size=10, color='green'),
+            name='Buy Signal'
+        ))
+        fig.add_trace(go.Scatter(
+            x=sell_signals.index,
+            y=sell_signals['Price'],
+            mode='markers',
+            marker=dict(symbol='triangle-down', size=10, color='red'),
+            name='Sell Signal'
+        ))
+
+        fig.update_layout(
+            title=f'{ticker} Trading Signals',
+            xaxis_title='Date',
+            yaxis_title='Price',
+            template="plotly_white",
+            xaxis_rangeslider_visible=False
+        )
+        return fig
+    except Exception as e:
+        logging.error(f"Plot signals error: {e}")
+        return None
+
+def plot_backtest(equity_df, trades_df, ticker):
+    try:
+        fig = go.Figure()
+        fig.add_trace(go.Scatter(
+            x=equity_df.index,
+            y=equity_df['Equity'],
+            mode='lines',
+            name='Equity Curve'
+        ))
+
+        buy_trades = trades_df[trades_df['Type'] == 'Buy']
+        sell_trades = trades_df[trades_df['Type'] == 'Sell']
+        exit_trades = trades_df[trades_df['Type'] == 'Exit']
+
+        fig.add_trace(go.Scatter(
+            x=buy_trades['Date'],
+            y=buy_trades['Price'],
+            mode='markers',
+            marker=dict(symbol='triangle-up', size=10, color='green'),
+            name='Buy Trades'
+        ))
+        fig.add_trace(go.Scatter(
+            x=sell_trades['Date'],
+            y=sell_trades['Price'],
+            mode='markers',
+            marker=dict(symbol='triangle-down', size=10, color='red'),
+            name='Sell Trades'
+        ))
+        fig.add_trace(go.Scatter(
+            x=exit_trades['Date'],
+            y=exit_trades['Price'],
+            mode='markers',
+            marker=dict(symbol='circle', size=8, color='blue'),
+            name='Exit Trades'
+        ))
+
+        fig.update_layout(
+            title=f'{ticker} Backtest Equity Curve and Trades',
+            xaxis_title='Date',
+            yaxis_title='Equity / Price',
+            template="plotly_white",
+            xaxis_rangeslider_visible=False
+        )
+        return fig
+    except Exception as e:
+        logging.error(f"Plot backtest error: {e}")
+        return None
+

core/signals (1).py

@@ -0,0 +1,142 @@
+import pandas as pd
+import numpy as np
+import logging
+
+logging.basicConfig(level=logging.DEBUG, filename="debug.log", filemode="a")
+
+def generate_signals(df, result, volatility_window=14):
+    try:
+        signals_df = pd.DataFrame(index=df.index)
+        signals_df["Price"] = df["value"]
+        signals_df["Signal"] = "Hold"
+        signals_df["Position_Size"] = 0.0
+        signals_df["Stop_Loss"] = np.nan
+        signals_df["Take_Profit"] = np.nan
+
+        rsi_key = "rsi_14"
+        macd_key = "macdh_12_26_9"
+        adx_key = "adx_14"
+        pdi_key = "pdi_14"
+        mdi_key = "mdi_14"
+        atr_key = "atr_14"
+        sentiment_key = "sentiment"
+
+        for i in range(1, len(df)):
+            vote = 0
+            rsi_signal = macd_signal = adx_signal = sentiment_signal = model_signal = 0
+
+            if rsi_key in df.columns and not pd.isna(df[rsi_key].iloc[i]):
+                rsi = df[rsi_key].iloc[i]
+                rsi_signal = 1 if rsi < 50 else -1 if rsi > 50 else 0
+                vote += rsi_signal
+                logging.debug(f"RSI at {df.index[i]}: value={rsi:.2f}, signal={rsi_signal}")
+
+            if macd_key in df.columns and not pd.isna(df[macd_key].iloc[i]):
+                macd = df[macd_key].iloc[i]
+                macd_prev = df[macd_key].iloc[i-1] if i > 0 else 0
+                macd_signal = 1 if macd > 0 and macd_prev <= 0 else -1 if macd < 0 and macd_prev >= 0 else 0
+                vote += macd_signal
+                logging.debug(f"MACD at {df.index[i]}: value={macd:.2f}, prev={macd_prev:.2f}, signal={macd_signal}")
+
+            if adx_key in df.columns and pdi_key in df.columns and mdi_key in df.columns:
+                adx = df[adx_key].iloc[i]
+                pdi = df[pdi_key].iloc[i]
+                mdi = df[mdi_key].iloc[i]
+                if not pd.isna(adx) and adx > 20:
+                    adx_signal = 1 if pdi > mdi else -1 if mdi > pdi else 0
+                    vote += adx_signal
+                logging.debug(f"ADX at {df.index[i]}: adx={adx:.2f}, pdi={pdi:.2f}, mdi={mdi:.2f}, signal={adx_signal}")
+
+            if sentiment_key in df.columns and not pd.isna(df[sentiment_key].iloc[i]):
+                sentiment = df[sentiment_key].iloc[i]
+                sentiment_signal = 1 if sentiment > 0.1 else -1 if sentiment < -0.1 else 0
+                vote += sentiment_signal
+                logging.debug(f"Sentiment at {df.index[i]}: value={sentiment:.2f}, signal={sentiment_signal}")
+
+            if "forecast" in result and len(result["forecast"]) > i:
+                forecast = result["forecast"][i]
+                actual = df["value"].iloc[i]
+                model_signal = 1 if forecast > actual * 1.01 else -1 if forecast < actual * 0.99 else 0
+                vote += model_signal
+                logging.debug(f"Model at {df.index[i]}: forecast={forecast:.2f}, actual={actual:.2f}, signal={model_signal}")
+
+            signals_df.loc[df.index[i], "Signal"] = "Buy" if vote >= 2 else "Sell" if vote <= -2 else "Hold"
+            signals_df.loc[df.index[i], "Position_Size"] = min(0.1 * abs(vote), 1.0)
+
+            if atr_key in df.columns and not pd.isna(df[atr_key].iloc[i]):
+                atr = df[atr_key].iloc[i]
+                signals_df.loc[df.index[i], "Stop_Loss"] = df["value"].iloc[i] - 2 * atr if vote >= 2 else df["value"].iloc[i] + 2 * atr if vote <= -2 else np.nan
+                signals_df.loc[df.index[i], "Take_Profit"] = df["value"].iloc[i] + 3 * atr if vote >= 2 else df["value"].iloc[i] - 3 * atr if vote <= -2 else np.nan
+
+            current_signal = signals_df.iloc[i]["Signal"]
+            logging.debug(f"Signal at {df.index[i]}: RSI={rsi_signal}, MACD={macd_signal}, ADX={adx_signal}, Sentiment={sentiment_signal}, Model={model_signal}, Vote={vote}, Signal={current_signal}")
+
+        trades_df, equity_df = backtest_signals(signals_df, df)
+        signals_df["Equity"] = equity_df["Equity"]
+
+        signal_counts = signals_df["Signal"].value_counts().to_dict()
+        total = sum(signal_counts.values())
+        signal_dist = {k: f"{v} ({v/total*100:.2f}%)" for k, v in signal_counts.items()}
+        signal_dist_str = ", ".join([f'{k}={v}' for k, v in signal_dist.items()])
+        logging.info(f"Signal distribution: {signal_dist_str}")
+        logging.info(f"Signals generated: {signal_counts}")
+        return signals_df, trades_df, equity_df
+    except Exception as e:
+        logging.error(f"Error in generate_signals: {e}")
+        return pd.DataFrame(), pd.DataFrame(), pd.DataFrame()
+
+def backtest_signals(signals_df, df, initial_balance=10000):
+    try:
+        balance = initial_balance
+        position = 0
+        trades = []
+        equity_curve = [balance]
+        entry_price = 0
+        
+        # Iterate through the original DataFrame's index to ensure equity_curve aligns
+        for idx, row in df.iterrows():
+            # Find the corresponding signal for this date
+            signal_row = signals_df.loc[idx] if idx in signals_df.index else None
+
+            if signal_row is not None:
+                price = signal_row["Price"]
+                signal = signal_row["Signal"]
+                position_size = signal_row["Position_Size"]
+                stop_loss = signal_row["Stop_Loss"]
+                take_profit = signal_row["Take_Profit"]
+                
+                if signal == "Buy" and position == 0:
+                    shares = position_size * balance / price
+                    position = shares
+                    entry_price = price
+                    trades.append({"Date": str(idx.date()), "Type": "Buy", "Price": price, "Shares": shares})
+                    logging.debug(f"Buy at {price:.2f}, Shares: {shares:.2f}")
+                
+                elif signal == "Sell" and position > 0:
+                    balance += position * (price - entry_price)
+                    trades.append({"Date": str(idx.date()), "Type": "Sell", "Price": price, "Shares": position, "Profit": position * (price - entry_price)})
+                    position = 0
+                    profit_val = trades[-1]["Profit"]
+                    logging.debug(f"Sell at {price:.2f}, Profit: {profit_val:.2f}")
+                
+                if position > 0 and not pd.isna(stop_loss) and not pd.isna(take_profit):
+                    if price <= stop_loss or price >= take_profit:
+                        balance += position * (price - entry_price)
+                        trades.append({"Date": str(idx.date()), "Type": "Exit", "Price": price, "Shares": position, "Profit": position * (price - entry_price)})
+                        position = 0
+                        profit_val = trades[-1]["Profit"]
+                        logging.debug(f"Exit at {price:.2f}, Profit: {profit_val:.2f}")
+            
+            current_equity = balance + position * (row["value"] - entry_price) if position > 0 else balance
+            equity_curve.append(current_equity)
+        
+        # The first element of equity_curve is the initial balance, remove it to align with df.index
+        equity_curve = equity_curve[1:]
+
+        trades_df = pd.DataFrame(trades)
+        equity_df = pd.DataFrame({"Equity": equity_curve}, index=df.index)
+        logging.info(f"Backtest completed: {len(trades)} trades, Final Balance: {balance:.2f}")
+        return trades_df, equity_df
+    except Exception as e:
+        logging.error(f"Backtest error: {e}")
+        return pd.DataFrame(), pd.DataFrame(), pd.DataFrame(), pd.DataFrame()

core/train_eval (3).py

@@ -0,0 +1,324 @@
+import numpy as np
+import pandas as pd
+import torch
+from torch import nn, optim
+from torch.utils.data import DataLoader, TensorDataset
+from sklearn.preprocessing import MinMaxScaler
+from sklearn.metrics import (
+    mean_squared_error,
+    mean_absolute_error,
+    r2_score,
+    precision_score,
+    recall_score,
+)
+from sklearn.decomposition import PCA
+from sklearn.ensemble import RandomForestRegressor
+import logging
+import torch.optim.lr_scheduler as lr_scheduler
+from io import StringIO
+import sys
+
+try:
+    from torchsummary import summary
+except Exception:
+    summary = None
+
+logging.basicConfig(
+    level=logging.DEBUG,
+    filename="/tmp/app_log.txt",
+    filemode="a",
+    format="%(asctime)s - %(levelname)s - %(message)s",
+)
+
+# ---------------- Utility metrics ----------------
+def mean_absolute_percentage_error(y_true, y_pred):
+    y_true, y_pred = np.array(y_true), np.array(y_pred)
+    non_zero = np.abs(y_true) > 0
+    if np.sum(non_zero) == 0:
+        logging.warning("All true values are zero in MAPE calculation")
+        return np.nan
+    return np.mean(np.abs((y_true[non_zero] - y_pred[non_zero]) / y_true[non_zero])) * 100
+
+def directional_accuracy(y_true, y_pred):
+    true_diff = np.diff(y_true)
+    pred_diff = np.diff(y_pred)
+    if len(true_diff) == 0:
+        logging.warning("Insufficient data for directional accuracy")
+        return np.nan
+    return np.mean(np.sign(true_diff) == np.sign(pred_diff))
+
+def mase(y_true, y_pred, y_train):
+    mae_val = mean_absolute_error(y_true, y_pred)
+    naive_mae = mean_absolute_error(y_train[1:], y_train[:-1]) if len(y_train) > 1 else np.nan
+    if naive_mae == 0:
+        logging.warning("Naive MAE is zero in MASE calculation")
+        return np.nan
+    return mae_val / naive_mae
+
+def compute_volatility(y_pred):
+    returns = np.diff(y_pred) / y_pred[:-1]
+    if len(returns) == 0:
+        logging.warning("Insufficient data for volatility calculation")
+        return np.nan
+    return np.std(returns) * np.sqrt(252)
+
+def compute_sharpe_ratio(y_pred, risk_free_rate=0.01):
+    returns = np.diff(y_pred) / y_pred[:-1]
+    if len(returns) == 0:
+        logging.warning("Insufficient data for Sharpe ratio calculation")
+        return np.nan
+    mean_return = np.mean(returns)
+    std_return = np.std(returns)
+    if std_return == 0:
+        logging.warning("Standard deviation of returns is zero in Sharpe ratio")
+        return np.nan
+    return (mean_return - risk_free_rate) / std_return
+
+def compute_precision_recall(y_true, y_pred):
+    true_diff = np.sign(np.diff(y_true))
+    pred_diff = np.sign(np.diff(y_pred))
+    if len(true_diff) == 0:
+        logging.warning("Insufficient data for precision/recall calculation")
+        return np.nan, np.nan
+    precision = precision_score(true_diff > 0, pred_diff > 0, zero_division=0)
+    recall = recall_score(true_diff > 0, pred_diff > 0, zero_division=0)
+    return precision, recall
+
+# ---------------- Feature selection ----------------
+def select_features(df, features, target, selector_method, importance_threshold):
+    logging.info(
+        f"Selecting features with method: {selector_method}, threshold: {importance_threshold}"
+    )
+    if selector_method == "RandomForest":
+        try:
+            X = df[features].dropna()
+            y = df[target].loc[X.index]
+            rf = RandomForestRegressor(n_estimators=100, random_state=42)
+            rf.fit(X, y)
+            importances = pd.Series(rf.feature_importances_, index=features)
+            selected_features = importances[importances >= importance_threshold].index.tolist()
+            logging.debug(f"RandomForest selected features: {selected_features}, importances: {importances.to_dict()}")
+            return selected_features if selected_features else features
+        except Exception as e:
+            logging.error(f"RandomForest feature selection failed: {str(e)}")
+            return features
+    elif selector_method == "PCA":
+        try:
+            X = df[features].dropna()
+            scaler = MinMaxScaler()
+            X_scaled = scaler.fit_transform(X)
+            n_components = min(len(features), X_scaled.shape[0], 10)
+            pca = PCA(n_components=n_components)
+            pca.fit(X_scaled)
+            explained_variance_ratio = pca.explained_variance_ratio_.cumsum()
+            n_selected = sum(explained_variance_ratio < 0.95) + 1 if any(explained_variance_ratio < 0.95) else n_components
+            selected_features = features[:n_selected]
+            logging.debug(f"PCA selected features: {selected_features}, explained variance: {explained_variance_ratio.tolist()}")
+            return selected_features if selected_features else features
+        except Exception as e:
+            logging.error(f"PCA feature selection failed: {str(e)}")
+            return features
+    else:
+        logging.warning(f"Unsupported selector_method: {selector_method}, using all features")
+        return features
+
+def train_and_evaluate(
+    df,
+    features,
+    target,
+    model_cls,
+    horizon=1,
+    hidden=64,
+    layers=1,
+    epochs=50,
+    lr=0.001,
+    beta1=0.9,
+    beta2=0.999,
+    weight_decay=0.01,
+    dropout=0.2,
+    window=30,
+    test_split=0.2,
+    selector_method="RandomForest",
+    importance_threshold=0.0,
+    scheduler_type="None",
+    device='cpu',
+    verbose=True
+):
+    try:
+        logging.info(f"Starting train_and_evaluate: model={model_cls.__name__}, features={len(features)}, window={window}, horizon={horizon}, scheduler={scheduler_type}, selector_method={selector_method}")
+        from .data import preprocess_data
+        
+        selected_features = select_features(df, features, target, selector_method, importance_threshold)
+        logging.info(f"Selected features: {selected_features}")
+        
+        X, y, feature_scaler, target_scaler, full_features, target_idx, pca, updated_feature_cols = preprocess_data(df, selected_features, target, window, horizon)
+        logging.debug(f"Preprocess: type(X)={type(X)}, example={X if isinstance(X, tuple) else X.shape}, type(y)={type(y)}, example={y if isinstance(y, tuple) else y.shape}")
+        
+        if X.shape[0] < 10:
+            logging.error(f"Insufficient data samples: {X.shape[0]}")
+            return {"error": f"Insufficient data samples: {X.shape[0]}"}
+
+        train_size = int((1 - test_split) * len(X))
+        X_train, X_test = X[:train_size], X[train_size:]
+        y_train, y_test = y[:train_size], y[train_size:]
+        logging.debug(f"Train size: {len(X_train)}, Test size: {len(X_test)}")
+        logging.debug(f"X_train type: {type(X_train)}, shape: {X_train.shape if isinstance(X_train, np.ndarray) else 'not ndarray'}")
+        logging.debug(f"X_test type: {type(X_test)}, shape: {X_test.shape if isinstance(X_test, np.ndarray) else 'not ndarray'}")
+
+        train_dataset = TensorDataset(torch.tensor(X_train, dtype=torch.float32).to(device), 
+                                     torch.tensor(y_train, dtype=torch.float32).to(device))
+        test_dataset = TensorDataset(torch.tensor(X_test, dtype=torch.float32).to(device), 
+                                    torch.tensor(y_test, dtype=torch.float32).to(device))
+        train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
+        test_loader = DataLoader(test_dataset, batch_size=32, shuffle=False)
+
+        # Debug DataLoader output
+        for batch_X, batch_y in train_loader:
+            logging.debug(f"DataLoader train batch: X_type={type(batch_X)}, X_shape={batch_X.shape}, y_type={type(batch_y)}, y_shape={batch_y.shape}")
+            break
+        for batch_X, batch_y in test_loader:
+            logging.debug(f"DataLoader test batch: X_type={type(batch_X)}, X_shape={batch_X.shape}, y_type={type(batch_y)}, y_shape={batch_y.shape}")
+            break
+
+        input_size = X.shape[2]
+        model = model_cls(input_size=input_size, hidden_size=hidden, num_layers=layers, output_size=horizon, dropout=dropout).to(device)
+        logging.debug(f"Model initialized: {model_cls.__name__}, input_size={input_size}, hidden={hidden}, layers={layers}")
+
+        # if verbose and summary:
+        #     try:
+        #         output = StringIO()
+        #         sys.stdout = output
+        #         summary(model, input_size=(window, input_size))
+        #         sys.stdout = sys.__stdout__
+        #         logging.debug(f"Model summary:\n{output.getvalue()}")
+        #     except Exception as e:
+        #         logging.warning(f"Failed to generate model summary: {str(e)}")
+
+        optimizer = optim.Adam(model.parameters(), lr=lr, betas=(beta1, beta2), weight_decay=weight_decay)
+        criterion = nn.MSELoss()
+        scheduler = None
+        if scheduler_type == "ReduceLROnPlateau":
+            scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=10, verbose=verbose)
+            logging.debug("Initialized ReduceLROnPlateau scheduler")
+        elif scheduler_type != "None":
+            logging.warning(f"Unsupported scheduler type: {scheduler_type}, using None")
+
+        train_losses = []
+        val_losses = []
+
+        for epoch in range(epochs):
+            model.train()
+            train_loss = 0.0
+            for batch_X, batch_y in train_loader:
+                logging.debug(f"Training Batch_X type: {type(batch_X)}, shape: {batch_X.shape}")
+                logging.debug(f"Training Batch_Y type: {type(batch_y)}, shape: {batch_y.shape}")
+                optimizer.zero_grad()
+                logging.debug(f"Training input to model: type={type(batch_X)}, example={batch_X if isinstance(batch_X, tuple) else batch_X.shape}")
+                try:
+                    outputs = model(batch_X)
+                    logging.debug(f"Training model output shape: {outputs.shape}")
+                except Exception as e:
+                    logging.error(f"Training model forward error: {str(e)}, batch_X_type={type(batch_X)}, batch_X_shape={batch_X.shape}")
+                    raise
+                loss = criterion(outputs, batch_y)
+                loss.backward()
+                optimizer.step()
+                train_loss += loss.item() * batch_X.size(0)
+            train_loss /= len(train_loader.dataset)
+            train_losses.append(train_loss)
+
+            model.eval()
+            val_loss = 0.0
+            with torch.no_grad():
+                for batch_X, batch_y in test_loader:
+                    logging.debug(f"Validation Batch_X type: {type(batch_X)}, shape: {batch_X.shape}")
+                    logging.debug(f"Validation Batch_Y type: {type(batch_y)}, shape: {batch_y.shape}")
+                    logging.debug(f"Validation input to model: type={type(batch_X)}, example={batch_X if isinstance(batch_X, tuple) else batch_X.shape}")
+                    try:
+                        outputs = model(batch_X)
+                        logging.debug(f"Validation model output shape: {outputs.shape}")
+                    except Exception as e:
+                        logging.error(f"Validation model forward error: {str(e)}, batch_X_type={type(batch_X)}, batch_X_shape={batch_X.shape}")
+                        raise
+                    loss = criterion(outputs, batch_y)
+                    val_loss += loss.item() * batch_X.size(0)
+                val_loss /= len(test_loader.dataset)
+                val_losses.append(val_loss)
+
+            if scheduler:
+                scheduler.step(val_loss)
+                current_lr = optimizer.param_groups[0]['lr']
+                logging.debug(f"Epoch {epoch+1}/{epochs}, Train Loss: {train_loss:.6f}, Val Loss: {val_loss:.6f}, LR: {current_lr}")
+            else:
+                logging.debug(f"Epoch {epoch+1}/{epochs}, Train Loss: {train_loss:.6f}, Val Loss: {val_loss:.6f}")
+
+        # ---------------- Evaluation ----------------
+        model.eval()
+        with torch.no_grad():
+            X_test_tensor = torch.tensor(X_test, dtype=torch.float32).to(device)
+            logging.debug(f"Eval model call: type={type(X_test_tensor)}, example={X_test_tensor if isinstance(X_test_tensor, tuple) else X_test_tensor.shape}")
+            try:
+                y_pred_scaled = model(X_test_tensor).cpu().numpy()
+                logging.debug(f"Eval model output shape: {y_pred_scaled.shape}")
+            except Exception as e:
+                logging.error(f"Eval model forward error: {str(e)}, X_test_type={type(X_test_tensor)}, X_test_shape={X_test_tensor.shape}")
+                raise
+
+        y_test_unscaled = target_scaler.inverse_transform(y_test.reshape(-1, horizon)).flatten()
+        y_pred_unscaled = target_scaler.inverse_transform(y_pred_scaled.reshape(-1, horizon)).flatten()
+
+        precision, recall = compute_precision_recall(y_test_unscaled, y_pred_unscaled)
+
+        metrics = {
+            "R2": float(r2_score(y_test_unscaled, y_pred_unscaled)),
+            "MAPE": float(mean_absolute_percentage_error(y_test_unscaled, y_pred_unscaled)),
+            "RMSE": float(np.sqrt(mean_squared_error(y_test_unscaled, y_pred_unscaled))),
+            "MAE": float(mean_absolute_error(y_test_unscaled, y_pred_unscaled)),
+            "DirAcc": float(directional_accuracy(y_test_unscaled, y_pred_unscaled)),
+            "MASE": float(
+                mase(
+                    y_test_unscaled,
+                    y_pred_unscaled,
+                    target_scaler.inverse_transform(y_train.reshape(-1, horizon)).flatten(),
+                )
+            ),
+            "Volatility": float(compute_volatility(y_pred_unscaled)),
+            "Sharpe": float(compute_sharpe_ratio(y_pred_unscaled)),
+            "Precision": float(np.nan if np.isnan(precision) else precision),
+            "Recall": float(np.nan if np.isnan(recall) else recall),
+        }
+
+        # Latest prediction (use last window from original X)
+        latest_data = torch.tensor(X[-1:], dtype=torch.float32).to(device)
+        with torch.no_grad():
+            logging.debug(f"Latest prediction input: type={type(latest_data)}, shape={latest_data.shape}")
+            latest_prediction_scaled = model(latest_data).cpu().numpy()
+            latest_prediction = target_scaler.inverse_transform(
+                latest_prediction_scaled.reshape(-1, horizon)
+            ).flatten()
+
+        result = {
+            "model": model,
+            "train_loss": train_losses,
+            "val_loss": val_losses,
+            "metrics": metrics,
+            "actual": y_test_unscaled,
+            "forecast": y_pred_unscaled,
+            "latest_prediction": latest_prediction,
+            "arch": {
+                "input_size": input_size,
+                "hidden": hidden,
+                "layers": layers,
+                "dropout": dropout,
+                "window": window,
+            },
+            "scalers": {"feature_scaler": feature_scaler, "target_scaler": target_scaler},
+            "features": updated_feature_cols,
+        }
+
+        logging.info("Training and evaluation completed successfully")
+        return result
+
+    except Exception as e:
+        logging.error(f"Error in train_and_evaluate: {str(e)}")
+        return {"error": str(e)}