forecasting_analysis.py

import pandas as pd
import numpy as np
from sklearn.preprocessing import MinMaxScaler
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Dense
from statsmodels.tsa.arima.model import ARIMA
import plotly.graph_objects as go
# from datetime import timedelta, date

# # --- 1. CONFIGURATION ---
# # IMPORTANT: Ensure this file path matches the location of your data file.
# FILE_PATH = 'daily_oilseeds_full_ml_dataset_2015_01_01_2025_12_02.csv'
# TARGET_PRODUCT = 'Castor' # Targeting the 'Castor' product as requested. Change this for other products.
# FORECAST_MONTH_YEAR = '2026-01' # Target month for prediction
# TEST_SIZE_RATIO = 0.20 # 20% for testing
# LOOK_BACK = 60 # Number of previous days (time steps) for LSTM to look at

# # --- HELPER FUNCTION: Convert data into sequences for LSTM ---
# def create_sequences(data, look_back):
#     """Creates lagged sequences for LSTM model training."""
#     X, Y = [], []
#     for i in range(len(data) - look_back):
#         # X is the sequence of LOOK_BACK prices
#         X.append(data[i:(i + look_back), 0])
#         # Y is the price immediately following the sequence
#         Y.append(data[i + look_back, 0])
#     return np.array(X), np.array(Y)

# # --- 2. DATA LOADING AND PREPARATION ---
# print("--- Starting Data Loading and Preprocessing ---")
# try:
#     # Load the data
#     df = pd.read_csv(FILE_PATH)
# except FileNotFoundError:
#     print(f"Error: File not found at {FILE_PATH}. Please ensure the CSV file is in the correct directory.")
#     # Exit gracefully if the file is missing
#     exit()

# # We need to find the correct date column, trying common names based on context
# DATE_COLUMN = None
# # Prioritize 'Expiry Date' based on previous structure
# if 'Expiry Date' in df.columns:
#     DATE_COLUMN = 'Expiry Date'
# # Try 'Expiry_Date' as a common alternative
# elif 'Expiry_Date' in df.columns:
#     DATE_COLUMN = 'Expiry_Date'
# # Check if the date is encoded as 'Date' or 'DATE'
# elif 'Date' in df.columns:
#     DATE_COLUMN = 'Date'
# elif 'DATE' in df.columns:
#     DATE_COLUMN = 'DATE'
# else:
#      # Fallback to general date column detection
#      date_cols = [col for col in df.columns if 'date' in col.lower() or 'expiry' in col.lower()]
#      if date_cols:
#          DATE_COLUMN = date_cols[0]
#      else:
#          print("Error: Could not find a recognizable Date column in the CSV file (looked for 'Expiry Date', 'Expiry_Date', 'Date', etc.).")
#          exit()

# print(f"Using Date Column: {DATE_COLUMN}")
# print(f"Using Target Product: {TARGET_PRODUCT}")

# # Convert Date column to datetime and filter for the target product
# df[DATE_COLUMN] = pd.to_datetime(df[DATE_COLUMN])
# df_filtered = df[df['Product'] == TARGET_PRODUCT].sort_values(by=DATE_COLUMN)

# # Select the 'Close' price as the target series and aggregate by Date
# # Aggregation is crucial if the same product has multiple entries per day (e.g., different contracts/expiry dates)
# data = df_filtered.groupby(DATE_COLUMN)['Close'].mean().to_frame()
# print(f"Total unique dates with data for {TARGET_PRODUCT}: {len(data)}")

# # Handle missing values by filling with the previous day's close price (after aggregation)
# data = data.fillna(method='ffill')

# # Resample data to a daily frequency and fill missing values for a continuous time series
# if not data.empty:
#     full_date_range = pd.date_range(start=data.index.min(), end=data.index.max(), freq='D')
#     data = data.reindex(full_date_range)
#     # Fill any NaNs introduced by reindexing (forward fill, then backward fill for initial gaps)
#     data = data.fillna(method='ffill') 
#     data = data.fillna(method='bfill')

# # Remove any remaining NaNs (e.g., if the entire series was empty)
# data = data.dropna()

# print(f"Total data points after resampling for {TARGET_PRODUCT}: {len(data)}")

# # --- 3. TIME-BASED DATA SPLITTING (80% Train / 20% Test) ---
# if len(data) == 0:
#     print("Error: Filtered and cleaned data is empty. Cannot proceed with modeling.")
#     exit()

# # Calculate the split point
# train_size = int(len(data) * (1 - TEST_SIZE_RATIO))

# # Split the data chronologically
# train_data = data[:train_size]
# test_data = data[train_size:]

# print(f"Training data size (80%): {len(train_data)} points, up to {train_data.index[-1].date()}")
# print(f"Testing data size (20%): {len(test_data)} points, starting from {test_data.index[0].date()}")
# print("---" * 15)

# # --- 4. ARIMA MODELING AND FORECAST ---
# print("--- Running ARIMA Model ---")

# arima_pred_test = pd.Series([], dtype='float64')
# arima_pred_future = pd.Series([], dtype='float64')
# forecast_dates = [] # Initialize forecast_dates for use in the LSTM section

# try:
#     # Check if train_data is not empty before fitting ARIMA model
#     if not train_data.empty:
#         # Setting 'freq' explicitly to 'D' (daily) to help ARIMA with frequency inference
#         # Using a simplified (5, 1, 0) order, which is common for initial price series fitting
#         arima_model = ARIMA(train_data['Close'], order=(5, 1, 0), freq='D')
#         arima_fit = arima_model.fit()

#         # Forecast on the 20% test data
#         if not test_data.empty:
#             arima_pred_test = arima_fit.predict(start=test_data.index[0], end=test_data.index[-1], dynamic=False)
        
#         # Determine future forecast dates (Jan 2026)
#         last_date = data.index[-1]
#         forecast_end_date = pd.to_datetime(FORECAST_MONTH_YEAR) + pd.offsets.MonthEnd(0)
#         forecast_dates = pd.date_range(start=last_date + timedelta(days=1), end=forecast_end_date, freq='D')

#         # Check if we need to make a forecast for the future
#         if len(forecast_dates) > 0:
#             arima_pred_future = arima_fit.predict(start=forecast_dates[0], end=forecast_dates[-1], dynamic=False)
#             arima_pred_future = pd.Series(arima_pred_future, index=forecast_dates)
        
#     else:
#         print("Train data is empty, skipping ARIMA model.")

# except Exception as e:
#     print(f"ARIMA Model failed to fit: {e}. Skipping ARIMA forecast for Jan 2026.")


# # --- 5. LSTM MODELING AND FORECAST ---
# print("--- Running LSTM Model ---")
# # Scaling and sequence preparation is essential for LSTMs
# lstm_pred_test = pd.Series([], dtype='float64')
# lstm_pred_future = pd.Series([], dtype='float64')

# if not data.empty and len(data) > LOOK_BACK:
#     scaler = MinMaxScaler(feature_range=(0, 1))
#     scaled_data = scaler.fit_transform(data['Close'].values.reshape(-1, 1))

#     # Split scaled data
#     train_scaled = scaled_data[:train_size]
#     test_scaled = scaled_data[train_size:]

#     # Check if there's enough data for sequence creation
#     if len(train_scaled) > LOOK_BACK and len(test_scaled) > LOOK_BACK:

#         X_train, y_train = create_sequences(train_scaled, LOOK_BACK)
#         X_test, y_test = create_sequences(test_scaled, LOOK_BACK)

#         # Reshape input to be [samples, time steps, features] = [samples, 60, 1]
#         X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
#         X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))

#         # Build and train the LSTM model
#         lstm_model = Sequential()
#         lstm_model.add(LSTM(units=50, return_sequences=True, input_shape=(LOOK_BACK, 1)))
#         lstm_model.add(LSTM(units=50, return_sequences=False))
#         lstm_model.add(Dense(units=1))
#         lstm_model.compile(optimizer='adam', loss='mean_squared_error')

#         # Train the model (simplified training for a runnable example)
#         try:
#             # epochs=5 is a small number for quick testing; increase for better accuracy
#             lstm_model.fit(X_train, y_train, epochs=5, batch_size=32, verbose=0)

#             # --- LSTM Prediction on Test Data ---
#             lstm_pred_test_scaled = lstm_model.predict(X_test, verbose=0)
#             lstm_pred_test = scaler.inverse_transform(lstm_pred_test_scaled)
#             # Ensure the index matches the length of the predictions, accounting for LOOK_BACK offset
#             lstm_pred_test = pd.Series(lstm_pred_test.flatten(), index=test_data.index[LOOK_BACK:])

#             # --- LSTM Forecast for Jan 2026 (Iterative Prediction) ---
#             if len(forecast_dates) > 0 and len(scaled_data) >= LOOK_BACK:
#                 last_look_back = scaled_data[-LOOK_BACK:]
#                 future_forecast = last_look_back
#                 future_predictions = []

#                 for _ in range(len(forecast_dates)):
#                     x_input = future_forecast.reshape((1, LOOK_BACK, 1))
#                     next_day_scaled = lstm_model.predict(x_input, verbose=0)
#                     future_predictions.append(next_day_scaled[0, 0])
#                     # Update the input sequence by appending the new prediction and dropping the oldest value
#                     future_forecast = np.append(future_forecast[1:], next_day_scaled, axis=0)

#                 lstm_pred_future = scaler.inverse_transform(np.array(future_predictions).reshape(-1, 1))
#                 lstm_pred_future = pd.Series(lstm_pred_future.flatten(), index=forecast_dates)
        
#         except Exception as e:
#             print(f"LSTM training or prediction failed: {e}")
            
#     else:
#         print("Not enough historical data to create LSTM sequences (Train/Test size is too small after split).")
# else:
#     print("Historical data is empty or too short for LSTM model.")

# print("LSTM model trained and forecast generated.")
# print("---" * 15)

# # --- 6. AGGREGATE RESULTS AND PLOT (TradingView Style with Plotly) ---
# print("--- Generating Interactive Plotly Graph ---")

# # Combine all actual and predicted values for plotting
# plot_df = pd.DataFrame({
#     'Actual Price': data['Close'],
#     'ARIMA Prediction (Test)': arima_pred_test,
#     'LSTM Prediction (Test)': lstm_pred_test,
# })

# # Add future forecasts (Jan 2026)
# plot_df = pd.concat([
#     plot_df,
#     pd.DataFrame({
#         'ARIMA Forecast (Jan 2026)': arima_pred_future,
#         'LSTM Forecast (Jan 2026)': lstm_pred_future
#     })
# ])
# plot_df = plot_df.sort_index()

# # Create Plotly figure
# fig = go.Figure()

# # --- Trace 1: Actual Historical Price (Black) ---
# fig.add_trace(go.Scatter(
#     x=plot_df.index,
#     y=plot_df['Actual Price'],
#     mode='lines',
#     name='Actual Close Price',
#     line=dict(color='black', width=2),
#     hoverinfo='x+y',
#     legendgroup='actual'
# ))

# # --- Trace 2: ARIMA Predictions (Orange) ---
# # ARIMA Test (Dotted)
# fig.add_trace(go.Scatter(
#     x=plot_df['ARIMA Prediction (Test)'].dropna().index,
#     y=plot_df['ARIMA Prediction (Test)'].dropna(),
#     mode='lines',
#     name='ARIMA Test Prediction',
#     line=dict(color='orange', width=1, dash='dot'),
#     hoverinfo='x+y',
#     legendgroup='arima'
# ))
# # ARIMA Future Forecast (Solid)
# fig.add_trace(go.Scatter(
#     x=plot_df['ARIMA Forecast (Jan 2026)'].dropna().index,
#     y=plot_df['ARIMA Forecast (Jan 2026)'].dropna(),
#     mode='lines',
#     name='ARIMA Forecast (Jan 2026)',
#     line=dict(color='orange', width=2),
#     hoverinfo='x+y',
#     legendgroup='arima'
# ))

# # --- Trace 3: LSTM Predictions (Blue) ---
# # LSTM Test (Dotted)
# fig.add_trace(go.Scatter(
#     x=plot_df['LSTM Prediction (Test)'].dropna().index,
#     y=plot_df['LSTM Prediction (Test)'].dropna(),
#     mode='lines',
#     name='LSTM Test Prediction',
#     line=dict(color='blue', width=1, dash='dot'),
#     hoverinfo='x+y',
#     legendgroup='lstm'
# ))
# # LSTM Future Forecast (Solid)
# fig.add_trace(go.Scatter(
#     x=plot_df['LSTM Forecast (Jan 2026)'].dropna().index,
#     y=plot_df['LSTM Forecast (Jan 2026)'].dropna(),
#     mode='lines',
#     name='LSTM Forecast (Jan 2026)',
#     line=dict(color='blue', width=2),
#     hoverinfo='x+y',
#     legendgroup='lstm'
# ))


# # --- Layout Configuration (TradingView Aesthetic) ---
# fig.update_layout(
#     title=f'{TARGET_PRODUCT} Price Forecasting (Actual vs. ARIMA vs. LSTM)',
#     xaxis_title='Date',
#     yaxis_title=f'{TARGET_PRODUCT} Close Price (₹)',
#     xaxis_rangeslider_visible=True, # Key TradingView-like feature
#     hovermode='x unified',
#     template='plotly_dark', # Dark theme for a TradingView-like look
#     legend=dict(
#         orientation="h",
#         yanchor="bottom",
#         y=1.02,
#         xanchor="right",
#         x=1
#     ),
#     height=600
# )

# # Add a vertical line to show the split point and the start of the forecast
# if not test_data.empty:
#     test_start_date = test_data.index[0]
#     fig.add_shape(type="line", x0=test_start_date, y0=0, x1=test_start_date, y1=1, 
#                   xref="x", yref="paper", line=dict(color="red", width=1, dash="dash"),
#                   name="Start of Test Data")
#     fig.add_annotation(x=test_start_date, y=1, text="Start of Test/Prediction Data", 
#                        showarrow=True, arrowhead=2, ax=0, ay=-40, xref="x", yref="paper", bgcolor="red", opacity=0.7)

# if len(forecast_dates) > 0:
#     forecast_start_date = forecast_dates[0]
#     fig.add_shape(type="line", x0=forecast_start_date, y0=0, x1=forecast_start_date, y1=1,
#                   xref="x", yref="paper", line=dict(color="green", width=1, dash="dash"),
#                   name="Start of Forecast")
#     fig.add_annotation(x=forecast_start_date, y=0, text="Start of Jan 2026 Forecast",
#                        showarrow=True, arrowhead=2, ax=0, ay=40, xref="x", yref="paper", bgcolor="green", opacity=0.7)


# # Use fig.write_html for local machine execution, which generates an interactive HTML file
# # You can open this file in any web browser.
# try:
#     output_filename = f'{TARGET_PRODUCT}_Price_Forecast_Chart.html'
#     fig.write_html(output_filename)
#     print(f"\nInteractive Plotly chart saved to {output_filename}")
# except Exception as e:
#     print(f"Could not save Plotly chart to HTML: {e}")

# print("--- Analysis Complete ---")
# print(f"\nPredicted prices for {TARGET_PRODUCT} in January 2026:")
# print("\nARIMA Forecast:")
# # Check if ARIMA forecast is not empty before printing
# if not arima_pred_future.empty:
#     print(arima_pred_future.to_string())
# else:
#     print("ARIMA forecast failed or no future dates were generated.")

# print("\nLSTM Forecast:")
# # Check if LSTM forecast is not empty before printing
# if not lstm_pred_future.empty:
#     print(lstm_pred_future.to_string())
# else:
#     print("LSTM forecast failed or no future dates were generated.")

import pandas as pd
import numpy as np
from sklearn.preprocessing import MinMaxScaler
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Dense
from statsmodels.tsa.arima.model import ARIMA
import plotly.graph_objects as go
from datetime import timedelta, date

# --- 1. CONFIGURATION ---
# IMPORTANT: Ensure this file path matches the location of your data file.
FILE_PATH = 'daily_oilseeds_full_ml_dataset_2015_01_01_2025_12_02.csv'
TARGET_PRODUCT = 'Castor' # Targeting the 'Castor' product as requested. Change this for other products.
TEST_SIZE_RATIO = 0.20 # 20% for testing
LOOK_BACK = 60 # Number of previous days (time steps) for LSTM to look at

# NEW FORECAST RANGE (Requested: All of 2026 with weekly breakdowns)
FORECAST_START_DATE_REQ = '2026-01-01'
FORECAST_END_DATE_REQ = '2026-12-31'

# --- HELPER FUNCTION: Convert data into sequences for LSTM ---
def create_sequences(data, look_back):
    """Creates lagged sequences for LSTM model training."""
    X, Y = [], []
    for i in range(len(data) - look_back):
        # X is the sequence of LOOK_BACK prices
        X.append(data[i:(i + look_back), 0])
        # Y is the price immediately following the sequence
        Y.append(data[i + look_back, 0])
    return np.array(X), np.array(Y)

# --- 2. DATA LOADING AND PREPARATION ---
print("--- Starting Data Loading and Preprocessing ---")
try:
    # Load the data
    df = pd.read_csv(FILE_PATH)
except FileNotFoundError:
    print(f"Error: File not found at {FILE_PATH}. Please ensure the CSV file is in the correct directory.")
    # Exit gracefully if the file is missing
    exit()

# We need to find the correct date column, trying common names based on context
DATE_COLUMN = None
# Prioritize 'Expiry Date' based on previous structure
if 'Expiry Date' in df.columns:
    DATE_COLUMN = 'Expiry Date'
# Try 'Expiry_Date' as a common alternative
elif 'Expiry_Date' in df.columns:
    DATE_COLUMN = 'Expiry_Date'
# Check if the date is encoded as 'Date' or 'DATE'
elif 'Date' in df.columns:
    DATE_COLUMN = 'Date'
elif 'DATE' in df.columns:
    DATE_COLUMN = 'DATE'
else:
     # Fallback to general date column detection
     date_cols = [col for col in df.columns if 'date' in col.lower() or 'expiry' in col.lower()]
     if date_cols:
         DATE_COLUMN = date_cols[0]
     else:
         print("Error: Could not find a recognizable Date column in the CSV file (looked for 'Expiry Date', 'Expiry_Date', 'Date', etc.).")
         exit()

print(f"Using Date Column: {DATE_COLUMN}")
print(f"Using Target Product: {TARGET_PRODUCT}")

# Convert Date column to datetime and filter for the target product
df[DATE_COLUMN] = pd.to_datetime(df[DATE_COLUMN])
df_filtered = df[df['Product'] == TARGET_PRODUCT].sort_values(by=DATE_COLUMN)

# Select the 'Close' price as the target series and aggregate by Date
# Aggregation is crucial if the same product has multiple entries per day (e.g., different contracts/expiry dates)
data = df_filtered.groupby(DATE_COLUMN)['Close'].mean().to_frame()
print(f"Total unique dates with data for {TARGET_PRODUCT}: {len(data)}")

# Handle missing values by filling with the previous day's close price (after aggregation)
data = data.fillna(method='ffill')

# Resample data to a daily frequency and fill missing values for a continuous time series
if not data.empty:
    full_date_range = pd.date_range(start=data.index.min(), end=data.index.max(), freq='D')
    data = data.reindex(full_date_range)
    # Fill any NaNs introduced by reindexing (forward fill, then backward fill for initial gaps)
    data = data.fillna(method='ffill') 
    data = data.fillna(method='bfill')

# Remove any remaining NaNs (e.g., if the entire series was empty)
data = data.dropna()

print(f"Total data points after resampling for {TARGET_PRODUCT}: {len(data)}")

# --- 3. TIME-BASED DATA SPLITTING (80% Train / 20% Test) ---
if len(data) == 0:
    print("Error: Filtered and cleaned data is empty. Cannot proceed with modeling.")
    exit()

# Calculate the split point
train_size = int(len(data) * (1 - TEST_SIZE_RATIO))

# Split the data chronologically
train_data = data[:train_size]
test_data = data[train_size:]

print(f"Training data size (80%): {len(train_data)} points, up to {train_data.index[-1].date()}")
print(f"Testing data size (20%): {len(test_data)} points, starting from {test_data.index[0].date()}")
print("---" * 15)

# --- 4. ARIMA MODELING AND FORECAST ---
print("--- Running ARIMA Model ---")

arima_pred_test = pd.Series([], dtype='float64')
arima_pred_future = pd.Series([], dtype='float64')
forecast_dates = [] # Initialize forecast_dates for use in the LSTM section

try:
    # Check if train_data is not empty before fitting ARIMA model
    if not train_data.empty:
        # Setting 'freq' explicitly to 'D' (daily) to help ARIMA with frequency inference
        # Using a simplified (5, 1, 0) order, which is common for initial price series fitting
        arima_model = ARIMA(train_data['Close'], order=(5, 1, 0), freq='D')
        arima_fit = arima_model.fit()

        # Forecast on the 20% test data
        if not test_data.empty:
            arima_pred_test = arima_fit.predict(start=test_data.index[0], end=test_data.index[-1], dynamic=False)
        
        # Determine future forecast dates (Custom Range)
        last_date = data.index[-1]
        
        # Calculate the actual forecast start date: the later of (last historical date + 1 day) or requested start date
        forecast_start_date = max(last_date + timedelta(days=1), pd.to_datetime(FORECAST_START_DATE_REQ))
        forecast_end_date = pd.to_datetime(FORECAST_END_DATE_REQ)
        
        # Generate the date range for the future forecast
        if forecast_start_date <= forecast_end_date:
             forecast_dates = pd.date_range(start=forecast_start_date, end=forecast_end_date, freq='D')

        # Check if we need to make a forecast for the future
        if len(forecast_dates) > 0:
            arima_pred_future = arima_fit.predict(start=forecast_dates[0], end=forecast_dates[-1], dynamic=False)
            arima_pred_future = pd.Series(arima_pred_future, index=forecast_dates)
        
    else:
        print("Train data is empty, skipping ARIMA model.")

except Exception as e:
    print(f"ARIMA Model failed to fit: {e}. Skipping ARIMA forecast.")


# --- 5. LSTM MODELING AND FORECAST ---
print("--- Running LSTM Model ---")
# Scaling and sequence preparation is essential for LSTMs
lstm_pred_test = pd.Series([], dtype='float64')
lstm_pred_future = pd.Series([], dtype='float64')

if not data.empty and len(data) > LOOK_BACK:
    scaler = MinMaxScaler(feature_range=(0, 1))
    scaled_data = scaler.fit_transform(data['Close'].values.reshape(-1, 1))

    # Split scaled data
    train_scaled = scaled_data[:train_size]
    test_scaled = scaled_data[train_size:]

    # Check if there's enough data for sequence creation
    if len(train_scaled) > LOOK_BACK and len(test_scaled) > LOOK_BACK:

        X_train, y_train = create_sequences(train_scaled, LOOK_BACK)
        X_test, y_test = create_sequences(test_scaled, LOOK_BACK)

        # Reshape input to be [samples, time steps, features] = [samples, 60, 1]
        X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
        X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))

        # Build and train the LSTM model
        lstm_model = Sequential()
        lstm_model.add(LSTM(units=50, return_sequences=True, input_shape=(LOOK_BACK, 1)))
        lstm_model.add(LSTM(units=50, return_sequences=False))
        lstm_model.add(Dense(units=1))
        lstm_model.compile(optimizer='adam', loss='mean_squared_error')

        # Train the model (simplified training for a runnable example)
        try:
            # epochs=5 is a small number for quick testing; increase for better accuracy
            lstm_model.fit(X_train, y_train, epochs=5, batch_size=32, verbose=0)

            # --- LSTM Prediction on Test Data ---
            lstm_pred_test_scaled = lstm_model.predict(X_test, verbose=0)
            lstm_pred_test = scaler.inverse_transform(lstm_pred_test_scaled)
            # Ensure the index matches the length of the predictions, accounting for LOOK_BACK offset
            lstm_pred_test = pd.Series(lstm_pred_test.flatten(), index=test_data.index[LOOK_BACK:])

            # --- LSTM Forecast for Custom Range (Iterative Prediction) ---
            if len(forecast_dates) > 0 and len(scaled_data) >= LOOK_BACK:
                last_look_back = scaled_data[-LOOK_BACK:]
                future_forecast = last_look_back
                future_predictions = []

                for _ in range(len(forecast_dates)):
                    x_input = future_forecast.reshape((1, LOOK_BACK, 1))
                    next_day_scaled = lstm_model.predict(x_input, verbose=0)
                    future_predictions.append(next_day_scaled[0, 0])
                    # Update the input sequence by appending the new prediction and dropping the oldest value
                    future_forecast = np.append(future_forecast[1:], next_day_scaled, axis=0)

                lstm_pred_future = scaler.inverse_transform(np.array(future_predictions).reshape(-1, 1))
                lstm_pred_future = pd.Series(lstm_pred_future.flatten(), index=forecast_dates)
        
        except Exception as e:
            print(f"LSTM training or prediction failed: {e}")
            
    else:
        print("Not enough historical data to create LSTM sequences (Train/Test size is too small after split).")
else:
    print("Historical data is empty or too short for LSTM model.")

print("LSTM model trained and forecast generated.")
print("---" * 15)

# --- 6. AGGREGATE RESULTS AND PLOT (TradingView Style with Plotly) ---
print("--- Generating Interactive Plotly Graph ---")

# Combine all actual and predicted values for plotting
plot_df = pd.DataFrame({
    'Actual Price': data['Close'],
    'ARIMA Prediction (Test)': arima_pred_test,
    'LSTM Prediction (Test)': lstm_pred_test,
})

# Add future forecasts (Custom Range)
plot_df = pd.concat([
    plot_df,
    pd.DataFrame({
        'ARIMA Forecast (Custom)': arima_pred_future,
        'LSTM Forecast (Custom)': lstm_pred_future
    })
])
plot_df = plot_df.sort_index()

# Create Plotly figure
fig = go.Figure()

# --- Trace 1: Actual Historical Price (Black) ---
fig.add_trace(go.Scatter(
    x=plot_df.index,
    y=plot_df['Actual Price'],
    mode='lines',
    name='Actual Close Price',
    line=dict(color='black', width=2),
    hoverinfo='x+y',
    legendgroup='actual'
))

# --- Trace 2: ARIMA Predictions (Orange) ---
# ARIMA Test (Dotted)
fig.add_trace(go.Scatter(
    x=plot_df['ARIMA Prediction (Test)'].dropna().index,
    y=plot_df['ARIMA Prediction (Test)'].dropna(),
    mode='lines',
    name='ARIMA Test Prediction',
    line=dict(color='orange', width=1, dash='dot'),
    hoverinfo='x+y',
    legendgroup='arima'
))
# ARIMA Future Forecast (Solid)
fig.add_trace(go.Scatter(
    x=plot_df['ARIMA Forecast (Custom)'].dropna().index,
    y=plot_df['ARIMA Forecast (Custom)'].dropna(),
    mode='lines',
    name='ARIMA Forecast (Dec 2025 - Jan 2026)',
    line=dict(color='orange', width=2),
    hoverinfo='x+y',
    legendgroup='arima'
))

# --- Trace 3: LSTM Predictions (Blue) ---
# LSTM Test (Dotted)
fig.add_trace(go.Scatter(
    x=plot_df['LSTM Prediction (Test)'].dropna().index,
    y=plot_df['LSTM Prediction (Test)'].dropna(),
    mode='lines',
    name='LSTM Test Prediction',
    line=dict(color='blue', width=1, dash='dot'),
    hoverinfo='x+y',
    legendgroup='lstm'
))
# LSTM Future Forecast (Solid)
fig.add_trace(go.Scatter(
    x=plot_df['LSTM Forecast (Custom)'].dropna().index,
    y=plot_df['LSTM Forecast (Custom)'].dropna(),
    mode='lines',
    name='LSTM Forecast (Dec 2025 - Jan 2026)',
    line=dict(color='blue', width=2),
    hoverinfo='x+y',
    legendgroup='lstm'
))


# --- Layout Configuration (TradingView Aesthetic) ---
forecast_range_label = f"Dec 2025 to Jan 2026 Forecast ({TARGET_PRODUCT})"
fig.update_layout(
    title=f'{TARGET_PRODUCT} Price Forecasting (Actual vs. Models)',
    xaxis_title='Date',
    yaxis_title=f'{TARGET_PRODUCT} Close Price (₹)',
    xaxis_rangeslider_visible=True, # Key TradingView-like feature
    hovermode='x unified',
    template='plotly', # White background theme
    plot_bgcolor='white',
    paper_bgcolor='white',
    legend=dict(
        orientation="h",
        yanchor="bottom",
        y=1.02,
        xanchor="right",
        x=1
    ),
    height=600
)

# Add a vertical line to show the split point (Start of Test Data)
if not test_data.empty:
    test_start_date = test_data.index[0]
    fig.add_shape(type="line", x0=test_start_date, y0=0, x1=test_start_date, y1=1, 
                  xref="x", yref="paper", line=dict(color="red", width=1, dash="dash"),
                  name="Start of Test Data")
    fig.add_annotation(x=test_start_date, y=1, text="Start of Test/Prediction Data", 
                       showarrow=True, arrowhead=2, ax=0, ay=-40, xref="x", yref="paper", 
                       bgcolor="rgba(255, 0, 0, 0.5)", bordercolor="red", borderwidth=1, opacity=0.8, font=dict(color="white"))

# Add a vertical line to show the start of the future forecast range
if len(forecast_dates) > 0:
    forecast_start_date_plot = forecast_dates[0]
    fig.add_shape(type="line", x0=forecast_start_date_plot, y0=0, x1=forecast_start_date_plot, y1=1,
                  xref="x", yref="paper", line=dict(color="green", width=1, dash="dash"),
                  name="Start of Forecast")
    fig.add_annotation(x=forecast_start_date_plot, y=0, text=f"Start of {FORECAST_START_DATE_REQ} Forecast",
                       showarrow=True, arrowhead=2, ax=0, ay=40, xref="x", yref="paper", 
                       bgcolor="rgba(0, 128, 0, 0.5)", bordercolor="green", borderwidth=1, opacity=0.8, font=dict(color="white"))


# Use fig.write_html for local machine execution, which generates an interactive HTML file
# You can open this file in any web browser.
try:
    output_filename = f'{TARGET_PRODUCT}_Price_Forecast_Chart_Custom_Range.html'
    fig.write_html(output_filename)
    print(f"\nInteractive Plotly chart saved to {output_filename}")
except Exception as e:
    print(f"Could not save Plotly chart to HTML: {e}")

print("--- Analysis Complete ---")
print(f"\nPredicted prices for {TARGET_PRODUCT} from {FORECAST_START_DATE_REQ} to {FORECAST_END_DATE_REQ}:")
print("\nARIMA Forecast:")
# Check if ARIMA forecast is not empty before printing
if not arima_pred_future.empty:
    print(arima_pred_future.to_string())
else:
    print("ARIMA forecast failed or no future dates were generated.")

print("\nLSTM Forecast:")
# Check if LSTM forecast is not empty before printing
if not lstm_pred_future.empty:
    print(lstm_pred_future.to_string())
else:
    print("LSTM forecast failed or no future dates were generated.")