app.py

# New version ---
import streamlit as st
import pandas as pd
import os
import numpy as np
from datetime import date, datetime
import plotly.graph_objects as go
import itertools
import json
# --- CORRECTED IMPORTS: Moved MACD ---
from ta.volatility import BollingerBands
from ta.momentum import RSIIndicator # Removed MACD from here
from ta.trend import ADXIndicator, MACD # Added MACD here
# --- [NEW] ADDED ATR FOR INTELLIGENT EXIT ---
from ta.volatility import AverageTrueRange
# --- [END NEW] ---
from multiprocessing import Pool, cpu_count
from functools import partial
from dateutil.relativedelta import relativedelta
from datetime import timedelta

# --- 0. Settings Management Functions ---
CONFIG_FILE = "config.json"
VETO_CONFIG_FILE = "veto_config.json"
TOP_SETUPS_FILE = "top_setups.json"
USER_SETUPS_FILE = "user_advisor_setups.json"
MARKOV_SETUP_FILE = "best_markov.json"

def save_settings(params_to_save):
    with open(CONFIG_FILE, 'w', encoding='utf-8') as f:
        json.dump(params_to_save, f, indent=4)
    st.sidebar.success("Settings saved as default!")

# --- Replacement for load_settings function ---
def load_settings():
    default_structure = {
        "large_ma_period": 70, 
        "bband_period": 32, 
        "bband_std_dev": 1.4,
        "confidence_threshold": 95,
        
        "long_entry_threshold_pct": 0.01, 
        "long_exit_ma_threshold_pct": 0.0, 
        "long_trailing_stop_loss_pct": 0.2, 
        "long_delay_days": 0,
        
        "short_entry_threshold_pct": 0.0, 
        "short_exit_ma_threshold_pct": 0.0, 
        "short_trailing_stop_loss_pct": 0.2, 
        "short_delay_days": 0,
        
        "use_rsi": True, "rsi_w": 1.5,
        "rsi_logic": "Level",
        "primary_driver": "Bollinger Bands",
        
        "exit_logic_type": "Intelligent (ADX/MACD/ATR)", 
        "exit_confidence_threshold": 50, 
        "smart_trailing_stop_pct": 5.0, 
        "smart_exit_atr_period": 14,
        "smart_exit_atr_multiplier": 3.0,
        "intelligent_tsl_pct": 0.4,
        "use_ma_floor_filter": False,
        
        "use_vol": False, "vol_w": 0.5,
        "use_trend": False, "trend_w": 2.0,
        "use_volume": False, "volume_w": 0.5,
        
        "use_adx_filter": False, "adx_threshold": 10.0,
        "adx_period": 14,
        
        "use_macd": False, "macd_w": 2.0,
        "use_ma_slope": False, "ma_slope_w": 0.5,
        "use_markov": False, "markov_w": 1.0, 
        
        "max_trading_days": 60,
        "max_long_duration": 60,
        "max_short_duration": 3
    }

    if os.path.exists(CONFIG_FILE):
        try:
            with open(CONFIG_FILE, 'r', encoding='utf-8') as f:
                loaded = json.load(f)
                for key in default_structure:
                    if key in loaded:
                        default_structure[key] = loaded[key]
        except (json.JSONDecodeError, Exception) as e:
            print(f"Error loading config.json: {e}. Using default settings.")
    return default_structure

# --- UPDATED: Save a list of veto setups ---
def save_veto_setup(veto_setups_list): # Takes a list now
    with open(VETO_CONFIG_FILE, 'w', encoding='utf-8') as f:
        # Save the list directly
        json.dump(veto_setups_list, f, indent=4)
    st.sidebar.success(f"Saved {len(veto_setups_list)} Veto filter(s) as default!")

# --- UPDATED: Load a list of veto setups, robustly ---
def load_veto_setup():
    veto_list = [] # Default to empty list
    if os.path.exists(VETO_CONFIG_FILE):
        try:
            with open(VETO_CONFIG_FILE, 'r', encoding='utf-8') as f:
                loaded_data = json.load(f)
                # Ensure it's loaded as a list
                if isinstance(loaded_data, list):
                    veto_list = loaded_data
                elif isinstance(loaded_data, dict): # Handle old single dict format
                    veto_list = [loaded_data]
        except (json.JSONDecodeError, Exception) as e:
            print(f"Error loading veto config: {e}. Using empty list.")
            # Keep veto_list as empty
            pass
    return veto_list

# --- UPDATED: Includes new factors, num_setups=8, removed infinity filter ---
def save_top_setups(results_df, side, num_setups=8): # Default is now 8
    df = results_df.copy()

    if df.empty:
        st.sidebar.warning(f"No valid setups found for {side.title()} to save.")
        return

    deduplication_cols = [
        'Conf. Threshold', 'Avg Profit/Trade', 'Ticker G/B Ratio', 'Trade G/B Ratio',
        'Winning Tickers', 'Losing Tickers', 'Avg Entry Conf.',
        'Good Score', 'Bad Score', 'Norm. Score %', 'Total Trades'
    ]

    factor_cols = ['RSI', 'Volatility', 'TREND', 'Volume', 'MACD', 'MA Slope']
    existing_factor_cols = [col for col in factor_cols if col in df.columns]
    if existing_factor_cols:
        df['FactorsOn'] = df[existing_factor_cols].apply(lambda row: (row == 'On').sum(), axis=1)
    else:
        df['FactorsOn'] = 0

    # --- CONSISTENCY FIX: Sort by the Weighted Score ("Norm. Score %") ---
    # If Norm. Score % exists, use it. Otherwise fall back to Strategy Score or Trade G/B Ratio.
    sort_col = 'Trade G/B Ratio' # Default fallback
    if 'Norm. Score %' in df.columns:
        sort_col = 'Norm. Score %'
    elif 'Strategy Score' in df.columns:
        sort_col = 'Strategy Score'

    if sort_col in df.columns:
        df[sort_col] = df[sort_col].fillna(-np.inf)
        # Sort by Score (Descending), then by Complexity (Ascending - simpler is better)
        df = df.sort_values(
            by=[sort_col, 'FactorsOn'],
            ascending=[False, True]
        )
    else:
        st.sidebar.error(f"Could not sort top setups: '{sort_col}' column missing.")
        return

    existing_cols_for_dedup = [col for col in deduplication_cols if col in df.columns]
    deduplicated_df = df.drop_duplicates(subset=existing_cols_for_dedup, keep='first')

    top_setups = deduplicated_df.head(num_setups).to_dict('records')

    if os.path.exists(TOP_SETUPS_FILE):
        try:
            with open(TOP_SETUPS_FILE, 'r', encoding='utf-8') as f:
                all_top_setups = json.load(f)
        except json.JSONDecodeError:
            all_top_setups = {}
    else:
        all_top_setups = {}

    all_top_setups[side] = top_setups

    with open(TOP_SETUPS_FILE, 'w', encoding='utf-8') as f:
        json.dump(all_top_setups, f, indent=4)

    st.sidebar.success(f"Top {len(top_setups)} unique {side.title()} setups saved! (Sorted by {sort_col})")

def load_top_setups():
    if os.path.exists(TOP_SETUPS_FILE):
        try:
            with open(TOP_SETUPS_FILE, 'r', encoding='utf-8') as f:
                return json.load(f)
        except json.JSONDecodeError:
             return None # Return None if file is corrupt
    return None

@st.cache_data # Use caching for efficiency
def load_proverbs():
    # Default to an empty list
    proverbs_list = []
    if os.path.exists("proverbs.json"):
        try:
            with open("proverbs.json", 'r', encoding='utf-8') as f:
                loaded_data = json.load(f)
                # Ensure it's a list
                if isinstance(loaded_data, list):
                    proverbs_list = loaded_data
        except (json.JSONDecodeError, Exception) as e:
            print(f"Error loading proverbs.json: {e}. Using fallback.")
            # Keep proverbs_list empty or provide a default
            # proverbs_list = ["Error loading proverbs."]
    # Add a fallback if the list is empty after trying to load
    if not proverbs_list:
        proverbs_list = ["Have a great trading day!"] # Fallback message
    return proverbs_list

def load_user_setups():
    """Loads user-defined advisor setups from a JSON file."""
    
    # 1. Update Template to include Notes
    default_row_template = {
        "Run": False,
        "Notes": "", # <--- NEW FIELD FOR ADVICE
        "RSI": "Off", "Volatility": "Off", "TREND": "Off", "Volume": "Off", 
        "MACD": "Off", "MA Slope": "Off", "Markov": "Off", 
        "ADX Filter": "Off",
        "Conf. Threshold": 50,
        "Large MA Period": 50, "Bollinger Band Period": 20, "Bollinger Band Std Dev": 2.0,
        "Long Entry Threshold (%)": 0.0, "Long Exit Threshold (%)": 0.0, "Long Stop Loss (%)": 8.0, "Long Delay (Days)": 0,
        "Short Entry Threshold (%)": 0.0, "Short Exit Threshold (%)": 0.0, "Short Stop Loss (%)": 8.0, "Short Delay (Days)": 0,
        "Z_Avg_Profit": 0.0, "Z_Num_Trades": 0, "Z_WL_Ratio": 0.0
    }

    # Default Examples
    default_setup = [default_row_template.copy()]

    # Pad with defaults
    while len(default_setup) < 20:
        default_setup.append(default_row_template.copy())

    if os.path.exists(USER_SETUPS_FILE):
        try:
            with open(USER_SETUPS_FILE, 'r', encoding='utf-8') as f:
                user_setups = json.load(f)
                if isinstance(user_setups, list):
                    processed_setups = []
                    for setup in user_setups:
                        full_setup = default_row_template.copy()
                        full_setup.update(setup) # Merge loaded data
                        processed_setups.append(full_setup)

                    while len(processed_setups) < 20:
                        processed_setups.append(default_row_template.copy())
                    return processed_setups[:20]
                else:
                    return default_setup
        except Exception as e:
            print(f"Error loading {USER_SETUPS_FILE}: {e}. Using defaults.")
            return default_setup
    return default_setup

# --- UPDATED: Callback to Save Setup (Added Notes) ---
def add_setup_to_user_list():
    try:
        stats = st.session_state.get('last_run_stats', {})
        if not stats:
            st.toast("⚠️ No stats found. Run analysis first.", icon="⚠️")
            return

        def get_weight_or_off(toggle_key, weight_key):
            if st.session_state.get(toggle_key, False):
                return round(st.session_state.get(weight_key, 1.0), 2)
            return "Off"

        adx_val = "Off"
        if st.session_state.get("use_adx_filter", False):
            val = st.session_state.get("adx_threshold", 25.0)
            adx_val = max(20.0, min(30.0, val))

        new_setup = {
            "Run": True,
            "Notes": "Auto-Saved Setup", # <--- Default note for auto-saves
            "RSI": get_weight_or_off('use_rsi', 'rsi_w'),
            "Volatility": get_weight_or_off('use_vol', 'vol_w'),
            "TREND": get_weight_or_off('use_trend', 'trend_w'),
            "Volume": get_weight_or_off('use_volume', 'volume_w'),
            "MACD": get_weight_or_off('use_macd', 'macd_w'),
            "MA Slope": get_weight_or_off('use_ma_slope', 'ma_slope_w'),
            "Markov": get_weight_or_off('use_markov', 'markov_w'),
            "ADX Filter": adx_val,

            "Conf. Threshold": st.session_state.confidence_slider,
            "Large MA Period": st.session_state.ma_period,
            "Bollinger Band Period": st.session_state.bb_period,
            "Bollinger Band Std Dev": st.session_state.bb_std,
            
            "Long Entry Threshold (%)": st.session_state.long_entry,
            "Long Exit Threshold (%)": st.session_state.long_exit,
            "Long Stop Loss (%)": st.session_state.long_sl,
            "Long Delay (Days)": st.session_state.long_delay,
            
            "Short Entry Threshold (%)": st.session_state.short_entry,
            "Short Exit Threshold (%)": st.session_state.short_exit,
            "Short Stop Loss (%)": st.session_state.short_sl,
            "Short Delay (Days)": st.session_state.short_delay,
            
            "Z_Avg_Profit": stats.get("Z_Avg_Profit", 0.0) * 100.0,
            "Z_Num_Trades": stats.get("Z_Num_Trades", 0),
            "Z_WL_Ratio": stats.get("Z_WL_Ratio", 0.0)
        }
        
        current_setups = st.session_state.get("user_setups_data", [])
        non_empty_setups = [s for s in current_setups if not is_row_blank(s)]
        default_row_template = {k:v for k,v in load_user_setups()[0].items()}
        non_empty_setups.append(new_setup)
        while len(non_empty_setups) < 20: non_empty_setups.append(default_row_template.copy())
            
        final_setups = non_empty_setups[:20]
        save_user_setups(final_setups)
        
        # Refresh State
        processed_setups = []
        for s in final_setups:
            processed_setups.append(s.copy()) # Simple copy is enough now
        st.session_state["user_setups_data"] = processed_setups
        
        st.toast("✅ Setup saved!", icon="✅")
        st.session_state.run_user_advisor_setup = True
        st.rerun()

    except Exception as e:
        st.error(f"Could not save setup: {e}")

def save_user_setups(setups_list):
    """Saves user-defined advisor setups to a JSON file."""
    try:
        # Ensure we only save 20
        with open(USER_SETUPS_FILE, 'w', encoding='utf-8') as f:
            json.dump(setups_list[:20], f, indent=4) # <-- CHANGED TO 20
        st.success("User-defined setups saved!")
    except Exception as e:
        st.error(f"Error saving user setups: {e}")

def save_markov_setup(setup_dict):
    """Saves the best Markov setup to a JSON file."""
    try:
        with open(MARKOV_SETUP_FILE, 'w', encoding='utf-8') as f:
            json.dump(setup_dict, f, indent=4)
        st.sidebar.success("Best Markov setup saved as default!")
    except Exception as e:
        st.sidebar.error(f"Error saving Markov setup: {e}")

def load_markov_setup():
    """Loads the best Markov setup from a JSON file."""
    if os.path.exists(MARKOV_SETUP_FILE):
        try:
            with open(MARKOV_SETUP_FILE, 'r', encoding='utf-8') as f:
                return json.load(f)
        except Exception as e:
            print(f"Error loading {MARKOV_SETUP_FILE}: {e}")
            return None
    return None

# --- UPDATED: Helper to accept filename (Replaces the old load_completed_setups) ---
def load_completed_setups(filename):
    """Reads the existing CSV and returns a set of completed configurations."""
    completed_configs = set()
    if os.path.exists(filename):
        try:
            # We only need to read the config columns: Factors and Weights
            df_completed = pd.read_csv(filename, usecols=['RSI', 'Volatility', 'TREND', 'Volume', 'MACD', 'MA Slope', 'Markov',
                                                          'RSI W', 'Volatility W', 'TREND W', 'Volume W', 'MACD W', 'MA Slope W', 'Markov W'], 
                                       dtype={'RSI': str, 'Volatility': str, 'TREND': str, 'Volume': str, 'MACD': str, 'MA Slope': str, 'Markov': str})
            
            for index, row in df_completed.iterrows():
                toggles = tuple(row[['RSI', 'Volatility', 'TREND', 'Volume', 'MACD', 'MA Slope', 'Markov']].values)
                weights = tuple(row[['RSI W', 'Volatility W', 'TREND W', 'Volume W', 'MACD W', 'MA Slope W', 'Markov W']].values)
                completed_configs.add((toggles, weights))
        except Exception as e:
            print(f"Warning: Error loading CSV for checkpointing: {e}. Starting from scratch.")
    return completed_configs

# --- 1. Data Loading and Cleaning Functions ---
@st.cache_data
def load_all_data(folder_path):
    all_files = [f for f in os.listdir(folder_path) if f.endswith('.csv')]
    if not all_files:
        st.error("No CSV files found in the 'csv_data' folder.")
        return None, None

    df_list = []
    error_messages = []

    for file_name in all_files:
        file_path = os.path.join(folder_path, file_name)
        try:
            # Using dayfirst=True, parse_dates=True, index_col=0
            df = pd.read_csv(file_path, header=0, index_col=0, dayfirst=True, parse_dates=True, encoding='utf-8')
            if df.empty or df.index.empty:
                error_messages.append(f"Warning: Skipped {file_name}, no data loaded or index missing.")
                continue
            df_list.append(df)
        except Exception as e:
            error_messages.append(f"Could not read or process {file_name}. Error: {e}")

    if not df_list:
        for msg in error_messages: st.error(msg)
        return None, "No data could be loaded successfully from the CSV files."

    for msg in error_messages: st.warning(msg)

    # --- REVERTED: Use pd.concat without axis=1 (defaults to axis=0 - stacking rows) ---
    master_df = pd.concat(df_list)
    # --- END REVERT ---

    master_df.index = pd.to_datetime(master_df.index, errors='coerce')
    master_df = master_df[master_df.index.notna()]

    # Handle duplicate index entries (keep last) AFTER concatenating rows
    if master_df.index.has_duplicates:
        master_df = master_df.loc[~master_df.index.duplicated(keep='last')]

    master_df.sort_index(inplace=True)

    # Convert columns to numeric *after* all concatenation and sorting
    for col in master_df.columns:
        master_df[col] = pd.to_numeric(master_df[col], errors='coerce')

    return master_df, f"Successfully combined data from {len(df_list)} files."

def clean_data_and_report_outliers(df):
    outlier_report = []
    # Identify price columns (exclude _Volume, _High, _Low if present)
    price_columns = [col for col in df.columns if not ('_Volume' in str(col) or '_High' in str(col) or '_Low' in str(col))]

    for ticker in price_columns:
        # Check if column exists
        if ticker in df.columns:
            # Ensure it's numeric before pct_change
            numeric_prices = pd.to_numeric(df[ticker], errors='coerce').replace(0, np.nan)

            if numeric_prices.isna().all(): continue # Skip fully NaN columns

            daily_pct_change = numeric_prices.pct_change().abs()
            outlier_days = daily_pct_change[daily_pct_change > 1.0].index
            if not outlier_days.empty:
                outlier_report.append({'Ticker': ticker, 'Outliers Removed': len(outlier_days)})
                df.loc[outlier_days, ticker] = np.nan # Set outliers to NaN
    return df, outlier_report

# --- 2. Custom Backtesting Engine ---

# --- [NEW] 7-Factor Version (with Markov State and RAW GRADIENT scoring) ---
def calculate_confidence_score(df, primary_driver, # <-- This exists
                                use_rsi, use_volatility, use_trend, use_volume, use_macd, use_ma_slope, use_markov, # 7 toggles
                                rsi_w, vol_w, trend_w, vol_w_val, macd_w, ma_slope_w, markov_w, # 7 weights
                                bband_params,
                                best_markov_setup=None):

    long_score = pd.Series(0.0, index=df.index)
    short_score = pd.Series(0.0, index=df.index)

    # --- Pre-calculate Markov State if needed ---
    if use_markov and best_markov_setup and 'RunUp_State' not in df.columns:
        run_up_period = best_markov_setup.get('Run-Up Period', 10)
        df['RunUp_Return'] = df['Close'].pct_change(periods=run_up_period)
        df['RunUp_State'] = df['RunUp_Return'].apply(lambda x: 'Up' if x > 0 else 'Down')

    # --- BBand Factor (Still uses gradient logic as it's a "how far" metric) ---
    if primary_driver != 'Bollinger Bands' and 'bband_lower' in df.columns:
        bb_weight = 1.0 # This factor has a fixed weight of 1.0
        long_entry_pct = bband_params.get('long_entry_threshold_pct', 0.0)
        short_entry_pct = bband_params.get('short_entry_threshold_pct', 0.0)
        long_bb_trigger_price = df['bband_lower'] * (1 - long_entry_pct)
        short_bb_trigger_price = df['bband_upper'] * (1 + short_entry_pct)
        long_score_range = (df['large_ma'] - long_bb_trigger_price).replace(0, np.nan)
        short_score_range = (short_bb_trigger_price - df['large_ma']).replace(0, np.nan)
        long_score_pct = ((df['large_ma'] - df['Close']) / long_score_range).clip(0, 1).fillna(0)
        short_score_pct = ((df['Close'] - df['large_ma']) / short_score_range).clip(0, 1).fillna(0)
        long_score += long_score_pct * bb_weight
        short_score += short_score_pct * bb_weight

    # --- [RESTORED] Gradient Factor Scoring ---

    # Factor 1: RSI
    if primary_driver != 'RSI Crossover' and use_rsi and 'RSI' in df.columns and not df['RSI'].isna().all():
        long_score += ((30 - df['RSI']) / 30).clip(0, 1).fillna(0) * rsi_w
        short_score += ((df['RSI'] - 70) / 30).clip(0, 1).fillna(0) * rsi_w

    # Factor 2: Volatility
    if use_volatility and 'Volatility_p' in df.columns and not df['Volatility_p'].isna().all():
        # This one is binary
        vol_signal = (df['Volatility_p'] > 0.025).astype(float) * vol_w
        long_score += vol_signal
        short_score += vol_signal

    # Factor 3: Trend (Distance from SMA200)
    if use_trend and 'SMA_200' in df.columns and 'Close' in df.columns \
            and not df['SMA_200'].isna().all() and not df['Close'].isna().all():
        valid_sma = (df['SMA_200'] != 0) & df['SMA_200'].notna() & df['Close'].notna()
        pct_dist = pd.Series(0.0, index=df.index)
        pct_dist.loc[valid_sma] = df.loc[valid_sma].apply(lambda row: (row['Close'] - row['SMA_200']) / row['SMA_200'] if row['SMA_200'] != 0 else 0, axis=1)
        long_score += (pct_dist / 0.10).clip(0, 1).fillna(0) * trend_w
        short_score += (-pct_dist / 0.10).clip(0, 1).fillna(0) * trend_w

    # Factor 4: Volume
    if use_volume and 'Volume_Ratio' in df.columns and not df['Volume_Ratio'].isna().all():
        # This one is a gradient
        vol_spike_signal = ((df['Volume_Ratio'] - 1.75) / 2.25).clip(0, 1).fillna(0) * vol_w_val
        long_score += vol_spike_signal
        short_score += vol_spike_signal

    # Factor 5: MACD Signals
    if primary_driver != 'MACD Crossover' and use_macd and 'MACD_line' in df.columns and 'MACD_signal' in df.columns and 'MACD_hist' in df.columns \
            and not df['MACD_line'].isna().all() and not df['MACD_signal'].isna().all() and not df['MACD_hist'].isna().all():
        # This is a hybrid gradient/binary
        macd_cross_long = (df['MACD_line'].shift(1) < df['MACD_signal'].shift(1)) & (df['MACD_line'] >= df['MACD_signal'])
        macd_cross_short = (df['MACD_line'].shift(1) > df['MACD_signal'].shift(1)) & (df['MACD_line'] <= df['MACD_signal'])
        long_score += macd_cross_long.astype(float) * macd_w * 0.6
        short_score += macd_cross_short.astype(float) * macd_w * 0.6
        hist_positive = df['MACD_hist'] > 0
        hist_negative = df['MACD_hist'] < 0
        long_score += hist_positive.astype(float) * macd_w * 0.4
        short_score += hist_negative.astype(float) * macd_w * 0.4

    # Factor 6: MA Slope
    if primary_driver != 'MA Slope' and use_ma_slope and 'ma_slope' in df.columns and not df['ma_slope'].isna().all():
        # This is binary
        long_score += (df['ma_slope'] > 0).astype(float) * ma_slope_w
        short_score += (df['ma_slope'] < 0).astype(float) * ma_slope_w

    # Factor 7: Markov State
    if primary_driver != 'Markov State' and use_markov and best_markov_setup and 'RunUp_State' in df.columns:
        strategy = best_markov_setup.get('Strategy')
        if strategy == 'Down -> Up':
            long_score += (df['RunUp_State'] == 'Down').astype(float) * markov_w
        elif strategy == 'Up -> Up':
            long_score += (df['RunUp_State'] == 'Up').astype(float) * markov_w
        elif strategy == 'Up -> Down':
            short_score += (df['RunUp_State'] == 'Up').astype(float) * markov_w
        elif strategy == 'Down -> Down':
            short_score += (df['RunUp_State'] == 'Down').astype(float) * markov_w
    # --- [END] ---

    # --- [REMOVED NORMALIZATION] ---
    # Return the raw, un-normalized scores
    return long_score.fillna(0), short_score.fillna(0)

# --- FINAL VERSION: Merged Intelligent Exit + All Missing Features ---

# --- FULL REPLACEMENT FOR run_backtest FUNCTION ---
def run_backtest(data, params,
                 use_rsi, use_volatility, use_trend, use_volume, use_macd, use_ma_slope, use_markov, 
                 rsi_w, vol_w, trend_w, vol_w_val, macd_w, ma_slope_w, markov_w, 
                 use_adx_filter, adx_threshold, rsi_logic, 
                 adx_period=14, 
                 veto_setups_list=None,
                 primary_driver='Bollinger Bands', 
                 markov_setup=None, 
                 exit_logic_type='Standard (Price-Based)', 
                 exit_confidence_threshold=50,
                 smart_trailing_stop_pct=0.05,
                 long_score_95_percentile=None,
                 short_score_95_percentile=None,
                 smart_exit_atr_period=14,
                 smart_exit_atr_multiplier=3.0,
                 intelligent_tsl_pct=1.0,
                 analysis_start_date=None): # <--- NEW ARGUMENT AT THE END

    df = data.copy()
    required_cols = ['Close']
    if 'High' not in df.columns: df['High'] = df['Close']
    if 'Low' not in df.columns: df['Low'] = df['Close']
    required_cols.extend(['High', 'Low'])
    if 'Volume' in df.columns: required_cols.append('Volume')

    df['Close'] = pd.to_numeric(df['Close'], errors='coerce').replace(0, np.nan)
    df.dropna(subset=['Close'], inplace=True)

    min_ma_period = params.get('large_ma_period', 50)
    
    min_lookback_needed = min_ma_period
    if (primary_driver == 'Markov State' or use_markov) and markov_setup is not None:
        min_lookback_needed = max(min_ma_period, markov_setup.get('Run-Up Period', 10))
        
    if len(df) < min_lookback_needed or len(df) < params.get('bband_period', 20) or len(df) < 30:
        return 0, 0, 0.0, 0.0, None, ([], [], [], []), [], (0, 0, 0, 0), ([], []), (None, None, None, None), (0, 0, 0, 0, 0, 0)

    # --- Indicator Calculations ---
    df['large_ma'] = df['Close'].rolling(window=min_ma_period).mean().ffill()
    df['ma_slope'] = df['large_ma'].diff(periods=3).ffill()

    bband_period = params.get('bband_period', 20)
    if len(df) >= bband_period:
        try:
            indicator_bb = BollingerBands(close=df['Close'], window=bband_period, window_dev=params['bband_std_dev'])
            df['bband_lower'] = indicator_bb.bollinger_lband()
            df['bband_upper'] = indicator_bb.bollinger_hband()
        except Exception as e: df['bband_lower'], df['bband_upper'] = np.nan, np.nan
    else: df['bband_lower'], df['bband_upper'] = np.nan, np.nan

    if len(df) >= 14:
        try: 
            indicator_rsi = RSIIndicator(close=df['Close'], window=14)
            df['RSI'] = indicator_rsi.rsi()
        except Exception: df['RSI'] = np.nan
            
        df['Volatility_p'] = df['Close'].pct_change().rolling(window=14).std()
        
        try:
            indicator_adx = ADXIndicator(high=df['High'], low=df['Low'], close=df['Close'], window=adx_period, fillna=True)
            df['ADX'] = indicator_adx.adx().ffill()
        except Exception: df['ADX'] = np.nan
            
        try:
            atr_window = int(smart_exit_atr_period)
            indicator_atr = AverageTrueRange(high=df['High'], low=df['Low'], close=df['Close'], window=atr_window, fillna=True)
            df['ATR'] = indicator_atr.average_true_range()
        except Exception: df['ATR'] = np.nan
    else: 
        df['RSI'], df['Volatility_p'], df['ADX'], df['ATR'] = np.nan, np.nan, np.nan, np.nan

    df['SMA_200'] = df['Close'].rolling(window=200, min_periods=1).mean()

    if 'Volume' in df.columns:
        df['Volume'] = pd.to_numeric(df['Volume'], errors='coerce').fillna(0)
        df['Volume_MA50'] = df['Volume'].rolling(window=50, min_periods=1).mean()
        df['Volume_Ratio'] = df.apply(lambda row: row['Volume'] / row['Volume_MA50'] if row['Volume_MA50'] > 0 else 0, axis=1)
        df['Volume_Ratio'] = df['Volume_Ratio'].replace([np.inf, -np.inf], 0).fillna(0)
    else: df['Volume_Ratio'] = 0.0

    if len(df) >= 26:
        indicator_macd = MACD(close=df['Close'], window_slow=26, window_fast=12, window_sign=9, fillna=True)
        df['MACD_line'] = indicator_macd.macd().ffill()
        df['MACD_signal'] = indicator_macd.macd_signal().ffill()
        df['MACD_hist'] = indicator_macd.macd_diff().ffill()
    else: df['MACD_line'], df['MACD_signal'], df['MACD_hist'] = np.nan, np.nan, np.nan
    
    if (primary_driver == 'Markov State' or use_markov) and markov_setup is not None:
        run_up_period = markov_setup.get('Run-Up Period', 10)
        df['RunUp_Return'] = df['Close'].pct_change(periods=run_up_period)
        df['RunUp_State'] = df['RunUp_Return'].apply(lambda x: 'Up' if x > 0 else 'Down')
    
    bband_params_for_score = {
        'long_entry_threshold_pct': params.get('long_entry_threshold_pct', 0.0),
        'short_entry_threshold_pct': params.get('short_entry_threshold_pct', 0.0)
    }

    raw_long_score, raw_short_score = calculate_confidence_score(df,
        primary_driver,
        use_rsi, use_volatility, use_trend, use_volume, use_macd, use_ma_slope, use_markov, 
        rsi_w, vol_w, trend_w, vol_w_val, macd_w, ma_slope_w, markov_w, 
        bband_params_for_score,
        best_markov_setup=markov_setup 
    )

    if long_score_95_percentile is None:
        long_scores_gt_zero = raw_long_score[raw_long_score > 0]
        long_95 = long_scores_gt_zero.quantile(0.95) if not long_scores_gt_zero.empty else 1.0
    else:
        long_95 = long_score_95_percentile

    if short_score_95_percentile is None:
        short_scores_gt_zero = raw_short_score[raw_short_score > 0]
        short_95 = short_scores_gt_zero.quantile(0.95) if not short_scores_gt_zero.empty else 1.0 
    else:
        short_95 = short_score_95_percentile

    df['long_confidence_score'] = (raw_long_score / long_95 * 100).clip(0, 100) if long_95 > 0 else 0.0
    df['short_confidence_score'] = (raw_short_score / short_95 * 100).clip(0, 100) if short_95 > 0 else 0.0

    apply_veto = bool(veto_setups_list)
    if apply_veto:
        df['any_long_veto_trigger'] = False; df['any_short_veto_trigger'] = False
        for veto_setup in veto_setups_list:
            veto_threshold = veto_setup.get('Conf. Threshold', 0)
            veto_rsi_on = veto_setup.get('RSI') == 'On'; veto_vol_on = veto_setup.get('Volatility') == 'On'
            veto_trend_on = veto_setup.get('TREND') == 'On'; veto_volume_on = veto_setup.get('Volume') == 'On'
            veto_macd_on = veto_setup.get('MACD', 'Off') == 'On'; veto_ma_slope_on = veto_setup.get('MA Slope', 'Off') == 'On'
            veto_markov_on = False 
            
            veto_long_raw, veto_short_raw = calculate_confidence_score(df, 
                'Bollinger Bands', 
                veto_rsi_on, veto_vol_on, veto_trend_on, veto_volume_on, veto_macd_on, veto_ma_slope_on, veto_markov_on,
                1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 
                bband_params_for_score,
                best_markov_setup=markov_setup
            )
            veto_long_norm = (veto_long_raw / 3.0) * 100 
            
            current_long_veto_trigger = (veto_long_norm >= veto_threshold); current_short_veto_trigger = (veto_long_norm >= veto_threshold)
            df['any_long_veto_trigger'] |= current_long_veto_trigger; df['any_short_veto_trigger'] |= current_long_veto_trigger

    # --- Entry Trigger Logic ---
    bb_long_signal = df['Close'] < (df['bband_lower'] * (1 - params['long_entry_threshold_pct']))
    bb_short_signal = df['Close'] > (df['bband_upper'] * (1 + params['short_entry_threshold_pct']))

    if rsi_logic == "Crossover":
        rsi_long_signal = (df['RSI'].shift(1) < 30) & (df['RSI'] >= 30) & df['RSI'].notna()
        rsi_short_signal = (df['RSI'].shift(1) > 70) & (df['RSI'] <= 70) & df['RSI'].notna()
    else: # Level
        rsi_long_signal = (df['RSI'] <= 30) & df['RSI'].notna()
        rsi_short_signal = (df['RSI'] >= 70) & df['RSI'].notna()

    macd_long_signal = (df['MACD_line'].shift(1) < df['MACD_signal'].shift(1)) & (df['MACD_line'] >= df['MACD_signal'])
    macd_short_signal = (df['MACD_line'].shift(1) > df['MACD_signal'].shift(1)) & (df['MACD_line'] <= df['MACD_signal'])
    ma_slope_long_signal = (df['ma_slope'].shift(1) <= 0) & (df['ma_slope'] > 0)
    ma_slope_short_signal = (df['ma_slope'].shift(1) >= 0) & (df['ma_slope'] < 0)

    if primary_driver == 'RSI Crossover':
        base_long_trigger = rsi_long_signal; base_short_trigger = rsi_short_signal
    elif primary_driver == 'MACD Crossover':
        base_long_trigger = macd_long_signal; base_short_trigger = macd_short_signal
    elif primary_driver == 'MA Slope':
        base_long_trigger = ma_slope_long_signal; base_short_trigger = ma_slope_short_signal
    elif primary_driver == 'Markov State' and markov_setup is not None:
        strategy = markov_setup.get('Strategy')
        if strategy == 'Down -> Up':
            base_long_trigger = (df['RunUp_State'] == 'Down'); base_short_trigger = pd.Series(False, index=df.index) 
        elif strategy == 'Up -> Up':
            base_long_trigger = (df['RunUp_State'] == 'Up'); base_short_trigger = pd.Series(False, index=df.index) 
        elif strategy == 'Up -> Down':
            base_long_trigger = pd.Series(False, index=df.index); base_short_trigger = (df['RunUp_State'] == 'Up')
        elif strategy == 'Down -> Down':
            base_long_trigger = pd.Series(False, index=df.index); base_short_trigger = (df['RunUp_State'] == 'Down')
        else: 
            base_long_trigger = pd.Series(False, index=df.index); base_short_trigger = pd.Series(False, index=df.index)
    else: 
        base_long_trigger = bb_long_signal; base_short_trigger = bb_short_signal

    if use_adx_filter and 'ADX' in df.columns and df['ADX'].notna().any():
        adx_allows_entry = (df['ADX'] < adx_threshold).fillna(False)
    else: adx_allows_entry = True

    ma_is_valid = pd.notna(df['large_ma']) 
    
    long_entry_trigger = base_long_trigger & adx_allows_entry & (df['long_confidence_score'] >= params['confidence_threshold']) & ma_is_valid
    short_entry_trigger = base_short_trigger & adx_allows_entry & (df['short_confidence_score'] >= params['confidence_threshold']) & ma_is_valid

    # --- [NEW] DATE FILTER APPLICATION ---
    if analysis_start_date is not None:
        date_mask = df.index >= pd.Timestamp(analysis_start_date)
        long_entry_trigger &= date_mask
        short_entry_trigger &= date_mask
    # -------------------------------------

    if apply_veto:
        long_entry_trigger &= ~df['any_long_veto_trigger']
        short_entry_trigger &= ~df['any_short_veto_trigger']

    potential_long_price_exit = (df['Close'] >= (df['large_ma'] * (1 + params['long_exit_ma_threshold_pct']))) | (df['Close'] >= df['bband_upper'])
    potential_short_price_exit = (df['Close'] <= (df['large_ma'] * (1 - params['short_exit_ma_threshold_pct']))) | (df['Close'] <= df['bband_lower'])

    long_entry_prices = df['Close'].where(long_entry_trigger).ffill()
    short_entry_prices = df['Close'].where(short_entry_trigger).ffill()
    
    if exit_logic_type == 'Intelligent (ADX/MACD/ATR)':
        adx_rising = (df['ADX'] > df['ADX'].shift(1)).fillna(False)
        adx_strong = (df['ADX'] > adx_threshold)
        macd_bullish = (df['MACD_line'] > df['MACD_signal'])
        stay_in_long_trade = (adx_rising | adx_strong) & macd_bullish
        base_long_exit_trigger = potential_long_price_exit & (~stay_in_long_trade)

        macd_bearish = (df['MACD_line'] < df['MACD_signal'])
        stay_in_short_trade = (adx_rising | adx_strong) & macd_bearish
        base_short_exit_trigger = potential_short_price_exit & (~stay_in_short_trade)
    else: 
        long_is_in_profit = (df['Close'] > long_entry_prices).fillna(False) 
        short_is_in_profit = (df['Close'] < short_entry_prices).fillna(False)
        base_long_exit_trigger = potential_long_price_exit & long_is_in_profit
        base_short_exit_trigger = potential_short_price_exit & short_is_in_profit
    
    df['long_signal'] = np.nan; df.loc[long_entry_trigger, 'long_signal'] = 1; df.loc[base_long_exit_trigger, 'long_signal'] = 0 
    df['short_signal'] = np.nan; df.loc[short_entry_trigger, 'short_signal'] = -1; df.loc[base_short_exit_trigger, 'short_signal'] = 0 

    if primary_driver == 'Markov State' and markov_setup is not None:
        limit_long = markov_setup.get('Future Period', 5); limit_short = markov_setup.get('Future Period', 5)
    else:
        limit_long = params.get('max_long_duration', 120); limit_short = params.get('max_short_duration', 120)
        
    temp_long_pos = df['long_signal'].ffill().fillna(0)
    temp_short_pos = df['short_signal'].ffill().fillna(0)
    
    long_groups = (~(temp_long_pos == 1)).cumsum(); df['days_in_long_trade'] = df.groupby(long_groups).cumcount(); df.loc[temp_long_pos == 0, 'days_in_long_trade'] = 0
    short_groups = (~(temp_short_pos == -1)).cumsum(); df['days_in_short_trade'] = df.groupby(short_groups).cumcount(); df.loc[temp_short_pos == 0, 'days_in_short_trade'] = 0
    
    long_time_exit_trigger = (df['days_in_long_trade'] > limit_long) & (temp_long_pos == 1)
    short_time_exit_trigger = (df['days_in_short_trade'] > limit_short) & (temp_short_pos == -1)
    
    df.loc[long_time_exit_trigger, 'long_signal'] = 0; df.loc[short_time_exit_trigger, 'short_signal'] = 0

    df['long_entry_price_static'] = df['Close'].where(df['long_signal'].shift(1) == 0).ffill().bfill()
    df['short_entry_price_static'] = df['Close'].where(df['short_signal'].shift(1) == 0).ffill().bfill()

    long_breakeven_floor = df['long_entry_price_static']; short_breakeven_floor = df['short_entry_price_static'] 
    use_ma_floor_filter = params.get('use_ma_floor_filter', False) 
    
    standard_tsl_pct_long = params.get('long_trailing_stop_loss_pct', 0)
    standard_tsl_pct_short = params.get('short_trailing_stop_loss_pct', 0)
    
    intelligent_tsl_pct_long = intelligent_tsl_pct * 0.60 if exit_logic_type == 'Intelligent (ADX/MACD/ATR)' else 0.0
    intelligent_tsl_pct_short = intelligent_tsl_pct * 0.60 if exit_logic_type == 'Intelligent (ADX/MACD/ATR)' else 0.0
    
    long_tsl_exit = pd.Series(False, index=df.index); short_tsl_exit = pd.Series(False, index=df.index)

    if primary_driver != 'Markov State':
        # LONG TSL
        has_hit_target = potential_long_price_exit
        tsl_to_use_long = np.where(has_hit_target, intelligent_tsl_pct_long, standard_tsl_pct_long) if use_ma_floor_filter else (intelligent_tsl_pct_long if exit_logic_type == 'Intelligent (ADX/MACD/ATR)' else standard_tsl_pct_long)
        if not isinstance(tsl_to_use_long, pd.Series): tsl_to_use_long = pd.Series(tsl_to_use_long, index=df.index)

        if (tsl_to_use_long > 0).any():
            in_long_trade = (df['long_signal'].ffill().fillna(0) == 1)
            long_high_water_mark = df['High'].where(in_long_trade).groupby((~in_long_trade).cumsum()).cummax()
            tsl_from_hwm = long_high_water_mark * (1 - tsl_to_use_long)
            long_tsl_price = np.maximum(tsl_from_hwm, long_breakeven_floor) if use_ma_floor_filter else tsl_from_hwm
            long_tsl_exit = in_long_trade & (df['Close'] < long_tsl_price)
            df.loc[long_tsl_exit, 'long_signal'] = 0

        # SHORT TSL
        has_hit_target = potential_short_price_exit
        tsl_to_use_short = np.where(has_hit_target, intelligent_tsl_pct_short, standard_tsl_pct_short) if use_ma_floor_filter else (intelligent_tsl_pct_short if exit_logic_type == 'Intelligent (ADX/MACD/ATR)' else standard_tsl_pct_short)
        if not isinstance(tsl_to_use_short, pd.Series): tsl_to_use_short = pd.Series(tsl_to_use_short, index=df.index)

        if (tsl_to_use_short > 0).any():
            in_short_trade = (df['short_signal'].ffill().fillna(0) == -1)
            short_low_water_mark = df['Low'].where(in_short_trade).groupby((~in_short_trade).cumsum()).cummin()
            tsl_from_lwm = short_low_water_mark * (1 + tsl_to_use_short)
            short_tsl_price = np.minimum(tsl_from_lwm, short_breakeven_floor) if use_ma_floor_filter else tsl_from_lwm
            short_tsl_exit = in_short_trade & (df['Close'] > short_tsl_price)
            df.loc[short_tsl_exit, 'short_signal'] = 0
   
    df['long_position'] = df['long_signal'].ffill().fillna(0)
    df['short_position'] = df['short_signal'].ffill().fillna(0)
    
    if params['long_delay_days'] > 0: df['long_position'] = df['long_position'].shift(params['long_delay_days']).fillna(0)
    if params['short_delay_days'] > 0: df['short_position'] = df['short_position'].shift(params['short_delay_days']).fillna(0)
    
    df['daily_return'] = df['Close'].pct_change()
    df['long_strategy_return'] = df['long_position'].shift(1) * df['daily_return']
    df['short_strategy_return'] = df['short_position'].shift(1) * df['daily_return']
    final_long_pnl = (1 + df['long_strategy_return'].fillna(0)).prod(skipna=True) - 1
    final_short_pnl = (1 + df['short_strategy_return'].fillna(0)).prod(skipna=True) - 1

    long_entries = df[(df['long_position'] == 1) & (df['long_position'].shift(1) == 0)]
    long_exits = df[(df['long_position'] == 0) & (df['long_position'].shift(1) == 1)]
    short_entries = df[(df['short_position'] == -1) & (df['short_position'].shift(1) == 0)]
    short_exits = df[(df['short_position'] == 0) & (df['short_position'].shift(1) == -1)]
    
    if not df.empty:
        end_date = df.index.max(); one_month_ago = end_date - timedelta(days=30)
    else: end_date = pd.NaT; one_month_ago = pd.NaT
    recently_closed_trades = []

    long_trade_profits, long_durations, first_long_entry_date, last_long_exit_date = [], [], None, None
    short_trade_profits, short_durations, first_short_entry_date, last_short_exit_date = [], [], None, None
    long_profit_take_count, long_tsl_count, long_time_exit_count = 0, 0, 0 
    short_profit_take_count, short_tsl_count, short_time_exit_count = 0, 0, 0 
    df_indices = pd.Series(range(len(df)), index=df.index)
    
    for idx, row in long_entries.iterrows():
        if first_long_entry_date is None: first_long_entry_date = idx
        future_exits = long_exits[long_exits.index > idx]
        if not future_exits.empty:
            exit_row = future_exits.iloc[0]; last_long_exit_date = exit_row.name
            exit_date = exit_row.name
            is_tsl = long_tsl_exit.loc[exit_row.name]; is_time = long_time_exit_trigger.loc[exit_row.name]
            
            if is_tsl: long_tsl_count += 1
            elif is_time: long_time_exit_count += 1
            else: long_profit_take_count += 1 

            profit = (exit_row['Close'] / row['Close']) - 1 if pd.notna(exit_row['Close']) and pd.notna(row['Close']) and row['Close'] != 0 else np.nan
            long_trade_profits.append(profit)

            if exit_date >= one_month_ago:
                recently_closed_trades.append({'Side': 'Long', 'Date Open': idx, 'Date Closed': exit_date, 'Start Confidence': row.get('long_confidence_score', np.nan), 'Final % P/L': profit, 'Status': 'Closed', 'Exit Reason': 'TSL' if is_tsl else ('Time' if is_time else 'Profit')})
            try: long_durations.append(df_indices.loc[exit_row.name] - df_indices.loc[idx])
            except KeyError: long_durations.append(np.nan)
    avg_long_profit_per_trade = np.nanmean(long_trade_profits) if long_trade_profits else 0.0

    for idx, row in short_entries.iterrows():
        if first_short_entry_date is None: first_short_entry_date = idx
        future_exits = short_exits[short_exits.index > idx]
        if not future_exits.empty:
            exit_row = future_exits.iloc[0]; last_short_exit_date = exit_row.name
            exit_date = exit_row.name
            is_tsl = short_tsl_exit.loc[exit_row.name]; is_time = short_time_exit_trigger.loc[exit_row.name]
            
            if is_tsl: short_tsl_count += 1
            elif is_time: short_time_exit_count += 1
            else: short_profit_take_count += 1 
            
            profit = ((exit_row['Close'] / row['Close']) - 1) * -1 if pd.notna(exit_row['Close']) and pd.notna(row['Close']) and row['Close'] != 0 else np.nan
            short_trade_profits.append(profit)

            if exit_date >= one_month_ago:
                recently_closed_trades.append({'Side': 'Short', 'Date Open': idx, 'Date Closed': exit_date, 'Start Confidence': row.get('short_confidence_score', np.nan), 'Final % P/L': profit, 'Status': 'Closed', 'Exit Reason': 'TSL' if is_tsl else ('Time' if is_time else 'Profit')})
            try: short_durations.append(df_indices.loc[exit_row.name] - df_indices.loc[idx])
            except KeyError: short_durations.append(np.nan)
    avg_short_profit_per_trade = np.nanmean(short_trade_profits) if short_trade_profits else 0.0

    long_wins = sum(1 for p in long_trade_profits if pd.notna(p) and p > 0); long_losses = sum(1 for p in long_trade_profits if pd.notna(p) and p < 0)
    short_wins = sum(1 for p in short_trade_profits if pd.notna(p) and p > 0); short_losses = sum(1 for p in short_trade_profits if pd.notna(p) and p < 0)

    long_trades_log = [{'date': idx, 'price': row['Close'], 'confidence': row.get('long_confidence_score', np.nan)} for idx, row in long_entries.iterrows()]
    short_trades_log = [{'date': idx, 'price': row['Close'], 'confidence': row.get('short_confidence_score', np.nan)} for idx, row in short_entries.iterrows()]
    
    open_trades = []
    if not df.empty and 'Close' in df.columns and df['Close'].notna().any():
        last_close = df['Close'].ffill().bfill().iloc[-1]
        if pd.notna(last_close):
            if df['long_position'].iloc[-1] == 1 and not long_entries.empty:
                last_entry_time = long_entries.index[-1]; last_entry = long_entries.loc[last_entry_time]
                if pd.notna(last_entry['Close']) and last_entry['Close'] != 0: 
                    pnl = (last_close / last_entry['Close']) - 1
                    open_trades.append({'Side': 'Long', 'Date Open': last_entry.name, 'Date Closed': pd.NaT, 'Start Confidence': last_entry.get('long_confidence_score', np.nan), 'Final % P/L': pnl, 'Status': 'Open', 'Exit Reason': 'N/A (Open)'})
            if df['short_position'].iloc[-1] == -1 and not short_entries.empty:
                last_entry_time = short_entries.index[-1]; last_entry = short_entries.loc[last_entry_time]
                if pd.notna(last_entry['Close']) and last_entry['Close'] != 0: 
                    pnl = ((last_close / last_entry['Close']) - 1) * -1
                    open_trades.append({'Side': 'Short', 'Date Open': last_entry.name, 'Date Closed': pd.NaT, 'Start Confidence': last_entry.get('short_confidence_score', np.nan), 'Final % P/L': pnl, 'Status': 'Open', 'Exit Reason': 'N/A (Open)'})
    open_trades.extend(recently_closed_trades)
    df.sort_index(inplace=True)

    trade_dates = (first_long_entry_date, last_long_exit_date, first_short_entry_date, last_short_exit_date)
    long_durations = [d for d in long_durations if pd.notna(d)]; short_durations = [d for d in short_durations if pd.notna(d)]
    avg_long_profit = float(avg_long_profit_per_trade) if pd.notna(avg_long_profit_per_trade) else 0.0
    avg_short_profit = float(avg_short_profit_per_trade) if pd.notna(avg_short_profit_per_trade) else 0.0
    final_long_pnl_float = float(final_long_pnl) if pd.notna(final_long_pnl) else 0.0
    final_short_pnl_float = float(final_short_pnl) if pd.notna(final_short_pnl) else 0.0
    final_trade_logs = (long_trades_log, long_exits.index, short_trades_log, short_exits.index)
    exit_breakdown = (long_profit_take_count, long_tsl_count, long_time_exit_count, short_profit_take_count, short_tsl_count, short_time_exit_count)

    return final_long_pnl_float, final_short_pnl_float, avg_long_profit, avg_short_profit, df, final_trade_logs, open_trades, (long_wins, long_losses, short_wins, short_losses), (long_durations, short_durations), trade_dates, exit_breakdown

# --- 3. Charting and Display Functions ---
def generate_long_plot(df, trades, ticker):
    fig = go.Figure()
    # Add Price and MA Lines
    fig.add_trace(go.Scatter(x=df.index, y=df['Close'], mode='lines', name='Close Price', line=dict(color='blue')))
    if 'large_ma' in df.columns:
        fig.add_trace(go.Scatter(x=df.index, y=df['large_ma'], mode='lines', name='Large MA', line=dict(color='orange', dash='dash')))
    # Add Bollinger Bands
    if 'bband_upper' in df.columns and 'bband_lower' in df.columns:
        fig.add_trace(go.Scatter(x=df.index, y=df['bband_upper'], mode='lines', name='Upper Band', line=dict(color='gray', width=0.5)))
        fig.add_trace(go.Scatter(x=df.index, y=df['bband_lower'], mode='lines', name='Lower Band', line=dict(color='gray', width=0.5), fill='tonexty', fillcolor='rgba(211,211,211,0.2)'))

    # Add Trades
    long_entries_log, long_exits, _, _ = trades
    if long_entries_log:
        dates = [t['date'] for t in long_entries_log]
        prices = [t['price'] for t in long_entries_log]
        scores = [f"Confidence: {t['confidence']:.0f}%" for t in long_entries_log]
        fig.add_trace(go.Scatter(x=dates, y=prices, mode='markers', name='Long Entry', marker=dict(color='green', symbol='triangle-up', size=12), text=scores, hoverinfo='text'))
    if not long_exits.empty and 'Close' in df.columns: # Check if Close exists
        exit_prices = df.loc[long_exits,'Close'].dropna() # Drop exits where price might be NaN
        fig.add_trace(go.Scatter(x=exit_prices.index, y=exit_prices, mode='markers', name='Long Exit', marker=dict(color='darkgreen', symbol='x', size=8)))

    fig.update_layout(title=f'Long Trades for {ticker}', xaxis_title='Date', yaxis_title='Price', legend_title="Indicator"); return fig

def generate_short_plot(df, trades, ticker):
    fig = go.Figure()
    # Add Price and MA Lines
    fig.add_trace(go.Scatter(x=df.index, y=df['Close'], mode='lines', name='Close Price', line=dict(color='blue')))
    if 'large_ma' in df.columns:
        fig.add_trace(go.Scatter(x=df.index, y=df['large_ma'], mode='lines', name='Large MA', line=dict(color='orange', dash='dash')))
     # Add Bollinger Bands
    if 'bband_upper' in df.columns and 'bband_lower' in df.columns:
        fig.add_trace(go.Scatter(x=df.index, y=df['bband_upper'], mode='lines', name='Upper Band', line=dict(color='gray', width=0.5)))
        fig.add_trace(go.Scatter(x=df.index, y=df['bband_lower'], mode='lines', name='Lower Band', line=dict(color='gray', width=0.5), fill='tonexty', fillcolor='rgba(211,211,211,0.2)'))

    # Add Trades
    _, _, short_entries_log, short_exits = trades
    if short_entries_log:
        dates = [t['date'] for t in short_entries_log]
        prices = [t['price'] for t in short_entries_log]
        scores = [f"Confidence: {t['confidence']:.0f}%" for t in short_entries_log]
        fig.add_trace(go.Scatter(x=dates, y=prices, mode='markers', name='Short Entry', marker=dict(color='red', symbol='triangle-down', size=12), text=scores, hoverinfo='text'))
    if not short_exits.empty and 'Close' in df.columns: # Check if Close exists
        exit_prices = df.loc[short_exits,'Close'].dropna() # Drop exits where price might be NaN
        fig.add_trace(go.Scatter(x=exit_prices.index, y=exit_prices, mode='markers', name='Short Exit', marker=dict(color='darkred', symbol='x', size=8)))

    fig.update_layout(title=f'Short Trades for {ticker}', xaxis_title='Date', yaxis_title='Price', legend_title="Indicator"); return fig

def normalise_strategy_score(raw_score, benchmark_for_100_percent=0.25):
    if not np.isfinite(raw_score) or raw_score <= 0: return 0.0 # Handle NaN/inf
    return min((raw_score / benchmark_for_100_percent) * 100, 100.0)

def calculate_strategy_score(avg_profit, gb_ratio, total_trades):
    """
    Calculates a Weighted Strategy Score with a 'Quality Gate'.
    
    1. Weights: Profit (42.5%), Win/Loss (42.5%), Trades (15%).
    2. Targets: Profit 3.0%, Win/Loss 5.0, Trades 3000.
    3. QUALITY GATE: If Win/Loss Ratio < 2.0, Profit Score is CAPPED at 1.0.
       (You cannot get 'extra credit' for high profit if the structure is risky).
    """
    # 1. Define Targets
    target_profit = 0.03   # 3.0% per trade
    target_gb = 5.0        # 5.0 Win/Loss Ratio
    target_trades = 3000.0 # Perfect trade count

    # 2. Define Weights (Sum = 1.0)
    w_profit = 0.425 
    w_gb = 0.425     
    w_trades = 0.15  

    # 3. Calculate Scores
    
    # Profit: Uncapped Base
    s_profit = avg_profit / target_profit
    if s_profit < 0: s_profit = 0

    # G/B Ratio: Uncapped Base
    s_gb = gb_ratio / target_gb
    if s_gb < 0: s_gb = 0

    # --- QUALITY GATE ---
    # If the strategy has a poor Win/Loss ratio (< 2.0), 
    # we cap the Profit score at 1.0 (100%).
    # This prevents high volatility/lucky profit from masking a bad ratio.
    if gb_ratio < 2.0:
        s_profit = min(s_profit, 1.0)

    # Trade Count: Pyramid Penalty
    dist = abs(total_trades - target_trades)
    s_trades = 1.0 - (dist / target_trades)
    if s_trades < 0: s_trades = 0

    # 4. Final Weighted Score
    final_score = (s_profit * w_profit) + (s_gb * w_gb) + (s_trades * w_trades)

    return final_score * 100

def display_summary_analytics(summary_df):
    st.subheader("Overall Strategy Performance")
    trade_counts = st.session_state.get('trade_counts', {})
    trade_durations = st.session_state.get('trade_durations', {})
    exit_totals = st.session_state.get('exit_breakdown_totals', {})

    # --- Clear last run stats before calculating ---
    st.session_state.last_run_stats = {}

    col1, col2 = st.columns(2)
    for side in ["Long", "Short"]:
        # Ensure summary_df exists and has the required columns
        req_cols = [f'Num {side} Trades', f'Avg {side} Profit per Trade', f'Cumulative {side} P&L', f'Avg {side} Confidence']
        if summary_df is None or not all(col in summary_df.columns for col in req_cols):
             st.warning(f"Summary data for {side} trades is missing or incomplete.")
             continue 

        active_trades_df = summary_df[summary_df[f'Num {side} Trades'] > 0]
        container = col1 if side == "Long" else col2
        with container:
            st.subheader(f"{side} Trades")
            if not active_trades_df.empty:
                total_trades = active_trades_df[f'Num {side} Trades'].sum()
                avg_trade_profit = (active_trades_df[f'Avg {side} Profit per Trade'].fillna(0) * active_trades_df[f'Num {side} Trades']).sum() / total_trades if total_trades > 0 else 0
                avg_cumulative_profit = active_trades_df[f'Cumulative {side} P&L'].mean()
                avg_confidence = active_trades_df[f'Avg {side} Confidence'].mean()
                if pd.isna(avg_confidence): avg_confidence = 0

                good_tickers = (active_trades_df[f'Cumulative {side} P&L'] > 0).sum()
                bad_tickers = (active_trades_df[f'Cumulative {side} P&L'] < 0).sum()
                ticker_good_bad_ratio = good_tickers / bad_tickers if bad_tickers > 0 else 99999.0

                display_score = calculate_strategy_score(avg_trade_profit, ticker_good_bad_ratio, total_trades)

                st.metric("Strategy Score", f"{display_score:.2f}%")
                st.metric("Avg Profit per Trade (Active Tickers)", f"{avg_trade_profit:.2%}")
                st.metric(f"Average Entry Confidence", f"{avg_confidence:.0f}%")

                st.text(f"Profitable Tickers: {good_tickers}")
                st.text(f"Losing Tickers: {bad_tickers}")
                st.text(f"Ticker Good/Bad Ratio: {ticker_good_bad_ratio:.2f}")

                side_lower = side.lower()
                wins = trade_counts.get(f"{side_lower}_wins", 0)
                losses = trade_counts.get(f"{side_lower}_losses", 0)
                
                # --- NEW: Calculate Open Trades explicitly ---
                open_trades_count = int(total_trades) - (wins + losses)

                # --- TRADING STATS ---
                st.markdown("---")
                st.text(f"Total Individual Trades: {int(total_trades)}")
                st.text(f"Winning Trades: {wins}")
                st.text(f"Losing Trades: {losses}")
                st.text(f"Open Trades: {open_trades_count}")

                trade_win_loss_ratio = 0.0
                if wins > 0 or losses > 0:
                    trade_win_loss_ratio = wins / losses if losses > 0 else 99999.0
                    st.text(f"Trade Win/Loss Ratio: {trade_win_loss_ratio:.2f}")

                if side == "Long":
                    st.session_state.last_run_stats = {
                        "Z_Avg_Profit": avg_trade_profit,
                        "Z_Num_Trades": int(total_trades),
                        "Z_WL_Ratio": trade_win_loss_ratio
                    }

                # --- EXIT BREAKDOWN ---
                if exit_totals:
                    st.markdown("---") 
                    st.subheader("Exit Breakdown")
                    
                    if side == "Long":
                        profit_count = exit_totals.get('long_profit_take_count', 0)
                        tsl_count = exit_totals.get('long_tsl_count', 0)
                        time_count = exit_totals.get('long_time_exit_count', 0)
                        # Get Long Timeout Setting
                        limit_days = st.session_state.get('max_long_duration', 60)
                    else:
                        profit_count = exit_totals.get('short_profit_take_count', 0)
                        tsl_count = exit_totals.get('short_tsl_count', 0)
                        time_count = exit_totals.get('short_time_exit_count', 0)
                        # Get Short Timeout Setting
                        limit_days = st.session_state.get('max_short_duration', 10)

                    exit_total = profit_count + tsl_count + time_count
                    
                    if exit_total > 0:
                        st.markdown(f"**Profit Take:** {profit_count} ({profit_count/exit_total:.1%})") 
                        st.markdown(f"**Stop Loss:** {tsl_count} ({tsl_count/exit_total:.1%})")
                        # Dynamic Label
                        st.markdown(f"**Time Out ({limit_days}d):** {time_count} ({time_count/exit_total:.1%})")

                # --- DURATION STATS ---
                avg_duration = trade_durations.get(f"avg_{side_lower}_duration", 0)
                max_duration = trade_durations.get(f"max_{side_lower}_duration", 0)
                if avg_duration > 0 or max_duration > 0:
                    st.text(f"Avg Trade Duration: {avg_duration:.1f} days")
                    st.text(f"Longest Trade: {max_duration:.0f} days")

            else: st.info("No trades found for this side with current settings.")
# --- 4. Optimisation Functions (Parallelised) ---

# --- FULL REPLACEMENT WORKER: ACCEPTS A SINGLE DICTIONARY ARGUMENT ---
def run_single_parameter_test(task_data):
    # Unpack all arguments from the single dictionary provided by the multiprocessing pool
    params = task_data['params']
    confidence_settings = task_data['confidence_settings']
    master_df = task_data['master_df']
    optimise_for = task_data['optimise_for']
    tickers = task_data['tickers']
    date_range = task_data['date_range']
    power = task_data['power']

    # --- FIX: Correctly unpack ALL 7 items from the list ---
    toggles_list = list(confidence_settings['toggles'])
    weights_list = list(confidence_settings['weights'])
    
    # Unpack all 7 factors (matches the order sent by generate_and_run_optimisation)
    use_rsi, use_vol, use_trend, use_volume, use_macd, use_ma_slope, use_markov = toggles_list
    rsi_w, vol_w, trend_w, volume_w, macd_w, ma_slope_w, markov_w = weights_list
    # --- END FIX ---

    # --- Rest of Unpack ---
    use_adx_filter, adx_threshold, adx_period = confidence_settings['adx_settings']
    rsi_logic = confidence_settings['rsi_logic'] 
    primary_driver = confidence_settings['primary_driver']
    markov_setup = confidence_settings['markov_setup'] 
    exit_logic = confidence_settings['exit_logic'] 
    exit_thresh = confidence_settings['exit_thresh']
    smart_trailing_stop = confidence_settings['smart_trailing_stop']
    smart_exit_atr_p = confidence_settings['smart_exit_atr_period']
    smart_exit_atr_m = confidence_settings['smart_exit_atr_multiplier']
    intelligent_tsl_pct = confidence_settings['intelligent_tsl_pct'] 
    long_95_percentile = confidence_settings['long_95_percentile'] 
    short_95_percentile = confidence_settings['short_95_percentile']
    veto_list = confidence_settings['veto_list']
    # --- End Unpack ---

    total_profit_weighted_avg, total_trades, winning_tickers, losing_tickers = 0, 0, 0, 0
    total_wins, total_losses = 0, 0
    all_confidences = []

    PROFIT_THRESHOLD = 1.0
    excluded_tickers = []
    
    # Initialize exit breakdown counters (Long and Short)
    total_exit_breakdown = [0, 0, 0, 0, 0, 0] # LP, LT, LE, SP, ST, SE

    if not isinstance(tickers, list): tickers = [tickers]
    for ticker in tickers:
        cols_to_use = [ticker]
        if f'{ticker}_High' in master_df.columns: cols_to_use.append(f'{ticker}_High')
        if f'{ticker}_Low' in master_df.columns: cols_to_use.append(f'{ticker}_Low')
        if f'{ticker}_Volume' in master_df.columns: cols_to_use.append(f'{ticker}_Volume')

        existing_cols = [col for col in cols_to_use if col in master_df.columns]
        if ticker not in existing_cols: continue

        ticker_data_full = master_df.loc[:, existing_cols]
        ticker_data = ticker_data_full.loc[date_range[0]:date_range[1]]

        rename_dict = {
            ticker: 'Close', f'{ticker}_High': 'High',
            f'{ticker}_Low': 'Low', f'{ticker}_Volume': 'Volume'
        }
        rename_dict_filtered = {k: v for k, v in rename_dict.items() if k in existing_cols}
        ticker_data = ticker_data.rename(columns=rename_dict_filtered)

        if not ticker_data.empty and 'Close' in ticker_data.columns and not ticker_data['Close'].isna().all():
            # --- CAPTURE FIX: run_backtest now returns exit_breakdown (11th return value) ---
            long_pnl, short_pnl, avg_long_trade, avg_short_trade, _, trades, _, trade_counts, _, _, exit_breakdown = run_backtest(
                ticker_data, params,
                use_rsi, use_vol, use_trend, use_volume, use_macd, use_ma_slope, use_markov,
                rsi_w, vol_w, trend_w, volume_w, macd_w, ma_slope_w, markov_w,
                use_adx_filter, adx_threshold, rsi_logic,
                adx_period,
                veto_setups_list=veto_list,
                primary_driver=primary_driver,
                markov_setup=markov_setup,
                exit_logic_type=exit_logic, 
                exit_confidence_threshold=exit_thresh,
                smart_trailing_stop_pct=smart_trailing_stop,
                smart_exit_atr_period=smart_exit_atr_p,
                smart_exit_atr_multiplier=smart_exit_atr_m,
                intelligent_tsl_pct=intelligent_tsl_pct,
                long_score_95_percentile=long_95_percentile,
                short_score_95_percentile=short_95_percentile
            )

            if abs(long_pnl) > PROFIT_THRESHOLD or abs(short_pnl) > PROFIT_THRESHOLD or \
                    (avg_long_trade is not None and pd.notna(avg_long_trade) and abs(avg_long_trade) > PROFIT_THRESHOLD) or \
                    (avg_short_trade is not None and pd.notna(avg_short_trade) and abs(avg_short_trade) > PROFIT_THRESHOLD):
                excluded_tickers.append(ticker)
                continue

            # Aggregate exit breakdown counters
            total_exit_breakdown = [sum(x) for x in zip(total_exit_breakdown, exit_breakdown)]

            if optimise_for == 'long': 
                pnl, avg_trade_profit, trade_log = long_pnl, avg_long_trade, trades[0]
                total_wins += trade_counts[0]; total_losses += trade_counts[1]
            else: 
                pnl, avg_trade_profit, trade_log = short_pnl, avg_short_trade, trades[2]
                total_wins += trade_counts[2]; total_losses += trade_counts[3]
            
            num_trades = len(trade_log)

            if num_trades > 0 and avg_trade_profit is not None and pd.notna(avg_trade_profit):
                total_trades += num_trades
                total_profit_weighted_avg += avg_trade_profit * num_trades
                if pnl > 0: winning_tickers += 1
                elif pnl < 0: losing_tickers += 1
                all_confidences.extend([trade['confidence'] for trade in trade_log if pd.notna(trade.get('confidence'))])

    overall_avg_profit = 0.0
    good_bad_ratio = 0.0
    trade_good_bad_ratio = 0.0

    if total_trades > 0:
        overall_avg_profit = total_profit_weighted_avg / total_trades
        if losing_tickers > 0: 
            good_bad_ratio = winning_tickers / losing_tickers
        elif winning_tickers > 0: 
            good_bad_ratio = 99999.0
        
        if total_losses > 0:
            trade_good_bad_ratio = total_wins / total_losses
        elif total_wins > 0:
            trade_good_bad_ratio = 99999.0
            
    avg_entry_confidence = np.mean(all_confidences) if all_confidences else 0.0
    
    # Return a single dictionary containing all calculated metrics and the configuration data
    return {
        # CORE METRICS
        "Avg Profit/Trade": overall_avg_profit,
        "Ticker G/B Ratio": good_bad_ratio,
        "Trade G/B Ratio": trade_good_bad_ratio,
        "Total Trades": total_trades,
        "Avg Entry Conf.": avg_entry_confidence,
        "Winning Tickers": winning_tickers,
        "Losing Tickers": losing_tickers,
        "Exit Breakdown": total_exit_breakdown, # NEW: Return the aggregated breakdown
        
        # CONFIG DATA (for Orchestrator unpacking)
        "params": params,
        "confidence_settings": confidence_settings
    }

# --- UPDATED: Calculation Only (Saves to Session State) ---
def generate_and_run_optimisation(master_df, main_content_placeholder, optimise_for, use_squared_weighting):
    # Clear previous results to avoid confusion
    st.session_state.param_results_df = None 
    st.session_state.best_params = None 
    
    # Clear other sections
    st.session_state.summary_df = None
    st.session_state.single_ticker_results = None
    st.session_state.confidence_results_df = None
    st.session_state.open_trades_df = None
    st.session_state.advisor_df = None

    with main_content_placeholder.container():
        veto_list_to_use = st.session_state.get('veto_setup_list', [])
        if veto_list_to_use:
            st.info(f"{len(veto_list_to_use)} Veto filter(s) are ACTIVE for this optimisation run.")

        st.info("Calibrating confidence scores (0-100%)...")
        
        # --- 1. Gather Settings & Run Calibration (Same as before) ---
        use_rsi = st.session_state.use_rsi; use_vol = st.session_state.use_vol; use_trend = st.session_state.use_trend
        use_volume = st.session_state.use_volume; use_macd = st.session_state.use_macd
        use_ma_slope = st.session_state.use_ma_slope; use_markov = st.session_state.use_markov
        
        rsi_w = st.session_state.rsi_w; vol_w = st.session_state.vol_w; trend_w = st.session_state.trend_w
        vol_w_val = st.session_state.volume_w; macd_w = st.session_state.macd_w
        ma_slope_w = st.session_state.ma_slope_w; markov_w = st.session_state.markov_w
        
        calibration_params = {
            "large_ma_period": st.session_state.ma_period, "bband_period": st.session_state.bb_period, 
            "bband_std_dev": st.session_state.bb_std, "confidence_threshold": st.session_state.confidence_slider, 
            "long_entry_threshold_pct": st.session_state.long_entry / 100.0, "long_exit_ma_threshold_pct": st.session_state.long_exit / 100.0, 
            "long_trailing_stop_loss_pct": st.session_state.long_sl / 100.0, "long_delay_days": st.session_state.long_delay, 
            "short_entry_threshold_pct": st.session_state.short_entry / 100.0, "short_exit_ma_threshold_pct": st.session_state.short_exit / 100.0, 
            "short_trailing_stop_loss_pct": st.session_state.short_sl / 100.0, "short_delay_days": st.session_state.short_delay,
            "max_trading_days": st.session_state.max_duration
        }
        
        markov_setup = st.session_state.get('best_markov_setup')
        primary_driver = st.session_state.primary_driver
        
        calib_adx_filter = st.session_state.use_adx_filter
        calib_adx_thresh = st.session_state.adx_threshold
        calib_adx_period = st.session_state.adx_period
        calib_rsi_logic = st.session_state.rsi_logic
        calib_exit_logic = st.session_state.exit_logic_type
        calib_exit_thresh = st.session_state.exit_confidence_threshold
        calib_smart_trailing_stop = st.session_state.smart_trailing_stop_pct / 100.0
        calib_smart_atr_p = st.session_state.smart_exit_atr_period
        calib_smart_atr_m = st.session_state.smart_exit_atr_multiplier
        calib_intelligent_tsl = st.session_state.intelligent_tsl_pct / 100.0
        
        all_long_scores = []; all_short_scores = []
        tickers_to_run_calib = [col for col in master_df.columns if not ('_Volume' in str(col) or '_High' in str(col) or '_Low' in str(col))]
        date_range_calib = (pd.Timestamp(st.session_state.start_date), pd.Timestamp(st.session_state.end_date))

        for ticker_symbol in tickers_to_run_calib:
            cols_to_use = [ticker_symbol, f'{ticker_symbol}_High', f'{ticker_symbol}_Low', f'{ticker_symbol}_Volume']
            existing_cols = [col for col in cols_to_use if col in master_df.columns]
            if ticker_symbol not in existing_cols: continue
            
            ticker_data_full = master_df.loc[:, existing_cols]
            ticker_data = ticker_data_full.loc[date_range_calib[0]:date_range_calib[1]]
            rename_dict = {ticker_symbol: 'Close', f'{ticker_symbol}_High': 'High', f'{ticker_symbol}_Low': 'Low', f'{ticker_symbol}_Volume': 'Volume'}
            rename_dict_filtered = {k: v for k, v in rename_dict.items() if k in existing_cols}
            ticker_data = ticker_data.rename(columns=rename_dict_filtered)
            
            l_pnl, s_pnl, al, as_, _, _, _, _, _, _, _ = run_backtest(
                data=ticker_data, params=calibration_params, 
                use_rsi=use_rsi, use_volatility=use_vol, use_trend=use_trend, use_volume=use_volume, 
                use_macd=use_macd, use_ma_slope=use_ma_slope, use_markov=use_markov,
                rsi_w=rsi_w, vol_w=vol_w, trend_w=trend_w, vol_w_val=vol_w_val, macd_w=macd_w, ma_slope_w=ma_slope_w, markov_w=markov_w,
                use_adx_filter=calib_adx_filter, adx_threshold=calib_adx_thresh, rsi_logic=calib_rsi_logic, adx_period=calib_adx_period,
                veto_setups_list=veto_list_to_use, primary_driver=primary_driver, markov_setup=markov_setup, 
                exit_logic_type=calib_exit_logic, exit_confidence_threshold=calib_exit_thresh, 
                smart_trailing_stop_pct=calib_smart_trailing_stop, smart_exit_atr_period=calib_smart_atr_p, 
                smart_exit_atr_multiplier=calib_smart_atr_m, intelligent_tsl_pct=calib_intelligent_tsl,
                long_score_95_percentile=1.0, short_score_95_percentile=1.0
            )
            
            raw_long, raw_short = calculate_confidence_score(
                ticker_data, primary_driver,
                use_rsi, use_vol, use_trend, use_volume, use_macd, use_ma_slope, use_markov,
                rsi_w, vol_w, trend_w, vol_w_val, macd_w, ma_slope_w, markov_w,
                calibration_params, markov_setup
            )
            all_long_scores.append(raw_long[raw_long > 0])
            all_short_scores.append(raw_short[raw_short > 0])

        long_95 = pd.concat(all_long_scores).quantile(0.95) if all_long_scores else 1.0
        short_95 = pd.concat(all_short_scores).quantile(0.95) if all_short_scores else 1.0
        if pd.isna(long_95) or long_95 == 0: long_95 = 1.0
        if pd.isna(short_95) or short_95 == 0: short_95 = 1.0
        st.info(f"Confidence calibrated: Long 95th percentile = {long_95:.2f}, Short 95th percentile = {short_95:.2f}")

        # --- 2. Build Combinations (Same as before) ---
        ma_range = range(st.session_state.ma_start, st.session_state.ma_end + 1, st.session_state.ma_step) if st.session_state.opt_ma_cb else [st.session_state.ma_period]
        bb_range = range(st.session_state.bb_start, st.session_state.bb_end + 1, st.session_state.bb_step) if st.session_state.opt_bb_cb else [st.session_state.bb_period]
        std_range = np.arange(st.session_state.std_start, st.session_state.std_end + 0.001, st.session_state.std_step) if st.session_state.opt_std_cb else [st.session_state.bb_std]
        sl_range = np.arange(st.session_state.sl_start, st.session_state.sl_end + 0.001, st.session_state.sl_step) / 100 if st.session_state.opt_sl_cb else [st.session_state.long_sl / 100]
        delay_range = range(st.session_state.delay_start, st.session_state.delay_end + 1, st.session_state.delay_step) if st.session_state.opt_delay_cb else [st.session_state.long_delay]
        entry_range = np.arange(st.session_state.entry_start, st.session_state.entry_end + 0.001, st.session_state.entry_step) / 100 if st.session_state.opt_entry_cb else [st.session_state.long_entry / 100]
        exit_range = np.arange(st.session_state.exit_start, st.session_state.exit_end + 0.001, st.session_state.exit_step) / 100 if st.session_state.opt_exit_cb else [st.session_state.long_exit / 100]
        conf_range = range(st.session_state.conf_start, st.session_state.conf_end + 1, st.session_state.conf_step) if st.session_state.opt_conf_cb else [st.session_state.confidence_slider]
        dur_range = range(st.session_state.dur_start, st.session_state.dur_end + 1, st.session_state.dur_step) if st.session_state.opt_duration_cb else [st.session_state.max_duration]

        param_product = itertools.product(ma_range, bb_range, std_range, sl_range, delay_range, entry_range, exit_range, conf_range, dur_range)
        param_combinations = [{
            "large_ma_period": p[0], "bband_period": p[1], "bband_std_dev": p[2],
            "long_trailing_stop_loss_pct": p[3], "short_trailing_stop_loss_pct": p[3],
            "long_delay_days": p[4], "short_delay_days": p[4],
            "long_entry_threshold_pct": p[5], "short_entry_threshold_pct": p[5],
            "long_exit_ma_threshold_pct": p[6], "short_exit_ma_threshold_pct": p[6],
            "confidence_threshold": p[7], "max_trading_days": p[8]
        } for p in param_product]
        
        total_combinations = len(param_combinations)
        if total_combinations <= 1:
            st.warning("No optimisation parameters selected."); return

        confidence_settings = {
            'toggles': (use_rsi, use_vol, use_trend, use_volume, use_macd, use_ma_slope, use_markov), 
            'weights': (rsi_w, vol_w, trend_w, vol_w_val, macd_w, ma_slope_w, markov_w),
            'adx_settings': (st.session_state.use_adx_filter, st.session_state.adx_threshold, st.session_state.adx_period),
            'rsi_logic': st.session_state.rsi_logic, 'primary_driver': primary_driver, 'markov_setup': markov_setup, 
            'exit_logic': st.session_state.exit_logic_type, 'exit_thresh': st.session_state.exit_confidence_threshold,
            'smart_trailing_stop': st.session_state.smart_trailing_stop_pct / 100.0,
            'smart_exit_atr_period': st.session_state.smart_exit_atr_period, 'smart_exit_atr_multiplier': st.session_state.smart_exit_atr_multiplier,
            'intelligent_tsl_pct': st.session_state.intelligent_tsl_pct / 100.0,
            'long_95_percentile': long_95, 'short_95_percentile': short_95, 'veto_list': veto_list_to_use 
        }

        # --- 3. Execution ---
        num_cores = cpu_count()
        st.info(f"Starting {optimise_for.upper()} optimisation on {num_cores} cores... Testing {total_combinations} combinations.")

        if st.session_state.run_mode.startswith("Analyse Full List"):
            tickers_to_run = [col for col in master_df.columns if not ('_Volume' in str(col) or '_High' in str(col) or '_Low' in str(col))]
        else:
            tickers_to_run = [st.session_state.ticker_select]

        date_range = (pd.Timestamp(st.session_state.start_date), pd.Timestamp(st.session_state.end_date))
        power = 2 if use_squared_weighting else 1 
        
        status_text = st.empty(); status_text.text("Optimisation starting...")
        progress_bar = st.progress(0)
        
        tasks = []
        for p in param_combinations:
            tasks.append({
                'params': p, 'confidence_settings': confidence_settings, 'master_df': master_df,
                'optimise_for': optimise_for, 'tickers': tickers_to_run, 'date_range': date_range, 'power': power
            })

        results_list = []
        with Pool(processes=num_cores) as pool:
            try:
                iterator = pool.imap_unordered(run_single_parameter_test, tasks)
                for i, result_dict in enumerate(iterator, 1):
                    results_list.append(result_dict)
                    progress_bar.progress(i / total_combinations, text=f"Optimising... {i}/{total_combinations} combinations complete.")
            except Exception as e:
                st.error(f"An error occurred during multiprocessing: {e}")
                status_text.text("Optimisation failed due to an error."); return
        
        status_text.text("Optimisation complete. Processing results...")

        if not results_list:
            st.warning("Optimisation finished, but no valid results were found."); return

        flattened_results = []
        for r in results_list:
            flat_row = r.copy()
            del flat_row['params']; del flat_row['confidence_settings']
            flat_row.update(r['params'])
            flattened_results.append(flat_row)

        results_df = pd.DataFrame(flattened_results)
        
        min_trades_threshold = 10
        if 'Total Trades' in results_df.columns:
             results_df = results_df[results_df['Total Trades'] >= min_trades_threshold].copy()

        if results_df.empty:
            st.warning(f"No results found with at least {min_trades_threshold} trades. Try a smaller threshold or different settings."); return

        results_df['Strategy Score'] = results_df.apply(
            lambda row: calculate_strategy_score(row['Avg Profit/Trade'], row['Ticker G/B Ratio'], row['Total Trades']), axis=1
        )
        
        # --- 4. PRE-SORT (Descending Duration) ---
        # We sort by Score (Desc) AND Duration (Desc) so the "Biggest Number" comes first.
        if 'max_trading_days' in results_df.columns:
            results_df = results_df.sort_values(by=['Strategy Score', 'max_trading_days'], ascending=[False, False])
        else:
            results_df = results_df.sort_values(by='Strategy Score', ascending=False)
        
        # Save the full results to session state so `main()` can display them
        st.session_state.param_results_df = results_df
        
        # Save best params (first row is best score + highest duration)
        best_setup_series = results_df.iloc[0]
        st.session_state.best_params = {k: best_setup_series[k] for k in param_combinations[0].keys()}
        
        status_text.empty()
        st.success(f"Optimisation Complete! Best Strategy Score: {best_setup_series['Strategy Score']:.2f}%")
        st.subheader("Optimal Parameters Found (based on Strategy Score):")
        st.json(st.session_state.best_params)
        # Note: We do NOT display the table here anymore. main() handles it.

# --- 4.5. NEW COMBINED OPTIMISATION FUNCTION (Optimized: 32k -> 3k Tasks) ---
def generate_and_run_combined_optimisation(master_df, main_content_placeholder, optimise_for):
    st.session_state.summary_df = None
    st.session_state.single_ticker_results = None
    st.session_state.confidence_results_df = None
    st.session_state.open_trades_df = None
    st.session_state.advisor_df = None
    st.session_state.best_params = None
    st.session_state.best_weights = None

    # Define side-specific filename to separate Long/Short results
    results_file = f"combined_optimisation_results_{optimise_for}.csv"

    with main_content_placeholder.container():
        # --- Visual Feedback ---
        st.header(f"⚡ Running Combined Factor & Weight Optimisation ({optimise_for.title()})")
        st.caption(f"Results will be saved to: {results_file}")
        
        # --- FIXED PARAMETERS ---
        fixed_params = {
            "large_ma_period": st.session_state.ma_period, "bband_period": st.session_state.bb_period, 
            "bband_std_dev": st.session_state.bb_std, "confidence_threshold": 50,
            "long_entry_threshold_pct": st.session_state.long_entry / 100, "short_entry_threshold_pct": st.session_state.short_entry / 100,
            "long_exit_ma_threshold_pct": st.session_state.long_exit / 100, "short_exit_ma_threshold_pct": st.session_state.short_exit / 100,
            "long_trailing_stop_loss_pct": st.session_state.long_sl / 100, "short_trailing_stop_loss_pct": st.session_state.short_sl / 100,
            "long_delay_days": st.session_state.long_delay, "short_delay_days": st.session_state.short_delay
        }
        
        # --- CHECKPOINT LOADING ---
        completed_configs = load_completed_setups(results_file)
        st.info(f"Loaded {len(completed_configs)} completed combinations from {results_file}. Resuming job...")
        
        # --- DYNAMIC FACTOR/WEIGHT GENERATION (OPTIMIZED) ---
        # Instead of looping Toggles and Weights separately (which creates duplicates),
        # we treat 0.0 as "Off" and any other value as "On + Weight".
        # 5 States per Factor: [Off, 0.5, 1.0, 1.5, 2.0]
        
        factors = ['RSI', 'Volatility', 'Volume', 'MACD', 'MA Slope'] 
        possible_states = [0.0, 0.5, 1.0, 1.5, 2.0] # 0.0 = Off
        
        # Generate all 3,125 unique combinations (5^5)
        all_combos = list(itertools.product(possible_states, repeat=len(factors)))

        all_tasks = []

        fixed_adx_settings = (st.session_state.use_adx_filter, st.session_state.adx_threshold, st.session_state.adx_period)
        fixed_markov_setup = st.session_state.get('best_markov_setup') 
        fixed_exit_settings = {
            'rsi_logic': st.session_state.rsi_logic, 'primary_driver': st.session_state.primary_driver,
            'exit_logic': st.session_state.exit_logic_type, 'exit_thresh': st.session_state.exit_confidence_threshold,
            'smart_trailing_stop': st.session_state.smart_trailing_stop_pct / 100.0,
            'smart_exit_atr_period': st.session_state.smart_exit_atr_period,
            'smart_exit_atr_multiplier': st.session_state.smart_exit_atr_multiplier,
            'intelligent_tsl_pct': st.session_state.intelligent_tsl_pct / 100.0
        }
        
        tickers_to_run = [col for col in master_df.columns if not ('_Volume' in str(col) or '_High' in str(col) or '_Low' in str(col))]
        date_range = (pd.Timestamp(st.session_state.start_date), pd.Timestamp(st.session_state.end_date))

        for combo in all_combos: 
            weights_5 = list(combo)
            toggles_5 = [w > 0.0 for w in weights_5] # If weight > 0, it's On. If 0.0, it's Off.
            
            # --- EXPAND TO 7 ITEMS (Insert False/0.0 for Trend and Markov) ---
            # Standard Order: RSI, Vol, TREND, Volume, MACD, MA Slope, Markov
            
            toggles_7 = (
                toggles_5[0], toggles_5[1], False, toggles_5[2], toggles_5[3], toggles_5[4], False
            )
            
            weights_7 = (
                weights_5[0], weights_5[1], 0.0, weights_5[2], weights_5[3], weights_5[4], 0.0
            )

            # --- CHECKPOINT LOGIC ---
            toggles_str = tuple(["On" if t else "Off" for t in toggles_7])
            weights_round = tuple([round(w, 2) for w in weights_7])
            task_key = (toggles_str, weights_round)
            
            if task_key not in completed_configs:
                confidence_settings = {
                    'toggles': toggles_7, 'weights': weights_7,
                    'adx_settings': fixed_adx_settings, 
                    'markov_setup': fixed_markov_setup, 'confidence_threshold': fixed_params['confidence_threshold'],
                    'long_95_percentile': 1.0, 'short_95_percentile': 1.0, 'veto_list': st.session_state.get('veto_setup_list', []),
                }
                confidence_settings.update(fixed_exit_settings)

                all_tasks.append({
                    'params': fixed_params,
                    'confidence_settings': confidence_settings,
                    'master_df': master_df,
                    'optimise_for': optimise_for,
                    'tickers': tickers_to_run,
                    'date_range': date_range,
                    'power': 1
                })

        # --- Multiprocessing Execution Setup ---

        total_combinations_theoretical = 3125 # Corrected to 5^5
        tasks_remaining = len(all_tasks)
        
        if tasks_remaining == 0:
            st.success(f"Optimisation complete. All {total_combinations_theoretical} combinations for {optimise_for.title()} have been processed."); 
            pass 

        current_progress_base = total_combinations_theoretical - tasks_remaining
        BATCH_SIZE = 50 # Smaller batch for smoother progress

        if tasks_remaining > 0:
            num_cores = cpu_count()
            st.info(f"Starting COMBINED optimisation ({optimise_for.title()}) on {num_cores} cores... {tasks_remaining} unique tasks remaining.")
            
            results_list = []
            status_text = st.empty(); status_text.text("Optimisation starting...")
            progress_bar = st.progress(0)

            with Pool(processes=cpu_count()) as pool:
                try:
                    iterator = pool.imap_unordered(run_single_parameter_test, all_tasks) 
                    
                    for i, result_dict in enumerate(iterator, 1):
                        results_list.append(result_dict)
                        total_progress = (current_progress_base + i) / total_combinations_theoretical
                        progress_bar.progress(total_progress, text=f"Optimising {optimise_for.title()}... {i}/{tasks_remaining} processed.")

                        if i % BATCH_SIZE == 0 or i == tasks_remaining:
                            factors_csv_order = ['RSI', 'Volatility', 'TREND', 'Volume', 'MACD', 'MA Slope', 'Markov']
                            
                            def unpack_csv_config(row):
                                conf_settings = row['confidence_settings']
                                result = {}
                                togs = conf_settings['toggles']
                                wgts = conf_settings['weights']
                                for idx, name in enumerate(factors_csv_order):
                                    result[name] = "On" if togs[idx] else "Off"
                                    result[name + ' W'] = wgts[idx]
                                return pd.Series(result)

                            config_df = pd.DataFrame(results_list).apply(unpack_csv_config, axis=1)
                            results_df_batch = pd.concat([pd.DataFrame(results_list).drop(['params', 'confidence_settings'], axis=1), config_df], axis=1)
                            
                            results_df_batch['Strategy Score'] = results_df_batch.apply(
                                lambda row: calculate_strategy_score(row['Avg Profit/Trade'], row['Trade G/B Ratio'], row['Total Trades']), axis=1
                            )
                            
                            write_header = not os.path.exists(results_file)
                            results_df_batch.to_csv(results_file, mode='a', header=write_header, index=False)
                            
                            status_text.text(f"CHECKPOINT: Saved {i} new {optimise_for} combinations. Total: {current_progress_base + i} / {total_combinations_theoretical}")
                            results_list = []

                except Exception as e:
                    st.error(f"FATAL ERROR during multiprocessing. Checkpoint saved. Error: {e}")
                    status_text.text("Optimization stopped.")
                    return

            status_text.empty()
            st.success(f"Optimization finished. All {optimise_for.title()} results saved.")
        
        # --- FINAL DISPLAY ---
        if os.path.exists(results_file):
            final_df = pd.read_csv(results_file)
            final_df = final_df.sort_values(by='Strategy Score', ascending=False)
            
            st.subheader(f"Top 20 Complete {optimise_for.title()} Setups:")
            
            display_cols = ['Strategy Score', 'Avg Profit/Trade', 'Trade G/B Ratio', 'Total Trades']
            factors_display = ['RSI', 'Volatility', 'TREND', 'Volume', 'MACD', 'MA Slope', 'Markov']
            display_cols.extend(factors_display)
            display_cols.extend([f + ' W' for f in factors_display])
            display_cols = [c for c in display_cols if c in final_df.columns]
            
            display_df = final_df.head(20)
            st.dataframe(display_df[display_cols].style.format({
                "Strategy Score": "{:.2f}%",
                "Avg Profit/Trade": "{:.2%}",
                "Trade G/B Ratio": "{:.2f}",
            }))
        else:
            st.info(f"No results found yet for {optimise_for.title()}.")


# --- CORRECTED: Ensures ALL arguments are correctly passed to run_backtest ---
def run_single_confidence_test(task, base_params, master_df, date_range, tickers_to_run, optimise_for, factor_weights):
    # Unpack the factor combo tuple (now has 7 elements)
    combo, threshold, _ = task
    use_rsi_combo, use_volatility_combo, use_trend_combo, use_volume_combo, use_macd_combo, use_ma_slope_combo, use_markov_combo = combo 

    test_params = base_params.copy()
    test_params["confidence_threshold"] = threshold

    total_profit_weighted_avg, total_trades, winning_tickers, losing_tickers = 0, 0, 0, 0
    all_confidences = []
    total_wins, total_losses = 0, 0

    PROFIT_THRESHOLD = 1.0
    excluded_tickers_conf = []

    # --- Get STATIC settings from factor_weights ---
    rsi_logic = factor_weights.get('rsi_logic', 'Crossover')
    use_adx_filter = factor_weights.get('use_adx', True)
    adx_threshold = factor_weights.get('adx_thresh', 25.0)
    adx_period = factor_weights.get('adx_period', 14)
    primary_driver = factor_weights.get('primary_driver', 'Bollinger Bands')
    markov_setup = factor_weights.get('markov_setup') 
    exit_logic = factor_weights.get('exit_logic') 
    exit_thresh = factor_weights.get('exit_thresh')
    smart_trailing_stop = factor_weights.get('smart_trailing_stop')
    smart_exit_atr_p = factor_weights.get('smart_exit_atr_period', 14)
    smart_exit_atr_m = factor_weights.get('smart_exit_atr_multiplier', 3.0)
    intelligent_tsl_pct = factor_weights.get('intelligent_tsl_pct', 0.60)
    # Get weights
    rsi_w = factor_weights.get('rsi', 1.0); vol_w = factor_weights.get('vol', 1.0)
    trend_w = factor_weights.get('trend', 1.0); volume_w = factor_weights.get('volume', 1.0)
    macd_w = factor_weights.get('macd', 1.0); ma_slope_w = factor_weights.get('ma_slope', 1.0)
    markov_w = factor_weights.get('markov', 1.0)


    for ticker in tickers_to_run:
        cols_to_use = [ticker]
        if f'{ticker}_High' in master_df.columns: cols_to_use.append(f'{ticker}_High')
        if f'{ticker}_Low' in master_df.columns: cols_to_use.append(f'{ticker}_Low')
        if f'{ticker}_Volume' in master_df.columns: cols_to_use.append(f'{ticker}_Volume')

        existing_cols = [col for col in cols_to_use if col in master_df.columns]
        if ticker not in existing_cols: continue

        ticker_data_full = master_df.loc[:, existing_cols]
        ticker_data = ticker_data_full.loc[date_range[0]:date_range[1]]

        rename_dict = {ticker: 'Close', f'{ticker}_High': 'High', f'{ticker}_Low': 'Low', f'{ticker}_Volume': 'Volume'}
        rename_dict_filtered = {k: v for k, v in rename_dict.items() if k in existing_cols}
        ticker_data = ticker_data.rename(columns=rename_dict_filtered)


        if not ticker_data.empty and 'Close' in ticker_data.columns and not ticker_data['Close'].isna().all():
            # --- CORRECTED run_backtest CALL ---
            long_pnl, short_pnl, avg_long_trade, avg_short_trade, _, trades, _, trade_counts, _, _ = run_backtest(
                ticker_data, test_params, 
                use_rsi_combo, use_volatility_combo, use_trend_combo, use_volume_combo, use_macd_combo, use_ma_slope_combo, use_markov_combo,
                rsi_w, vol_w, trend_w, volume_w, macd_w, ma_slope_w, markov_w,
                use_adx_filter, adx_threshold,
                rsi_logic,
                adx_period,
                veto_setups_list=None, 
                primary_driver=primary_driver,
                markov_setup=markov_setup,
                exit_logic_type=exit_logic, 
                exit_confidence_threshold=exit_thresh,
                smart_trailing_stop_pct=smart_trailing_stop,
                smart_exit_atr_period=smart_exit_atr_p,
                smart_exit_atr_multiplier=smart_exit_atr_m,
                intelligent_tsl_pct=intelligent_tsl_pct 
            )

            if abs(long_pnl) > PROFIT_THRESHOLD or abs(short_pnl) > PROFIT_THRESHOLD or \
                   (avg_long_trade is not None and pd.notna(avg_long_trade) and abs(avg_long_trade) > PROFIT_THRESHOLD) or \
                   (avg_short_trade is not None and pd.notna(avg_short_trade) and abs(avg_short_trade) > PROFIT_THRESHOLD):
                excluded_tickers_conf.append(ticker); continue

            if optimise_for == 'long':
                pnl, avg_trade_profit, trade_log = long_pnl, avg_long_trade, trades[0]
                total_wins += trade_counts[0]; total_losses += trade_counts[1]
            else:
                pnl, avg_trade_profit, trade_log = short_pnl, avg_short_trade, trades[2]
                total_wins += trade_counts[2]; total_losses += trade_counts[3]

            num_trades = len(trade_log)

            if num_trades > 0 and avg_trade_profit is not None and pd.notna(avg_trade_profit):
                total_trades += num_trades
                total_profit_weighted_avg += avg_trade_profit * num_trades
                if pnl > 0: winning_tickers += 1
                elif pnl < 0: losing_tickers += 1
                all_confidences.extend([trade['confidence'] for trade in trade_log if pd.notna(trade.get('confidence'))])

    # --- [NEW SCORING LOGIC] ---
    overall_avg_profit = 0.0
    ticker_good_bad_ratio = 0.0
    badness_score = 0.0
    
    if total_trades > 0:
        overall_avg_profit = total_profit_weighted_avg / total_trades
        if losing_tickers > 0:
            ticker_good_bad_ratio = winning_tickers / losing_tickers
        elif winning_tickers > 0:
            ticker_good_bad_ratio = 99999.0
        
        if winning_tickers > 0 and overall_avg_profit < 0:
            badness_score = (losing_tickers / winning_tickers) * abs(overall_avg_profit)
            
    avg_entry_confidence = np.mean(all_confidences) if all_confidences else 0.0
    trade_good_bad_ratio = total_wins / total_losses if total_losses > 0 else 99999.0
    if pd.isna(avg_entry_confidence): avg_entry_confidence = 0.0

    # Use the new 3-Factor Score
    # We pass TRADE G/B ratio as the second argument because it is more granular/reliable
    norm_score = calculate_strategy_score(overall_avg_profit, trade_good_bad_ratio, total_trades)
    
    # For sorting consistency, we set "Good Score" to the same value as Norm Score
    raw_score = norm_score 

    return {
        "RSI": use_rsi_combo, "Volatility": use_volatility_combo, "TREND": use_trend_combo, "Volume": use_volume_combo,
        "MACD": use_macd_combo, "MA Slope": use_ma_slope_combo, "Markov": use_markov_combo,
        "Conf. Threshold": threshold, "Avg Profit/Trade": overall_avg_profit if pd.notna(overall_avg_profit) else 0.0,
        "Ticker G/B Ratio": ticker_good_bad_ratio if pd.notna(ticker_good_bad_ratio) else 0.0,
        "Trade G/B Ratio": trade_good_bad_ratio if pd.notna(trade_good_bad_ratio) else 0.0,
        "Winning Tickers": winning_tickers, "Losing Tickers": losing_tickers,
        "Avg Entry Conf.": avg_entry_confidence,
        "Good Score": raw_score if pd.notna(raw_score) else 0.0,
        "Bad Score": badness_score if pd.notna(badness_score) else 0.0,
        "Norm. Score %": norm_score, 
        "Total Trades": total_trades
    }

def run_single_weight_test(confidence_settings, base_params, master_df, optimise_for, tickers, date_range, power):
    """
    Worker function for weight optimisation.
    'base_params' (MA, BB, etc.) are fixed.
    'confidence_settings' (weights) are variable.
    """
    total_profit_weighted_avg, total_trades, winning_tickers, losing_tickers = 0, 0, 0, 0
    total_wins, total_losses = 0, 0
    all_confidences = []

    # --- UNPACK all settings from the variable dictionary ---
    use_rsi, use_vol, use_trend, use_volume, use_macd, use_ma_slope, use_markov = confidence_settings['toggles']
    rsi_w, vol_w, trend_w, volume_w, macd_w, ma_slope_w, markov_w = confidence_settings['weights']
    use_adx_filter, adx_threshold, adx_period = confidence_settings['adx_settings'] # <-- MODIFIED
    rsi_logic = confidence_settings['rsi_logic']
    primary_driver = confidence_settings['primary_driver']
    markov_setup = confidence_settings['markov_setup'] 
    exit_logic = confidence_settings['exit_logic'] 
    exit_thresh = confidence_settings['exit_thresh']
    smart_trailing_stop = confidence_settings['smart_trailing_stop']
    smart_exit_atr_p = confidence_settings['smart_exit_atr_period']
    smart_exit_atr_m = confidence_settings['smart_exit_atr_multiplier']
    intelligent_tsl_pct = confidence_settings['intelligent_tsl_pct'] # <-- ADDED
    # --- End Unpack ---

    PROFIT_THRESHOLD = 1.0
    
    if not isinstance(tickers, list): tickers = [tickers]
    for ticker in tickers:
        cols_to_use = [ticker]
        if f'{ticker}_High' in master_df.columns: cols_to_use.append(f'{ticker}_High')
        if f'{ticker}_Low' in master_df.columns: cols_to_use.append(f'{ticker}_Low')
        if f'{ticker}_Volume' in master_df.columns: cols_to_use.append(f'{ticker}_Volume')

        existing_cols = [col for col in cols_to_use if col in master_df.columns]
        if ticker not in existing_cols: continue

        ticker_data_full = master_df.loc[:, existing_cols]
        ticker_data = ticker_data_full.loc[date_range[0]:date_range[1]]

        rename_dict = {
            ticker: 'Close', f'{ticker}_High': 'High',
            f'{ticker}_Low': 'Low', f'{ticker}_Volume': 'Volume'
        }
        rename_dict_filtered = {k: v for k, v in rename_dict.items() if k in existing_cols}
        ticker_data = ticker_data.rename(columns=rename_dict_filtered)

        if not ticker_data.empty and 'Close' in ticker_data.columns and not ticker_data['Close'].isna().all():
            long_pnl, short_pnl, avg_long_trade, avg_short_trade, _, trades, _, trade_counts, _, _ = run_backtest(
                ticker_data, base_params,
                use_rsi, use_vol, use_trend, use_volume, use_macd, use_ma_slope, use_markov,
                rsi_w, vol_w, trend_w, volume_w, macd_w, ma_slope_w, markov_w,
                use_adx_filter, adx_threshold, rsi_logic,
                adx_period, # <-- ADDED
                veto_setups_list=None, # Veto is OFF during optimisation
                primary_driver=primary_driver,
                markov_setup=markov_setup,
                exit_logic_type=exit_logic, 
                exit_confidence_threshold=exit_thresh,
                smart_trailing_stop_pct=smart_trailing_stop,
                smart_exit_atr_period=smart_exit_atr_p,
                smart_exit_atr_multiplier=smart_exit_atr_m,
                intelligent_tsl_pct=intelligent_tsl_pct # <-- ADDED
            )

            if abs(long_pnl) > PROFIT_THRESHOLD or abs(short_pnl) > PROFIT_THRESHOLD or \
                   (avg_long_trade is not None and pd.notna(avg_long_trade) and abs(avg_long_trade) > PROFIT_THRESHOLD) or \
                   (avg_short_trade is not None and pd.notna(avg_short_trade) and abs(avg_short_trade) > PROFIT_THRESHOLD):
                continue # Skip outlier tickers

            if optimise_for == 'long': 
                pnl, avg_trade_profit, trade_log = long_pnl, avg_long_trade, trades[0]
                total_wins += trade_counts[0]; total_losses += trade_counts[1]
            else: 
                pnl, avg_trade_profit, trade_log = short_pnl, avg_short_trade, trades[2]
                total_wins += trade_counts[2]; total_losses += trade_counts[3]

            num_trades = len(trade_log)

            if num_trades > 0 and avg_trade_profit is not None and pd.notna(avg_trade_profit):
                total_trades += num_trades
                total_profit_weighted_avg += avg_trade_profit * num_trades
                if pnl > 0: winning_tickers += 1
                elif pnl < 0: losing_tickers += 1
                all_confidences.extend([trade['confidence'] for trade in trade_log if pd.notna(trade.get('confidence'))])

    overall_avg_profit = 0.0
    good_bad_ratio = 0.0
    trade_good_bad_ratio = 0.0

    if total_trades > 0:
        overall_avg_profit = total_profit_weighted_avg / total_trades
        if losing_tickers > 0: 
            good_bad_ratio = winning_tickers / losing_tickers
        elif winning_tickers > 0: 
            good_bad_ratio = 99999.0
        
        if total_losses > 0:
            trade_good_bad_ratio = total_wins / total_losses
        elif total_wins > 0:
            trade_good_bad_ratio = 99999.0
            
    avg_entry_confidence = np.mean(all_confidences) if all_confidences else 0.0

    weights_tested = {
        "rsi_w": rsi_w, "vol_w": vol_w, "trend_w": trend_w,
        "volume_w": volume_w, "macd_w": macd_w, "ma_slope_w": ma_slope_w,
        "markov_w": markov_w
    }
    
    return {
        "weights": weights_tested,
        "Avg Profit/Trade": overall_avg_profit,
        "Ticker G/B Ratio": good_bad_ratio,
        "Trade G/B Ratio": trade_good_bad_ratio,
        "Total Trades": total_trades,
        "Avg Entry Conf.": avg_entry_confidence,
        "Winning Tickers": winning_tickers,
        "Losing Tickers": losing_tickers
    }

def run_advisor_scan(main_df, setups_to_run, advisor_type="Advisor"):
    """
    Scans tickers for open trades using a list of setups.
    Updated to provide real-time granular feedback per ticker.
    """
    st.info(f"Scanning tickers for open trades based on {len(setups_to_run)} {advisor_type} setups...")
    
    base_params = {
        "large_ma_period": st.session_state.ma_period, 
        "bband_period": st.session_state.bb_period, 
        "bband_std_dev": st.session_state.bb_std,
        "long_entry_threshold_pct": st.session_state.long_entry / 100, 
        "long_exit_ma_threshold_pct": st.session_state.long_exit / 100, 
        "long_trailing_stop_loss_pct": st.session_state.long_sl / 100, 
        "long_delay_days": st.session_state.long_delay, 
        "short_entry_threshold_pct": st.session_state.short_entry / 100, 
        "short_exit_ma_threshold_pct": st.session_state.short_exit / 100, 
        "short_trailing_stop_loss_pct": st.session_state.short_sl / 100, 
        "short_delay_days": st.session_state.short_delay
    }

    lookbacks = [
        base_params.get('large_ma_period', 50),
        base_params.get('bband_period', 20),
        200, 50, 26, 14,
        st.session_state.adx_period
    ]
    
    markov_setup_to_use = None
    if st.session_state.primary_driver == 'Markov State' or st.session_state.use_markov:
        if 'best_markov_setup' in st.session_state and st.session_state.best_markov_setup:
            markov_setup_to_use = st.session_state.best_markov_setup
            if st.session_state.primary_driver == 'Markov State':
                st.info("Using saved Best Markov Setup as Primary Driver.")
            lookbacks.append(markov_setup_to_use.get('Run-Up Period', 10))
        else:
            st.error("Markov State is active, but no Best Markov Setup found. Run Section 7 first.")
            st.stop()

    if advisor_type == "User-Defined":
        for setup in setups_to_run:
            try:
                lookbacks.append(int(setup.get("Large MA Period", 50)))
                lookbacks.append(int(setup.get("Bollinger Band Period", 20)))
            except (ValueError, TypeError):
                lookbacks.append(50)
                lookbacks.append(20)

    max_lookback_period = max(lookbacks)
    active_scan_days = 120
    buffer_days = 10
    total_days_needed = max_lookback_period + active_scan_days + buffer_days
    
    scan_end_date = pd.Timestamp(st.session_state.end_date)
    scan_start_date = scan_end_date - timedelta(days=total_days_needed)
    earliest_data_date = main_df.index.min()
    if scan_start_date < earliest_data_date: scan_start_date = earliest_data_date
        
    st.caption(f"Analysing data from {scan_start_date.date()} to {scan_end_date.date()}.")
    
    factor_settings = {
        "use_adx": st.session_state.use_adx_filter, "adx_thresh": st.session_state.adx_threshold,
        "adx_period": st.session_state.adx_period,
        "rsi_logic": st.session_state.get('rsi_logic', 'Crossover'),
        "primary_driver": st.session_state.get('primary_driver', 'Bollinger Bands'), 
        "markov_setup": markov_setup_to_use, 
        "exit_logic": st.session_state.exit_logic_type, 
        "exit_thresh": st.session_state.exit_confidence_threshold,
        "smart_trailing_stop": st.session_state.smart_trailing_stop_pct / 100.0,
        "rsi_w": st.session_state.rsi_w, "vol_w": st.session_state.vol_w,
        "trend_w": st.session_state.trend_w, "volume_w": st.session_state.volume_w,
        "macd_w": st.session_state.macd_w, "ma_slope_w": st.session_state.ma_slope_w,
        "markov_w": st.session_state.markov_w,
        'smart_exit_atr_period': st.session_state.smart_exit_atr_period,
        'smart_exit_atr_multiplier': st.session_state.smart_exit_atr_multiplier,
        'intelligent_tsl_pct': st.session_state.intelligent_tsl_pct / 100.0
    }

    all_advisor_trades = []
    ticker_list = sorted([col for col in main_df.columns if not ('_Volume' in str(col) or '_High' in str(col) or '_Low' in str(col))])
    
    # --- [NEW] Better Progress Tracking ---
    num_setups = len(setups_to_run)
    num_tickers = len(ticker_list)
    total_ops = num_setups * num_tickers
    current_op = 0

    progress_bar = st.progress(0, text="Initializing scan...")
    status_text = st.empty() # Placeholder for rolling ticker names

    for i, setup in enumerate(setups_to_run):
        
        if advisor_type == "Top Setups":
            params_for_run = base_params.copy()
            params_for_run['confidence_threshold'] = setup.get('Conf. Threshold', 50)
            setup_use_rsi = setup.get('RSI', 'Off') == 'On'
            setup_use_vol = setup.get('Volatility', 'Off') == 'On'
            setup_use_trend = setup.get('TREND', 'Off') == 'On'
            setup_use_volume = setup.get('Volume', 'Off') == 'On'
            setup_use_macd = setup.get('MACD', 'Off') == 'On'
            setup_use_ma_slope = setup.get('MA Slope', 'Off') == 'On'
            setup_use_markov = setup.get('Markov', 'Off') == 'On' 
            scan_rsi_w = factor_settings['rsi_w']; scan_vol_w = factor_settings['vol_w']
            scan_trend_w = factor_settings['trend_w']; scan_volume_w = factor_settings['volume_w']
            scan_macd_w = factor_settings['macd_w']; scan_ma_slope_w = factor_settings['ma_slope_w']
            scan_markov_w = factor_settings['markov_w'] 

        elif advisor_type == "User-Defined":
            try: ma_p = int(setup.get("Large MA Period", 50))
            except: ma_p = 50
            try: bb_p = int(setup.get("Bollinger Band Period", 20))
            except: bb_p = 20
            try: long_d = int(setup.get("Long Delay (Days)", 0))
            except: long_d = 0
            try: short_d = int(setup.get("Short Delay (Days)", 0))
            except: short_d = 0

            params_for_run = {
                "large_ma_period": ma_p, "bband_period": bb_p, "long_delay_days": long_d, "short_delay_days": short_d,
                "bband_std_dev": setup.get("Bollinger Band Std Dev", 2.0),
                "confidence_threshold": setup.get("Conf. Threshold", 50),
                "long_entry_threshold_pct": setup.get("Long Entry Threshold (%)", 0.0) / 100.0,
                "long_exit_ma_threshold_pct": setup.get("Long Exit Threshold (%)", 0.0) / 100.0,
                "long_trailing_stop_loss_pct": setup.get("Long Stop Loss (%)", 0.0) / 100.0,
                "short_entry_threshold_pct": setup.get("Short Entry Threshold (%)", 0.0) / 100.0,
                "short_exit_ma_threshold_pct": setup.get("Short Exit Threshold (%)", 0.0) / 100.0,
                "short_trailing_stop_loss_pct": setup.get("Short Stop Loss (%)", 0.0) / 100.0,
                "use_ma_floor_filter": st.session_state.use_ma_floor_filter # Pass this through
            }

            def get_weight_toggle(val):
                try:
                    w = float(val)
                    if w > 0: return w, True
                except: pass
                return 0.0, False

            scan_rsi_w, setup_use_rsi = get_weight_toggle(setup.get('RSI', 'Off'))
            scan_vol_w, setup_use_vol = get_weight_toggle(setup.get('Volatility', 'Off'))
            scan_trend_w, setup_use_trend = get_weight_toggle(setup.get('TREND', 'Off'))
            scan_volume_w, setup_use_volume = get_weight_toggle(setup.get('Volume', 'Off'))
            scan_macd_w, setup_use_macd = get_weight_toggle(setup.get('MACD', 'Off'))
            scan_ma_slope_w, setup_use_ma_slope = get_weight_toggle(setup.get('MA Slope', 'Off'))
            scan_markov_w, setup_use_markov = get_weight_toggle(setup.get('Markov', 'Off')) 
        else:
            continue

        # --- INNER LOOP ---
        for ticker_symbol in ticker_list:
            # Update progress per ticker
            current_op += 1
            if current_op % 10 == 0: # Update visual every 10 items to be faster
                progress_bar.progress(current_op / total_ops, text=f"Setup {i+1}/{num_setups}: Scanning {ticker_symbol}...")

            cols_to_use = [ticker_symbol]
            if f'{ticker_symbol}_High' in main_df.columns: cols_to_use.append(f'{ticker_symbol}_High')
            if f'{ticker_symbol}_Low' in main_df.columns: cols_to_use.append(f'{ticker_symbol}_Low')
            if f'{ticker_symbol}_Volume' in main_df.columns: cols_to_use.append(f'{ticker_symbol}_Volume')

            existing_cols = [col for col in cols_to_use if col in main_df.columns]
            if ticker_symbol not in existing_cols: continue

            data_for_backtest_full = main_df.loc[:, existing_cols]
            data_for_scan = data_for_backtest_full.loc[scan_start_date:scan_end_date]
            rename_dict = {ticker_symbol: 'Close', f'{ticker_symbol}_High': 'High', f'{ticker_symbol}_Low': 'Low', f'{ticker_symbol}_Volume': 'Volume'}
            rename_dict_filtered = {k: v for k, v in rename_dict.items() if k in existing_cols}
            data_for_scan = data_for_scan.rename(columns=rename_dict_filtered)

            if advisor_type == "User-Defined":
                current_lookback = max(params_for_run.get('large_ma_period', 50), params_for_run.get('bband_period', 20), 200, 50, 26, 14, factor_settings['adx_period'])
            else:
                current_lookback = max_lookback_period
                
            if (factor_settings['primary_driver'] == 'Markov State' or setup_use_markov) and factor_settings['markov_setup']:
                current_lookback = max(current_lookback, factor_settings['markov_setup'].get('Run-Up Period', 10))

            if not data_for_scan.empty and 'Close' in data_for_scan.columns and not data_for_scan['Close'].isna().all() and len(data_for_scan) >= current_lookback :
                try:
                    # --- FIX APPLIED HERE: Added the 11th underscore ---
                    _, _, _, _, _, _, open_trades, _, _, _, _ = run_backtest(
                        data_for_scan, params_for_run,
                        setup_use_rsi, setup_use_vol, setup_use_trend, setup_use_volume,
                        setup_use_macd, setup_use_ma_slope, setup_use_markov, 
                        scan_rsi_w, scan_vol_w, scan_trend_w, scan_volume_w,
                        scan_macd_w, scan_ma_slope_w, scan_markov_w, 
                        factor_settings['use_adx'], factor_settings['adx_thresh'],
                        factor_settings['rsi_logic'],
                        factor_settings['adx_period'],
                        veto_setups_list=None,
                        primary_driver=factor_settings['primary_driver'], 
                        markov_setup=factor_settings['markov_setup'],
                        exit_logic_type=factor_settings['exit_logic'], 
                        exit_confidence_threshold=factor_settings['exit_thresh'],
                        smart_trailing_stop_pct=factor_settings['smart_trailing_stop'],
                        smart_exit_atr_period=factor_settings['smart_exit_atr_period'],
                        smart_exit_atr_multiplier=factor_settings['smart_exit_atr_multiplier'],
                        intelligent_tsl_pct=factor_settings['intelligent_tsl_pct']
                    )
                except Exception as e:
                    print(f"Error during advisor backtest for {ticker_symbol} with setup {i+1}: {e}")
                    continue

                if open_trades:
                    for trade in open_trades:
                        trade['Ticker'] = ticker_symbol
                        trade['Setup Rank'] = i + 1
                        if advisor_type == "Top Setups":
                            trade['Setup G/B Ratio'] = setup.get('Ticker G/B Ratio', np.nan)
                        all_advisor_trades.append(trade)

    progress_bar.empty()
    status_text.empty()
    
    print(f"\nFound {len(all_advisor_trades)} total trades before de-duplication.")
    best_trades = {}
    for trade in all_advisor_trades:
        try:
            trade_key = (trade.get('Ticker'), trade.get('Entry Date'), trade.get('Trade Type'))
        except Exception as e:
            print(f"Warning: Could not create trade key for {trade}. Error: {e}")
            continue
        try:
            current_rank = int(trade.get('Setup Rank', 999))
        except (ValueError, TypeError):
            current_rank = 999 

        if trade_key not in best_trades:
            best_trades[trade_key] = trade
        else:
            existing_rank = int(best_trades[trade_key].get('Setup Rank', 999))
            if current_rank < existing_rank:
                best_trades[trade_key] = trade
                
    deduplicated_trades = list(best_trades.values())
    
    if all_advisor_trades:
        raw_advisor_df = pd.DataFrame(all_advisor_trades)
        deduped_advisor_df = pd.DataFrame(deduplicated_trades)
        cols_order = ['Ticker', 'Status', 'Setup Rank', 'Final % P/L', 'Side', 'Date Open', 'Date Closed', 'Start Confidence']

        if advisor_type == "Top Setups":
            cols_order.append('Setup G/B Ratio')
        
        if advisor_type == "User-Defined":
            param_keys = [k for k in setups_to_run[0].keys() if k not in ["RSI", "Volatility", "TREND", "Volume", "MACD", "MA Slope", "Conf. Threshold", "Markov"]]
            cols_order.extend(param_keys)
            setups_df = pd.DataFrame(setups_to_run)
            setups_df['Setup Rank'] = setups_df.index + 1
            raw_advisor_df = pd.merge(raw_advisor_df, setups_df, on='Setup Rank', how='left')
            deduped_advisor_df = pd.merge(deduped_advisor_df, setups_df, on='Setup Rank', how='left')

        existing_cols_final = [col for col in cols_order if col in raw_advisor_df.columns] 
        st.session_state.raw_df = raw_advisor_df[existing_cols_final].sort_values(by=['Status', 'Date Open'], ascending=[True, False])
        st.session_state.deduped_df = deduped_advisor_df[existing_cols_final].sort_values(by=['Status', 'Date Open'], ascending=[True, False])
        st.session_state.advisor_type = advisor_type
    else:
        st.session_state.raw_df = None
        st.session_state.deduped_df = None
        st.session_state.advisor_type = advisor_type

    st.session_state.run_advanced_advisor = False
    st.session_state.run_user_advisor_setup = False
    st.session_state.run_scan_user_setups = False 

    st.rerun()

def generate_and_run_weight_optimisation(master_df, main_content_placeholder, side, use_sq_weighting):
    """
    Runs weight optimisation using multiprocessing.
    """
    st.session_state.summary_df = None
    st.session_state.single_ticker_results = None
    st.session_state.confidence_results_df = None
    st.session_state.open_trades_df = None
    st.session_state.advisor_df = None
    st.session_state.best_params = None
    st.session_state.best_weights = None 

    with main_content_placeholder.container():
        # Get FIXED strategy parameters
        base_params = {
            "large_ma_period": st.session_state.ma_period, "bband_period": st.session_state.bb_period, 
            "bband_std_dev": st.session_state.bb_std, "long_entry_threshold_pct": st.session_state.long_entry / 100, 
            "long_exit_ma_threshold_pct": st.session_state.long_exit / 100, 
            "long_trailing_stop_loss_pct": st.session_state.long_sl / 100, 
            "long_delay_days": st.session_state.long_delay, "short_entry_threshold_pct": st.session_state.short_entry / 100, 
            "short_exit_ma_threshold_pct": st.session_state.short_exit / 100, 
            "short_trailing_stop_loss_pct": st.session_state.short_sl / 100, 
            "short_delay_days": st.session_state.short_delay, "confidence_threshold": st.session_state.confidence_slider
        }
        
        # Get FIXED confidence settings
        fixed_toggles = (st.session_state.use_rsi, st.session_state.use_vol, st.session_state.use_trend, 
                         st.session_state.use_volume, st.session_state.use_macd, st.session_state.use_ma_slope, st.session_state.use_markov)
        fixed_adx_settings = (st.session_state.use_adx_filter, st.session_state.adx_threshold, st.session_state.adx_period)
        fixed_rsi_logic = st.session_state.rsi_logic
        fixed_primary_driver = st.session_state.primary_driver
        fixed_markov_setup = st.session_state.get('best_markov_setup') 
        fixed_exit_logic = st.session_state.exit_logic_type 
        fixed_exit_thresh = st.session_state.exit_confidence_threshold
        fixed_smart_trailing_stop = st.session_state.smart_trailing_stop_pct / 100.0
        smart_atr_p = st.session_state.smart_exit_atr_period
        smart_atr_m = st.session_state.smart_exit_atr_multiplier
        # Check if key exists, default to 0.6 if not (backward compatibility)
        fixed_intelligent_tsl = st.session_state.get('intelligent_tsl_pct', 60.0) / 100.0 

        # Dynamic Weight Optimisation Logic
        weight_step = 0.5 
        weight_range = np.arange(0.5, 2.0 + weight_step, weight_step) 

        all_factor_info = {
            'RSI': {'toggle_key': 'use_rsi', 'weight_key': 'rsi_w'},
            'Volatility': {'toggle_key': 'use_vol', 'weight_key': 'vol_w'},
            'TREND': {'toggle_key': 'use_trend', 'weight_key': 'trend_w'},
            'Volume': {'toggle_key': 'use_volume', 'weight_key': 'volume_w'},
            'MACD': {'toggle_key': 'use_macd', 'weight_key': 'macd_w'},
            'MA Slope': {'toggle_key': 'use_ma_slope', 'weight_key': 'ma_slope_w'},
            'Markov': {'toggle_key': 'use_markov', 'weight_key': 'markov_w'}
        }
        
        driver_map = {'RSI Crossover': 'RSI', 'MACD Crossover': 'MACD', 'MA Slope': 'MA Slope', 'Markov State': 'Markov', 'Bollinger Bands': None}
        primary_factor_key = driver_map.get(fixed_primary_driver)

        factors_to_optimise = []
        for factor_key, info in all_factor_info.items():
            is_active = st.session_state.get(info['toggle_key'], False)
            is_primary = (factor_key == primary_factor_key)
            
            if factor_key == 'Markov' and not fixed_markov_setup:
                st.warning("Skipping Markov weight optimisation: No 'Best Markov Setup' found. Please run Section 7 first.")
                is_active = False
            
            if is_active and not is_primary:
                factors_to_optimise.append(factor_key)
        
        if not factors_to_optimise:
            st.warning("No active, non-primary factors to optimise. Toggle some factors 'On' in Section 2.")
            st.session_state.run_weight_optimisation = False 
            return

        st.info(f"Optimising weights for: {', '.join(factors_to_optimise)}")

        weight_product = itertools.product(weight_range, repeat=len(factors_to_optimise))
        
        base_weight_keys = ('rsi_w', 'vol_w', 'trend_w', 'volume_w', 'macd_w', 'ma_slope_w', 'markov_w')
        base_weights_tuple = (st.session_state.rsi_w, st.session_state.vol_w, st.session_state.trend_w, st.session_state.volume_w, st.session_state.macd_w, st.session_state.ma_slope_w, st.session_state.markov_w)

        confidence_combinations = []
        for weight_tuple in weight_product:
            current_weights_map = dict(zip(base_weight_keys, base_weights_tuple))
            for factor_key, new_weight_value in zip(factors_to_optimise, weight_tuple):
                weight_key = all_factor_info[factor_key]['weight_key']
                current_weights_map[weight_key] = new_weight_value
            
            final_weights_tuple = (current_weights_map['rsi_w'], current_weights_map['vol_w'], current_weights_map['trend_w'], current_weights_map['volume_w'], current_weights_map['macd_w'], current_weights_map['ma_slope_w'], current_weights_map['markov_w'])

            confidence_combinations.append({
                'toggles': fixed_toggles, 'weights': final_weights_tuple, 'adx_settings': fixed_adx_settings, 
                'rsi_logic': fixed_rsi_logic, 'primary_driver': fixed_primary_driver, 'markov_setup': fixed_markov_setup, 
                'exit_logic': fixed_exit_logic, 'exit_thresh': fixed_exit_thresh, 'smart_trailing_stop': fixed_smart_trailing_stop,
                'smart_exit_atr_period': smart_atr_p, 'smart_exit_atr_multiplier': smart_atr_m, 'intelligent_tsl_pct': fixed_intelligent_tsl
            })

        total_combinations = len(confidence_combinations)
        num_cores = cpu_count()
        st.info(f"Starting {side.upper()} weight optimisation on {num_cores} cores... Testing {total_combinations} weight combinations.")

        if st.session_state.run_mode.startswith("Analyse Full List"):
            tickers_to_run = [col for col in master_df.columns if not ('_Volume' in str(col) or '_High' in str(col) or '_Low' in str(col))]
        else:
            tickers_to_run = [st.session_state.ticker_select]

        date_range = (pd.Timestamp(st.session_state.start_date), pd.Timestamp(st.session_state.end_date))
        power = 2 if use_sq_weighting else 1
        
        status_text = st.empty(); status_text.text("Optimisation starting...")
        progress_bar = st.progress(0)

        worker_func = partial(run_single_weight_test, base_params=base_params, master_df=master_df, optimise_for=side, tickers=tickers_to_run, date_range=date_range, power=power)

        results_list = []
        with Pool(processes=num_cores) as pool:
            try:
                iterator = pool.imap_unordered(worker_func, confidence_combinations)
                for i, result_dict in enumerate(iterator, 1):
                    results_list.append(result_dict)
                    progress_bar.progress(i / total_combinations, text=f"Optimising... {i}/{total_combinations} combinations complete.")
            except Exception as e:
                st.error(f"An error occurred during multiprocessing: {e}")
                status_text.text("Optimisation failed due to an error."); st.session_state.run_weight_optimisation = False; return

        status_text.text("Optimisation complete. Calculating scores...")

        if not results_list:
            st.warning("Optimisation finished, but no valid results were found."); st.session_state.run_weight_optimisation = False; return

        results_df = pd.DataFrame(results_list)
        weights_df = results_df['weights'].apply(pd.Series)
        results_df = pd.concat([results_df.drop('weights', axis=1), weights_df], axis=1)

        min_trades_threshold = 10
        if 'Total Trades' in results_df.columns: results_df = results_df[results_df['Total Trades'] >= min_trades_threshold].copy()

        if results_df.empty:
            st.warning(f"No results found with at least {min_trades_threshold} trades."); st.session_state.run_weight_optimisation = False; return

        results_df['Strategy Score'] = results_df.apply(lambda row: calculate_strategy_score(row['Avg Profit/Trade'], row['Ticker G/B Ratio'], row['Total Trades']), axis=1)
        results_df = results_df.sort_values(by='Strategy Score', ascending=False)
        
        best_setup_series = results_df.iloc[0]
        best_metric = best_setup_series['Strategy Score']
        best_weights_dict = {key: float(best_setup_series[key]) for key in base_weight_keys}
        
        status_text.empty()
        st.success(f"Weight Optimisation Complete! Best Strategy Score: {best_metric:.2f}%")
        st.subheader("Optimal Weights Found (based on Strategy Score):"); st.json(best_weights_dict)
        st.session_state.best_weights = best_weights_dict

        st.subheader("Full Weight Optimisation Results:")
        display_cols = ["Strategy Score", "Avg Profit/Trade", "Ticker G/B Ratio", "Total Trades"] + list(base_weight_keys)
        display_cols = [col for col in display_cols if col in results_df.columns]
        formatters = {"Strategy Score": "{:.2f}%", "Avg Profit/Trade": "{:.2%}", "Ticker G/B Ratio": "{:.2f}"}
        for w_key in base_weight_keys: formatters[w_key] = "{:.1f}"
        
        st.dataframe(results_df[display_cols].style.format(formatters))
        st.session_state.run_weight_optimisation = False

def apply_best_weights_to_widgets():
    """Loads optimised weights from st.session_state.best_weights into the sidebar widgets."""
    if 'best_weights' in st.session_state and st.session_state.best_weights:
        weights = st.session_state.best_weights
        
        if 'rsi_w' in weights: st.session_state.rsi_w = weights['rsi_w']
        if 'vol_w' in weights: st.session_state.vol_w = weights['vol_w']
        if 'trend_w' in weights: st.session_state.trend_w = weights['trend_w']
        if 'volume_w' in weights: st.session_state.volume_w = weights['volume_w']
        if 'macd_w' in weights: st.session_state.macd_w = weights['macd_w']
        if 'ma_slope_w' in weights: st.session_state.ma_slope_w = weights['ma_slope_w']
        
        st.sidebar.success("Optimal weights loaded into sidebar!")
        st.rerun()
    else:
        st.sidebar.error("No optimal weights found in session state.")

def apply_best_params_to_widgets():
    """Loads parameters from st.session_state.best_params into the sidebar widgets."""
    if 'best_params' in st.session_state and st.session_state.best_params:
        params = st.session_state.best_params
        
        if 'large_ma_period' in params: st.session_state.ma_period = params['large_ma_period']
        if 'bband_period' in params: st.session_state.bb_period = params['bband_period']
        if 'bband_std_dev' in params: st.session_state.bb_std = params['bband_std_dev']
        if 'confidence_threshold' in params: st.session_state.confidence_slider = params['confidence_threshold']
        
        if 'long_entry_threshold_pct' in params: st.session_state.long_entry = params['long_entry_threshold_pct'] * 100
        if 'long_exit_ma_threshold_pct' in params: st.session_state.long_exit = params['long_exit_ma_threshold_pct'] * 100
        if 'long_trailing_stop_loss_pct' in params: st.session_state.long_sl = params['long_trailing_stop_loss_pct'] * 100
        if 'long_delay_days' in params: st.session_state.long_delay = params['long_delay_days']
        
        if 'long_entry_threshold_pct' in params: st.session_state.short_entry = params['long_entry_threshold_pct'] * 100
        if 'long_exit_ma_threshold_pct' in params: st.session_state.short_exit = params['long_exit_ma_threshold_pct'] * 100
        if 'long_trailing_stop_loss_pct' in params: st.session_state.short_sl = params['long_trailing_stop_loss_pct'] * 100
        if 'long_delay_days' in params: st.session_state.short_delay = params['long_delay_days']
        
        # --- NEW: Load Max Duration ---
        if 'max_trading_days' in params: st.session_state.max_duration = params['max_trading_days']
        
        st.sidebar.success("Optimal parameters loaded into sidebar!")
        st.rerun() 
    else:
        st.sidebar.error("No optimal parameters found in session state.")

# --- 5. Streamlit User Interface ---
def update_state():
    """
    Callback to update the main session state from the 'widget_' keys.
    This synchronizes all widgets.
    """
    # Get all keys from session state
    keys = list(st.session_state.keys())
    
    # Loop through all keys that start with 'widget_'
    for widget_key in keys:
        if widget_key.startswith('widget_'):
            # Find the main key (e.g., 'widget_ma_period' -> 'ma_period')
            main_key = widget_key[len('widget_'):]
            
            # Update the main key with the widget's value
            if main_key in st.session_state:
                st.session_state[main_key] = st.session_state[widget_key]

# --- UPDATED: Helper to check for blank rows (Checks Factors, Stats, Notes, AND Run status) ---
def is_row_blank(s):
    """Checks if a row from the user-defined table is blank."""
    factors = ["RSI", "Volatility", "TREND", "Volume", "MACD", "MA Slope", "Markov", "ADX Filter"]
    
    # Check 1: Are all factors Off?
    all_factors_off = all(str(s.get(f, "Off")).lower() in ["off", "0", "0.0"] for f in factors)
    
    # Check 2: Are stats zero?
    stats_are_zero = s.get("Z_Num_Trades", 0) == 0
    
    # Check 3: Is the note empty?
    note_is_empty = str(s.get("Notes", "")).strip() == ""

    # Check 4: Is the Run box unchecked?
    run_is_unchecked = s.get("Run", False) is False
    
    # The row is considered blank ONLY if ALL conditions are met
    return all_factors_off and stats_are_zero and note_is_empty and run_is_unchecked

# --- [NEW] 6. Markov Chain Optimisation Functions ---
def run_single_markov_test(params, master_df, tickers, date_range):
    """
    Worker function for the Markov optimisation.
    Tests a single combination of (run_up_period, future_period).
    """
    run_up_period = params['run_up']
    future_period = params['future']

    total_results = {
        'Up -> Up': {'profit': 0.0, 'count': 0},
        'Up -> Down': {'profit': 0.0, 'count': 0},
        'Down -> Up': {'profit': 0.0, 'count': 0},
        'Down -> Down': {'profit': 0.0, 'count': 0}
    }

    for ticker in tickers:
        if ticker not in master_df.columns:
            continue

        # Get only the 'Close' price for this ticker
        ticker_data = master_df[ticker].loc[date_range[0]:date_range[1]].to_frame(name='Close')

        # --- [NEW] CLEANING LOGIC TO PREVENT INFINITY ---
        ticker_data['Close'] = pd.to_numeric(ticker_data['Close'], errors='coerce').replace(0, np.nan)
        ticker_data.dropna(subset=['Close'], inplace=True)
        # --- [END NEW] ---

        if ticker_data.empty or ticker_data['Close'].isna().all():
            continue

        # 1. Calculate Run-Up state (past)
        # pct_change(N) calculates (price[t] / price[t-N]) - 1
        ticker_data['RunUp_Return'] = ticker_data['Close'].pct_change(periods=run_up_period)
        ticker_data['RunUp_State'] = ticker_data['RunUp_Return'].apply(lambda x: 'Up' if x > 0 else 'Down')

        # 2. Calculate Future state (what *actually* happened)
        # shift(-N) looks *forward* N periods
        ticker_data['Future_Return'] = (ticker_data['Close'].shift(-future_period) / ticker_data['Close']) - 1

        # 3. Drop NaNs created by the shifts/pct_change
        ticker_data.dropna(subset=['RunUp_State', 'Future_Return'], inplace=True)

        if ticker_data.empty:
            continue

        # 4. Tally results
        # We use .values for speed
        runup_states = ticker_data['RunUp_State'].values
        future_returns = ticker_data['Future_Return'].values

        for i in range(len(runup_states)):
            state = runup_states[i]
            ret = future_returns[i]

            # --- [NEW] Check for infinity returns ---
            if not np.isfinite(ret):
                continue
            # --- [END NEW] ---

            if state == 'Up':
                # Strategy: Bet on Up
                total_results['Up -> Up']['profit'] += ret
                total_results['Up -> Up']['count'] += 1
                # Strategy: Bet on Down
                total_results['Up -> Down']['profit'] += (ret * -1)
                total_results['Up -> Down']['count'] += 1
            else: # State is 'Down'
                # Strategy: Bet on Up
                total_results['Down -> Up']['profit'] += ret
                total_results['Down -> Up']['count'] += 1
                # Strategy: Bet on Down
                total_results['Down -> Down']['profit'] += (ret * -1)
                total_results['Down -> Down']['count'] += 1

    # 5. Compile final metrics
    final_report = []
    for strategy, results in total_results.items():
        if results['count'] > 0:
            avg_pnl = (results['profit'] / results['count'])
            # We use a simple score: Avg P&L * log(count) to value consistency
            score = avg_pnl * np.log10(results['count'] + 1)
        else:
            avg_pnl = 0
            score = 0

        final_report.append({
            'Run-Up Period': run_up_period,
            'Future Period': future_period,
            'Strategy': strategy,
            'Avg. P/L': avg_pnl,
            'Total Occurrences': results['count'],
            'Total P/L': results['profit'],
            'Score': score
        })

    return final_report


def generate_and_run_markov_optimisation(master_df, main_content_placeholder, side):
    """
    Main UI and orchestrator for Markov Chain optimisation.
    [NEW] This version calculates an 'Alpha Score per Day' to find the
    most efficient predictive edge.
    """
    st.session_state.summary_df = None
    st.session_state.single_ticker_results = None
    st.session_state.confidence_results_df = None
    st.session_state.open_trades_df = None
    st.session_state.advisor_df = None; st.session_state.raw_df = None; st.session_state.deduped_df = None
    st.session_state.best_params = None
    st.session_state.best_weights = None
    st.session_state.markov_results_df = None # Clear previous Markov results
    st.session_state.best_markov_setup = None

    with main_content_placeholder.container():

        st.header(f"🔮 Finding Best Markov Probabilities ({side.title()})")

        # --- Get UI inputs from session state ---
        run_up_start = st.session_state.markov_run_up_start
        run_up_end = st.session_state.markov_run_up_end
        run_up_step = st.session_state.markov_run_up_step
        future_start = st.session_state.markov_future_start
        future_end = st.session_state.markov_future_end
        future_step = st.session_state.markov_future_step

        # --- Create parameter combinations ---
        run_up_range = range(run_up_start, run_up_end + 1, run_up_step)
        future_range = range(future_start, future_end + 1, future_step)

        param_product = itertools.product(run_up_range, future_range)
        param_combinations = [{
            "run_up": p[0], "future": p[1]
        } for p in param_product]

        total_combinations = len(param_combinations)
        if total_combinations == 0:
            st.warning("No combinations to test. Check your ranges in the sidebar.")
            st.session_state.run_markov_optimisation = False # Reset flag
            st.stop()

        num_cores = cpu_count()
        st.info(f"Starting Markov optimisation on {num_cores} cores... Testing {total_combinations} period combinations.")

        tickers_to_run = [col for col in master_df.columns if not ('_Volume' in str(col) or '_High' in str(col) or '_Low' in str(col))]
        date_range = (pd.Timestamp(st.session_state.start_date), pd.Timestamp(st.session_state.end_date))

        status_text = st.empty(); status_text.text("Optimisation starting...")
        progress_bar = st.progress(0) # <-- The progress bar

        worker_func = partial(run_single_markov_test, master_df=master_df, tickers=tickers_to_run, date_range=date_range)

        results_list = []
        with Pool(processes=num_cores) as pool:
            try:
                iterator = pool.imap_unordered(worker_func, param_combinations)
                for i, result_group in enumerate(iterator, 1):
                    results_list.extend(result_group) # Add the 4 strategies
                    progress_bar.progress(i / total_combinations, text=f"Optimising... {i}/{total_combinations} combinations complete.")
            except Exception as e:
                st.error(f"An error occurred during multiprocessing: {e}")
                status_text.text("Optimisation failed due to an error.");
                st.session_state.run_markov_optimisation = False # Reset flag
                return

        status_text.text("Optimisation complete. Compiling results...")

        if not results_list:
            st.warning("Optimisation finished, but no valid results were found.")
            st.session_state.run_markov_optimisation = False # Reset flag
            return

        results_df = pd.DataFrame(results_list)

        # --- [NEW ALPHA SCORE PER DAY LOGIC] ---
        # 1. Pivot the data to get all 4 strategies in one row per time period
        pivot_df = results_df.pivot_table(
            index=['Run-Up Period', 'Future Period'],
            columns='Strategy',
            values='Avg. P/L' # We use Avg. P/L for the alpha calculation
        ).reset_index()

        # 2. Calculate the "Alpha Scores"
        pivot_df['Long Alpha Score'] = pivot_df.get('Down -> Up', 0) - pivot_df.get('Up -> Up', 0)
        pivot_df['Short Alpha Score'] = pivot_df.get('Up -> Down', 0) - pivot_df.get('Down -> Down', 0)

        # 3. Calculate the "Alpha Score per Day"
        pivot_df['Long Alpha Score per Day'] = pivot_df['Long Alpha Score'] / pivot_df['Future Period']
        pivot_df['Short Alpha Score per Day'] = pivot_df['Short Alpha Score'] / pivot_df['Future Period']

        # 4. Join this back to the original results to get counts, etc.
        results_df = results_df.set_index(['Run-Up Period', 'Future Period'])
        pivot_df = pivot_df.set_index(['Run-Up Period', 'Future Period'])

        # Join the new Alpha Scores to the main results
        results_df = results_df.join(pivot_df[['Long Alpha Score', 'Short Alpha Score', 'Long Alpha Score per Day', 'Short Alpha Score per Day']])
        results_df = results_df.reset_index()

        # 5. Determine which Alpha Score and strategies to show
        if side == 'long':
            score_to_use = 'Long Alpha Score per Day'
            target_strategies = ['Down -> Up', 'Up -> Up']
        else: # short
            score_to_use = 'Short Alpha Score per Day'
            target_strategies = ['Up -> Down', 'Down -> Down']

        # Filter for the strategies we care about for this 'side'
        final_df = results_df[results_df['Strategy'].isin(target_strategies)].copy()

        if final_df.empty:
            st.warning(f"No valid results found for {side}-biased strategies.")
            st.session_state.run_markov_optimisation = False # Reset flag
            return

        # 6. Sort by the new "Alpha Score per Day"
        final_df = final_df.sort_values(by=score_to_use, ascending=False)

        # Get the best row (which will be the one with the highest Alpha per Day)
        best_setup_series = final_df.iloc[0]

        st.success(f"Markov Optimisation Complete! Best 'Alpha' strategy found:")
        st.subheader("Best Markov Setup (by Alpha Score per Day):")

        # Save the best setup to session state
        st.session_state.best_markov_setup = best_setup_series.to_dict()
        save_markov_setup(st.session_state.best_markov_setup) # <-- SAVE TO FILE
        st.json(st.session_state.best_markov_setup)

        st.subheader(f"Top 10 Setups ({side.title()}-biased, sorted by {score_to_use}):")
        display_cols = [
            'Strategy', 'Run-Up Period', 'Future Period', score_to_use, 'Avg. P/L', 'Total Occurrences'
        ]

        st.dataframe(final_df.head(10)[display_cols].style.format({
            score_to_use: "{:.4%}", # Format as percentage
            "Avg. P/L": "{:.4%}",
        }))

        # --- [NEW] HEATMAP INFOGRAPHIC ---
        st.subheader(f"Markov '{side.title()}' Alpha Score per Day Heatmap")
        st.caption("This heatmap shows the most *efficient* signals (highest predictive value per day held).")
        try:
            # We use the pivot_df we created earlier, which has the scores
            heatmap_data = pivot_df.pivot_table(
                index='Run-Up Period',
                columns='Future Period',
                values=score_to_use # Plot the Alpha Score per Day
            )

            # Create the heatmap
            st.dataframe(
                heatmap_data.style
                .background_gradient(cmap='RdYlGn', axis=None) # Red-Yellow-Green colormap
                .format("{:.4%}", na_rep='-') # Format as percentage
            )
        except Exception as e:
            st.warning(f"Could not generate heatmap. Error: {e}")
        # --- [END NEW] ---

        st.session_state.markov_results_df = final_df # Save the sorted results
        st.session_state.run_markov_optimisation = False # Reset flag

# --- Corrected: Ensures rsi_logic AND primary_driver are passed ---
def run_confidence_optimisation(optimise_for, find_mode, master_df, main_content_placeholder, veto_factors):
    st.session_state.summary_df = None
    st.session_state.single_ticker_results = None
    st.session_state.open_trades_df = None
    st.session_state.best_params = None
    st.session_state.advisor_df = None
    st.session_state.worst_confidence_setups_list = []

    with main_content_placeholder.container():
        num_cores = cpu_count()
        st.info(f"Starting to find **{find_mode.upper()}** {optimise_for.upper()} setups on {num_cores} CPU cores...")
        st.caption("Note: This process runs in parallel for maximum speed. The status below updates as each strategy combination completes.")

        # --- [NEW] Added Markov ---
        factors = ['RSI', 'Volatility', 'TREND', 'Volume', 'MACD', 'MA Slope', 'Markov']
        num_factors = len(factors)

        if find_mode == 'worst':
            if veto_factors is None or len(veto_factors) != num_factors:
                st.error("Internal error: Veto factors not provided correctly.")
                return
            target_combo = veto_factors
            on_off_combos = [c for c in itertools.product([False, True], repeat=num_factors) if c == target_combo]
            if not on_off_combos or not any(on_off_combos[0]):
                st.warning("Please select at least one factor for the Veto search."); return
        else:
            on_off_combos = [c for c in itertools.product([False, True], repeat=num_factors) if any(c)]

        thresholds_to_test = [20, 25, 30, 35, 40, 45, 50]
        tasks = list(itertools.product(on_off_combos, thresholds_to_test, [1.0]))
        total_tasks = len(tasks)

        base_params = { "large_ma_period": st.session_state.ma_period, "bband_period": st.session_state.bb_period, "bband_std_dev": st.session_state.bb_std, "long_entry_threshold_pct": st.session_state.long_entry / 100, "long_exit_ma_threshold_pct": st.session_state.long_exit / 100, "long_trailing_stop_loss_pct": st.session_state.long_sl / 100, "long_delay_days": st.session_state.long_delay, "short_entry_threshold_pct": st.session_state.short_entry / 100, "short_exit_ma_threshold_pct": st.session_state.short_exit / 100, "short_trailing_stop_loss_pct": st.session_state.short_sl / 100, "short_delay_days": st.session_state.short_delay, }
        tickers_to_run = sorted([col for col in master_df.columns if not ('_Volume' in str(col) or '_High' in str(col) or '_Low' in str(col))])
        date_range = (pd.Timestamp(st.session_state.start_date), pd.Timestamp(st.session_state.end_date))

        factor_weights = {
            "rsi": st.session_state.rsi_w, "vol": st.session_state.vol_w,
            "trend": st.session_state.trend_w, "volume": st.session_state.volume_w,
            "macd": st.session_state.macd_w, "ma_slope": st.session_state.ma_slope_w,
            "markov": st.session_state.markov_w,
            "use_adx": st.session_state.use_adx_filter,
            "adx_thresh": st.session_state.adx_threshold,
            "adx_period": st.session_state.adx_period,
            "rsi_logic": st.session_state.rsi_logic, 
            "primary_driver": st.session_state.primary_driver,
            'markov_setup': st.session_state.get('best_markov_setup'), 
            'exit_logic': st.session_state.exit_logic_type, 
            'exit_thresh': st.session_state.exit_confidence_threshold,
            'smart_trailing_stop': st.session_state.smart_trailing_stop_pct / 100.0,
            'smart_exit_atr_period': st.session_state.smart_exit_atr_period,
            'smart_exit_atr_multiplier': st.session_state.smart_exit_atr_multiplier,
            'intelligent_tsl_pct': st.session_state.intelligent_tsl_pct / 100.0
        }

        worker_func = partial(run_single_confidence_test, base_params=base_params, master_df=master_df, date_range=date_range, tickers_to_run=tickers_to_run, optimise_for=optimise_for, factor_weights=factor_weights)

        results_list = []
        
        # --- UI Feedback Elements ---
        progress_bar = st.progress(0, text="Initializing parallel engines...")
        status_text = st.empty()
        
        with Pool(processes=num_cores) as pool:
            try:
                iterator = pool.imap_unordered(worker_func, tasks)
                for i, result in enumerate(iterator, 1):
                    if isinstance(result, dict) and "Trade G/B Ratio" in result:
                         results_list.append(result)
                         
                         # --- DYNAMIC STATUS UPDATE ---
                         # This creates the "Screaming Past" effect for strategies
                         active = []
                         if result['RSI']: active.append("RSI")
                         if result['Volatility']: active.append("Vol")
                         if result['TREND']: active.append("Trend")
                         if result['Volume']: active.append("VolSpike")
                         if result['MACD']: active.append("MACD")
                         if result['MA Slope']: active.append("Slope")
                         if result['Markov']: active.append("Markov")
                         
                         status_msg = f"Analyzed: [{' + '.join(active)}] @ Thresh {result['Conf. Threshold']} -> Profit: {result['Avg Profit/Trade']:.2%} ({result['Total Trades']} trades)"
                         status_text.text(status_msg)
                         
                    else: print(f"Warning: Worker returned invalid result format: {result}")
                    
                    # Update progress bar
                    progress_bar.progress(i / total_tasks, text=f"Optimising... {i}/{total_tasks} combinations complete.")
                    
            except Exception as e:
                st.error(f"An error occurred during multiprocessing: {e}")
                progress_bar.empty(); return

        if results_list:
            results_df = pd.DataFrame(results_list)
            
            # --- [NEW] FILTER OUT "INFINITY" RESULTS ---
            # We exclude results where Trade G/B Ratio is > 50000 (the placeholder for 0 losses).
            # This prevents low-volume "perfect" trades from gaming the Uncapped Scoring system.
            if 'Trade G/B Ratio' in results_df.columns:
                 results_df = results_df[results_df['Trade G/B Ratio'] < 50000].copy()
            
            if results_df.empty:
                 st.warning("No valid setups found after removing infinite ratio outliers.")
                 st.session_state.run_confidence_optimisation = False
                 return

            # --- Sort by the NEW 3-Factor Weighted Score ---
            sort_col = "Good Score" if find_mode == 'best' else "Bad Score"
            
            if sort_col in results_df.columns:
                fill_value = -np.inf if find_mode == 'best' else 0
                results_df[sort_col] = results_df[sort_col].fillna(fill_value)
                results_df = results_df.sort_values(by=sort_col, ascending=False).reset_index(drop=True)
            else: st.error(f"Sorting column '{sort_col}' not found."); return

            for factor in factors:
                if factor in results_df.columns:
                    results_df[factor] = results_df[factor].apply(lambda x: "On" if x else "Off")
            
            # Save all results to state so they persist
            st.session_state.confidence_results_df = results_df 

            if find_mode == 'best':
                st.subheader(f"📊 Top 60 Confidence Setups ({optimise_for.title()} Trades)")
                if not results_df.empty:
                    best_setup = results_df.iloc[0]
                    st.session_state.best_confidence_setup = best_setup.to_dict()
                    
                    # Save TOP setups using the same DF (which is now sorted by Weighted Score)
                    save_top_setups(results_df, optimise_for)
                else: 
                    st.warning("No valid 'best' setups found.")
                    st.session_state.best_confidence_setup = None
            else: # 'worst' mode
                valid_worst_df = results_df[results_df['Trade G/B Ratio'] < 99999.0].copy()
                if not valid_worst_df.empty:
                    valid_worst_df['Bad Score'] = valid_worst_df['Bad Score'].fillna(0)
                    valid_worst_df = valid_worst_df.sort_values(by="Bad Score", ascending=False).reset_index(drop=True)
                    st.session_state.worst_confidence_setups_list = valid_worst_df.head(4).to_dict('records')
                    st.info(f"Top {len(st.session_state.worst_confidence_setups_list)} valid 'worst' setups ready.")
                    st.subheader(f"🏆 Top 60 Valid Worst Setups ({optimise_for.title()} Trades)")
                    results_df = valid_worst_df # Use this for display
                else: 
                    st.warning("No valid 'worst' setups found.")
                    st.session_state.worst_confidence_setups_list = []
            
            # --- Display the table ---
            display_df_conf = results_df.head(60)
            conf_formatters = { "Avg Profit/Trade": "{:.2%}", "Ticker G/B Ratio": "{:.2f}", "Trade G/B Ratio": "{:.2f}", "Avg Entry Conf.": "{:.1f}%", "Good Score": "{:.4f}", "Bad Score": "{:.4f}", "Norm. Score %": "{:.2f}%" }
            if 'MACD' in display_df_conf.columns: conf_formatters['MACD'] = '{}'
            if 'MA Slope' in display_df_conf.columns: conf_formatters['MA Slope'] = '{}'
            if 'Markov' in display_df_conf.columns: conf_formatters['Markov'] = '{}'
            
            valid_conf_formatters = {k: (lambda val, fmt=v: fmt.format(val) if pd.notna(val) else '-')
                                     for k, v in conf_formatters.items() if k in display_df_conf.columns}
            st.dataframe(display_df_conf.style.format(valid_conf_formatters, na_rep='-'))

        else:
            st.warning("Optimisation completed but no results were generated."); st.session_state.confidence_results_df = None
        
        st.session_state.run_confidence_optimisation = False # Reset flag

def generate_user_advisor_ui_and_run(main_df):
    st.header("⚙️ User-Defined Advisor Setups")
    data_key = "user_setups_data"
    widget_key = "user_setups_editor_state"
    
    def load_saved_setups_callback():
        st.session_state[data_key] = load_user_setups()
        if widget_key in st.session_state: del st.session_state[widget_key]

    def clear_all_setups_callback():
        blank_setups = [load_user_setups()[0].copy() for _ in range(20)]
        st.session_state[data_key] = blank_setups
        if widget_key in st.session_state: del st.session_state[widget_key]

    st.caption("Add advice/notes in the 'Notes' column. Check 'Run' to test.")

    col_config = {
        "Run": st.column_config.CheckboxColumn("Run", width="small"),
        # Added Notes Column
        "Notes": st.column_config.TextColumn("Notes", help="Advice/Description for this setup", width="medium"), 
        
        "RSI": st.column_config.TextColumn("RSI", width="small"),
        "Volatility": st.column_config.TextColumn("Volatility", width="small"),
        "TREND": st.column_config.TextColumn("TREND", width="small"),
        "Volume": st.column_config.TextColumn("Volume", width="small"),
        "MACD": st.column_config.TextColumn("MACD", width="small"),
        "MA Slope": st.column_config.TextColumn("MA Slope", width="small"),
        "Markov": st.column_config.TextColumn("Markov", width="small"),
        "ADX Filter": st.column_config.TextColumn("ADX Filter", width="small"),
        "Conf. Threshold": st.column_config.NumberColumn("Conf.", width="small"),
        "Large MA Period": st.column_config.NumberColumn("MA", width="small"),
        "Bollinger Band Period": st.column_config.NumberColumn("BB", width="small"),
        "Bollinger Band Std Dev": st.column_config.NumberColumn("Std", format="%.1f", width="small"),
        "Long Entry Threshold (%)": st.column_config.NumberColumn("L Entry", format="%.1f", width="small"),
        "Long Exit Threshold (%)": st.column_config.NumberColumn("L Exit", format="%.1f", width="small"),
        "Long Stop Loss (%)": st.column_config.NumberColumn("L TSL", format="%.1f", width="small"),
        "Long Delay (Days)": st.column_config.NumberColumn("L Delay", width="small"),
        "Short Entry Threshold (%)": st.column_config.NumberColumn("S Entry", format="%.1f", width="small"),
        "Short Exit Threshold (%)": st.column_config.NumberColumn("S Exit", format="%.1f", width="small"),
        "Short Stop Loss (%)": st.column_config.NumberColumn("S TSL", format="%.1f", width="small"),
        "Short Delay (Days)": st.column_config.NumberColumn("S Delay", width="small"),
        "Z_Avg_Profit": st.column_config.NumberColumn("Avg %", format="%.2f%%", disabled=True, width="small"),
        "Z_Num_Trades": st.column_config.NumberColumn("Trades", disabled=True, width="small"),
        "Z_WL_Ratio": st.column_config.NumberColumn("W/L", format="%.2f", disabled=True, width="small"),
    }

    edited_setups_list = st.data_editor(
        st.session_state[data_key],
        column_config=col_config,
        num_rows="dynamic",
        key=widget_key,
        column_order=[
            "Run", "Notes", # <--- Notes column appears here (2nd)
            "RSI", "Volatility", "TREND", "Volume", "MACD", "MA Slope", "Markov", "ADX Filter",
            "Conf. Threshold",
            "Large MA Period", "Bollinger Band Period", "Bollinger Band Std Dev",
            "Long Entry Threshold (%)", "Long Exit Threshold (%)", "Long Stop Loss (%)", "Long Delay (Days)",
            "Short Entry Threshold (%)", "Short Exit Threshold (%)", "Short Stop Loss (%)", "Short Delay (Days)",
            "Z_Avg_Profit", "Z_Num_Trades", "Z_WL_Ratio"
        ]
    )

    col1, col2, col3, col4, col5 = st.columns([1, 1, 1, 2, 2])
    if col1.button("💾 Save Setups"):
        save_user_setups(edited_setups_list[:20])
        st.session_state[data_key] = edited_setups_list[:20]
    col2.button("🔄 Load Saved", on_click=load_saved_setups_callback)
    col3.button("🗑️ Clear All Setups", on_click=clear_all_setups_callback)

    if col4.button("Scan with User Setups", type="primary"):
        current_setups_list = edited_setups_list[:20]
        setups_to_run = [s for s in current_setups_list if s.get("Run") == True]
        if len(setups_to_run) == 0: st.error("No setups selected.")
        elif len(setups_to_run) > 8: st.error(f"Selected {len(setups_to_run)} setups. Max is 8.")
        else:
            st.session_state.run_scan_user_setups = True
            st.session_state.run_user_advisor_setup = False 
            st.session_state.advisor_df = None
            st.session_state.setups_to_scan = setups_to_run
            st.rerun()

    if col5.button("Back to Main Analysis"):
        st.session_state.run_user_advisor_setup = False; st.session_state.advisor_df = None; st.rerun()

def generate_advisor_report(main_df):
    """Handles the UI for running the 'Top Setups' advisor."""
    st.header("📈 Advanced Advisor Report (Top 8 Setups)")
    top_setups = load_top_setups()

    if not top_setups:
        st.warning("No saved top setups found. Run 'Find Best Confidence' first.")
        if st.button("Back"):
            st.session_state.run_advanced_advisor = False
            st.rerun()
        return

    side = st.radio("Generate report for which saved setups?", ("Long", "Short"), horizontal=True, key="advisor_side_select_top")
    setups_to_run_all = top_setups.get(side.lower())

    if not setups_to_run_all:
        st.warning(f"No saved top {side.lower()} setups found.")
        if st.button("Back"):
            st.session_state.run_advanced_advisor = False
            st.rerun()
        return

    if st.button(f"Scan using Top {side} Setups", type="primary"):
         st.session_state.advisor_df = None; st.session_state.raw_df = None; st.session_state.deduped_df = None
         run_advisor_scan(main_df, setups_to_run_all, "Top Setups") # Call the scanner

    if st.button("Back"):
        st.session_state.run_advanced_advisor = False
        st.session_state.advisor_df = None; st.session_state.raw_df = None; st.session_state.deduped_df = None
        st.rerun()

def apply_best_weights_to_widgets():
    """Loads optimised weights from st.session_state.best_weights into the sidebar widgets."""
    if 'best_weights' in st.session_state and st.session_state.best_weights:
        weights = st.session_state.best_weights
        
        if 'rsi_w' in weights: st.session_state.rsi_w = weights['rsi_w']
        if 'vol_w' in weights: st.session_state.vol_w = weights['vol_w']
        if 'trend_w' in weights: st.session_state.trend_w = weights['trend_w']
        if 'volume_w' in weights: st.session_state.volume_w = weights['volume_w']
        if 'macd_w' in weights: st.session_state.macd_w = weights['macd_w']
        if 'ma_slope_w' in weights: st.session_state.ma_slope_w = weights['ma_slope_w']
        
        st.sidebar.success("Optimal weights loaded into sidebar!")
        st.rerun()
    else:
        st.sidebar.error("No optimal weights found in session state.")

def apply_best_params_to_widgets():
    """Loads parameters from st.session_state.best_params into the sidebar widgets."""
    if 'best_params' in st.session_state and st.session_state.best_params:
        params = st.session_state.best_params
        
        if 'large_ma_period' in params: st.session_state.ma_period = params['large_ma_period']
        if 'bband_period' in params: st.session_state.bb_period = params['bband_period']
        if 'bband_std_dev' in params: st.session_state.bb_std = params['bband_std_dev']
        if 'confidence_threshold' in params: st.session_state.confidence_slider = params['confidence_threshold']
        
        if 'long_entry_threshold_pct' in params: st.session_state.long_entry = params['long_entry_threshold_pct'] * 100
        if 'long_exit_ma_threshold_pct' in params: st.session_state.long_exit = params['long_exit_ma_threshold_pct'] * 100
        if 'long_trailing_stop_loss_pct' in params: st.session_state.long_sl = params['long_trailing_stop_loss_pct'] * 100
        if 'long_delay_days' in params: st.session_state.long_delay = params['long_delay_days']
        
        if 'long_entry_threshold_pct' in params: st.session_state.short_entry = params['long_entry_threshold_pct'] * 100
        if 'long_exit_ma_threshold_pct' in params: st.session_state.short_exit = params['long_exit_ma_threshold_pct'] * 100
        if 'long_trailing_stop_loss_pct' in params: st.session_state.short_sl = params['long_trailing_stop_loss_pct'] * 100
        if 'long_delay_days' in params: st.session_state.long_delay = params['long_delay_days']
        
        st.sidebar.success("Optimal parameters loaded into sidebar!")
        st.rerun()
    else:
        st.sidebar.error("No optimal parameters found in session state.")

# --- 5. Streamlit User Interface ---
def update_state():
    """
    Callback to update the main session state from the 'widget_' keys.
    This synchronizes all widgets.
    """
    # Get all keys from session state
    keys = list(st.session_state.keys())
    
    # Loop through all keys that start with 'widget_'
    for widget_key in keys:
        if widget_key.startswith('widget_'):
            # Find the main key (e.g., 'widget_ma_period' -> 'ma_period')
            main_key = widget_key[len('widget_'):]
            
            # Update the main key with the widget's value
            if main_key in st.session_state:
                st.session_state[main_key] = st.session_state[widget_key]

def main():
    st.set_page_config(page_title="Dave's Quant System", page_icon="🔴", layout="wide")

    # --- [FIX 1: Robust Initialization] ---
    # We check for 'run_mode' explicitly to fix the crash if session state is stale
    if 'first_run' not in st.session_state or 'run_mode' not in st.session_state:
        st.session_state.first_run = True
        defaults = load_settings() 
        st.session_state.widget_defaults = defaults 
        st.session_state.veto_setup_list = load_veto_setup()

        st.session_state.use_ma_floor_filter = defaults.get("use_ma_floor_filter", True)

        # Unpack all widget defaults into session state
        st.session_state.ticker_select = None 
        st.session_state.run_mode = "Analyse Full List"  # <--- This was missing in your stale state
        
        st.session_state.start_date = None
        st.session_state.end_date = None

        # Section 2 Defaults
        st.session_state.use_rsi = defaults.get('use_rsi', True)
        st.session_state.rsi_w = defaults.get('rsi_w', 1.0)
        st.session_state.rsi_logic = defaults.get('rsi_logic', 'Crossover')
        st.session_state.primary_driver = defaults.get('primary_driver', 'Bollinger Bands')
        st.session_state.exit_logic_type = defaults.get('exit_logic_type', 'Standard (Price-Based)')
        st.session_state.exit_confidence_threshold = defaults.get('exit_confidence_threshold', 50)
        st.session_state.smart_trailing_stop_pct = defaults.get('smart_trailing_stop_pct', 5.0)
        st.session_state.use_vol = defaults.get('use_vol', True)
        st.session_state.vol_w = defaults.get('vol_w', 1.0)
        st.session_state.use_trend = defaults.get('use_trend', True)
        st.session_state.trend_w = defaults.get('trend_w', 1.5)
        st.session_state.use_volume = defaults.get('use_volume', True)
        st.session_state.volume_w = defaults.get('volume_w', 1.0)
        st.session_state.use_adx_filter = defaults.get('use_adx_filter', True)
        st.session_state.adx_threshold = defaults.get('adx_threshold', 25.0)
        st.session_state.adx_period = defaults.get('adx_period', 14) 
        st.session_state.use_macd = defaults.get('use_macd', True)
        st.session_state.macd_w = defaults.get('macd_w', 1.0)
        st.session_state.use_ma_slope = defaults.get('use_ma_slope', True)
        st.session_state.ma_slope_w = defaults.get('ma_slope_w', 0.5)
        st.session_state.use_markov = defaults.get('use_markov', False)
        st.session_state.markov_w = defaults.get('markov_w', 1.0)
        st.session_state.confidence_slider = defaults.get("confidence_threshold", 50)
        
        st.session_state.smart_exit_atr_period = defaults.get("smart_exit_atr_period", 14)
        st.session_state.smart_exit_atr_multiplier = defaults.get("smart_exit_atr_multiplier", 3.0)
        st.session_state.intelligent_tsl_pct = defaults.get("intelligent_tsl_pct", 0.60) * 100 

        # Section 3 Defaults
        st.session_state.ma_period = defaults.get("large_ma_period", 50)
        st.session_state.bb_period = defaults.get("bband_period", 20)
        st.session_state.bb_std = defaults.get("bband_std_dev", 2.0)
        st.session_state.long_entry = defaults.get("long_entry_threshold_pct", 0.0) * 100
        st.session_state.long_exit = defaults.get("long_exit_ma_threshold_pct", 0.0) * 100
        st.session_state.long_sl = defaults.get("long_trailing_stop_loss_pct", 8.0) * 100
        st.session_state.long_delay = defaults.get("long_delay_days", 0)
        st.session_state.short_entry = defaults.get("short_entry_threshold_pct", 0.0) * 100
        st.session_state.short_exit = defaults.get("short_exit_ma_threshold_pct", 0.0) * 100
        st.session_state.short_sl = defaults.get("short_trailing_stop_loss_pct", 8.0) * 100
        st.session_state.short_delay = defaults.get("short_delay_days", 0)
        st.session_state.max_duration = defaults.get("max_trading_days", 60)
        st.session_state.max_long_duration = defaults.get("max_long_duration", 60)
        st.session_state.max_short_duration = defaults.get("max_short_duration", 10)

        # Section 4/5/6/7 Init (Optimisation variables)
        st.session_state.sq_params_toggle = False
        st.session_state.opt_ma_cb = False; st.session_state.opt_bb_cb = False; st.session_state.opt_std_cb = False
        st.session_state.opt_conf_cb = False; st.session_state.opt_sl_cb = False; st.session_state.opt_delay_cb = False
        st.session_state.opt_entry_cb = False; st.session_state.opt_exit_cb = False; st.session_state.opt_duration_cb = False
        
        # Init Opt Ranges (Defaults)
        st.session_state.ma_start = 50; st.session_state.ma_end = 55; st.session_state.ma_step = 5
        st.session_state.bb_start = 20; st.session_state.bb_end = 25; st.session_state.bb_step = 5
        st.session_state.std_start = 2.0; st.session_state.std_end = 2.1; st.session_state.std_step = 0.1
        st.session_state.conf_start = 50; st.session_state.conf_end = 60; st.session_state.conf_step = 5
        st.session_state.sl_start = 0.0; st.session_state.sl_end = 2.0; st.session_state.sl_step = 0.5
        st.session_state.delay_start = 0; st.session_state.delay_end = 1; st.session_state.delay_step = 1
        st.session_state.entry_start = 0.0; st.session_state.entry_end = 0.5; st.session_state.entry_step = 0.1
        st.session_state.exit_start = 0.0; st.session_state.exit_end = 0.5; st.session_state.exit_step = 0.1
        st.session_state.dur_start = 30; st.session_state.dur_end = 90; st.session_state.dur_step = 10

        st.session_state.veto_rsi_cb = True; st.session_state.veto_vol_cb = True; st.session_state.veto_trend_cb = True
        st.session_state.veto_volume_cb = True; st.session_state.veto_macd_cb = False; st.session_state.veto_ma_slope_cb = False
        st.session_state.sq_weights_toggle = False
        
        # Markov Init
        st.session_state.markov_run_up_start = 5; st.session_state.markov_run_up_end = 20; st.session_state.markov_run_up_step = 2
        st.session_state.markov_future_start = 3; st.session_state.markov_future_end = 10; st.session_state.markov_future_step = 1
        st.session_state.best_markov_setup = load_markov_setup()

        # Data Editor Init
        st.session_state["user_setups_data"] = []
        loaded_setups = load_user_setups()
        processed = []
        factor_cols = ["RSI", "Volatility", "TREND", "Volume", "MACD", "MA Slope", "Markov"]
        for s in loaded_setups:
            new_s = s.copy()
            for k, v in s.items():
                if k in factor_cols: new_s[k] = str(v)
                elif k == "Run": new_s[k] = bool(v)
            processed.append(new_s)
        st.session_state["user_setups_data"] = processed

        # --- Password Auth Init ---
        st.session_state.dev_authenticated = False

    # --- [Load Data Logic] ---
    if 'master_df' not in st.session_state or st.session_state.master_df is None:
        with st.spinner("Loading and cleaning data..."):
            master_df, load_message = load_all_data('csv_data')
            if master_df is None: st.error(load_message); st.stop()
            master_df, outlier_report = clean_data_and_report_outliers(master_df)
            st.session_state.master_df = master_df
            st.session_state.ticker_list = sorted([col for col in master_df.columns if not ('_Volume' in str(col) or '_High' in str(col) or '_Low' in str(col))])

            min_date = master_df.index.min().date()
            max_date = master_df.index.max().date()
            
            # --- NEW: Set Default Start to 1 Year Ago from TODAY ---
            target_start_date = date.today() - relativedelta(years=1)
            
            # Safety Check: Ensure the target date is valid within your CSV data range
            if target_start_date < min_date:
                st.session_state.start_date = min_date
            elif target_start_date > max_date:
                st.session_state.start_date = min_date # Fallback if data is older than 1 year ago
            else:
                st.session_state.start_date = target_start_date
                
            st.session_state.end_date = max_date
            
            if st.session_state.ticker_select is None and st.session_state.ticker_list:
                st.session_state.ticker_select = st.session_state.ticker_list[0]
            st.session_state.load_message = load_message
            st.session_state.outlier_report = outlier_report

    master_df = st.session_state.master_df
    ticker_list = st.session_state.ticker_list

    # --- HEADER / GREETING ---
    st.title("🔴 🚀 Dave's Quant system")
    
    current_hour = datetime.now().hour
    if 5 <= current_hour < 12: greeting = "Good morning!"
    elif 12 <= current_hour < 19: greeting = "Good afternoon!"
    else: greeting = "Good evening!"
    
    today = date.today()
    day_str = today.strftime('%A, %d %B %Y') 
    
    proverbs = load_proverbs()
    day_of_year = today.timetuple().tm_yday
    proverb_index = (day_of_year - 1) % len(proverbs) if proverbs else 0
    daily_proverb = proverbs[proverb_index] if proverbs else "Have a profitable day."
    
    st.success(f"{greeting} Today is {day_str}. | *{daily_proverb}*")

    main_content_placeholder = st.empty()

    # --- SIDEBAR TOGGLE & PASSWORD LOGIC ---
    # Default is TRUE (Production)
    production_mode_toggle = st.sidebar.toggle("Production Mode", value=True, key="production_mode_toggle")
    
    if not production_mode_toggle:
        if not st.session_state.dev_authenticated:
            st.sidebar.warning("Restricted Access")
            pwd = st.sidebar.text_input("Enter Developer Password:", type="password", key="dev_password_input")
            if pwd == "492211":
                st.session_state.dev_authenticated = True
                st.rerun()
            else:
                production_mode = True 
        else:
            production_mode = False
    else:
        production_mode = True

    if production_mode:
        st.markdown("""<style>div[data-testid="stSidebar"] { background-color: #f0f2f6; }</style>""", unsafe_allow_html=True)
    
    st.markdown("""<style>
        div[data-testid="stSidebar"] button[kind="primary"] { width: 100%; }
        div[data-testid="stSidebar"] div[data-testid="stButton"] button { width: 100%; }
        </style>""", unsafe_allow_html=True)

    # --- DISPLAY LOAD MESSAGES (Moved here to check Production Mode) ---
    if 'load_message' in st.session_state:
        with main_content_placeholder.container():
            # Always show the main load success/fail message
            st.info(st.session_state.load_message)
            
            # Only show Data Cleaning details if NOT in Production Mode
            if st.session_state.outlier_report and not production_mode:
                st.info(f"Data Cleaning: Found and removed price spikes >100% in {len(st.session_state.outlier_report)} tickers.")
        
        del st.session_state.load_message
        if 'outlier_report' in st.session_state: del st.session_state.outlier_report


    # --- SIDEBAR SECTION 1: Select Test Mode & Dates (Common to both) ---
    st.sidebar.header("1. Select Test Mode & Dates")
    full_list_label = f"Analyse Full List ({len(ticker_list)} Tickers)"
    if st.session_state.run_mode == "Analyse Full List":
        st.session_state.run_mode = full_list_label
    
    st.sidebar.radio("Mode:", ("Analyse Single Ticker", full_list_label), key='run_mode')

    if st.session_state.get('run_mode') == "Analyse Single Ticker":
        if not ticker_list: 
            st.sidebar.warning("No tickers loaded.")
        else:
            ticker_index = 0 
            try: ticker_index = ticker_list.index(st.session_state.ticker_select)
            except ValueError: ticker_index = 0
            st.sidebar.selectbox("Select a Ticker:", ticker_list, index=ticker_index, key='widget_ticker_select', on_change=update_state)

    st.sidebar.date_input("Start Date", value=st.session_state.start_date, key='widget_start_date', on_change=update_state)
    st.sidebar.date_input("End Date", value=st.session_state.end_date, key='widget_end_date', on_change=update_state)
    
    st.sidebar.markdown("---")

    # --- PRIMARY TRIGGER (Hidden in Production, Forced to BBands) ---
    if production_mode:
        st.session_state.primary_driver = "Bollinger Bands" # Force logic
        # No UI element shown
    else:
        st.sidebar.selectbox(
            "Select Primary Trigger:",
            ("Bollinger Bands", "RSI Crossover", "MACD Crossover", "MA Slope", "Markov State"),
            key='widget_primary_driver',
            on_change=update_state,
            help="Select the main indicator that will trigger a trade."
        )
    st.sidebar.markdown("---")

    # --- RUN ACTIONS (Dual Mode Logic) ---
    st.sidebar.subheader("Run Actions")

    if production_mode:
        # --- PRODUCTION BUTTONS ---
        
        # 1. Run Analysis Button
        if st.sidebar.button("Run Analysis", type="primary", key="run_analysis_prod", 
                             help="Runs the test using the manual settings currently visible in Sections 2 & 3 below."):
            st.session_state.run_analysis_button = True
            st.session_state.run_advanced_advisor = False
            st.session_state.run_user_advisor_setup = False
            st.session_state.run_scan_user_setups = False
            st.session_state.advisor_df = None; st.session_state.raw_df = None; st.session_state.deduped_df = None 
            st.session_state.param_results_df = None; st.session_state.confidence_results_df = None
            st.session_state.summary_df = None; st.session_state.single_ticker_results = None
            st.session_state.open_trades_df = None; st.session_state.last_run_stats = None
            st.session_state.markov_results_df = None
            st.rerun()

        st.sidebar.markdown("---")
        
        # 2. Dropdown for User Setups (With "Default" option)
        user_setups_raw = st.session_state.get("user_setups_data", [])
        valid_user_setups = [s for s in user_setups_raw if not is_row_blank(s)]
        
        setup_options = {"Default (Reset to Original)": -1}
        for i, s in enumerate(valid_user_setups):
            # Build Label with Notes
            note = s.get('Notes', '')
            if note:
                if len(note) > 50: note = note[:47] + "..."
                note_display = f" - {note}"
            else:
                note_display = ""
                
            label = f"Setup {i+1}{note_display}"
            setup_options[label] = i
        
        selected_setup_label = st.sidebar.selectbox("Select User Setup:", list(setup_options.keys()))

        # 3. Run User-Selected Option Button
        if st.sidebar.button("Run User-Selected Option", type="primary", 
                             help="Runs the specific setup selected in the dropdown above, overriding manual settings."):
            idx = setup_options[selected_setup_label]
            
            def sync_param(main_key, value):
                st.session_state[main_key] = value
                st.session_state[f"widget_{main_key}"] = value

            # --- CASE A: DEFAULT (RESET) ---
            if idx == -1:
                defaults = st.session_state.get('widget_defaults', {}) or load_settings()
                
                # Reset Factors
                sync_param('use_rsi', defaults.get('use_rsi', True))
                sync_param('rsi_w', defaults.get('rsi_w', 1.0))
                sync_param('use_vol', defaults.get('use_vol', True))
                sync_param('vol_w', defaults.get('vol_w', 1.0))
                sync_param('use_trend', defaults.get('use_trend', True))
                sync_param('trend_w', defaults.get('trend_w', 1.5))
                sync_param('use_volume', defaults.get('use_volume', True))
                sync_param('volume_w', defaults.get('volume_w', 1.0))
                sync_param('use_macd', defaults.get('use_macd', True))
                sync_param('macd_w', defaults.get('macd_w', 1.0))
                sync_param('use_ma_slope', defaults.get('use_ma_slope', True))
                sync_param('ma_slope_w', defaults.get('ma_slope_w', 0.5))
                sync_param('use_markov', defaults.get('use_markov', False))
                sync_param('markov_w', defaults.get('markov_w', 1.0))
                
                # Reset ADX (With Clamping for Production Mode)
                sync_param('use_adx_filter', defaults.get('use_adx_filter', True))
                raw_adx = defaults.get('adx_threshold', 25.0)
                # FIX: Clamp value to be within 20-30 range to prevent widget crash
                clamped_adx = max(20.0, min(30.0, raw_adx)) 
                sync_param('adx_threshold', clamped_adx)

                # Reset Strategies
                sync_param('ma_period', defaults.get("large_ma_period", 50))
                sync_param('bb_period', defaults.get("bband_period", 20))
                sync_param('bb_std', defaults.get("bband_std_dev", 2.0))
                sync_param('confidence_slider', defaults.get("confidence_threshold", 50))
                
                sync_param('long_entry', defaults.get("long_entry_threshold_pct", 0.0) * 100)
                sync_param('long_exit', defaults.get("long_exit_ma_threshold_pct", 0.0) * 100)
                sync_param('long_sl', defaults.get("long_trailing_stop_loss_pct", 8.0) * 100)
                sync_param('long_delay', defaults.get("long_delay_days", 0))
                
                sync_param('short_entry', defaults.get("short_entry_threshold_pct", 0.0) * 100)
                sync_param('short_exit', defaults.get("short_exit_ma_threshold_pct", 0.0) * 100)
                sync_param('short_sl', defaults.get("short_trailing_stop_loss_pct", 8.0) * 100)
                sync_param('short_delay', defaults.get("short_delay_days", 0))
                
                st.toast("Settings reset to Default. Running Analysis...", icon="🔄")

            # --- CASE B: USER SETUP ---
            else:
                selected_setup_data = valid_user_setups[idx]
                
                def apply_factor_loading(key, main_key, weight_key):
                    val = selected_setup_data.get(key, 'Off')
                    if str(val).lower() in ['off', '0', '0.0', 'false']:
                        sync_param(main_key, False)
                    else:
                        sync_param(main_key, True)
                        try: w = float(val); sync_param(weight_key, w)
                        except: sync_param(weight_key, 1.0)

                apply_factor_loading('RSI', 'use_rsi', 'rsi_w')
                apply_factor_loading('Volatility', 'use_vol', 'vol_w')
                apply_factor_loading('TREND', 'use_trend', 'trend_w')
                apply_factor_loading('Volume', 'use_volume', 'volume_w')
                apply_factor_loading('MACD', 'use_macd', 'macd_w')
                apply_factor_loading('MA Slope', 'use_ma_slope', 'ma_slope_w')
                apply_factor_loading('Markov', 'use_markov', 'markov_w')

                # --- ADX FILTER LOGIC (With Clamping) ---
                adx_val = selected_setup_data.get('ADX Filter', 'Off')
                if str(adx_val).lower() in ['off', '0', '0.0', 'false']:
                    sync_param('use_adx_filter', False)
                else:
                    sync_param('use_adx_filter', True)
                    try:
                        thresh = float(adx_val)
                        # FIX: Clamp value here too, in case a User Setup has < 20
                        clamped_thresh = max(20.0, min(30.0, thresh))
                        sync_param('adx_threshold', clamped_thresh)
                    except:
                        sync_param('adx_threshold', 25.0)

                def get_num(key, default, type_func):
                    try: return type_func(selected_setup_data.get(key, default))
                    except: return default

                sync_param('ma_period', get_num('Large MA Period', st.session_state.ma_period, int))
                sync_param('bb_period', get_num('Bollinger Band Period', st.session_state.bb_period, int))
                sync_param('bb_std', get_num('Bollinger Band Std Dev', st.session_state.bb_std, float))
                sync_param('confidence_slider', get_num('Conf. Threshold', st.session_state.confidence_slider, int))
                
                sync_param('long_entry', get_num('Long Entry Threshold (%)', st.session_state.long_entry, float))
                sync_param('long_exit', get_num('Long Exit Threshold (%)', st.session_state.long_exit, float))
                sync_param('long_sl', get_num('Long Stop Loss (%)', st.session_state.long_sl, float))
                sync_param('long_delay', get_num('Long Delay (Days)', st.session_state.long_delay, int))
                
                sync_param('short_entry', get_num('Short Entry Threshold (%)', st.session_state.short_entry, float))
                sync_param('short_exit', get_num('Short Exit Threshold (%)', st.session_state.short_exit, float))
                sync_param('short_sl', get_num('Short Stop Loss (%)', st.session_state.short_sl, float))
                sync_param('short_delay', get_num('Short Delay (Days)', st.session_state.short_delay, int))

                st.toast(f"Loaded {selected_setup_label}. Running Analysis...", icon="✅")

            st.session_state.run_analysis_button = True
            st.session_state.run_scan_user_setups = False
            st.session_state.run_user_advisor_setup = False 
            st.session_state.advisor_df = None
            st.rerun()

    else:
        # --- DEVELOPER BUTTONS (Original) ---
        if st.sidebar.button("🚀 Run Analysis (Developer)", type="primary", key="run_analysis_dev"):
            st.session_state.run_analysis_button = True
            st.session_state.run_advanced_advisor = False
            st.session_state.run_user_advisor_setup = False
            st.session_state.run_scan_user_setups = False
            st.session_state.advisor_df = None; st.session_state.raw_df = None; st.session_state.deduped_df = None 
            st.session_state.param_results_df = None; st.session_state.confidence_results_df = None
            st.session_state.summary_df = None; st.session_state.single_ticker_results = None
            st.session_state.open_trades_df = None; st.session_state.last_run_stats = None
            st.session_state.markov_results_df = None
            st.rerun() 

        if st.sidebar.button("👨‍💼 Run Advanced Advisor (Find Trades)", key="run_adv_find_trades"):
            st.session_state.run_advanced_advisor = True
            st.session_state.run_analysis_button = False
            st.session_state.run_user_advisor_setup = False
            st.session_state.run_scan_user_setups = False
            st.session_state.advisor_df = None; st.session_state.raw_df = None; st.session_state.deduped_df = None
            st.session_state.param_results_df = None; st.session_state.confidence_results_df = None
            st.session_state.summary_df = None
            st.rerun() 

        if st.sidebar.button("⚙️ Run User-Defined Advisor (Find Trades)", key="run_user_find_trades"):
            st.session_state.run_user_advisor_setup = True
            st.session_state.run_analysis_button = False
            st.session_state.run_advanced_advisor = False
            st.session_state.run_scan_user_setups = False
            st.session_state.advisor_df = None; st.session_state.raw_df = None; st.session_state.deduped_df = None
            st.session_state.param_results_df = None; st.session_state.confidence_results_df = None
            st.session_state.summary_df = None
            st.rerun()
 
    st.sidebar.markdown("---")

    # --- SECTION 2: Confidence Score Factors ---
    st.sidebar.header("2. Confidence Score Factors")

    if production_mode:
        # --- PRODUCTION VIEW (Simplified) ---
        
        # Force hidden factors OFF
        st.session_state.use_vol = False
        st.session_state.use_trend = False
        st.session_state.use_ma_slope = False
        st.session_state.use_markov = False
        st.session_state.rsi_logic = "Level"

        # RSI (Momentum)
        if 'widget_use_rsi' not in st.session_state: st.session_state.widget_use_rsi = st.session_state.use_rsi
        st.sidebar.toggle("Use Momentum (RSI)", key='widget_use_rsi', on_change=update_state, help="Enable Relative Strength Index (Momentum) factor.")
        
        rsi_val = st.session_state.rsi_w if st.session_state.rsi_w != 1.0 else 0.5
        if 'widget_rsi_w' not in st.session_state: st.session_state.widget_rsi_w = rsi_val
        st.sidebar.number_input("RSI Weight", 0.1, 5.0, step=0.1, key='widget_rsi_w', on_change=update_state, disabled=not st.session_state.get('use_rsi', True))
        
        # Volume Spike
        if 'widget_use_volume' not in st.session_state: st.session_state.widget_use_volume = st.session_state.use_volume
        st.sidebar.toggle("Use Volume Spike", key='widget_use_volume', on_change=update_state, help="Enable Volume Spike detection.")
        
        vol_val = st.session_state.volume_w if st.session_state.volume_w != 1.0 else 0.5
        if 'widget_volume_w' not in st.session_state: st.session_state.widget_volume_w = vol_val
        st.sidebar.number_input("Volume Weight", 0.1, 5.0, step=0.1, key='widget_volume_w', on_change=update_state, disabled=not st.session_state.get('use_volume', True))
        
        # ADX Filter
        if 'widget_use_adx_filter' not in st.session_state: st.session_state.widget_use_adx_filter = st.session_state.use_adx_filter
        st.sidebar.toggle("Use ADX Filter (<)", key='widget_use_adx_filter', on_change=update_state, help="Enable Trend Strength filter (ADX). Good for hunting Shorts, bad for Longs")
        
        curr_adx = max(20.0, min(30.0, st.session_state.adx_threshold))
        if 'widget_adx_threshold' not in st.session_state: st.session_state.widget_adx_threshold = curr_adx
        st.sidebar.number_input("ADX Threshold", 20.0, 30.0, step=1.0, key='widget_adx_threshold', on_change=update_state, disabled=not st.session_state.get('use_adx_filter', True))
        
        # MACD
        if 'widget_use_macd' not in st.session_state: st.session_state.widget_use_macd = st.session_state.use_macd
        st.sidebar.toggle("Use MACD Signals", key='widget_use_macd', on_change=update_state, help="Enable MACD crossover/histogram signals.")
        
        macd_val = st.session_state.macd_w if st.session_state.macd_w != 1.0 else 2.0
        if 'widget_macd_w' not in st.session_state: st.session_state.widget_macd_w = macd_val
        st.sidebar.number_input("MACD Weight", 0.1, 5.0, step=0.1, key='widget_macd_w', on_change=update_state, disabled=not st.session_state.get('use_macd', True))
        
        # Confidence Slider
        if 'widget_confidence_slider' not in st.session_state: st.session_state.widget_confidence_slider = st.session_state.confidence_slider
        st.sidebar.slider("Minimum Confidence Threshold (%)", 0, 100, step=5, key='widget_confidence_slider', on_change=update_state, help="Acts as a quality filter. Higher values increase trade quality but reduce the number of trades found.")

    else:
        # --- DEVELOPER VIEW (Original) ---
        if 'widget_use_rsi' not in st.session_state: st.session_state.widget_use_rsi = st.session_state.use_rsi
        st.sidebar.toggle("Use Momentum (RSI)", key='widget_use_rsi', on_change=update_state)
        
        if 'widget_rsi_w' not in st.session_state: st.session_state.widget_rsi_w = st.session_state.rsi_w
        st.sidebar.number_input("RSI Weight", 0.1, 5.0, step=0.1, key='widget_rsi_w', on_change=update_state, disabled=not st.session_state.get('use_rsi', True))
        
        rsi_logic_options = ["Crossover", "Level"]
        rsi_logic_index = rsi_logic_options.index(st.session_state.rsi_logic) if st.session_state.rsi_logic in rsi_logic_options else 0
        st.sidebar.radio("RSI Entry Logic:", rsi_logic_options, index=rsi_logic_index, key='widget_rsi_logic', on_change=update_state,
            help="Level: Enter <= 30/>= 70. Crossover: Enter on cross back.",
            disabled=not st.session_state.get('use_rsi', True))
        
        if 'widget_use_vol' not in st.session_state: st.session_state.widget_use_vol = st.session_state.use_vol
        st.sidebar.toggle("Use Volatility", key='widget_use_vol', on_change=update_state)
        
        if 'widget_vol_w' not in st.session_state: st.session_state.widget_vol_w = st.session_state.vol_w
        st.sidebar.number_input("Volatility Weight", 0.1, 5.0, step=0.1, key='widget_vol_w', on_change=update_state, disabled=not st.session_state.get('use_vol', True))
        
        if 'widget_use_trend' not in st.session_state: st.session_state.widget_use_trend = st.session_state.use_trend
        st.sidebar.toggle("Use Trend (200d MA)", key='widget_use_trend', on_change=update_state)
        
        if 'widget_trend_w' not in st.session_state: st.session_state.widget_trend_w = st.session_state.trend_w
        st.sidebar.number_input("Trend Weight", 0.1, 5.0, step=0.1, key='widget_trend_w', on_change=update_state, disabled=not st.session_state.get('use_trend', True))
        
        if 'widget_use_volume' not in st.session_state: st.session_state.widget_use_volume = st.session_state.use_volume
        st.sidebar.toggle("Use Volume Spike", key='widget_use_volume', on_change=update_state)
        
        if 'widget_volume_w' not in st.session_state: st.session_state.widget_volume_w = st.session_state.volume_w
        st.sidebar.number_input("Volume Weight", 0.1, 5.0, step=0.1, key='widget_volume_w', on_change=update_state, disabled=not st.session_state.get('use_volume', True))
        
        if 'widget_use_adx_filter' not in st.session_state: st.session_state.widget_use_adx_filter = st.session_state.use_adx_filter
        st.sidebar.toggle("Use ADX Filter (<)", key='widget_use_adx_filter', on_change=update_state)
        
        if 'widget_adx_threshold' not in st.session_state: st.session_state.widget_adx_threshold = st.session_state.adx_threshold
        st.sidebar.number_input("ADX Threshold", 10.0, 50.0, step=1.0, key='widget_adx_threshold', on_change=update_state, disabled=not st.session_state.get('use_adx_filter', True), help="Allow entries only when ADX is BELOW this value.")
        
        if 'widget_adx_period' not in st.session_state: st.session_state.widget_adx_period = st.session_state.adx_period
        st.sidebar.number_input("ADX Period", 5, 50, step=1, key='widget_adx_period', on_change=update_state, help="The lookback period for the ADX calculation.")
        
        if 'widget_use_macd' not in st.session_state: st.session_state.widget_use_macd = st.session_state.use_macd
        st.sidebar.toggle("Use MACD Signals", key='widget_use_macd', on_change=update_state)
        
        if 'widget_macd_w' not in st.session_state: st.session_state.widget_macd_w = st.session_state.macd_w
        st.sidebar.number_input("MACD Weight", 0.1, 5.0, step=0.1, key='widget_macd_w', on_change=update_state, disabled=not st.session_state.get('use_macd', True))
        
        if 'widget_use_ma_slope' not in st.session_state: st.session_state.widget_use_ma_slope = st.session_state.use_ma_slope
        st.sidebar.toggle("Use MA Slope", key='widget_use_ma_slope', on_change=update_state)
        
        if 'widget_ma_slope_w' not in st.session_state: st.session_state.widget_ma_slope_w = st.session_state.ma_slope_w
        st.sidebar.number_input("MA Slope Weight", 0.1, 5.0, step=0.1, key='widget_ma_slope_w', on_change=update_state, disabled=not st.session_state.get('use_ma_slope', True))
        
        if 'widget_use_markov' not in st.session_state: st.session_state.widget_use_markov = st.session_state.use_markov
        st.sidebar.toggle("Use Markov State", key='widget_use_markov', on_change=update_state, help="Use the best-found Markov state as a confidence factor. You must run 'Find Best Markov Setup' (Section 7) first.")
        
        if 'widget_markov_w' not in st.session_state: st.session_state.widget_markov_w = st.session_state.markov_w
        st.sidebar.number_input("Markov Weight", 0.1, 5.0, step=0.1, key='widget_markov_w', on_change=update_state, disabled=not st.session_state.get('use_markov', True))
        
        if 'widget_confidence_slider' not in st.session_state: st.session_state.widget_confidence_slider = st.session_state.confidence_slider
        st.sidebar.slider("Minimum Confidence Threshold (%)", 0, 100, step=5, key='widget_confidence_slider', on_change=update_state)

    st.sidebar.markdown("---")

    st.sidebar.header("3. Strategy Parameters")

    if production_mode:
        # --- PRODUCTION VIEW (Simplified) ---
        # Large MA and BB Period are HIDDEN here (values persist in session_state)
        
        # Dropdown for Std Dev (1.4, 1.6, 2.2)
        std_options = [1.4, 1.6, 2.2]
        current_std = st.session_state.bb_std
        std_index = std_options.index(current_std) if current_std in std_options else 2
        
        st.sidebar.selectbox(
            "Bollinger Band Std Dev", 
            std_options, 
            index=std_index, 
            key='widget_bb_std', 
            on_change=update_state,
            help="Controls volatility sensitivity. Lower (1.4/1.6) = more trades, Higher (2.2) = fewer, stricter trades."
        )

    else:
        # --- DEVELOPER VIEW (Original) ---
        st.sidebar.number_input("Large MA Period", 10, 200, value=st.session_state.ma_period, step=1, key='widget_ma_period', on_change=update_state)
        st.sidebar.number_input("Bollinger Band Period", 10, 100, value=st.session_state.bb_period, step=1, key='widget_bb_period', on_change=update_state)
        st.sidebar.number_input("Bollinger Band Std Dev", 1.0, 5.0, value=st.session_state.bb_std, step=0.1, key='widget_bb_std', on_change=update_state)
        
        st.sidebar.subheader("Long Trade Logic")
        st.sidebar.slider("Entry Threshold (%)", 0.0, 10.0, value=st.session_state.long_entry, step=0.1, key='widget_long_entry', on_change=update_state)
        st.sidebar.slider("Exit MA Threshold (%)", 0.0, 10.0, value=st.session_state.long_exit, step=0.1, key='widget_long_exit', on_change=update_state)
        st.sidebar.slider("Trailing Stop Loss (%)", 0.0, 30.0, value=st.session_state.long_sl, step=0.5, key='widget_long_sl', on_change=update_state, help="A trailing stop-loss set from the trade's high-water mark. Set to 0 to disable.")
        st.sidebar.number_input("Delay Entry (days)", 0, 10, value=st.session_state.long_delay, step=1, key='widget_long_delay', on_change=update_state)
        
        st.sidebar.markdown("---")
        st.sidebar.subheader("Exit Logic")

        exit_options = ["Standard (Price-Based)", "Intelligent (ADX/MACD/ATR)"]
        if st.session_state.exit_logic_type not in exit_options: st.session_state.exit_logic_type = "Standard (Price-Based)"
        exit_index = exit_options.index(st.session_state.exit_logic_type)

        st.sidebar.selectbox(
            "Exit Logic Type:", exit_options, index=exit_index, key='widget_exit_logic_type', on_change=update_state,
            help="Standard: Exit on MA/BB cross. Intelligent: ADX/MACD profit-take with an optional TSL."
        )
        
        st.sidebar.slider(
            "Intelligent: Trailing Stop Loss (%)", 0.0, 60.0, value=st.session_state.intelligent_tsl_pct, step=1.0, key='widget_intelligent_tsl_pct', on_change=update_state,
            format="%.1f%%", disabled=(st.session_state.exit_logic_type != 'Intelligent (ADX/MACD/ATR)'),
            help="The TSL for the 'Intelligent' exit. Set to 60.0% to replicate the 'Profit-Take or Time-Out' strategy."
        )

        st.sidebar.toggle("Apply MA/BB Price Lock Floor (Catcher)", value=st.session_state.use_ma_floor_filter, key='widget_use_ma_floor_filter', on_change=update_state,
            help="If ON, the Trailing Stop Loss is prevented from dropping below the Mean Reversion price.")
        st.sidebar.markdown("---")

        st.sidebar.subheader("Short Trade Logic")
        st.sidebar.slider("Entry Threshold (%)", 0.0, 10.0, value=st.session_state.short_entry, step=0.1, key='widget_short_entry', on_change=update_state)
        st.sidebar.slider("Exit MA Threshold (%)", 0.0, 10.0, value=st.session_state.short_exit, step=0.1, key='widget_short_exit', on_change=update_state)
        st.sidebar.slider("Trailing Stop Loss (%)", 0.0, 30.0, value=st.session_state.short_sl, step=0.5, key='widget_short_sl', on_change=update_state, help="A trailing stop-loss set from the trade's low-water mark. Set to 0 to disable.")
        st.sidebar.number_input("Delay Entry (days)", 0, 10, value=st.session_state.short_delay, step=1, key='widget_short_delay', on_change=update_state)
        
        st.sidebar.markdown("---")
        st.sidebar.subheader("Time Limits (Days)")
        c_time1, c_time2 = st.sidebar.columns(2)
        st.session_state.max_long_duration = c_time1.number_input("Max Long Duration", min_value=1, max_value=365, value=st.session_state.get('max_long_duration', 60), step=1, key='widget_max_long_duration', on_change=update_state)
        st.session_state.max_short_duration = c_time2.number_input("Max Short Duration", min_value=1, max_value=365, value=st.session_state.get('max_short_duration', 10), step=1, key='widget_max_short_duration', on_change=update_state)

    if 'best_params' not in st.session_state: st.session_state.best_params = None
    
    # --- SECTIONS 4, 5, 6, 7 (HIDDEN IN PRODUCTION) ---
    if not production_mode:
        st.sidebar.markdown("---")
        st.sidebar.header("4. Find Best Parameters")
        with st.sidebar.expander("Set Optimisation Ranges"):
            use_squared_weighting = st.toggle("Prioritise Profit per Trade (Params)", value=st.session_state.sq_params_toggle, key="widget_sq_params_toggle", on_change=update_state)
            st.markdown("---")
            
            optimise_ma = st.checkbox("Optimise MA Period", value=st.session_state.opt_ma_cb, key="widget_opt_ma_cb", on_change=update_state); c1,c2,c3 = st.columns(3)
            c1.number_input("MA Start", 10, 200, value=st.session_state.ma_start, step=5, disabled=not optimise_ma, key='widget_ma_start', on_change=update_state)
            c2.number_input("MA End", 10, 200, value=st.session_state.ma_end, step=5, disabled=not optimise_ma, key='widget_ma_end', on_change=update_state)
            c3.number_input("MA Step", 1, 20, value=st.session_state.ma_step, step=1, disabled=not optimise_ma, key='widget_ma_step', on_change=update_state)
            
            optimise_bb = st.checkbox("Optimise BB Period", value=st.session_state.opt_bb_cb, key="widget_opt_bb_cb", on_change=update_state); c1,c2,c3 = st.columns(3)
            c1.number_input("BB Start", 10, 100, value=st.session_state.bb_start, step=5, disabled=not optimise_bb, key='widget_bb_start', on_change=update_state)
            c2.number_input("BB End", 10, 100, value=st.session_state.bb_end, step=5, disabled=not optimise_bb, key='widget_bb_end', on_change=update_state)
            c3.number_input("BB Step", 1, 10, value=st.session_state.bb_step, step=1, disabled=not optimise_bb, key='widget_bb_step', on_change=update_state)
            
            optimise_std = st.checkbox("Optimise BB Std Dev", value=st.session_state.opt_std_cb, key="widget_opt_std_cb", on_change=update_state); c1,c2,c3 = st.columns(3)
            c1.number_input("Std Start", 1.0, 5.0, value=st.session_state.std_start, step=0.1, format="%.1f", disabled=not optimise_std, key='widget_std_start', on_change=update_state)
            c2.number_input("Std End", 1.0, 5.0, value=st.session_state.std_end, step=0.1, format="%.1f", disabled=not optimise_std, key='widget_std_end', on_change=update_state)
            c3.number_input("Std Step", 0.1, 1.0, value=st.session_state.std_step, step=0.1, format="%.1f", disabled=not optimise_std, key='widget_std_step', on_change=update_state)
            
            st.markdown("---")
            optimise_conf = st.checkbox("Optimise Confidence Threshold", value=st.session_state.opt_conf_cb, key="widget_opt_conf_cb", on_change=update_state); c1,c2,c3 = st.columns(3)
            c1.number_input("Conf Start", 0, 100, value=st.session_state.conf_start, step=5, disabled=not optimise_conf, key='widget_conf_start', on_change=update_state)
            c2.number_input("Conf End", 0, 100, value=st.session_state.conf_end, step=5, disabled=not optimise_conf, key='widget_conf_end', on_change=update_state)
            c3.number_input("Conf Step", 5, 25, value=st.session_state.conf_step, step=5, disabled=not optimise_conf, key='widget_conf_step', on_change=update_state)
            
            optimise_sl = st.checkbox("Optimise Stop Loss %", value=st.session_state.opt_sl_cb, key="widget_opt_sl_cb", on_change=update_state); c1,c2,c3 = st.columns(3)
            c1.number_input("SL Start", 0.0, 30.0, value=st.session_state.sl_start, step=0.5, disabled=not optimise_sl, key='widget_sl_start', on_change=update_state)
            c2.number_input("SL End", 0.0, 30.0, value=st.session_state.sl_end, step=0.5, disabled=not optimise_sl, key='widget_sl_end', on_change=update_state)
            c3.number_input("SL Step", 0.1, 5.0, value=st.session_state.sl_step, step=0.5, disabled=not optimise_sl, key='widget_sl_step', on_change=update_state)
            
            optimise_delay = st.checkbox("Optimise Delay Days", value=st.session_state.opt_delay_cb, key="widget_opt_delay_cb", on_change=update_state); c1,c2,c3 = st.columns(3)
            c1.number_input("Delay Start", 0, 10, value=st.session_state.delay_start, step=1, disabled=not optimise_delay, key='widget_delay_start', on_change=update_state)
            c2.number_input("Delay End", 0, 10, value=st.session_state.delay_end, step=1, disabled=not optimise_delay, key='widget_delay_end', on_change=update_state)
            c3.number_input("Delay Step", 1, 5, value=st.session_state.delay_step, step=1, disabled=not optimise_delay, key='widget_delay_step', on_change=update_state)
            
            optimise_entry = st.checkbox("Optimise Entry %", value=st.session_state.opt_entry_cb, key="widget_opt_entry_cb", on_change=update_state); c1,c2,c3 = st.columns(3)
            c1.number_input("Entry Start", 0.0, 10.0, value=st.session_state.entry_start, step=0.1, disabled=not optimise_entry, key='widget_entry_start', on_change=update_state)
            c2.number_input("Entry End", 0.0, 10.0, value=st.session_state.entry_end, step=0.1, disabled=not optimise_entry, key='widget_entry_end', on_change=update_state)
            c3.number_input("Entry Step", 0.1, 1.0, value=st.session_state.entry_step, step=0.1, disabled=not optimise_entry, key='widget_entry_step', on_change=update_state)
            
            optimise_exit = st.checkbox("Optimise Exit MA %", value=st.session_state.opt_exit_cb, key="widget_opt_exit_cb", on_change=update_state); c1,c2,c3 = st.columns(3)
            c1.number_input("Exit Start", 0.0, 10.0, value=st.session_state.exit_start, step=0.1, disabled=not optimise_exit, key='widget_exit_start', on_change=update_state)
            c2.number_input("Exit End", 0.0, 10.0, value=st.session_state.exit_end, step=0.1, disabled=not optimise_exit, key='widget_exit_end', on_change=update_state)
            c3.number_input("Exit Step", 0.1, 1.0, value=st.session_state.exit_step, step=0.1, disabled=not optimise_exit, key='widget_exit_step', on_change=update_state)
            
            st.markdown("---")
            optimise_dur = st.checkbox("Optimise Max Duration", value=st.session_state.opt_duration_cb, key="widget_opt_duration_cb", on_change=update_state); c1,c2,c3 = st.columns(3)
            c1.number_input("Dur Start", 1, 365, value=st.session_state.dur_start, step=1, disabled=not optimise_dur, key='widget_dur_start', on_change=update_state)
            c2.number_input("Dur End", 1, 365, value=st.session_state.dur_end, step=1, disabled=not optimise_dur, key='widget_dur_end', on_change=update_state)
            c3.number_input("Dur Step", 1, 30, value=st.session_state.dur_step, step=1, disabled=not optimise_dur, key='widget_dur_step', on_change=update_state)
            
            st.markdown("---")
            col1, col2 = st.columns(2)
            if col1.button("💡 Find Best Long"): generate_and_run_optimisation(master_df, main_content_placeholder, 'long', use_squared_weighting)
            if col2.button("💡 Find Best Short"): generate_and_run_optimisation(master_df, main_content_placeholder, 'short', use_squared_weighting)
            st.markdown("---")
            col_comb_1, col_comb_2 = st.columns(2)
            if col_comb_1.button("🔥 Find Combined Best Long", key="find_combined_long", use_container_width=True):
                st.session_state.run_combined_optimisation = True; st.session_state.optimise_side = 'long'; st.rerun()
            if col_comb_2.button("🔥 Find Combined Best Short", key="find_combined_short", use_container_width=True):
                st.session_state.run_combined_optimisation = True; st.session_state.optimise_side = 'short'; st.rerun()

        st.sidebar.markdown("---")
        st.sidebar.header("5. Find Best/Worst Confidence Setup")
        with st.sidebar.expander("Optimise Confidence Factors"):
            st.info("Finds good setups (using Section 2 factors) or bad setups (using the factors below).")
            st.write("**Find Best Setups (High Profit)**"); c1, c2 = st.columns(2)
            if c1.button("💡 Find Best Long Confidence", key="find_conf_long"):
                st.session_state.run_confidence_optimisation = True; st.session_state.confidence_optimise_side = 'long'; st.session_state.confidence_optimise_mode = 'best'; st.session_state.confidence_optimise_veto_factors = None; st.rerun()
            if c2.button("💡 Find Best Short Confidence", key="find_conf_short"):
                st.session_state.run_confidence_optimisation = True; st.session_state.confidence_optimise_side = 'short'; st.session_state.confidence_optimise_mode = 'best'; st.session_state.confidence_optimise_veto_factors = None; st.rerun()

            st.markdown("---")
            st.write("**Find Worst Setups (for Veto Filter)**")
            st.caption("Select the factors to test for the Veto signal:")

            c1_veto, c2_veto = st.columns(2)
            veto_rsi = c1_veto.toggle("Veto RSI", value=st.session_state.veto_rsi_cb, key="widget_veto_rsi_cb", on_change=update_state)
            veto_vol = c2_veto.toggle("Veto Volatility", value=st.session_state.veto_vol_cb, key="widget_veto_vol_cb", on_change=update_state)
            veto_trend = c1_veto.toggle("Veto Trend", value=st.session_state.veto_trend_cb, key="widget_veto_trend_cb", on_change=update_state)
            veto_volume = c2_veto.toggle("Veto Volume", value=st.session_state.veto_volume_cb, key="widget_veto_volume_cb", on_change=update_state)
            veto_macd = c1_veto.toggle("Veto MACD", value=st.session_state.veto_macd_cb, key="widget_veto_macd_cb", on_change=update_state)
            veto_ma_slope = c2_veto.toggle("Veto MA Slope", value=st.session_state.veto_ma_slope_cb, key="widget_veto_ma_slope_cb", on_change=update_state)
            veto_markov = c1_veto.toggle("Veto Markov", value=st.session_state.get('veto_markov_cb', False), key="widget_veto_markov_cb", on_change=update_state)

            veto_factors = (veto_rsi, veto_vol, veto_trend, veto_volume, veto_macd, veto_ma_slope, veto_markov)

            c1_veto_btn, c2_veto_btn = st.columns(2)
            if c1_veto_btn.button("❌ Find Worst Long", key="find_worst_long"):
                st.session_state.run_confidence_optimisation = True; st.session_state.confidence_optimise_side = 'long'; st.session_state.confidence_optimise_mode = 'worst'; st.session_state.confidence_optimise_veto_factors = veto_factors; st.rerun()
            if c2_veto_btn.button("❌ Find Worst Short", key="find_worst_short"):
                st.session_state.run_confidence_optimisation = True; st.session_state.confidence_optimise_side = 'short'; st.session_state.confidence_optimise_mode = 'worst'; st.session_state.confidence_optimise_veto_factors = veto_factors; st.rerun()

        st.sidebar.markdown("---")
        st.sidebar.header("6. Find Best Weights")
        with st.sidebar.expander("Optimise Specific Factor Weights"):
            st.info("Dynamically optimises weights (0.5 to 2.0) for all active, non-primary factors. Uses other sidebar settings as fixed values.")
            use_sq_weighting_for_weights = st.toggle("Prioritise Profit per Trade (Weights)", value=st.session_state.sq_weights_toggle, key="widget_sq_weights_toggle", on_change=update_state)
            col1_w, col2_w = st.columns(2)
            if col1_w.button("⚖️ Find Best Long Weights", key="find_weights_long"):
                st.session_state.run_weight_optimisation = True; st.session_state.weight_optimise_side = 'long'; st.rerun()
            if col2_w.button("⚖️ Find Best Short Weights", key="find_weights_short"):
                st.session_state.run_weight_optimisation = True; st.session_state.weight_optimise_side = 'short'; st.rerun()

        st.sidebar.markdown("---")
        st.sidebar.header("7. Find Best Markov Setup")
        with st.sidebar.expander("Optimise Markov Probabilities"):
            st.info("Brute-forces Run-Up/Future periods to find the highest-probability trading states.")
            st.write("Run-Up Period (Past)")
            c1, c2, c3 = st.columns(3)
            c1.number_input("Start", 1, 100, value=st.session_state.markov_run_up_start, key='widget_markov_run_up_start', on_change=update_state)
            c2.number_input("End", 1, 100, value=st.session_state.markov_run_up_end, key='widget_markov_run_up_end', on_change=update_state)
            c3.number_input("Step", 1, 10, value=st.session_state.markov_run_up_step, key='widget_markov_run_up_step', on_change=update_state)
            
            st.write("Future Period (Prediction)")
            c4, c5, c6 = st.columns(3)
            c4.number_input("Start", 1, 100, value=st.session_state.markov_future_start, key='widget_markov_future_start', on_change=update_state)
            c5.number_input("End", 1, 100, value=st.session_state.markov_future_end, key='widget_markov_future_end', on_change=update_state)
            c6.number_input("Step", 1, 10, value=st.session_state.markov_future_step, key='widget_markov_future_step', on_change=update_state)

            c1_markov, c2_markov = st.columns(2)
            if c1_markov.button("🔮 Find Best Long Markov", key="markov_long_button"):
                st.session_state.run_markov_optimisation = True; st.session_state.markov_side = 'long'; st.rerun()
            if c2_markov.button("🔮 Find Best Short Markov", key="markov_short_button"):
                st.session_state.run_markov_optimisation = True; st.session_state.markov_side = 'short'; st.rerun()

        # --- Veto / Save Settings (Only in Developer Mode) ---
        st.sidebar.markdown("---")
        current_veto_list = st.session_state.get('veto_setup_list', [])
        if current_veto_list:
            st.sidebar.header("Veto Filter(s)")
            st.sidebar.success(f"{len(current_veto_list)} Veto filter(s) ACTIVE.")
            with st.sidebar.expander("Show Veto Filters"): st.json(current_veto_list)
            if st.sidebar.button("💾 Save Veto Filters as Default"): save_veto_setup(current_veto_list)
            if st.sidebar.button("Clear Veto Filters"):
                st.session_state.veto_setup_list = [];
                if os.path.exists(VETO_CONFIG_FILE):
                    try: os.remove(VETO_CONFIG_FILE)
                    except OSError as e: print(f"Error removing veto file: {e}")
                st.rerun()
            st.sidebar.markdown("---")
    
    # --- SAVE SETTINGS (Hidden in Production Mode) ---
    if not production_mode:
        if st.sidebar.button("💾 Save Settings as Default"):
            settings_to_save = {
                "large_ma_period": st.session_state.ma_period, "bband_period": st.session_state.bb_period,
                "bband_std_dev": st.session_state.bb_std, "confidence_threshold": st.session_state.confidence_slider,
                "long_entry_threshold_pct": st.session_state.long_entry / 100, "long_exit_ma_threshold_pct": st.session_state.long_exit / 100,
                "long_trailing_stop_loss_pct": st.session_state.long_sl / 100, "long_delay_days": st.session_state.long_delay,
                "short_entry_threshold_pct": st.session_state.short_entry / 100, "short_exit_ma_threshold_pct": st.session_state.short_exit / 100,
                "short_trailing_stop_loss_pct": st.session_state.short_sl / 100, "short_delay_days": st.session_state.short_delay,
                "use_rsi": st.session_state.use_rsi, "rsi_w": st.session_state.rsi_w,
                "rsi_logic": st.session_state.get('rsi_logic', 'Crossover'),
                "primary_driver": st.session_state.get('primary_driver', 'Bollinger Bands'),
                "exit_logic_type": st.session_state.get('exit_logic_type', 'Standard (Price-Based)'),
                "use_ma_floor_filter": st.session_state.use_ma_floor_filter, 
                "exit_confidence_threshold": st.session_state.get('exit_confidence_threshold', 50),
                "smart_trailing_stop_pct": st.session_state.get('smart_trailing_stop_pct', 5.0),
                "smart_exit_atr_period": st.session_state.get('smart_exit_atr_period', 14),
                "smart_exit_atr_multiplier": st.session_state.get('smart_exit_atr_multiplier', 3.0),
                "intelligent_tsl_pct": st.session_state.intelligent_tsl_pct / 100.0, 
                "use_vol": st.session_state.use_vol, "vol_w": st.session_state.vol_w,
                "use_trend": st.session_state.use_trend, "trend_w": st.session_state.trend_w,
                "use_volume": st.session_state.use_volume, "volume_w": st.session_state.volume_w,
                "use_adx_filter": st.session_state.use_adx_filter, "adx_threshold": st.session_state.adx_threshold,
                "adx_period": st.session_state.adx_period, 
                "use_macd": st.session_state.use_macd, "macd_w": st.session_state.macd_w,
                "use_ma_slope": st.session_state.use_ma_slope, "ma_slope_w": st.session_state.ma_slope_w,
                "use_markov": st.session_state.use_markov, "markov_w": st.session_state.markov_w,
                "max_trading_days": st.session_state.max_duration,
                "max_long_duration": st.session_state.max_long_duration,
                "max_short_duration": st.session_state.max_short_duration
            }
            save_settings(settings_to_save)

    # --- End of Sidebar Definitions ---

    # --- MAIN CONTENT AREA LOGIC ---
    with main_content_placeholder.container():
        
        # 1. User Defined Advisor Setup UI (Developer Only)
        if st.session_state.get('run_user_advisor_setup'):
            st.session_state.advisor_df = None; st.session_state.raw_df = None; st.session_state.deduped_df = None; st.session_state.single_ticker_results = None
            st.session_state.open_trades_df = None; st.session_state.confidence_results_df = None
            st.session_state.best_params = None; st.session_state.last_run_stats = None
            generate_user_advisor_ui_and_run(master_df)

        # 2. Run User Defined Advisor Scan (Used by BOTH Dev "Scan with User Setups" AND Prod "Run User-Selected Option")
        elif st.session_state.get('run_scan_user_setups'):
            st.session_state.run_scan_user_setups = False 
            st.session_state.last_run_stats = None
            setups_for_scan = st.session_state.get('setups_to_scan', [])
            
            if setups_for_scan:
                with st.spinner("Running advisor scan..."):
                    # Determine label based on count
                    label = "User-Defined" if len(setups_for_scan) > 1 else "Selected-Setup"
                    run_advisor_scan(master_df, setups_for_scan, label)
            else:
                st.warning("No setup selected for scanning.")
        
        # 3. Confidence Optimisation (Developer Only)
        elif st.session_state.get('run_confidence_optimisation'):
            st.session_state.run_advanced_advisor = False; st.session_state.run_analysis_button = False
            st.session_state.run_user_advisor_setup = False; st.session_state.run_scan_user_setups = False
            st.session_state.run_markov_optimisation = False; st.session_state.run_weight_optimisation = False
            st.session_state.advisor_df = None; st.session_state.raw_df = None; st.session_state.deduped_df = None
            st.session_state.markov_results_df = None; st.session_state.summary_df = None; st.session_state.last_run_stats = None
            
            side_to_run = st.session_state.get('confidence_optimise_side', 'long')
            mode_to_run = st.session_state.get('confidence_optimise_mode', 'best')
            veto_factors_to_run = st.session_state.get('confidence_optimise_veto_factors', None)
            run_confidence_optimisation(side_to_run, mode_to_run, master_df, main_content_placeholder, veto_factors_to_run)

        # 4. Weight Optimisation (Developer Only)
        elif st.session_state.get('run_weight_optimisation'):
            st.session_state.run_advanced_advisor = False; st.session_state.run_analysis_button = False
            st.session_state.run_user_advisor_setup = False; st.session_state.run_scan_user_setups = False
            st.session_state.run_markov_optimisation = False; st.session_state.advisor_df = None
            st.session_state.raw_df = None; st.session_state.deduped_df = None; st.session_state.markov_results_df = None
            st.session_state.summary_df = None; st.session_state.confidence_results_df = None; st.session_state.last_run_stats = None
            
            side_to_run = st.session_state.get('weight_optimise_side', 'long')
            use_sq = st.session_state.get('sq_weights_toggle', False)
            generate_and_run_weight_optimisation(master_df, main_content_placeholder, side_to_run, use_sq)

        # 5. Parameter Optimisation Results Display (Developer Only)
        elif st.session_state.get('param_results_df') is not None:
            st.subheader("🏆 Parameter Optimisation Results")
            st.caption("Results are pre-sorted by Score (High to Low).")
            if st.session_state.get('best_params'):
                st.subheader("Optimal Parameters Found:")
                st.json(st.session_state.best_params)
            
            c_view1, c_view2 = st.columns([1, 2])
            dedupe_params = c_view1.checkbox("De-duplicate (Show unique scores only)", value=True)
            results_df = st.session_state.param_results_df.copy()
            if dedupe_params:
                results_df = results_df.drop_duplicates(subset=['Strategy Score', 'Avg Profit/Trade'], keep='first')

            display_cols = ["Strategy Score", "Avg Profit/Trade", "Ticker G/B Ratio", "Total Trades", "Avg Entry Conf."]
            if 'max_trading_days' in results_df.columns: display_cols.append('max_trading_days')
            
            optional_cols = ["confidence_threshold", "bband_period", "large_ma_period", "long_entry_threshold_pct", "long_exit_ma_threshold_pct", "long_delay_days", "long_trailing_stop_loss_pct"]
            for col in optional_cols:
                if col in results_df.columns:
                     if results_df[col].nunique() > 1 or st.session_state.get(f"opt_{col[:3]}_cb", True):
                        display_cols.append(col)

            st.dataframe(results_df.head(100)[display_cols].style.format({
                "Strategy Score": "{:.2f}%", "Avg Profit/Trade": "{:.2%}", "Ticker G/B Ratio": "{:.2f}",
                "Avg Entry Conf.": "{:.1f}%", "max_trading_days": "{:.0f}"
            }, na_rep='-'))
            
            csv_data = results_df.to_csv(index=False).encode('utf-8')
            st.download_button(label="⬇️ Download Full Results (CSV)", data=csv_data, file_name="parameter_optimisation_results.csv", mime="text/csv")
            if st.button("Close Results"):
                st.session_state.param_results_df = None
                st.rerun()

        # 6. Combined Optimisation (Developer Only)
        elif st.session_state.get('run_combined_optimisation'):
            st.session_state.run_advanced_advisor = False; st.session_state.run_analysis_button = False
            st.session_state.run_user_advisor_setup = False; st.session_state.run_scan_user_setups = False
            st.session_state.run_markov_optimisation = False; st.session_state.run_weight_optimisation = False
            st.session_state.run_confidence_optimisation = False; st.session_state.advisor_df = None
            st.session_state.raw_df = None; st.session_state.deduped_df = None; st.session_state.markov_results_df = None
            st.session_state.summary_df = None; st.session_state.confidence_results_df = None; st.session_state.last_run_stats = None
            
            side_to_run = st.session_state.get('optimise_side', 'long')
            generate_and_run_combined_optimisation(master_df, main_content_placeholder, side_to_run)
            st.session_state.run_combined_optimisation = False 
        
        # 7. Markov Optimisation (Developer Only)
        elif st.session_state.get('run_markov_optimisation'):
            st.session_state.run_advanced_advisor = False; st.session_state.run_analysis_button = False
            st.session_state.run_user_advisor_setup = False; st.session_state.run_scan_user_setups = False
            st.session_state.advisor_df = None; st.session_state.raw_df = None; st.session_state.deduped_df = None
            st.session_state.last_run_stats = None
            side_to_run = st.session_state.get('markov_side', 'long')
            generate_and_run_markov_optimisation(master_df, main_content_placeholder, side_to_run)

        # 8. Top Setups Advisor UI (Developer Only)
        elif st.session_state.get('run_advanced_advisor'):
            st.session_state.summary_df = None; st.session_state.single_ticker_results = None
            st.session_state.open_trades_df = None; st.session_state.confidence_results_df = None
            st.session_state.best_params = None; st.session_state.last_run_stats = None
            generate_advisor_report(master_df)

        # 9. RUN ANALYSIS (Used by BOTH Dev "Rocket" button AND Prod "Run Analysis" button)
        elif st.session_state.get("run_analysis_button"):
            st.session_state.run_analysis_button = False # Reset flag
            
            # Clear all results
            st.session_state.summary_df = None; st.session_state.single_ticker_results = None
            st.session_state.open_trades_df = None; st.session_state.confidence_results_df = None
            st.session_state.advisor_df = None; st.session_state.raw_df = None; st.session_state.deduped_df = None
            st.session_state.best_params = None; st.session_state.worst_confidence_setup = None
            st.session_state.worst_confidence_setups_list = []; st.session_state.markov_results_df = None 
            st.session_state.last_run_stats = None 

            with st.spinner("Running analysis..."):
                # --- VETO LOGIC FIX ---
                if production_mode:
                    veto_list_to_use = [] # Disable veto in simplified mode
                else:
                    veto_list_to_use = st.session_state.get('veto_setup_list', [])
                    if veto_list_to_use: st.info(f"{len(veto_list_to_use)} Veto filter(s) active.")
                
                # Gather parameters
                manual_params = {
                    "large_ma_period": st.session_state.get('ma_period', 50), 
                    "bband_period": st.session_state.get('bb_period', 20), 
                    "bband_std_dev": st.session_state.get('bb_std', 2.0), 
                    "confidence_threshold": st.session_state.get('confidence_slider', 50), 
                    "long_entry_threshold_pct": st.session_state.get('long_entry', 0.0) / 100, 
                    "long_exit_ma_threshold_pct": st.session_state.get('long_exit', 0.0) / 100, 
                    "long_trailing_stop_loss_pct": st.session_state.get('long_sl', 8.0) / 100, 
                    "long_delay_days": st.session_state.get('long_delay', 0), 
                    "short_entry_threshold_pct": st.session_state.get('short_entry', 0.0) / 100, 
                    "short_exit_ma_threshold_pct": st.session_state.get('short_exit', 0.0) / 100, 
                    "short_trailing_stop_loss_pct": st.session_state.get('short_sl', 8.0) / 100, 
                    "short_delay_days": st.session_state.get('short_delay', 0),
                    "use_ma_floor_filter": st.session_state.get('use_ma_floor_filter', True),
                    "max_long_duration": st.session_state.get('max_long_duration', 60),
                    "max_short_duration": st.session_state.get('max_short_duration', 10)
                }

                markov_setup_to_use = None
                if st.session_state.primary_driver == 'Markov State' or st.session_state.use_markov:
                    if 'best_markov_setup' in st.session_state and st.session_state.best_markov_setup:
                        markov_setup_to_use = st.session_state.best_markov_setup
                    elif not production_mode:
                        st.error("Markov State selected but no setup found. Run Section 7."); st.stop()
                
                exit_logic = st.session_state.exit_logic_type
                exit_thresh = st.session_state.exit_confidence_threshold
                smart_trailing_stop = st.session_state.smart_trailing_stop_pct / 100.0
                smart_atr_p = st.session_state.smart_exit_atr_period
                smart_atr_m = st.session_state.smart_exit_atr_multiplier
                intelligent_tsl = st.session_state.intelligent_tsl_pct / 100.0 

                # --- NEW: Date Buffer Logic ---
                # We load data starting 365 days BEFORE the user selected start date
                # to ensure indicators (like 200MA) are fully calculated by the time the analysis starts.
                user_start_date = pd.Timestamp(st.session_state.start_date)
                end_date = pd.Timestamp(st.session_state.end_date)
                warmup_delta = timedelta(days=365) # Safe buffer
                data_load_start = user_start_date - warmup_delta

                # A) Single Ticker Analysis
                if st.session_state.run_mode == "Analyse Single Ticker":
                    selected_ticker = st.session_state.get('ticker_select', ticker_list[0] if ticker_list else None)
                    if not selected_ticker: st.error("No ticker selected."); st.stop()
                    
                    cols_to_use = [selected_ticker]
                    if f'{selected_ticker}_High' in master_df.columns: cols_to_use.append(f'{selected_ticker}_High')
                    if f'{selected_ticker}_Low' in master_df.columns: cols_to_use.append(f'{selected_ticker}_Low')
                    if f'{selected_ticker}_Volume' in master_df.columns: cols_to_use.append(f'{selected_ticker}_Volume')
                    existing_cols = [col for col in cols_to_use if col in master_df.columns]
                    
                    # Load data with WARMUP
                    data_for_backtest_full = master_df.loc[:, existing_cols]
                    data_for_backtest = data_for_backtest_full.loc[data_load_start:end_date]
                    
                    rename_dict = {selected_ticker: 'Close', f'{selected_ticker}_High': 'High', f'{selected_ticker}_Low': 'Low', f'{selected_ticker}_Volume': 'Volume'}
                    rename_dict_filtered = {k: v for k, v in rename_dict.items() if k in existing_cols}
                    data_for_backtest = data_for_backtest.rename(columns=rename_dict_filtered)
                    
                    if not data_for_backtest.empty and 'Close' in data_for_backtest.columns and not data_for_backtest['Close'].isna().all():
                         long_pnl, short_pnl, avg_long_trade, avg_short_trade, results_df, trades, open_trades, trade_counts, durations, trade_dates, exit_breakdown = run_backtest(
                            data_for_backtest, manual_params,
                            st.session_state.use_rsi, st.session_state.use_vol, st.session_state.use_trend, st.session_state.use_volume,
                            st.session_state.use_macd, st.session_state.use_ma_slope, st.session_state.use_markov,
                            st.session_state.rsi_w, st.session_state.vol_w, st.session_state.trend_w, st.session_state.volume_w,
                            st.session_state.macd_w, st.session_state.ma_slope_w, st.session_state.markov_w,
                            st.session_state.use_adx_filter, st.session_state.adx_threshold,
                            st.session_state.get('rsi_logic', 'Crossover'), st.session_state.adx_period, 
                            veto_setups_list=veto_list_to_use, primary_driver=st.session_state.primary_driver,
                            markov_setup=markov_setup_to_use, exit_logic_type=exit_logic, exit_confidence_threshold=exit_thresh,
                            smart_trailing_stop_pct=smart_trailing_stop, smart_exit_atr_period=smart_atr_p,
                            smart_exit_atr_multiplier=smart_atr_m, intelligent_tsl_pct=intelligent_tsl,
                            analysis_start_date=user_start_date # <--- Pass the filter date
                        )
                         st.session_state.single_ticker_results = {"long_pnl": long_pnl, "short_pnl": short_pnl, "avg_long_trade": avg_long_trade, "avg_short_trade": avg_short_trade, "results_df": results_df, "trades": trades}
                         st.session_state.open_trades_df = pd.DataFrame(open_trades) if open_trades else pd.DataFrame()
                         st.session_state.exit_breakdown_totals = {'long_profit_take_count': exit_breakdown[0], 'long_tsl_count': exit_breakdown[1], 'long_time_exit_count': exit_breakdown[2], 'short_profit_take_count': exit_breakdown[3], 'short_tsl_count': exit_breakdown[4], 'short_time_exit_count': exit_breakdown[5]}
                    else: st.warning("No data for ticker.")

                # B) Full List Analysis
                elif st.session_state.run_mode.startswith("Analyse Full List"):
                    summary_results, all_open_trades = [], []
                    total_long_wins, total_long_losses, total_short_wins, total_short_losses = 0, 0, 0, 0
                    all_long_durations = []; all_short_durations = []
                    total_exit_breakdown = [0, 0, 0, 0, 0, 0]
                    PROFIT_THRESHOLD = 1.0; excluded_analysis_tickers = []
                    progress_bar = st.progress(0, text="Starting analysis...")
                    num_tickers = len(ticker_list)
                    
                    for i, ticker_symbol in enumerate(ticker_list):
                        progress_bar.progress((i + 1) / num_tickers, text=f"Analysing {ticker_symbol}...")
                        cols_to_use = [ticker_symbol]
                        if f'{ticker_symbol}_High' in master_df.columns: cols_to_use.append(f'{ticker_symbol}_High')
                        if f'{ticker_symbol}_Low' in master_df.columns: cols_to_use.append(f'{ticker_symbol}_Low')
                        if f'{ticker_symbol}_Volume' in master_df.columns: cols_to_use.append(f'{ticker_symbol}_Volume')
                        existing_cols = [col for col in cols_to_use if col in master_df.columns]
                        if ticker_symbol not in existing_cols: continue
                        
                        # Load data with WARMUP
                        ticker_data_series = master_df.loc[data_load_start:end_date, existing_cols]
                        
                        rename_dict = {ticker_symbol: 'Close', f'{ticker_symbol}_High': 'High', f'{ticker_symbol}_Low': 'Low', f'{ticker_symbol}_Volume': 'Volume'}
                        ticker_data_series = ticker_data_series.rename(columns={k:v for k,v in rename_dict.items() if k in existing_cols})

                        if not ticker_data_series.empty and 'Close' in ticker_data_series.columns and not ticker_data_series['Close'].isna().all():
                            long_pnl, short_pnl, avg_long_trade, avg_short_trade, _, trades, open_trades, trade_counts, durations, trade_dates, exit_breakdown = run_backtest(
                                ticker_data_series, manual_params,
                                st.session_state.use_rsi, st.session_state.use_vol, st.session_state.use_trend, st.session_state.use_volume,
                                st.session_state.use_macd, st.session_state.use_ma_slope, st.session_state.use_markov,
                                st.session_state.rsi_w, st.session_state.vol_w, st.session_state.trend_w, st.session_state.volume_w,
                                st.session_state.macd_w, st.session_state.ma_slope_w, st.session_state.markov_w,
                                st.session_state.use_adx_filter, st.session_state.adx_threshold, st.session_state.get('rsi_logic', 'Crossover'), st.session_state.adx_period, 
                                veto_setups_list=veto_list_to_use, primary_driver=st.session_state.primary_driver,
                                markov_setup=markov_setup_to_use, exit_logic_type=exit_logic, exit_confidence_threshold=exit_thresh,
                                smart_trailing_stop_pct=smart_trailing_stop, smart_exit_atr_period=smart_atr_p,
                                smart_exit_atr_multiplier=smart_atr_m, intelligent_tsl_pct=intelligent_tsl,
                                analysis_start_date=user_start_date # <--- Pass the filter date
                            )

                            if abs(long_pnl) > PROFIT_THRESHOLD or abs(short_pnl) > PROFIT_THRESHOLD or \
                                (avg_long_trade is not None and pd.notna(avg_long_trade) and abs(avg_long_trade) > PROFIT_THRESHOLD) or \
                                (avg_short_trade is not None and pd.notna(avg_short_trade) and abs(avg_short_trade) > PROFIT_THRESHOLD):
                                excluded_analysis_tickers.append(ticker_symbol); continue
                            
                            total_exit_breakdown = [sum(x) for x in zip(total_exit_breakdown, exit_breakdown)]
                            long_wins, long_losses, short_wins, short_losses = trade_counts
                            long_durations, short_durations = durations
                            first_long_entry, last_long_exit, first_short_entry, last_short_exit = trade_dates
                            total_long_wins += long_wins; total_long_losses += long_losses
                            total_short_wins += short_wins; total_short_losses += short_losses
                            all_long_durations.extend(long_durations); all_short_durations.extend(short_durations)
                            
                            long_conf = np.mean([t['confidence'] for t in trades[0] if pd.notna(t.get('confidence'))]) if trades[0] else 0
                            short_conf = np.mean([t['confidence'] for t in trades[2] if pd.notna(t.get('confidence'))]) if trades[2] else 0
                            avg_long_dur_ticker = np.mean(long_durations) if long_durations else 0
                            avg_short_dur_ticker = np.mean(short_durations) if short_durations else 0
                            
                            summary_results.append({
                                "Ticker": ticker_symbol, "Cumulative Long P&L": long_pnl, "Avg Long Profit per Trade": avg_long_trade,
                                "Num Long Trades": len(trades[0]), "Avg Long Confidence": long_conf, "Avg Long Duration (Days)": avg_long_dur_ticker,
                                "First Long Entry": first_long_entry, "Last Long Exit": last_long_exit,
                                "Cumulative Short P&L": short_pnl, "Avg Short Profit per Trade": avg_short_trade,
                                "Num Short Trades": len(trades[2]), "Avg Short Confidence": short_conf, "Avg Short Duration (Days)": avg_short_dur_ticker,
                                "First Short Entry": first_short_entry, "Last Short Exit": last_short_exit
                            })
                            if open_trades:
                                for trade in open_trades: trade['Ticker'] = ticker_symbol; all_open_trades.append(trade)
                    
                    progress_bar.empty()
                    if excluded_analysis_tickers and not production_mode: st.warning(f"Excluded {len(excluded_analysis_tickers)} tickers due to unrealistic profit.")
                    
                    if summary_results:
                        st.session_state.summary_df = pd.DataFrame(summary_results).set_index('Ticker')
                        st.session_state.trade_counts = {"long_wins": total_long_wins, "long_losses": total_long_losses, "short_wins": total_short_wins, "short_losses": total_short_losses}
                        st.session_state.trade_durations = { "avg_long_duration": np.mean(all_long_durations) if all_long_durations else 0, "max_long_duration": np.max(all_long_durations) if all_long_durations else 0, "avg_short_duration": np.mean(all_short_durations) if all_short_durations else 0, "max_short_duration": np.max(all_short_durations) if all_short_durations else 0 }
                        st.session_state.exit_breakdown_totals = {
                            'long_profit_take_count': total_exit_breakdown[0], 'long_tsl_count': total_exit_breakdown[1], 'long_time_exit_count': total_exit_breakdown[2], 
                            'short_profit_take_count': total_exit_breakdown[3], 'short_tsl_count': total_exit_breakdown[4], 'short_time_exit_count': total_exit_breakdown[5]
                        }
                    else:
                        st.warning("No trades found.")
                        st.session_state.trade_durations = {}
                        st.session_state.exit_breakdown_totals = {}

                    st.session_state.open_trades_df = pd.DataFrame(all_open_trades) if all_open_trades else pd.DataFrame()
            st.rerun()

        # 10. Display Advisor Scan Results (raw_df)
        elif 'raw_df' in st.session_state and st.session_state.raw_df is not None:
            advisor_type = st.session_state.get('advisor_type', 'Advisor')
            st.subheader(f"👨‍💼 {advisor_type} Advisor: Trade Signals")
            c_view1, c_view2 = st.columns(2)
            dedupe_view = c_view1.checkbox("De-duplicate trades", value=True, key="advisor_dedupe_check")
            filter_trades = c_view2.checkbox("Hide Empty Tickers", value=True, key="advisor_filter_check")
            display_df = st.session_state.deduped_df if dedupe_view else st.session_state.raw_df
            
            if display_df is not None and not display_df.empty:
                if filter_trades and 'Status' in display_df.columns: display_df = display_df[display_df['Status'].notna()]
                
                format_dict = {
                    "Final % P/L": lambda x: f"{x:.2%}" if pd.notna(x) else '-',
                    "Date Open": lambda x: x.strftime('%Y-%m-%d') if pd.notna(x) else '-',
                    "Date Closed": lambda x: x.strftime('%Y-%m-%d') if pd.notna(x) else '-',
                    "Start Confidence": lambda x: f"{x:.0f}%" if pd.notna(x) else '-'
                }
                if "Setup G/B Ratio" in display_df.columns: format_dict["Setup G/B Ratio"] = lambda x: f"{x:.2f}" if pd.notna(x) else '-'
                for col in display_df.columns:
                     if any(x in col for x in ["Threshold", "Std Dev", "Ratio"]) and col not in format_dict:
                         format_dict[col] = lambda x: f"{x:.2f}" if pd.notna(x) and isinstance(x, (int, float)) else x
                st.dataframe(display_df.style.format(format_dict, na_rep='-'))
            else: st.info("No trades found.")

            if st.button("Back to Main Analysis"):
                st.session_state.advisor_df = None; st.session_state.raw_df = None; st.session_state.deduped_df = None
                st.session_state.run_advanced_advisor = False; st.session_state.run_user_advisor_setup = False
                st.session_state.run_scan_user_setups = False; st.rerun()

        # 11. Display Confidence Optimisation Results
        elif st.session_state.get('confidence_results_df') is not None and not st.session_state.confidence_results_df.empty:
            st.subheader("📊 Confidence Setup Optimisation Results")
            display_df_conf = st.session_state.confidence_results_df.head(60)
            conf_formatters = { "Avg Profit/Trade": "{:.2%}", "Ticker G/B Ratio": "{:.2f}", "Trade G/B Ratio": "{:.2f}", "Avg Entry Conf.": "{:.1f}%", "Good Score": "{:.4f}", "Bad Score": "{:.4f}", "Norm. Score %": "{:.2f}%" }
            if 'MACD' in display_df_conf.columns: conf_formatters['MACD'] = '{}'
            if 'MA Slope' in display_df_conf.columns: conf_formatters['MA Slope'] = '{}'
            if 'Markov' in display_df_conf.columns: conf_formatters['Markov'] = '{}'
            valid_conf_formatters = {k: (lambda val, fmt=v: fmt.format(val) if pd.notna(val) else '-') for k, v in conf_formatters.items() if k in display_df_conf.columns}
            st.dataframe(display_df_conf.style.format(valid_conf_formatters, na_rep='-'))

        # 12. Display Single Ticker Results
        elif st.session_state.get('single_ticker_results') is not None:
            res = st.session_state.single_ticker_results
            st.subheader(f"Results for {st.session_state.get('ticker_select')}")
            c1, c2, c3, c4 = st.columns(4)
            c1.metric("Cumulative Long P&L", f"{res.get('long_pnl', 0):.2%}")
            c2.metric("Avg Long Trade P&L", f"{res.get('avg_long_trade', 0):.2%}")
            c3.metric("Cumulative Short P&L", f"{res.get('short_pnl', 0):.2%}")
            c4.metric("Avg Short Trade P&L", f"{res.get('avg_short_trade', 0):.2%}")
            if res.get('results_df') is not None and not res['results_df'].empty:
                try:
                    st.plotly_chart(generate_long_plot(res['results_df'], res['trades'], st.session_state.get('ticker_select')), use_container_width=True)
                    st.plotly_chart(generate_short_plot(res['results_df'], res['trades'], st.session_state.get('ticker_select')), use_container_width=True)
                except Exception as e: st.error(f"Error generating plot: {e}")
            else: st.info("No chart data available.")

        # 13. Display Full Analysis Summary
        elif st.session_state.get('summary_df') is not None and not st.session_state.summary_df.empty:
            display_summary_analytics(st.session_state.summary_df)
            st.subheader("Results per Ticker")
            if st.checkbox("Only show tickers with trades", value=True):
                df_to_display = st.session_state.summary_df[(st.session_state.summary_df['Num Long Trades'] > 0) | (st.session_state.summary_df['Num Short Trades'] > 0)].copy()
            else:
                df_to_display = st.session_state.summary_df.copy()

            date_cols = ["First Long Entry", "Last Long Exit", "First Short Entry", "Last Short Exit"]
            for col in date_cols:
                if col in df_to_display.columns: df_to_display[col] = pd.to_datetime(df_to_display[col], errors='coerce').dt.strftime('%Y-%m-%d')
            df_to_display.fillna('-', inplace=True)
            st.dataframe(df_to_display.style.format({ "Cumulative Long P&L": "{:.2%}", "Avg Long Profit per Trade": "{:.2%}", "Avg Long Duration (Days)": "{:.1f}", "Cumulative Short P&L": "{:.2%}", "Avg Short Profit per Trade": "{:.2%}", "Avg Short Duration (Days)": "{:.1f}", "Avg Long Confidence": "{:.0f}%", "Avg Short Confidence": "{:.0f}%" }, na_rep='-'))
            
            if not production_mode:
                if st.button("💾 Add these settings to User-Defined List", key="save_setup_from_analysis", on_click=add_setup_to_user_list): pass

            if st.session_state.get('open_trades_df') is not None and not st.session_state.open_trades_df.empty:
                # --- UPDATED HEADER WITH ICON ---
                st.subheader("👨‍💼 Open Positions & Recently Closed", 
                             help="This table displays all currently ACTIVE trades, plus any trades that closed within the last 30 days. Older closed trades are included in the statistics above but hidden here to keep the list relevant.")
                
                display_open_df = st.session_state.open_trades_df.copy()
                display_open_df.sort_values(by=['Status', 'Date Open'], ascending=[True, False], inplace=True)
                cols_order_manual = ['Ticker', 'Status', 'Final % P/L', 'Side', 'Date Open', 'Date Closed', 'Start Confidence']
                existing_cols_open = [col for col in cols_order_manual if col in display_open_df.columns]
                if existing_cols_open:
                     st.dataframe(display_open_df[existing_cols_open].style.format({
                        "Final % P/L": lambda x: f"{x:.2%}" if pd.notna(x) else '-',
                        "Date Open": lambda x: x.strftime('%Y-%m-%d') if pd.notna(x) else '-',
                        "Date Closed": lambda x: x.strftime('%Y-%m-%d') if pd.notna(x) else '-',
                        "Start Confidence": lambda x: f"{x:.0f}%" if pd.notna(x) else '-'
                    }, na_rep='-'))

        # 14. Default Message
        else:
            if not any([st.session_state.get(k) for k in ['run_advanced_advisor','run_user_advisor_setup','advisor_df','confidence_results_df','single_ticker_results','summary_df','load_message']]): 
                st.info("Click a 'Run' button in the sidebar to start.")

        # 15. Footer Buttons (Hidden in Production)
        if not production_mode:
            st.markdown("---")
            col_load1, col_load2, col_load3 = st.columns(3)
            with col_load1:
                if st.session_state.get('best_params'): st.button("⬇️ Load Optimal Parameters", on_click=apply_best_params_to_widgets, use_container_width=True)
            with col_load2:
                if 'apply_best_weights_to_widgets' in locals() and st.session_state.get('best_weights'): st.button("⬇️ Load Optimal Weights", on_click=apply_best_weights_to_widgets, use_container_width=True)
            with col_load3:
                if st.session_state.get('worst_confidence_setups_list'):
                    if st.button("Apply Top Worst Setups as Veto Filter", use_container_width=True):
                        st.session_state.veto_setup_list = st.session_state.worst_confidence_setups_list
                        st.session_state.worst_confidence_setups_list = None
                        st.sidebar.info(f"Applying {len(st.session_state.veto_setup_list)} Veto filters.")
                        st.rerun()

            col_load4 = st.columns(1)[0]
            with col_load4:
                 is_markov_relevant = (st.session_state.primary_driver == 'Markov State') or st.session_state.use_markov
                 if st.session_state.get('best_markov_setup') and is_markov_relevant:
                    if st.button("⬇️ Load Best Markov Setup", on_click=None, use_container_width=True):
                        st.session_state.primary_driver = "Markov State"
                        st.session_state.use_markov = True 
                        st.sidebar.success("Best Markov setup loaded!")
                        st.rerun()

if __name__ == "__main__":
    main()