src/streamlit_app.py

# -----------------------
# streamlit_trading_dashboard.py
# Added: NSE market calendar check (pandas_market_calendars when available; yf fallback)
# -----------------------
import warnings
import logging
import urllib3
import streamlit as st # type: ignore
import pandas as pd
import numpy as np
import yfinance as yf
import time
import threading
from datetime import datetime, timedelta, date
import plotly.graph_objects as go

from sklearn.linear_model import LinearRegression, ElasticNet
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor, ExtraTreesRegressor
from sklearn.svm import SVR
from sklearn.neighbors import KNeighborsRegressor
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_absolute_error, mean_squared_error
from statsmodels.tsa.arima.model import ARIMA
from statsmodels.tsa.statespace.sarimax import SARIMAX
from statsmodels.tsa.holtwinters import ExponentialSmoothing

# optional tensorflow / transformers
TENSORFLOW_AVAILABLE = False
try:
    import tensorflow as tf
    from tensorflow.keras.models import Sequential # type: ignore
    from tensorflow.keras.layers import LSTM, Dense, Input # type: ignore
    TENSORFLOW_AVAILABLE = True
except Exception:
    TENSORFLOW_AVAILABLE = False

TRANSFORMERS_AVAILABLE = False
try:
    from transformers import pipeline
    TRANSFORMERS_AVAILABLE = True
except Exception:
    TRANSFORMERS_AVAILABLE = False

# silence some noisy output
warnings.filterwarnings("ignore", category=DeprecationWarning)
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
logging.getLogger("yfinance").setLevel(logging.ERROR)
logging.getLogger("urllib3").setLevel(logging.ERROR)

st.set_page_config(page_title="Trading Dashboard", layout="wide")
st.markdown(
    """
    <style>
    .topbar { padding: 10px 16px; border-radius: 8px; background: linear-gradient(90deg,#0f172a,#0b1220); color: white; margin-bottom: 8px;}
    .stButton>button {width:100% !important; height:40px; margin-bottom:6px;}
    table {border-collapse: collapse;}
    th, td {padding:6px 8px;}
    .arrow-up { color: green; font-weight:700 }
    .arrow-down { color: red; font-weight:700 }
    </style>
    """,
    unsafe_allow_html=True,
)
st.markdown('<div class="topbar"><h2>Trading Dashboard </h2></div>', unsafe_allow_html=True)

# simple global live snapshot + thread control
LIVE_PRICE = None
LIVE_LOCK = threading.Lock()
LIVE_STOP_EVENT = None
LIVE_THREAD = None
LIVE_WS = None  # <<-- hold reference to websocket object so we can close it

# -----------------------
# cached helpers
# -----------------------
@st.cache_data(show_spinner=False)
def safe_history(ticker, period="10y", interval="1d"):
    t = yf.Ticker(ticker)
    try:
        hist = t.history(period=period, interval=interval, auto_adjust=False)
    except Exception:
        hist = pd.DataFrame()
    return hist

@st.cache_data(show_spinner=False)
def safe_info(ticker):
    t = yf.Ticker(ticker)
    try:
        return t.info
    except Exception:
        return {}

@st.cache_data(show_spinner=False)
def safe_news(ticker, count=200):
    t = yf.Ticker(ticker)
    try:
        n = getattr(t, "get_news", None)
        if callable(n):
            return n(count=count)
        return getattr(t, "news", []) or []
    except Exception:
        return []

@st.cache_data(show_spinner=False)
def safe_financials(ticker):
    # avoid fundamentals calls for index tickers to prevent 404 noise
    if isinstance(ticker, str) and ticker.startswith("^"):
        return None, None, None
    t = yf.Ticker(ticker)
    def sg(attr):
        try:
            a = getattr(t, attr)
            return a() if callable(a) else a
        except Exception:
            return None
    balance = sg("get_balance_sheet")
    if balance is None or (hasattr(balance, "empty") and balance.empty):
        balance = sg("balance_sheet")
    income = sg("get_income_stmt")
    if income is None or (hasattr(income, "empty") and income.empty):
        income = sg("income_stmt")
    cashflow = sg("get_cashflow")
    if cashflow is None or (hasattr(cashflow, "empty") and cashflow.empty):
        cashflow = sg("cashflow")
    return balance, income, cashflow

@st.cache_resource(show_spinner=False)
def load_finbert_pipeline():
    if not TRANSFORMERS_AVAILABLE:
        return None
    try:
        return pipeline("sentiment-analysis", model="ProsusAI/finbert", tokenizer="ProsusAI/finbert", device=-1)
    except Exception:
        try:
            return pipeline("sentiment-analysis", device=-1)
        except Exception:
            return None

def finbert_score_from_label(item):
    label = item.get("label", "").lower()
    score = float(item.get("score", 0.0))
    if "neg" in label:
        return -score
    if "pos" in label:
        return score
    return 0.0

def compute_news_sentiment(news_list, backend="finbert"):
    if not news_list:
        return pd.Series(dtype=float)
    items = []
    for it in news_list:
        c = it.get("content", {}) if isinstance(it, dict) else {}
        title = (c.get("title") or it.get("title") or "")
        summary = (c.get("summary") or it.get("summary") or "")
        text = (title + " " + summary).strip()
        pdstr = c.get("pubDate") or c.get("displayTime") or it.get("pubDate") or it.get("displayTime")
        try:
            ts = pd.to_datetime(pdstr) if pdstr else pd.Timestamp.now()
        except Exception:
            ts = pd.Timestamp.now()
        items.append({"dt": ts.normalize(), "text": text[:2048]})
    dfn = pd.DataFrame(items).set_index("dt")
    if dfn.empty:
        return pd.Series(dtype=float)
    if backend == "finbert":
        pipe = load_finbert_pipeline()
        if pipe is None:
            return pd.Series(dtype=float)
        scores = []
        for txt in dfn["text"].tolist():
            try:
                out = pipe(txt[:1024])
                out0 = out[0] if isinstance(out, list) else out
                scores.append(finbert_score_from_label(out0))
            except Exception:
                scores.append(np.nan)
        dfn["score"] = scores
        dfn_mean = dfn.groupby(dfn.index.date)["score"].mean()
        dfn_mean.index = pd.to_datetime(dfn_mean.index)
        return dfn_mean.sort_index()
    return pd.Series(dtype=float)

# -----------------------
# Market open check for Indian market (NSE)
# -----------------------
import yfinance as yf
import pandas as pd
import requests
from datetime import date, datetime, timedelta


HOLIDAY_CACHE = None


def load_nse_holidays():
    global HOLIDAY_CACHE
    if HOLIDAY_CACHE is not None:
        return HOLIDAY_CACHE

    url = "https://www.nseindia.com/api/holiday-master?type=trading"
    headers = {"User-Agent": "Mozilla/5.0", "Accept": "application/json"}

    r = requests.get(url, headers=headers, timeout=10)
    r.raise_for_status()
    data = r.json()

    holidays = set()
    for item in data.get("CBM", []) + data.get("TRADING", []) + data.get("CLEARING", []):
        try:
            d = datetime.strptime(item["tradingDate"], "%d-%b-%Y").date()
            holidays.add(d)
        except:
            pass

    HOLIDAY_CACHE = holidays
    return holidays

def fallback(value, last_close):
    try:
        v = float(value)
        if pd.isna(v):
            return float(last_close)
        return v
    except:
        return float(last_close)

def to_returns(series):
    return series.pct_change().dropna()

def price_from_returns(last_price, future_returns):
    price = last_price
    for r in future_returns:
        price *= (1 + r)
    return float(price)

def is_nse_open_on(check_date: date):
    """
    Returns: (is_open, reason)
    """

    # 1️⃣ weekend
    if check_date.weekday() >= 5:
        return False, "weekend"

    # 2️⃣ official NSE holidays
    try:
        if check_date in load_nse_holidays():
            return False, "nse_official_holiday"
    except:
        pass

    # 3️⃣ market calendar (authoritative for trading sessions)
    calendar_says_open = None
    try:
        import pandas_market_calendars as mcal

        try:
            cal = mcal.get_calendar("NSE")
        except:
            cal = mcal.get_calendar("XNSE")

        start = (check_date - timedelta(days=2)).strftime("%Y-%m-%d")
        end   = (check_date + timedelta(days=2)).strftime("%Y-%m-%d")

        sched = cal.schedule(start_date=start, end_date=end)

        if sched is not None and not sched.empty:
            session_days = {pd.to_datetime(x).date() for x in sched.index}
            if check_date in session_days:
                calendar_says_open = True
            else:
                return False, "nse_calendar:holiday"
    except:
        calendar_says_open = None

    # 4️⃣ Yahoo (used ONLY as a sanity check — never to block predictions)
    try:
        t = yf.Ticker("^NSEI")
        win_start = (pd.Timestamp(check_date) - pd.Timedelta(days=10)).strftime("%Y-%m-%d")
        win_end   = (pd.Timestamp(check_date) + pd.Timedelta(days=2)).strftime("%Y-%m-%d")
        hist = t.history(start=win_start, end=win_end, interval="1d")

        if not hist.empty:
            traded_days = {pd.to_datetime(x).date() for x in hist.index}
            if check_date in traded_days:
                return True, "history"

        # ⭐️ IMPORTANT CHANGE:
        # If calendar says open but Yahoo has no data (future days) → STILL OPEN
        if calendar_says_open:
            return True, "calendar_open_no_yahoo_data"

        # otherwise treat as holiday
        return False, "no_data_yahoo"

    except Exception:
        # if Yahoo fails entirely but calendar says open → STILL OPEN
        if calendar_says_open:
            return True, "calendar_open_yahoo_error"
        return False, "yahoo_error"


# small utilities
def human_readable(n):
    try:
        n = float(n)
    except Exception:
        return n
    units = [(1e12, "T"), (1e9, "B"), (1e6, "M"), (1e3, "K")]
    for div, suf in units:
        if abs(n) >= div:
            return f"{n/div:,.2f}{suf}"
    return f"{n:,.2f}"

def create_supervised_features(series: pd.Series, lags: int = 5, exog: pd.DataFrame = None):
    df = pd.DataFrame({"y": series})
    for lag in range(1, lags + 1):
        df[f"lag_{lag}"] = df["y"].shift(lag)
    if exog is not None:
        exog = exog.reindex(df.index).ffill().fillna(0.0)
        for c in exog.columns:
            df[c] = exog[c].values
    df = df.dropna()
    return df

def evaluate_preds(y_true, y_pred):
    y_true = np.array(y_true).astype(float)
    y_pred = np.array(y_pred).astype(float)
    mae = float(np.mean(np.abs(y_true - y_pred)))
    rmse = float(np.sqrt(np.mean((y_true - y_pred) ** 2)))
    denom = np.where(np.abs(y_true) < 1e-8, 1e-8, np.abs(y_true))
    mape = float(np.mean(np.abs((y_true - y_pred) / denom))) * 100.0
    return {"MAE": mae, "RMSE": rmse, "MAPE": mape}

# -----------------------
# session defaults & top UI
# -----------------------
if "loaded_ticker" not in st.session_state:
    st.session_state.loaded_ticker = None
if "hist" not in st.session_state:
    st.session_state.hist = pd.DataFrame()
if "info" not in st.session_state:
    st.session_state.info = {}
if "news" not in st.session_state:
    st.session_state.news = []
if "balance" not in st.session_state:
    st.session_state.balance = None
if "income" not in st.session_state:
    st.session_state.income = None
if "cashflow" not in st.session_state:
    st.session_state.cashflow = None
if "selected_chart" not in st.session_state:
    st.session_state.selected_chart = "Price"
if "live_running" not in st.session_state:
    st.session_state.live_running = False
if "news_sentiment" not in st.session_state:
    st.session_state.news_sentiment = pd.Series(dtype=float)

# Top controls layout (uses st.toggle for live)
col1, col2, col3 = st.columns([3, 2, 2])
with col1:
    ticker = st.text_input("Ticker (e.g., ICICIBANK.NS or ^NSEI)", value=st.session_state.loaded_ticker or "^NSEI", key="ticker_input")
    load_btn = st.button("Load / Refresh")
with col2:
    period = st.selectbox("History period", ["6mo", "1y", "2y", "5y", "10y"], index=4, key="period_input")
    interval = st.selectbox("Interval", ["1d", "1wk", "1mo"], index=0, key="interval_input")
with col3:
    # Use st.toggle (user requested) — it's a boolean control
    try:
        live_toggle = st.toggle("Live stream (toggle ON/OFF)", value=st.session_state.get("live_running", False), key="live_toggle")
    except Exception:
        # fallback if st.toggle is not available: use checkbox (keeps functionality)
        live_toggle = st.checkbox("Live stream (toggle ON/OFF)", value=st.session_state.get("live_running", False), key="live_toggle_fallback")
    poll_seconds = st.number_input("Poll interval (s)", min_value=5, max_value=60, value=10, step=1, key="poll_interval")
    refresh_now = st.button("Refresh Live Now")

def load_ticker(ticker_symbol, period, interval):
    hist = safe_history(ticker_symbol, period=period, interval=interval)
    info = safe_info(ticker_symbol)
    news = safe_news(ticker_symbol, count=200)
    bal, inc, cf = safe_financials(ticker_symbol)
    if not hist.empty and not isinstance(hist.index, pd.DatetimeIndex):
        hist.index = pd.to_datetime(hist.index)
    st.session_state.loaded_ticker = ticker_symbol
    st.session_state.hist = hist
    st.session_state.info = info
    st.session_state.news = news
    st.session_state.balance = bal
    st.session_state.income = inc
    st.session_state.cashflow = cf
    st.session_state.news_sentiment = pd.Series(dtype=float)
    global LIVE_PRICE
    with LIVE_LOCK:
        LIVE_PRICE = LIVE_PRICE

if load_btn:
    load_ticker(ticker, period, interval)
if st.session_state.loaded_ticker is None:
    load_ticker(ticker, period, interval)

# --- Live stream (fixed stop) ---
def ws_listener(ticker_sym, stop_event):
    WS = getattr(yf, "WebSocket", None)
    if WS is None:
        return False

    try:
        with WS() as ws:
            ws.subscribe([ticker_sym])

            for message in ws.listen():
                if stop_event.is_set():
                    try:
                        ws.unsubscribe([ticker_sym])
                        ws.close()          
                    except Exception:
                        pass
                    break

                try:
                    if isinstance(message, dict):
                        price = message.get("price") or message.get("close")
                        ts = message.get("time") or datetime.now().isoformat()

                        try:
                            if isinstance(ts, str) and ts.isdigit() and len(ts) >= 10:
                                ts = datetime.fromtimestamp(
                                    int(ts) / (1000.0 if len(ts) >= 13 else 1)
                                ).isoformat()
                        except Exception:
                            pass

                        if price is not None:
                            with LIVE_LOCK:
                                global LIVE_PRICE
                                LIVE_PRICE = {"price": float(price), "time": str(ts)}
                except Exception:
                    continue

        return True

    except Exception:
        return False


def poller(ticker_sym, interval_s, stop_event):
    while not stop_event.is_set():
        try:
            t = yf.Ticker(ticker_sym)
            recent = t.history(period="1d", interval="1m", auto_adjust=False)
            if not recent.empty:
                price = float(recent["Close"].iloc[-1])
                ts = str(recent.index[-1])
                with LIVE_LOCK:
                    global LIVE_PRICE
                    LIVE_PRICE = {"price": price, "time": ts}
            else:
                h = st.session_state.hist
                if not h.empty:
                    with LIVE_LOCK:
                        LIVE_PRICE = {"price": float(h["Close"].iloc[-1]), "time": str(h.index[-1])}
        except Exception:
            with LIVE_LOCK:
                LIVE_PRICE = {"error": "poll error"}
        # wait so we can exit quickly when stop_event is set
        stop_event.wait(timeout=interval_s)

def start_live_thread(ticker_sym, interval_s):
    global LIVE_STOP_EVENT, LIVE_THREAD, LIVE_WS
    # If there's an existing run, stop it first (defensive)
    if LIVE_STOP_EVENT is not None and not LIVE_STOP_EVENT.is_set():
        stop_live_thread()

    LIVE_STOP_EVENT = threading.Event()
    LIVE_WS = None

    def runner(ev, tck, interval_local):
        used = False
        try:
            used = ws_listener(tck, ev)
        except Exception:
            used = False
        if not used and not ev.is_set():
            poller(tck, interval_local, ev)
        # thread exit cleanup
        try:
            # ensure WS closed
            if LIVE_WS is not None:
                try:
                    if hasattr(LIVE_WS, "close"):
                        LIVE_WS.close()
                except Exception:
                    pass
                finally:
                    try:
                        # try to close underlying socket if accessible
                        sock = getattr(LIVE_WS, "_sock", None) or getattr(LIVE_WS, "sock", None)
                        if sock is not None:
                            try:
                                sock.close()
                            except Exception:
                                pass
                    except Exception:
                        pass
                    try:
                        # ensure cleared
                        globals()['LIVE_WS'] = None
                    except Exception:
                        pass
        except Exception:
            pass
        finally:
            try:
                st.session_state.live_running = False
            except Exception:
                pass

    LIVE_THREAD = threading.Thread(target=runner, args=(LIVE_STOP_EVENT, ticker_sym, interval_s), daemon=True)
    LIVE_THREAD.start()
    st.session_state.live_running = True

def stop_live_thread():
    global LIVE_STOP_EVENT, LIVE_THREAD, LIVE_WS
    # set the stop event
    if LIVE_STOP_EVENT is not None:
        try:
            LIVE_STOP_EVENT.set()
        except Exception:
            pass

    # Best-effort: close the websocket object to unblock ws.listen()
    if LIVE_WS is not None:
        try:
            if hasattr(LIVE_WS, "close"):
                try:
                    LIVE_WS.close()
                except Exception:
                    pass
            # try underlying socket objects too
            sock = getattr(LIVE_WS, "_sock", None) or getattr(LIVE_WS, "sock", None)
            if sock is not None:
                try:
                    sock.close()
                except Exception:
                    pass
        except Exception:
            pass
        finally:
            try:
                globals()['LIVE_WS'] = None
            except Exception:
                pass

    # try to join thread briefly
    try:
        if LIVE_THREAD is not None:
            LIVE_THREAD.join(timeout=1.0)
    except Exception:
        pass

    LIVE_THREAD = None
    st.session_state.live_running = False

# react to toggle state change
# If toggle is True and thread not running -> start it
# If toggle is False and thread running -> stop it
if live_toggle and not st.session_state.get("live_running", False):
    start_live_thread(st.session_state.loaded_ticker or ticker, int(poll_seconds))
if (not live_toggle) and st.session_state.get("live_running", False):
    stop_live_thread()

# Refresh now
if refresh_now:
    try:
        t_temp = yf.Ticker(st.session_state.loaded_ticker or ticker)
        recent = t_temp.history(period="1d", interval="1m", auto_adjust=False)
        if not recent.empty:
            price = float(recent["Close"].iloc[-1])
            ts = str(recent.index[-1])
            with LIVE_LOCK:
                LIVE_PRICE = {"price": price, "time": ts}
        else:
            h = st.session_state.hist
            if not h.empty:
                with LIVE_LOCK:
                    LIVE_PRICE = {"price": float(h["Close"].iloc[-1]), "time": str(h.index[-1])}
    except Exception:
        with LIVE_LOCK:
            LIVE_PRICE = {"error": "refresh failed"}

with LIVE_LOCK:
    live_snapshot = dict(LIVE_PRICE) if LIVE_PRICE is not None else None

tabs = st.tabs(["Overview", "Financials", "News", "Charts", "Forecast"])

# Overview
with tabs[0]:
    info = st.session_state.info or {}
    hist = st.session_state.hist.copy()
    st.subheader(info.get("shortName") or info.get("longName") or st.session_state.loaded_ticker)
    if hist.empty:
        st.warning("No historical data available for this ticker.")
    else:
        last_close = float(hist["Close"].iloc[-1])
        prev_close = float(hist["Close"].iloc[-2]) if len(hist) > 1 else last_close
        change = last_close - prev_close
        pct = (change / prev_close * 100) if prev_close != 0 else 0.0
        live_price_val = live_snapshot.get("price") if live_snapshot else None
        live_time_val = live_snapshot.get("time") if live_snapshot else None

        chart_col, metrics_col = st.columns([8, 6])
        with chart_col:
            fig = go.Figure()
            fig.add_trace(go.Scatter(x=hist.index, y=hist["Close"], name="Close", line=dict(width=2)))
            fig.update_layout(height=560, margin=dict(l=0, r=0, t=20, b=0))
            st.plotly_chart(fig, use_container_width=True)

        with metrics_col:
            r1c1, r1c2 = st.columns([1, 1])
            with r1c1:
                if live_price_val is not None:
                    st.metric("Live Price", f"{live_price_val:.2f}", delta=f"{(live_price_val - last_close):.2f}")
                else:
                    st.metric("Live Price", "—")
                if live_time_val:
                    st.caption(f"Live update: {live_time_val}")
            with r1c2:
                st.metric("Last Close", f"{last_close:.2f}", delta=f"{change:.2f}")
                st.markdown(f"**Pct change:** {pct:.2f}%")
            st.markdown("---")
            r2c1, r2c2 = st.columns([1, 1])
            with r2c1:
                try:
                    st.metric("Open", f"{float(hist['Open'].iloc[-1]):.2f}")
                except Exception:
                    st.metric("Open", "—")
            with r2c2:
                st.metric("Previous Close", f"{prev_close:.2f}")
            st.markdown("---")
            r3c1, r3c2, r3c3 = st.columns([1, 1, 1])
            with r3c1:
                st.metric("Market Cap", human_readable(info.get("marketCap", "—")))
            with r3c2:
                st.metric("PE (TTM)", f"{info.get('trailingPE', '—')}")
            with r3c3:
                st.metric("Beta", f"{info.get('beta', '—')}")
            st.markdown("---")
            r4c1, r4c2 = st.columns([1, 1])
            with r4c1:
                st.metric("52W High", f"{info.get('fiftyTwoWeekHigh', '—')}")
            with r4c2:
                st.metric("52W Low", f"{info.get('fiftyTwoWeekLow', '—')}")

# Financials
with tabs[1]:
    st.header("Financial Statements & Key Data")
    bal, inc, cf = st.session_state.balance, st.session_state.income, st.session_state.cashflow
    if bal is None and inc is None and cf is None:
        st.info("No financial statements available (likely an index ticker or missing data).")
    else:
        with st.expander("Balance Sheet"):
            st.dataframe(bal if bal is not None else pd.DataFrame())
        with st.expander("Income Statement"):
            st.dataframe(inc if inc is not None else pd.DataFrame())
        with st.expander("Cash Flow"):
            st.dataframe(cf if cf is not None else pd.DataFrame())
    if not (isinstance(st.session_state.loaded_ticker, str) and st.session_state.loaded_ticker.startswith("^")):
        t = yf.Ticker(st.session_state.loaded_ticker)
        try:
            rec = t.get_recommendations() if getattr(t, "get_recommendations", None) else getattr(t, "recommendations", None)
            if rec is not None and len(rec) > 0:
                st.subheader("Analyst Recommendations")
                st.dataframe(rec.head(20))
        except Exception:
            pass
        try:
            earnings = t.get_earnings() if getattr(t, "get_earnings", None) else getattr(t, "earnings", None)
            if earnings is not None and len(earnings) > 0:
                st.subheader("Earnings History")
                st.dataframe(earnings.tail(10))
        except Exception:
            pass
        try:
            capg = t.get_capital_gains() if getattr(t, "get_capital_gains", None) else getattr(t, "capital_gains", None)
            if capg is not None:
                st.subheader("Capital Gains")
                st.dataframe(capg)
        except Exception:
            pass
    else:
        st.info("Skipping recommendations/earnings/capital gains for index tickers (e.g., ^NSEI).")

# News & FinBERT
with tabs[2]:
    st.header("News & Sentiment")
    news_list = st.session_state.news or []
    if not news_list:
        st.info("No news available for this ticker.")
    else:
        use_finbert = st.checkbox("Enable FinBERT sentiment (optional)", value=False)
        N = st.slider("Number of news items to show", 1, min(100, len(news_list)), min(10, len(news_list)))
        items = news_list[:N]
        pipe = None
        if use_finbert:
            with st.spinner("Loading FinBERT..."):
                pipe = load_finbert_pipeline()
            if pipe is None:
                st.warning("FinBERT not available (install transformers & torch).")
        for it in items:
            c = it.get("content", {}) if isinstance(it, dict) else {}
            title = c.get("title") or it.get("title") or "No title"
            summary = c.get("summary") or it.get("summary") or ""
            provider = (c.get("provider") or {}).get("displayName") or ""
            published = c.get("pubDate") or c.get("displayTime") or ""
            url = (c.get("canonicalUrl") or {}).get("url") or (c.get("clickThroughUrl") or {}).get("url") or ""
            st.markdown(f"### [{title}]({url})")
            st.caption(f"{provider} • {published}")
            if summary:
                st.write(summary)
            if pipe is not None:
                try:
                    out = pipe((title + " " + summary)[:1024])
                    out0 = out[0] if isinstance(out, list) else out
                    score = finbert_score_from_label(out0)
                    badge = "Bullish 🔺" if score > 0.05 else ("Bearish 🔻" if score < -0.05 else "Neutral")
                    st.markdown(f"**FinBERT:** {badge} ({score:.3f})")
                except Exception as e:
                    st.write("FinBERT error:", e)
            st.divider()

# Charts
with tabs[3]:
    st.header("Charts")
    left_col, right_col = st.columns([1, 4])
    charts = ["Price", "Candlestick", "EMA", "SMA", "RSI", "MACD", "Volume"]
    with left_col:
        for ch in charts:
            if st.button(ch, key=f"chart_btn_{ch}"):
                st.session_state.selected_chart = ch
    with right_col:
        sel = st.session_state.selected_chart or "Price"
        st.markdown(f"### {sel}")
        hist = st.session_state.hist
        if hist.empty:
            st.info("No history to plot.")
        else:
            if sel == "Price":
                fig = go.Figure(); fig.add_trace(go.Scatter(x=hist.index, y=hist["Close"], name="Close")); st.plotly_chart(fig, use_container_width=True)
            elif sel == "Candlestick":
                fig = go.Figure(); fig.add_trace(go.Candlestick(x=hist.index, open=hist["Open"], high=hist["High"], low=hist["Low"], close=hist["Close"])); st.plotly_chart(fig, use_container_width=True)
            elif sel == "EMA":
                hist["EMA20"] = hist["Close"].ewm(span=20).mean(); hist["EMA50"] = hist["Close"].ewm(span=50).mean()
                fig = go.Figure(); fig.add_trace(go.Scatter(x=hist.index, y=hist["EMA20"], name="EMA20")); fig.add_trace(go.Scatter(x=hist.index, y=hist["EMA50"], name="EMA50")); st.plotly_chart(fig, use_container_width=True)
            elif sel == "SMA":
                hist["SMA50"] = hist["Close"].rolling(50).mean(); fig = go.Figure(); fig.add_trace(go.Scatter(x=hist.index, y=hist["SMA50"], name="SMA50")); st.plotly_chart(fig, use_container_width=True)
            elif sel == "RSI":
                delta = hist["Close"].diff()
                up = delta.clip(lower=0).rolling(14).mean()
                down = -delta.clip(upper=0).rolling(14).mean()
                rsi = 100 - (100 / (1 + (up / down).replace([np.inf, -np.inf], np.nan)))
                fig = go.Figure(); fig.add_trace(go.Scatter(x=hist.index, y=rsi, name="RSI")); st.plotly_chart(fig, use_container_width=True)
            elif sel == "MACD":
                ema12 = hist["Close"].ewm(span=12).mean(); ema26 = hist["Close"].ewm(span=26).mean()
                macd = ema12 - ema26; signal = macd.ewm(span=9).mean()
                fig = go.Figure(); fig.add_trace(go.Scatter(x=hist.index, y=macd, name="MACD")); fig.add_trace(go.Scatter(x=hist.index, y=signal, name="Signal")); st.plotly_chart(fig, use_container_width=True)
            elif sel == "Volume":
                fig = go.Figure(); fig.add_trace(go.Bar(x=hist.index, y=hist["Volume"], name="Volume")); st.plotly_chart(fig, use_container_width=True)

# helper: naive exog forecast
def forecast_exog_from_price(last_close_series, days_ahead, ema_spans=(10, 50), last_volume=None, last_sentiment=0.0):
    if last_close_series.empty or days_ahead <= 0:
        return pd.DataFrame(columns=['EMA10', 'EMA50', 'Volume_fill', 'sentiment'])
    y = last_close_series.values
    n = len(y)
    if n >= 3:
        t = np.arange(n).reshape(-1, 1)
        lr = LinearRegression().fit(t, y)
        future_t = np.arange(n, n + days_ahead).reshape(-1, 1)
        price_fore = lr.predict(future_t)
    else:
        k = min(5, n)
        mv = np.mean(y[-k:]) if n > 0 else 0
        price_fore = np.array([mv] * days_ahead)
    ema_vals = {}
    for span in ema_spans:
        alpha = 2.0 / (span + 1.0)
        last_ema = last_close_series.ewm(span=span).mean().iloc[-1]
        arr = []
        prev = last_ema
        for p in price_fore:
            prev = alpha * p + (1 - alpha) * prev
            arr.append(prev)
        ema_vals[f"EMA{span}"] = arr
    volumes = [float(last_volume) if last_volume is not None else 0.0] * days_ahead
    sentiments = [float(last_sentiment)] * days_ahead
    future_index = [ (last_close_series.index[-1].date() + timedelta(days=i+1)) for i in range(days_ahead) ]
    df = pd.DataFrame({
        "EMA10": ema_vals.get("EMA10", [np.nan]*days_ahead),
        "EMA50": ema_vals.get("EMA50", [np.nan]*days_ahead),
        "Volume_fill": volumes,
        "sentiment": sentiments
    }, index=pd.to_datetime(future_index))
    return df

# -----------------------
# Forecast & Backtest (models with NaN models) - REPLACEMENT BLOCK
# Paste this entire block in place of your current "with tabs[4]:" section
# -----------------------
# Forecast & Backtest (with trimming NaN models)
with tabs[4]:
    st.header("Forecast & Backtest (models with NaN removed)")
    hist = st.session_state.hist.copy()
    if hist.empty:
        st.info("No historical data to forecast.")
    else:
        col_a, col_b = st.columns([3, 1])
        with col_a:
            target_date = st.date_input("Forecast target date", value=date.today() + timedelta(days=1))
            available_models = [
                'Naive', 'SeasonalNaive', 'MovingAverage', 'LinearReg', 'RandomForest',
                'ARIMA', 'ARIMAX', 'LSTM', 'GradientBoosting', 'ExtraTrees',
                'ElasticNet', 'KNN', 'SVR', 'HoltWinters', 'EnsembleMean'
            ]
            models_sel = st.multiselect("Models", available_models, default=['LSTM','ARIMA','LinearReg','RandomForest','HoltWinters'])
            use_finbert_for_forecast = st.checkbox("Use FinBERT sentiment as exogenous feature (optional)", value=False)
            run = st.button("Run Forecasts")
            st.markdown("### Backtesting (walk-forward 1-step)")
            backtest_run = st.button("Run Backtest")
            n_obs = len(hist)
            default_train = min(252, max(30, n_obs - 10)) if n_obs > 40 else max(10, n_obs - 2)
            bt_window = st.number_input("Backtest train window (days)", min_value=10, max_value=max(10, n_obs-1), value=int(default_train))
            default_steps = max(1, min(100, max(1, n_obs - int(bt_window) - 1)))
            bt_steps = st.number_input("Backtest steps (max test points)", min_value=1, max_value=max(1, n_obs-1), value=int(default_steps))
            lags = st.number_input("Lag features (for ML models)", min_value=1, max_value=30, value=5)
        with col_b:
            st.metric("Last Close", f"{float(hist['Close'].iloc[-1]):.2f}")

        # compute sentiment if needed
        if use_finbert_for_forecast:
            with st.spinner("Computing FinBERT sentiment series..."):
                s_series = compute_news_sentiment(st.session_state.news, backend="finbert")
                if not s_series.empty:
                    sentiment_map = {pd.Timestamp(d).date(): v for d, v in s_series.items()}
                    hist['sentiment'] = [sentiment_map.get(d.date(), np.nan) for d in hist.index]
                    hist['sentiment'] = pd.Series(hist['sentiment']).ffill().fillna(0.0).values
                else:
                    hist['sentiment'] = 0.0
        else:
            hist['sentiment'] = 0.0

        hist['EMA10'] = hist['Close'].ewm(span=10).mean()
        hist['EMA50'] = hist['Close'].ewm(span=50).mean()
        hist['Volume_fill'] = hist['Volume'].ffill().fillna(0.0)

        last_dt = hist.index[-1]
        days_ahead = (target_date - last_dt.date()).days
        if days_ahead < 0:
            days_ahead = 0
        future_exog = forecast_exog_from_price(hist['Close'], max(days_ahead,1), ema_spans=(10,50),
                                              last_volume=float(hist['Volume'].iloc[-1]) if not hist['Volume'].empty else 0.0,
                                              last_sentiment=float(hist['sentiment'].iloc[-1]) if 'sentiment' in hist.columns else 0.0)

        # run forecasts
        if run:
            if days_ahead == 0:
                st.warning("Target date must be after last available historical date for forecasting.")
            else:
                # check NSE market calendar before making predictions
                is_open, reason = is_nse_open_on(target_date)
                if not is_open:
                    st.warning(f"Forecast skipped — market appears closed on {target_date} (reason: {reason}).")
                else:
                    close = hist['Close']
                    results = {}

                    def try_float(x):
                        try:
                            return float(x)
                        except Exception:
                            return np.nan

                    # Naive
                    if 'Naive' in models_sel:
                        results['Naive'] = fallback(close.iloc[-1], close.iloc[-1])

                    # SeasonalNaive
                    if 'SeasonalNaive' in models_sel:
                        try:
                            period = 5 if len(close) >= 5 else 1
                            idx = -period if len(close) >= period else -1
                            val = close.iloc[idx]
                        except:
                            val = close.iloc[-1]
                        
                        results['SeasonalNaive'] = fallback(val, close.iloc[-1])

                    # MovingAverage
                    if 'MovingAverage' in models_sel:
                        mv = close.rolling(20).mean().iloc[-1] if len(close) >= 20 else close.mean()
                        results['MovingAverage'] = fallback(mv, close.iloc[-1])


                    # LinearReg
                    if 'LinearReg' in models_sel:
                        try:
                            df_lr = close.reset_index(drop=True).reset_index()
                            df_lr.columns = ['t','y']
                            lr = LinearRegression().fit(df_lr[['t']], df_lr['y'])
                            t_future = len(df_lr) + days_ahead
                            pred = lr.predict([[t_future]])[0]
                        except:
                            pred = close.iloc[-1]
                        
                        results['LinearReg'] = fallback(pred, close.iloc[-1])


                    # helper for tree-like forecasts with lags + exog
                    def forecast_via_tree_return(model_factory):
                        try:
                            # build returns target
                            ret = to_returns(close)
                            df = pd.DataFrame({'r': ret})
                    
                            ex = hist[['EMA10','EMA50','Volume_fill','sentiment']].iloc[-len(ret):].reset_index(drop=True)
                            df = df.join(ex)
                    
                            for lag in range(1, lags+1):
                                df[f'lag_{lag}'] = df['r'].shift(lag)
                    
                            df = df.dropna().reset_index(drop=True)
                            if df.shape[0] < max(20, lags+2):
                                return float(close.iloc[-1])
                    
                            X = df.drop(columns=['r'])
                            y = df['r']
                    
                            model = model_factory()
                            model.fit(X, y)
                    
                            cur = X.iloc[[-1]].copy()
                            cols = X.columns
                            preds = []
                    
                            for step in range(days_ahead):
                                cur = cur[cols]
                                rhat = float(model.predict(cur)[0])
                                preds.append(rhat)
                    
                                for i in range(lags, 1, -1):
                                    cur[f'lag_{i}'] = cur[f'lag_{i-1}']
                                cur['lag_1'] = rhat
                    
                                if step < len(future_exog):
                                    fe = future_exog.iloc[step]
                                    for c in ['EMA10','EMA50','Volume_fill','sentiment']:
                                        cur[c] = float(fe[c])
                    
                            return price_from_returns(float(close.iloc[-1]), preds)
                    
                        except:
                            return float(close.iloc[-1])

                    if 'RandomForest' in models_sel:
                        val = forecast_via_tree_return(
                            lambda: RandomForestRegressor(
                                n_estimators=200,
                                max_depth=6,
                                min_samples_leaf=10,
                                random_state=0
                            )
                        )
                        results['RandomForest'] = try_float(val)                    
                    if 'GradientBoosting' in models_sel:
                        val = forecast_via_tree_return(lambda: GradientBoostingRegressor())
                        results['GradientBoosting'] = try_float(val)
                    if 'ExtraTrees' in models_sel:
                        val = forecast_via_tree(lambda: ExtraTreesRegressor(n_estimators=100, random_state=0))
                        results['ExtraTrees'] = try_float(val)
                    if 'ElasticNet' in models_sel:
                        # ElasticNet needs special instantiation
                        try:
                            df_rf = pd.DataFrame({'y': close})
                            ex = hist[['EMA10','EMA50','Volume_fill','sentiment']].reset_index(drop=True)
                            df_rf = df_rf.join(ex)
                            for lag in range(1, lags+1):
                                df_rf[f'lag_{lag}'] = df_rf['y'].shift(lag)
                            df_rf = df_rf.dropna().reset_index(drop=True)
                            if df_rf.shape[0] < max(10, lags+2):
                                results['ElasticNet'] = np.nan
                            else:
                                X = df_rf.drop(columns=['y'])
                                y = df_rf['y']
                                en = ElasticNet(random_state=0).fit(X, y)
                                cur = X.iloc[[-1]].copy()
                                preds = []
                                for step in range(days_ahead):
                                    cur = cur[X.columns]
                                    p = float(en.predict(cur)[0])
                                    preds.append(p)
                                    for i in range(lags, 1, -1):
                                        if f'lag_{i}' in cur.columns and f'lag_{i-1}' in cur.columns:
                                            cur[f'lag_{i}'] = cur[f'lag_{i-1}']
                                    if 'lag_1' in cur.columns:
                                        cur['lag_1'] = p
                                    if step < len(future_exog):
                                        fe = future_exog.iloc[step]
                                        for col in ['EMA10','EMA50','Volume_fill','sentiment']:
                                            if col in cur.columns:
                                                cur[col] = float(fe[col])
                                results['ElasticNet'] = try_float(preds[-1])
                        except Exception:
                            results['ElasticNet'] = np.nan
                    if 'KNN' in models_sel:
                        val = forecast_via_tree(lambda: KNeighborsRegressor(n_neighbors=5))
                        results['KNN'] = try_float(val)
                    if 'SVR' in models_sel:
                        val = forecast_via_tree(lambda: SVR())
                        results['SVR'] = try_float(val)

                    if 'ARIMA' in models_sel:
                        try:
                            m = ARIMA(close, order=(5,1,0)).fit()
                            pred = m.forecast(steps=days_ahead).iloc[-1]
                        except:
                            pred = close.iloc[-1]
                        
                        results['ARIMA'] = fallback(pred, close.iloc[-1])

                    if 'ARIMAX' in models_sel:
                        try:
                            exog = hist[['EMA10','EMA50','Volume_fill','sentiment']]
                            m = SARIMAX(close, exog=exog, order=(1,1,1)).fit(disp=False)
                            future_ex = future_exog[['EMA10','EMA50','Volume_fill','sentiment']]
                            pred = m.predict(start=len(close), end=len(close)+days_ahead-1, exog=future_ex).iloc[-1]
                        except:
                            pred = close.iloc[-1]
                        
                        results['ARIMAX'] = fallback(pred, close.iloc[-1])

                    if 'HoltWinters' in models_sel:
                        try:
                            hw = ExponentialSmoothing(close, trend="add", seasonal=None).fit()
                            hw_pred = hw.forecast(days_ahead)
                            results['HoltWinters'] = try_float(hw_pred.iloc[-1])
                        except Exception:
                            results['HoltWinters'] = np.nan

                    if 'LSTM' in models_sel:
                        if TENSORFLOW_AVAILABLE:
                            try:
                                seq = close.values.reshape(-1,1)
                                scaler = MinMaxScaler()
                                seq_s = scaler.fit_transform(seq)

                                # use returns
                                rets = np.diff(seq_s.squeeze(), prepend=seq_s[0]).reshape(-1,1)

                                window = min(30, len(seq_s)-1)
                                Xs, ys = [], []
                        
                                for i in range(len(rets)-window):
                                    Xs.append(seq_s[i:i+window])
                                    ys.append(rets[i+window][0])
                        
                                if len(Xs) < 5:
                                    pred = close.iloc[-1]
                                else:
                                    Xs = np.array(Xs).reshape(-1,window,1)
                                    ys = np.array(ys)
                        
                                    model = Sequential([Input(shape=(window,1)), LSTM(64), Dense(1)])
                                    model.compile(optimizer='adam', loss='mse')
                                    model.fit(Xs, ys, epochs=5, verbose=0)
                        
                                    cur = seq_s[-window:].reshape(1,window,1)
                                    preds = []
                                    for _ in range(days_ahead):
                                        p = model.predict(cur, verbose=0)[0,0]
                                        preds.append(p)
                                        cur = np.roll(cur, -1, axis=1)
                                        cur[0,-1,0] = cur[0,-2,0] + p
                        
                                    pred_vals = scaler.inverse_transform(cur.reshape(window,1)).flatten()
                                    pred = pred_vals[-1]
                            except:
                                pred = close.iloc[-1]
                        else:
                            pred = close.iloc[-1]
                        
                        results['LSTM'] = fallback(pred, close.iloc[-1])

                    # remove models that produced NaN before preparing ensemble
                    cleaned_results = {k: v for k,v in results.items() if pd.notna(v)}
                    if len(cleaned_results) == 0:
                        st.warning("All selected models produced missing predictions (NaN). Try fewer models, smaller lag, or more history.")
                    else:
                        # ensemble mean only across cleaned numeric values
                        if 'EnsembleMean' in models_sel:
                            numeric_vals = [v for v in cleaned_results.values() if isinstance(v, (int,float,np.floating)) and not np.isnan(v)]
                            cleaned_results['EnsembleMean'] = float(np.mean(numeric_vals)) if numeric_vals else np.nan
                            # remove if ensemble becomes NaN
                            if pd.isna(cleaned_results['EnsembleMean']):
                                cleaned_results.pop('EnsembleMean', None)

                        # build dataframe and show
                        res_df = pd.DataFrame.from_dict(cleaned_results, orient='index', columns=['pred'])
                        res_df['last_close'] = float(hist['Close'].iloc[-1])
                        res_df['delta'] = res_df['pred'] - res_df['last_close']
                        
                        # save for side-by-side display later
                        st.session_state['forecast_results'] = res_df.copy()

        # Backtest (1-step walk-forward)
        if backtest_run:
            st.info("Running backtest — this may take time depending on models & steps.")
            close = hist['Close'].copy()
            n = len(close)
            train_window = int(bt_window)
            steps_to_run = int(min(bt_steps, max(0, n - train_window - 1)))
            if steps_to_run <= 0:
                st.warning(f"Not enough data for requested backtest window/steps. n={n}, train_window={train_window}. Adjust window or steps.")
            else:
                metrics = {}
                progress = st.progress(0)
                model_list = [m for m in models_sel]
                for mi, model_name in enumerate(model_list):
                    y_trues = []
                    y_preds = []
                    for i in range(0, steps_to_run):
                        train_start = i
                        train_end = i + train_window
                        test_idx = train_end
                        if test_idx >= n:
                            break
                        train_series = close.iloc[train_start:train_end].copy()
                        exog_hist = hist[['EMA10','EMA50','Volume_fill','sentiment']].iloc[train_start:train_end].copy()
                        yhat = None
                        try:
                            if model_name == 'Naive':
                                yhat = float(train_series.iloc[-1])
                            elif model_name == 'SeasonalNaive':
                                period = 5 if len(train_series)>=5 else 1
                                idx = -period if len(train_series)>=period else -1
                                yhat = float(train_series.iloc[idx])
                            elif model_name == 'MovingAverage':
                                yhat = float(train_series.rolling(20).mean().iloc[-1]) if len(train_series) >= 20 else float(train_series.mean())
                            elif model_name == 'LinearReg':
                                df_lr = train_series.reset_index(drop=True).reset_index(); df_lr.columns=['t','y']
                                lr = LinearRegression().fit(df_lr[['t']].values, df_lr['y'].values)
                                yhat = float(lr.predict([[len(df_lr)]])[0])
                            elif model_name in ('RandomForest','GradientBoosting','ExtraTrees','ElasticNet','KNN','SVR'):
                                df_sup = create_supervised_features(train_series, lags=int(lags), exog=exog_hist)
                                if df_sup.shape[0] < max(5, int(lags)+1):
                                    yhat = float(train_series.iloc[-1])
                                else:
                                    X_train = df_sup.drop(columns=['y'])
                                    y_train = df_sup['y'].values
                                    if model_name == 'RandomForest':
                                        mdl = RandomForestRegressor(n_estimators=100, random_state=0)
                                    elif model_name == 'GradientBoosting':
                                        mdl = GradientBoostingRegressor(n_estimators=100, random_state=0)
                                    elif model_name == 'ExtraTrees':
                                        mdl = ExtraTreesRegressor(n_estimators=100, random_state=0)
                                    elif model_name == 'ElasticNet':
                                        mdl = ElasticNet(random_state=0)
                                    elif model_name == 'KNN':
                                        mdl = KNeighborsRegressor(n_neighbors=5)
                                    elif model_name == 'SVR':
                                        mdl = SVR()
                                    else:
                                        mdl = RandomForestRegressor(n_estimators=100, random_state=0)
                                    mdl.fit(X_train, y_train)
                                    last_lags = train_series.iloc[-int(lags):].values[::-1] if len(train_series)>=int(lags) else np.concatenate([np.full(int(lags)-len(train_series), train_series.iloc[0]), train_series.values])
                                    X_test_row = {}
                                    for col in X_train.columns:
                                        if col.startswith("lag_"):
                                            idx = int(col.split("_")[1]) - 1
                                            X_test_row[col] = float(last_lags[idx]) if idx < len(last_lags) else float(last_lags[-1])
                                        else:
                                            try:
                                                next_exog = hist[['EMA10','EMA50','Volume_fill','sentiment']].iloc[train_end]
                                                X_test_row[col] = float(next_exog[col])
                                            except Exception:
                                                X_test_row[col] = float(exog_hist.iloc[-1][col]) if col in exog_hist.columns else 0.0
                                    X_test_df = pd.DataFrame([X_test_row], columns=X_train.columns)
                                    yhat = float(mdl.predict(X_test_df)[0])
                            elif model_name == 'ARIMA':
                                try:
                                    m = ARIMA(train_series, order=(5,1,0)).fit()
                                    yhat = float(m.forecast(steps=1).iloc[0])
                                except Exception:
                                    yhat = float(train_series.iloc[-1])
                            elif model_name == 'ARIMAX':
                                try:
                                    ex = exog_hist
                                    m = SARIMAX(train_series, exog=ex, order=(1,1,1)).fit(disp=False)
                                    try:
                                        next_ex = hist[['EMA10','EMA50','Volume_fill','sentiment']].iloc[train_end].values.reshape(1,-1)
                                    except Exception:
                                        next_ex = ex.iloc[[-1]].values.reshape(1,-1)
                                    yhat = float(m.predict(start=len(train_series), end=len(train_series), exog=next_ex).iloc[0])
                                except Exception:
                                    yhat = float(train_series.iloc[-1])
                            elif model_name == 'HoltWinters':
                                try:
                                    m = ExponentialSmoothing(train_series, trend="add", seasonal=None).fit()
                                    yhat = float(m.forecast(1).iloc[0])
                                except Exception:
                                    yhat = float(train_series.iloc[-1])
                            elif model_name == 'LSTM':
                                if TENSORFLOW_AVAILABLE:
                                    try:
                                        seq = train_series.values.reshape(-1,1)
                                        scaler = MinMaxScaler()
                                        seq_s = scaler.fit_transform(seq)
                                        window = min(30, len(seq_s)-1)
                                        Xs, ys = [], []
                                        for k in range(len(seq_s)-window):
                                            Xs.append(seq_s[k:k+window])
                                            ys.append(seq_s[k+window][0])
                                        if len(Xs) < 5:
                                            yhat = float(train_series.iloc[-1])
                                        else:
                                            Xs = np.array(Xs).reshape(-1,window,1)
                                            ys = np.array(ys)
                                            m = Sequential([Input(shape=(window,1)), LSTM(32), Dense(1)])
                                            m.compile(optimizer='adam', loss='mse')
                                            m.fit(Xs, ys, epochs=3, verbose=0)
                                            last_win = seq_s[-window:].reshape(1,window,1)
                                            p = m.predict(last_win, verbose=0)[0,0]
                                            yhat = float(scaler.inverse_transform(np.array([[p]]))[0,0])
                                    except Exception:
                                        yhat = float(train_series.iloc[-1])
                                else:
                                    yhat = float(train_series.iloc[-1])
                            else:
                                yhat = float(train_series.iloc[-1])
                        except Exception:
                            yhat = float(train_series.iloc[-1])
                        y_trues.append(float(close.iloc[test_idx]))
                        y_preds.append(float(yhat))
                    if len(y_trues) > 0:
                        metrics[model_name] = evaluate_preds(y_trues, y_preds)
                    else:
                        metrics[model_name] = {"MAE": np.nan, "RMSE": np.nan, "MAPE": np.nan}
                    progress.progress(int((mi+1)/max(1,len(model_list))*100))
                    st.write(f"Backtested {model_name} — points={len(y_trues)}")
                # filter out models where metrics are NaN (no numeric MAE)
                metrics_clean = {k:v for k,v in metrics.items() if pd.notna(v.get("MAE", np.nan))}
                if len(metrics_clean) == 0:
                    st.warning("No backtest results (all models had insufficient data).")
                else:
                    metrics_df = pd.DataFrame(metrics_clean).T
                    st.subheader("Backtest Results (1-step walk-forward)")
                    st.dataframe(metrics_df)

                    # BEST MODELS (rank by MAE, tie-break RMSE)
                    ranked = metrics_df.sort_values(by=["MAE","RMSE"])
                    best = ranked.head(10)

                    # save for joint view
                    st.session_state['best_backtest'] = best.copy()

                    st.caption(
                        "Models ranked primarily by MAE (accuracy). "
                        "RMSE is used as a tie-breaker."
                    )

    # --- SIDE-BY-SIDE SUMMARY ---
    if 'forecast_results' in st.session_state or 'best_backtest' in st.session_state:
        st.markdown("## 📊 Forecast vs Backtest — quick comparison")

        col1, col2 = st.columns(2)

        with col1:
            st.subheader(" Forecast Results")
            if 'forecast_results' in st.session_state:
                fr = st.session_state['forecast_results'].copy()

                def arrow_html(x):
                    if pd.isna(x):
                        return "—"
                    if x > 0:
                        return f"<span class='arrow-up'>▲ {x:.2f}</span>"
                    if x < 0:
                        return f"<span class='arrow-down'>▼ {abs(x):.2f}</span>"
                    return f"{x:.2f}"

                html = """
                <style>
                    table {width:100%; border-collapse:collapse;}
                    th, td {padding:6px 8px; border-bottom:1px solid #333;}
                    .arrow-up {color:#2ecc71; font-weight:600;}
                    .arrow-down {color:#e74c3c; font-weight:600;}
                    th {text-align:left;}
                </style>
                """

                html += "<table><tr><th>Model</th><th style='text-align:right'>Prediction</th><th style='text-align:right'>Delta vs last</th></tr>"

                for idx, row in fr.iterrows():
                    pred = f"{row['pred']:.2f}" if not pd.isna(row['pred']) else "—"
                    html += (
                        f"<tr>"
                        f"<td>{idx}</td>"
                        f"<td style='text-align:right'>{pred}</td>"
                        f"<td style='text-align:right'>{arrow_html(row['delta'])}</td>"
                        f"</tr>"
                    )

                html += "</table>"
                st.markdown(html, unsafe_allow_html=True)

            else:
                st.info("Run a forecast first.")


        with col2:
            st.subheader(" Best Backtest Models")
            if 'best_backtest' in st.session_state:
                st.table(st.session_state['best_backtest'])
            else:
                st.info("Run the backtest to see best models.")

st.markdown("---")
st.caption("Notes: This app removes any models that return missing predictions (NaN) from final tables and ensembles. If many models are removed, reduce lag/backtest window or increase historical data. FinBERT/TF are optional and may slow startup.")