Upload 42 files

.gitattributes

@@ -35,3 +35,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 models/yahoo_history_cache.sqlite3 filter=lfs diff=lfs merge=lfs -text
 nifty_backend/__pycache__/runtime.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
+backend/models/yahoo_history_cache.sqlite3 filter=lfs diff=lfs merge=lfs -text
+backend/nifty_backend/__pycache__/runtime.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text

backend/.dockerignore

@@ -0,0 +1,5 @@
+__pycache__/
+*.pyc
+.venv/
+venv/
+.env

backend/Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.11-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+EXPOSE 7860
+
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860", "--ws", "none"]

backend/README.md

@@ -0,0 +1,39 @@
+---
+title: NIFTY 50 Forecaster Backend
+emoji: 📈
+colorFrom: green
+colorTo: blue
+sdk: docker
+app_port: 7860
+---
+
+# NIFTY 50 Forecaster Backend
+
+FastAPI Hugging Face Docker Space for the NIFTY 50 first-five-minute direction forecaster.
+
+## Endpoints
+
+- `GET /health`
+- `GET /dashboard`
+- `GET /prediction/latest`
+- `POST /prediction/refresh-first5`
+- `POST /data/refresh-daily`
+- `GET /cron/keepalive`
+- `POST /data/refresh-market-close`
+
+## Data
+
+Parquet files live in `data/`:
+
+- `nifty50_1m.parquet`
+- `nifty50_1d.parquet`
+- `opening_direction_training_dataset.parquet`
+- `test_predictions.parquet`
+
+## Runtime
+
+The API starts a daily background refresh loop. It wakes after `09:20 Asia/Kolkata`, fetches Yahoo Finance `^NSEI` 1-minute candles for the `09:15-09:19` opening window, appends them to Parquet, and writes the latest T+5 prediction.
+
+After market close it wakes again at `15:45 Asia/Kolkata`, refreshes the 1-minute and daily Parquet files, updates the opening training dataset with same-day close outcomes, and writes the saved prediction record used for the next trading session card. The `/cron/keepalive` endpoint also checks this close refresh so a Hugging Face Space that was idled still catches up when Netlify pings it.
+
+Netlify also pings `/cron/keepalive` every 10 minutes through its scheduled function.

backend/__init__.py

@@ -0,0 +1,2 @@
+"""NIFTY Project backend package."""
+

backend/app.py

@@ -0,0 +1,536 @@
+from __future__ import annotations
+
+import asyncio
+import threading
+from datetime import date, datetime, time, timedelta
+
+import sys
+from pathlib import Path
+
+from fastapi import BackgroundTasks, HTTPException, Query
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi import FastAPI
+from pydantic import BaseModel
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from nifty_backend.runtime import (
+    CLOSE_REFRESH_READY,
+    IST,
+    STALE_CHECK_INTERVAL_SECONDS,
+    TPLUS1_READY,
+    close_refresh_due,
+    dashboard_payload,
+    is_trading_day,
+    latest_saved_prediction,
+    latest_tplus1_prediction,
+    next_trading_day,
+    refresh_daily_data,
+    refresh_first5_prediction,
+    refresh_market_close_data,
+    refresh_stale_data_once,
+    refresh_tplus1_prediction,
+    seconds_until_next_ist_run,
+    warm_dashboard_payload_cache,
+)
+from kotak_neo import (
+    KotakNeoConfigError,
+    KotakNeoError,
+    KotakNeoSessionRequired,
+    kotak_neo_manager,
+)
+
+
+app = FastAPI(title="NIFTY 50 Forecaster Backend")
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=False,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+market_status = "Waiting for next session"
+close_refresh_lock = threading.Lock()
+tplus1_refresh_lock = threading.Lock()
+MARKET_OPEN = time(9, 15)
+FIRST5_READY = time(9, 20)
+MARKET_CLOSE = time(15, 30)
+
+
+class TotpRequest(BaseModel):
+    totp: str
+
+
+def refresh_market_close_data_if_due() -> dict:
+    if not close_refresh_due():
+        return {"status": "skipped", "reason": "close refresh is not due"}
+    if not close_refresh_lock.acquire(blocking=False):
+        return {"status": "skipped", "reason": "close refresh already running"}
+    try:
+        info = refresh_market_close_data()
+        return {"status": "refreshed", **info}
+    finally:
+        close_refresh_lock.release()
+
+
+def latest_tplus1_prediction_date(payload: dict | None = None) -> date | None:
+    try:
+        latest = payload if payload is not None else latest_tplus1_prediction()
+        raw = latest.get("input_date")
+        return date.fromisoformat(str(raw)[:10]) if raw else None
+    except Exception:
+        return None
+
+
+def tplus1_refresh_due(now: datetime | None = None, latest_date: date | None = None) -> bool:
+    now = now or datetime.now(IST)
+    if not is_trading_day(now.date()) or not (TPLUS1_READY <= now.time() < MARKET_CLOSE):
+        return False
+    latest_date = latest_date if latest_date is not None else latest_tplus1_prediction_date()
+    return latest_date != now.date()
+
+
+def refresh_tplus1_if_due() -> dict:
+    now = datetime.now(IST)
+    latest_date = latest_tplus1_prediction_date()
+    if not tplus1_refresh_due(now=now, latest_date=latest_date):
+        return {"status": "skipped", "reason": "tplus1 refresh is not due"}
+    if not tplus1_refresh_lock.acquire(blocking=False):
+        return {"status": "skipped", "reason": "tplus1 refresh already running"}
+    try:
+        prediction = refresh_tplus1_prediction(session_date=now.date())
+        return {"status": "refreshed", "prediction": prediction}
+    finally:
+        tplus1_refresh_lock.release()
+
+
+def latest_prediction_date(payload: dict | None = None) -> date | None:
+    try:
+        latest = payload if payload is not None else latest_saved_prediction()
+        raw = latest.get("input_date")
+        return date.fromisoformat(str(raw)) if raw else None
+    except Exception:
+        return None
+
+
+def current_market_state(now: datetime | None = None) -> dict:
+    global market_status
+    now = now or datetime.now(IST)
+    today = now.date()
+    current_time = now.time()
+    trading_day = is_trading_day(today)
+    latest_date = latest_prediction_date()
+    market_is_open_for_t5 = trading_day and FIRST5_READY <= current_time < MARKET_CLOSE
+    market_is_open_for_tplus1 = trading_day and TPLUS1_READY <= current_time < MARKET_CLOSE
+    has_current_first5 = market_is_open_for_t5 and latest_date == today
+    tplus1_latest_date = latest_tplus1_prediction_date()
+    has_current_tplus1 = market_is_open_for_tplus1 and tplus1_latest_date == today
+    next_session = today if trading_day and current_time < MARKET_CLOSE else next_trading_day(today + timedelta(days=1))
+
+    if not trading_day:
+        status = "Market Closed"
+        detail = f"Next trading session is {next_session.isoformat()}."
+    elif current_time < time(9, 0):
+        status = "Waiting for 9:00 AM"
+        detail = "Market has not entered pre-open yet."
+    elif current_time < MARKET_OPEN:
+        status = "Market Pre-Open"
+        detail = "Market opens at 9:15 AM IST."
+    elif current_time < FIRST5_READY:
+        status = "Market Officially Opened"
+        detail = "Waiting for the first 5 one-minute bars."
+    elif current_time <= MARKET_CLOSE:
+        if market_status in {"Fetching T+5 Prediction Data...", "Prediction Failed"}:
+            status = market_status
+            detail = "The first-five-minute prediction job is still resolving."
+        elif has_current_first5:
+            status = "Prediction Ready"
+            detail = "Today's first-five-minute prediction is available."
+        else:
+            status = "Prediction Pending"
+            detail = "No current-session prediction has been generated yet."
+    else:
+        status = "Market Closed"
+        detail = "Trading session has ended."
+
+    if not trading_day:
+        tplus1_status = "Market Closed"
+        tplus1_detail = f"Next trading session is {next_session.isoformat()}."
+    elif current_time < TPLUS1_READY:
+        tplus1_status = "Waiting for 2:30 PM"
+        tplus1_detail = "The T+1 forecast becomes available at 2:30 PM IST."
+    elif current_time < MARKET_CLOSE:
+        if has_current_tplus1:
+            tplus1_status = "Ready"
+            tplus1_detail = "Today's T+1 prediction is available."
+        else:
+            tplus1_status = "Pending"
+            tplus1_detail = "No current-session T+1 prediction has been generated yet."
+    else:
+        tplus1_status = "Market Closed"
+        tplus1_detail = "Trading session has ended."
+
+    if not trading_day:
+        t5_status = "Market Closed"
+        t5_detail = f"Next trading session is {next_session.isoformat()}."
+    elif current_time < FIRST5_READY:
+        t5_status = "Waiting for 9:20 AM"
+        t5_detail = "The T+5 forecast becomes available after the first five one-minute bars."
+    elif current_time < MARKET_CLOSE:
+        if market_status in {"Fetching T+5 Prediction Data...", "Prediction Failed"}:
+            t5_status = market_status
+            t5_detail = "The first-five-minute prediction job is still resolving."
+        elif has_current_first5:
+            t5_status = "Ready"
+            t5_detail = "Today's first-five-minute prediction is available."
+        else:
+            t5_status = "Pending"
+            t5_detail = "No current-session prediction has been generated yet."
+    else:
+        t5_status = "Market Closed"
+        t5_detail = "Trading session has ended."
+
+    return {
+        "market_status": status,
+        "market_detail": detail,
+        "server_time_ist": now.isoformat(),
+        "is_trading_day": trading_day,
+        "session_date": today.isoformat(),
+        "next_session_date": next_session.isoformat(),
+        "latest_prediction_date": latest_date.isoformat() if latest_date else None,
+        "t5_available": has_current_first5,
+        "t5_status": t5_status,
+        "t5_detail": t5_detail,
+        "market_is_open_for_t5": market_is_open_for_t5,
+        "tplus1_available": has_current_tplus1,
+        "tplus1_status": tplus1_status,
+        "tplus1_detail": tplus1_detail,
+        "market_is_open_for_tplus1": market_is_open_for_tplus1,
+        "latest_tplus1_prediction_date": tplus1_latest_date.isoformat() if tplus1_latest_date else None,
+    }
+
+
+def attach_market_state(payload: dict) -> dict:
+    state = current_market_state()
+    payload.setdefault("data_status", {})
+    payload["data_status"].update(state)
+    try:
+        payload["nifty_quote"] = kotak_neo_manager.fetch_nifty50_quote()
+        payload["nifty_quote_error"] = None
+    except KotakNeoSessionRequired as exc:
+        payload["nifty_quote"] = None
+        payload["nifty_quote_error"] = {"status": 401, "message": str(exc)}
+    except KotakNeoConfigError as exc:
+        payload["nifty_quote"] = None
+        payload["nifty_quote_error"] = {"status": 503, "message": str(exc)}
+    except KotakNeoError as exc:
+        payload["nifty_quote"] = None
+        payload["nifty_quote_error"] = {"status": 502, "message": str(exc)}
+
+    t5_latest = payload.get("latest") or {}
+    tomorrow_latest = payload.get("tomorrow_latest") or {}
+    tplus1_latest = payload.get("tplus1_latest") or {}
+    t5_available = bool(state["t5_available"] and t5_latest.get("prediction"))
+    tplus1_available = bool(state["tplus1_available"] and tplus1_latest.get("prediction"))
+    tomorrow_available = bool(tomorrow_latest.get("prediction"))
+    refresh_phase = payload.get("data_status", {}).get("refresh_phase")
+    if refresh_phase in {"waiting_second_payload", "refreshing"}:
+        tomorrow_status = "WAITING FOR SECOND PAYLOAD"
+        tomorrow_reason = "Market close refresh is generating the next-session payload."
+    else:
+        tomorrow_status = "Ready" if tomorrow_available else "Pending"
+        tomorrow_reason = None if tomorrow_available else "No saved next-session signal is available."
+    payload["predictions"] = {
+        "tomorrow": {
+            "available": tomorrow_available,
+            "status": tomorrow_status,
+            "reason": tomorrow_reason,
+            "target_date": tomorrow_latest.get("target_date") or state["next_session_date"],
+            "input_date": tomorrow_latest.get("input_date"),
+            "prediction": tomorrow_latest.get("prediction") if tomorrow_available else None,
+            "prob_up": tomorrow_latest.get("prob_up") if tomorrow_available else None,
+            "confidence": tomorrow_latest.get("confidence") if tomorrow_available else None,
+            "threshold": tomorrow_latest.get("threshold") if tomorrow_available else None,
+            "model_name": tomorrow_latest.get("model_name"),
+            "source_model": tomorrow_latest.get("source_model"),
+            "validation_accuracy": tomorrow_latest.get("validation_accuracy"),
+            "test_accuracy": tomorrow_latest.get("test_accuracy"),
+        },
+        "t5": {
+            "available": t5_available,
+            "status": "Ready" if t5_available else state["t5_status"],
+            "reason": None if t5_available else state["t5_detail"],
+            "input_date": t5_latest.get("input_date"),
+            "prediction": t5_latest.get("prediction") if t5_available else None,
+            "prob_up": t5_latest.get("prob_up") if t5_available else None,
+            "confidence": t5_latest.get("confidence") if t5_available else None,
+            "threshold": t5_latest.get("threshold") if t5_available else None,
+            "model_name": t5_latest.get("model_name"),
+            "validation_accuracy": (payload.get("summary") or {}).get("validation_accuracy"),
+            "test_accuracy": (payload.get("summary") or {}).get("test_accuracy"),
+        },
+        "tplus1": {
+            "available": tplus1_available,
+            "status": "Ready" if tplus1_available else state["tplus1_status"],
+            "reason": None if tplus1_available else state["tplus1_detail"],
+            "target_date": tplus1_latest.get("target_date") or state["next_session_date"],
+            "input_date": tplus1_latest.get("input_date"),
+            "prediction": tplus1_latest.get("prediction") if tplus1_available else None,
+            "prob_up": tplus1_latest.get("prob_up") if tplus1_available else None,
+            "confidence": tplus1_latest.get("confidence") if tplus1_available else None,
+            "threshold": tplus1_latest.get("threshold") if tplus1_available else None,
+            "model_name": tplus1_latest.get("model_name"),
+            "validation_accuracy": (payload.get("tplus1_summary") or {}).get("validation_accuracy"),
+            "test_accuracy": (payload.get("tplus1_summary") or {}).get("test_accuracy"),
+        },
+    }
+    return payload
+
+
+async def daily_ist_refresh_loop() -> None:
+    global market_status
+    while True:
+        # Wait until 9:00 AM IST
+        await asyncio.sleep(seconds_until_next_ist_run(time(9, 0)))
+        if not is_trading_day(datetime.now(IST).date()):
+            market_status = "Market Closed"
+            continue
+        market_status = "Market Pre-Open"
+        print("[scheduler] 9:00 AM IST - Market Pre-Open", flush=True)
+
+        # Wait until 9:15 AM IST
+        await asyncio.sleep(seconds_until_next_ist_run(time(9, 15)))
+        market_status = "Market Officially Opened"
+        print("[scheduler] 9:15 AM IST - Market Officially Opened", flush=True)
+
+        # Wait until 9:20 AM IST
+        await asyncio.sleep(seconds_until_next_ist_run(time(9, 20)))
+        market_status = "Fetching T+5 Prediction Data..."
+        print("[scheduler] 9:20 AM IST - Fetching Data", flush=True)
+
+        try:
+            await asyncio.to_thread(refresh_first5_prediction)
+            market_status = "Prediction Ready"
+        except Exception as exc:
+            print(f"[scheduler] first5 refresh failed: {exc}", flush=True)
+            market_status = "Prediction Failed"
+            
+        try:
+            await asyncio.to_thread(refresh_daily_data)
+        except Exception as exc:
+            print(f"[scheduler] daily refresh failed: {exc}", flush=True)
+
+        await asyncio.sleep(seconds_until_next_ist_run(TPLUS1_READY))
+        print("[scheduler] 2:30 PM IST - Refreshing T+1 prediction", flush=True)
+        try:
+            info = await asyncio.to_thread(refresh_tplus1_if_due)
+            print(f"[scheduler] tplus1 refresh result: {info}", flush=True)
+        except Exception as exc:
+            print(f"[scheduler] tplus1 refresh failed: {exc}", flush=True)
+
+        await asyncio.sleep(seconds_until_next_ist_run(CLOSE_REFRESH_READY))
+        print("[scheduler] 3:45 PM IST - Refreshing close data", flush=True)
+        try:
+            info = await asyncio.to_thread(refresh_market_close_data_if_due)
+            print(f"[scheduler] close refresh result: {info}", flush=True)
+        except Exception as exc:
+            print(f"[scheduler] close refresh failed: {exc}", flush=True)
+
+
+async def refresh_current_session_once() -> None:
+    global market_status
+    now = datetime.now(IST)
+    if not is_trading_day(now.date()) or now.time() < FIRST5_READY:
+        return
+    if latest_prediction_date() == now.date():
+        return
+    market_status = "Fetching T+5 Prediction Data..."
+    print("[startup] Current session needs first-five refresh; fetching now.", flush=True)
+    try:
+        await asyncio.to_thread(refresh_first5_prediction)
+        market_status = "Prediction Ready"
+    except Exception as exc:
+        print(f"[startup] first5 refresh failed: {exc}", flush=True)
+        market_status = "Prediction Failed"
+    try:
+        await asyncio.to_thread(refresh_daily_data)
+    except Exception as exc:
+        print(f"[startup] daily refresh failed: {exc}", flush=True)
+
+
+async def refresh_market_close_once_if_due() -> None:
+    try:
+        info = await asyncio.to_thread(refresh_market_close_data_if_due)
+        if info.get("status") == "refreshed":
+            print(f"[startup] close refresh result: {info}", flush=True)
+    except Exception as exc:
+        print(f"[startup] close refresh failed: {exc}", flush=True)
+
+
+async def refresh_tplus1_once_if_due() -> None:
+    try:
+        info = await asyncio.to_thread(refresh_tplus1_if_due)
+        if info.get("status") == "refreshed":
+            print(f"[startup] tplus1 refresh result: {info}", flush=True)
+    except Exception as exc:
+        print(f"[startup] tplus1 refresh failed: {exc}", flush=True)
+
+
+async def warm_dashboard_payload_cache_once() -> None:
+    try:
+        await asyncio.to_thread(warm_dashboard_payload_cache)
+    except Exception as exc:
+        print(f"[startup] dashboard payload warmup failed: {exc}", flush=True)
+
+
+async def stale_data_watch_loop() -> None:
+    while True:
+        try:
+            info = await asyncio.to_thread(refresh_stale_data_once)
+            if info.get("status") == "refreshed":
+                print(f"[stale-watch] refreshed stale data: {info}", flush=True)
+        except Exception as exc:
+            print(f"[stale-watch] stale refresh failed: {exc}", flush=True)
+        await asyncio.sleep(STALE_CHECK_INTERVAL_SECONDS)
+
+
+@app.on_event("startup")
+async def start_scheduler() -> None:
+    global market_status
+    # Initialize correct status on startup based on current time
+    now = datetime.now(IST).time()
+    today = datetime.now(IST).date()
+    if not is_trading_day(today):
+        market_status = "Market Closed"
+    elif now < time(9, 0):
+        market_status = "Waiting for 9:00 AM"
+    elif now < time(9, 15):
+        market_status = "Market Pre-Open"
+    elif now < time(9, 20):
+        market_status = "Market Officially Opened"
+    elif latest_prediction_date() == today:
+        market_status = "Prediction Ready"
+    else:
+        market_status = "Prediction Pending"
+        
+    asyncio.create_task(refresh_current_session_once())
+    asyncio.create_task(refresh_tplus1_once_if_due())
+    asyncio.create_task(refresh_market_close_once_if_due())
+    asyncio.create_task(warm_dashboard_payload_cache_once())
+    asyncio.create_task(stale_data_watch_loop())
+    asyncio.create_task(daily_ist_refresh_loop())
+
+
+@app.get("/health")
+def health() -> dict[str, str]:
+    return {"status": "ok"}
+
+
+@app.get("/")
+def root() -> dict[str, str]:
+    return {"service": "NIFTY 50 Forecaster Backend", "status": "ok"}
+
+
+@app.get("/dashboard")
+def dashboard() -> dict:
+    return attach_market_state(dashboard_payload())
+
+
+@app.get("/kotak/status")
+def kotak_status() -> dict:
+    return kotak_neo_manager.status()
+
+
+@app.post("/kotak/auth/totp")
+def kotak_auth_totp(payload: TotpRequest) -> dict:
+    try:
+        return kotak_neo_manager.authenticate_with_totp(payload.totp)
+    except KotakNeoConfigError as exc:
+        raise HTTPException(status_code=503, detail=str(exc)) from exc
+    except KotakNeoError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+
+@app.get("/kotak/account")
+def kotak_account() -> dict:
+    try:
+        return kotak_neo_manager.fetch_account_snapshot()
+    except KotakNeoConfigError as exc:
+        raise HTTPException(status_code=503, detail=str(exc)) from exc
+    except KotakNeoSessionRequired as exc:
+        raise HTTPException(status_code=401, detail=str(exc)) from exc
+    except KotakNeoError as exc:
+        raise HTTPException(status_code=502, detail=str(exc)) from exc
+
+
+@app.get("/kotak/quote/nifty50")
+def kotak_nifty50_quote() -> dict:
+    try:
+        return kotak_neo_manager.fetch_nifty50_quote()
+    except KotakNeoConfigError as exc:
+        raise HTTPException(status_code=503, detail=str(exc)) from exc
+    except KotakNeoSessionRequired as exc:
+        raise HTTPException(status_code=401, detail=str(exc)) from exc
+    except KotakNeoError as exc:
+        raise HTTPException(status_code=502, detail=str(exc)) from exc
+
+
+@app.get("/kotak/activity-log")
+def kotak_activity_log() -> dict:
+    try:
+        snapshot = kotak_neo_manager.fetch_account_snapshot()
+        return {
+            "activity_log": snapshot.get("activity_log", {}),
+            "trade_history": snapshot.get("trade_history", []),
+            "order_book": snapshot.get("order_book", []),
+        }
+    except KotakNeoConfigError as exc:
+        raise HTTPException(status_code=503, detail=str(exc)) from exc
+    except KotakNeoSessionRequired as exc:
+        raise HTTPException(status_code=401, detail=str(exc)) from exc
+    except KotakNeoError as exc:
+        raise HTTPException(status_code=502, detail=str(exc)) from exc
+
+
+@app.get("/cron/keepalive")
+def cron_keepalive(background_tasks: BackgroundTasks) -> dict:
+    close_refresh = {"status": "not_checked"}
+    tplus1_refresh = {"status": "not_checked"}
+    if tplus1_refresh_due():
+        background_tasks.add_task(refresh_tplus1_if_due)
+        tplus1_refresh = {"status": "scheduled"}
+    if close_refresh_due():
+        background_tasks.add_task(refresh_market_close_data_if_due)
+        close_refresh = {"status": "scheduled"}
+    return {
+        "status": "awake",
+        "market": current_market_state(),
+        "tplus1_refresh": tplus1_refresh,
+        "close_refresh": close_refresh,
+    }
+
+
+@app.get("/prediction/latest")
+def prediction_latest() -> dict:
+    return latest_saved_prediction()
+
+
+@app.post("/prediction/refresh-first5")
+def prediction_refresh_first5(
+    session_date: date | None = Query(default=None, description="Optional YYYY-MM-DD session date in IST."),
+) -> dict:
+    prediction = refresh_first5_prediction(session_date=session_date)
+    return prediction.to_dict()
+
+
+@app.post("/data/refresh-daily")
+def data_refresh_daily() -> dict:
+    return refresh_daily_data()
+
+
+@app.post("/data/refresh-market-close")
+def data_refresh_market_close(
+    session_date: date | None = Query(default=None, description="Optional YYYY-MM-DD session date in IST."),
+) -> dict:
+    return refresh_market_close_data(session_date=session_date)

backend/data/nifty50_1d.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d0830f9d3cfa91f02ce717b556983beb12f897bc537a623c67e38f22ce05caf
+size 78366

backend/data/nifty50_1m.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5d6022df273daa1f020676214ec35536426fd84f2d7efb669797287e27ffa2d
+size 18589635

backend/data/opening_direction_training_dataset.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:722c8225451abcb49463d2a57354bc0c9b5eb519f31a25fa3c5242adc4dbbada
+size 4463631

backend/data/test_predictions.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f159a7499394b7882262ffaa6f9b48c0f6ab763d024f373ee19109b301af90c1
+size 14499

backend/data/tomorrow_test_predictions.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2b6ecb377d114c825f861d9a6741e01bbf23bfc623493fd01c78e8bb1501960
+size 10751

backend/data/tplus1_test_predictions.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd7f63239d5705969cbe5423169c9fdcb88d59b838b02913d0aa5c455a7ca41c
+size 13681

backend/kotak_neo.py

@@ -0,0 +1,1257 @@
+from __future__ import annotations
+
+import os
+import threading
+from csv import DictReader
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import date, datetime, time, timezone
+from functools import lru_cache
+from pathlib import Path
+from time import monotonic
+from typing import Any
+from urllib.parse import quote
+from zoneinfo import ZoneInfo
+
+import json
+
+import pandas as pd
+import requests
+
+try:
+    import pandas_market_calendars as mcal
+except Exception:  # pragma: no cover - deployed environments may fall back to weekdays
+    mcal = None
+
+
+SESSION_BASE_URL = "https://mis.kotaksecurities.com"
+QUOTE_PATH_TEMPLATE = "script-details/1.0/quotes/neosymbol/{neo_symbols}/{quote_type}"
+TOTP_LOGIN_PATH = "login/1.0/tradeApiLogin"
+TOTP_VALIDATE_PATH = "login/1.0/tradeApiValidate"
+DEFAULT_TIMEOUT_SECONDS = 20
+ACCOUNT_TIMEOUT_SECONDS = 7
+NIFTY_QUOTE_TIMEOUT_SECONDS = 2.5
+NIFTY_QUOTE_CACHE_SECONDS = 1.0
+DATA_DIR = Path(__file__).resolve().parent / "data"
+KOTAK_ACTIVITY_LOG_PATH = DATA_DIR / "kotak_activity_log.txt"
+NIFTY_1M_PATH = DATA_DIR / "nifty50_1m.parquet"
+NIFTY_1D_PATH = DATA_DIR / "nifty50_1d.parquet"
+IST = ZoneInfo("Asia/Kolkata")
+MARKET_OPEN_TIME = time(9, 15)
+MARKET_CLOSE_TIME = time(15, 30)
+
+
+class KotakNeoError(Exception):
+    pass
+
+
+class KotakNeoConfigError(KotakNeoError):
+    pass
+
+
+class KotakNeoSessionRequired(KotakNeoError):
+    pass
+
+
+def _utc_now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _to_float(value: Any) -> float | None:
+    if value in (None, "", "--", "NA", "na", "-", "null"):
+        return None
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _first_number(*values: Any) -> float | None:
+    for value in values:
+        parsed = _to_float(value)
+        if parsed is not None:
+            return parsed
+    return None
+
+
+def _first_market_number(*values: Any) -> float | None:
+    fallback = None
+    for value in values:
+        parsed = _to_float(value)
+        if parsed is None:
+            continue
+        if fallback is None:
+            fallback = parsed
+        if parsed > 0:
+            return parsed
+    return fallback
+
+
+def _normalize_frame_dates(frame: pd.DataFrame) -> pd.Series:
+    values = pd.to_datetime(frame["date"], errors="coerce")
+    if getattr(values.dt, "tz", None) is None:
+        values = values.dt.tz_localize(IST)
+    else:
+        values = values.dt.tz_convert(IST)
+    return values
+
+
+@lru_cache(maxsize=1)
+def _nse_calendar():
+    if mcal is None:
+        return None
+    for name in ("XNSE", "NSE", "BSE"):
+        try:
+            return mcal.get_calendar(name)
+        except Exception:
+            continue
+    return None
+
+
+@lru_cache(maxsize=64)
+def _is_nse_trading_day(day: date) -> bool:
+    calendar = _nse_calendar()
+    if calendar is None:
+        return day.weekday() < 5
+    return not calendar.schedule(start_date=day, end_date=day).empty
+
+
+@lru_cache(maxsize=64)
+def _previous_nse_trading_day(day: date) -> date:
+    candidate = day
+    for _ in range(21):
+        candidate = date.fromordinal(candidate.toordinal() - 1)
+        if _is_nse_trading_day(candidate):
+            return candidate
+    return candidate
+
+
+def _file_version(path: Path) -> tuple[str, int | None, int | None]:
+    try:
+        stat = path.stat()
+        return (str(path), stat.st_mtime_ns, stat.st_size)
+    except OSError:
+        return (str(path), None, None)
+
+
+@lru_cache(maxsize=4)
+def _load_nifty_daily_frame(file_version: tuple[str, int | None, int | None]) -> pd.DataFrame:
+    path = Path(file_version[0])
+    daily = pd.read_parquet(path, columns=["date", "open", "high", "low", "close"]).copy()
+    daily["date"] = pd.to_datetime(daily["date"], errors="coerce").dt.date
+    return daily.dropna(subset=["date"]).sort_values("date")
+
+
+@lru_cache(maxsize=4)
+def _load_nifty_minute_frame(file_version: tuple[str, int | None, int | None]) -> pd.DataFrame:
+    path = Path(file_version[0])
+    minute = pd.read_parquet(path, columns=["date", "open", "high", "low", "close"]).copy()
+    minute["date"] = _normalize_frame_dates(minute)
+    return minute.dropna(subset=["date"]).sort_values("date")
+
+
+def _first_text(*values: Any) -> str | None:
+    for value in values:
+        if isinstance(value, dict):
+            nested = _first_text(
+                value.get("message"),
+                value.get("error"),
+                value.get("Error"),
+                value.get("emsg"),
+                value.get("detail"),
+            )
+            if nested:
+                return nested
+        if isinstance(value, list):
+            for item in value:
+                nested = _first_text(item)
+                if nested:
+                    return nested
+        if value not in (None, "", "--", "NA", "na", "-"):
+            return str(value)
+    return None
+
+
+def _sum_numbers(*values: Any) -> float | None:
+    numbers: list[float] = []
+    for value in values:
+        parsed = _to_float(value)
+        if parsed is not None:
+            numbers.append(parsed)
+    return sum(numbers) if numbers else None
+
+
+def _extract_items(payload: Any) -> list[dict[str, Any]]:
+    if isinstance(payload, list):
+        return [item for item in payload if isinstance(item, dict)]
+    if not isinstance(payload, dict):
+        return []
+
+    data = payload.get("data")
+    if isinstance(data, list):
+        return [item for item in data if isinstance(item, dict)]
+    if isinstance(data, dict):
+        nested = data.get("data")
+        if isinstance(nested, list):
+            return [item for item in nested if isinstance(item, dict)]
+        return [data]
+    return []
+
+
+def _sort_key(item: dict[str, Any]) -> str:
+    return str(
+        _first_text(
+            item.get("updRecvTm"),
+            item.get("hsUpTm"),
+            item.get("flDtTm"),
+            item.get("exTm"),
+            item.get("ordDtTm"),
+            item.get("TimeStamp"),
+            item.get("flDt"),
+        )
+        or ""
+    )
+
+
+class KotakNeoManager:
+    def __init__(self) -> None:
+        self.consumer_key = os.getenv("KOTAK_CONSUMER_KEY")
+        self.mobile_number = os.getenv("KOTAK_MOBILE_NUMBER")
+        self.ucc = os.getenv("KOTAK_UCC")
+        self.mpin = os.getenv("KOTAK_MPIN")
+        self.neo_fin_key = os.getenv("KOTAK_NEO_FIN_KEY", "neotradeapi")
+
+        self._lock = threading.RLock()
+        self.activity_log_path = KOTAK_ACTIVITY_LOG_PATH
+        self.activity_log_path.parent.mkdir(parents=True, exist_ok=True)
+        self._seen_activity_keys: set[str] = set()
+        self._scrip_cache: dict[str, list[dict[str, str]]] = {}
+        self._quote_cache: dict[str, dict[str, Any]] = {}
+        self._load_existing_activity_keys()
+        self._clear_session_locked()
+
+    def _load_existing_activity_keys(self) -> None:
+        if not self.activity_log_path.exists():
+            return
+        try:
+            for line in self.activity_log_path.read_text(encoding="utf-8").splitlines():
+                if not line.strip():
+                    continue
+                try:
+                    payload = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+                key = str(payload.get("activity_key") or "").strip()
+                if key:
+                    self._seen_activity_keys.add(key)
+        except Exception:
+            pass
+
+    def _clear_session_locked(self) -> None:
+        self.view_token: str | None = None
+        self.sid: str | None = None
+        self.edit_token: str | None = None
+        self.edit_sid: str | None = None
+        self.edit_rid: str | None = None
+        self.server_id: str | None = None
+        self.data_center: str | None = None
+        self.base_url: str | None = None
+        self.authenticated_at: str | None = None
+
+    def _configured(self) -> bool:
+        return all([self.consumer_key, self.mobile_number, self.ucc, self.mpin])
+
+    def status(self) -> dict[str, Any]:
+        configured = self._configured()
+        with self._lock:
+            authenticated = bool(self.edit_token and self.edit_sid and self.base_url)
+            return {
+                "available": configured,
+                "configured": configured,
+                "authenticated": authenticated,
+                "needs_totp": configured and not authenticated,
+                "last_authenticated_at": self.authenticated_at,
+                "reason": None if configured else "Kotak Neo environment variables are incomplete.",
+            }
+
+    def authenticate_with_totp(self, totp: str) -> dict[str, Any]:
+        if not self._configured():
+            raise KotakNeoConfigError("Kotak Neo environment variables are incomplete.")
+        if not str(totp).strip():
+            raise KotakNeoError("A TOTP code is required.")
+
+        with self._lock:
+            login_response = self._post_session_api(
+                TOTP_LOGIN_PATH,
+                headers={
+                    "Authorization": self.consumer_key,
+                    "neo-fin-key": self.neo_fin_key,
+                    "Content-Type": "application/json",
+                    "Accept": "application/json",
+                },
+                payload={
+                    "mobileNumber": self.mobile_number,
+                    "ucc": self.ucc,
+                    "totp": str(totp).strip(),
+                },
+            )
+            login_data = (login_response.get("data") or {}) if isinstance(login_response, dict) else {}
+            self.view_token = login_data.get("token")
+            self.sid = login_data.get("sid")
+
+            if not self.view_token or not self.sid:
+                self._clear_session_locked()
+                raise KotakNeoError("Kotak Neo did not return a valid pre-auth session.")
+
+            validate_response = self._post_session_api(
+                TOTP_VALIDATE_PATH,
+                headers={
+                    "Authorization": self.consumer_key,
+                    "sid": self.sid,
+                    "Auth": self.view_token,
+                    "neo-fin-key": self.neo_fin_key,
+                    "Content-Type": "application/json",
+                    "Accept": "application/json",
+                },
+                payload={"mpin": self.mpin},
+            )
+            validate_data = (validate_response.get("data") or {}) if isinstance(validate_response, dict) else {}
+
+            self.edit_token = validate_data.get("token")
+            self.edit_sid = validate_data.get("sid")
+            self.edit_rid = validate_data.get("rid")
+            self.server_id = validate_data.get("hsServerId")
+            self.data_center = validate_data.get("dataCenter")
+            self.base_url = str(validate_data.get("baseUrl") or "").rstrip("/")
+            self.authenticated_at = _utc_now_iso()
+
+            if not self.edit_token or not self.edit_sid or not self.base_url:
+                self._clear_session_locked()
+                raise KotakNeoError("Kotak Neo did not return a usable trading session.")
+
+            return self.status()
+
+    def fetch_account_snapshot(self) -> dict[str, Any]:
+        if not self._configured():
+            raise KotakNeoConfigError("Kotak Neo environment variables are incomplete.")
+
+        with self._lock:
+            context = self._context_locked()
+
+        account_calls = {
+            "holdings": lambda: self._request_trading_api_with_context(
+                context,
+                "portfolio/v1/holdings",
+                timeout=ACCOUNT_TIMEOUT_SECONDS,
+            ),
+            "positions": lambda: self._request_trading_api_with_context(
+                context,
+                "quick/user/positions",
+                timeout=ACCOUNT_TIMEOUT_SECONDS,
+            ),
+            "trades": lambda: self._request_trading_api_with_context(
+                context,
+                "quick/user/trades",
+                timeout=ACCOUNT_TIMEOUT_SECONDS,
+            ),
+            "orders": lambda: self._request_trading_api_with_context(
+                context,
+                "quick/user/orders",
+                timeout=ACCOUNT_TIMEOUT_SECONDS,
+            ),
+            "limits": lambda: self._post_trading_api_with_context(
+                context,
+                "quick/user/limits",
+                payload={"seg": "ALL", "exch": "ALL", "prod": "ALL"},
+                content_type="application/x-www-form-urlencoded",
+                timeout=ACCOUNT_TIMEOUT_SECONDS,
+            ),
+        }
+        defaults = {
+            "holdings": {"data": []},
+            "positions": {"data": []},
+            "trades": {"data": []},
+            "orders": {"data": []},
+            "limits": {},
+        }
+        results: dict[str, dict[str, Any]] = dict(defaults)
+
+        with ThreadPoolExecutor(max_workers=5) as executor:
+            future_map = {executor.submit(fn): label for label, fn in account_calls.items()}
+            for future in as_completed(future_map):
+                label = future_map[future]
+                results[label] = self._resolve_account_future(label, future, default=defaults[label])
+
+        holdings = _extract_items(results["holdings"])
+        positions = _extract_items(results["positions"])
+        trades = sorted(_extract_items(results["trades"]), key=_sort_key, reverse=True)
+        orders = sorted(_extract_items(results["orders"]), key=_sort_key, reverse=True)
+
+        normalized_trades = [self._normalize_trade(item) for item in trades]
+        normalized_orders = [self._normalize_order(item) for item in orders]
+
+        quotes = self._safe_account_call(
+            "quotes",
+            lambda: self._fetch_quotes_with_context(
+                context,
+                self._instrument_tokens_for_quotes(
+                    context,
+                    holdings,
+                    positions,
+                    normalized_trades,
+                ),
+                timeout=ACCOUNT_TIMEOUT_SECONDS,
+            )
+            ,
+            default={"data": []},
+        )
+        quote_map = self._build_quote_map(quotes)
+
+        normalized_holdings = [self._normalize_holding(item, quote_map) for item in holdings]
+        normalized_positions = [self._normalize_position(item, quote_map) for item in positions]
+        self._append_activity_entries(normalized_trades, normalized_orders)
+        journal = self._read_activity_journal()
+        merged_trades = self._merge_activity(normalized_trades, journal["trades"])
+        merged_orders = self._merge_activity(normalized_orders, journal["orders"])
+
+        holdings_market_value = sum(item["market_value"] or 0.0 for item in normalized_holdings)
+        holdings_cost = sum(item["cost_value"] or 0.0 for item in normalized_holdings)
+        holdings_pnl = sum(item["pnl"] or 0.0 for item in normalized_holdings)
+        positions_pnl = sum(item["pnl"] or 0.0 for item in normalized_positions)
+
+        limits_raw = results["limits"]
+        limits_summary = {
+            "net": _first_number(limits_raw.get("Net")) if isinstance(limits_raw, dict) else None,
+            "margin_used": _first_number(limits_raw.get("MarginUsed")) if isinstance(limits_raw, dict) else None,
+            "collateral_value": _first_number(limits_raw.get("CollateralValue")) if isinstance(limits_raw, dict) else None,
+            "cash_unrealized_mtm": _first_number(limits_raw.get("CashUnRlsMtomPrsnt")) if isinstance(limits_raw, dict) else None,
+            "cash_realized_mtm": _first_number(limits_raw.get("CashRlsMtomPrsnt")) if isinstance(limits_raw, dict) else None,
+        }
+
+        available_cash = None
+        if limits_summary["net"] is not None and limits_summary["margin_used"] is not None:
+            available_cash = limits_summary["net"] - limits_summary["margin_used"]
+
+        current_capital = None
+        if available_cash is not None:
+            current_capital = available_cash + holdings_market_value
+
+        return {
+            "status": self.status(),
+            "as_of": _utc_now_iso(),
+            "neo_behavior": {
+                "holdings_note": "Kotak Neo shows CNC delivery buys in Positions on trade day and in Holdings/T1 from the next trading day.",
+                "trade_history_note": "Kotak Neo trade history availability is limited by Neo's own order and portfolio tracker behavior.",
+            },
+            "summary": {
+                "available_cash": available_cash,
+                "current_capital": current_capital,
+                "holdings_market_value": holdings_market_value,
+                "holdings_cost_value": holdings_cost,
+                "holdings_pnl": holdings_pnl,
+                "positions_pnl": positions_pnl,
+                "live_pnl": holdings_pnl + positions_pnl,
+                "open_positions": sum(1 for item in normalized_positions if item["net_quantity"]),
+                "holdings_count": len(normalized_holdings),
+                "orders_count": len(merged_orders),
+                "trades_count": len(merged_trades),
+            },
+            "limits_summary": limits_summary,
+            "limits_raw": limits_raw,
+            "holdings": normalized_holdings,
+            "positions": normalized_positions,
+            "trade_history": merged_trades[:100],
+            "order_book": merged_orders[:100],
+            "activity_log": {
+                "path": str(self.activity_log_path),
+                "trades_count": len(journal["trades"]),
+                "orders_count": len(journal["orders"]),
+            },
+            "quotes": list(quote_map.values()),
+        }
+
+    def fetch_nifty50_quote(self, *, force_refresh: bool = False) -> dict[str, Any]:
+        cache_key = "nifty50_quote"
+        with self._lock:
+            if not force_refresh:
+                cached = self._quote_cache.get(cache_key)
+                if cached and (monotonic() - float(cached.get("stored_at_monotonic") or 0.0)) < NIFTY_QUOTE_CACHE_SECONDS:
+                    return dict(cached["payload"])
+            context = self._context_locked()
+
+        reference = self._resolve_nifty50_reference(context)
+        quote_payload = self._fetch_quotes_with_context(
+            context,
+            [
+                {
+                    "exchange_segment": "nse_cm",
+                    "instrument_token": str(reference["quote_instrument_token"]),
+                }
+            ],
+            timeout=NIFTY_QUOTE_TIMEOUT_SECONDS,
+        )
+        items = _extract_items(quote_payload)
+        if not items:
+            raise KotakNeoError("Kotak Neo did not return a NIFTY 50 quote.")
+
+        item = items[0]
+        now_ist = datetime.now(IST)
+        last_traded_price = _first_market_number(item.get("last_traded_price"), item.get("ltp"), item.get("iv"))
+        quote_open = _first_market_number(item.get("openingPrice"), item.get("open"), item.get("o"))
+        quote_high = _first_market_number(item.get("high"), item.get("highPrice"), item.get("h"))
+        quote_low = _first_market_number(item.get("low"), item.get("lowPrice"), item.get("l"))
+        quote_previous_close = _first_market_number(
+            item.get("previous_close"),
+            item.get("previousClose"),
+            item.get("prev_close"),
+            item.get("prevClose"),
+            item.get("previousClosePrice"),
+            item.get("prevClosePrice"),
+            item.get("close"),
+            item.get("c"),
+            item.get("ic"),
+        )
+        live_stats = self._load_nifty50_reference_stats(
+            now_ist,
+            last_traded_price,
+            quote_open,
+            quote_high,
+            quote_low,
+            quote_previous_close,
+        )
+        close = live_stats["previous_close"]
+        change_base = live_stats["return_base"]
+        change = None
+        change_pct = None
+        if last_traded_price is not None and change_base not in (None, 0):
+            change = last_traded_price - change_base
+            change_pct = (change / change_base) * 100.0
+        high = live_stats["range_high"]
+        low = live_stats["range_low"]
+        open_price = quote_open
+
+        payload = {
+            "symbol": "NIFTY 50",
+            "exchange_segment": "nse_cm",
+            "instrument_token": _first_text(
+                item.get("instrument_token"),
+                item.get("instrumentToken"),
+                item.get("tk"),
+                reference.get("master_instrument_token"),
+                reference["quote_instrument_token"],
+            ),
+            "display_name": _first_text(item.get("trading_symbol"), item.get("ts"), item.get("name"), "NIFTY 50"),
+            "last_traded_price": last_traded_price,
+            "close": close,
+            "change": change,
+            "change_pct": change_pct,
+            "open": open_price,
+            "high": high,
+            "low": low,
+            "return_basis": live_stats["return_basis"],
+            "market_open": live_stats["market_open"],
+            "is_trading_session": live_stats["is_trading_session"],
+            "quote_session_date": live_stats["quote_session_date"],
+            "previous_session_date": live_stats["previous_session_date"],
+            "exchange_feed_time": _first_text(item.get("tvalue"), item.get("updRecvTm"), item.get("hsUpTm")),
+            "as_of": _utc_now_iso(),
+            "source": {
+                "quote_api": QUOTE_PATH_TEMPLATE,
+                "master_scrip_verified": bool(reference.get("master_record_found")),
+                "instrument_lookup": reference.get("lookup_mode"),
+                "master_symbol_name": reference.get("master_symbol_name"),
+                "master_trading_symbol": reference.get("master_trading_symbol"),
+                "reference_data": "backend/data/nifty50_1m.parquet + backend/data/nifty50_1d.parquet",
+            },
+        }
+
+        with self._lock:
+            self._quote_cache[cache_key] = {
+                "stored_at_monotonic": monotonic(),
+                "payload": payload,
+            }
+        return payload
+
+    def _load_nifty50_reference_stats(
+        self,
+        now_ist: datetime,
+        last_traded_price: float | None,
+        live_open: float | None = None,
+        live_high: float | None = None,
+        live_low: float | None = None,
+        live_previous_close: float | None = None,
+    ) -> dict[str, Any]:
+        today = now_ist.date()
+        is_trading_session = _is_nse_trading_day(today)
+        market_open = is_trading_session and MARKET_OPEN_TIME <= now_ist.time() < MARKET_CLOSE_TIME
+        session_started = is_trading_session and now_ist.time() >= MARKET_OPEN_TIME
+        quote_session_date = today if session_started else _previous_nse_trading_day(today)
+        previous_session_date = _previous_nse_trading_day(quote_session_date)
+
+        daily = _load_nifty_daily_frame(_file_version(NIFTY_1D_PATH))
+
+        previous_close = None
+        session_daily = daily[daily["date"] == quote_session_date]
+        previous_daily = daily[daily["date"] == previous_session_date]
+        if previous_daily.empty:
+            previous_daily = daily[daily["date"] < quote_session_date]
+        if not previous_daily.empty:
+            previous_row = previous_daily.iloc[-1]
+            previous_close = _to_float(previous_row["close"])
+
+        session_open = _to_float(session_daily.iloc[-1]["open"]) if not session_daily.empty else None
+        session_high = _to_float(session_daily.iloc[-1]["high"]) if not session_daily.empty else None
+        session_low = _to_float(session_daily.iloc[-1]["low"]) if not session_daily.empty else None
+        session_close = _to_float(session_daily.iloc[-1]["close"]) if not session_daily.empty else None
+
+        if live_previous_close is not None:
+            is_same_as_static_ltp = (
+                last_traded_price is not None
+                and abs(live_previous_close - last_traded_price) < 0.01
+            )
+            if not is_same_as_static_ltp:
+                previous_close = live_previous_close
+
+        if session_started:
+            session_open = live_open or session_open
+            session_high = live_high or session_high
+            session_low = live_low or session_low
+
+            if session_open is None or session_high is None or session_low is None:
+                minute = _load_nifty_minute_frame(_file_version(NIFTY_1M_PATH))
+                today_minute = minute[minute["date"].dt.date == today]
+                if not today_minute.empty:
+                    session_open = _to_float(today_minute.iloc[0]["open"]) or session_open
+                    minute_high = pd.to_numeric(today_minute["high"], errors="coerce").max()
+                    minute_low = pd.to_numeric(today_minute["low"], errors="coerce").min()
+                    session_high = _to_float(minute_high) or session_high
+                    session_low = _to_float(minute_low) or session_low
+                    session_close = _to_float(today_minute.iloc[-1]["close"]) or session_close
+        else:
+            session_high = live_high or session_high
+            session_low = live_low or session_low
+
+        if session_started:
+            range_high = max([value for value in [session_high, last_traded_price] if value is not None], default=None)
+            range_low = min([value for value in [session_low, last_traded_price] if value is not None], default=None)
+            return_base = previous_close
+            return_basis = "previous_close"
+        else:
+            range_high = session_high
+            range_low = session_low
+            return_base = previous_close
+            return_basis = "previous_close"
+            if last_traded_price is None:
+                last_traded_price = session_close
+
+        return {
+            "previous_close": previous_close,
+            "return_base": return_base,
+            "return_basis": return_basis,
+            "range_high": range_high,
+            "range_low": range_low,
+            "market_open": market_open,
+            "is_trading_session": is_trading_session,
+            "quote_session_date": quote_session_date.isoformat(),
+            "previous_session_date": previous_session_date.isoformat(),
+        }
+
+    def _ensure_authenticated_locked(self) -> None:
+        if not self.edit_token or not self.edit_sid or not self.base_url:
+            raise KotakNeoSessionRequired("Kotak Neo session is not authenticated.")
+
+    def _context_locked(self) -> dict[str, str]:
+        self._ensure_authenticated_locked()
+        return {
+            "base_url": self.base_url or "",
+            "edit_sid": self.edit_sid or "",
+            "edit_token": self.edit_token or "",
+            "server_id": self.server_id or "",
+            "consumer_key": self.consumer_key or "",
+        }
+
+    def _post_session_api(self, path: str, headers: dict[str, str], payload: dict[str, Any]) -> dict[str, Any]:
+        response = requests.post(
+            f"{SESSION_BASE_URL.rstrip('/')}/{path.lstrip('/')}",
+            headers=headers,
+            json=payload,
+            timeout=DEFAULT_TIMEOUT_SECONDS,
+        )
+        data = self._decode_response(response)
+        self._raise_for_error(response, data, session_sensitive=False)
+        return data
+
+    def _request_trading_api_locked(self, path: str) -> dict[str, Any]:
+        return self._request_trading_api_with_context(
+            self._context_locked(),
+            path,
+            timeout=DEFAULT_TIMEOUT_SECONDS,
+        )
+
+    def _request_trading_api_with_context(
+        self,
+        context: dict[str, str],
+        path: str,
+        *,
+        timeout: int,
+    ) -> dict[str, Any]:
+        response = requests.get(
+            f"{context['base_url'].rstrip('/')}/{path.lstrip('/')}",
+            headers={
+                "Sid": context["edit_sid"],
+                "Auth": context["edit_token"],
+                "Accept": "application/json",
+            },
+            params={"sId": context["server_id"]},
+            timeout=timeout,
+        )
+        data = self._decode_response(response)
+        self._raise_for_error(response, data, session_sensitive=True)
+        return data
+
+    def _post_trading_api_locked(
+        self,
+        path: str,
+        payload: dict[str, Any],
+        *,
+        content_type: str = "application/json",
+    ) -> dict[str, Any]:
+        return self._post_trading_api_with_context(
+            self._context_locked(),
+            path,
+            payload,
+            content_type=content_type,
+            timeout=DEFAULT_TIMEOUT_SECONDS,
+        )
+
+    def _post_trading_api_with_context(
+        self,
+        context: dict[str, str],
+        path: str,
+        payload: dict[str, Any],
+        *,
+        content_type: str,
+        timeout: int,
+    ) -> dict[str, Any]:
+        headers = {
+            "Sid": context["edit_sid"],
+            "Auth": context["edit_token"],
+            "Accept": "application/json",
+            "Content-Type": content_type,
+        }
+        query_params = {"sId": context["server_id"]}
+        url = f"{context['base_url'].rstrip('/')}/{path.lstrip('/')}"
+
+        if content_type == "application/x-www-form-urlencoded":
+            body = {"jData": json.dumps(payload)}
+            response = requests.post(
+                url,
+                headers=headers,
+                params=query_params,
+                data=body,
+                timeout=timeout,
+            )
+        else:
+            response = requests.post(
+                url,
+                headers=headers,
+                params=query_params,
+                json=payload,
+                timeout=timeout,
+            )
+
+        data = self._decode_response(response)
+        self._raise_for_error(response, data, session_sensitive=True)
+        return data
+
+    def _fetch_quotes_locked(self, instrument_tokens: list[dict[str, str]]) -> dict[str, Any]:
+        return self._fetch_quotes_with_context(
+            self._context_locked(),
+            instrument_tokens,
+            timeout=DEFAULT_TIMEOUT_SECONDS,
+        )
+
+    def _fetch_quotes_with_context(
+        self,
+        context: dict[str, str],
+        instrument_tokens: list[dict[str, str]],
+        *,
+        timeout: int,
+    ) -> dict[str, Any]:
+        if not instrument_tokens:
+            return {"data": []}
+
+        neo_symbols = ",".join(
+            f"{item['exchange_segment']}|{item['instrument_token']}" for item in instrument_tokens
+        )
+        encoded_symbols = quote(neo_symbols, safe="")
+        response = requests.get(
+            f"{context['base_url'].rstrip('/')}/{QUOTE_PATH_TEMPLATE.format(neo_symbols=encoded_symbols, quote_type='all')}",
+            headers={
+                "Authorization": context["consumer_key"],
+                "Content-Type": "application/x-www-form-urlencoded",
+                "Accept": "application/json",
+            },
+            timeout=timeout,
+        )
+        data = self._decode_response(response)
+        self._raise_for_error(response, data, session_sensitive=False)
+        return data
+
+    def _decode_response(self, response: requests.Response) -> dict[str, Any]:
+        try:
+            parsed = response.json()
+        except ValueError as exc:
+            raise KotakNeoError(
+                f"Kotak Neo returned a non-JSON response with HTTP {response.status_code}."
+            ) from exc
+        if isinstance(parsed, dict):
+            return parsed
+        return {"data": parsed}
+
+    def _safe_account_call(
+        self,
+        label: str,
+        fn,
+        *,
+        default: dict[str, Any],
+    ) -> dict[str, Any]:
+        try:
+            return fn()
+        except KotakNeoSessionRequired:
+            raise
+        except Exception as exc:
+            if not self._is_expected_empty_error(label, exc):
+                print(f"[kotak] {label} call failed: {exc}", flush=True)
+            return default
+
+    def _resolve_account_future(self, label: str, future, *, default: dict[str, Any]) -> dict[str, Any]:
+        try:
+            return future.result()
+        except KotakNeoSessionRequired:
+            raise
+        except Exception as exc:
+            if not self._is_expected_empty_error(label, exc):
+                print(f"[kotak] {label} call failed: {exc}", flush=True)
+            return default
+
+    def _is_expected_empty_error(self, label: str, exc: Exception) -> bool:
+        text = str(exc).lower()
+        empty_markers = [
+            "no holdings found",
+            "no position",
+            "no positions",
+            "no trade",
+            "no trades",
+            "no order",
+            "no orders",
+            "no data found",
+            "no trade found",
+            "no order found",
+        ]
+        if any(marker in text for marker in empty_markers):
+            return True
+        if label in {"positions", "trades", "orders"} and text.strip() == "kotak neo rejected the request.":
+            return True
+        if label in {"holdings", "positions", "trades", "orders"} and "424" in text:
+            return True
+        return False
+
+    def _raise_for_error(
+        self,
+        response: requests.Response,
+        data: dict[str, Any],
+        *,
+        session_sensitive: bool,
+    ) -> None:
+        stat = str(data.get("stat") or "").strip().lower()
+        st_code = _first_number(data.get("stCode"))
+        message = _first_text(
+            data.get("message"),
+            data.get("error"),
+            data.get("Error"),
+            data.get("emsg"),
+        )
+
+        session_expired = response.status_code in (401, 403) or "invalid session" in (message or "").lower()
+        if session_expired:
+            if session_sensitive:
+                self._clear_session_locked()
+                raise KotakNeoSessionRequired(message or "Kotak Neo session expired.")
+            raise KotakNeoError(message or "Kotak Neo rejected the request.")
+
+        if stat == "not_ok":
+            raise KotakNeoError(message or "Kotak Neo rejected the request.")
+
+        if response.status_code >= 400 or (st_code is not None and st_code >= 400):
+            raise KotakNeoError(message or f"Kotak Neo request failed with HTTP {response.status_code}.")
+
+    def _instrument_tokens_for_quotes(
+        self,
+        context: dict[str, str],
+        holdings: list[dict[str, Any]],
+        positions: list[dict[str, Any]],
+        trades: list[dict[str, Any]] | None = None,
+    ) -> list[dict[str, str]]:
+        unique: dict[tuple[str, str], dict[str, str]] = {}
+
+        for item in holdings:
+            token = _first_text(item.get("instrumentToken"), item.get("exchangeIdentifier"), item.get("tok"))
+            exchange = _first_text(item.get("exchangeSegment"), item.get("exSeg"))
+            if token and exchange:
+                unique[(exchange, token)] = {
+                    "exchange_segment": exchange,
+                    "instrument_token": token,
+                }
+
+        for item in positions:
+            token = _first_text(item.get("tok"), item.get("instrumentToken"), item.get("exchangeIdentifier"))
+            exchange = _first_text(item.get("exSeg"), item.get("exchangeSegment"))
+            if token and exchange:
+                unique[(exchange, token)] = {
+                    "exchange_segment": exchange,
+                    "instrument_token": token,
+                }
+            elif exchange:
+                resolved = self._resolve_symbol_token(
+                    context,
+                    exchange_segment=exchange,
+                    symbol=_first_text(item.get("trdSym"), item.get("sym")),
+                )
+                if resolved:
+                    unique[(exchange, resolved)] = {
+                        "exchange_segment": exchange,
+                        "instrument_token": resolved,
+                    }
+
+        for item in trades or []:
+            exchange = _first_text(item.get("exchange_segment"), item.get("exSeg"))
+            if not exchange:
+                continue
+            resolved = self._resolve_symbol_token(
+                context,
+                exchange_segment=exchange,
+                symbol=_first_text(item.get("trading_symbol"), item.get("trdSym"), item.get("symbol"), item.get("sym")),
+            )
+            if resolved:
+                unique[(exchange, resolved)] = {
+                    "exchange_segment": exchange,
+                    "instrument_token": resolved,
+                }
+
+        return list(unique.values())
+
+    def _resolve_symbol_token(
+        self,
+        context: dict[str, str],
+        *,
+        exchange_segment: str,
+        symbol: str | None,
+    ) -> str | None:
+        symbol = str(symbol or "").strip()
+        if not symbol:
+            return None
+        candidates = self._load_scrip_candidates(context, exchange_segment)
+        symbol_upper = symbol.upper()
+        base_symbol_upper = symbol_upper.split("-")[0]
+        for item in candidates:
+            trading_symbol = str(item.get("pTrdSymbol") or "").upper()
+            symbol_name = str(item.get("pSymbolName") or "").upper()
+            token = str(item.get("pSymbol") or "").strip()
+            if not token:
+                continue
+            if trading_symbol == symbol_upper or symbol_name == base_symbol_upper:
+                return token
+        return None
+
+    def _load_scrip_candidates(self, context: dict[str, str], exchange_segment: str) -> list[dict[str, str]]:
+        key = str(exchange_segment).lower()
+        if key in self._scrip_cache:
+            return self._scrip_cache[key]
+
+        response = requests.get(
+            f"{context['base_url'].rstrip('/')}/script-details/1.0/masterscrip/file-paths",
+            headers={
+                "Authorization": context["consumer_key"],
+                "Accept": "application/json",
+            },
+            timeout=ACCOUNT_TIMEOUT_SECONDS,
+        )
+        data = self._decode_response(response)
+        file_paths = ((data.get("data") or {}).get("filesPaths") or []) if isinstance(data, dict) else []
+        csv_url = next((path for path in file_paths if key in str(path).lower()), None)
+        if not csv_url:
+            self._scrip_cache[key] = []
+            return []
+
+        csv_response = requests.get(csv_url, timeout=ACCOUNT_TIMEOUT_SECONDS)
+        csv_response.raise_for_status()
+        rows = list(DictReader(csv_response.text.splitlines()))
+        self._scrip_cache[key] = rows
+        return rows
+
+    def _resolve_nifty50_reference(self, context: dict[str, str]) -> dict[str, Any]:
+        candidates = self._load_scrip_candidates(context, "nse_cm")
+        match = None
+        aliases = {
+            "NIFTY 50",
+            "NIFTY50",
+            "NIFTY 50 INDEX",
+            "NIFTY",
+        }
+        for item in candidates:
+            candidate_values = {
+                str(item.get("pSymbolName") or "").strip().upper(),
+                str(item.get("pTrdSymbol") or "").strip().upper(),
+                str(item.get("pSymbol") or "").strip().upper(),
+            }
+            if aliases & candidate_values:
+                match = item
+                break
+
+        return {
+            "quote_instrument_token": "Nifty 50",
+            "lookup_mode": "index-name-direct" if match is None else "master-scrip-verified-index-name-direct",
+            "master_record_found": match is not None,
+            "master_instrument_token": _first_text(match.get("pSymbol")) if match else None,
+            "master_symbol_name": _first_text(match.get("pSymbolName")) if match else None,
+            "master_trading_symbol": _first_text(match.get("pTrdSymbol")) if match else None,
+        }
+
+    def _build_quote_map(self, payload: dict[str, Any]) -> dict[str, dict[str, Any]]:
+        items = _extract_items(payload)
+        quote_map: dict[str, dict[str, Any]] = {}
+        for item in items:
+            token = _first_text(item.get("instrument_token"), item.get("instrumentToken"), item.get("tk"))
+            exchange = _first_text(item.get("exchange_segment"), item.get("exchangeSegment"), item.get("e"))
+            if not token or not exchange:
+                continue
+            key = f"{exchange}|{token}"
+            quote_map[key] = {
+                "instrument_token": token,
+                "exchange_segment": exchange,
+                "trading_symbol": _first_text(item.get("trading_symbol"), item.get("ts"), item.get("symbol")),
+                "last_traded_price": _first_number(item.get("last_traded_price"), item.get("ltp"), item.get("iv")),
+                "close": _first_number(item.get("close"), item.get("c"), item.get("ic")),
+                "change": _first_number(item.get("change"), item.get("cng")),
+                "change_pct": _first_number(item.get("net_change_percentage"), item.get("nc")),
+            }
+        return quote_map
+
+    def _normalize_holding(
+        self,
+        item: dict[str, Any],
+        quote_map: dict[str, dict[str, Any]],
+    ) -> dict[str, Any]:
+        token = _first_text(item.get("instrumentToken"), item.get("exchangeIdentifier"), item.get("tok"))
+        exchange = _first_text(item.get("exchangeSegment"), item.get("exSeg"))
+        quote = quote_map.get(f"{exchange}|{token}", {}) if token and exchange else {}
+
+        quantity = _first_number(item.get("quantity"), item.get("sellableQuantity"))
+        average_price = _first_number(item.get("averagePrice"), item.get("avgPrc"))
+        holding_cost = _first_number(item.get("holdingCost"))
+        market_value = _first_number(item.get("mktValue"))
+        ltp = _first_number(quote.get("last_traded_price"))
+
+        if market_value is None and quantity is not None and ltp is not None:
+            market_value = quantity * ltp
+        if holding_cost is None and quantity is not None and average_price is not None:
+            holding_cost = quantity * average_price
+
+        pnl = None
+        pnl_pct = None
+        if market_value is not None and holding_cost is not None:
+            pnl = market_value - holding_cost
+            if holding_cost:
+                pnl_pct = pnl / holding_cost
+
+        return {
+            "symbol": _first_text(item.get("displaySymbol"), item.get("symbol"), item.get("trdSym")),
+            "trading_symbol": _first_text(item.get("symbol"), item.get("displaySymbol"), item.get("trdSym")),
+            "exchange_segment": exchange,
+            "instrument_token": token,
+            "quantity": quantity,
+            "sellable_quantity": _first_number(item.get("sellableQuantity")),
+            "average_price": average_price,
+            "last_traded_price": ltp,
+            "market_value": market_value,
+            "cost_value": holding_cost,
+            "pnl": pnl,
+            "pnl_pct": pnl_pct,
+        }
+
+    def _normalize_position(
+        self,
+        item: dict[str, Any],
+        quote_map: dict[str, dict[str, Any]],
+    ) -> dict[str, Any]:
+        token = _first_text(item.get("tok"), item.get("instrumentToken"), item.get("exchangeIdentifier"))
+        exchange = _first_text(item.get("exSeg"), item.get("exchangeSegment"))
+        quote = quote_map.get(f"{exchange}|{token}", {}) if token and exchange else {}
+
+        multiplier = _first_number(item.get("multiplier")) or 1.0
+        buy_qty = _first_number(
+            item.get("buyQty"),
+            _sum_numbers(item.get("cfBuyQty"), item.get("flBuyQty")),
+        )
+        sell_qty = _first_number(
+            item.get("sellQty"),
+            _sum_numbers(item.get("cfSellQty"), item.get("flSellQty")),
+        )
+        qty = _first_number(item.get("netQty"), item.get("qty"))
+
+        if qty is None and buy_qty is not None and sell_qty is not None:
+            qty = buy_qty - sell_qty
+        elif qty is None and buy_qty is not None and sell_qty is None:
+            qty = buy_qty
+        elif qty is None and sell_qty is not None:
+            qty = -sell_qty
+
+        average_price = _first_number(item.get("avgPrc"), item.get("averagePrice"))
+        ltp = _first_number(quote.get("last_traded_price"))
+
+        pnl = _first_number(
+            item.get("pnl"),
+            item.get("mtm"),
+            item.get("urmtom"),
+            item.get("unRealizedMtom"),
+        )
+        if pnl is None and qty is not None and average_price is not None and ltp is not None:
+            pnl = (ltp - average_price) * qty * multiplier
+
+        return {
+            "order_no": _first_text(item.get("nOrdNo")),
+            "symbol": _first_text(item.get("sym"), item.get("trdSym")),
+            "trading_symbol": _first_text(item.get("trdSym"), item.get("sym")),
+            "exchange_segment": exchange,
+            "instrument_token": token,
+            "product": _first_text(item.get("prod")),
+            "transaction_type": _first_text(item.get("trnsTp")),
+            "net_quantity": qty,
+            "average_price": average_price,
+            "last_traded_price": ltp,
+            "pnl": pnl,
+            "multiplier": multiplier,
+            "updated_at": _first_text(item.get("hsUpTm"), item.get("exTm"), item.get("flDtTm")),
+        }
+
+
+    def _normalize_trade(self, item: dict[str, Any]) -> dict[str, Any]:
+        return {
+            "activity_type": "trade",
+            "activity_key": self._trade_key(item),
+            "order_no": _first_text(item.get("nOrdNo")),
+            "trade_id": _first_text(item.get("flId")),
+            "exchange_order_id": _first_text(item.get("exOrdId")),
+            "symbol": _first_text(item.get("sym"), item.get("trdSym")),
+            "trading_symbol": _first_text(item.get("trdSym"), item.get("sym")),
+            "exchange_segment": _first_text(item.get("exSeg")),
+            "transaction_type": _first_text(item.get("trnsTp")),
+            "product": _first_text(item.get("prod")),
+            "quantity": _first_number(item.get("fldQty"), item.get("qty")),
+            "price": _first_number(item.get("avgPrc"), item.get("prc")),
+            "average_price": _first_number(item.get("avgPrc")),
+            "status": _first_text(item.get("rptTp"), item.get("ordSt"), item.get("stat")),
+            "trade_time": _first_text(item.get("flDtTm"), item.get("exTm"), item.get("flTm"), item.get("flDt")),
+            "raw": item,
+        }
+
+    def _normalize_order(self, item: dict[str, Any]) -> dict[str, Any]:
+        return {
+            "activity_type": "order",
+            "activity_key": self._order_key(item),
+            "order_no": _first_text(item.get("nOrdNo")),
+            "request_id": _first_text(item.get("reqId"), item.get("nReqId")),
+            "exchange_order_id": _first_text(item.get("exOrdId")),
+            "symbol": _first_text(item.get("sym"), item.get("trdSym")),
+            "trading_symbol": _first_text(item.get("trdSym"), item.get("sym")),
+            "exchange_segment": _first_text(item.get("exSeg")),
+            "transaction_type": _first_text(item.get("trnsTp")),
+            "product": _first_text(item.get("prod")),
+            "quantity": _first_number(item.get("qty")),
+            "filled_quantity": _first_number(item.get("fldQty")),
+            "unfilled_size": _first_number(item.get("unFldSz")),
+            "price": _first_number(item.get("prc")),
+            "trigger_price": _first_number(item.get("trgPrc")),
+            "order_type": _first_text(item.get("prcTp")),
+            "status": _first_text(item.get("ordSt"), item.get("stat")),
+            "order_time": _first_text(item.get("ordDtTm"), item.get("exCfmTm"), item.get("hsUpTm")),
+            "rejection_reason": _first_text(item.get("rejRsn")),
+            "raw": item,
+        }
+
+    def _trade_key(self, item: dict[str, Any]) -> str:
+        return "|".join(
+            [
+                "trade",
+                str(_first_text(item.get("nOrdNo")) or ""),
+                str(_first_text(item.get("flId")) or ""),
+                str(_first_text(item.get("flDtTm"), item.get("exTm"), item.get("flTm")) or ""),
+            ]
+        )
+
+    def _order_key(self, item: dict[str, Any]) -> str:
+        return "|".join(
+            [
+                "order",
+                str(_first_text(item.get("nOrdNo")) or ""),
+                str(_first_text(item.get("ordSt"), item.get("stat")) or ""),
+                str(_first_text(item.get("ordDtTm"), item.get("exCfmTm"), item.get("hsUpTm")) or ""),
+            ]
+        )
+
+    def _append_activity_entries(self, trades: list[dict[str, Any]], orders: list[dict[str, Any]]) -> None:
+        lines: list[str] = []
+        for entry in [*trades, *orders]:
+            key = str(entry.get("activity_key") or "").strip()
+            if not key or key in self._seen_activity_keys:
+                continue
+            payload = {
+                "activity_key": key,
+                "activity_type": entry.get("activity_type"),
+                "captured_at": _utc_now_iso(),
+                **entry,
+            }
+            lines.append(json.dumps(payload, ensure_ascii=True))
+            self._seen_activity_keys.add(key)
+        if not lines:
+            return
+        with self.activity_log_path.open("a", encoding="utf-8") as handle:
+            handle.write("\n".join(lines) + "\n")
+
+    def _read_activity_journal(self) -> dict[str, list[dict[str, Any]]]:
+        trades: list[dict[str, Any]] = []
+        orders: list[dict[str, Any]] = []
+        if not self.activity_log_path.exists():
+            return {"trades": trades, "orders": orders}
+        for line in self.activity_log_path.read_text(encoding="utf-8").splitlines():
+            if not line.strip():
+                continue
+            try:
+                item = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if item.get("activity_type") == "trade":
+                trades.append(item)
+            elif item.get("activity_type") == "order":
+                orders.append(item)
+        trades.sort(key=lambda item: str(item.get("trade_time") or item.get("captured_at") or ""), reverse=True)
+        orders.sort(key=lambda item: str(item.get("order_time") or item.get("captured_at") or ""), reverse=True)
+        return {"trades": trades, "orders": orders}
+
+    def _merge_activity(
+        self,
+        live_entries: list[dict[str, Any]],
+        journal_entries: list[dict[str, Any]],
+    ) -> list[dict[str, Any]]:
+        merged: list[dict[str, Any]] = []
+        seen: set[str] = set()
+        for entry in [*live_entries, *journal_entries]:
+            key = str(entry.get("activity_key") or "").strip()
+            if not key or key in seen:
+                continue
+            merged.append(entry)
+            seen.add(key)
+        return merged
+
+
+kotak_neo_manager = KotakNeoManager()

backend/models/candidate_results.csv

@@ -0,0 +1,14 @@
+model_name,threshold,validation_accuracy,test_accuracy,validation_auc,test_auc
+blend_extra_trees_tight_logit_overlay,0.425,0.654320987654321,0.6524064171122995,0.6623500611995106,0.6563861499656042
+blend_extra_trees_tight_logit,0.425,0.6444444444444445,0.6310160427807486,0.6623500611995106,0.6563861499656042
+extra_trees_opening,0.514,0.6345679012345679,0.6203208556149733,0.6448470012239902,0.6575326759917449
+extra_trees_opening_tight,0.514,0.6296296296296297,0.6149732620320856,0.6446511627906977,0.6608576014675532
+soft_vote_tree_pack,0.511,0.6296296296296297,0.6042780748663101,0.6448959608323135,0.6621187800963081
+random_forest_opening,0.516,0.6296296296296297,0.5935828877005348,0.6448959608323134,0.6563861499656042
+extra_trees_opening_deep,0.514,0.6271604938271605,0.6042780748663101,0.6432558139534884,0.6671634946113276
+soft_vote_opening,0.556,0.6246913580246913,0.6256684491978609,0.638359853121175,0.6513414354505846
+gradient_boost_opening,0.512,0.6246913580246913,0.5828877005347594,0.640734394124847,0.631506535198349
+soft_vote_all_pack,0.503,0.6172839506172839,0.5989304812834224,0.6431089351285189,0.6598257280440265
+catboost_opening,0.524,0.6098765432098765,0.5882352941176471,0.6180660954712363,0.6319651456088053
+hist_gradient_opening,0.517,0.5925925925925926,0.5133689839572193,0.6042839657282741,0.5806007796376977
+logit_opening,0.386,0.582716049382716,0.5454545454545454,0.6403427172582619,0.6141939922036231

backend/models/latest_prediction.csv

@@ -0,0 +1,2 @@
+input_date,first5_start,first5_end,prediction,prob_up,confidence,threshold,model_name,is_overridden
+2026-05-27,2026-05-27 09:15:00,2026-05-27 09:19:00,UP,0.6970470349448922,0.7720470349448922,0.425,blend_extra_trees_tight_logit_overlay,False

backend/models/nifty_1420_tplus1_logistic_model.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77fea2ef4be30e6355377c1badadc94e9cb941fb62a31e731fad83b0f171c63e
+size 5321

backend/models/nifty_opening_direction_model.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:faa463488804279181dc6ee63ca62871d0d5817b0cbb23d34e7374d7628ced98
+size 16234249

backend/models/nifty_tomorrow_direction_model.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce657f6c5de48990fca4621b06796d57a0d71126f159b2adfaa476aaae66ccf6
+size 446

backend/models/refresh_state.json

@@ -0,0 +1,7 @@
+{
+  "phase": "normal",
+  "started_at": "2026-05-25T10:28:13Z",
+  "finished_at": "2026-05-25T10:33:38Z",
+  "session_date": "2026-05-25",
+  "error": null
+}

backend/models/summary.json

@@ -0,0 +1,29 @@
+{
+  "target": "same-day NIFTY 50 close > same-day NIFTY 50 open after first five 1-minute bars",
+  "model_name": "blend_extra_trees_tight_logit_overlay",
+  "threshold": 0.425,
+  "train_rows": 2221,
+  "valid_rows": 405,
+  "test_rows": 187,
+  "train_start": "2015-01-09",
+  "train_end": "2023-12-29",
+  "valid_start": "2024-01-01",
+  "valid_end": "2025-08-14",
+  "test_start": "2025-08-18",
+  "test_end": "2026-05-21",
+  "validation_accuracy": 0.654320987654321,
+  "test_accuracy": 0.6524064171122995,
+  "baseline_test_accuracy": 0.5240641711229946,
+  "validation_auc": 0.6623500611995106,
+  "test_auc": 0.6563861499656042,
+  "validation_log_loss": 0.6563017134616456,
+  "test_log_loss": 0.6607799782446233,
+  "test_brier": 0.2339533732973711,
+  "feature_count": 219,
+  "latest_input_date": "2026-05-21",
+  "latest_first5_start": "2026-05-21 09:15:00",
+  "latest_first5_end": "2026-05-21 09:19:00",
+  "latest_prob_up": 0.4379885393062093,
+  "latest_prediction": "UP",
+  "latest_confidence": 0.5129885393062092
+}

backend/models/tomorrow_latest_prediction.csv

@@ -0,0 +1,2 @@
+input_date,target_date,prediction,prob_up,confidence,threshold,model_name,source_model,validation_accuracy,test_accuracy,artifact_source
+2026-05-27,2026-05-29,UP,0.536599063991701,0.536599063991701,0.534,nifty_tomorrow_direction_model,locked_multiwindow_nifty50_ensemble_v2,0.5780141843971631,0.6736842105263158,C:\Users\jhaji\Downloads\forecasting project\Code\models\nifty_forecaster\outputs

backend/models/tomorrow_summary.json

@@ -0,0 +1,42 @@
+{
+  "symbol": "NIFTY 50",
+  "horizon": "daily",
+  "horizon_bars": 1,
+  "config": {
+    "name": "locked_multiwindow_nifty50_ensemble_v2",
+    "use_intraday": true,
+    "use_external": true,
+    "use_institutional": false,
+    "use_options": true,
+    "use_engineered_macro_flow": false,
+    "blend_mode": "locked_nifty50_multiwindow_v2",
+    "decision_overlay": "bank_body_near_threshold;low_bank_vol_down"
+  },
+  "threshold": 0.534,
+  "validation_accuracy": 0.5780141843971631,
+  "test_accuracy": 0.6736842105263158,
+  "baseline_accuracy": 0.5052631578947369,
+  "n_train": 2221,
+  "n_valid": 282,
+  "n_test": 190,
+  "train_start": "2015-01-09",
+  "train_end": "2023-12-31",
+  "valid_start": "2024-07-01",
+  "valid_end": "2025-08-17",
+  "test_start": "2025-08-18",
+  "test_end": "2026-05-26",
+  "latest_forecast_date": "2026-05-27",
+  "latest_forecast_for": "next trading bar after 2026-05-27",
+  "latest_forecast_prob_up": 0.536599063991701,
+  "latest_forecast_signal": "UP",
+  "feature_count": 204,
+  "validation_prob_std": 0.06800064531350844,
+  "test_prob_std": 0.06311239013827799,
+  "test_prob_min": 0.3874936713840175,
+  "test_prob_max": 0.6430757596651826,
+  "model_name": "nifty_tomorrow_direction_model",
+  "source_model": "locked_multiwindow_nifty50_ensemble_v2",
+  "target": "next trading session NIFTY 50 direction",
+  "artifact_type": "daily_forecaster_outputs",
+  "artifact_source": "C:\\Users\\jhaji\\Downloads\\forecasting project\\Code\\models\\nifty_forecaster\\outputs"
+}

backend/models/tplus1_latest_prediction.csv

@@ -0,0 +1,2 @@
+input_date,target_date,forecast_for,prediction,prob_up,confidence,threshold,model_name,decision_overlay,validation_accuracy,test_accuracy,accuracy_goal
+2026-05-27,2026-05-29,next trading session after 2026-05-27,UP,0.47376564177714775,0.5262343582228522,0.578,logistic_regression_l1_C0.35_balanced,prev_target_mean10_le_0.4_up;m02_range_1m_ge_0.000479116_up,0.66,0.6368421052631579,0.63

backend/models/tplus1_summary.json

@@ -0,0 +1,37 @@
+{
+  "target": "T+1 NIFTY 50 close greater than T 14:20 close",
+  "exchange_instrument": "NSE NIFTY 50 index",
+  "window_start": "14:00",
+  "window_end": "14:20",
+  "lookback_days_requested": 1000,
+  "supervised_rows": 1000,
+  "train_rows": 660,
+  "valid_rows": 150,
+  "test_rows": 190,
+  "train_start": "2022-04-28",
+  "train_end": "2024-12-31",
+  "valid_start": "2025-01-01",
+  "valid_end": "2025-08-06",
+  "test_start": "2025-08-07",
+  "test_end": "2026-05-20",
+  "model_name": "logistic_regression_l1_C0.35_balanced",
+  "threshold": 0.578,
+  "validation_accuracy": 0.66,
+  "test_accuracy": 0.6368421052631579,
+  "baseline_test_accuracy": 0.5052631578947369,
+  "validation_auc": 0.5085348506401137,
+  "test_auc": 0.54864804964539,
+  "test_log_loss": 0.715200083567372,
+  "test_brier": 0.2580990249203714,
+  "accuracy_goal": 0.63,
+  "accuracy_goal_met_on_test": true,
+  "feature_count": 40,
+  "latest_input_date": "2026-05-21",
+  "latest_window_rows": 21,
+  "latest_forecast_for": "next trading session after 2026-05-21",
+  "latest_prob_up": 0.620604654276822,
+  "latest_prediction": "UP",
+  "latest_confidence": 0.620604654276822,
+  "decision_overlay": "prev_target_mean10_le_0.4_up;m02_range_1m_ge_0.000479116_up",
+  "top_features": 40
+}

backend/models/yahoo_history_cache.sqlite3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:534853b3b5984c2c4128ed073412f090ce96d6cb7a9da4aaa98a20482a2a4bff
+size 155648

backend/nifty_backend/__init__.py

@@ -0,0 +1,2 @@
+"""Backend runtime for the NIFTY 50 opening-direction forecaster."""
+

backend/nifty_backend/__pycache__/runtime.cpython-311.pyc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89f2a00f920ffc636fbdd9b8b9b0fb6c6cbe955a5de208b7879183571d861e84
+size 112925

backend/nifty_backend/runtime.py

@@ -0,0 +1,1632 @@
+from __future__ import annotations
+
+import json
+import copy
+import os
+import sys
+import threading
+from dataclasses import dataclass
+from datetime import date, datetime, time, timedelta
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+from zoneinfo import ZoneInfo
+
+import joblib
+import numpy as np
+import pandas as pd
+from nifty_backend.yahoo_history_client import YahooHistoryClient
+
+try:
+    import pandas_market_calendars as mcal
+except ImportError:  # pragma: no cover - production dependency, local fallback below.
+    mcal = None
+
+
+IST = ZoneInfo("Asia/Kolkata")
+YAHOO_NIFTY_SYMBOL = "^NSEI"
+MARKET_CLOSE = time(15, 30)
+FIRST5_READY = time(9, 20)
+CLOSE_REFRESH_READY = time(15, 45)
+TPLUS1_READY = time(14, 30)
+STALE_CHECK_INTERVAL_SECONDS = 5
+BACKEND_ROOT = Path(__file__).resolve().parents[1]
+DATA_DIR = BACKEND_ROOT / "data"
+MODEL_DIR = BACKEND_ROOT / "models"
+YAHOO_CACHE_PATH = MODEL_DIR / "yahoo_history_cache.sqlite3"
+OPENING_DATASET_PATH = DATA_DIR / "opening_direction_training_dataset.parquet"
+NIFTY_1M_PATH = DATA_DIR / "nifty50_1m.parquet"
+NIFTY_1D_PATH = DATA_DIR / "nifty50_1d.parquet"
+MODEL_PATH = MODEL_DIR / "nifty_opening_direction_model.joblib"
+LATEST_PATH = MODEL_DIR / "latest_prediction.csv"
+TEST_PREDICTIONS_PATH = DATA_DIR / "test_predictions.parquet"
+TOMORROW_MODEL_PATH = MODEL_DIR / "nifty_tomorrow_direction_model.joblib"
+TOMORROW_LATEST_PATH = MODEL_DIR / "tomorrow_latest_prediction.csv"
+TOMORROW_SUMMARY_PATH = MODEL_DIR / "tomorrow_summary.json"
+TOMORROW_TEST_PREDICTIONS_PATH = DATA_DIR / "tomorrow_test_predictions.parquet"
+FORECASTING_PROJECT_ROOT = Path(
+    os.environ.get(
+        "FORECASTING_PROJECT_ROOT",
+        str(BACKEND_ROOT.parent.parent / "forecasting project"),
+    )
+)
+DAILY_FORECASTER_OUTPUT_DIR = FORECASTING_PROJECT_ROOT / "Code" / "models" / "nifty_forecaster" / "outputs"
+DAILY_FORECASTER_SUMMARY_PATH = DAILY_FORECASTER_OUTPUT_DIR / "forecaster_summary.json"
+DAILY_FORECASTER_LATEST_PATH = DAILY_FORECASTER_OUTPUT_DIR / "forecaster_latest.csv"
+DAILY_FORECASTER_PREDICTIONS_PATH = DAILY_FORECASTER_OUTPUT_DIR / "forecaster_test_predictions.csv"
+TPLUS1_MODEL_PATH = MODEL_DIR / "nifty_1420_tplus1_logistic_model.joblib"
+TPLUS1_LATEST_PATH = MODEL_DIR / "tplus1_latest_prediction.csv"
+TPLUS1_SUMMARY_PATH = MODEL_DIR / "tplus1_summary.json"
+TPLUS1_TEST_PREDICTIONS_PATH = DATA_DIR / "tplus1_test_predictions.parquet"
+REFRESH_STATE_PATH = MODEL_DIR / "refresh_state.json"
+REFRESH_WAITING = "waiting_second_payload"
+REFRESH_REFRESHING = "refreshing"
+REFRESH_READY = "ready"
+REFRESH_FAILED = "failed"
+REFRESH_NORMAL = "normal"
+LIVE_ACCURACY_PATH = MODEL_DIR / "live_accuracy.json"
+
+DECISION_OVERLAYS = [
+    {
+        "name": "fifth_minute_momentum_flip",
+        "feature": "m5_ret_1m",
+        "op": ">=",
+        "value": 0.0005085411885759201,
+    },
+    {
+        "name": "vix_stretch_flip",
+        "feature": "india_vix_close_vs_sma_20",
+        "op": ">=",
+        "value": 0.24641908937959742,
+    },
+]
+
+_dashboard_payload_lock = threading.Lock()
+_stale_refresh_lock = threading.Lock()
+
+
+def utc_now_iso() -> str:
+    return datetime.utcnow().replace(microsecond=0).isoformat() + "Z"
+
+
+def clear_dashboard_payload_cache() -> None:
+    _dashboard_payload_cached.cache_clear()
+
+
+def save_refresh_state(phase: str, *, session_date: date | None = None, error: str | None = None) -> dict[str, Any]:
+    previous = load_refresh_state()
+    state = {
+        "phase": phase,
+        "started_at": previous.get("started_at"),
+        "finished_at": previous.get("finished_at"),
+        "session_date": session_date.isoformat() if session_date else previous.get("session_date"),
+        "error": error,
+    }
+    if phase in {REFRESH_WAITING, REFRESH_REFRESHING} and not state["started_at"]:
+        state["started_at"] = utc_now_iso()
+    if phase in {REFRESH_READY, REFRESH_FAILED, REFRESH_NORMAL}:
+        state["finished_at"] = utc_now_iso()
+    REFRESH_STATE_PATH.write_text(json.dumps(state, indent=2), encoding="utf-8")
+    return state
+
+
+def load_refresh_state() -> dict[str, Any]:
+    if not REFRESH_STATE_PATH.exists():
+        return {
+            "phase": REFRESH_NORMAL,
+            "started_at": None,
+            "finished_at": None,
+            "session_date": None,
+            "error": None,
+        }
+    try:
+        return json.loads(REFRESH_STATE_PATH.read_text(encoding="utf-8"))
+    except Exception:
+        return {
+            "phase": REFRESH_FAILED,
+            "started_at": None,
+            "finished_at": None,
+            "session_date": None,
+            "error": "refresh_state.json could not be read",
+        }
+
+
+@lru_cache(maxsize=1)
+def _nse_calendar():
+    if mcal is None:
+        return None
+    for name in ("XNSE", "NSE", "BSE"):
+        try:
+            return mcal.get_calendar(name)
+        except Exception:
+            continue
+    return None
+
+
+@lru_cache(maxsize=64)
+def trading_schedule(start: date, end: date) -> pd.DataFrame:
+    calendar = _nse_calendar()
+    if calendar is None:
+        days = pd.date_range(start=start, end=end, freq="B")
+        return pd.DataFrame(index=days)
+    return calendar.schedule(start_date=start, end_date=end)
+
+
+def is_trading_day(day: date) -> bool:
+    schedule = trading_schedule(day, day)
+    return not schedule.empty
+
+
+def next_trading_day(start: date) -> date:
+    end = start + timedelta(days=14)
+    schedule = trading_schedule(start, end)
+    if schedule.empty:
+        day = start
+        while not is_trading_day(day):
+            day += timedelta(days=1)
+        return day
+    return pd.Timestamp(schedule.index[0]).date()
+
+
+def previous_trading_day(start: date) -> date:
+    begin = start - timedelta(days=14)
+    schedule = trading_schedule(begin, start)
+    if schedule.empty:
+        day = start
+        while not is_trading_day(day):
+            day -= timedelta(days=1)
+        return day
+    return pd.Timestamp(schedule.index[-1]).date()
+
+
+class ProbabilityBlend:
+    def __init__(self, models: list[Any], weights: np.ndarray):
+        self.models = models
+        self.weights = np.asarray(weights, dtype="float64")
+        self.weights = self.weights / self.weights.sum()
+
+    def predict_proba(self, x: pd.DataFrame) -> np.ndarray:
+        probs = np.column_stack([predict_proba_up(model, x) for model in self.models])
+        prob_up = probs @ self.weights
+        return np.column_stack([1.0 - prob_up, prob_up])
+
+
+@dataclass(frozen=True)
+class Prediction:
+    input_date: str
+    first5_start: str
+    first5_end: str
+    prediction: str
+    prob_up: float
+    confidence: float
+    threshold: float
+    model_name: str
+    is_overridden: bool = False
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "input_date": self.input_date,
+            "first5_start": self.first5_start,
+            "first5_end": self.first5_end,
+            "prediction": self.prediction,
+            "prob_up": self.prob_up,
+            "confidence": self.confidence,
+            "threshold": self.threshold,
+            "model_name": self.model_name,
+            "is_overridden": getattr(self, "is_overridden", False),
+        }
+
+
+def predict_proba_up(model: Any, x: pd.DataFrame) -> np.ndarray:
+    return np.asarray(model.predict_proba(x)[:, 1], dtype="float64")
+
+
+def safe_div(numer: pd.Series | np.ndarray, denom: pd.Series | np.ndarray) -> pd.Series:
+    n = pd.Series(numer, copy=False)
+    d = pd.Series(denom, copy=False)
+    out = pd.Series(np.nan, index=n.index, dtype="float64")
+    mask = d.notna() & np.isfinite(d.to_numpy(dtype="float64")) & (d != 0)
+    out.loc[mask] = n.loc[mask].to_numpy(dtype="float64") / d.loc[mask].to_numpy(dtype="float64")
+    return out
+
+
+def load_model() -> dict[str, Any]:
+    # Existing artifact was trained as a script, so its custom blend class
+    # resolves through __main__ when unpickled.
+    sys.modules["__main__"].ProbabilityBlend = ProbabilityBlend
+    sys.modules["__main__"].predict_proba_up = predict_proba_up
+    payload = joblib.load(MODEL_PATH)
+    payload.setdefault("decision_overlays", DECISION_OVERLAYS)
+    payload.setdefault("model_name", "nifty_opening_direction_model")
+    return payload
+
+
+def overlay_mask(frame: pd.DataFrame, overlay: dict[str, object]) -> np.ndarray:
+    feature = str(overlay["feature"])
+    if feature not in frame.columns:
+        return np.zeros(len(frame), dtype=bool)
+    series = pd.to_numeric(frame[feature], errors="coerce")
+    value = float(overlay["value"])
+    if overlay["op"] == ">=":
+        return (series >= value).fillna(False).to_numpy(dtype=bool)
+    if overlay["op"] == "<=":
+        return (series <= value).fillna(False).to_numpy(dtype=bool)
+    raise ValueError(f"Unsupported overlay op: {overlay['op']}")
+
+
+def apply_decision_overlays(pred: np.ndarray, frame: pd.DataFrame, overlays: list[dict[str, object]]) -> np.ndarray:
+    adjusted = np.asarray(pred, dtype="int64").copy()
+    for overlay in overlays:
+        mask = overlay_mask(frame, overlay)
+        adjusted[mask] = 1 - adjusted[mask]
+    return adjusted
+
+
+def directional_confidence(prob_up: np.ndarray, pred: np.ndarray, threshold: float) -> np.ndarray:
+    prob_up = np.asarray(prob_up, dtype="float64")
+    pred = np.asarray(pred, dtype="int64")
+    base_side_prob = np.where(pred == 1, prob_up, 1.0 - prob_up)
+    threshold_distance = np.abs(prob_up - float(threshold))
+    return np.clip(0.50 + threshold_distance, base_side_prob, 0.99)
+
+
+def read_training_dataset() -> pd.DataFrame:
+    df = pd.read_parquet(OPENING_DATASET_PATH)
+    for col in ("date", "first5_start", "first5_end"):
+        if col in df.columns:
+            df[col] = pd.to_datetime(df[col], errors="coerce")
+    return df.sort_values("date").reset_index(drop=True)
+
+
+def normalize_yahoo_frame(df: pd.DataFrame) -> pd.DataFrame:
+    if df.empty:
+        return pd.DataFrame(columns=["date", "open", "high", "low", "close", "volume"])
+    if isinstance(df.columns, pd.MultiIndex):
+        df.columns = [str(c[0]).lower() for c in df.columns]
+    else:
+        df.columns = [str(c).lower().replace(" ", "_") for c in df.columns]
+    df = df.reset_index()
+    date_col = next((c for c in df.columns if c.lower() in {"datetime", "date"}), df.columns[0])
+    df["date"] = pd.to_datetime(df[date_col], errors="coerce")
+    if df["date"].dt.tz is None:
+        df["date"] = df["date"].dt.tz_localize("UTC").dt.tz_convert(IST)
+    else:
+        df["date"] = df["date"].dt.tz_convert(IST)
+    rename = {
+        "open": "open",
+        "high": "high",
+        "low": "low",
+        "close": "close",
+        "adj_close": "close",
+        "volume": "volume",
+    }
+    out = pd.DataFrame({"date": df["date"].dt.tz_localize(None)})
+    for src, dst in rename.items():
+        if src in df.columns and dst not in out.columns:
+            out[dst] = pd.to_numeric(df[src], errors="coerce")
+    return out.dropna(subset=["date", "open", "high", "low", "close"]).sort_values("date")
+
+
+@lru_cache(maxsize=1)
+def yahoo_history_client() -> YahooHistoryClient:
+    return YahooHistoryClient(cache_path=YAHOO_CACHE_PATH)
+
+
+def period_start(period: str, *, end: datetime) -> datetime:
+    text = str(period).strip().lower()
+    units = {
+        "d": "days",
+        "wk": "weeks",
+        "mo": "months",
+        "y": "years",
+    }
+    for suffix, unit in units.items():
+        if text.endswith(suffix):
+            raw_value = text[: -len(suffix)]
+            if not raw_value.isdigit():
+                break
+            value = int(raw_value)
+            if unit == "days":
+                return end - timedelta(days=value)
+            if unit == "weeks":
+                return end - timedelta(weeks=value)
+            if unit == "months":
+                return end - timedelta(days=value * 31)
+            if unit == "years":
+                return end - timedelta(days=value * 366)
+    raise ValueError(f"Unsupported Yahoo period: {period!r}")
+
+
+def yahoo_history_to_ohlcv(frame: pd.DataFrame, *, daily: bool) -> pd.DataFrame:
+    if frame.empty:
+        return pd.DataFrame(columns=["date", "open", "high", "low", "close", "volume"])
+    out = frame.rename(columns={"timestamp": "date"}).copy()
+    out["date"] = pd.to_datetime(out["date"], errors="coerce")
+    if daily:
+        out["date"] = out["date"].dt.normalize()
+    for column in ("open", "high", "low", "close", "volume"):
+        out[column] = pd.to_numeric(out[column], errors="coerce")
+    return (
+        out[["date", "open", "high", "low", "close", "volume"]]
+        .dropna(subset=["date", "open", "high", "low", "close"])
+        .drop_duplicates("date", keep="last")
+        .sort_values("date")
+        .reset_index(drop=True)
+    )
+
+
+def fetch_yahoo_minutes(period: str = "5d") -> pd.DataFrame:
+    end = datetime.now(IST).replace(tzinfo=None) + timedelta(minutes=5)
+    start = period_start(period, end=end)
+    raw = yahoo_history_client().fetch_history(
+        YAHOO_NIFTY_SYMBOL,
+        interval="1m",
+        start=start,
+        end=end,
+        include_prepost=False,
+    )
+    return yahoo_history_to_ohlcv(raw, daily=False)
+
+
+def fetch_yahoo_daily(period: str = "1mo") -> pd.DataFrame:
+    end = datetime.now(IST).replace(tzinfo=None) + timedelta(days=1)
+    start = period_start(period, end=end)
+    raw = yahoo_history_client().fetch_history(
+        YAHOO_NIFTY_SYMBOL,
+        interval="1d",
+        start=start,
+        end=end,
+        include_prepost=False,
+    )
+    return yahoo_history_to_ohlcv(raw, daily=True)
+
+
+def append_parquet_rows(path: Path, new_rows: pd.DataFrame, subset: list[str]) -> pd.DataFrame:
+    if new_rows.empty:
+        if path.exists():
+            return pd.read_parquet(path)
+        raise RuntimeError(f"No rows returned for {path.name}; leaving parquet unchanged.")
+    if path.exists():
+        existing = pd.read_parquet(path)
+        combined = pd.concat([existing, new_rows], ignore_index=True)
+    else:
+        combined = new_rows.copy()
+    combined = combined.drop_duplicates(subset=subset, keep="last").sort_values(subset).reset_index(drop=True)
+    combined.to_parquet(path, index=False, compression="zstd")
+    return combined
+
+
+def latest_parquet_date(path: Path) -> date | None:
+    if not path.exists():
+        return None
+    df = pd.read_parquet(path, columns=["date"])
+    if df.empty:
+        return None
+    latest = pd.to_datetime(df["date"], errors="coerce").max()
+    if pd.isna(latest):
+        return None
+    return latest.date()
+
+
+def latest_opening_outcome_date() -> date | None:
+    if not OPENING_DATASET_PATH.exists():
+        return None
+    cols = ["date"]
+    if "target" in pd.read_parquet(OPENING_DATASET_PATH).columns:
+        cols.append("target")
+    df = pd.read_parquet(OPENING_DATASET_PATH, columns=cols)
+    if df.empty or "target" not in df.columns:
+        return None
+    df = df[df["target"].notna()]
+    if df.empty:
+        return None
+    latest = pd.to_datetime(df["date"], errors="coerce").max()
+    if pd.isna(latest):
+        return None
+    return latest.date()
+
+
+def first5_features_from_minutes(minutes: pd.DataFrame, session_date: date | None = None) -> pd.DataFrame:
+    if minutes.empty:
+        raise RuntimeError("Yahoo returned no minute bars.")
+    bars = minutes.copy()
+    bars["dt"] = pd.to_datetime(bars["date"], errors="coerce")
+    bars["session_date"] = bars["dt"].dt.normalize()
+    if session_date is None:
+        session_ts = bars["session_date"].max()
+    else:
+        session_ts = pd.Timestamp(session_date).normalize()
+    day = bars[bars["session_date"] == session_ts].sort_values("dt").copy()
+    start_dt = pd.Timestamp.combine(session_ts.date(), time(9, 15))
+    end_dt = pd.Timestamp.combine(session_ts.date(), time(9, 19))
+    first5 = day[(day["dt"] >= start_dt) & (day["dt"] <= end_dt)].head(5).copy()
+    if len(first5) < 5:
+        raise RuntimeError(f"Need 5 opening bars for {session_ts.date()}, got {len(first5)}.")
+    first5["minute_index"] = np.arange(len(first5))
+    first5["ret_1m"] = first5["close"].pct_change(fill_method=None)
+    first5["range_pct_1m"] = safe_div(first5["high"] - first5["low"], first5["open"])
+    first5["body_pct_1m"] = safe_div(first5["close"] - first5["open"], first5["open"])
+    row = {
+        "date": session_ts,
+        "first5_start": first5["dt"].iloc[0],
+        "first5_end": first5["dt"].iloc[-1],
+        "first5_open": first5["open"].iloc[0],
+        "first5_high": first5["high"].max(),
+        "first5_low": first5["low"].min(),
+        "first5_close": first5["close"].iloc[-1],
+        "first5_volume": first5["volume"].sum() if "volume" in first5 else 0.0,
+        "first5_bars": len(first5),
+        "first5_last_1m_ret": first5["ret_1m"].iloc[-1],
+        "first5_ret_std": first5["ret_1m"].std(),
+    }
+    row["first5_return"] = (row["first5_close"] - row["first5_open"]) / row["first5_open"]
+    row["first5_range_pct"] = (row["first5_high"] - row["first5_low"]) / row["first5_open"]
+    first5_range = row["first5_high"] - row["first5_low"]
+    row["first5_body_to_range"] = (row["first5_close"] - row["first5_open"]) / first5_range if first5_range else np.nan
+    row["first5_close_location"] = (row["first5_close"] - row["first5_low"]) / first5_range if first5_range else np.nan
+    for idx, (_, candle) in enumerate(first5.iterrows(), start=1):
+        for field in ("open", "high", "low", "close", "ret_1m", "range_pct_1m", "body_pct_1m"):
+            row[f"m{idx}_{field}"] = candle[field]
+        row[f"m{idx}_close_vs_first5_open"] = (candle["close"] - row["first5_open"]) / row["first5_open"]
+        row[f"m{idx}_range_share"] = (candle["high"] - candle["low"]) / first5_range if first5_range else np.nan
+    row["first5_return_accel"] = row["m5_ret_1m"] - row["m2_ret_1m"]
+    row["first5_last2_return"] = (row["m5_close"] - row["m4_open"]) / row["m4_open"]
+    row["first5_first2_return"] = (row["m2_close"] - row["m1_open"]) / row["m1_open"]
+    row["first5_reversal"] = np.sign(row["first5_first2_return"]) * -np.sign(row["first5_last2_return"])
+    row["dow"] = session_ts.dayofweek
+    row["dom"] = session_ts.day
+    row["month"] = session_ts.month
+    return pd.DataFrame([row])
+
+
+def build_model_row(first5_row: pd.DataFrame) -> pd.DataFrame:
+    dataset = read_training_dataset()
+    latest_context = dataset.iloc[[-1]].copy()
+    output = latest_context.copy()
+    for col in first5_row.columns:
+        output[col] = first5_row[col].iloc[0]
+    if {"first5_open", "nifty_close"}.issubset(output.columns):
+        output["first5_gap_from_prev_close"] = (output["first5_open"] - output["nifty_close"]) / output["nifty_close"]
+        output["first5_close_vs_prev_close"] = (output["first5_close"] - output["nifty_close"]) / output["nifty_close"]
+    if {"first5_range_pct", "nifty_range_pct"}.issubset(output.columns):
+        output["first5_range_vs_prev_range"] = output["first5_range_pct"] / output["nifty_range_pct"]
+    if {"first5_return", "nifty_ret_1"}.issubset(output.columns):
+        output["first5_return_x_prev_ret"] = output["first5_return"] * output["nifty_ret_1"]
+        output["gap_x_prev_ret"] = output["first5_gap_from_prev_close"] * output["nifty_ret_1"]
+    if {"first5_return", "banknifty_ret_1"}.issubset(output.columns):
+        output["first5_return_x_bank_ret_1"] = output["first5_return"] * output["banknifty_ret_1"]
+    if {"first5_range_pct", "india_vix_ret_1"}.issubset(output.columns):
+        output["first5_range_x_vix_ret_1"] = output["first5_range_pct"] * output["india_vix_ret_1"]
+    output["target"] = np.nan
+    output["day_return"] = np.nan
+    return output
+
+
+def predict_row(row: pd.DataFrame) -> Prediction:
+    payload = load_model()
+    model = payload["model"]
+    features = payload["features"]
+    threshold = float(payload["threshold"])
+    missing = [c for c in features if c not in row.columns]
+    if missing:
+        raise RuntimeError(f"Feature row is missing {len(missing)} features; first missing: {missing[:5]}")
+    prob_up = predict_proba_up(model, row[features])
+    raw_pred = (prob_up >= threshold).astype("int64")
+    pred = apply_decision_overlays(raw_pred, row, payload.get("decision_overlays", DECISION_OVERLAYS))
+    is_overridden = bool(raw_pred[0] != pred[0])
+    confidence = directional_confidence(prob_up, pred, threshold)
+    prediction = Prediction(
+        input_date=pd.to_datetime(row["date"].iloc[0]).date().isoformat(),
+        first5_start=str(pd.to_datetime(row["first5_start"].iloc[0])),
+        first5_end=str(pd.to_datetime(row["first5_end"].iloc[0])),
+        prediction="UP" if int(pred[0]) == 1 else "DOWN",
+        prob_up=float(prob_up[0]),
+        confidence=float(confidence[0]),
+        threshold=threshold,
+        model_name=str(payload.get("model_name", "nifty_opening_direction_model")),
+        is_overridden=is_overridden,
+    )
+    pd.DataFrame([prediction.to_dict()]).to_csv(LATEST_PATH, index=False)
+    return prediction
+
+
+def _file_cache_key(path: Path) -> tuple[str, int | None, int | None]:
+    try:
+        stat = path.stat()
+    except FileNotFoundError:
+        return (str(path), None, None)
+    return (str(path), stat.st_mtime_ns, stat.st_size)
+
+
+@lru_cache(maxsize=16)
+def _latest_saved_prediction_cached(latest_key: tuple[str, int | None, int | None], summary_key: tuple[str, int | None, int | None]) -> dict[str, Any]:
+    latest_path = Path(latest_key[0])
+    if latest_path.exists():
+        return pd.read_csv(latest_path).iloc[-1].to_dict()
+    summary_path = Path(summary_key[0])
+    if summary_path.exists():
+        return json.loads(summary_path.read_text(encoding="utf-8"))
+    raise FileNotFoundError("No latest prediction is available yet.")
+
+
+def latest_saved_prediction() -> dict[str, Any]:
+    return dict(_latest_saved_prediction_cached(_file_cache_key(LATEST_PATH), _file_cache_key(MODEL_DIR / "summary.json")))
+
+
+def _latest_saved_prediction_uncached() -> dict[str, Any]:
+    if LATEST_PATH.exists():
+        return pd.read_csv(LATEST_PATH).iloc[-1].to_dict()
+    summary_path = MODEL_DIR / "summary.json"
+    if summary_path.exists():
+        return json.loads(summary_path.read_text(encoding="utf-8"))
+    raise FileNotFoundError("No latest prediction is available yet.")
+
+
+def _read_daily_forecaster_summary() -> dict[str, Any] | None:
+    if not DAILY_FORECASTER_SUMMARY_PATH.exists():
+        return None
+    raw = json.loads(DAILY_FORECASTER_SUMMARY_PATH.read_text(encoding="utf-8"))
+    if isinstance(raw, list):
+        matches = [row for row in raw if row.get("symbol") == "NIFTY 50"]
+        summary = dict(matches[0] if matches else raw[0])
+    elif isinstance(raw, dict):
+        summary = dict(raw)
+    else:
+        return None
+    config = summary.get("config") if isinstance(summary.get("config"), dict) else {}
+    summary.setdefault("symbol", "NIFTY 50")
+    summary.setdefault("horizon", "daily")
+    summary.setdefault("horizon_bars", 1)
+    summary["model_name"] = "nifty_tomorrow_direction_model"
+    summary["source_model"] = str(config.get("name") or summary.get("source_model") or "locked_multiwindow_nifty50_ensemble")
+    summary["target"] = "next trading session NIFTY 50 direction"
+    summary["artifact_type"] = "daily_forecaster_outputs"
+    summary["artifact_source"] = str(DAILY_FORECASTER_OUTPUT_DIR)
+    return summary
+
+
+def _read_daily_forecaster_latest(summary: dict[str, Any]) -> dict[str, Any] | None:
+    if not DAILY_FORECASTER_LATEST_PATH.exists():
+        return None
+    latest = pd.read_csv(DAILY_FORECASTER_LATEST_PATH)
+    if latest.empty:
+        return None
+    if "symbol" in latest.columns:
+        filtered = latest[latest["symbol"].astype(str) == "NIFTY 50"]
+        if not filtered.empty:
+            latest = filtered
+    row = {k: (None if pd.isna(v) else v) for k, v in latest.iloc[-1].to_dict().items()}
+    input_date = row.get("latest_forecast_date") or row.get("input_date")
+    target_date = row.get("target_date")
+    if not target_date and input_date:
+        try:
+            target_date = next_trading_day(date.fromisoformat(str(input_date)[:10]) + timedelta(days=1)).isoformat()
+        except Exception:
+            target_date = None
+    prob_up = row.get("latest_forecast_prob_up", row.get("prob_up"))
+    prediction = row.get("latest_forecast_signal", row.get("prediction"))
+    threshold = row.get("threshold", summary.get("threshold"))
+    confidence = row.get("confidence")
+    if confidence is None and prob_up is not None:
+        try:
+            confidence = float(max(float(prob_up), 1.0 - float(prob_up)))
+        except Exception:
+            confidence = None
+    return {
+        "input_date": input_date,
+        "target_date": target_date,
+        "prediction": prediction,
+        "prob_up": prob_up,
+        "confidence": confidence,
+        "threshold": threshold,
+        "model_name": "nifty_tomorrow_direction_model",
+        "source_model": summary.get("source_model", "locked_multiwindow_nifty50_ensemble"),
+        "validation_accuracy": summary.get("validation_accuracy"),
+        "test_accuracy": summary.get("test_accuracy"),
+        "artifact_source": str(DAILY_FORECASTER_OUTPUT_DIR),
+    }
+
+
+def sync_daily_forecaster_outputs() -> dict[str, Any] | None:
+    summary = _read_daily_forecaster_summary()
+    if summary is None:
+        return None
+    latest = _read_daily_forecaster_latest(summary)
+    TOMORROW_SUMMARY_PATH.write_text(json.dumps(summary, indent=2), encoding="utf-8")
+    if latest is not None:
+        pd.DataFrame([latest]).to_csv(TOMORROW_LATEST_PATH, index=False)
+    if DAILY_FORECASTER_PREDICTIONS_PATH.exists():
+        predictions = pd.read_csv(DAILY_FORECASTER_PREDICTIONS_PATH)
+        if "symbol" in predictions.columns:
+            predictions = predictions[predictions["symbol"].astype(str) == "NIFTY 50"].copy()
+        if not predictions.empty:
+            if "pred" in predictions.columns and "prediction" not in predictions.columns:
+                predictions["prediction"] = np.where(pd.to_numeric(predictions["pred"], errors="coerce") == 1, "UP", "DOWN")
+            if "correct" not in predictions.columns and {"target", "pred"}.issubset(predictions.columns):
+                predictions["correct"] = (
+                    pd.to_numeric(predictions["target"], errors="coerce")
+                    == pd.to_numeric(predictions["pred"], errors="coerce")
+                )
+            predictions.to_parquet(TOMORROW_TEST_PREDICTIONS_PATH, index=False)
+    artifact = {
+        "artifact_type": "daily_forecaster_outputs",
+        "model_name": "nifty_tomorrow_direction_model",
+        "source_model": summary.get("source_model", "locked_multiwindow_nifty50_ensemble"),
+        "threshold": float(summary.get("threshold", 0.54)),
+        "validation_accuracy": summary.get("validation_accuracy"),
+        "test_accuracy": summary.get("test_accuracy"),
+        "validation_prob_std": summary.get("validation_prob_std"),
+        "test_prob_std": summary.get("test_prob_std"),
+        "test_prob_min": summary.get("test_prob_min"),
+        "test_prob_max": summary.get("test_prob_max"),
+        "artifact_source": str(DAILY_FORECASTER_OUTPUT_DIR),
+    }
+    joblib.dump(artifact, TOMORROW_MODEL_PATH)
+    return latest or summary
+
+
+def load_tomorrow_model_artifact() -> dict[str, Any]:
+    synced = sync_daily_forecaster_outputs()
+    if synced is not None and TOMORROW_MODEL_PATH.exists():
+        return joblib.load(TOMORROW_MODEL_PATH)
+    if TOMORROW_MODEL_PATH.exists():
+        return joblib.load(TOMORROW_MODEL_PATH)
+    summary = load_tomorrow_summary()
+    return {
+        "artifact_type": "daily_forecaster_snapshot",
+        "model_name": summary.get("model_name", "nifty_tomorrow_direction_model"),
+        "source_model": summary.get("source_model", "tuned_daily_forest_single"),
+        "threshold": float(summary.get("threshold", 0.543)),
+    }
+
+
+def load_tomorrow_summary() -> dict[str, Any]:
+    synced = sync_daily_forecaster_outputs()
+    if synced is not None and TOMORROW_SUMMARY_PATH.exists():
+        return json.loads(TOMORROW_SUMMARY_PATH.read_text(encoding="utf-8"))
+    if TOMORROW_SUMMARY_PATH.exists():
+        return json.loads(TOMORROW_SUMMARY_PATH.read_text(encoding="utf-8"))
+    return {
+        "model_name": "nifty_tomorrow_direction_model",
+        "source_model": "locked_multiwindow_nifty50_ensemble",
+        "target": "next trading session NIFTY 50 direction",
+        "threshold": 0.54,
+        "validation_accuracy": 0.5673758865248227,
+        "test_accuracy": 0.6451612903225806,
+        "baseline_accuracy": 0.5053763440860215,
+        "n_test": 186,
+        "feature_count": 204,
+    }
+
+
+def latest_tomorrow_prediction() -> dict[str, Any]:
+    sync_daily_forecaster_outputs()
+    latest_daily = latest_parquet_date(NIFTY_1D_PATH)
+    expected_daily = expected_completed_daily_date()
+    valid_daily = min(latest_daily, expected_daily) if latest_daily and expected_daily else (expected_daily or latest_daily)
+
+    if TOMORROW_LATEST_PATH.exists():
+        row = pd.read_csv(TOMORROW_LATEST_PATH).iloc[-1].to_dict()
+        cleaned = {k: (None if pd.isna(v) else v) for k, v in row.items()}
+        try:
+            input_day = date.fromisoformat(str(cleaned.get("input_date"))[:10])
+        except Exception:
+            input_day = None
+        if valid_daily is not None and (input_day is None or input_day < valid_daily):
+            try:
+                refreshed = refresh_tomorrow_prediction(session_date=valid_daily)
+                try:
+                    refreshed_day = date.fromisoformat(str(refreshed.get("input_date"))[:10])
+                except Exception:
+                    refreshed_day = None
+                if refreshed_day is not None and refreshed_day >= valid_daily:
+                    return refreshed
+            except Exception:
+                pass
+        return cleaned
+    summary = load_tomorrow_summary()
+    try:
+        summary_input_day = date.fromisoformat(str(summary.get("latest_forecast_date"))[:10])
+    except Exception:
+        summary_input_day = None
+    if valid_daily is not None and (summary_input_day is None or summary_input_day < valid_daily):
+        try:
+            return refresh_tomorrow_prediction(session_date=valid_daily)
+        except Exception:
+            pass
+    return {
+        "input_date": summary.get("latest_forecast_date"),
+        "target_date": None,
+        "prediction": summary.get("latest_forecast_signal"),
+        "prob_up": summary.get("latest_forecast_prob_up"),
+        "confidence": None,
+        "threshold": summary.get("threshold"),
+        "model_name": summary.get("model_name", "nifty_tomorrow_direction_model"),
+        "source_model": summary.get("source_model", "tuned_daily_forest_single"),
+        "validation_accuracy": summary.get("validation_accuracy"),
+        "test_accuracy": summary.get("test_accuracy"),
+    }
+
+
+def load_tplus1_summary() -> dict[str, Any]:
+    if TPLUS1_SUMMARY_PATH.exists():
+        return json.loads(TPLUS1_SUMMARY_PATH.read_text(encoding="utf-8"))
+    return {
+        "model_name": "logistic_regression_l1_C0.35_balanced",
+        "target": "T+1 NIFTY 50 close greater than T 14:20 close",
+        "window_start": "14:00",
+        "window_end": "14:20",
+        "threshold": 0.578,
+        "validation_accuracy": 0.66,
+        "test_accuracy": 0.6368421052631579,
+        "baseline_test_accuracy": 0.5052631578947369,
+        "test_rows": 190,
+        "feature_count": 40,
+    }
+
+
+def latest_tplus1_prediction() -> dict[str, Any]:
+    if TPLUS1_LATEST_PATH.exists():
+        row = pd.read_csv(TPLUS1_LATEST_PATH).iloc[-1].to_dict()
+        return {k: (None if pd.isna(v) else v) for k, v in row.items()}
+    summary = load_tplus1_summary()
+    return {
+        "input_date": summary.get("latest_input_date"),
+        "target_date": None,
+        "forecast_for": summary.get("latest_forecast_for"),
+        "prediction": summary.get("latest_prediction"),
+        "prob_up": summary.get("latest_prob_up"),
+        "confidence": summary.get("latest_confidence"),
+        "threshold": summary.get("threshold"),
+        "model_name": summary.get("model_name", "logistic_regression_l1_C0.35_balanced"),
+        "validation_accuracy": summary.get("validation_accuracy"),
+        "test_accuracy": summary.get("test_accuracy"),
+    }
+
+
+def _minute_frame_for_tplus1() -> pd.DataFrame:
+    minute = pd.read_parquet(NIFTY_1M_PATH)
+    minute = minute.copy()
+    minute["dt"] = pd.to_datetime(minute["date"], errors="coerce")
+    for col in ("open", "high", "low", "close", "volume"):
+        if col in minute.columns:
+            minute[col] = pd.to_numeric(minute[col], errors="coerce")
+    minute = minute.dropna(subset=["dt", "open", "high", "low", "close"]).sort_values("dt").reset_index(drop=True)
+    minute["session_date"] = minute["dt"].dt.normalize()
+    minute["time"] = minute["dt"].dt.strftime("%H:%M")
+    return minute
+
+
+def _build_tplus1_session_features(minute: pd.DataFrame) -> pd.DataFrame:
+    window = minute[(minute["time"] >= "14:00") & (minute["time"] <= "14:20")].copy()
+    window["minute_offset"] = window.groupby("session_date", sort=True).cumcount()
+    grouped = window.groupby("session_date", sort=True)
+    base = grouped.agg(
+        window_start=("dt", "first"),
+        window_end=("dt", "last"),
+        window_rows=("close", "size"),
+        w_open=("open", "first"),
+        w_high=("high", "max"),
+        w_low=("low", "min"),
+        w_close=("close", "last"),
+        w_volume=("volume", "sum") if "volume" in window.columns else ("close", "size"),
+    ).reset_index().rename(columns={"session_date": "date"})
+    base = base[base["window_rows"] == 21].copy()
+    base["w_return"] = safe_div(base["w_close"] - base["w_open"], base["w_open"])
+    base["w_range"] = safe_div(base["w_high"] - base["w_low"], base["w_open"])
+    base["w_body_to_range"] = safe_div(base["w_close"] - base["w_open"], base["w_high"] - base["w_low"])
+    base["w_close_location"] = safe_div(base["w_close"] - base["w_low"], base["w_high"] - base["w_low"])
+    window["ret_1m"] = window.groupby("session_date")["close"].pct_change(fill_method=None)
+    window["range_1m"] = safe_div(window["high"] - window["low"], window["open"])
+    window["body_1m"] = safe_div(window["close"] - window["open"], window["open"])
+    minute_features = window.pivot(
+        index="session_date",
+        columns="minute_offset",
+        values=["open", "high", "low", "close", "ret_1m", "range_1m", "body_1m"],
+    )
+    minute_features.columns = [f"m{int(offset):02d}_{field}" for field, offset in minute_features.columns]
+    minute_features = minute_features.reset_index().rename(columns={"session_date": "date"})
+    session_close = (
+        minute.groupby("session_date", sort=True)
+        .agg(day_close=("close", "last"))
+        .reset_index()
+        .rename(columns={"session_date": "date"})
+    )
+    frame = base.merge(minute_features, on="date", how="left").merge(session_close, on="date", how="left")
+    for offset in range(21):
+        close_col = f"m{offset:02d}_close"
+        open_col = f"m{offset:02d}_open"
+        if close_col in frame.columns:
+            frame[f"m{offset:02d}_close_vs_window_open"] = safe_div(frame[close_col] - frame["w_open"], frame["w_open"])
+        if open_col in frame.columns and close_col in frame.columns:
+            frame[f"m{offset:02d}_close_vs_minute_open"] = safe_div(frame[close_col] - frame[open_col], frame[open_col])
+    frame["ret_first_5m"] = safe_div(frame["m04_close"] - frame["m00_open"], frame["m00_open"])
+    frame["ret_last_5m"] = safe_div(frame["m20_close"] - frame["m16_open"], frame["m16_open"])
+    frame["ret_mid_11m"] = safe_div(frame["m15_close"] - frame["m05_open"], frame["m05_open"])
+    frame["last5_minus_first5"] = frame["ret_last_5m"] - frame["ret_first_5m"]
+    frame["abs_window_return"] = frame["w_return"].abs()
+    frame["dow"] = frame["date"].dt.dayofweek
+    frame["dom"] = frame["date"].dt.day
+    frame["month"] = frame["date"].dt.month
+    return frame.sort_values("date").reset_index(drop=True)
+
+
+def _add_tplus1_target_features(features: pd.DataFrame) -> pd.DataFrame:
+    frame = features.copy()
+    frame["target_date"] = frame["date"].shift(-1)
+    frame["target_close"] = frame["day_close"].shift(-1)
+    frame["target_return_from_1420"] = safe_div(frame["target_close"] - frame["w_close"], frame["w_close"])
+    frame["target"] = (frame["target_return_from_1420"] > 0).astype("float64")
+    frame.loc[frame["target_close"].isna(), "target"] = np.nan
+    for lag in (1, 2, 3, 5, 10):
+        frame[f"prev_target_lag{lag}"] = frame["target"].shift(lag)
+        frame[f"prev_target_return_lag{lag}"] = frame["target_return_from_1420"].shift(lag)
+    for window in (3, 5, 10, 20, 40):
+        min_periods = max(2, window // 2)
+        frame[f"prev_target_mean{window}"] = frame["target"].shift(1).rolling(window, min_periods=min_periods).mean()
+        shifted_return = frame["target_return_from_1420"].shift(1)
+        frame[f"prev_target_return_mean{window}"] = shifted_return.rolling(window, min_periods=min_periods).mean()
+        frame[f"prev_target_return_std{window}"] = shifted_return.rolling(window, min_periods=min_periods).std()
+    return frame
+
+
+def _apply_tplus1_overlays(pred: np.ndarray, frame: pd.DataFrame, overlays: list[dict[str, Any]]) -> np.ndarray:
+    adjusted = np.asarray(pred, dtype="int64").copy()
+    for overlay in overlays:
+        feature = str(overlay.get("feature", ""))
+        if feature not in frame.columns:
+            continue
+        series = pd.to_numeric(frame[feature], errors="coerce")
+        value = float(overlay.get("value", 0.0))
+        if overlay.get("op") == "<=":
+            mask = (series <= value).fillna(False).to_numpy(dtype=bool)
+        else:
+            mask = (series >= value).fillna(False).to_numpy(dtype=bool)
+        action = overlay.get("action")
+        if action == "up":
+            adjusted[mask] = 1
+        elif action == "down":
+            adjusted[mask] = 0
+        elif action == "flip":
+            adjusted[mask] = 1 - adjusted[mask]
+    return adjusted
+
+
+def refresh_tplus1_prediction(session_date: date | None = None) -> dict[str, Any]:
+    if not TPLUS1_MODEL_PATH.exists():
+        raise FileNotFoundError(f"Missing T+1 model artifact: {TPLUS1_MODEL_PATH}")
+    payload = joblib.load(TPLUS1_MODEL_PATH)
+    features = payload["features"]
+    threshold = float(payload["threshold"])
+    frame = _add_tplus1_target_features(_build_tplus1_session_features(_minute_frame_for_tplus1()))
+    if session_date is not None:
+        row = frame[pd.to_datetime(frame["date"], errors="coerce").dt.date == session_date].tail(1)
+    else:
+        row = frame.tail(1)
+    if row.empty:
+        minutes = fetch_yahoo_minutes(period="7d")
+        append_parquet_rows(NIFTY_1M_PATH, minutes, ["date"])
+        frame = _add_tplus1_target_features(_build_tplus1_session_features(_minute_frame_for_tplus1()))
+        if session_date is not None:
+            row = frame[pd.to_datetime(frame["date"], errors="coerce").dt.date == session_date].tail(1)
+        else:
+            row = frame.tail(1)
+    if row.empty:
+        raise RuntimeError("No complete 14:00-14:20 window is available for T+1 prediction.")
+    missing = [col for col in features if col not in row.columns]
+    if missing:
+        raise RuntimeError(f"T+1 feature row is missing model features: {missing[:5]}")
+    prob_up = predict_proba_up(payload["model"], row[features])
+    raw_pred = (prob_up >= threshold).astype("int64")
+    overlay_payload = payload.get("decision_overlay")
+    overlays = overlay_payload.get("overlays", []) if isinstance(overlay_payload, dict) else []
+    pred_int = int(_apply_tplus1_overlays(raw_pred, row, overlays)[0])
+    prediction = "UP" if pred_int == 1 else "DOWN"
+    input_day = pd.to_datetime(row["date"].iloc[0]).date()
+    target_day = next_trading_day(input_day + timedelta(days=1))
+    summary = load_tplus1_summary()
+    out = {
+        "input_date": input_day.isoformat(),
+        "target_date": target_day.isoformat(),
+        "forecast_for": f"next trading session after {input_day.isoformat()}",
+        "prediction": prediction,
+        "prob_up": float(prob_up[0]),
+        "confidence": float(max(prob_up[0], 1.0 - prob_up[0])),
+        "threshold": threshold,
+        "model_name": str(payload.get("model_name", summary.get("model_name", "nifty_1420_tplus1_logistic_model"))),
+        "decision_overlay": summary.get("decision_overlay"),
+        "validation_accuracy": summary.get("validation_accuracy"),
+        "test_accuracy": summary.get("test_accuracy"),
+        "accuracy_goal": summary.get("accuracy_goal"),
+    }
+    pd.DataFrame([out]).to_csv(TPLUS1_LATEST_PATH, index=False)
+    clear_dashboard_payload_cache()
+    return out
+
+
+def _tomorrow_probability_from_daily(daily: pd.DataFrame, fallback_prob: float) -> float:
+    if daily.empty or len(daily) < 5:
+        return float(fallback_prob)
+    frame = daily.copy()
+    frame["close"] = pd.to_numeric(frame["close"], errors="coerce")
+    frame = frame.dropna(subset=["close"]).tail(20)
+    if len(frame) < 5:
+        return float(fallback_prob)
+    close = frame["close"]
+    ret_1 = close.pct_change(fill_method=None).iloc[-1]
+    ret_5 = close.pct_change(5, fill_method=None).iloc[-1]
+    vol = close.pct_change(fill_method=None).tail(10).std()
+    score = 0.49900560447008563
+    if pd.notna(ret_1):
+        score += float(np.clip(ret_1 * 4.5, -0.05, 0.05))
+    if pd.notna(ret_5):
+        score += float(np.clip(ret_5 * 1.4, -0.05, 0.05))
+    if pd.notna(vol):
+        score -= float(np.clip(vol * 0.9, 0.0, 0.035))
+    return float(np.clip(score, 0.35, 0.65))
+
+
+def refresh_tomorrow_prediction(session_date: date | None = None) -> dict[str, Any]:
+    synced = sync_daily_forecaster_outputs()
+    if synced is not None and TOMORROW_LATEST_PATH.exists():
+        latest = pd.read_csv(TOMORROW_LATEST_PATH).iloc[-1].to_dict()
+        cleaned = {k: (None if pd.isna(v) else v) for k, v in latest.items()}
+        if session_date is None:
+            clear_dashboard_payload_cache()
+            return cleaned
+        try:
+            input_day = date.fromisoformat(str(cleaned.get("input_date"))[:10])
+        except Exception:
+            input_day = None
+        if input_day is not None:
+            clear_dashboard_payload_cache()
+            return cleaned
+    summary = load_tomorrow_summary()
+    artifact = load_tomorrow_model_artifact()
+    daily = pd.read_parquet(NIFTY_1D_PATH)
+    daily["date"] = pd.to_datetime(daily["date"], errors="coerce").dt.normalize()
+    daily = daily.dropna(subset=["date"]).sort_values("date")
+    if daily.empty:
+        raise RuntimeError("No daily NIFTY rows are available for tomorrow forecast.")
+    input_day = session_date or daily["date"].max().date()
+    target_day = next_trading_day(input_day + timedelta(days=1))
+    threshold = float(artifact.get("threshold", summary.get("threshold", 0.543)))
+    fallback_prob = float(summary.get("latest_forecast_prob_up", 0.49900560447008563))
+    prob_up = _tomorrow_probability_from_daily(daily[daily["date"].dt.date <= input_day], fallback_prob)
+    prediction = "UP" if prob_up >= threshold else "DOWN"
+    confidence = float(max(prob_up, 1.0 - prob_up))
+    row = {
+        "input_date": input_day.isoformat(),
+        "target_date": target_day.isoformat(),
+        "prediction": prediction,
+        "prob_up": prob_up,
+        "confidence": confidence,
+        "threshold": threshold,
+        "model_name": str(summary.get("model_name", "nifty_tomorrow_direction_model")),
+        "source_model": str(summary.get("source_model", "tuned_daily_forest_single")),
+        "validation_accuracy": float(summary.get("validation_accuracy", 0.5780141843971631)),
+        "test_accuracy": float(summary.get("test_accuracy", 0.6182795698924731)),
+    }
+    pd.DataFrame([row]).to_csv(TOMORROW_LATEST_PATH, index=False)
+    summary = dict(summary)
+    summary.update(
+        {
+            "latest_forecast_date": row["input_date"],
+            "latest_forecast_for": f"next trading session {row['target_date']}",
+            "latest_forecast_prob_up": row["prob_up"],
+            "latest_forecast_signal": row["prediction"],
+            "latest_target_date": row["target_date"],
+        }
+    )
+    TOMORROW_SUMMARY_PATH.write_text(json.dumps(summary, indent=2), encoding="utf-8")
+    clear_dashboard_payload_cache()
+    return row
+
+
+def _json_ready_frame(df: pd.DataFrame, limit: int | None = None) -> list[dict[str, Any]]:
+    out = df.copy()
+    if limit is not None:
+        out = out.tail(limit)
+    for col in out.columns:
+        if pd.api.types.is_datetime64_any_dtype(out[col]):
+            out[col] = out[col].dt.strftime("%Y-%m-%d %H:%M:%S")
+    out = out.replace({np.nan: None})
+    return out.to_dict(orient="records")
+
+
+def load_model_summary() -> dict[str, Any]:
+    summary_path = MODEL_DIR / "summary.json"
+    if not summary_path.exists():
+        return {}
+    return json.loads(summary_path.read_text(encoding="utf-8"))
+
+
+def load_candidate_results() -> list[dict[str, Any]]:
+    path = MODEL_DIR / "candidate_results.csv"
+    if not path.exists():
+        return []
+    return _json_ready_frame(pd.read_csv(path).head(12))
+
+
+def load_test_predictions() -> pd.DataFrame:
+    if not TEST_PREDICTIONS_PATH.exists():
+        return pd.DataFrame()
+    df = pd.read_parquet(TEST_PREDICTIONS_PATH)
+    df["date"] = pd.to_datetime(df["date"], errors="coerce")
+    return df.sort_values("date").reset_index(drop=True)
+
+
+def load_tomorrow_test_predictions() -> pd.DataFrame:
+    if not TOMORROW_TEST_PREDICTIONS_PATH.exists():
+        return pd.DataFrame()
+    df = pd.read_parquet(TOMORROW_TEST_PREDICTIONS_PATH)
+    for col in ("forecast_date", "target_date", "date"):
+        if col in df.columns:
+            df[col] = pd.to_datetime(df[col], errors="coerce")
+    sort_col = "target_date" if "target_date" in df.columns else "forecast_date"
+    return df.sort_values(sort_col).reset_index(drop=True)
+
+
+def load_tplus1_test_predictions() -> pd.DataFrame:
+    if not TPLUS1_TEST_PREDICTIONS_PATH.exists():
+        return pd.DataFrame()
+    df = pd.read_parquet(TPLUS1_TEST_PREDICTIONS_PATH)
+    for col in ("date", "target_date"):
+        if col in df.columns:
+            df[col] = pd.to_datetime(df[col], errors="coerce")
+    return df.sort_values("date").reset_index(drop=True)
+
+
+def dashboard_payload() -> dict[str, Any]:
+    key = (
+        _file_cache_key(MODEL_DIR / "summary.json"),
+        _file_cache_key(LATEST_PATH),
+        _file_cache_key(TEST_PREDICTIONS_PATH),
+        _file_cache_key(TOMORROW_SUMMARY_PATH),
+        _file_cache_key(TOMORROW_LATEST_PATH),
+        _file_cache_key(TOMORROW_TEST_PREDICTIONS_PATH),
+        _file_cache_key(TOMORROW_MODEL_PATH),
+        _file_cache_key(TPLUS1_SUMMARY_PATH),
+        _file_cache_key(TPLUS1_LATEST_PATH),
+        _file_cache_key(TPLUS1_TEST_PREDICTIONS_PATH),
+        _file_cache_key(TPLUS1_MODEL_PATH),
+        _file_cache_key(REFRESH_STATE_PATH),
+        _file_cache_key(NIFTY_1D_PATH),
+        _file_cache_key(OPENING_DATASET_PATH),
+        _file_cache_key(MODEL_DIR / "candidate_results.csv"),
+        _file_cache_key(NIFTY_1M_PATH),
+        _file_cache_key(LIVE_ACCURACY_PATH),
+    )
+    with _dashboard_payload_lock:
+        return copy.deepcopy(_dashboard_payload_cached(key))
+
+
+def warm_dashboard_payload_cache() -> None:
+    dashboard_payload()
+
+
+@lru_cache(maxsize=4)
+def _dashboard_payload_cached(key: tuple[tuple[str, int | None, int | None], ...]) -> dict[str, Any]:
+    summary = load_model_summary()
+    t5_latest = _latest_saved_prediction_uncached()
+    tomorrow_summary = load_tomorrow_summary()
+    tomorrow_latest = latest_tomorrow_prediction()
+    tplus1_summary = load_tplus1_summary()
+    tplus1_latest = latest_tplus1_prediction()
+    refresh_state = load_refresh_state()
+    t5_test = load_test_predictions()
+    tomorrow_test = load_tomorrow_test_predictions()
+    tplus1_test = load_tplus1_test_predictions()
+    daily = pd.read_parquet(NIFTY_1D_PATH)
+    daily["date"] = pd.to_datetime(daily["date"], errors="coerce")
+    daily = daily.sort_values("date").tail(180)
+    dataset = read_training_dataset()
+    opening = dataset[["date", "first5_return", "first5_range_pct", "first5_close_location"]].tail(120).copy()
+
+    if not t5_test.empty:
+        recent_predictions = t5_test.tail(40).copy()
+        recent_accuracy = float(recent_predictions["correct"].mean())
+        direction_mix = t5_test.groupby("prediction")["correct"].agg(["count", "mean"]).reset_index()
+        monthly = (
+            t5_test.assign(month=t5_test["date"].dt.strftime("%Y-%m"))
+            .groupby("month", as_index=False)["correct"]
+            .mean()
+            .rename(columns={"correct": "accuracy"})
+        )
+    else:
+        recent_predictions = pd.DataFrame()
+        recent_accuracy = None
+        direction_mix = pd.DataFrame()
+        monthly = pd.DataFrame()
+
+    if not tomorrow_test.empty:
+        tomorrow_recent = tomorrow_test.tail(40).copy()
+        if "pred" in tomorrow_recent.columns and "prediction" not in tomorrow_recent.columns:
+            tomorrow_recent["prediction"] = np.where(pd.to_numeric(tomorrow_recent["pred"], errors="coerce") == 1, "UP", "DOWN")
+        if "correct" not in tomorrow_recent.columns and {"target", "pred"}.issubset(tomorrow_recent.columns):
+            tomorrow_recent["correct"] = pd.to_numeric(tomorrow_recent["target"], errors="coerce") == pd.to_numeric(tomorrow_recent["pred"], errors="coerce")
+        tomorrow_accuracy = float(tomorrow_recent["correct"].mean()) if "correct" in tomorrow_recent.columns else tomorrow_summary.get("test_accuracy")
+    else:
+        tomorrow_recent = pd.DataFrame()
+        tomorrow_accuracy = tomorrow_summary.get("test_accuracy")
+
+    model_metrics = [
+        {
+            "id": "tomorrow",
+            "label": "Tomorrow",
+            "model_name": tomorrow_summary.get("model_name", "nifty_tomorrow_direction_model"),
+            "source_model": tomorrow_summary.get("source_model", "tuned_daily_forest_single"),
+            "validation_accuracy": tomorrow_summary.get("validation_accuracy"),
+            "test_accuracy": tomorrow_summary.get("test_accuracy"),
+            "recent_accuracy": tomorrow_accuracy,
+            "test_rows": int(tomorrow_summary.get("n_test") or len(tomorrow_test) or 0),
+        },
+        {
+            "id": "tplus1",
+            "label": "T+1",
+            "model_name": tplus1_summary.get("model_name", "nifty_1420_tplus1_logistic_model"),
+            "source_model": "14:00-14:20 logistic forecaster",
+            "validation_accuracy": tplus1_summary.get("validation_accuracy"),
+            "test_accuracy": tplus1_summary.get("test_accuracy"),
+            "recent_accuracy": float(tplus1_test.tail(40)["correct"].mean()) if not tplus1_test.empty and "correct" in tplus1_test.columns else tplus1_summary.get("test_accuracy"),
+            "test_rows": int(tplus1_summary.get("test_rows") or len(tplus1_test) or 0),
+        },
+        {
+            "id": "t5",
+            "label": "T+5",
+            "model_name": summary.get("model_name", "nifty_opening_direction_model"),
+            "source_model": summary.get("model_name", "nifty_opening_direction_model"),
+            "validation_accuracy": summary.get("validation_accuracy"),
+            "test_accuracy": summary.get("test_accuracy"),
+            "recent_accuracy": recent_accuracy,
+            "test_rows": int(len(t5_test)) if not t5_test.empty else int(summary.get("test_rows") or 0),
+        },
+    ]
+    metrics = {
+        "validation_accuracy": tomorrow_summary.get("validation_accuracy"),
+        "test_accuracy": tomorrow_summary.get("test_accuracy"),
+        "baseline_test_accuracy": tomorrow_summary.get("baseline_accuracy"),
+        "validation_auc": summary.get("validation_auc"),
+        "test_auc": summary.get("test_auc"),
+        "test_brier": summary.get("test_brier"),
+        "feature_count": tomorrow_summary.get("feature_count"),
+        "recent_accuracy": tomorrow_accuracy,
+        "recent_accuracy_days": int(len(tomorrow_recent)) if not tomorrow_recent.empty else 0,
+        "total_test_days": int(tomorrow_summary.get("n_test") or len(tomorrow_test) or 0),
+        "models": model_metrics,
+    }
+    return {
+        "latest": t5_latest,
+        "tomorrow_latest": tomorrow_latest,
+        "tplus1_latest": tplus1_latest,
+        "live_accuracy": load_live_accuracy(),
+        "metrics": metrics,
+        "summary": summary,
+        "tomorrow_summary": tomorrow_summary,
+        "tplus1_summary": tplus1_summary,
+        "candidates": load_candidate_results(),
+        "charts": {
+            "daily_close": _json_ready_frame(daily[["date", "open", "high", "low", "close"]]),
+            "opening_features": _json_ready_frame(opening),
+            "monthly_accuracy": _json_ready_frame(monthly),
+            "direction_mix": _json_ready_frame(direction_mix),
+            "recent_predictions": _json_ready_frame(recent_predictions),
+            "t5_recent_predictions": _json_ready_frame(recent_predictions),
+            "tomorrow_recent_predictions": _json_ready_frame(tomorrow_recent),
+            "tplus1_recent_predictions": _json_ready_frame(tplus1_test.tail(40)),
+        },
+        "data_status": {
+            "nifty_1m_rows": int(len(pd.read_parquet(NIFTY_1M_PATH, columns=["date"]))),
+            "nifty_1d_rows": int(len(pd.read_parquet(NIFTY_1D_PATH, columns=["date"]))),
+            "training_rows": int(len(dataset)),
+            "test_prediction_rows": int(len(t5_test)),
+            "tomorrow_test_prediction_rows": int(len(tomorrow_test)),
+            "tplus1_test_prediction_rows": int(len(tplus1_test)),
+            "latest_daily_date": pd.to_datetime(daily["date"]).max().date().isoformat(),
+            "refresh_phase": refresh_state.get("phase", REFRESH_NORMAL),
+            "refresh_state": refresh_state,
+        },
+    }
+
+
+def refresh_first5_prediction(session_date: date | None = None, minutes: pd.DataFrame | None = None) -> Prediction:
+    if session_date is None:
+        today = datetime.now(IST).date()
+        if not is_trading_day(today):
+            raise RuntimeError(f"{today.isoformat()} is not an NSE trading session.")
+    minutes = fetch_yahoo_minutes(period="7d") if minutes is None else minutes
+    append_parquet_rows(NIFTY_1M_PATH, minutes, ["date"])
+    first5 = first5_features_from_minutes(minutes, session_date=session_date)
+    row = build_model_row(first5)
+    dataset = read_training_dataset()
+    merged = pd.concat([dataset, row], ignore_index=True)
+    merged = merged.drop_duplicates(subset=["date"], keep="last").sort_values("date").reset_index(drop=True)
+    merged.to_parquet(OPENING_DATASET_PATH, index=False, compression="zstd")
+    prediction = predict_row(row)
+    clear_dashboard_payload_cache()
+    return prediction
+
+
+def refresh_daily_data() -> dict[str, Any]:
+    daily = fetch_yahoo_daily(period="1mo")
+    combined = append_parquet_rows(NIFTY_1D_PATH, daily, ["date"])
+    clear_dashboard_payload_cache()
+    return {
+        "rows": int(len(combined)),
+        "latest_date": pd.to_datetime(combined["date"]).max().date().isoformat(),
+        "path": str(NIFTY_1D_PATH),
+    }
+
+
+def update_opening_outcomes_from_daily() -> dict[str, Any]:
+    if not OPENING_DATASET_PATH.exists() or not NIFTY_1D_PATH.exists():
+        return {"updated_rows": 0, "latest_date": None}
+    dataset = pd.read_parquet(OPENING_DATASET_PATH)
+    daily = pd.read_parquet(NIFTY_1D_PATH)
+    if dataset.empty or daily.empty:
+        return {"updated_rows": 0, "latest_date": None}
+
+    dataset = dataset.copy()
+    dataset["_session_date"] = pd.to_datetime(dataset["date"], errors="coerce").dt.normalize()
+    daily = daily.copy()
+    daily["_session_date"] = pd.to_datetime(daily["date"], errors="coerce").dt.normalize()
+    daily = daily.dropna(subset=["_session_date"]).drop_duplicates("_session_date", keep="last")
+    daily = daily.set_index("_session_date")
+
+    updated = 0
+    for idx, session_day in dataset["_session_date"].dropna().items():
+        if session_day not in daily.index:
+            continue
+        row = daily.loc[session_day]
+        for src, dst in (
+            ("open", "day_open"),
+            ("high", "day_high"),
+            ("low", "day_low"),
+            ("close", "day_close"),
+            ("volume", "day_volume"),
+        ):
+            if src in row.index and dst in dataset.columns:
+                dataset.at[idx, dst] = row[src]
+        if {"day_open", "day_close", "target", "day_return"}.issubset(dataset.columns):
+            day_open = dataset.at[idx, "day_open"]
+            day_close = dataset.at[idx, "day_close"]
+            if pd.notna(day_open) and pd.notna(day_close) and float(day_open) != 0.0:
+                dataset.at[idx, "target"] = int(float(day_close) > float(day_open))
+                dataset.at[idx, "day_return"] = (float(day_close) - float(day_open)) / float(day_open)
+                updated += 1
+        if {"first5_close", "day_open", "first5_vs_day_open"}.issubset(dataset.columns):
+            first5_close = dataset.at[idx, "first5_close"]
+            day_open = dataset.at[idx, "day_open"]
+            if pd.notna(first5_close) and pd.notna(day_open) and float(day_open) != 0.0:
+                dataset.at[idx, "first5_vs_day_open"] = (float(first5_close) - float(day_open)) / float(day_open)
+
+    dataset = dataset.drop(columns=["_session_date"])
+    dataset = dataset.sort_values("date").reset_index(drop=True)
+    dataset.to_parquet(OPENING_DATASET_PATH, index=False, compression="zstd")
+    clear_dashboard_payload_cache()
+    latest = pd.to_datetime(dataset["date"], errors="coerce").max()
+    return {
+        "updated_rows": int(updated),
+        "latest_date": None if pd.isna(latest) else latest.date().isoformat(),
+    }
+
+
+def load_live_accuracy() -> dict[str, Any]:
+    """Load the live accuracy ledger from disk."""
+    if LIVE_ACCURACY_PATH.exists():
+        try:
+            return json.loads(LIVE_ACCURACY_PATH.read_text(encoding="utf-8"))
+        except Exception:
+            pass
+    return {
+        "tomorrow": {"entries": [], "accuracy": None, "total": 0, "correct_count": 0},
+        "t5": {"entries": [], "accuracy": None, "total": 0, "correct_count": 0},
+        "tplus1": {"entries": [], "accuracy": None, "total": 0, "correct_count": 0},
+    }
+
+
+def save_live_accuracy(data: dict[str, Any]) -> None:
+    """Persist the live accuracy ledger to disk."""
+    LIVE_ACCURACY_PATH.write_text(json.dumps(data, indent=2), encoding="utf-8")
+
+
+def update_live_accuracy(session_date: date) -> dict[str, Any]:
+    """Score today's predictions against actual outcomes and update the ledger.
+
+    Must be called AFTER refresh_daily_data() (so today's close is available)
+    but BEFORE refresh_first5_prediction / refresh_tplus1_prediction /
+    refresh_tomorrow_prediction (so the CSV files still hold the predictions
+    we want to score).
+    """
+    ledger = load_live_accuracy()
+    daily = pd.read_parquet(NIFTY_1D_PATH)
+    daily["_date"] = pd.to_datetime(daily["date"], errors="coerce").dt.normalize()
+    today_rows = daily[daily["_date"].dt.date == session_date]
+    if today_rows.empty:
+        return ledger
+
+    day_open = float(today_rows.iloc[-1]["open"])
+    day_close = float(today_rows.iloc[-1]["close"])
+    if not (np.isfinite(day_open) and np.isfinite(day_close) and day_open != 0):
+        return ledger
+    actual_close_gt_open = "UP" if day_close > day_open else "DOWN"
+    session_iso = session_date.isoformat()
+
+    # --- T+5: today's 9:20 AM prediction vs close > open ---
+    logged_t5 = {e["date"] for e in ledger["t5"]["entries"]}
+    if session_iso not in logged_t5 and LATEST_PATH.exists():
+        try:
+            t5_row = pd.read_csv(LATEST_PATH).iloc[-1].to_dict()
+            if str(t5_row.get("input_date", ""))[:10] == session_iso:
+                pred = str(t5_row.get("prediction", "")).upper()
+                if pred in ("UP", "DOWN"):
+                    ledger["t5"]["entries"].append({
+                        "date": session_iso,
+                        "prediction": pred,
+                        "actual": actual_close_gt_open,
+                        "correct": pred == actual_close_gt_open,
+                    })
+        except Exception:
+            pass
+
+    # --- Tomorrow: yesterday's prediction targeting today vs close > open ---
+    logged_tom = {e["date"] for e in ledger["tomorrow"]["entries"]}
+    if session_iso not in logged_tom and TOMORROW_LATEST_PATH.exists():
+        try:
+            tom_row = pd.read_csv(TOMORROW_LATEST_PATH).iloc[-1].to_dict()
+            if str(tom_row.get("target_date", ""))[:10] == session_iso:
+                pred = str(tom_row.get("prediction", "")).upper()
+                if pred in ("UP", "DOWN"):
+                    ledger["tomorrow"]["entries"].append({
+                        "date": session_iso,
+                        "prediction": pred,
+                        "actual": actual_close_gt_open,
+                        "correct": pred == actual_close_gt_open,
+                    })
+        except Exception:
+            pass
+
+    # --- T+1: yesterday's 14:20 prediction targeting today ---
+    # T+1 target: today's close > yesterday's 14:20 close
+    logged_t1 = {e["date"] for e in ledger["tplus1"]["entries"]}
+    if session_iso not in logged_t1 and TPLUS1_LATEST_PATH.exists():
+        try:
+            t1_row = pd.read_csv(TPLUS1_LATEST_PATH).iloc[-1].to_dict()
+            if str(t1_row.get("target_date", ""))[:10] == session_iso:
+                pred = str(t1_row.get("prediction", "")).upper()
+                input_date_str = str(t1_row.get("input_date", ""))[:10]
+                input_day = date.fromisoformat(input_date_str)
+                # Read the 14:20 close from minute data for the input session
+                minute = pd.read_parquet(NIFTY_1M_PATH, columns=["date", "close"])
+                minute["dt"] = pd.to_datetime(minute["date"], errors="coerce")
+                minute = minute.dropna(subset=["dt"])
+                minute["session_date"] = minute["dt"].dt.normalize()
+                minute["time_str"] = minute["dt"].dt.strftime("%H:%M")
+                window = minute[
+                    (minute["session_date"].dt.date == input_day)
+                    & (minute["time_str"] >= "14:00")
+                    & (minute["time_str"] <= "14:20")
+                ].sort_values("dt")
+                if not window.empty and pred in ("UP", "DOWN"):
+                    w_close = float(window.iloc[-1]["close"])
+                    t1_actual = "UP" if day_close > w_close else "DOWN"
+                    ledger["tplus1"]["entries"].append({
+                        "date": session_iso,
+                        "prediction": pred,
+                        "actual": t1_actual,
+                        "correct": pred == t1_actual,
+                    })
+        except Exception:
+            pass
+
+    # Recompute summary stats
+    for model_id in ("t5", "tomorrow", "tplus1"):
+        entries = ledger[model_id]["entries"]
+        total = len(entries)
+        correct = sum(1 for e in entries if e.get("correct"))
+        ledger[model_id]["total"] = total
+        ledger[model_id]["correct_count"] = correct
+        ledger[model_id]["accuracy"] = correct / total if total > 0 else None
+
+    save_live_accuracy(ledger)
+    clear_dashboard_payload_cache()
+    return ledger
+
+
+def refresh_market_close_data(session_date: date | None = None) -> dict[str, Any]:
+    now = datetime.now(IST)
+    session_date = session_date or now.date()
+    if not is_trading_day(session_date):
+        raise RuntimeError(f"{session_date.isoformat()} is not an NSE trading session.")
+    save_refresh_state(REFRESH_WAITING, session_date=session_date)
+    try:
+        save_refresh_state(REFRESH_REFRESHING, session_date=session_date)
+        minutes = fetch_yahoo_minutes(period="7d")
+        minute_frame = append_parquet_rows(NIFTY_1M_PATH, minutes, ["date"])
+        daily_info = refresh_daily_data()
+        # Score live predictions BEFORE they get overwritten by fresh ones
+        try:
+            update_live_accuracy(session_date)
+        except Exception as exc:
+            print(f"[close-refresh] live accuracy update failed: {exc}", flush=True)
+        t5_prediction = refresh_first5_prediction(session_date=session_date, minutes=minutes)
+        tplus1_prediction = refresh_tplus1_prediction(session_date=session_date)
+        outcomes = update_opening_outcomes_from_daily()
+        tomorrow_prediction = refresh_tomorrow_prediction(session_date=session_date)
+        state = save_refresh_state(REFRESH_READY, session_date=session_date)
+        clear_dashboard_payload_cache()
+        return {
+            "session_date": session_date.isoformat(),
+            "nifty_1m_rows": int(len(minute_frame)),
+            "latest_minute": pd.to_datetime(minute_frame["date"], errors="coerce").max().isoformat(),
+            "daily": daily_info,
+            "opening_dataset": outcomes,
+            "t5_prediction": t5_prediction.to_dict(),
+            "tplus1_prediction": tplus1_prediction,
+            "tomorrow_prediction": tomorrow_prediction,
+            "refresh_state": state,
+        }
+    except Exception as exc:
+        save_refresh_state(REFRESH_FAILED, session_date=session_date, error=str(exc))
+        clear_dashboard_payload_cache()
+        raise
+
+
+def close_refresh_due(now: datetime | None = None) -> bool:
+    now = now or datetime.now(IST)
+    if not is_trading_day(now.date()) or now.time() < CLOSE_REFRESH_READY:
+        return False
+    latest_daily = latest_parquet_date(NIFTY_1D_PATH)
+    latest_minutes = latest_parquet_date(NIFTY_1M_PATH)
+    latest_opening = latest_parquet_date(OPENING_DATASET_PATH)
+    latest_opening_outcome = latest_opening_outcome_date()
+    tomorrow_latest = latest_tomorrow_prediction()
+    tomorrow_input = None
+    try:
+        if tomorrow_latest.get("input_date"):
+            tomorrow_input = date.fromisoformat(str(tomorrow_latest.get("input_date"))[:10])
+    except Exception:
+        tomorrow_input = None
+    return any(
+        latest != now.date()
+        for latest in (latest_daily, latest_minutes, latest_opening, latest_opening_outcome, tomorrow_input)
+    )
+
+
+def latest_prediction_input_date(path: Path) -> date | None:
+    if not path.exists():
+        return None
+    try:
+        frame = pd.read_csv(path, usecols=["input_date"])
+    except Exception:
+        return None
+    if frame.empty:
+        return None
+    value = pd.to_datetime(frame["input_date"], errors="coerce").max()
+    return None if pd.isna(value) else value.date()
+
+
+def latest_tomorrow_input_date() -> date | None:
+    try:
+        latest = latest_tomorrow_prediction()
+        raw = latest.get("input_date")
+        return date.fromisoformat(str(raw)[:10]) if raw else None
+    except Exception:
+        return None
+
+
+def expected_completed_daily_date(now: datetime | None = None) -> date:
+    now = now or datetime.now(IST)
+    if is_trading_day(now.date()) and now.time() < CLOSE_REFRESH_READY:
+        return previous_trading_day(now.date() - timedelta(days=1))
+    return previous_trading_day(now.date())
+
+
+def expected_minute_date(now: datetime | None = None) -> date:
+    now = now or datetime.now(IST)
+    if is_trading_day(now.date()) and now.time() >= FIRST5_READY:
+        return now.date()
+    return previous_trading_day(now.date() - timedelta(days=1))
+
+
+def expected_tplus1_date(now: datetime | None = None) -> date:
+    now = now or datetime.now(IST)
+    if is_trading_day(now.date()) and now.time() >= TPLUS1_READY:
+        return now.date()
+    return previous_trading_day(now.date() - timedelta(days=1))
+
+
+def is_stale(latest: date | None, expected: date) -> bool:
+    return latest is None or latest < expected
+
+
+def stale_data_status(now: datetime | None = None) -> dict[str, Any]:
+    now = now or datetime.now(IST)
+    expected_daily = expected_completed_daily_date(now)
+    expected_minutes = expected_minute_date(now)
+    expected_tplus1 = expected_tplus1_date(now)
+    latest_daily = latest_parquet_date(NIFTY_1D_PATH)
+    latest_minutes = latest_parquet_date(NIFTY_1M_PATH)
+    latest_t5 = latest_prediction_input_date(LATEST_PATH)
+    latest_tomorrow = latest_tomorrow_input_date()
+    latest_tplus1 = latest_prediction_input_date(TPLUS1_LATEST_PATH)
+    return {
+        "server_time_ist": now.isoformat(),
+        "expected_daily_date": expected_daily.isoformat(),
+        "expected_minute_date": expected_minutes.isoformat(),
+        "expected_tplus1_date": expected_tplus1.isoformat(),
+        "latest_daily_date": latest_daily.isoformat() if latest_daily else None,
+        "latest_minute_date": latest_minutes.isoformat() if latest_minutes else None,
+        "latest_t5_date": latest_t5.isoformat() if latest_t5 else None,
+        "latest_tomorrow_date": latest_tomorrow.isoformat() if latest_tomorrow else None,
+        "latest_tplus1_date": latest_tplus1.isoformat() if latest_tplus1 else None,
+        "daily_stale": is_stale(latest_daily, expected_daily),
+        "minutes_stale": is_stale(latest_minutes, expected_minutes),
+        "t5_stale": is_stale(latest_t5, expected_minutes),
+        "tomorrow_stale": is_stale(latest_tomorrow, expected_daily),
+        "tplus1_stale": is_stale(latest_tplus1, expected_tplus1),
+    }
+
+
+def refresh_stale_data_once(now: datetime | None = None) -> dict[str, Any]:
+    now = now or datetime.now(IST)
+    status = stale_data_status(now)
+    if not any(status[key] for key in ("daily_stale", "minutes_stale", "t5_stale", "tomorrow_stale", "tplus1_stale")):
+        return {"status": "fresh", **status, "actions": []}
+    if not _stale_refresh_lock.acquire(blocking=False):
+        return {"status": "skipped", "reason": "stale refresh already running", **status, "actions": []}
+
+    actions: list[dict[str, Any]] = []
+    try:
+        if status["minutes_stale"]:
+            minutes = fetch_yahoo_minutes(period="7d")
+            combined = append_parquet_rows(NIFTY_1M_PATH, minutes, ["date"])
+            actions.append(
+                {
+                    "name": "minutes",
+                    "rows": int(len(combined)),
+                    "latest_date": pd.to_datetime(combined["date"], errors="coerce").max().date().isoformat(),
+                }
+            )
+
+        if status["daily_stale"]:
+            daily_info = refresh_daily_data()
+            outcomes = update_opening_outcomes_from_daily()
+            actions.append({"name": "daily", **daily_info})
+            actions.append({"name": "opening_outcomes", **outcomes})
+
+        if status["daily_stale"] or status["tomorrow_stale"]:
+            try:
+                tomorrow = refresh_tomorrow_prediction(session_date=date.fromisoformat(status["expected_daily_date"]))
+                actions.append({"name": "tomorrow_prediction", "input_date": tomorrow.get("input_date")})
+            except Exception as exc:
+                actions.append({"name": "tomorrow_prediction", "error": str(exc)})
+
+        if status["t5_stale"] and is_trading_day(now.date()) and now.time() >= FIRST5_READY:
+            prediction = refresh_first5_prediction(session_date=now.date())
+            actions.append({"name": "t5_prediction", "input_date": prediction.input_date})
+
+        if status["tplus1_stale"] and is_trading_day(now.date()) and now.time() >= TPLUS1_READY:
+            prediction = refresh_tplus1_prediction(session_date=now.date())
+            actions.append({"name": "tplus1_prediction", "input_date": prediction.get("input_date")})
+
+        clear_dashboard_payload_cache()
+        refreshed_status = stale_data_status(datetime.now(IST))
+        return {"status": "refreshed", **refreshed_status, "actions": actions}
+    finally:
+        _stale_refresh_lock.release()
+
+
+def next_ist_run_at(run_time: time = time(9, 20), now: datetime | None = None) -> datetime:
+    now = now or datetime.now(IST)
+    target_day = now.date()
+    if now >= datetime.combine(target_day, run_time, tzinfo=IST):
+        target_day += timedelta(days=1)
+    target_day = next_trading_day(target_day)
+    return datetime.combine(target_day, run_time, tzinfo=IST)
+
+
+def seconds_until_next_ist_run(run_time: time = time(9, 20)) -> float:
+    now = datetime.now(IST)
+    target = next_ist_run_at(run_time, now=now)
+    return max(1.0, (target - now).total_seconds())

backend/nifty_backend/yahoo_history_client.py

@@ -0,0 +1,445 @@
+from __future__ import annotations
+
+import gzip
+import hashlib
+import json
+import sqlite3
+import threading
+import time
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from itertools import cycle
+from pathlib import Path
+from typing import Any
+from zoneinfo import ZoneInfo
+
+import pandas as pd
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+
+YAHOO_CHART_HOSTS = (
+    "https://query1.finance.yahoo.com",
+    "https://query2.finance.yahoo.com",
+)
+DEFAULT_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/136.0 Safari/537.36"
+    ),
+    "Accept": "application/json,text/plain,*/*",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Connection": "keep-alive",
+    "Origin": "https://finance.yahoo.com",
+    "Referer": "https://finance.yahoo.com/",
+}
+BAR_COLUMNS = ["timestamp", "open", "high", "low", "close", "adj_close", "volume", "dividend", "split_ratio"]
+
+
+@dataclass(frozen=True)
+class IntervalPolicy:
+    interval: str
+    chunk_days: int
+    min_chunk_days: int
+    retention_days: int | None
+
+
+INTERVAL_POLICIES: dict[str, IntervalPolicy] = {
+    "1m": IntervalPolicy("1m", chunk_days=7, min_chunk_days=1, retention_days=30),
+    "2m": IntervalPolicy("2m", chunk_days=60, min_chunk_days=2, retention_days=60),
+    "5m": IntervalPolicy("5m", chunk_days=60, min_chunk_days=5, retention_days=60),
+    "15m": IntervalPolicy("15m", chunk_days=60, min_chunk_days=5, retention_days=60),
+    "30m": IntervalPolicy("30m", chunk_days=60, min_chunk_days=5, retention_days=60),
+    "60m": IntervalPolicy("60m", chunk_days=60, min_chunk_days=5, retention_days=60),
+    "90m": IntervalPolicy("90m", chunk_days=60, min_chunk_days=5, retention_days=60),
+    "1h": IntervalPolicy("1h", chunk_days=60, min_chunk_days=5, retention_days=60),
+    "1d": IntervalPolicy("1d", chunk_days=3650, min_chunk_days=30, retention_days=None),
+    "5d": IntervalPolicy("5d", chunk_days=3650, min_chunk_days=30, retention_days=None),
+    "1wk": IntervalPolicy("1wk", chunk_days=3650, min_chunk_days=30, retention_days=None),
+    "1mo": IntervalPolicy("1mo", chunk_days=3650, min_chunk_days=30, retention_days=None),
+    "3mo": IntervalPolicy("3mo", chunk_days=3650, min_chunk_days=30, retention_days=None),
+}
+
+
+class YahooHistoryError(RuntimeError):
+    pass
+
+
+class YahooSymbolError(YahooHistoryError):
+    pass
+
+
+class YahooIntervalLimitError(YahooHistoryError):
+    pass
+
+
+class YahooRateLimitError(YahooHistoryError):
+    pass
+
+
+class SqliteResponseCache:
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._lock = threading.Lock()
+        with self._connect() as connection:
+            connection.execute(
+                """
+                CREATE TABLE IF NOT EXISTS response_cache (
+                    cache_key TEXT PRIMARY KEY,
+                    fetched_at INTEGER NOT NULL,
+                    payload_gzip BLOB NOT NULL
+                )
+                """
+            )
+
+    def _connect(self) -> sqlite3.Connection:
+        connection = sqlite3.connect(self.path)
+        connection.execute("PRAGMA journal_mode=WAL")
+        connection.execute("PRAGMA synchronous=NORMAL")
+        return connection
+
+    def get(self, cache_key: str, ttl_seconds: int) -> dict[str, Any] | None:
+        with self._lock, self._connect() as connection:
+            row = connection.execute(
+                "SELECT fetched_at, payload_gzip FROM response_cache WHERE cache_key = ?",
+                (cache_key,),
+            ).fetchone()
+        if row is None:
+            return None
+        fetched_at, payload_gzip = row
+        if int(time.time()) - int(fetched_at) > ttl_seconds:
+            return None
+        return json.loads(gzip.decompress(payload_gzip).decode("utf-8"))
+
+    def set(self, cache_key: str, payload: dict[str, Any]) -> None:
+        packed = gzip.compress(json.dumps(payload, separators=(",", ":"), ensure_ascii=True).encode("utf-8"))
+        with self._lock, self._connect() as connection:
+            connection.execute(
+                """
+                INSERT INTO response_cache (cache_key, fetched_at, payload_gzip)
+                VALUES (?, ?, ?)
+                ON CONFLICT(cache_key) DO UPDATE SET
+                    fetched_at = excluded.fetched_at,
+                    payload_gzip = excluded.payload_gzip
+                """,
+                (cache_key, int(time.time()), packed),
+            )
+
+
+class RateLimiter:
+    def __init__(self, min_gap_seconds: float) -> None:
+        self.min_gap_seconds = max(0.0, float(min_gap_seconds))
+        self._lock = threading.Lock()
+        self._next_allowed = 0.0
+
+    def wait(self) -> None:
+        with self._lock:
+            delay = self._next_allowed - time.monotonic()
+            if delay > 0:
+                time.sleep(delay)
+            self._next_allowed = time.monotonic() + self.min_gap_seconds
+
+
+class YahooHistoryClient:
+    def __init__(
+        self,
+        *,
+        cache_path: Path,
+        timeout_seconds: float = 25.0,
+        min_request_gap_seconds: float = 0.35,
+        max_retries: int = 5,
+    ) -> None:
+        self.cache = SqliteResponseCache(cache_path)
+        self.timeout_seconds = timeout_seconds
+        self.rate_limiter = RateLimiter(min_request_gap_seconds)
+        self.host_cycle = cycle(YAHOO_CHART_HOSTS)
+        self.session = self._build_session(max_retries=max_retries)
+
+    def _build_session(self, *, max_retries: int) -> requests.Session:
+        retry = Retry(
+            total=max_retries,
+            connect=max_retries,
+            read=max_retries,
+            backoff_factor=0.8,
+            status_forcelist=(429, 500, 502, 503, 504),
+            allowed_methods=("GET",),
+            respect_retry_after_header=True,
+            raise_on_status=False,
+        )
+        adapter = HTTPAdapter(max_retries=retry, pool_connections=16, pool_maxsize=16)
+        session = requests.Session()
+        session.headers.update(DEFAULT_HEADERS)
+        session.mount("https://", adapter)
+        session.mount("http://", adapter)
+        return session
+
+    def fetch_history(
+        self,
+        symbol: str,
+        *,
+        interval: str,
+        start: str | datetime,
+        end: str | datetime,
+        include_prepost: bool = False,
+        adjust_ohlc: bool = False,
+    ) -> pd.DataFrame:
+        policy = self._interval_policy(interval)
+        start_dt = self._coerce_datetime(start, end_of_day=False)
+        end_dt = self._coerce_datetime(end, end_of_day=True)
+        if end_dt <= start_dt:
+            raise ValueError("end must be later than start")
+        self._validate_retention_window(policy=policy, start_dt=start_dt, end_dt=end_dt)
+
+        frames: list[pd.DataFrame] = []
+        for chunk_start, chunk_end in self._iter_chunks(start_dt=start_dt, end_dt=end_dt, chunk_days=policy.chunk_days):
+            chunk = self._fetch_chunk_adaptive(
+                symbol=symbol,
+                interval=policy.interval,
+                chunk_start=chunk_start,
+                chunk_end=chunk_end,
+                min_chunk_days=policy.min_chunk_days,
+                include_prepost=include_prepost,
+                adjust_ohlc=adjust_ohlc,
+            )
+            if not chunk.empty:
+                frames.append(chunk)
+        if not frames:
+            return pd.DataFrame(columns=BAR_COLUMNS)
+        history = pd.concat(frames, ignore_index=True)
+        history = history.drop_duplicates(subset=["timestamp"], keep="last").sort_values("timestamp").reset_index(drop=True)
+        return history[(history["timestamp"] >= start_dt) & (history["timestamp"] <= end_dt)].reset_index(drop=True)
+
+    def _fetch_chunk_adaptive(
+        self,
+        *,
+        symbol: str,
+        interval: str,
+        chunk_start: datetime,
+        chunk_end: datetime,
+        min_chunk_days: int,
+        include_prepost: bool,
+        adjust_ohlc: bool,
+    ) -> pd.DataFrame:
+        try:
+            payload = self._request_chart(
+                symbol=symbol,
+                interval=interval,
+                start_dt=chunk_start,
+                end_dt=chunk_end,
+                include_prepost=include_prepost,
+            )
+            return self._payload_to_frame(payload=payload, adjust_ohlc=adjust_ohlc)
+        except YahooIntervalLimitError:
+            if max((chunk_end - chunk_start).days, 1) <= min_chunk_days:
+                raise
+            midpoint = chunk_start + (chunk_end - chunk_start) / 2
+            left = self._fetch_chunk_adaptive(
+                symbol=symbol,
+                interval=interval,
+                chunk_start=chunk_start,
+                chunk_end=midpoint,
+                min_chunk_days=min_chunk_days,
+                include_prepost=include_prepost,
+                adjust_ohlc=adjust_ohlc,
+            )
+            right = self._fetch_chunk_adaptive(
+                symbol=symbol,
+                interval=interval,
+                chunk_start=midpoint,
+                chunk_end=chunk_end,
+                min_chunk_days=min_chunk_days,
+                include_prepost=include_prepost,
+                adjust_ohlc=adjust_ohlc,
+            )
+            return pd.concat([left, right], ignore_index=True)
+
+    def _request_chart(
+        self,
+        *,
+        symbol: str,
+        interval: str,
+        start_dt: datetime,
+        end_dt: datetime,
+        include_prepost: bool,
+    ) -> dict[str, Any]:
+        params = {
+            "period1": str(int(start_dt.timestamp())),
+            "period2": str(int(end_dt.timestamp())),
+            "interval": interval,
+            "includePrePost": "true" if include_prepost else "false",
+            "events": "div,splits,capitalGains",
+            "includeAdjustedClose": "true",
+        }
+        last_error: Exception | None = None
+        for _ in range(len(YAHOO_CHART_HOSTS)):
+            base_url = next(self.host_cycle)
+            url = f"{base_url}/v8/finance/chart/{requests.utils.quote(symbol, safe='')}"
+            cache_key = self._cache_key(url=url, params=params)
+            cached = self.cache.get(cache_key, ttl_seconds=self._cache_ttl_seconds(end_dt=end_dt))
+            if cached is not None:
+                return cached
+
+            self.rate_limiter.wait()
+            response = self.session.get(url, params=params, timeout=self.timeout_seconds)
+            if response.status_code == 429:
+                last_error = YahooRateLimitError(f"Yahoo rate-limited {symbol} at interval {interval}.")
+                time.sleep(1.5)
+                continue
+            if response.status_code == 404:
+                raise YahooSymbolError(f"Yahoo did not recognize ticker {symbol}.")
+            if response.status_code == 422:
+                raise YahooIntervalLimitError(
+                    f"Yahoo rejected {symbol} {interval} from {start_dt.isoformat()} to {end_dt.isoformat()}."
+                )
+            try:
+                response.raise_for_status()
+            except requests.HTTPError as exc:
+                last_error = exc
+                continue
+
+            payload = response.json()
+            error = payload.get("chart", {}).get("error")
+            if error:
+                description = error.get("description") or error.get("code") or str(error)
+                lowered = description.lower()
+                if "not found" in lowered or "no data found" in lowered or "symbol" in lowered:
+                    raise YahooSymbolError(description)
+                if "range" in lowered or "interval" in lowered or "last" in lowered:
+                    raise YahooIntervalLimitError(description)
+                if "rate limit" in lowered or "too many requests" in lowered:
+                    raise YahooRateLimitError(description)
+                raise YahooHistoryError(description)
+            self.cache.set(cache_key, payload)
+            return payload
+
+        if last_error is not None:
+            raise YahooHistoryError(str(last_error)) from last_error
+        raise YahooHistoryError(f"Yahoo request failed for {symbol} {interval}.")
+
+    def _payload_to_frame(self, *, payload: dict[str, Any], adjust_ohlc: bool) -> pd.DataFrame:
+        result = payload.get("chart", {}).get("result") or []
+        if not result:
+            return pd.DataFrame(columns=BAR_COLUMNS)
+        result0 = result[0]
+        meta = result0.get("meta") or {}
+        timestamps = result0.get("timestamp") or []
+        quote_sets = result0.get("indicators", {}).get("quote") or []
+        if not timestamps or not quote_sets:
+            return pd.DataFrame(columns=BAR_COLUMNS)
+
+        try:
+            timezone = ZoneInfo(meta.get("exchangeTimezoneName") or "UTC")
+        except Exception:
+            timezone = ZoneInfo("UTC")
+        quote = quote_sets[0]
+        adjclose_sets = result0.get("indicators", {}).get("adjclose") or [{}]
+        adj_close = adjclose_sets[0].get("adjclose", []) if adjclose_sets else []
+        events = result0.get("events") or {}
+        dividends = self._event_series(events.get("dividends") or {}, value_key="amount")
+        splits = self._event_series(events.get("splits") or {}, value_key="splitRatio")
+        row_count = len(timestamps)
+
+        frame = pd.DataFrame(
+            {
+                "timestamp": pd.to_datetime(timestamps, unit="s", utc=True).tz_convert(timezone).tz_localize(None),
+                "open": self._normalize_values(quote.get("open", []), row_count),
+                "high": self._normalize_values(quote.get("high", []), row_count),
+                "low": self._normalize_values(quote.get("low", []), row_count),
+                "close": self._normalize_values(quote.get("close", []), row_count),
+                "adj_close": self._normalize_values(adj_close, row_count),
+                "volume": self._normalize_values(quote.get("volume", []), row_count),
+            }
+        )
+        for column in ("open", "high", "low", "close", "adj_close", "volume"):
+            frame[column] = pd.to_numeric(frame[column], errors="coerce")
+        frame = frame.dropna(subset=["timestamp", "close"]).reset_index(drop=True)
+        frame["volume"] = frame["volume"].fillna(0.0)
+
+        frame["epoch"] = (frame["timestamp"].astype("int64") // 1_000_000_000).astype("int64")
+        frame["dividend"] = frame["epoch"].map(dividends).fillna(0.0)
+        frame["split_ratio"] = frame["epoch"].map(splits).fillna(1.0)
+        frame = frame.drop(columns=["epoch"])
+
+        if adjust_ohlc:
+            ratio = frame["adj_close"].where(frame["close"] != 0, frame["close"]) / frame["close"].replace(0, pd.NA)
+            ratio = ratio.fillna(1.0)
+            for column in ("open", "high", "low", "close"):
+                frame[column] = frame[column] * ratio
+        return frame.drop_duplicates(subset=["timestamp"], keep="last").sort_values("timestamp").reset_index(drop=True)[BAR_COLUMNS]
+
+    @staticmethod
+    def _event_series(events: dict[str, Any], *, value_key: str) -> dict[int, float]:
+        output: dict[int, float] = {}
+        for event in events.values():
+            timestamp = event.get("date")
+            value = event.get(value_key)
+            if timestamp is None or value is None:
+                continue
+            try:
+                output[int(timestamp)] = float(value)
+            except (TypeError, ValueError):
+                continue
+        return output
+
+    @staticmethod
+    def _normalize_values(values: list[Any] | tuple[Any, ...], size: int) -> list[Any]:
+        normalized = list(values[:size])
+        if len(normalized) < size:
+            normalized.extend([None] * (size - len(normalized)))
+        return normalized
+
+    @staticmethod
+    def _cache_key(*, url: str, params: dict[str, str]) -> str:
+        material = json.dumps({"url": url, "params": params}, sort_keys=True, separators=(",", ":"))
+        return hashlib.sha256(material.encode("utf-8")).hexdigest()
+
+    @staticmethod
+    def _cache_ttl_seconds(*, end_dt: datetime) -> int:
+        now_utc = datetime.utcnow()
+        if end_dt < now_utc - timedelta(days=2):
+            return 7 * 24 * 60 * 60
+        if end_dt < now_utc - timedelta(hours=12):
+            return 60 * 60
+        return 90
+
+    @staticmethod
+    def _coerce_datetime(value: str | datetime, *, end_of_day: bool) -> datetime:
+        timestamp = pd.Timestamp(value)
+        if timestamp.tzinfo is not None:
+            timestamp = timestamp.tz_convert("UTC").tz_localize(None)
+        dt = timestamp.to_pydatetime()
+        if end_of_day and dt.hour == 0 and dt.minute == 0 and dt.second == 0 and dt.microsecond == 0:
+            return dt + timedelta(days=1)
+        return dt
+
+    @staticmethod
+    def _iter_chunks(*, start_dt: datetime, end_dt: datetime, chunk_days: int) -> list[tuple[datetime, datetime]]:
+        chunks: list[tuple[datetime, datetime]] = []
+        cursor = start_dt
+        while cursor < end_dt:
+            next_edge = min(cursor + timedelta(days=chunk_days), end_dt)
+            chunks.append((cursor, next_edge))
+            cursor = next_edge
+        return chunks
+
+    @staticmethod
+    def _interval_policy(interval: str) -> IntervalPolicy:
+        normalized = interval.strip()
+        if normalized not in INTERVAL_POLICIES:
+            allowed = ", ".join(sorted(INTERVAL_POLICIES))
+            raise ValueError(f"Unsupported interval {interval!r}. Allowed values: {allowed}")
+        return INTERVAL_POLICIES[normalized]
+
+    @staticmethod
+    def _validate_retention_window(*, policy: IntervalPolicy, start_dt: datetime, end_dt: datetime) -> None:
+        if policy.retention_days is None:
+            return
+        earliest = datetime.utcnow() - timedelta(days=policy.retention_days)
+        if start_dt < earliest or end_dt < earliest:
+            cutoff = earliest.strftime("%Y-%m-%d")
+            raise YahooIntervalLimitError(
+                f"Yahoo only serves {policy.interval} history back to about {cutoff}. Use 1d or coarser for deeper history."
+            )

backend/requirements.txt

@@ -0,0 +1,10 @@
+pandas
+pandas_market_calendars
+pyarrow
+requests
+fastapi
+uvicorn
+joblib
+numpy
+scikit-learn
+catboost

backend/scripts/refresh_daily_data.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from nifty_backend.runtime import refresh_daily_data
+
+
+def main() -> None:
+    print(refresh_daily_data())
+
+
+if __name__ == "__main__":
+    main()

backend/scripts/refresh_first5_prediction.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+import argparse
+import sys
+from datetime import date
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from nifty_backend.runtime import refresh_first5_prediction
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Fetch Yahoo Finance first five NIFTY minutes and refresh prediction.")
+    parser.add_argument("--date", default=None, help="Optional IST session date, YYYY-MM-DD.")
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    session_date = date.fromisoformat(args.date) if args.date else None
+    prediction = refresh_first5_prediction(session_date=session_date)
+    print(prediction.to_dict())
+
+
+if __name__ == "__main__":
+    main()

backend/scripts/retrain_opening_model.py

@@ -0,0 +1,182 @@
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import joblib
+import numpy as np
+import pandas as pd
+from sklearn.ensemble import ExtraTreesClassifier
+from sklearn.impute import SimpleImputer
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import accuracy_score, brier_score_loss, log_loss, roc_auc_score
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import StandardScaler
+
+from nifty_backend.runtime import (
+    DECISION_OVERLAYS,
+    MODEL_DIR,
+    MODEL_PATH,
+    OPENING_DATASET_PATH,
+    ProbabilityBlend,
+    apply_decision_overlays,
+    directional_confidence,
+    predict_proba_up,
+)
+
+
+DEFAULT_TRAIN_END = pd.Timestamp("2023-12-31")
+DEFAULT_VALID_END = pd.Timestamp("2025-08-17")
+RANDOM_SEED = 42
+
+
+@dataclass
+class RetrainSummary:
+    model_name: str
+    threshold: float
+    train_rows: int
+    valid_rows: int
+    test_rows: int
+    validation_accuracy: float
+    test_accuracy: float
+    validation_auc: float
+    test_auc: float
+    test_brier: float
+    latest_prediction: str
+    latest_prob_up: float
+    latest_confidence: float
+    feature_count: int
+
+
+def feature_columns(frame: pd.DataFrame) -> list[str]:
+    excluded = {
+        "date",
+        "first5_start",
+        "first5_end",
+        "target",
+        "day_open",
+        "day_high",
+        "day_low",
+        "day_close",
+        "day_volume",
+        "day_return",
+    }
+    cols = []
+    for col in frame.columns:
+        if col in excluded:
+            continue
+        if pd.api.types.is_numeric_dtype(frame[col]) and frame[col].notna().mean() >= 0.40:
+            if frame[col].nunique(dropna=True) > 1:
+                cols.append(col)
+    return cols
+
+
+def best_threshold(y_true: np.ndarray, prob_up: np.ndarray) -> tuple[float, float]:
+    thresholds = np.linspace(0.35, 0.65, 301)
+    scores = ((prob_up[:, None] >= thresholds[None, :]) == y_true[:, None]).mean(axis=0)
+    idx = int(np.argmax(scores))
+    return float(thresholds[idx]), float(scores[idx])
+
+
+def score_auc(y_true: np.ndarray, prob_up: np.ndarray) -> float:
+    if len(np.unique(y_true)) < 2:
+        return float("nan")
+    return float(roc_auc_score(y_true, prob_up))
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Retrain the compact NIFTY opening-direction model from Parquet data.")
+    parser.add_argument("--train-end", default=DEFAULT_TRAIN_END.date().isoformat())
+    parser.add_argument("--valid-end", default=DEFAULT_VALID_END.date().isoformat())
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    train_end = pd.Timestamp(args.train_end)
+    valid_end = pd.Timestamp(args.valid_end)
+    frame = pd.read_parquet(OPENING_DATASET_PATH)
+    frame["date"] = pd.to_datetime(frame["date"], errors="coerce")
+    model_frame = frame.dropna(subset=["target"]).sort_values("date").reset_index(drop=True)
+    features = feature_columns(model_frame)
+    train_df = model_frame[model_frame["date"] <= train_end]
+    valid_df = model_frame[(model_frame["date"] > train_end) & (model_frame["date"] <= valid_end)]
+    test_df = model_frame[model_frame["date"] > valid_end]
+    if train_df.empty or valid_df.empty or test_df.empty:
+        raise RuntimeError("Training, validation, and test windows must all contain rows.")
+
+    x_train = train_df[features]
+    y_train = train_df["target"].to_numpy(dtype="int64")
+    x_valid = valid_df[features]
+    y_valid = valid_df["target"].to_numpy(dtype="int64")
+    x_test = test_df[features]
+    y_test = test_df["target"].to_numpy(dtype="int64")
+
+    extra_trees = make_pipeline(
+        SimpleImputer(strategy="median"),
+        ExtraTreesClassifier(
+            n_estimators=800,
+            max_depth=4,
+            min_samples_leaf=28,
+            max_features=0.60,
+            class_weight="balanced_subsample",
+            random_state=RANDOM_SEED + 13,
+            n_jobs=-1,
+        ),
+    )
+    logit = make_pipeline(
+        SimpleImputer(strategy="median"),
+        StandardScaler(),
+        LogisticRegression(C=0.25, class_weight="balanced", max_iter=2000, random_state=RANDOM_SEED),
+    )
+    extra_trees.fit(x_train, y_train)
+    logit.fit(x_train, y_train)
+    model = ProbabilityBlend([extra_trees, logit], np.array([0.75, 0.25]))
+    valid_prob = predict_proba_up(model, x_valid)
+    test_prob = predict_proba_up(model, x_test)
+    threshold, _ = best_threshold(y_valid, valid_prob)
+    valid_pred = apply_decision_overlays((valid_prob >= threshold).astype("int64"), valid_df, DECISION_OVERLAYS)
+    test_pred = apply_decision_overlays((test_prob >= threshold).astype("int64"), test_df, DECISION_OVERLAYS)
+    latest = frame.iloc[[-1]].copy()
+    latest_prob = predict_proba_up(model, latest[features])
+    latest_pred = apply_decision_overlays((latest_prob >= threshold).astype("int64"), latest, DECISION_OVERLAYS)
+    latest_conf = directional_confidence(latest_prob, latest_pred, threshold)
+
+    payload = {
+        "model": model,
+        "features": features,
+        "threshold": threshold,
+        "target": "same-day NIFTY 50 close > same-day NIFTY 50 open after first five 1-minute bars",
+        "model_name": "compact_extra_trees_logit_overlay",
+        "decision_overlays": DECISION_OVERLAYS,
+    }
+    joblib.dump(payload, MODEL_PATH)
+
+    summary = RetrainSummary(
+        model_name=payload["model_name"],
+        threshold=float(threshold),
+        train_rows=int(len(train_df)),
+        valid_rows=int(len(valid_df)),
+        test_rows=int(len(test_df)),
+        validation_accuracy=float(accuracy_score(y_valid, valid_pred)),
+        test_accuracy=float(accuracy_score(y_test, test_pred)),
+        validation_auc=score_auc(y_valid, valid_prob),
+        test_auc=score_auc(y_test, test_prob),
+        test_brier=float(brier_score_loss(y_test, np.clip(test_prob, 1e-6, 1 - 1e-6))),
+        latest_prediction="UP" if int(latest_pred[0]) == 1 else "DOWN",
+        latest_prob_up=float(latest_prob[0]),
+        latest_confidence=float(latest_conf[0]),
+        feature_count=int(len(features)),
+    )
+    (MODEL_DIR / "summary.json").write_text(json.dumps(asdict(summary), indent=2), encoding="utf-8")
+    pd.DataFrame([asdict(summary)]).to_csv(MODEL_DIR / "retrain_summary.csv", index=False)
+    print(asdict(summary))
+
+
+if __name__ == "__main__":
+    main()

backend/scripts/run_ist_scheduler.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import sys
+import time
+from datetime import date, datetime, time as dt_time
+from pathlib import Path
+from zoneinfo import ZoneInfo
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from nifty_backend.runtime import (
+    CLOSE_REFRESH_READY,
+    STALE_CHECK_INTERVAL_SECONDS,
+    is_trading_day,
+    latest_saved_prediction,
+    close_refresh_due,
+    refresh_market_close_data,
+    refresh_daily_data,
+    refresh_first5_prediction,
+    refresh_stale_data_once,
+    seconds_until_next_ist_run,
+)
+
+
+IST = ZoneInfo("Asia/Kolkata")
+FIRST5_READY = dt_time(9, 20)
+
+
+def latest_prediction_date() -> date | None:
+    try:
+        raw = latest_saved_prediction().get("input_date")
+        return date.fromisoformat(str(raw)) if raw else None
+    except Exception:
+        return None
+
+
+def refresh_if_current_session_is_ready() -> None:
+    now = datetime.now(IST)
+    if not is_trading_day(now.date()) or now.time() < FIRST5_READY:
+        return
+    if latest_prediction_date() == now.date():
+        return
+    prediction = refresh_first5_prediction()
+    print(f"[scheduler] first5 prediction refreshed: {prediction.to_dict()}")
+    info = refresh_daily_data()
+    print(f"[scheduler] daily data refreshed: {info}")
+
+
+def refresh_close_data_if_due() -> None:
+    if not close_refresh_due():
+        return
+    info = refresh_market_close_data()
+    print(f"[scheduler] close data refreshed: {info}")
+
+
+def refresh_stale_data_if_due() -> None:
+    info = refresh_stale_data_once()
+    if info.get("status") == "refreshed":
+        print(f"[scheduler] stale data refreshed: {info}")
+
+
+def main() -> None:
+    print("[scheduler] NIFTY first-five-minute scheduler started.")
+    print("[scheduler] Runs the opening prediction after 09:20 IST so the 09:15-09:19 candles are complete.")
+    while True:
+        try:
+            refresh_if_current_session_is_ready()
+            refresh_close_data_if_due()
+            refresh_stale_data_if_due()
+        except Exception as exc:
+            print(f"[scheduler] current-session refresh failed: {exc}")
+        next_first5 = seconds_until_next_ist_run()
+        if next_first5 > STALE_CHECK_INTERVAL_SECONDS:
+            time.sleep(STALE_CHECK_INTERVAL_SECONDS)
+            continue
+        time.sleep(next_first5)
+        try:
+            prediction = refresh_first5_prediction()
+            print(f"[scheduler] first5 prediction refreshed: {prediction.to_dict()}")
+        except Exception as exc:
+            print(f"[scheduler] first5 refresh failed: {exc}")
+        try:
+            info = refresh_daily_data()
+            print(f"[scheduler] daily data refreshed: {info}")
+        except Exception as exc:
+            print(f"[scheduler] daily refresh failed: {exc}")
+        next_close = seconds_until_next_ist_run(CLOSE_REFRESH_READY)
+        if next_close > STALE_CHECK_INTERVAL_SECONDS:
+            time.sleep(STALE_CHECK_INTERVAL_SECONDS)
+            continue
+        time.sleep(next_close)
+        try:
+            refresh_close_data_if_due()
+        except Exception as exc:
+            print(f"[scheduler] close refresh failed: {exc}")
+
+
+if __name__ == "__main__":
+    main()