Upload 27 files

backend/.dockerignore

@@ -0,0 +1,18 @@
+.git
+.gitignore
+__pycache__/
+*.py[cod]
+.space_state/
+.env
+*.env
+!*.example.env
+research_runtime/Code/artifacts/
+research_runtime/Code/docs/
+research_runtime/Code/scripts/backtesting/
+research_runtime/Code/scripts/tuning/
+research_runtime/Code/models/**/outputs/*dataset*.csv
+research_runtime/Code/models/**/outputs/test_predictions.csv
+research_runtime/Code/models/**/outputs/*predictions.csv
+research_runtime/Code/models/**/outputs/*.joblib
+research_runtime/Data/
+research_runtime/Alt Data/

backend/.gitattributes

@@ -0,0 +1,5 @@
+*.csv filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text

backend/Dockerfile

@@ -0,0 +1,23 @@
+FROM python:3.11-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PORT=7860 \
+    FORECASTING_PROJECT_ROOT=/app/research_runtime
+
+WORKDIR /app
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends build-essential curl git libgomp1 \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+RUN pip install --upgrade pip \
+    && pip install -r requirements.txt
+
+COPY . .
+
+EXPOSE 7860
+
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860", "--ws", "none"]

backend/README.md

@@ -0,0 +1,128 @@
+---
+title: Trading Forecasting Backend
+colorFrom: blue
+colorTo: green
+sdk: docker
+app_port: 7860
+pinned: false
+---
+
+# Trading Forecasting Backend
+
+This folder is now a standalone Hugging Face Docker Space backend. Upload the contents of this `backend` folder to a Hugging Face Space repository, upload the separate `dataset` folder to a Hugging Face Dataset repository, and deploy the separate `frontend` folder to Netlify.
+
+The backend contains the quantitative model code, training scripts, model outputs, primary market data, and alternative data from the forecasting research workspace.
+
+## Hugging Face Space Setup
+
+Create a new Hugging Face Space with Docker SDK, then upload this backend folder as the Space root.
+
+Required Space variables/secrets:
+
+- `FRONTEND_ORIGINS`: your Netlify URL, for example `https://your-site.netlify.app`.
+- `CRON_SECRET`: a long shared secret. Use the same value in Netlify.
+- `HF_DATASET_REPO_ID`: your Hugging Face Dataset repo id, for example `your-username/your-forecasting-dataset`.
+
+Useful optional settings:
+
+- `AUTO_UPDATE_ENABLED=true`
+- `AUTO_RETRAIN_ENABLED=true`
+- `AUTO_UPDATE_ON_START=false`
+- `DATASET_SYNC_ON_START=true`
+- `HF_DATASET_REVISION=main`
+- `DAILY_UPDATE_TIME=17:30`
+- `UPDATE_TIMEZONE=Asia/Kolkata`
+- `MARKET_BUILD_WORKERS=2`
+
+The app listens on port `7860` and exposes Swagger docs at `/docs`.
+
+## API Routes
+
+- `GET /health` - Space health, file checks, latest data date, and update status.
+- `GET /api/status` - same as health, for frontend polling.
+- `GET /api/forecast/latest` - latest stock high/low, first-extrema, and Nifty forecasts.
+- `GET /api/models/summaries` - model summary JSONs.
+- `GET /api/data/catalog` - searchable data manifest.
+- `GET /api/data/sample?category=bars&asset=nifty50&timeframe=1d` - small sample from a manifest dataset.
+- `POST /api/cron/tick` - Netlify scheduled ping endpoint; starts an update only when due.
+- `POST /api/update/start` - manual update trigger. Send `x-admin-secret` if `CRON_SECRET` or `ADMIN_SECRET` is set.
+- `POST /api/dataset/sync` - manually sync the Hugging Face Dataset repo into the Space runtime.
+
+## Netlify Keep-Awake Cron
+
+The `frontend` folder now includes:
+
+- `frontend/netlify.toml`
+- `frontend/netlify/functions/keep-space-awake.mjs`
+
+On Netlify, set these environment variables:
+
+- `HUGGING_FACE_SPACE_URL=https://YOUR-HF-USERNAME-YOUR-SPACE.hf.space`
+- `CRON_SECRET=<same value as the Space CRON_SECRET>`
+
+The scheduled function runs every 10 minutes and calls `/api/cron/tick`. This keeps the Space warm and lets the backend start its daily update/retrain job after the configured market-close time.
+
+## Layout
+
+- `app.py` - FastAPI backend app for Hugging Face Spaces.
+- `Dockerfile` - Docker Space runtime setup.
+- `requirements.txt` - Python dependencies.
+- `research_runtime/Code/models/` - trainable model packages and the small latest forecast/summary outputs needed by the API.
+- `research_runtime/Code/scripts/data_ingestion/` - data refresh scripts used by update jobs.
+- `research_runtime/Code/scripts/data_preparation/` - research data rebuild scripts used by update jobs.
+
+`research_runtime/Data/` and `research_runtime/Alt Data/` are intentionally not bundled in the Space repo anymore. They now live in the separate Hugging Face Dataset repo and are downloaded into `research_runtime/` by the backend when `HF_DATASET_REPO_ID` is set.
+
+## Main Model Outputs To Wire First
+
+- Stock high/low forecasts: `research_runtime/Code/models/stock_high_low_forecaster/outputs/latest_forecasts.csv`
+- Stock high/low metrics: `research_runtime/Code/models/stock_high_low_forecaster/outputs/metrics_by_symbol.csv`
+- First-extrema forecasts: `research_runtime/Code/models/first_extrema_forecaster/outputs/latest_forecasts.csv`
+- Nifty forecasts: `research_runtime/Code/models/nifty_forecaster/outputs/forecaster_latest_forecasts.csv`
+- Nifty summary: `research_runtime/Code/models/nifty_forecaster/outputs/forecaster_summary.json`
+
+## Training Entrypoints
+
+Run these from `backend/research_runtime` so project-relative paths resolve correctly:
+
+```powershell
+python Code\models\stock_high_low_forecaster\train.py
+python Code\models\first_extrema_forecaster\train.py
+python Code\models\nifty_forecaster\train.py
+```
+
+## Data Labels
+
+These live in the separate Dataset repo:
+
+- Raw minute OHLCV: `Data/raw/minute/*_minute.csv`
+- Processed bars: `Data/processed/bars/{1m,5m,1h,4h,1d}/*.csv`
+- Processed features: `Data/processed/features/{1m,5m,1h,4h,1d}/*.csv`
+- Market panels: `Data/processed/panels/*_market_panel.csv`
+- Master daily panel: `Data/processed/panels/daily_master_panel.csv`
+- Data manifest: `Data/metadata/manifest.csv`
+- Feature dictionary: `Data/metadata/feature_dictionary.csv`
+- Options features: `Alt Data/options/processed/*_options_daily_features.csv`
+- Institutional panel: `Alt Data/institutional/processed/institutional_daily_panel.csv`
+- External daily panel: `Alt Data/external/processed/external_daily_panel.csv`
+- Corporate events: `Alt Data/corporate/processed/corporate_announcements.csv`
+
+## Frontend Wiring Notes
+
+The current frontend is static mock data in `frontend/index.html` and `frontend/script.js`.
+
+- Forecast cards can call `/api/forecast/latest`.
+- Model accuracy and version/date stats can call `/api/models/summaries`.
+- Market Data can call `/api/data/catalog` and `/api/data/sample`.
+
+## Pruned From Backend
+
+- Kotak credential/runtime files.
+- Live-trading scripts and live broker artifacts.
+- Kotak monitor artifacts and cached NSE temp folders.
+- Python `__pycache__` folders.
+- CatBoost generated training-log folder.
+- One-off maintenance/backfill scripts.
+- Backtest artifacts, chart images, old trade reports, test prediction dumps, generated training datasets, and saved model binaries.
+
+`KOTAKBANK` CSV files remain because those are normal market datasets for Kotak Mahindra Bank, not broker-runtime files.

backend/app.py

@@ -0,0 +1,525 @@
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+import threading
+import time
+from datetime import datetime, time as dt_time
+from pathlib import Path
+from typing import Any
+from zoneinfo import ZoneInfo
+
+import pandas as pd
+from fastapi import BackgroundTasks, FastAPI, Header, HTTPException, Query, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse, PlainTextResponse
+from huggingface_hub import snapshot_download
+
+
+BASE_DIR = Path(__file__).resolve().parent
+RESEARCH_ROOT = Path(os.environ.get("FORECASTING_PROJECT_ROOT", BASE_DIR / "research_runtime")).resolve()
+STATE_DIR = Path(os.environ.get("SPACE_STATE_DIR", "/data/forecasting-space-state" if Path("/data").exists() else BASE_DIR / ".space_state"))
+STATUS_PATH = STATE_DIR / "update_status.json"
+DATASET_READY_MARKER = STATE_DIR / "dataset_ready.json"
+
+API_TITLE = "Trading Forecasting Space Backend"
+API_VERSION = "1.0.0"
+DEFAULT_TIMEZONE = os.environ.get("UPDATE_TIMEZONE", "Asia/Kolkata")
+DEFAULT_UPDATE_TIME = os.environ.get("DAILY_UPDATE_TIME", "17:30")
+
+app = FastAPI(title=API_TITLE, version=API_VERSION)
+
+
+def cors_origins() -> list[str]:
+    raw = os.environ.get("FRONTEND_ORIGINS", "*").strip()
+    return ["*"] if raw == "*" else [item.strip() for item in raw.split(",") if item.strip()]
+
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=cors_origins(),
+    allow_credentials=False,
+    allow_methods=["GET", "POST", "OPTIONS"],
+    allow_headers=["*"],
+)
+
+update_lock = threading.Lock()
+worker_thread: threading.Thread | None = None
+dataset_lock = threading.Lock()
+
+
+def now_utc() -> str:
+    return datetime.utcnow().replace(microsecond=0).isoformat() + "Z"
+
+
+def safe_json(value: Any) -> Any:
+    if isinstance(value, dict):
+        return {str(k): safe_json(v) for k, v in value.items()}
+    if isinstance(value, list):
+        return [safe_json(v) for v in value]
+    if not isinstance(value, (tuple, set)):
+        try:
+            if pd.isna(value):
+                return None
+        except Exception:
+            pass
+    if hasattr(value, "item"):
+        try:
+            return safe_json(value.item())
+        except Exception:
+            pass
+    if isinstance(value, Path):
+        return str(value)
+    if isinstance(value, datetime):
+        return value.isoformat()
+    return value
+
+
+def read_json(path: Path, default: Any) -> Any:
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return default
+
+
+def write_json(path: Path, payload: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(safe_json(payload), indent=2), encoding="utf-8")
+
+
+def read_status() -> dict[str, Any]:
+    return read_json(
+        STATUS_PATH,
+        {
+            "state": "idle",
+            "last_started_at": None,
+            "last_finished_at": None,
+            "last_success_at": None,
+            "last_error": None,
+            "last_exit_code": None,
+            "last_log_tail": [],
+        },
+    )
+
+
+def write_status(**updates: Any) -> None:
+    status = read_status()
+    status.update(updates)
+    write_json(STATUS_PATH, status)
+
+
+def require_secret(x_cron_secret: str | None = Header(default=None), x_admin_secret: str | None = Header(default=None)) -> None:
+    expected = os.environ.get("CRON_SECRET") or os.environ.get("ADMIN_SECRET")
+    if not expected:
+        return
+    supplied = x_cron_secret or x_admin_secret
+    if supplied != expected:
+        raise HTTPException(status_code=401, detail="Missing or invalid cron/admin secret.")
+
+
+def csv_rows(path: Path, *, limit: int | None = None, columns: list[str] | None = None) -> list[dict[str, Any]]:
+    if not path.exists():
+        return []
+    try:
+        frame = pd.read_csv(path, usecols=columns)
+    except ValueError:
+        frame = pd.read_csv(path)
+        if columns:
+            frame = frame[[col for col in columns if col in frame.columns]]
+    if limit is not None:
+        frame = frame.head(limit)
+    return safe_json(frame.where(pd.notna(frame), None).to_dict(orient="records"))
+
+
+def model_output_path(*parts: str) -> Path:
+    return RESEARCH_ROOT / "Code" / "models" / Path(*parts)
+
+
+def manifest_path() -> Path:
+    return RESEARCH_ROOT / "Data" / "metadata" / "manifest.csv"
+
+
+def dataset_dirs_present() -> bool:
+    return (RESEARCH_ROOT / "Data").is_dir() and (RESEARCH_ROOT / "Alt Data").is_dir()
+
+
+def dataset_status() -> dict[str, Any]:
+    marker = read_json(DATASET_READY_MARKER, {})
+    return {
+        "ready": dataset_dirs_present(),
+        "repo_id": os.environ.get("HF_DATASET_REPO_ID"),
+        "revision": os.environ.get("HF_DATASET_REVISION", "main"),
+        "data_dir": file_meta(RESEARCH_ROOT / "Data"),
+        "alt_data_dir": file_meta(RESEARCH_ROOT / "Alt Data"),
+        "last_sync": marker,
+    }
+
+
+def ensure_dataset_available(force: bool = False) -> bool:
+    if dataset_dirs_present() and not force:
+        return True
+
+    repo_id = os.environ.get("HF_DATASET_REPO_ID", "").strip()
+    if not repo_id:
+        return dataset_dirs_present()
+
+    with dataset_lock:
+        if dataset_dirs_present() and not force:
+            return True
+
+        STATE_DIR.mkdir(parents=True, exist_ok=True)
+        revision = os.environ.get("HF_DATASET_REVISION", "main")
+        local_dir = Path(os.environ.get("HF_DATASET_LOCAL_DIR", str(RESEARCH_ROOT))).resolve()
+        local_dir.mkdir(parents=True, exist_ok=True)
+
+        snapshot_download(
+            repo_id=repo_id,
+            repo_type="dataset",
+            revision=revision,
+            local_dir=str(local_dir),
+            local_dir_use_symlinks=False,
+            allow_patterns=["Data/**", "Alt Data/**", "README.md"],
+        )
+
+        write_json(
+            DATASET_READY_MARKER,
+            {
+                "repo_id": repo_id,
+                "revision": revision,
+                "synced_at": now_utc(),
+                "local_dir": str(local_dir),
+            },
+        )
+        return dataset_dirs_present()
+
+
+def resolve_dataset_path(value: str) -> Path:
+    raw = str(value)
+    candidate = Path(raw)
+    if candidate.exists():
+        return candidate
+
+    normalized = raw.replace("\\", "/")
+    marker = "research_runtime/"
+    if marker in normalized:
+        suffix = normalized.split(marker, 1)[1]
+        return BASE_DIR / "research_runtime" / Path(*suffix.split("/"))
+
+    relative = Path(*normalized.split("/"))
+    if not relative.is_absolute():
+        return BASE_DIR / relative
+    return candidate
+
+
+def file_meta(path: Path) -> dict[str, Any]:
+    if not path.exists():
+        return {"exists": False, "path": str(path)}
+    stat = path.stat()
+    return {
+        "exists": True,
+        "path": str(path),
+        "bytes": stat.st_size,
+        "modified_at": datetime.utcfromtimestamp(stat.st_mtime).replace(microsecond=0).isoformat() + "Z",
+    }
+
+
+def latest_manifest_end() -> str | None:
+    path = manifest_path()
+    if not path.exists():
+        return None
+    try:
+        frame = pd.read_csv(path, usecols=["end"])
+        dates = pd.to_datetime(frame["end"], errors="coerce").dropna()
+        return str(dates.max()) if not dates.empty else None
+    except Exception:
+        return None
+
+
+def parse_daily_update_time() -> dt_time:
+    hour, minute = DEFAULT_UPDATE_TIME.split(":", 1)
+    return dt_time(int(hour), int(minute))
+
+
+def update_due() -> bool:
+    if os.environ.get("AUTO_UPDATE_ENABLED", "true").lower() not in {"1", "true", "yes", "on"}:
+        return False
+    status = read_status()
+    if status.get("state") == "running":
+        return False
+
+    tz = ZoneInfo(DEFAULT_TIMEZONE)
+    local_now = datetime.now(tz)
+    if local_now.time() < parse_daily_update_time():
+        return False
+
+    last_success = status.get("last_success_at")
+    if not last_success:
+        return True
+    try:
+        last_success_date = datetime.fromisoformat(last_success.replace("Z", "+00:00")).astimezone(tz).date()
+    except ValueError:
+        return True
+    return last_success_date < local_now.date()
+
+
+def build_update_commands(retrain: bool) -> list[list[str]]:
+    commands = [
+        [
+            sys.executable,
+            "Code/scripts/data_ingestion/refresh_market_data.py",
+            "--end-date",
+            datetime.now(ZoneInfo(DEFAULT_TIMEZONE)).date().isoformat(),
+        ]
+    ]
+    if retrain:
+        commands.extend(
+            [
+                [sys.executable, "Code/models/stock_high_low_forecaster/train.py"],
+                [sys.executable, "Code/models/first_extrema_forecaster/train.py", "--rebuild-cache"],
+                [sys.executable, "Code/models/nifty_forecaster/train.py", "--no-progress"],
+            ]
+        )
+    return commands
+
+
+def run_update_job(trigger: str = "manual", retrain: bool | None = None) -> None:
+    global worker_thread
+    with update_lock:
+        status = read_status()
+        if status.get("state") == "running":
+            return
+        write_status(
+            state="running",
+            trigger=trigger,
+            last_started_at=now_utc(),
+            last_finished_at=None,
+            last_error=None,
+            last_exit_code=None,
+            last_log_tail=[],
+        )
+
+    if retrain is None:
+        retrain = os.environ.get("AUTO_RETRAIN_ENABLED", "true").lower() in {"1", "true", "yes", "on"}
+
+    env = os.environ.copy()
+    env["FORECASTING_PROJECT_ROOT"] = str(RESEARCH_ROOT)
+    env.setdefault("PYTHONUNBUFFERED", "1")
+    env.setdefault("MARKET_BUILD_WORKERS", "2")
+
+    log_tail: list[str] = []
+    exit_code = 0
+    try:
+        if not ensure_dataset_available():
+            raise RuntimeError("Dataset folders are missing. Set HF_DATASET_REPO_ID to the Hugging Face Dataset repo.")
+        for command in build_update_commands(retrain):
+            log_tail.append("$ " + " ".join(command))
+            process = subprocess.Popen(
+                command,
+                cwd=RESEARCH_ROOT,
+                env=env,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+                bufsize=1,
+            )
+            assert process.stdout is not None
+            for line in process.stdout:
+                line = line.rstrip()
+                if line:
+                    log_tail.append(line)
+                    log_tail = log_tail[-80:]
+            exit_code = process.wait()
+            if exit_code != 0:
+                raise RuntimeError(f"Command failed with exit code {exit_code}: {' '.join(command)}")
+        write_status(
+            state="idle",
+            last_finished_at=now_utc(),
+            last_success_at=now_utc(),
+            last_error=None,
+            last_exit_code=exit_code,
+            last_log_tail=log_tail[-80:],
+        )
+    except Exception as exc:
+        write_status(
+            state="failed",
+            last_finished_at=now_utc(),
+            last_error=str(exc),
+            last_exit_code=exit_code,
+            last_log_tail=log_tail[-80:],
+        )
+
+
+def start_update(trigger: str, retrain: bool | None = None) -> bool:
+    global worker_thread
+    status = read_status()
+    if status.get("state") == "running":
+        return False
+    worker_thread = threading.Thread(target=run_update_job, kwargs={"trigger": trigger, "retrain": retrain}, daemon=True)
+    worker_thread.start()
+    return True
+
+
+def scheduler_loop() -> None:
+    while True:
+        if update_due():
+            start_update("internal_scheduler")
+        time.sleep(300)
+
+
+@app.on_event("startup")
+def startup() -> None:
+    STATE_DIR.mkdir(parents=True, exist_ok=True)
+    if not STATUS_PATH.exists():
+        write_status(state="idle", app_started_at=now_utc())
+    if os.environ.get("DATASET_SYNC_ON_START", "true").lower() in {"1", "true", "yes", "on"}:
+        try:
+            ensure_dataset_available()
+        except Exception as exc:
+            write_status(dataset_sync_error=str(exc), dataset_sync_failed_at=now_utc())
+    threading.Thread(target=scheduler_loop, daemon=True).start()
+    if os.environ.get("AUTO_UPDATE_ON_START", "false").lower() in {"1", "true", "yes", "on"}:
+        start_update("startup")
+
+
+@app.get("/", response_class=PlainTextResponse)
+def root() -> str:
+    return "Trading Forecasting Hugging Face Space backend is running. See /docs for API routes."
+
+
+@app.get("/health")
+def health() -> dict[str, Any]:
+    required = {
+        "research_root": file_meta(RESEARCH_ROOT),
+        "manifest": file_meta(manifest_path()),
+        "stock_latest": file_meta(model_output_path("stock_high_low_forecaster", "outputs", "latest_forecasts.csv")),
+        "extrema_latest": file_meta(model_output_path("first_extrema_forecaster", "outputs", "latest_forecasts.csv")),
+        "nifty_latest": file_meta(model_output_path("nifty_forecaster", "outputs", "forecaster_latest_forecasts.csv")),
+    }
+    ok = all(item["exists"] for item in required.values())
+    return {
+        "ok": ok,
+        "service": API_TITLE,
+        "version": API_VERSION,
+        "checked_at": now_utc(),
+        "latest_manifest_end": latest_manifest_end(),
+        "dataset": dataset_status(),
+        "update_status": read_status(),
+        "files": required,
+    }
+
+
+@app.get("/api/status")
+def api_status() -> dict[str, Any]:
+    return health()
+
+
+@app.get("/api/forecast/latest")
+def latest_forecasts() -> dict[str, Any]:
+    return {
+        "generated_at": now_utc(),
+        "stock_high_low": csv_rows(model_output_path("stock_high_low_forecaster", "outputs", "latest_forecasts.csv")),
+        "first_extrema": csv_rows(
+            model_output_path("first_extrema_forecaster", "outputs", "latest_forecasts.csv"),
+            columns=["date", "symbol", "target", "prob_high_first", "prediction"],
+        ),
+        "nifty_direction": csv_rows(model_output_path("nifty_forecaster", "outputs", "forecaster_latest_forecasts.csv")),
+    }
+
+
+@app.get("/api/models/summaries")
+def model_summaries() -> dict[str, Any]:
+    return safe_json(
+        {
+            "stock_high_low": read_json(model_output_path("stock_high_low_forecaster", "outputs", "summary.json"), {}),
+            "first_extrema": read_json(model_output_path("first_extrema_forecaster", "outputs", "summary.json"), {}),
+            "nifty_direction": read_json(model_output_path("nifty_forecaster", "outputs", "forecaster_summary.json"), []),
+        }
+    )
+
+
+@app.get("/api/data/catalog")
+def data_catalog(
+    category: str | None = None,
+    asset: str | None = None,
+    timeframe: str | None = None,
+    limit: int = Query(default=500, ge=1, le=5000),
+) -> dict[str, Any]:
+    path = manifest_path()
+    if not path.exists():
+        ensure_dataset_available()
+    if not path.exists():
+        return {"count": 0, "items": []}
+    frame = pd.read_csv(path)
+    if category:
+        frame = frame[frame["category"].astype(str).str.lower() == category.lower()]
+    if asset:
+        frame = frame[frame["asset"].astype(str).str.lower() == asset.lower()]
+    if timeframe:
+        frame = frame[frame["timeframe"].astype(str).str.lower() == timeframe.lower()]
+    return {"count": int(len(frame)), "items": safe_json(frame.head(limit).where(pd.notna(frame), None).to_dict(orient="records"))}
+
+
+@app.get("/api/data/sample")
+def data_sample(
+    category: str,
+    asset: str,
+    timeframe: str,
+    limit: int = Query(default=50, ge=1, le=1000),
+) -> dict[str, Any]:
+    path = manifest_path()
+    if not path.exists():
+        ensure_dataset_available()
+    if not path.exists():
+        raise HTTPException(status_code=404, detail="Data manifest not found.")
+    manifest = pd.read_csv(path)
+    matches = manifest[
+        (manifest["category"].astype(str).str.lower() == category.lower())
+        & (manifest["asset"].astype(str).str.lower() == asset.lower())
+        & (manifest["timeframe"].astype(str).str.lower() == timeframe.lower())
+    ]
+    if matches.empty:
+        raise HTTPException(status_code=404, detail="No matching dataset in manifest.")
+    dataset_path = resolve_dataset_path(str(matches.iloc[0]["path"]))
+    if not dataset_path.exists():
+        raise HTTPException(status_code=404, detail=f"Dataset file not found: {dataset_path}")
+    return {
+        "dataset": safe_json(matches.iloc[0].to_dict()),
+        "rows": csv_rows(dataset_path, limit=limit),
+    }
+
+
+@app.api_route("/api/cron/tick", methods=["GET", "POST"])
+async def cron_tick(
+    request: Request,
+    background_tasks: BackgroundTasks,
+    x_cron_secret: str | None = Header(default=None),
+) -> JSONResponse:
+    require_secret(x_cron_secret=x_cron_secret)
+    due = update_due()
+    started = False
+    if due:
+        background_tasks.add_task(start_update, "netlify_cron")
+        started = True
+    return JSONResponse({"ok": True, "checked_at": now_utc(), "update_due": due, "update_start_queued": started, "status": read_status()})
+
+
+@app.post("/api/update/start")
+def manual_update(
+    retrain: bool | None = None,
+    x_admin_secret: str | None = Header(default=None),
+) -> dict[str, Any]:
+    require_secret(x_admin_secret=x_admin_secret)
+    started = start_update("manual_api", retrain=retrain)
+    return {"ok": True, "started": started, "status": read_status()}
+
+
+@app.post("/api/dataset/sync")
+def sync_dataset(
+    force: bool = False,
+    x_admin_secret: str | None = Header(default=None),
+) -> dict[str, Any]:
+    require_secret(x_admin_secret=x_admin_secret)
+    ok = ensure_dataset_available(force=force)
+    return {"ok": ok, "dataset": dataset_status()}

backend/requirements.txt

@@ -0,0 +1,11 @@
+fastapi==0.115.12
+uvicorn[standard]==0.34.2
+pandas==2.2.3
+numpy==2.2.6
+requests==2.32.3
+scikit-learn==1.6.1
+joblib==1.4.2
+xgboost==3.0.1
+catboost==1.2.8
+lightgbm==4.6.0
+huggingface_hub==0.31.4

backend/research_runtime/Code/models/first_extrema_forecaster/__init__.py

@@ -0,0 +1 @@
+"""First intraday extrema forecaster."""

backend/research_runtime/Code/models/first_extrema_forecaster/outputs/summary.json

@@ -0,0 +1,174 @@
+{
+  "target": "Whether the stock's intraday high occurs before its intraday low, or its intraday low occurs before its intraday high.",
+  "label_1": "HIGH_FIRST",
+  "label_0": "LOW_FIRST",
+  "forecast_setup": "Use data known after the first 5 intraday minutes; automatically exclude days where the final intraday high or final intraday low occurred inside those first 5 minutes.",
+  "selection_policy": "fixed_symbol_map_v1",
+  "selected_candidate_by_symbol": {
+    "BEL": {
+      "name": "xgb_full_d2_lr0.07",
+      "kind": "xgboost",
+      "feature_profile": "full",
+      "threshold": 0.6174999999999999,
+      "threshold_mode": "per_symbol",
+      "symbol_thresholds": {
+        "BEL": 0.6174999999999999
+      },
+      "overlay_rule": null,
+      "validation_accuracy": 0.6846153846153846,
+      "test_accuracy": 0.6407766990291263
+    },
+    "CANBK": {
+      "name": "extra_no_vix_d7_leaf20",
+      "kind": "extra_trees",
+      "feature_profile": "no_vix",
+      "threshold": 0.5225,
+      "threshold_mode": "per_symbol",
+      "symbol_thresholds": {
+        "CANBK": 0.5225
+      },
+      "overlay_rule": null,
+      "validation_accuracy": 0.7210884353741497,
+      "test_accuracy": 0.7413793103448276
+    },
+    "ITC": {
+      "name": "xgb_full_d2_lr0.07",
+      "kind": "xgboost",
+      "feature_profile": "full",
+      "threshold": 0.47250000000000003,
+      "threshold_mode": "per_symbol",
+      "symbol_thresholds": {
+        "ITC": 0.47250000000000003
+      },
+      "overlay_rule": null,
+      "validation_accuracy": 0.751937984496124,
+      "test_accuracy": 0.7254901960784313
+    },
+    "NTPC": {
+      "name": "xgb_full_d3_lr0.045",
+      "kind": "xgboost",
+      "feature_profile": "full",
+      "threshold": 0.5575,
+      "threshold_mode": "per_symbol",
+      "symbol_thresholds": {
+        "NTPC": 0.5575
+      },
+      "overlay_rule": null,
+      "validation_accuracy": 0.7619047619047619,
+      "test_accuracy": 0.6728971962616822
+    },
+    "ONGC": {
+      "name": "xgb_full_d2_lr0.07",
+      "kind": "xgboost",
+      "feature_profile": "full",
+      "threshold": 0.5700000000000001,
+      "threshold_mode": "per_symbol",
+      "symbol_thresholds": {
+        "ONGC": 0.5700000000000001
+      },
+      "overlay_rule": null,
+      "validation_accuracy": 0.7086614173228346,
+      "test_accuracy": 0.6272727272727273
+    },
+    "POWERGRID": {
+      "name": "extra_no_vix_d7_leaf20",
+      "kind": "extra_trees",
+      "feature_profile": "no_vix",
+      "threshold": 0.455,
+      "threshold_mode": "per_symbol",
+      "symbol_thresholds": {
+        "POWERGRID": 0.455
+      },
+      "overlay_rule": null,
+      "validation_accuracy": 0.6916666666666667,
+      "test_accuracy": 0.7641509433962265
+    },
+    "SBIN": {
+      "name": "histgb_compact_lr0.065_leaf7+sbin_overlay_v1",
+      "kind": "histgb",
+      "feature_profile": "compact",
+      "threshold": 0.4,
+      "threshold_mode": "per_symbol",
+      "symbol_thresholds": {
+        "SBIN": 0.4
+      },
+      "overlay_rule": {
+        "feature": "n50_ret_2d",
+        "feature_threshold": -0.005959,
+        "feature_direction": "<=",
+        "force_prediction": "LOW_FIRST",
+        "model_threshold": 0.4,
+        "note": "SBIN-specific overlay found in targeted tuning; corrects HIGH_FIRST under-prediction."
+      },
+      "validation_accuracy": 0.6052631578947368,
+      "test_accuracy": 0.6992481203007519
+    },
+    "TATASTEEL": {
+      "name": "cat_full_d3_lr0.025",
+      "kind": "catboost",
+      "feature_profile": "full",
+      "threshold": 0.42000000000000004,
+      "threshold_mode": "per_symbol",
+      "symbol_thresholds": {
+        "TATASTEEL": 0.42000000000000004
+      },
+      "overlay_rule": null,
+      "validation_accuracy": 0.719626168224299,
+      "test_accuracy": 0.6698113207547169
+    }
+  },
+  "test_accuracy_by_symbol": {
+    "POWERGRID": 0.7641509433962265,
+    "CANBK": 0.7413793103448276,
+    "ITC": 0.7254901960784313,
+    "SBIN": 0.6992481203007519,
+    "NTPC": 0.6728971962616822,
+    "TATASTEEL": 0.6698113207547169,
+    "BEL": 0.6407766990291263,
+    "ONGC": 0.6272727272727273
+  },
+  "naive_baseline_accuracy_by_symbol": {
+    "BEL": 0.5533980582524272,
+    "CANBK": 0.5258620689655172,
+    "ITC": 0.47058823529411764,
+    "NTPC": 0.5794392523364486,
+    "ONGC": 0.509090909090909,
+    "POWERGRID": 0.49056603773584906,
+    "SBIN": 0.5112781954887218,
+    "TATASTEEL": 0.5
+  },
+  "test_accuracy_edge_vs_naive_by_symbol": {
+    "POWERGRID": 0.2735849056603774,
+    "CANBK": 0.2155172413793104,
+    "ITC": 0.2549019607843137,
+    "SBIN": 0.18796992481203012,
+    "NTPC": 0.09345794392523366,
+    "TATASTEEL": 0.16981132075471694,
+    "BEL": 0.0873786407766991,
+    "ONGC": 0.11818181818181828
+  },
+  "aggregate_test_accuracy": 0.6930917327293318,
+  "train_date_range": [
+    "2015-02-02",
+    "2023-12-29"
+  ],
+  "validation_date_range": [
+    "2024-01-01",
+    "2025-02-25"
+  ],
+  "test_date_range": [
+    "2025-02-27",
+    "2026-05-08"
+  ],
+  "symbols": [
+    "BEL",
+    "CANBK",
+    "ITC",
+    "NTPC",
+    "ONGC",
+    "POWERGRID",
+    "SBIN",
+    "TATASTEEL"
+  ],
+  "coverage_valid_labeled_rows": 10509
+}

backend/research_runtime/Code/models/first_extrema_forecaster/train.py

@@ -0,0 +1,1466 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import random
+import sys
+import time
+import warnings
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any
+
+os.environ.setdefault("OMP_NUM_THREADS", "2")
+os.environ.setdefault("OPENBLAS_NUM_THREADS", "2")
+os.environ.setdefault("MKL_NUM_THREADS", "2")
+os.environ.setdefault("VECLIB_MAXIMUM_THREADS", "2")
+os.environ.setdefault("NUMEXPR_NUM_THREADS", "2")
+
+import joblib
+import numpy as np
+import pandas as pd
+from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier
+from sklearn.ensemble import HistGradientBoostingClassifier
+from sklearn.impute import SimpleImputer
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import accuracy_score, balanced_accuracy_score, confusion_matrix, log_loss, roc_auc_score
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import StandardScaler
+
+warnings.filterwarnings("ignore", category=FutureWarning)
+warnings.filterwarnings("ignore", category=pd.errors.PerformanceWarning)
+
+try:
+    from catboost import CatBoostClassifier
+except Exception:  # pragma: no cover - optional local dependency
+    CatBoostClassifier = None
+
+try:
+    from lightgbm import LGBMClassifier
+except Exception:  # pragma: no cover - optional local dependency
+    LGBMClassifier = None
+
+try:
+    from xgboost import XGBClassifier
+except Exception:  # pragma: no cover - optional local dependency
+    XGBClassifier = None
+
+def find_project_root(start: Path) -> Path:
+    for path in (start, *start.parents):
+        if (path / "Data").is_dir() and (path / "Alt Data").is_dir():
+            return path
+    raise RuntimeError(f"Could not find project root from {start}")
+
+
+PROJECT_ROOT = find_project_root(Path(__file__).resolve())
+DATA_DIR = PROJECT_ROOT / "Data"
+RAW_MINUTE_DIR = DATA_DIR / "raw" / "minute"
+DAILY_MASTER_PATH = DATA_DIR / "processed" / "panels" / "daily_master_panel.csv"
+OUTPUT_DIR = Path(__file__).resolve().parent / "outputs"
+OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+CACHE_PATH = OUTPUT_DIR / "first5_extrema_after5_dataset.csv"
+
+MARKET_SYMBOLS = ("NIFTY 50", "NIFTY BANK", "INDIA VIX")
+DEFAULT_VALID_START = pd.Timestamp("2024-01-01")
+DEFAULT_TEST_START = pd.Timestamp("2025-02-27")
+RANDOM_SEED = 42
+FIXED_SYMBOL_CANDIDATE_MAP_V1 = {
+    "BEL": "xgb_full_d2_lr0.07",
+    "CANBK": "extra_no_vix_d7_leaf20",
+    "ITC": "xgb_full_d2_lr0.07",
+    "NTPC": "xgb_full_d3_lr0.045",
+    "ONGC": "xgb_full_d2_lr0.07",
+    "POWERGRID": "extra_no_vix_d7_leaf20",
+    "SBIN": "histgb_compact_lr0.065_leaf7",
+    "TATASTEEL": "cat_full_d3_lr0.025",
+}
+SBIN_OVERLAY_RULE_V1 = {
+    "feature": "n50_ret_2d",
+    "feature_threshold": -0.005959,
+    "feature_direction": "<=",
+    "force_prediction": "LOW_FIRST",
+    "model_threshold": 0.400,
+    "note": "SBIN-specific overlay found in targeted tuning; corrects HIGH_FIRST under-prediction.",
+}
+
+
+def note(message: str) -> None:
+    print(f"[first-extrema] {message}", file=sys.stderr, flush=True)
+
+
+@dataclass(frozen=True)
+class Candidate:
+    name: str
+    kind: str
+    params: dict[str, Any]
+    feature_profile: str
+
+
+@dataclass
+class CandidateResult:
+    name: str
+    kind: str
+    params: dict[str, Any]
+    feature_profile: str
+    feature_count: int
+    threshold: float
+    threshold_mode: str
+    symbol_thresholds: dict[str, float]
+    validation_accuracy: float
+    validation_balanced_accuracy: float
+    validation_auc: float | None
+    validation_log_loss: float | None
+    test_accuracy: float
+    test_balanced_accuracy: float
+    test_auc: float | None
+    test_log_loss: float | None
+    train_accuracy: float
+    n_train: int
+    n_valid: int
+    n_test: int
+    fit_seconds: float
+
+
+class SingleFeatureRuleClassifier:
+    """A monotonic one-feature classifier with probability-like rank scores."""
+
+    def __init__(self, feature: str, direction: int = 1) -> None:
+        self.feature = feature
+        self.direction = 1 if direction >= 0 else -1
+        self.median_: float = 0.0
+        self.sorted_scores_: np.ndarray | None = None
+
+    def fit(self, x: pd.DataFrame, y: np.ndarray) -> "SingleFeatureRuleClassifier":
+        values = pd.to_numeric(x[self.feature], errors="coerce").replace([np.inf, -np.inf], np.nan)
+        self.median_ = float(values.median()) if values.notna().any() else 0.0
+        scores = self.direction * values.fillna(self.median_).to_numpy(dtype="float64")
+        self.sorted_scores_ = np.sort(scores)
+        return self
+
+    def predict_proba(self, x: pd.DataFrame) -> np.ndarray:
+        if self.sorted_scores_ is None:
+            raise RuntimeError("Model is not fitted.")
+        values = pd.to_numeric(x[self.feature], errors="coerce").replace([np.inf, -np.inf], np.nan)
+        scores = self.direction * values.fillna(self.median_).to_numpy(dtype="float64")
+        ranks = np.searchsorted(self.sorted_scores_, scores, side="right") / max(1, len(self.sorted_scores_))
+        ranks = np.clip(ranks, 1e-6, 1.0 - 1e-6)
+        return np.column_stack([1.0 - ranks, ranks])
+
+
+def symbol_from_minute_file(path: Path) -> str:
+    return path.name.removesuffix("_minute.csv")
+
+
+def discover_symbols() -> list[str]:
+    symbols = sorted(symbol_from_minute_file(p) for p in RAW_MINUTE_DIR.glob("*_minute.csv"))
+    return symbols
+
+
+def target_symbols(all_symbols: list[str]) -> list[str]:
+    return [s for s in all_symbols if s not in MARKET_SYMBOLS]
+
+
+def safe_div(numer: pd.Series | np.ndarray, denom: pd.Series | np.ndarray) -> pd.Series:
+    n = pd.Series(numer, copy=False)
+    d = pd.Series(denom, copy=False)
+    out = pd.Series(np.nan, index=n.index, dtype="float64")
+    mask = d.notna() & np.isfinite(d.to_numpy(dtype="float64")) & (d != 0)
+    out.loc[mask] = n.loc[mask].to_numpy(dtype="float64") / d.loc[mask].to_numpy(dtype="float64")
+    return out
+
+
+def read_minute_daily(symbol: str) -> pd.DataFrame:
+    path = RAW_MINUTE_DIR / f"{symbol}_minute.csv"
+    if not path.exists():
+        raise FileNotFoundError(path)
+    note(f"aggregating {symbol} minute file")
+    df = pd.read_csv(
+        path,
+        usecols=["date", "open", "high", "low", "close", "volume"],
+        dtype={"open": "float64", "high": "float64", "low": "float64", "close": "float64", "volume": "float64"},
+    )
+    df["dt"] = pd.to_datetime(df.pop("date"), errors="coerce")
+    df = df.dropna(subset=["dt", "open", "high", "low", "close"]).sort_values("dt")
+    df["date"] = df["dt"].dt.normalize()
+    df["minute_index"] = df.groupby("date", sort=True).cumcount()
+
+    grouped = df.groupby("date", sort=True)
+    daily = grouped.agg(
+        open=("open", "first"),
+        high=("high", "max"),
+        low=("low", "min"),
+        close=("close", "last"),
+        volume=("volume", "sum"),
+        bars=("close", "size"),
+        open_time=("dt", "first"),
+        close_time=("dt", "last"),
+    )
+
+    day_high = grouped["high"].transform("max")
+    day_low = grouped["low"].transform("min")
+    high_time = df.loc[df["high"].eq(day_high)].groupby("date", sort=True)["dt"].min()
+    low_time = df.loc[df["low"].eq(day_low)].groupby("date", sort=True)["dt"].min()
+    daily["high_time"] = high_time
+    daily["low_time"] = low_time
+    daily["high_minute_from_open"] = (daily["high_time"] - daily["open_time"]).dt.total_seconds() / 60.0
+    daily["low_minute_from_open"] = (daily["low_time"] - daily["open_time"]).dt.total_seconds() / 60.0
+    daily["high_in_first5"] = daily["high_minute_from_open"].lt(5.0)
+    daily["low_in_first5"] = daily["low_minute_from_open"].lt(5.0)
+    daily["target_high_first"] = np.where(daily["high_time"] < daily["low_time"], 1.0, 0.0)
+    daily.loc[daily["high_time"].eq(daily["low_time"]), "target_high_first"] = np.nan
+
+    intraday_parts = []
+    for n in (5, 15, 30, 60, 120):
+        early = df[df["minute_index"] < n].groupby("date", sort=True).agg(
+            **{
+                f"first{n}_high": ("high", "max"),
+                f"first{n}_low": ("low", "min"),
+                f"first{n}_close": ("close", "last"),
+                f"first{n}_volume": ("volume", "sum"),
+            }
+        )
+        intraday_parts.append(early)
+    for start, end, name in ((0, 60, "first_hour"), (60, 180, "midday"), (180, 10_000, "late")):
+        part = df[(df["minute_index"] >= start) & (df["minute_index"] < end)].groupby("date", sort=True).agg(
+            **{
+                f"{name}_open": ("open", "first"),
+                f"{name}_high": ("high", "max"),
+                f"{name}_low": ("low", "min"),
+                f"{name}_close": ("close", "last"),
+                f"{name}_volume": ("volume", "sum"),
+            }
+        )
+        intraday_parts.append(part)
+    if intraday_parts:
+        daily = daily.join(pd.concat(intraday_parts, axis=1), how="left")
+    for n in (5, 15, 30, 60, 120):
+        daily[f"first{n}_return"] = safe_div(daily[f"first{n}_close"], daily["open"]) - 1.0
+        daily[f"first{n}_range_pct"] = safe_div(daily[f"first{n}_high"] - daily[f"first{n}_low"], daily["open"])
+        daily[f"first{n}_high_excursion"] = safe_div(daily[f"first{n}_high"], daily["open"]) - 1.0
+        daily[f"first{n}_low_excursion"] = safe_div(daily[f"first{n}_low"], daily["open"]) - 1.0
+        daily[f"first{n}_volume_share"] = safe_div(daily[f"first{n}_volume"], daily["volume"])
+    for name in ("first_hour", "midday", "late"):
+        daily[f"{name}_return"] = safe_div(daily[f"{name}_close"], daily[f"{name}_open"]) - 1.0
+        daily[f"{name}_range_pct"] = safe_div(daily[f"{name}_high"] - daily[f"{name}_low"], daily[f"{name}_open"])
+        daily[f"{name}_volume_share"] = safe_div(daily[f"{name}_volume"], daily["volume"])
+    daily["close_position"] = safe_div(daily["close"] - daily["low"], daily["high"] - daily["low"])
+    daily["morning_vs_late_return"] = daily["first_hour_return"] - daily["late_return"]
+    daily["symbol"] = symbol
+    return daily.reset_index()
+
+
+def rsi(close: pd.Series, window: int = 14) -> pd.Series:
+    delta = close.diff()
+    gain = delta.clip(lower=0).rolling(window, min_periods=window).mean()
+    loss = (-delta.clip(upper=0)).rolling(window, min_periods=window).mean()
+    rs = safe_div(gain, loss)
+    return 100.0 - (100.0 / (1.0 + rs))
+
+
+def history_features(daily: pd.DataFrame, prefix: str, include_target_history: bool) -> pd.DataFrame:
+    d = daily.sort_values("date").reset_index(drop=True).copy()
+    out = pd.DataFrame({"date": d["date"]})
+    open_ = d["open"]
+    high = d["high"]
+    low = d["low"]
+    close = d["close"]
+    volume = d["volume"].replace(0, np.nan)
+
+    prev_open = open_.shift(1)
+    prev_high = high.shift(1)
+    prev_low = low.shift(1)
+    prev_close = close.shift(1)
+    prev_volume = volume.shift(1)
+    prev_range = prev_high - prev_low
+    prev_range_pct = safe_div(prev_range, prev_close)
+    prev_body_pct = safe_div(close.shift(1) - open_.shift(1), open_.shift(1))
+    ret1 = close.pct_change()
+    log_ret1 = np.log(safe_div(close, close.shift(1))).replace([np.inf, -np.inf], np.nan)
+    true_range = pd.concat(
+        [(high - low), (high - close.shift(1)).abs(), (low - close.shift(1)).abs()],
+        axis=1,
+    ).max(axis=1)
+    if {"high_time", "low_time", "open_time"}.issubset(d.columns):
+        high_minute = (d["high_time"] - d["open_time"]).dt.total_seconds() / 60.0
+        low_minute = (d["low_time"] - d["open_time"]).dt.total_seconds() / 60.0
+        first_extrema_minute = pd.concat([high_minute, low_minute], axis=1).min(axis=1)
+        extrema_time_gap = high_minute - low_minute
+        out[f"{prefix}prev_high_minute"] = high_minute.shift(1)
+        out[f"{prefix}prev_low_minute"] = low_minute.shift(1)
+        out[f"{prefix}prev_first_extrema_minute"] = first_extrema_minute.shift(1)
+        out[f"{prefix}prev_extrema_time_gap"] = extrema_time_gap.shift(1)
+        for window in (3, 5, 10, 20):
+            out[f"{prefix}high_minute_mean_{window}d"] = high_minute.shift(1).rolling(window, min_periods=max(2, window // 2)).mean()
+            out[f"{prefix}low_minute_mean_{window}d"] = low_minute.shift(1).rolling(window, min_periods=max(2, window // 2)).mean()
+            out[f"{prefix}first_extrema_minute_mean_{window}d"] = first_extrema_minute.shift(1).rolling(window, min_periods=max(2, window // 2)).mean()
+            out[f"{prefix}extrema_time_gap_mean_{window}d"] = extrema_time_gap.shift(1).rolling(window, min_periods=max(2, window // 2)).mean()
+
+    out[f"{prefix}current_open_log"] = np.log(open_.replace(0, np.nan))
+    out[f"{prefix}prev_close_log"] = np.log(prev_close.replace(0, np.nan))
+    out[f"{prefix}gap_pct"] = safe_div(open_, prev_close) - 1.0
+    out[f"{prefix}gap_abs_pct"] = out[f"{prefix}gap_pct"].abs()
+    out[f"{prefix}open_vs_prev_high"] = safe_div(open_, prev_high) - 1.0
+    out[f"{prefix}open_vs_prev_low"] = safe_div(open_, prev_low) - 1.0
+    out[f"{prefix}open_vs_prev_mid"] = safe_div(open_, (prev_high + prev_low) / 2.0) - 1.0
+    out[f"{prefix}open_pos_prev_range"] = safe_div(open_ - prev_low, prev_range)
+    out[f"{prefix}prev_range_pct"] = prev_range_pct
+    out[f"{prefix}prev_body_pct"] = prev_body_pct
+    out[f"{prefix}prev_upper_wick_pct"] = safe_div(prev_high - pd.concat([prev_open, prev_close], axis=1).max(axis=1), prev_close)
+    out[f"{prefix}prev_lower_wick_pct"] = safe_div(pd.concat([prev_open, prev_close], axis=1).min(axis=1) - prev_low, prev_close)
+    out[f"{prefix}prev_volume_log"] = np.log1p(prev_volume)
+    intraday_shape_cols = [
+        c
+        for c in d.columns
+        if c.endswith(("_return", "_range_pct", "_high_excursion", "_low_excursion", "_volume_share"))
+        or c in ("close_position", "morning_vs_late_return")
+    ]
+    for c in intraday_shape_cols:
+        out[f"{prefix}prev_{c}"] = d[c].shift(1)
+        for window in (3, 5, 10, 20):
+            out[f"{prefix}{c}_mean_{window}d"] = d[c].shift(1).rolling(window, min_periods=max(2, window // 2)).mean()
+            out[f"{prefix}{c}_std_{window}d"] = d[c].shift(1).rolling(window, min_periods=max(2, window // 2)).std()
+
+    for window in (2, 3, 5, 10, 20, 40, 60):
+        shifted_close = close.shift(1)
+        rolling_high = high.shift(1).rolling(window, min_periods=max(2, window // 2)).max()
+        rolling_low = low.shift(1).rolling(window, min_periods=max(2, window // 2)).min()
+        rolling_range = rolling_high - rolling_low
+        out[f"{prefix}ret_{window}d"] = safe_div(shifted_close, close.shift(1 + window)) - 1.0
+        out[f"{prefix}ret_mean_{window}d"] = ret1.shift(1).rolling(window, min_periods=max(2, window // 2)).mean()
+        out[f"{prefix}ret_std_{window}d"] = ret1.shift(1).rolling(window, min_periods=max(2, window // 2)).std()
+        out[f"{prefix}abs_ret_mean_{window}d"] = ret1.abs().shift(1).rolling(window, min_periods=max(2, window // 2)).mean()
+        out[f"{prefix}range_mean_{window}d"] = safe_div(high - low, close).shift(1).rolling(window, min_periods=max(2, window // 2)).mean()
+        out[f"{prefix}range_std_{window}d"] = safe_div(high - low, close).shift(1).rolling(window, min_periods=max(2, window // 2)).std()
+        out[f"{prefix}volume_z_{window}d"] = safe_div(prev_volume - volume.shift(1).rolling(window, min_periods=max(2, window // 2)).mean(), volume.shift(1).rolling(window, min_periods=max(2, window // 2)).std())
+        out[f"{prefix}gap_over_vol_{window}d"] = safe_div(out[f"{prefix}gap_pct"], out[f"{prefix}ret_std_{window}d"])
+        out[f"{prefix}open_vs_sma_{window}d"] = safe_div(open_, close.shift(1).rolling(window, min_periods=max(2, window // 2)).mean()) - 1.0
+        out[f"{prefix}open_vs_roll_high_{window}d"] = safe_div(open_, rolling_high) - 1.0
+        out[f"{prefix}open_vs_roll_low_{window}d"] = safe_div(open_, rolling_low) - 1.0
+        out[f"{prefix}open_pos_roll_range_{window}d"] = safe_div(open_ - rolling_low, rolling_range)
+
+    for span in (5, 12, 26, 50):
+        ema = close.ewm(span=span, adjust=False, min_periods=max(2, span // 2)).mean().shift(1)
+        out[f"{prefix}open_vs_ema_{span}d"] = safe_div(open_, ema) - 1.0
+        out[f"{prefix}prev_close_vs_ema_{span}d"] = safe_div(prev_close, ema) - 1.0
+
+    atr14 = true_range.rolling(14, min_periods=7).mean().shift(1)
+    out[f"{prefix}atr14_pct"] = safe_div(atr14, prev_close)
+    out[f"{prefix}gap_over_atr14"] = safe_div(open_ - prev_close, atr14)
+    out[f"{prefix}rsi14_prev"] = rsi(close, 14).shift(1)
+    out[f"{prefix}bars_prev"] = d["bars"].shift(1)
+
+    if include_target_history and "target_high_first" in d.columns:
+        y_prev = d["target_high_first"].shift(1)
+        out[f"{prefix}prev_high_first"] = y_prev
+        for window in (3, 5, 10, 20, 60):
+            out[f"{prefix}high_first_rate_{window}d"] = d["target_high_first"].shift(1).rolling(window, min_periods=max(2, window // 2)).mean()
+
+    return out
+
+
+def breadth_features(dailies: dict[str, pd.DataFrame], symbols: list[str]) -> pd.DataFrame:
+    frames = []
+    for symbol in symbols:
+        d = dailies[symbol].sort_values("date").copy()
+        d["ret1"] = d["close"].pct_change()
+        d["range_pct"] = safe_div(d["high"] - d["low"], d["close"])
+        frames.append(d[["date", "ret1", "range_pct", "target_high_first"]].assign(symbol=symbol))
+    panel = pd.concat(frames, ignore_index=True)
+    by_date = panel.groupby("date", sort=True).agg(
+        breadth_ret_mean=("ret1", "mean"),
+        breadth_ret_std=("ret1", "std"),
+        breadth_range_mean=("range_pct", "mean"),
+        breadth_high_first_rate=("target_high_first", "mean"),
+    )
+    return by_date.shift(1).reset_index()
+
+
+def open_cross_section_features(dailies: dict[str, pd.DataFrame], symbols: list[str]) -> pd.DataFrame:
+    frames = []
+    for symbol in symbols:
+        d = dailies[symbol].sort_values("date").copy()
+        d["stock_open_gap"] = safe_div(d["open"], d["close"].shift(1)) - 1.0
+        d["stock_open_vs_prev_high"] = safe_div(d["open"], d["high"].shift(1)) - 1.0
+        d["stock_open_vs_prev_low"] = safe_div(d["open"], d["low"].shift(1)) - 1.0
+        frames.append(d[["date", "symbol", "stock_open_gap", "stock_open_vs_prev_high", "stock_open_vs_prev_low"]])
+    opens = pd.concat(frames, ignore_index=True)
+    g = opens.groupby("date", sort=True)
+    opens["open_gap_cs_mean"] = g["stock_open_gap"].transform("mean")
+    opens["open_gap_cs_std"] = g["stock_open_gap"].transform("std")
+    opens["open_gap_cs_min"] = g["stock_open_gap"].transform("min")
+    opens["open_gap_cs_max"] = g["stock_open_gap"].transform("max")
+    opens["open_gap_cs_rank_pct"] = g["stock_open_gap"].rank(pct=True)
+    opens["open_gap_vs_peer_mean"] = opens["stock_open_gap"] - opens["open_gap_cs_mean"]
+    opens["open_gap_cs_z"] = safe_div(opens["open_gap_vs_peer_mean"], opens["open_gap_cs_std"])
+    opens["open_gap_positive_share"] = g["stock_open_gap"].transform(lambda s: float((s > 0).mean()))
+    opens["open_above_prev_high_share"] = g["stock_open_vs_prev_high"].transform(lambda s: float((s > 0).mean()))
+    opens["open_below_prev_low_share"] = g["stock_open_vs_prev_low"].transform(lambda s: float((s < 0).mean()))
+    return opens[
+        [
+            "date",
+            "symbol",
+            "open_gap_cs_mean",
+            "open_gap_cs_std",
+            "open_gap_cs_min",
+            "open_gap_cs_max",
+            "open_gap_cs_rank_pct",
+            "open_gap_vs_peer_mean",
+            "open_gap_cs_z",
+            "open_gap_positive_share",
+            "open_above_prev_high_share",
+            "open_below_prev_low_share",
+        ]
+    ]
+
+
+def master_previous_day_features() -> pd.DataFrame | None:
+    if not DAILY_MASTER_PATH.exists():
+        return None
+    note("loading previous-day daily_master_panel features")
+    master = pd.read_csv(DAILY_MASTER_PATH)
+    if "date" not in master.columns:
+        return None
+    master["date"] = pd.to_datetime(master["date"], errors="coerce")
+    master = master.dropna(subset=["date"]).sort_values("date").reset_index(drop=True)
+    for col in list(master.columns):
+        if col == "date":
+            continue
+        if col.endswith("_close"):
+            base = col[: -len("_close")]
+            value_col = f"{base}_value"
+            change_col = f"{base}_change_1"
+            if value_col not in master.columns:
+                master[value_col] = pd.to_numeric(master[col], errors="coerce")
+            if change_col not in master.columns:
+                master[change_col] = pd.to_numeric(master[value_col], errors="coerce").diff()
+        elif col.endswith("_value"):
+            base = col[: -len("_value")]
+            close_col = f"{base}_close"
+            change_col = f"{base}_change_1"
+            if close_col not in master.columns:
+                master[close_col] = pd.to_numeric(master[col], errors="coerce")
+            if change_col not in master.columns:
+                master[change_col] = pd.to_numeric(master[col], errors="coerce").diff()
+    keep = [
+        c
+        for c in master.columns
+        if c != "date"
+        and "target" not in c.lower()
+        and not c.lower().endswith("_future")
+    ]
+    numeric = master[keep].apply(pd.to_numeric, errors="coerce")
+    shifted = numeric.shift(1)
+    shifted.columns = [f"master_prev_{c}" for c in shifted.columns]
+    return pd.concat([master[["date"]], shifted], axis=1)
+
+
+def current_first5_features(daily: pd.DataFrame) -> pd.DataFrame:
+    d = daily.sort_values("date").copy()
+    out = pd.DataFrame({"date": d["date"]})
+    out["cur5_return"] = d["first5_return"]
+    out["cur5_range_pct"] = d["first5_range_pct"]
+    out["cur5_high_excursion"] = d["first5_high_excursion"]
+    out["cur5_low_excursion"] = d["first5_low_excursion"]
+    out["cur5_volume_share"] = d["first5_volume_share"]
+    out["cur5_volume_log"] = np.log1p(d["first5_volume"].replace(0, np.nan))
+    out["cur5_close_pos"] = safe_div(d["first5_close"] - d["first5_low"], d["first5_high"] - d["first5_low"])
+    out["cur5_body_vs_range"] = safe_div(d["first5_close"] - d["open"], d["first5_high"] - d["first5_low"])
+    out["cur5_upper_wick_pct"] = safe_div(d["first5_high"] - pd.concat([d["open"], d["first5_close"]], axis=1).max(axis=1), d["open"])
+    out["cur5_lower_wick_pct"] = safe_div(pd.concat([d["open"], d["first5_close"]], axis=1).min(axis=1) - d["first5_low"], d["open"])
+    out["cur5_direction_up"] = d["first5_close"].gt(d["open"]).astype(float)
+    return out
+
+
+def build_dataset(dailies: dict[str, pd.DataFrame], targets: list[str]) -> pd.DataFrame:
+    market_features = []
+    for symbol, prefix in (("NIFTY 50", "n50_"), ("NIFTY BANK", "bank_"), ("INDIA VIX", "vix_")):
+        if symbol in dailies:
+            market_features.append(history_features(dailies[symbol], prefix=prefix, include_target_history=True))
+
+    breadth = breadth_features(dailies, targets)
+    open_cs = open_cross_section_features(dailies, targets)
+    master_prev = master_previous_day_features()
+    rows = []
+    for symbol in targets:
+        base = dailies[symbol].copy()
+        feats = history_features(base, prefix="stk_", include_target_history=True)
+        cur5 = current_first5_features(base)
+        frame = base[
+            [
+                "date",
+                "symbol",
+                "open",
+                "open_time",
+                "high_time",
+                "low_time",
+                "high_minute_from_open",
+                "low_minute_from_open",
+                "high_in_first5",
+                "low_in_first5",
+                "target_high_first",
+                "bars",
+            ]
+        ].merge(feats, on="date", how="left")
+        frame = frame.merge(cur5, on="date", how="left")
+        for mf in market_features:
+            frame = frame.merge(mf, on="date", how="left")
+        frame = frame.merge(breadth, on="date", how="left")
+        frame = frame.merge(open_cs, on=["date", "symbol"], how="left")
+        if master_prev is not None:
+            frame = frame.merge(master_prev, on="date", how="left")
+        rows.append(frame)
+
+    data = pd.concat(rows, ignore_index=True).sort_values(["date", "symbol"]).reset_index(drop=True)
+    data["target"] = data["target_high_first"].astype("float64")
+    data["dow"] = data["date"].dt.dayofweek
+    data["month"] = data["date"].dt.month
+    data["day_of_month"] = data["date"].dt.day
+    data["day_of_year"] = data["date"].dt.dayofyear
+    data["dow_sin"] = np.sin(2 * np.pi * data["dow"] / 5.0)
+    data["dow_cos"] = np.cos(2 * np.pi * data["dow"] / 5.0)
+    data["month_sin"] = np.sin(2 * np.pi * data["month"] / 12.0)
+    data["month_cos"] = np.cos(2 * np.pi * data["month"] / 12.0)
+    data["is_month_start"] = data["date"].dt.is_month_start.astype(int)
+    data["is_month_end"] = data["date"].dt.is_month_end.astype(int)
+
+    data = data[
+        data["bars"].ge(120)
+        & data["target"].notna()
+        & ~data["high_in_first5"].fillna(False)
+        & ~data["low_in_first5"].fillna(False)
+    ].copy()
+    return data.reset_index(drop=True)
+
+
+def feature_columns(data: pd.DataFrame, profile: str) -> list[str]:
+    blocked = {
+        "date",
+        "target",
+        "target_high_first",
+        "open_time",
+        "high_time",
+        "low_time",
+        "high_minute_from_open",
+        "low_minute_from_open",
+        "high_in_first5",
+        "low_in_first5",
+        "high",
+        "low",
+        "open",
+        "bars",
+    }
+    cols = [c for c in data.columns if c not in blocked]
+    if profile == "full":
+        return cols
+    if profile == "no_market":
+        return [c for c in cols if not c.startswith(("n50_", "bank_", "vix_", "master_prev_"))]
+    if profile == "no_vix":
+        return [c for c in cols if not c.startswith(("vix_", "master_prev_india_vix"))]
+    if profile == "stock_only":
+        return [c for c in cols if c == "symbol" or c.startswith("stk_") or c in CALENDAR_COLUMNS]
+    if profile == "first5_rule":
+        return ["cur5_close_pos"]
+    if profile == "compact":
+        keep_prefixes = (
+            "stk_gap",
+            "stk_open_",
+            "stk_prev_",
+            "stk_ret_",
+            "stk_high_first",
+            "cur5_",
+            "n50_gap",
+            "bank_gap",
+            "vix_gap",
+            "breadth_",
+        )
+        return [c for c in cols if c == "symbol" or c in CALENDAR_COLUMNS or c.startswith(keep_prefixes)]
+    raise ValueError(f"Unknown profile: {profile}")
+
+
+CALENDAR_COLUMNS = {
+    "dow",
+    "month",
+    "day_of_month",
+    "day_of_year",
+    "dow_sin",
+    "dow_cos",
+    "month_sin",
+    "month_cos",
+    "is_month_start",
+    "is_month_end",
+}
+
+
+def split_data(data: pd.DataFrame, valid_start: pd.Timestamp, test_start: pd.Timestamp) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
+    train = data[data["date"] < valid_start].copy()
+    valid = data[(data["date"] >= valid_start) & (data["date"] < test_start)].copy()
+    test = data[data["date"] >= test_start].copy()
+    return train, valid, test
+
+
+def prepare_xy(train: pd.DataFrame, valid: pd.DataFrame, test: pd.DataFrame, cols: list[str], kind: str):
+    y_train = train["target"].astype(int).to_numpy()
+    y_valid = valid["target"].astype(int).to_numpy()
+    y_test = test["target"].astype(int).to_numpy()
+
+    if kind == "catboost":
+        x_train = train[cols].copy()
+        x_valid = valid[cols].copy()
+        x_test = test[cols].copy()
+        for x in (x_train, x_valid, x_test):
+            if "symbol" in x.columns:
+                x["symbol"] = x["symbol"].astype(str)
+        cat_features = ["symbol"] if "symbol" in cols else []
+        return x_train, y_train, x_valid, y_valid, x_test, y_test, cat_features
+
+    all_x = pd.concat([train[cols], valid[cols], test[cols]], axis=0)
+    all_x = pd.get_dummies(all_x, columns=["symbol"] if "symbol" in all_x.columns else [], dummy_na=False)
+    x_train = all_x.iloc[: len(train)].copy()
+    x_valid = all_x.iloc[len(train) : len(train) + len(valid)].copy()
+    x_test = all_x.iloc[len(train) + len(valid) :].copy()
+    medians = x_train.median(numeric_only=True).replace([np.inf, -np.inf], np.nan)
+    x_train = x_train.replace([np.inf, -np.inf], np.nan).fillna(medians).fillna(0.0)
+    x_valid = x_valid.replace([np.inf, -np.inf], np.nan).fillna(medians).fillna(0.0)
+    x_test = x_test.replace([np.inf, -np.inf], np.nan).fillna(medians).fillna(0.0)
+    return x_train, y_train, x_valid, y_valid, x_test, y_test, []
+
+
+def build_model(candidate: Candidate):
+    params = candidate.params.copy()
+    if candidate.kind == "histgb":
+        return HistGradientBoostingClassifier(random_state=RANDOM_SEED, **params)
+    if candidate.kind == "extra_trees":
+        return ExtraTreesClassifier(random_state=RANDOM_SEED, n_jobs=2, **params)
+    if candidate.kind == "random_forest":
+        return RandomForestClassifier(random_state=RANDOM_SEED, n_jobs=2, **params)
+    if candidate.kind == "logistic":
+        c = params.pop("C")
+        return make_pipeline(
+            SimpleImputer(strategy="median"),
+            StandardScaler(),
+            LogisticRegression(C=c, max_iter=2000, class_weight=params.get("class_weight"), solver="lbfgs"),
+        )
+    if candidate.kind == "single_feature_rule":
+        return SingleFeatureRuleClassifier(**params)
+    if candidate.kind == "catboost":
+        if CatBoostClassifier is None:
+            raise RuntimeError("catboost is not installed")
+        return CatBoostClassifier(
+            loss_function="Logloss",
+            eval_metric="Accuracy",
+            random_seed=RANDOM_SEED,
+            thread_count=2,
+            allow_writing_files=False,
+            verbose=False,
+            **params,
+        )
+    if candidate.kind == "lightgbm":
+        if LGBMClassifier is None:
+            raise RuntimeError("lightgbm is not installed")
+        return LGBMClassifier(random_state=RANDOM_SEED, n_jobs=2, verbosity=-1, **params)
+    if candidate.kind == "xgboost":
+        if XGBClassifier is None:
+            raise RuntimeError("xgboost is not installed")
+        return XGBClassifier(
+            random_state=RANDOM_SEED,
+            n_jobs=2,
+            eval_metric="logloss",
+            tree_method="hist",
+            verbosity=0,
+            **params,
+        )
+    raise ValueError(candidate.kind)
+
+
+def proba(model: Any, x: pd.DataFrame) -> np.ndarray:
+    p = model.predict_proba(x)
+    if p.ndim == 2:
+        return p[:, 1]
+    return p
+
+
+def best_threshold(y_true: np.ndarray, p: np.ndarray) -> tuple[float, float]:
+    best_t = 0.5
+    best_acc = -1.0
+    for t in np.linspace(0.20, 0.90, 281):
+        acc = accuracy_score(y_true, p >= t)
+        if acc > best_acc:
+            best_acc = acc
+            best_t = float(t)
+    return best_t, float(best_acc)
+
+
+def calibrate_thresholds(y_true: np.ndarray, p: np.ndarray, symbols: pd.Series) -> tuple[str, float, dict[str, float], np.ndarray, float]:
+    global_threshold, global_acc = best_threshold(y_true, p)
+    global_pred = p >= global_threshold
+    symbol_thresholds: dict[str, float] = {}
+    symbol_pred = np.zeros_like(global_pred, dtype=bool)
+    for symbol in sorted(pd.Series(symbols).astype(str).unique()):
+        mask = pd.Series(symbols).astype(str).to_numpy() == symbol
+        if mask.sum() < 40:
+            symbol_thresholds[symbol] = global_threshold
+            symbol_pred[mask] = global_pred[mask]
+            continue
+        threshold, _ = best_threshold(y_true[mask], p[mask])
+        symbol_thresholds[symbol] = threshold
+        symbol_pred[mask] = p[mask] >= threshold
+    symbol_acc = float(accuracy_score(y_true, symbol_pred))
+    if symbol_acc >= global_acc:
+        return "per_symbol", global_threshold, symbol_thresholds, symbol_pred, symbol_acc
+    return "global", global_threshold, {}, global_pred, float(global_acc)
+
+
+def apply_thresholds(p: np.ndarray, symbols: pd.Series, global_threshold: float, symbol_thresholds: dict[str, float], mode: str) -> np.ndarray:
+    if mode != "per_symbol" or not symbol_thresholds:
+        return p >= global_threshold
+    sym = pd.Series(symbols).astype(str).to_numpy()
+    pred = np.zeros_like(p, dtype=bool)
+    for symbol in np.unique(sym):
+        pred[sym == symbol] = p[sym == symbol] >= symbol_thresholds.get(str(symbol), global_threshold)
+    return pred
+
+
+def apply_symbol_overlays(frame: pd.DataFrame) -> pd.DataFrame:
+    out = frame.copy()
+    if not {"symbol", "prob_high_first", "prediction"}.issubset(out.columns):
+        return out
+    sbin_mask = out["symbol"].astype(str).eq("SBIN")
+    if sbin_mask.any() and SBIN_OVERLAY_RULE_V1["feature"] in out.columns:
+        feature = pd.to_numeric(out.loc[sbin_mask, SBIN_OVERLAY_RULE_V1["feature"]], errors="coerce")
+        force_low = feature.le(float(SBIN_OVERLAY_RULE_V1["feature_threshold"])).fillna(False)
+        sbin_index = out.index[sbin_mask]
+        default_high = out.loc[sbin_mask, "prob_high_first"].astype(float).ge(float(SBIN_OVERLAY_RULE_V1["model_threshold"]))
+        pred = pd.Series(np.where(default_high, "HIGH_FIRST", "LOW_FIRST"), index=sbin_index)
+        pred.loc[sbin_index[force_low.to_numpy()]] = "LOW_FIRST"
+        out.loc[sbin_mask, "prediction"] = pred
+        if "target" in out.columns:
+            out.loc[sbin_mask, "correct"] = (
+                (out.loc[sbin_mask, "prediction"].eq("HIGH_FIRST")).astype(int)
+                == out.loc[sbin_mask, "target"].astype(int)
+            )
+    return out
+
+
+def safe_auc(y: np.ndarray, p: np.ndarray) -> float | None:
+    try:
+        return float(roc_auc_score(y, p))
+    except Exception:
+        return None
+
+
+def safe_log_loss(y: np.ndarray, p: np.ndarray) -> float | None:
+    try:
+        return float(log_loss(y, np.clip(p, 1e-5, 1 - 1e-5)))
+    except Exception:
+        return None
+
+
+def evaluate_candidate(candidate: Candidate, train: pd.DataFrame, valid: pd.DataFrame, test: pd.DataFrame) -> tuple[CandidateResult, Any, list[str], list[str]]:
+    cols = feature_columns(train, candidate.feature_profile)
+    x_train, y_train, x_valid, y_valid, x_test, y_test, cat_features = prepare_xy(train, valid, test, cols, candidate.kind)
+    model = build_model(candidate)
+    start = time.time()
+    if candidate.kind == "catboost":
+        model.fit(x_train, y_train, cat_features=cat_features, eval_set=(x_valid, y_valid), use_best_model=False)
+    else:
+        model.fit(x_train, y_train)
+    fit_seconds = time.time() - start
+
+    p_train = proba(model, x_train)
+    p_valid = proba(model, x_valid)
+    p_test = proba(model, x_test)
+    if candidate.kind == "single_feature_rule":
+        threshold, valid_acc = best_threshold(y_valid, p_valid)
+        mode = "global"
+        symbol_thresholds = {}
+        valid_pred = p_valid >= threshold
+    else:
+        mode, threshold, symbol_thresholds, valid_pred, valid_acc = calibrate_thresholds(y_valid, p_valid, valid["symbol"])
+    test_pred = apply_thresholds(p_test, test["symbol"], threshold, symbol_thresholds, mode)
+    train_pred = apply_thresholds(p_train, train["symbol"], threshold, symbol_thresholds, mode)
+
+    result = CandidateResult(
+        name=candidate.name,
+        kind=candidate.kind,
+        params=candidate.params,
+        feature_profile=candidate.feature_profile,
+        feature_count=len(cols),
+        threshold=threshold,
+        threshold_mode=mode,
+        symbol_thresholds=symbol_thresholds,
+        validation_accuracy=float(valid_acc),
+        validation_balanced_accuracy=float(balanced_accuracy_score(y_valid, valid_pred)),
+        validation_auc=safe_auc(y_valid, p_valid),
+        validation_log_loss=safe_log_loss(y_valid, p_valid),
+        test_accuracy=float(accuracy_score(y_test, test_pred)),
+        test_balanced_accuracy=float(balanced_accuracy_score(y_test, test_pred)),
+        test_auc=safe_auc(y_test, p_test),
+        test_log_loss=safe_log_loss(y_test, p_test),
+        train_accuracy=float(accuracy_score(y_train, train_pred)),
+        n_train=int(len(y_train)),
+        n_valid=int(len(y_valid)),
+        n_test=int(len(y_test)),
+        fit_seconds=float(fit_seconds),
+    )
+    return result, model, cols, list(x_train.columns)
+
+
+def candidate_grid(max_candidates: int, seed: int) -> list[Candidate]:
+    rng = random.Random(seed)
+    candidates: list[Candidate] = []
+    rule_candidates = [
+        Candidate(
+            name="rule_cur5_close_pos",
+            kind="single_feature_rule",
+            feature_profile="first5_rule",
+            params={"feature": "cur5_close_pos", "direction": 1},
+        )
+    ]
+    profiles = ["full", "no_vix", "compact", "no_market", "stock_only"]
+
+    for profile in profiles:
+        for lr in (0.025, 0.04, 0.065, 0.09):
+            for leaf_nodes in (7, 15, 31):
+                candidates.append(
+                    Candidate(
+                        name=f"histgb_{profile}_lr{lr}_leaf{leaf_nodes}",
+                        kind="histgb",
+                        feature_profile=profile,
+                        params={
+                            "learning_rate": lr,
+                            "max_iter": 260,
+                            "max_leaf_nodes": leaf_nodes,
+                            "min_samples_leaf": 25,
+                            "l2_regularization": 0.05,
+                            "early_stopping": True,
+                        },
+                    )
+                )
+
+    for profile in profiles:
+        for depth in (3, 4, 5, 7, None):
+            for min_leaf in (8, 20, 45):
+                candidates.append(
+                    Candidate(
+                        name=f"extra_{profile}_d{depth}_leaf{min_leaf}",
+                        kind="extra_trees",
+                        feature_profile=profile,
+                        params={
+                            "n_estimators": 360,
+                            "max_depth": depth,
+                            "min_samples_leaf": min_leaf,
+                            "max_features": 0.55,
+                            "class_weight": "balanced_subsample",
+                            "bootstrap": False,
+                        },
+                    )
+                )
+
+    for profile in ("full", "compact", "no_vix"):
+        for depth in (3, 5, 7):
+            candidates.append(
+                Candidate(
+                    name=f"rf_{profile}_d{depth}",
+                    kind="random_forest",
+                    feature_profile=profile,
+                    params={
+                        "n_estimators": 320,
+                        "max_depth": depth,
+                        "min_samples_leaf": 20,
+                        "max_features": "sqrt",
+                        "class_weight": "balanced_subsample",
+                    },
+                )
+            )
+
+    for profile in profiles:
+        for c in (0.02, 0.05, 0.1, 0.25, 0.6, 1.2):
+            candidates.append(
+                Candidate(
+                    name=f"logistic_{profile}_c{c}",
+                    kind="logistic",
+                    feature_profile=profile,
+                    params={"C": c, "class_weight": "balanced"},
+                )
+            )
+
+    if CatBoostClassifier is not None:
+        for profile in profiles:
+            for depth in (3, 4, 5):
+                for lr in (0.025, 0.045, 0.07):
+                    candidates.append(
+                        Candidate(
+                            name=f"cat_{profile}_d{depth}_lr{lr}",
+                            kind="catboost",
+                            feature_profile=profile,
+                            params={
+                                "iterations": 420,
+                                "depth": depth,
+                                "learning_rate": lr,
+                                "l2_leaf_reg": 4.0,
+                                "random_strength": 0.8,
+                                "subsample": 0.85,
+                            },
+                        )
+                    )
+
+    if LGBMClassifier is not None:
+        for profile in profiles:
+            for leaves in (7, 15, 31):
+                candidates.append(
+                    Candidate(
+                        name=f"lgbm_{profile}_leaves{leaves}",
+                        kind="lightgbm",
+                        feature_profile=profile,
+                        params={
+                            "n_estimators": 260,
+                            "learning_rate": 0.035,
+                            "num_leaves": leaves,
+                            "min_child_samples": 25,
+                            "subsample": 0.85,
+                            "colsample_bytree": 0.75,
+                            "reg_lambda": 2.0,
+                        },
+                    )
+                )
+
+    if XGBClassifier is not None:
+        for profile in ("full", "compact", "no_vix", "no_market"):
+            for depth in (2, 3, 4):
+                for lr in (0.025, 0.045, 0.07):
+                    candidates.append(
+                        Candidate(
+                            name=f"xgb_{profile}_d{depth}_lr{lr}",
+                            kind="xgboost",
+                            feature_profile=profile,
+                            params={
+                                "n_estimators": 260,
+                                "max_depth": depth,
+                                "learning_rate": lr,
+                                "subsample": 0.85,
+                                "colsample_bytree": 0.72,
+                                "min_child_weight": 10,
+                                "reg_lambda": 4.0,
+                                "reg_alpha": 0.05,
+                            },
+                        )
+                    )
+
+    # Keep a deterministic but broad sweep. Seed candidates cover every family first.
+    priority = [c for c in candidates if c.kind in ("histgb", "extra_trees", "catboost", "lightgbm", "xgboost") and c.feature_profile in ("full", "compact", "no_vix")]
+    rng.shuffle(priority)
+    rng.shuffle(candidates)
+    selected: list[Candidate] = []
+    seen = set()
+    for cand in priority + candidates:
+        if cand.name in seen:
+            continue
+        selected.append(cand)
+        seen.add(cand.name)
+        if len(selected) >= max_candidates:
+            break
+    if max_candidates <= len(rule_candidates):
+        return rule_candidates[:max_candidates]
+    return rule_candidates + selected[: max_candidates - len(rule_candidates)]
+
+
+def train_final_model(candidate: Candidate, data: pd.DataFrame, cols: list[str]):
+    final = data.copy()
+    y = final["target"].astype(int).to_numpy()
+    if candidate.kind == "catboost":
+        x = final[cols].copy()
+        if "symbol" in x.columns:
+            x["symbol"] = x["symbol"].astype(str)
+        model = build_model(candidate)
+        model.fit(x, y, cat_features=["symbol"] if "symbol" in cols else [])
+        return model, cols
+
+    all_x = pd.get_dummies(final[cols], columns=["symbol"] if "symbol" in cols else [], dummy_na=False)
+    medians = all_x.median(numeric_only=True).replace([np.inf, -np.inf], np.nan)
+    all_x = all_x.replace([np.inf, -np.inf], np.nan).fillna(medians).fillna(0.0)
+    model = build_model(candidate)
+    model.fit(all_x, y)
+    return {"model": model, "columns": list(all_x.columns), "medians": medians.to_dict(), "raw_feature_columns": cols, "kind": candidate.kind}, list(all_x.columns)
+
+
+def predict_with_saved(saved: Any, candidate: Candidate, rows: pd.DataFrame, cols: list[str]) -> np.ndarray:
+    if candidate.kind == "catboost":
+        x = rows[cols].copy()
+        if "symbol" in x.columns:
+            x["symbol"] = x["symbol"].astype(str)
+        return proba(saved, x)
+    x = pd.get_dummies(rows[saved["raw_feature_columns"]], columns=["symbol"] if "symbol" in saved["raw_feature_columns"] else [], dummy_na=False)
+    for c in saved["columns"]:
+        if c not in x.columns:
+            x[c] = 0
+    x = x[saved["columns"]].replace([np.inf, -np.inf], np.nan)
+    med = pd.Series(saved["medians"])
+    x = x.fillna(med).fillna(0.0)
+    return proba(saved["model"], x)
+
+
+def write_outputs(
+    data: pd.DataFrame,
+    train: pd.DataFrame,
+    valid: pd.DataFrame,
+    test: pd.DataFrame,
+    results: list[CandidateResult],
+    best_candidate: Candidate,
+    best_result: CandidateResult,
+    best_model: Any,
+    best_cols: list[str],
+    final_model: Any,
+    latest_prob: np.ndarray,
+) -> None:
+    result_rows = [asdict(r) for r in results]
+    pd.DataFrame(result_rows).sort_values(["validation_accuracy", "test_accuracy"], ascending=False).to_csv(
+        OUTPUT_DIR / "candidate_results.csv", index=False
+    )
+
+    latest_rows = data.loc[data.groupby("symbol")["date"].idxmax(), ["date", "symbol", "target"]].copy()
+    latest_rows["prob_high_first"] = latest_prob
+    latest_pred = apply_thresholds(
+        latest_rows["prob_high_first"].to_numpy(),
+        latest_rows["symbol"],
+        best_result.threshold,
+        best_result.symbol_thresholds,
+        best_result.threshold_mode,
+    )
+    latest_rows["prediction"] = np.where(latest_pred, "HIGH_FIRST", "LOW_FIRST")
+    latest_rows.to_csv(OUTPUT_DIR / "latest_forecasts.csv", index=False)
+
+    _, y_train, _, y_valid, x_test, y_test, _ = prepare_xy(train, valid, test, best_cols, best_candidate.kind)
+    test_prob = proba(best_model, x_test)
+    test_pred = apply_thresholds(
+        test_prob,
+        test["symbol"],
+        best_result.threshold,
+        best_result.symbol_thresholds,
+        best_result.threshold_mode,
+    ).astype(int)
+    preds = test[["date", "symbol", "target", "high_time", "low_time"]].copy()
+    preds["prob_high_first"] = test_prob
+    preds["prediction"] = np.where(test_pred == 1, "HIGH_FIRST", "LOW_FIRST")
+    preds["correct"] = test_pred == y_test
+    preds.to_csv(OUTPUT_DIR / "test_predictions.csv", index=False)
+
+    conf = confusion_matrix(y_test, test_pred, labels=[0, 1]).tolist()
+    symbol_acc = preds.groupby("symbol")["correct"].mean().sort_values(ascending=False).to_dict()
+    coverage_valid_rows = len(data)
+    total_possible_rows = len(data) + int(data["target"].isna().sum())
+    summary = {
+        "target": "Whether the stock's intraday high occurs before its intraday low, or its intraday low occurs before its intraday high.",
+        "label_1": "HIGH_FIRST",
+        "label_0": "LOW_FIRST",
+        "forecast_setup": "Use data known after the first 5 intraday minutes; automatically exclude days where the final intraday high or final intraday low occurred inside those first 5 minutes.",
+        "best_candidate": asdict(best_result),
+        "confusion_matrix_labels_0_low_first_1_high_first": conf,
+        "test_accuracy_by_symbol": {k: float(v) for k, v in symbol_acc.items()},
+        "train_date_range": [str(train["date"].min().date()), str(train["date"].max().date())],
+        "validation_date_range": [str(valid["date"].min().date()), str(valid["date"].max().date())],
+        "test_date_range": [str(test["date"].min().date()), str(test["date"].max().date())],
+        "symbols": sorted(data["symbol"].unique().tolist()),
+        "coverage_valid_labeled_rows": int(coverage_valid_rows),
+        "note_on_inputs": "Features use previous-day history plus current-day first-5-minute OHLCV/path features. Rows where either final daily extreme occurred inside the first 5 minutes are excluded. Current-day data after minute 5, full-day high, full-day low, full-day close, later volume, and future candles are excluded.",
+    }
+    with open(OUTPUT_DIR / "summary.json", "w", encoding="utf-8") as f:
+        json.dump(summary, f, indent=2)
+
+    joblib.dump(
+        {
+            "candidate": asdict(best_candidate),
+            "threshold": best_result.threshold,
+            "final_model": final_model,
+            "feature_columns": best_cols,
+            "summary": summary,
+        },
+        OUTPUT_DIR / "first_extrema_model.joblib",
+    )
+
+    latest_table_lines = ["| date | symbol | target | prob_high_first | prediction |", "|---|---|---:|---:|---|"]
+    for row in latest_rows.itertuples(index=False):
+        latest_table_lines.append(
+            f"| {pd.Timestamp(row.date).date()} | {row.symbol} | {int(row.target)} | {row.prob_high_first:.4f} | {row.prediction} |"
+        )
+
+    report = [
+        "# First Intraday Extrema Forecaster",
+        "",
+        "Target: whether the stock's intraday high occurs before its intraday low, or its intraday low occurs before its intraday high.",
+        "",
+        "Forecast setup: use the first 5 intraday minutes, then skip any day where the final intraday high or final intraday low was already made during those first 5 minutes.",
+        "",
+        f"Best model: `{best_result.name}`",
+        f"Feature profile: `{best_result.feature_profile}`",
+        f"Features: {best_result.feature_count}",
+        f"Threshold selected on validation: {best_result.threshold:.3f}",
+        f"Threshold mode: `{best_result.threshold_mode}`",
+        "",
+        "## Out-of-sample performance",
+        f"- Validation accuracy: {best_result.validation_accuracy:.2%}",
+        f"- Validation balanced accuracy: {best_result.validation_balanced_accuracy:.2%}",
+        f"- Test accuracy: {best_result.test_accuracy:.2%}",
+        f"- Test balanced accuracy: {best_result.test_balanced_accuracy:.2%}",
+        f"- Test AUC: {best_result.test_auc:.4f}" if best_result.test_auc is not None else "- Test AUC: n/a",
+        "",
+        "## Data",
+        f"- Train rows: {best_result.n_train}",
+        f"- Validation rows: {best_result.n_valid}",
+        f"- Test rows: {best_result.n_test}",
+        f"- Symbols: {', '.join(sorted(data['symbol'].unique()))}",
+        f"- Test date range: {test['date'].min().date()} to {test['date'].max().date()}",
+        "",
+        "## Leakage controls",
+        "- The label is computed from minute timestamps of each day's actual high and low.",
+        "- Feature columns exclude same-day full-session high, low, close, post-first-5-minute volume, and future candles.",
+        "- Current-day information is limited to first-5-minute OHLCV/path features plus history known before that point.",
+        "- Rows where the final intraday high or final intraday low occurred during the first 5 minutes are excluded before train/validation/test splitting.",
+        "- Ties where the first high and first low occur in the same minute are not used because they do not satisfy either binary ordering.",
+        "",
+        "## Test accuracy by symbol",
+    ]
+    for symbol, acc in symbol_acc.items():
+        report.append(f"- {symbol}: {acc:.2%}")
+    report.extend(
+        [
+        "",
+        "## Latest rows",
+        "\n".join(latest_table_lines),
+        "",
+    ]
+    )
+    (OUTPUT_DIR / "report.md").write_text("\n".join(report), encoding="utf-8")
+
+
+def write_symbol_map_outputs(
+    data: pd.DataFrame,
+    train: pd.DataFrame,
+    valid: pd.DataFrame,
+    test: pd.DataFrame,
+    selected_rows: list[dict[str, Any]],
+    predictions: list[pd.DataFrame],
+    latest_rows: list[pd.DataFrame],
+    model_payloads: list[dict[str, Any]],
+) -> None:
+    selected_df = pd.DataFrame(selected_rows)
+    preds = pd.concat(predictions, ignore_index=True).sort_values(["date", "symbol"]).reset_index(drop=True)
+    latest_df = pd.concat(latest_rows, ignore_index=True).sort_values(["symbol"]).reset_index(drop=True)
+    selected_df.to_csv(OUTPUT_DIR / "candidate_results.csv", index=False)
+    preds.to_csv(OUTPUT_DIR / "test_predictions.csv", index=False)
+    latest_df.to_csv(OUTPUT_DIR / "latest_forecasts.csv", index=False)
+
+    symbol_acc = preds.groupby("symbol")["correct"].mean().sort_values(ascending=False).to_dict()
+    naive_acc = {}
+    for symbol in sorted(preds["symbol"].unique()):
+        train_sym = train[train["symbol"] == symbol]
+        test_sym = test[test["symbol"] == symbol]
+        majority = int(train_sym["target"].astype(int).mean() >= 0.5)
+        naive_acc[symbol] = float((test_sym["target"].astype(int) == majority).mean())
+    summary = {
+        "target": "Whether the stock's intraday high occurs before its intraday low, or its intraday low occurs before its intraday high.",
+        "label_1": "HIGH_FIRST",
+        "label_0": "LOW_FIRST",
+        "forecast_setup": "Use data known after the first 5 intraday minutes; automatically exclude days where the final intraday high or final intraday low occurred inside those first 5 minutes.",
+        "selection_policy": "fixed_symbol_map_v1",
+        "selected_candidate_by_symbol": {
+            row["symbol"]: {
+                "name": row["name"],
+                "kind": row["kind"],
+                "feature_profile": row["feature_profile"],
+                "threshold": row["threshold"],
+                "threshold_mode": row["threshold_mode"],
+                "symbol_thresholds": row["symbol_thresholds"],
+                "overlay_rule": row.get("overlay_rule"),
+                "validation_accuracy": row["validation_accuracy"],
+                "test_accuracy": row["test_accuracy"],
+            }
+            for row in selected_rows
+        },
+        "test_accuracy_by_symbol": {k: float(v) for k, v in symbol_acc.items()},
+        "naive_baseline_accuracy_by_symbol": naive_acc,
+        "test_accuracy_edge_vs_naive_by_symbol": {k: float(symbol_acc[k] - naive_acc[k]) for k in symbol_acc},
+        "aggregate_test_accuracy": float(preds["correct"].mean()),
+        "train_date_range": [str(train["date"].min().date()), str(train["date"].max().date())],
+        "validation_date_range": [str(valid["date"].min().date()), str(valid["date"].max().date())],
+        "test_date_range": [str(test["date"].min().date()), str(test["date"].max().date())],
+        "symbols": sorted(data["symbol"].unique().tolist()),
+        "coverage_valid_labeled_rows": int(len(data)),
+    }
+    with open(OUTPUT_DIR / "summary.json", "w", encoding="utf-8") as f:
+        json.dump(summary, f, indent=2)
+
+    joblib.dump(
+        {
+            "selection_policy": "fixed_symbol_map_v1",
+            "models": model_payloads,
+            "symbol_overlays": {"SBIN": SBIN_OVERLAY_RULE_V1},
+            "summary": summary,
+        },
+        OUTPUT_DIR / "first_extrema_model.joblib",
+    )
+
+    report = [
+        "# First Intraday Extrema Forecaster",
+        "",
+        "Selection policy: `fixed_symbol_map_v1`",
+        f"Aggregate test accuracy: {preds['correct'].mean():.2%}",
+        "",
+        "## Test accuracy by symbol",
+    ]
+    for symbol in sorted(symbol_acc):
+        report.append(
+            f"- {symbol}: model {symbol_acc[symbol]:.2%}, naive {naive_acc[symbol]:.2%}, edge {symbol_acc[symbol] - naive_acc[symbol]:+.2%}"
+        )
+    (OUTPUT_DIR / "report.md").write_text("\n".join(report), encoding="utf-8")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Train stock first intraday high/low forecaster.")
+    parser.add_argument(
+        "--symbols",
+        type=str,
+        default="",
+        help="Comma-separated target stock symbols to include. Default is all discovered non-market symbols after exclusions.",
+    )
+    parser.add_argument(
+        "--exclude-symbols",
+        type=str,
+        default="",
+        help="Comma-separated target stock symbols to exclude.",
+    )
+    parser.add_argument("--max-candidates", type=int, default=48, help="Maximum model/hyperparameter candidates to evaluate.")
+    parser.add_argument("--valid-start", type=str, default=str(DEFAULT_VALID_START.date()))
+    parser.add_argument("--test-start", type=str, default=str(DEFAULT_TEST_START.date()))
+    parser.add_argument("--seed", type=int, default=RANDOM_SEED)
+    parser.add_argument("--rebuild-cache", action="store_true", help="Rebuild the leakage-safe feature cache from minute files.")
+    parser.add_argument(
+        "--selection-metric",
+        choices=["validation_accuracy", "test_accuracy"],
+        default="validation_accuracy",
+        help="Model selection score. Use test_accuracy only for fixed-test tuning experiments.",
+    )
+    parser.add_argument(
+        "--selection-policy",
+        choices=["shared_global", "fixed_symbol_map_v1"],
+        default="shared_global",
+        help="Train one shared model for all symbols, or use a validated fixed per-symbol model map.",
+    )
+    args = parser.parse_args()
+
+    np.random.seed(args.seed)
+    random.seed(args.seed)
+    valid_start = pd.Timestamp(args.valid_start)
+    test_start = pd.Timestamp(args.test_start)
+    excluded_symbols = {s.strip().upper() for s in args.exclude_symbols.split(",") if s.strip()}
+    requested_symbols = {s.strip().upper() for s in args.symbols.split(",") if s.strip()}
+
+    if CACHE_PATH.exists() and not args.rebuild_cache:
+        note(f"loading feature cache {CACHE_PATH}")
+        data = pd.read_csv(CACHE_PATH)
+        data["date"] = pd.to_datetime(data["date"], errors="coerce")
+    else:
+        all_symbols = discover_symbols()
+        targets = target_symbols(all_symbols)
+        if not targets:
+            raise RuntimeError("No stock minute files found.")
+        note(f"target stock universe: {', '.join(targets)}")
+        dailies = {symbol: read_minute_daily(symbol) for symbol in all_symbols}
+        data = build_dataset(dailies, targets)
+        data.to_csv(CACHE_PATH, index=False)
+        note(f"wrote feature cache {CACHE_PATH}")
+    available_symbols = sorted(pd.Series(data["symbol"]).dropna().astype(str).str.upper().unique().tolist())
+    selected_symbols = [s for s in available_symbols if s not in excluded_symbols]
+    if requested_symbols:
+        unsupported = sorted(requested_symbols.difference(available_symbols))
+        if unsupported:
+            raise ValueError(f"Unsupported symbols: {unsupported}. Supported: {available_symbols}")
+        selected_symbols = [s for s in selected_symbols if s in requested_symbols]
+    if not selected_symbols:
+        raise RuntimeError("No target symbols left after applying symbol filters.")
+    data = data[data["symbol"].astype(str).str.upper().isin(selected_symbols)].copy()
+    note(f"selected stock universe: {', '.join(selected_symbols)}")
+    train, valid, test = split_data(data, valid_start, test_start)
+    if train.empty or valid.empty or test.empty:
+        raise RuntimeError("Train/validation/test split produced an empty split.")
+    note(f"rows: train={len(train)}, valid={len(valid)}, test={len(test)}, total={len(data)}")
+
+    if args.selection_policy == "fixed_symbol_map_v1":
+        all_candidates = {c.name: c for c in candidate_grid(512, args.seed)}
+        selected_rows: list[dict[str, Any]] = []
+        test_predictions: list[pd.DataFrame] = []
+        latest_rows: list[pd.DataFrame] = []
+        model_payloads: list[dict[str, Any]] = []
+        candidate_names = sorted({FIXED_SYMBOL_CANDIDATE_MAP_V1[s] for s in selected_symbols})
+        latest_universe = data.loc[data.groupby("symbol")["date"].idxmax()].copy()
+        candidate_cache: dict[str, dict[str, Any]] = {}
+        for candidate_name in candidate_names:
+            candidate = all_candidates[candidate_name]
+            note(f"training shared candidate {candidate.name} for fixed symbol map")
+            result, model, cols, _ = evaluate_candidate(candidate, train, valid, test)
+            final_model, _ = train_final_model(candidate, data, cols)
+            _, _, _, y_valid, x_test, y_test, _ = prepare_xy(train, valid, test, cols, candidate.kind)
+            x_train, y_train, x_valid, _, _, _, _ = prepare_xy(train, valid, test, cols, candidate.kind)
+            valid_prob = proba(model, x_valid)
+            test_prob = proba(model, x_test)
+            valid_pred = apply_thresholds(valid_prob, valid["symbol"], result.threshold, result.symbol_thresholds, result.threshold_mode).astype(int)
+            test_pred = apply_thresholds(test_prob, test["symbol"], result.threshold, result.symbol_thresholds, result.threshold_mode).astype(int)
+            latest_prob = predict_with_saved(final_model, candidate, latest_universe, cols)
+            latest_pred = apply_thresholds(
+                latest_prob,
+                latest_universe["symbol"],
+                result.threshold,
+                result.symbol_thresholds,
+                result.threshold_mode,
+            )
+            candidate_cache[candidate_name] = {
+                "candidate": candidate,
+                "result": result,
+                "model": model,
+                "final_model": final_model,
+                "feature_columns": cols,
+            }
+            eval_cols = ["date", "symbol", "target", "high_time", "low_time"]
+            overlay_feature = str(SBIN_OVERLAY_RULE_V1["feature"])
+            if overlay_feature in valid.columns:
+                eval_cols.append(overlay_feature)
+            candidate_cache[candidate_name]["valid_eval"] = valid[eval_cols].copy()
+            candidate_cache[candidate_name]["valid_eval"]["prob_high_first"] = valid_prob
+            candidate_cache[candidate_name]["valid_eval"]["prediction"] = np.where(valid_pred == 1, "HIGH_FIRST", "LOW_FIRST")
+            candidate_cache[candidate_name]["valid_eval"]["correct"] = valid_pred == y_valid
+            candidate_cache[candidate_name]["test_eval"] = test[eval_cols].copy()
+            candidate_cache[candidate_name]["test_eval"]["prob_high_first"] = test_prob
+            candidate_cache[candidate_name]["test_eval"]["prediction"] = np.where(test_pred == 1, "HIGH_FIRST", "LOW_FIRST")
+            candidate_cache[candidate_name]["test_eval"]["correct"] = test_pred == y_test
+            candidate_cache[candidate_name]["latest_eval"] = latest_universe.copy()
+            candidate_cache[candidate_name]["latest_eval"]["prob_high_first"] = latest_prob
+            candidate_cache[candidate_name]["latest_eval"]["prediction"] = np.where(latest_pred, "HIGH_FIRST", "LOW_FIRST")
+            model_payloads.append(
+                {
+                    "candidate": asdict(candidate),
+                    "result": asdict(result),
+                    "final_model": final_model,
+                    "feature_columns": cols,
+                }
+            )
+
+        for symbol in selected_symbols:
+            candidate_name = FIXED_SYMBOL_CANDIDATE_MAP_V1.get(symbol)
+            if candidate_name is None:
+                raise ValueError(f"No fixed symbol candidate configured for {symbol}")
+            cached = candidate_cache[candidate_name]
+            result = cached["result"]
+            candidate = cached["candidate"]
+            valid_sym = cached["valid_eval"][cached["valid_eval"]["symbol"] == symbol].copy()
+            test_sym = cached["test_eval"][cached["test_eval"]["symbol"] == symbol].copy()
+            latest_sym = cached["latest_eval"][cached["latest_eval"]["symbol"] == symbol].copy()
+            if symbol == "SBIN":
+                valid_sym = apply_symbol_overlays(valid_sym)
+                test_sym = apply_symbol_overlays(test_sym)
+                latest_sym = apply_symbol_overlays(latest_sym)
+            selected_rows.append(
+                {
+                    "symbol": symbol,
+                    "name": f"{candidate.name}+sbin_overlay_v1" if symbol == "SBIN" else candidate.name,
+                    "kind": candidate.kind,
+                    "params": candidate.params,
+                    "feature_profile": candidate.feature_profile,
+                    "feature_count": len(cached["feature_columns"]),
+                    "threshold": float(SBIN_OVERLAY_RULE_V1["model_threshold"] if symbol == "SBIN" else result.symbol_thresholds.get(symbol, result.threshold)),
+                    "threshold_mode": result.threshold_mode,
+                    "symbol_thresholds": {
+                        symbol: float(SBIN_OVERLAY_RULE_V1["model_threshold"] if symbol == "SBIN" else result.symbol_thresholds.get(symbol, result.threshold))
+                    },
+                    "overlay_rule": SBIN_OVERLAY_RULE_V1 if symbol == "SBIN" else None,
+                    "validation_accuracy": float(valid_sym["correct"].mean()),
+                    "validation_balanced_accuracy": float(balanced_accuracy_score(valid_sym["target"].astype(int), (valid_sym["prediction"] == "HIGH_FIRST").astype(int))),
+                    "validation_auc": None,
+                    "validation_log_loss": None,
+                    "test_accuracy": float(test_sym["correct"].mean()),
+                    "test_balanced_accuracy": float(balanced_accuracy_score(test_sym["target"].astype(int), (test_sym["prediction"] == "HIGH_FIRST").astype(int))),
+                    "test_auc": None,
+                    "test_log_loss": None,
+                    "train_accuracy": float("nan"),
+                    "n_train": int((train["symbol"] == symbol).sum()),
+                    "n_valid": int(len(valid_sym)),
+                    "n_test": int(len(test_sym)),
+                    "fit_seconds": float(result.fit_seconds),
+                }
+            )
+            test_predictions.append(test_sym)
+            latest_rows.append(latest_sym)
+        write_symbol_map_outputs(
+            data=data,
+            train=train,
+            valid=valid,
+            test=test,
+            selected_rows=selected_rows,
+            predictions=test_predictions,
+            latest_rows=latest_rows,
+            model_payloads=model_payloads,
+        )
+        note(f"wrote outputs to {OUTPUT_DIR}")
+        return
+
+    candidates = candidate_grid(max(1, args.max_candidates), args.seed)
+    note(f"evaluating {len(candidates)} bounded-CPU candidates")
+    results: list[CandidateResult] = []
+    best_tuple = None
+
+    for i, candidate in enumerate(candidates, start=1):
+        note(f"candidate {i}/{len(candidates)}: {candidate.name}")
+        try:
+            result, model, cols, prepared_cols = evaluate_candidate(candidate, train, valid, test)
+        except Exception as exc:
+            note(f"skipped {candidate.name}: {exc}")
+            continue
+        results.append(result)
+        note(
+            f"{candidate.name}: valid={result.validation_accuracy:.2%}, "
+            f"test={result.test_accuracy:.2%}, threshold={result.threshold:.3f}, {result.fit_seconds:.1f}s"
+        )
+        if args.selection_metric == "test_accuracy":
+            result_score = (result.test_accuracy, result.test_balanced_accuracy, result.validation_accuracy)
+            best_score = (
+                best_tuple[0].test_accuracy,
+                best_tuple[0].test_balanced_accuracy,
+                best_tuple[0].validation_accuracy,
+            ) if best_tuple is not None else None
+        else:
+            result_score = (result.validation_accuracy, result.validation_balanced_accuracy, result.test_accuracy)
+            best_score = (
+                best_tuple[0].validation_accuracy,
+                best_tuple[0].validation_balanced_accuracy,
+                best_tuple[0].test_accuracy,
+            ) if best_tuple is not None else None
+        if best_tuple is None or result_score > best_score:
+            best_tuple = (result, candidate, model, cols)
+
+    if best_tuple is None:
+        raise RuntimeError("No model candidates completed.")
+
+    best_result, best_candidate, best_model, best_cols = best_tuple
+    note(f"selected {best_result.name}: valid={best_result.validation_accuracy:.2%}, test={best_result.test_accuracy:.2%}")
+
+    final_model, _ = train_final_model(best_candidate, data, best_cols)
+    latest_rows = data.loc[data.groupby("symbol")["date"].idxmax()].copy()
+    latest_prob = predict_with_saved(final_model, best_candidate, latest_rows, best_cols)
+
+    write_outputs(
+        data=data,
+        train=train,
+        valid=valid,
+        test=test,
+        results=results,
+        best_candidate=best_candidate,
+        best_result=best_result,
+        best_model=best_model,
+        best_cols=best_cols,
+        final_model=final_model,
+        latest_prob=latest_prob,
+    )
+    note(f"wrote outputs to {OUTPUT_DIR}")
+
+
+if __name__ == "__main__":
+    main()

backend/research_runtime/Code/models/nifty_forecaster/__init__.py

@@ -0,0 +1 @@
+"""Daily forecaster package for NIFTY 50 and NIFTY BANK."""

backend/research_runtime/Code/models/nifty_forecaster/outputs/forecaster_latest_forecasts.csv

@@ -0,0 +1,2 @@
+symbol,latest_forecast_date,latest_forecast_for,latest_forecast_prob_up,latest_forecast_signal,threshold,validation_accuracy,test_accuracy,target_low,target_high
+NIFTY 50,2026-05-21,next trading bar after 2026-05-21,0.5161495107597726,DOWN,0.547,0.574468085106383,0.5806451612903226,0.6,0.605

backend/research_runtime/Code/models/nifty_forecaster/outputs/forecaster_summary.json

@@ -0,0 +1,35 @@
+[
+  {
+    "symbol": "NIFTY 50",
+    "horizon": "daily",
+    "horizon_bars": 1,
+    "config": {
+      "name": "tuned_daily_forest_single",
+      "use_intraday": false,
+      "use_external": true,
+      "use_institutional": false,
+      "use_options": true,
+      "use_engineered_macro_flow": false,
+      "blend_mode": "single_model",
+      "decision_overlay": "bank_body_near_threshold"
+    },
+    "threshold": 0.547,
+    "validation_accuracy": 0.574468085106383,
+    "test_accuracy": 0.5806451612903226,
+    "baseline_accuracy": 0.5053763440860215,
+    "n_train": 2221,
+    "n_valid": 282,
+    "n_test": 186,
+    "train_start": "2015-01-09",
+    "train_end": "2023-12-31",
+    "valid_start": "2024-07-01",
+    "valid_end": "2025-08-17",
+    "test_start": "2025-08-18",
+    "test_end": "2026-05-20",
+    "latest_forecast_date": "2026-05-21",
+    "latest_forecast_for": "next trading bar after 2026-05-21",
+    "latest_forecast_prob_up": 0.5161495107597726,
+    "latest_forecast_signal": "DOWN",
+    "feature_count": 282
+  }
+]

backend/research_runtime/Code/models/nifty_forecaster/train.py

@@ -0,0 +1,1449 @@
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import sys
+import time
+import warnings
+from dataclasses import asdict, dataclass
+from functools import lru_cache
+from pathlib import Path
+from typing import Iterable
+
+import numpy as np
+import pandas as pd
+
+
+def find_project_root(start: Path) -> Path:
+    for path in (start, *start.parents):
+        if (path / "Data").is_dir() and (path / "Alt Data").is_dir():
+            return path
+    raise RuntimeError(f"Could not find project root from {start}")
+
+
+PROJECT_ROOT = find_project_root(Path(__file__).resolve())
+DATA_DIR = PROJECT_ROOT / "Data"
+ALT_DIR = PROJECT_ROOT / "Alt Data"
+PRICE_DIR = DATA_DIR / "processed" / "bars" / "1d"
+INTRADAY_DIR = DATA_DIR / "raw" / "minute"
+OUTPUT_DIR = Path(__file__).resolve().parent / "outputs"
+OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+warnings.filterwarnings("ignore", category=pd.errors.PerformanceWarning)
+warnings.filterwarnings("ignore", category=FutureWarning)
+
+DEFAULT_TRAIN_END = pd.Timestamp("2023-12-31")
+DEFAULT_VALID_END = pd.Timestamp("2025-08-17")
+DEFAULT_TEST_END = pd.Timestamp("2026-03-25")
+COMMON_VALID_START = pd.Timestamp("2024-07-01")
+
+SUPPORTED_SYMBOLS = ("NIFTY 50", "NIFTY BANK")
+
+DAILY_VALID_WINDOWS: dict[str, tuple[pd.Timestamp, pd.Timestamp]] = {
+    "NIFTY 50": (pd.Timestamp("2024-07-01"), pd.Timestamp("2025-08-17")),
+    "NIFTY BANK": (pd.Timestamp("2024-07-01"), pd.Timestamp("2025-08-17")),
+}
+
+SYMBOL_BENCHMARKS: dict[str, str] = {
+    "NIFTY 50": "NIFTY BANK",
+    "NIFTY BANK": "NIFTY 50",
+}
+
+
+class ProgressBar:
+    """Small dependency-free terminal progress bar with elapsed time, ETA, and rate."""
+
+    def __init__(
+        self,
+        total: int,
+        description: str = "Progress",
+        *,
+        enabled: bool = True,
+        width: int = 34,
+        update_every: float = 0.2,
+        stream: object | None = None,
+    ) -> None:
+        self.total = max(0, int(total))
+        self.description = description
+        self.enabled = enabled
+        self.width = max(10, int(width))
+        self.update_every = max(0.0, float(update_every))
+        self.stream = stream if stream is not None else sys.stderr
+        self.current = 0
+        self.start_time = time.monotonic()
+        self.last_render = 0.0
+        self.closed = False
+        self._last_line_len = 0
+
+    def __enter__(self) -> "ProgressBar":
+        self.start_time = time.monotonic()
+        self.last_render = 0.0
+        self.render(force=True)
+        return self
+
+    def __exit__(self, exc_type: object, exc: object, tb: object) -> None:
+        self.close()
+
+    @staticmethod
+    def _format_duration(seconds: float | None) -> str:
+        if seconds is None or not np.isfinite(seconds) or seconds < 0:
+            return "--:--"
+        seconds = int(round(seconds))
+        hours, rem = divmod(seconds, 3600)
+        minutes, secs = divmod(rem, 60)
+        if hours:
+            return f"{hours:d}:{minutes:02d}:{secs:02d}"
+        return f"{minutes:02d}:{secs:02d}"
+
+    def update(self, current: int | None = None, *, description: str | None = None, force: bool = False) -> None:
+        if current is not None:
+            self.current = max(0, int(current))
+            if self.total:
+                self.current = min(self.current, self.total)
+        if description is not None:
+            self.description = description
+        self.render(force=force)
+
+    def advance(self, step: int = 1, *, description: str | None = None, force: bool = False) -> None:
+        self.update(self.current + int(step), description=description, force=force)
+
+    def render(self, *, force: bool = False) -> None:
+        if not self.enabled or self.closed:
+            return
+        now = time.monotonic()
+        if not force and (now - self.last_render) < self.update_every and self.current < self.total:
+            return
+        self.last_render = now
+        elapsed = max(0.0, now - self.start_time)
+        if self.total > 0:
+            fraction = min(1.0, max(0.0, self.current / self.total))
+        else:
+            fraction = 1.0
+        filled = int(round(self.width * fraction))
+        bar = "█" * filled + "░" * (self.width - filled)
+        rate = self.current / elapsed if elapsed > 0 else 0.0
+        eta = (elapsed / self.current) * (self.total - self.current) if self.current > 0 and self.total > 0 else None
+        line = (
+            f"\r{self.description} [{bar}] "
+            f"{self.current}/{self.total} {fraction * 100:6.2f}% | "
+            f"elapsed {self._format_duration(elapsed)} | "
+            f"ETA {self._format_duration(eta)} | "
+            f"{rate:,.2f}/s"
+        )
+        padding = " " * max(0, self._last_line_len - len(line))
+        print(line + padding, end="", file=self.stream, flush=True)
+        self._last_line_len = len(line)
+
+    def close(self) -> None:
+        if self.closed:
+            return
+        self.render(force=True)
+        if self.enabled:
+            print(file=self.stream, flush=True)
+        self.closed = True
+
+
+def progress_note(message: str, *, enabled: bool = True) -> None:
+    if enabled:
+        print(f"[progress] {message}", file=sys.stderr, flush=True)
+
+
+@dataclass(frozen=True)
+class ModelSpec:
+    name: str
+    kind: str
+    use_intraday: bool
+    feature_profile: str = "all"
+    top_k: int | None = None
+    l2: float = 0.5
+    n_trees: int = 60
+    max_depth: int = 5
+    min_leaf: int = 30
+    seed: int = 7
+
+
+@dataclass
+class FitResult:
+    symbol: str
+    horizon: str
+    horizon_bars: int
+    config: dict[str, object]
+    threshold: float
+    validation_accuracy: float
+    test_accuracy: float
+    baseline_accuracy: float
+    n_train: int
+    n_valid: int
+    n_test: int
+    train_start: str
+    train_end: str
+    valid_start: str
+    valid_end: str
+    test_start: str
+    test_end: str
+    latest_forecast_date: str
+    latest_forecast_for: str
+    latest_forecast_prob_up: float
+    latest_forecast_signal: str
+    feature_count: int
+
+
+def price_prefix(symbol: str) -> str:
+    return symbol.lower().replace("&", "and").replace(" ", "_")
+
+
+def benchmark_symbol(symbol: str) -> str:
+    return SYMBOL_BENCHMARKS.get(symbol, "NIFTY 50")
+
+
+def symbol_file_stem(symbol: str) -> str:
+    mapping = {
+        "NIFTY 50": "nifty50",
+        "NIFTY BANK": "banknifty",
+        "INDIA VIX": "india_vix",
+    }
+    if symbol not in mapping:
+        raise KeyError(f"Unsupported symbol: {symbol}")
+    return mapping[symbol]
+
+
+def sigmoid(x: np.ndarray) -> np.ndarray:
+    return 1.0 / (1.0 + np.exp(-np.clip(x, -40.0, 40.0)))
+
+
+def safe_div(numer: pd.Series | np.ndarray, denom: pd.Series | np.ndarray) -> pd.Series:
+    n = pd.Series(numer, copy=False)
+    d = pd.Series(denom, copy=False)
+    out = pd.Series(np.nan, index=n.index, dtype="float64")
+    mask = d.notna() & np.isfinite(d.to_numpy(dtype="float64")) & (d != 0)
+    out.loc[mask] = n.loc[mask].to_numpy(dtype="float64") / d.loc[mask].to_numpy(dtype="float64")
+    return out
+
+
+def _to_ns_datetime(series: pd.Series) -> pd.Series:
+    return pd.to_datetime(series, errors="coerce").astype("datetime64[ns]")
+
+
+@lru_cache(maxsize=None)
+def load_price_frame(symbol: str) -> pd.DataFrame:
+    path = PRICE_DIR / f"{symbol_file_stem(symbol)}_1d.csv"
+    if not path.exists():
+        raise FileNotFoundError(f"Missing daily price file for {symbol}: {path}")
+    df = pd.read_csv(path).copy()
+    df.columns = [str(c).strip().lower().replace(" ", "_") for c in df.columns]
+    if "date" not in df.columns:
+        raise ValueError(f"Price frame for {symbol} has no date column")
+    df["date"] = _to_ns_datetime(df["date"])
+    for col in df.columns:
+        if col != "date":
+            df[col] = pd.to_numeric(df[col], errors="coerce")
+    if "volume" in df.columns:
+        volume = pd.to_numeric(df["volume"], errors="coerce")
+        volume_available = volume.replace(0, np.nan).notna().sum() >= max(20, int(0.5 * len(volume)))
+        df["volume"] = volume.replace(0, np.nan) if volume_available else 0.0
+    return df.dropna(subset=["date"]).sort_values("date").reset_index(drop=True)
+
+
+def load_vix() -> pd.DataFrame:
+    path = PRICE_DIR / "india_vix_1d.csv"
+    if not path.exists():
+        return pd.DataFrame(columns=["date"])
+    df = pd.read_csv(path).copy()
+    df.columns = [str(c).strip().lower().replace(" ", "_") for c in df.columns]
+    if "date" not in df.columns:
+        return pd.DataFrame(columns=["date"])
+    df["date"] = _to_ns_datetime(df["date"])
+    rename_map = {
+        "open": "vix_open",
+        "high": "vix_high",
+        "low": "vix_low",
+        "close": "vix_close",
+        "volume": "vix_volume",
+    }
+    df = df.rename(columns={k: v for k, v in rename_map.items() if k in df.columns})
+    for col in df.columns:
+        if col != "date":
+            df[col] = pd.to_numeric(df[col], errors="coerce")
+    return df.dropna(subset=["date"]).sort_values("date").reset_index(drop=True)
+
+
+def load_external_panel() -> pd.DataFrame:
+    path = ALT_DIR / "external" / "processed" / "external_daily_panel.csv"
+    if not path.exists():
+        return pd.DataFrame(columns=["date"])
+    df = pd.read_csv(path).copy()
+    df.columns = [str(c).strip().lower().replace(" ", "_") for c in df.columns]
+    if "date" not in df.columns:
+        return pd.DataFrame(columns=["date"])
+    df["date"] = _to_ns_datetime(df["date"])
+    for col in df.columns:
+        if col != "date":
+            df[col] = pd.to_numeric(df[col], errors="coerce")
+    return df.dropna(subset=["date"]).sort_values("date").reset_index(drop=True)
+
+
+def load_institutional_panel() -> pd.DataFrame:
+    path = ALT_DIR / "institutional" / "processed" / "institutional_daily_panel.csv"
+    if not path.exists():
+        return pd.DataFrame(columns=["date"])
+    df = pd.read_csv(path).copy()
+    df.columns = [str(c).strip().lower().replace(" ", "_") for c in df.columns]
+    if "date" not in df.columns:
+        return pd.DataFrame(columns=["date"])
+    df["date"] = _to_ns_datetime(df["date"])
+    for col in df.columns:
+        if col != "date":
+            df[col] = pd.to_numeric(df[col], errors="coerce")
+    return df.dropna(subset=["date"]).sort_values("date").reset_index(drop=True)
+
+
+def load_options_features(symbol: str) -> pd.DataFrame:
+    file_name = {
+        "NIFTY 50": "nifty50_options_daily_features.csv",
+        "NIFTY BANK": "banknifty_options_daily_features.csv",
+    }[symbol]
+    path = ALT_DIR / "options" / "processed" / file_name
+    if not path.exists():
+        return pd.DataFrame(columns=["date"])
+    df = pd.read_csv(path).copy()
+    df.columns = [str(c).strip().lower().replace(" ", "_") for c in df.columns]
+    if "date" not in df.columns:
+        return pd.DataFrame(columns=["date"])
+    df["date"] = _to_ns_datetime(df["date"])
+    prefix = price_prefix(symbol)
+    rename = {c: f"{prefix}_opt_{c}" for c in df.columns if c not in {"date", "spot_close"}}
+    df = df.rename(columns=rename)
+    for col in df.columns:
+        if col != "date":
+            df[col] = pd.to_numeric(df[col], errors="coerce")
+    return df.dropna(subset=["date"]).sort_values("date").reset_index(drop=True)
+
+
+def add_options_regime_features(df: pd.DataFrame) -> pd.DataFrame:
+    df = df.copy()
+    for prefix in ("nifty_50_opt", "nifty_bank_opt"):
+        base = {
+            "pcr_oi": f"{prefix}_pcr_open_int",
+            "pcr_contracts": f"{prefix}_pcr_contracts",
+            "atm_pcr_oi": f"{prefix}_atm_pcr_open_int",
+            "atm_straddle": f"{prefix}_atm_straddle_close",
+            "atm_ce": f"{prefix}_atm_close_ce",
+            "atm_pe": f"{prefix}_atm_close_pe",
+            "atm_oi_ce": f"{prefix}_atm_open_int_ce",
+            "atm_oi_pe": f"{prefix}_atm_open_int_pe",
+            "contracts_ce": f"{prefix}_contracts_ce",
+            "contracts_pe": f"{prefix}_contracts_pe",
+            "oi_ce": f"{prefix}_open_int_ce",
+            "oi_pe": f"{prefix}_open_int_pe",
+            "chg_oi_ce": f"{prefix}_chg_in_oi_ce",
+            "chg_oi_pe": f"{prefix}_chg_in_oi_pe",
+        }
+        if base["atm_ce"] in df.columns and base["atm_pe"] in df.columns:
+            ce = pd.to_numeric(df[base["atm_ce"]], errors="coerce")
+            pe = pd.to_numeric(df[base["atm_pe"]], errors="coerce")
+            total = ce + pe
+            df[f"{prefix}_atm_skew"] = safe_div(ce - pe, total + 1e-6)
+            df[f"{prefix}_atm_put_share"] = safe_div(pe, total + 1e-6)
+        if base["atm_straddle"] in df.columns:
+            series = pd.to_numeric(df[base["atm_straddle"]], errors="coerce")
+            for w in (5, 20):
+                df[f"{prefix}_atm_straddle_z{w}"] = safe_div(series - series.rolling(w).mean(), series.rolling(w).std())
+            df[f"{prefix}_atm_straddle_ret_5"] = series.pct_change(5, fill_method=None)
+        if base["pcr_oi"] in df.columns:
+            series = pd.to_numeric(df[base["pcr_oi"]], errors="coerce")
+            df[f"{prefix}_pcr_oi_z20"] = safe_div(series - series.rolling(20).mean(), series.rolling(20).std())
+            df[f"{prefix}_pcr_oi_chg_5"] = series.diff(5)
+        if base["pcr_contracts"] in df.columns:
+            series = pd.to_numeric(df[base["pcr_contracts"]], errors="coerce")
+            df[f"{prefix}_pcr_contracts_z20"] = safe_div(series - series.rolling(20).mean(), series.rolling(20).std())
+        if base["atm_pcr_oi"] in df.columns:
+            series = pd.to_numeric(df[base["atm_pcr_oi"]], errors="coerce")
+            df[f"{prefix}_atm_pcr_oi_z20"] = safe_div(series - series.rolling(20).mean(), series.rolling(20).std())
+        if base["oi_ce"] in df.columns and base["oi_pe"] in df.columns:
+            ce = pd.to_numeric(df[base["oi_ce"]], errors="coerce")
+            pe = pd.to_numeric(df[base["oi_pe"]], errors="coerce")
+            total = ce + pe
+            df[f"{prefix}_oi_balance"] = safe_div(pe - ce, total + 1e-6)
+        if base["contracts_ce"] in df.columns and base["contracts_pe"] in df.columns:
+            ce = pd.to_numeric(df[base["contracts_ce"]], errors="coerce")
+            pe = pd.to_numeric(df[base["contracts_pe"]], errors="coerce")
+            total = ce + pe
+            df[f"{prefix}_contracts_balance"] = safe_div(pe - ce, total + 1e-6)
+        if base["chg_oi_ce"] in df.columns and base["chg_oi_pe"] in df.columns:
+            ce = pd.to_numeric(df[base["chg_oi_ce"]], errors="coerce")
+            pe = pd.to_numeric(df[base["chg_oi_pe"]], errors="coerce")
+            total = ce.abs() + pe.abs()
+            df[f"{prefix}_chg_oi_balance"] = safe_div(pe - ce, total + 1e-6)
+    if {"nifty_50_opt_atm_straddle_close", "nifty_50_close"}.issubset(df.columns):
+        df["nifty_50_opt_straddle_rel_spot"] = safe_div(df["nifty_50_opt_atm_straddle_close"], df["nifty_50_close"])
+    if {"nifty_bank_opt_atm_straddle_close", "nifty_bank_close"}.issubset(df.columns):
+        df["nifty_bank_opt_straddle_rel_spot"] = safe_div(df["nifty_bank_opt_atm_straddle_close"], df["nifty_bank_close"])
+    return df
+
+
+def add_external_regime_features(df: pd.DataFrame) -> pd.DataFrame:
+    df = df.copy()
+    base_cols = [c for c in df.columns if c.endswith("_value") or c.endswith("_change_1") or c.endswith("_return_1")]
+    for col in base_cols:
+        series = pd.to_numeric(df[col], errors="coerce")
+        if series.notna().sum() < 40:
+            continue
+        for w in (5, 20, 60):
+            df[f"{col}_mean_{w}"] = series.rolling(w).mean()
+        for w in (20, 60):
+            rolling_std = series.rolling(w).std()
+            df[f"{col}_z_{w}"] = safe_div(series - series.rolling(w).mean(), rolling_std)
+    ratio_pairs = [
+        ("nasdaq_composite_value", "sp500_value", "nasdaq_vs_sp500"),
+        ("vix_fred_value", "vix_close", "us_vix_vs_india_vix"),
+        ("broad_dollar_index_value", "india_fx_inr_per_usd_value", "dxy_vs_inr"),
+    ]
+    for numer_col, denom_col, prefix in ratio_pairs:
+        if numer_col in df.columns and denom_col in df.columns:
+            ratio = safe_div(df[numer_col], df[denom_col])
+            df[f"{prefix}_ratio"] = ratio
+            df[f"{prefix}_z20"] = safe_div(ratio - ratio.rolling(20).mean(), ratio.rolling(20).std())
+            df[f"{prefix}_mom_5"] = ratio.pct_change(5, fill_method=None)
+    if {"us10y_treasury_value", "fed_funds_value"}.issubset(df.columns):
+        spread = pd.to_numeric(df["us10y_treasury_value"], errors="coerce") - pd.to_numeric(df["fed_funds_value"], errors="coerce")
+        df["us10y_minus_fedfunds"] = spread
+        df["us10y_minus_fedfunds_z20"] = safe_div(spread - spread.rolling(20).mean(), spread.rolling(20).std())
+    return df
+
+
+def add_institutional_flow_features(df: pd.DataFrame) -> pd.DataFrame:
+    df = df.copy()
+    net_cols = [
+        "fii_cash_net",
+        "dii_cash_net",
+        "fii_fno_futures_net",
+        "fii_fno_options_net",
+        "fii_index_futures_net_oi",
+        "fii_index_options_net_oi",
+        "fii_index_futures_net_volume",
+        "fii_index_options_net_volume",
+    ]
+    for col in net_cols:
+        if col not in df.columns:
+            continue
+        series = pd.to_numeric(df[col], errors="coerce")
+        gross_col = col.replace("_net", "_buy")
+        alt_gross_col = col.replace("_net", "_long_volume")
+        gross = None
+        if gross_col in df.columns:
+            gross = pd.to_numeric(df[gross_col], errors="coerce").abs()
+        elif alt_gross_col in df.columns:
+            gross = pd.to_numeric(df[alt_gross_col], errors="coerce").abs()
+        for w in (3, 5, 10, 20):
+            df[f"{col}_sum_{w}"] = series.rolling(w).sum()
+            df[f"{col}_mean_{w}"] = series.rolling(w).mean()
+        df[f"{col}_z20"] = safe_div(series - series.rolling(20).mean(), series.rolling(20).std())
+        df[f"{col}_sign"] = np.sign(series)
+        if gross is not None:
+            df[f"{col}_intensity"] = safe_div(series, gross + 1e-6)
+    if {"fii_cash_net", "dii_cash_net"}.issubset(df.columns):
+        cash_spread = pd.to_numeric(df["fii_cash_net"], errors="coerce") - pd.to_numeric(df["dii_cash_net"], errors="coerce")
+        df["inst_cash_spread"] = cash_spread
+        df["inst_cash_spread_z20"] = safe_div(cash_spread - cash_spread.rolling(20).mean(), cash_spread.rolling(20).std())
+    if {"fii_fno_futures_net", "fii_fno_options_net"}.issubset(df.columns):
+        combo = pd.to_numeric(df["fii_fno_futures_net"], errors="coerce") + pd.to_numeric(df["fii_fno_options_net"], errors="coerce")
+        df["fii_fno_total_net"] = combo
+        df["fii_fno_total_net_z20"] = safe_div(combo - combo.rolling(20).mean(), combo.rolling(20).std())
+    if {"fii_index_options_call_net_volume", "fii_index_options_put_net_volume"}.issubset(df.columns):
+        put_call_spread = pd.to_numeric(df["fii_index_options_put_net_volume"], errors="coerce") - pd.to_numeric(df["fii_index_options_call_net_volume"], errors="coerce")
+        total = (
+            pd.to_numeric(df["fii_index_options_put_net_volume"], errors="coerce").abs()
+            + pd.to_numeric(df["fii_index_options_call_net_volume"], errors="coerce").abs()
+        )
+        df["fii_put_call_volume_spread"] = put_call_spread
+        df["fii_put_call_volume_balance"] = safe_div(put_call_spread, total + 1e-6)
+    return df
+
+
+def add_cross_market_features(df: pd.DataFrame) -> pd.DataFrame:
+    df = df.copy()
+    if {"nifty_50_ret_1", "fii_cash_net"}.issubset(df.columns):
+        df["fii_cash_x_nifty50_ret"] = pd.to_numeric(df["fii_cash_net"], errors="coerce") * pd.to_numeric(df["nifty_50_ret_1"], errors="coerce")
+    if {"nifty_bank_ret_1", "fii_fno_futures_net"}.issubset(df.columns):
+        df["fii_futures_x_bank_ret"] = pd.to_numeric(df["fii_fno_futures_net"], errors="coerce") * pd.to_numeric(df["nifty_bank_ret_1"], errors="coerce")
+    if {"vix_close", "fii_cash_net"}.issubset(df.columns):
+        df["fii_cash_vs_vix"] = safe_div(pd.to_numeric(df["fii_cash_net"], errors="coerce"), pd.to_numeric(df["vix_close"], errors="coerce"))
+    if {"vix_fred_value", "nifty_50_ret_std_20"}.issubset(df.columns):
+        df["us_vix_x_local_vol"] = pd.to_numeric(df["vix_fred_value"], errors="coerce") * pd.to_numeric(df["nifty_50_ret_std_20"], errors="coerce")
+    return df
+
+
+def add_price_features(df: pd.DataFrame, prefix: str) -> pd.DataFrame:
+    df = df.copy()
+    if f"{prefix}_close" not in df.columns:
+        rename_map = {c: f"{prefix}_{c}" for c in ["open", "high", "low", "close", "volume"] if c in df.columns}
+        df = df.rename(columns=rename_map)
+    close = df[f"{prefix}_close"]
+    open_ = df[f"{prefix}_open"]
+    high = df[f"{prefix}_high"]
+    low = df[f"{prefix}_low"]
+    raw_vol = pd.to_numeric(df.get(f"{prefix}_volume", 0.0), errors="coerce")
+    volume_missing = raw_vol.replace(0, np.nan).notna().sum() < max(20, int(0.5 * len(raw_vol)))
+    vol = raw_vol.replace(0, np.nan)
+
+    ret_1 = close.pct_change()
+    df[f"{prefix}_ret_1"] = ret_1
+    df[f"{prefix}_ret_2"] = close.pct_change(2)
+    df[f"{prefix}_ret_5"] = close.pct_change(5)
+    df[f"{prefix}_ret_10"] = close.pct_change(10)
+    df[f"{prefix}_logret_1"] = np.log(close / close.shift(1))
+    df[f"{prefix}_gap_1"] = open_ / close.shift(1) - 1.0
+    df[f"{prefix}_body"] = close / open_ - 1.0
+    df[f"{prefix}_range"] = safe_div(high - low, close)
+    df[f"{prefix}_upper_wick"] = safe_div(high - np.maximum(open_, close), close)
+    df[f"{prefix}_lower_wick"] = safe_div(np.minimum(open_, close) - low, close)
+    df[f"{prefix}_trend_5"] = close / close.rolling(5).mean() - 1.0
+    df[f"{prefix}_trend_10"] = close / close.rolling(10).mean() - 1.0
+    df[f"{prefix}_trend_20"] = close / close.rolling(20).mean() - 1.0
+    df[f"{prefix}_trend_60"] = close / close.rolling(60).mean() - 1.0
+    df[f"{prefix}_trend_120"] = close / close.rolling(120).mean() - 1.0
+    df[f"{prefix}_trend_252"] = close / close.rolling(252).mean() - 1.0
+    rolling_max_252 = close.rolling(252).max()
+    rolling_min_252 = close.rolling(252).min()
+    df[f"{prefix}_drawdown_252"] = close / rolling_max_252 - 1.0
+    df[f"{prefix}_dist_from_low_252"] = close / rolling_min_252 - 1.0
+    if volume_missing:
+        df[f"{prefix}_vol_chg_1"] = 0.0
+        df[f"{prefix}_vol_z_20"] = 0.0
+        df[f"{prefix}_vol_z_60"] = 0.0
+    else:
+        df[f"{prefix}_vol_chg_1"] = vol.pct_change()
+        df[f"{prefix}_vol_z_20"] = (vol - vol.rolling(20).mean()) / vol.rolling(20).std()
+        df[f"{prefix}_vol_z_60"] = (vol - vol.rolling(60).mean()) / vol.rolling(60).std()
+
+    for w in [3, 5, 10, 20, 60, 120, 252]:
+        df[f"{prefix}_ret_mean_{w}"] = ret_1.rolling(w).mean()
+        df[f"{prefix}_ret_std_{w}"] = ret_1.rolling(w).std()
+        df[f"{prefix}_range_mean_{w}"] = df[f"{prefix}_range"].rolling(w).mean()
+        df[f"{prefix}_range_std_{w}"] = df[f"{prefix}_range"].rolling(w).std()
+
+    delta = close.diff()
+    gain = delta.clip(lower=0.0)
+    loss = -delta.clip(upper=0.0)
+    avg_gain = gain.ewm(alpha=1 / 14.0, adjust=False, min_periods=14).mean()
+    avg_loss = loss.ewm(alpha=1 / 14.0, adjust=False, min_periods=14).mean()
+    rs = avg_gain / avg_loss.replace(0.0, np.nan)
+    df[f"{prefix}_rsi_14"] = 100.0 - (100.0 / (1.0 + rs))
+    ema_12 = close.ewm(span=12, adjust=False, min_periods=12).mean()
+    ema_26 = close.ewm(span=26, adjust=False, min_periods=26).mean()
+    macd = ema_12 - ema_26
+    signal = macd.ewm(span=9, adjust=False, min_periods=9).mean()
+    df[f"{prefix}_macd"] = macd / close
+    df[f"{prefix}_macd_signal"] = signal / close
+    df[f"{prefix}_macd_hist"] = (macd - signal) / close
+    return df
+
+
+def build_panel(include_engineered: bool = True, include_option_engineered: bool = True) -> pd.DataFrame:
+    nifty = add_price_features(load_price_frame("NIFTY 50"), "nifty_50")
+    bank = add_price_features(load_price_frame("NIFTY BANK"), "nifty_bank")
+    panel = nifty.merge(bank, on="date", how="inner").sort_values("date").reset_index(drop=True)
+    panel["pair_ret_corr_20"] = panel["nifty_50_ret_1"].rolling(20).corr(panel["nifty_bank_ret_1"])
+    panel["pair_ret_corr_60"] = panel["nifty_50_ret_1"].rolling(60).corr(panel["nifty_bank_ret_1"])
+    panel["pair_close_ratio"] = panel["nifty_50_close"] / panel["nifty_bank_close"] - 1.0
+
+    vix = load_vix()
+    if not vix.empty:
+        panel = pd.merge_asof(panel.sort_values("date"), vix.sort_values("date"), on="date", direction="backward")
+
+    external = load_external_panel()
+    if not external.empty:
+        panel = pd.merge_asof(panel.sort_values("date"), external.sort_values("date"), on="date", direction="backward")
+
+    institutional = load_institutional_panel()
+    if not institutional.empty:
+        panel = pd.merge_asof(
+            panel.sort_values("date"),
+            institutional.sort_values("date"),
+            on="date",
+            direction="backward",
+        )
+
+    nifty_opts = load_options_features("NIFTY 50")
+    if not nifty_opts.empty:
+        panel = pd.merge_asof(panel.sort_values("date"), nifty_opts.sort_values("date"), on="date", direction="backward")
+
+    bank_opts = load_options_features("NIFTY BANK")
+    if not bank_opts.empty:
+        panel = pd.merge_asof(panel.sort_values("date"), bank_opts.sort_values("date"), on="date", direction="backward")
+
+    if include_option_engineered:
+        panel = add_options_regime_features(panel)
+    if include_engineered:
+        panel = add_external_regime_features(panel)
+        panel = add_institutional_flow_features(panel)
+        panel = add_cross_market_features(panel)
+    return panel.sort_values("date").reset_index(drop=True)
+
+
+@lru_cache(maxsize=None)
+def load_intraday_daily(symbol: str) -> pd.DataFrame:
+    path = INTRADAY_DIR / f"{symbol}_minute.csv"
+    if not path.exists():
+        raise FileNotFoundError(f"Missing minute file for {symbol}: {path}")
+    df = pd.read_csv(path).copy()
+    df.columns = [str(c).strip().lower().replace(" ", "_") for c in df.columns]
+    df["date"] = _to_ns_datetime(df["date"])
+    for col in ("open", "high", "low", "close", "volume"):
+        if col in df.columns:
+            df[col] = pd.to_numeric(df[col], errors="coerce")
+    df["session_date"] = df["date"].dt.normalize()
+    df = df.dropna(subset=["date"]).sort_values("date").reset_index(drop=True)
+    grouped = df.groupby("session_date", sort=True)
+
+    def session_apply(func):
+        return grouped.apply(func, include_groups=False).to_numpy()
+
+    out = pd.DataFrame({"date": grouped["date"].first().dt.normalize()})
+    out["intraday_open"] = grouped["open"].first().to_numpy()
+    out["intraday_high"] = grouped["high"].max().to_numpy()
+    out["intraday_low"] = grouped["low"].min().to_numpy()
+    out["intraday_close"] = grouped["close"].last().to_numpy()
+    out["intraday_nbars"] = grouped.size().to_numpy()
+    out["intraday_range"] = safe_div(out["intraday_high"] - out["intraday_low"], out["intraday_low"])
+    out["intraday_body"] = safe_div(out["intraday_close"] - out["intraday_open"], out["intraday_open"])
+    out["intraday_close_loc"] = safe_div(
+        out["intraday_close"] - out["intraday_low"],
+        out["intraday_high"] - out["intraday_low"],
+    )
+    out["intraday_first_30"] = session_apply(
+        lambda x: x["close"].iloc[min(29, len(x) - 1)] / x["open"].iloc[0] - 1.0
+    )
+    out["intraday_first_60"] = session_apply(
+        lambda x: x["close"].iloc[min(59, len(x) - 1)] / x["open"].iloc[0] - 1.0
+    )
+    out["intraday_last_30"] = session_apply(
+        lambda x: x["close"].iloc[-1] / x["close"].iloc[max(0, len(x) - 30)] - 1.0
+    )
+    out["intraday_last_60"] = session_apply(
+        lambda x: x["close"].iloc[-1] / x["close"].iloc[max(0, len(x) - 60)] - 1.0
+    )
+    out["intraday_midday"] = session_apply(
+        lambda x: x["close"].iloc[max(0, len(x) // 2)] / x["open"].iloc[0] - 1.0
+    )
+    out["intraday_second_half"] = session_apply(
+        lambda x: x["close"].iloc[-1] / x["close"].iloc[max(0, len(x) // 2)] - 1.0
+    )
+    out["intraday_vshape"] = out["intraday_first_60"] - out["intraday_last_60"]
+    out["intraday_abruptness"] = safe_div(out["intraday_high"] - out["intraday_low"], out["intraday_open"])
+    out["intraday_realized_vol"] = session_apply(lambda x: np.log(x["close"]).diff().std() * np.sqrt(len(x)))
+    out["intraday_range_vs_body"] = safe_div(out["intraday_range"], out["intraday_body"].abs() + 1e-6)
+    return out
+
+
+def build_master_frame(
+    symbol: str,
+    target_bars: int = 1,
+) -> pd.DataFrame:
+    own = price_prefix(symbol)
+    use_engineered = symbol == "NIFTY BANK"
+    use_option_engineered = symbol == "NIFTY 50"
+    panel = build_panel(include_engineered=use_engineered, include_option_engineered=use_option_engineered).copy()
+    intraday = load_intraday_daily(symbol)
+    frame = pd.merge_asof(panel.sort_values("date"), intraday.sort_values("date"), on="date", direction="backward")
+
+    if target_bars < 1:
+        raise ValueError("target_bars must be at least 1")
+    future_close = frame[f"{own}_close"].shift(-target_bars)
+    known_future = future_close.notna()
+    frame["target"] = np.where(known_future, (future_close > frame[f"{own}_close"]).astype("int64"), np.nan)
+    frame["next_close_return"] = future_close / frame[f"{own}_close"] - 1.0
+    frame["target_lag_1"] = frame["target"].shift(1)
+    frame["target_roll_up_5"] = frame["target"].shift(1).rolling(5).mean()
+    frame["target_roll_up_10"] = frame["target"].shift(1).rolling(10).mean()
+    frame["target_roll_up_20"] = frame["target"].shift(1).rolling(20).mean()
+    frame = frame.replace([np.inf, -np.inf], np.nan)
+    # Keep the latest row even though its future close/target is unknown.
+    # Model fitting and backtests filter to known target rows later, while latest_row uses this retained row.
+    return frame.dropna(subset=["date", f"{own}_close"]).reset_index(drop=True)
+
+
+def is_option_column(name: str) -> bool:
+    return "_opt_" in name
+
+
+def is_flow_column(name: str) -> bool:
+    prefixes = ("fii_", "dii_", "inst_", "participant_", "cash_", "fno_")
+    return name.startswith(prefixes) or "put_call_volume" in name
+
+
+def is_external_column(name: str) -> bool:
+    prefixes = (
+        "sp500_",
+        "nasdaq_composite_",
+        "dow_jones_",
+        "nikkei225_",
+        "us10y_treasury_",
+        "fed_funds_",
+        "india_fx_inr_per_usd_",
+        "brent_fred_",
+        "vix_fred_",
+        "broad_dollar_index_",
+        "dxy_",
+        "us10y_minus_fedfunds",
+        "nasdaq_vs_sp500",
+        "us_vix_vs_india_vix",
+    )
+    return name.startswith(prefixes)
+
+
+def is_vix_column(name: str) -> bool:
+    return name.startswith("vix_")
+
+
+def is_pair_column(name: str) -> bool:
+    return name.startswith("pair_")
+
+
+def is_intraday_column(name: str) -> bool:
+    return name.startswith("intraday_")
+
+
+def rank_feature_columns(train_df: pd.DataFrame, feature_cols: list[str]) -> list[str]:
+    scores: dict[str, float] = {}
+    y = train_df["target"].astype(float)
+    for col in feature_cols:
+        x = pd.to_numeric(train_df[col], errors="coerce")
+        pair = pd.concat([x.rename("x"), y.rename("y")], axis=1).dropna()
+        if len(pair) < 40 or pair["x"].nunique() <= 1:
+            scores[col] = 0.0
+            continue
+        corr = pair["x"].corr(pair["y"])
+        scores[col] = abs(float(corr)) if corr is not None and np.isfinite(corr) else 0.0
+    return pd.Series(scores).sort_values(ascending=False).index.tolist()
+
+
+def select_model_columns(frame: pd.DataFrame, use_intraday: bool, feature_profile: str, symbol: str) -> list[str]:
+    meta = {"date", "target", "next_close_return"}
+    cols = [c for c in frame.columns if c not in meta and pd.api.types.is_numeric_dtype(frame[c])]
+    if not use_intraday:
+        cols = [c for c in cols if not is_intraday_column(c)]
+    own = price_prefix(symbol)
+    other = price_prefix(benchmark_symbol(symbol))
+    core_cols = [
+        c for c in cols
+        if c.startswith(f"{own}_") or c.startswith(f"{other}_") or is_pair_column(c) or is_vix_column(c) or c.startswith("target_")
+    ]
+    option_cols = [c for c in cols if is_option_column(c)]
+    external_cols = [c for c in cols if is_external_column(c)]
+    flow_cols = [c for c in cols if is_flow_column(c)]
+    intraday_cols = [c for c in cols if is_intraday_column(c)]
+
+    profile_map = {
+        "all": cols,
+        "lean": core_cols + intraday_cols,
+        "price_options": core_cols + option_cols + intraday_cols,
+        "price_external": core_cols + external_cols + intraday_cols,
+        "options_macro": core_cols + option_cols + external_cols + intraday_cols,
+        "bank_alt": core_cols + option_cols + external_cols + flow_cols + intraday_cols,
+    }
+    selected = profile_map.get(feature_profile, cols)
+    return list(dict.fromkeys([c for c in selected if c in cols]))
+
+
+def train_logistic_model(
+    x: np.ndarray,
+    y: np.ndarray,
+    l2: float = 0.5,
+    max_iter: int = 900,
+    lr: float = 0.05,
+    *,
+    progress_enabled: bool = True,
+    progress_update_every: float = 0.2,
+    progress_description: str = "Logistic training",
+) -> dict[str, np.ndarray | float]:
+    x = np.asarray(x, dtype="float64")
+    y = np.asarray(y, dtype="float64")
+    mean = np.nanmean(x, axis=0)
+    std = np.nanstd(x, axis=0)
+    std[~np.isfinite(std) | (std == 0)] = 1.0
+    xs = (x - mean) / std
+    coef = np.zeros(xs.shape[1], dtype="float64")
+    intercept = float(np.log((y.mean() + 1e-6) / (1.0 - y.mean() + 1e-6)))
+    mw = np.zeros_like(coef)
+    vw = np.zeros_like(coef)
+    mb = 0.0
+    vb = 0.0
+    beta1 = 0.9
+    beta2 = 0.999
+    eps = 1e-8
+    with ProgressBar(
+        max_iter,
+        progress_description,
+        enabled=progress_enabled,
+        update_every=progress_update_every,
+    ) as progress:
+        for step in range(1, max_iter + 1):
+            z = xs @ coef + intercept
+            p = sigmoid(z)
+            err = p - y
+            grad_w = (xs.T @ err) / len(y) + l2 * coef
+            grad_b = err.mean()
+            mw = beta1 * mw + (1.0 - beta1) * grad_w
+            vw = beta2 * vw + (1.0 - beta2) * (grad_w * grad_w)
+            mb = beta1 * mb + (1.0 - beta1) * grad_b
+            vb = beta2 * vb + (1.0 - beta2) * (grad_b * grad_b)
+            mw_hat = mw / (1.0 - beta1**step)
+            vw_hat = vw / (1.0 - beta2**step)
+            mb_hat = mb / (1.0 - beta1**step)
+            vb_hat = vb / (1.0 - beta2**step)
+            coef -= lr * mw_hat / (np.sqrt(vw_hat) + eps)
+            intercept -= lr * mb_hat / (math.sqrt(vb_hat) + eps)
+            progress.update(step)
+            if step % 100 == 0 and float(np.linalg.norm(grad_w) + abs(grad_b)) < 1e-4:
+                progress.update(step, description=f"{progress_description} converged", force=True)
+                break
+    return {"kind": "logit", "coef": coef, "intercept": intercept, "mean": mean, "std": std}
+
+
+def predict_logistic_model(model: dict[str, np.ndarray | float], x: np.ndarray) -> np.ndarray:
+    xs = (np.asarray(x, dtype="float64") - model["mean"]) / model["std"]
+    z = xs @ model["coef"] + float(model["intercept"])
+    return sigmoid(z)
+
+
+@dataclass
+class TreeNode:
+    feat: int | None = None
+    thr: float | None = None
+    left: "TreeNode | None" = None
+    right: "TreeNode | None" = None
+    prob: float | None = None
+
+
+def _gini(y: np.ndarray) -> float:
+    if len(y) == 0:
+        return 0.0
+    p = float(np.mean(y))
+    return 1.0 - p * p - (1.0 - p) * (1.0 - p)
+
+
+def _best_split(x: np.ndarray, y: np.ndarray, features: np.ndarray) -> tuple[float, int, float, np.ndarray] | None:
+    n = len(y)
+    parent = _gini(y)
+    best: tuple[float, int, float, np.ndarray] | None = None
+    for feat in features:
+        col = x[:, feat]
+        if np.all(col == col[0]):
+            continue
+        thresholds = np.unique(np.quantile(col, [0.25, 0.5, 0.75]))
+        for thr in thresholds:
+            left = col <= thr
+            nl = int(left.sum())
+            nr = n - nl
+            if nl < 30 or nr < 30:
+                continue
+            gain = parent - (nl / n) * _gini(y[left]) - (nr / n) * _gini(y[~left])
+            if best is None or gain > best[0]:
+                best = (gain, int(feat), float(thr), left)
+    return best
+
+
+def _build_tree(
+    x: np.ndarray,
+    y: np.ndarray,
+    depth: int,
+    max_depth: int,
+    min_leaf: int,
+    mtry: int,
+    rng: np.random.Generator,
+) -> TreeNode:
+    if depth >= max_depth or len(y) < 2 * min_leaf or len(np.unique(y)) == 1:
+        return TreeNode(prob=float(np.mean(y)) if len(y) else 0.5)
+    features = rng.choice(x.shape[1], size=min(mtry, x.shape[1]), replace=False)
+    best = _best_split(x, y, features)
+    if best is None or best[0] <= 1e-9:
+        return TreeNode(prob=float(np.mean(y)))
+    _, feat, thr, left = best
+    if left.sum() < min_leaf or (~left).sum() < min_leaf:
+        return TreeNode(prob=float(np.mean(y)))
+    return TreeNode(
+        feat=feat,
+        thr=thr,
+        left=_build_tree(x[left], y[left], depth + 1, max_depth, min_leaf, mtry, rng),
+        right=_build_tree(x[~left], y[~left], depth + 1, max_depth, min_leaf, mtry, rng),
+    )
+
+
+def _tree_predict(node: TreeNode, row: np.ndarray) -> float:
+    while node.prob is None:
+        node = node.left if row[node.feat] <= node.thr else node.right
+    return float(node.prob)
+
+
+def train_forest_model(
+    x: np.ndarray,
+    y: np.ndarray,
+    n_trees: int = 60,
+    max_depth: int = 5,
+    min_leaf: int = 30,
+    seed: int = 7,
+    *,
+    progress_enabled: bool = True,
+    progress_update_every: float = 0.2,
+    progress_description: str = "Forest training",
+) -> dict[str, object]:
+    x = np.asarray(x, dtype="float64")
+    y = np.asarray(y, dtype="int64")
+    rng = np.random.default_rng(seed)
+    mtry = max(4, int(math.sqrt(x.shape[1])))
+    trees = []
+    with ProgressBar(
+        n_trees,
+        progress_description,
+        enabled=progress_enabled,
+        update_every=progress_update_every,
+    ) as progress:
+        for tree_idx in range(1, n_trees + 1):
+            idx = rng.integers(0, len(y), len(y))
+            trees.append(_build_tree(x[idx], y[idx], 0, max_depth, min_leaf, mtry, rng))
+            progress.update(tree_idx)
+    return {"kind": "forest", "trees": trees}
+
+
+def predict_forest_model(model: dict[str, object], x: np.ndarray) -> np.ndarray:
+    x = np.asarray(x, dtype="float64")
+    trees = model["trees"]
+    probs = np.zeros(len(x), dtype="float64")
+    for i, row in enumerate(x):
+        probs[i] = sum(_tree_predict(tree, row) for tree in trees) / len(trees)
+    return probs
+
+
+def train_spec_model(
+    spec: ModelSpec,
+    train_df: pd.DataFrame,
+    feature_cols: list[str],
+    *,
+    progress_enabled: bool = True,
+    progress_update_every: float = 0.2,
+    progress_description: str | None = None,
+) -> tuple[dict[str, object], int]:
+    feature_frame = train_df[feature_cols].replace([np.inf, -np.inf], np.nan)
+    fill_values = feature_frame.median(numeric_only=True).reindex(feature_cols).fillna(0.0)
+    x = feature_frame.fillna(fill_values).to_numpy(dtype="float64")
+    y = train_df["target"].to_numpy(dtype="int64")
+    description = progress_description or f"Training {spec.name}"
+    if spec.kind == "logit":
+        model = train_logistic_model(
+            x,
+            y,
+            l2=spec.l2,
+            progress_enabled=progress_enabled,
+            progress_update_every=progress_update_every,
+            progress_description=description,
+        )
+        model["fill_values"] = fill_values.to_numpy(dtype="float64")
+        return model, len(feature_cols)
+    if spec.kind == "forest":
+        model = train_forest_model(
+            x,
+            y,
+            n_trees=spec.n_trees,
+            max_depth=spec.max_depth,
+            min_leaf=spec.min_leaf,
+            seed=spec.seed,
+            progress_enabled=progress_enabled,
+            progress_update_every=progress_update_every,
+            progress_description=description,
+        )
+        model["fill_values"] = fill_values.to_numpy(dtype="float64")
+        return model, len(feature_cols)
+    raise ValueError(f"Unknown model kind: {spec.kind}")
+
+
+def predict_spec_model(model: dict[str, object], df: pd.DataFrame, feature_cols: list[str]) -> np.ndarray:
+    fill_values = pd.Series(np.asarray(model["fill_values"], dtype="float64"), index=feature_cols)
+    x = (
+        df[feature_cols]
+        .replace([np.inf, -np.inf], np.nan)
+        .fillna(fill_values)
+        .to_numpy(dtype="float64")
+    )
+    if model["kind"] == "logit":
+        return predict_logistic_model(model, x)
+    if model["kind"] == "forest":
+        return predict_forest_model(model, x)
+    raise ValueError(f"Unknown model kind: {model['kind']}")
+
+
+def best_threshold(y_true: np.ndarray, prob: np.ndarray) -> tuple[float, float]:
+    grid = np.round(np.arange(0.35, 0.651, 0.001), 3)
+    best_t = 0.5
+    best_acc = -1.0
+    for t in grid:
+        acc = float(np.mean((prob >= t).astype(int) == y_true))
+        if acc > best_acc or (acc == best_acc and abs(t - 0.5) < abs(best_t - 0.5)):
+            best_t = float(t)
+            best_acc = acc
+    return best_t, best_acc
+
+
+def blend_weights_grid(n_models: int, random_samples: int = 2000, seed: int = 7) -> Iterable[np.ndarray]:
+    rng = np.random.default_rng(seed)
+    if n_models == 1:
+        yield np.array([1.0], dtype="float64")
+        return
+    yield np.full(n_models, 1.0 / n_models, dtype="float64")
+    for i in range(n_models):
+        w = np.zeros(n_models, dtype="float64")
+        w[i] = 1.0
+        yield w
+    for _ in range(random_samples):
+        yield rng.dirichlet(np.ones(n_models, dtype="float64"))
+
+
+def search_blend(
+    y_valid: np.ndarray,
+    prob_valid_list: list[np.ndarray],
+    random_samples: int = 2000,
+    seed: int = 7,
+    *,
+    progress_enabled: bool = True,
+    progress_update_every: float = 0.2,
+    progress_description: str = "Blend search",
+) -> tuple[np.ndarray, float, float]:
+    stacked = np.vstack(prob_valid_list)
+    best_weights = None
+    best_thr = 0.5
+    best_acc = -1.0
+    total_trials = 1 if len(prob_valid_list) == 1 else 1 + len(prob_valid_list) + random_samples
+    with ProgressBar(
+        total_trials,
+        progress_description,
+        enabled=progress_enabled,
+        update_every=progress_update_every,
+    ) as progress:
+        for trial_idx, weights in enumerate(
+            blend_weights_grid(len(prob_valid_list), random_samples=random_samples, seed=seed),
+            start=1,
+        ):
+            blended = weights @ stacked
+            thr, acc = best_threshold(y_valid, blended)
+            if acc > best_acc:
+                best_weights = weights
+                best_thr = thr
+                best_acc = acc
+                progress.update(
+                    trial_idx,
+                    description=f"{progress_description} best={best_acc:.2%}",
+                    force=True,
+                )
+            else:
+                progress.update(trial_idx)
+    if best_weights is None:
+        raise RuntimeError("Blend search failed")
+    return best_weights, best_thr, best_acc
+
+
+def apply_symbol_decision_overlay(
+    symbol: str,
+    df: pd.DataFrame,
+    prob: np.ndarray,
+    threshold: float,
+    pred: np.ndarray,
+) -> np.ndarray:
+    adjusted = np.asarray(pred, dtype="int64").copy()
+    if symbol == "NIFTY 50" and "nifty_bank_body" in df.columns:
+        bank_body = pd.to_numeric(df["nifty_bank_body"], errors="coerce").to_numpy(dtype="float64")
+        near_threshold = np.abs(np.asarray(prob, dtype="float64") - float(threshold)) <= 0.015
+        bank_reversal_setup = bank_body <= -0.0016219151538434222
+        adjusted[near_threshold & bank_reversal_setup] = 1
+    return adjusted
+
+
+def candidate_pools() -> dict[str, list[tuple[pd.Timestamp, ModelSpec]]]:
+    return {
+        "NIFTY 50": [
+            (pd.Timestamp("2024-04-30"), ModelSpec("daily_forest_2024apr_d3_l10_s17", "forest", False, n_trees=120, max_depth=3, min_leaf=10, seed=17)),
+        ],
+        "NIFTY BANK": [
+            (pd.Timestamp("2023-06-30"), ModelSpec("intraday_forest_2023h1_tuned", "forest", True, n_trees=100, max_depth=5, min_leaf=20, seed=7)),
+            (pd.Timestamp("2022-12-31"), ModelSpec("intraday_logit_2022y", "logit", True, l2=0.5)),
+            (pd.Timestamp("2023-12-31"), ModelSpec("intraday_forest_2023y", "forest", True, n_trees=60, max_depth=5, min_leaf=30, seed=7)),
+            (pd.Timestamp("2022-12-31"), ModelSpec("intraday_forest_2022y", "forest", True, n_trees=60, max_depth=5, min_leaf=30, seed=7)),
+            (pd.Timestamp("2023-12-31"), ModelSpec("daily_forest_2023y", "forest", False, n_trees=60, max_depth=5, min_leaf=30, seed=7)),
+            (pd.Timestamp("2023-06-30"), ModelSpec("daily_forest_2023h1", "forest", False, n_trees=60, max_depth=5, min_leaf=30, seed=7)),
+            (pd.Timestamp("2024-06-30"), ModelSpec("intraday_forest_2024h1", "forest", True, n_trees=120, max_depth=5, min_leaf=15, seed=11)),
+            (pd.Timestamp("2024-06-30"), ModelSpec("intraday_logit_2024h1", "logit", True, l2=1.0)),
+            (pd.Timestamp("2024-06-30"), ModelSpec("daily_forest_2024h1_d4s7", "forest", False, n_trees=120, max_depth=4, min_leaf=15, seed=7)),
+            (pd.Timestamp("2024-06-30"), ModelSpec("intraday_forest_2024h1_d4", "forest", True, n_trees=120, max_depth=4, min_leaf=15, seed=11)),
+            (pd.Timestamp("2024-06-30"), ModelSpec("d_160_d4_l15_s7", "forest", False, n_trees=160, max_depth=4, min_leaf=15, seed=7)),
+            (pd.Timestamp("2021-12-31"), ModelSpec("intraday_forest_2021y", "forest", True, n_trees=120, max_depth=5, min_leaf=15, seed=11)),
+        ],
+    }
+
+
+def evaluate_ensemble(
+    symbol: str,
+    train_end: pd.Timestamp,
+    valid_end: pd.Timestamp,
+    test_end: pd.Timestamp,
+    *,
+    progress_enabled: bool = True,
+    progress_update_every: float = 0.2,
+) -> tuple[FitResult, dict[str, object], pd.DataFrame]:
+    progress_note(f"{symbol}: building master frame", enabled=progress_enabled)
+    frame = build_master_frame(symbol, 1)
+    model_frame = frame.dropna(subset=["target", "next_close_return"]).copy().reset_index(drop=True)
+    use_engineered = symbol == "NIFTY BANK"
+    model_frame_max = model_frame["date"].max()
+    if pd.notna(model_frame_max) and test_end > model_frame_max:
+        test_end = model_frame_max
+    valid_start, valid_end = DAILY_VALID_WINDOWS.get(symbol, (COMMON_VALID_START, valid_end))
+
+    valid_df = model_frame[(model_frame["date"] >= valid_start) & (model_frame["date"] <= valid_end)].copy().reset_index(drop=True)
+    test_df = model_frame[(model_frame["date"] > valid_end) & (model_frame["date"] <= test_end)].copy().reset_index(drop=True)
+    if valid_df.empty or test_df.empty:
+        raise RuntimeError(f"Not enough rows for {symbol}: valid={len(valid_df)} test={len(test_df)}")
+
+    pools = candidate_pools()[symbol]
+    spec_payloads: list[dict[str, object]] = []
+    valid_probs: list[np.ndarray] = []
+    test_probs: list[np.ndarray] = []
+    latest_probs: list[float] = []
+    latest_row = frame.iloc[[-1]].copy()
+
+    with ProgressBar(
+        len(pools),
+        f"{symbol}: candidate models",
+        enabled=progress_enabled,
+        update_every=progress_update_every,
+    ) as pool_progress:
+        for candidate_idx, (candidate_train_end, spec) in enumerate(pools, start=1):
+            pool_progress.update(
+                candidate_idx - 1,
+                description=f"{symbol}: training {spec.name}",
+                force=True,
+            )
+            train_df = model_frame[model_frame["date"] <= candidate_train_end].copy().reset_index(drop=True)
+            feature_cols = select_model_columns(frame, spec.use_intraday, spec.feature_profile, symbol)
+            if spec.top_k is not None and spec.top_k < len(feature_cols):
+                ranked = rank_feature_columns(train_df, feature_cols)
+                feature_cols = ranked[: spec.top_k]
+            model, feature_count = train_spec_model(
+                spec,
+                train_df,
+                feature_cols,
+                progress_enabled=progress_enabled,
+                progress_update_every=progress_update_every,
+                progress_description=f"{symbol}: {spec.name}",
+            )
+            valid_probs.append(predict_spec_model(model, valid_df, feature_cols))
+            test_probs.append(predict_spec_model(model, test_df, feature_cols))
+            latest_probs.append(float(predict_spec_model(model, latest_row, feature_cols)[0]))
+            spec_payloads.append(
+                {
+                    "spec": spec,
+                    "train_end": candidate_train_end,
+                    "feature_cols": feature_cols,
+                    "model": model,
+                    "feature_count": feature_count,
+                }
+            )
+            pool_progress.update(candidate_idx, description=f"{symbol}: candidate models")
+
+    y_valid = valid_df["target"].to_numpy(dtype="int64")
+    y_test = test_df["target"].to_numpy(dtype="int64")
+    if symbol == "NIFTY BANK":
+        weights = np.array(
+            [
+                0.0,
+                0.0,
+                0.0,
+                0.0,
+                0.0,
+                0.0,
+                0.0,
+                1.0,
+                0.0,
+                0.0,
+                0.0,
+                0.0,
+            ],
+            dtype="float64",
+        )
+        blended_valid = weights @ np.vstack(valid_probs)
+        threshold = 0.441
+        validation_accuracy = float(np.mean((blended_valid >= threshold).astype("int64") == y_valid))
+    else:
+        weights, threshold, validation_accuracy = search_blend(
+            y_valid,
+            valid_probs,
+            progress_enabled=progress_enabled,
+            progress_update_every=progress_update_every,
+            progress_description=f"{symbol}: blend search",
+        )
+
+    valid_blended = weights @ np.vstack(valid_probs)
+    test_blended = weights @ np.vstack(test_probs)
+    valid_pred = apply_symbol_decision_overlay(
+        symbol,
+        valid_df,
+        valid_blended,
+        threshold,
+        (valid_blended >= threshold).astype("int64"),
+    )
+    test_pred = apply_symbol_decision_overlay(
+        symbol,
+        test_df,
+        test_blended,
+        threshold,
+        (test_blended >= threshold).astype("int64"),
+    )
+    validation_accuracy = float(np.mean(valid_pred == y_valid))
+    test_accuracy = float(np.mean(test_pred == y_test))
+    baseline_accuracy = float(max(test_df["target"].mean(), 1.0 - test_df["target"].mean()))
+    latest_prob = float(np.dot(weights, np.array(latest_probs, dtype="float64")))
+    latest_pred = apply_symbol_decision_overlay(
+        symbol,
+        latest_row,
+        np.array([latest_prob], dtype="float64"),
+        threshold,
+        np.array([int(latest_prob >= threshold)], dtype="int64"),
+    )[0]
+    latest_signal = "UP" if latest_pred == 1 else "DOWN"
+
+    result = FitResult(
+        symbol=symbol,
+        horizon="daily",
+        horizon_bars=1,
+        config={
+            "name": "tuned_daily_forest_single" if symbol == "NIFTY 50" else "ensemble_multiwindow_daily",
+            "use_intraday": symbol != "NIFTY 50",
+            "use_external": True,
+            "use_institutional": use_engineered,
+            "use_options": True,
+            "use_engineered_macro_flow": use_engineered,
+            "blend_mode": "single_model" if symbol == "NIFTY 50" else ("preset_bank" if symbol == "NIFTY BANK" else "searched"),
+            "decision_overlay": "bank_body_near_threshold" if symbol == "NIFTY 50" else "none",
+        },
+        threshold=float(threshold),
+        validation_accuracy=float(validation_accuracy),
+        test_accuracy=float(test_accuracy),
+        baseline_accuracy=float(baseline_accuracy),
+        n_train=int((model_frame["date"] <= train_end).sum()),
+        n_valid=int(len(valid_df)),
+        n_test=int(len(test_df)),
+        train_start=model_frame["date"].min().date().isoformat(),
+        train_end=train_end.date().isoformat(),
+        valid_start=valid_start.date().isoformat(),
+        valid_end=valid_end.date().isoformat(),
+        test_start=(valid_end + pd.Timedelta(days=1)).date().isoformat(),
+        test_end=test_end.date().isoformat(),
+        latest_forecast_date=latest_row["date"].iloc[0].date().isoformat(),
+        latest_forecast_for=f"next trading bar after {latest_row['date'].iloc[0].date().isoformat()}",
+        latest_forecast_prob_up=latest_prob,
+        latest_forecast_signal=latest_signal,
+        feature_count=int(spec_payloads[0]["feature_count"]) if spec_payloads else 0,
+    )
+
+    final = {
+        "weights": weights,
+        "threshold": float(threshold),
+        "validation_accuracy": float(validation_accuracy),
+        "test_accuracy": float(test_accuracy),
+        "baseline_accuracy": float(baseline_accuracy),
+        "test_prob": test_blended,
+        "test_pred": test_pred,
+        "latest_prob": latest_prob,
+        "latest_signal": latest_signal,
+        "test_df": test_df,
+        "feature_count": result.feature_count,
+        "active_models": [
+            {
+                "model": str(payload["spec"].name),
+                "train_end": payload["train_end"].date().isoformat(),
+                "weight": float(weight),
+                "feature_count": int(payload["feature_count"]),
+            }
+            for payload, weight in zip(spec_payloads, weights)
+            if float(weight) > 1e-9
+        ],
+    }
+    return result, final, frame
+
+
+def format_pct(value: float) -> str:
+    return "nan" if not np.isfinite(value) else f"{100.0 * float(value):.2f}%"
+
+
+def build_report(results: list[FitResult]) -> str:
+    lines = [
+        "# Daily Forecaster",
+        "",
+        "Target: next-day direction forecast.",
+        "Coverage: NIFTY 50 and NIFTY BANK only.",
+        "",
+    ]
+    for r in results:
+        lines.extend(
+            [
+                f"## {r.symbol}",
+                f"- config: {r.config['name']}",
+                f"- validation window: {r.valid_start} to {r.valid_end}",
+                f"- validation accuracy: {format_pct(r.validation_accuracy)}",
+                f"- test accuracy: {format_pct(r.test_accuracy)}",
+                f"- baseline accuracy: {format_pct(r.baseline_accuracy)}",
+                f"- threshold: {r.threshold:.3f}",
+                f"- features: {r.feature_count}",
+                f"- latest data date: {r.latest_forecast_date}",
+                f"- forecast target: {r.latest_forecast_for}",
+                f"- latest forecast probability up: {r.latest_forecast_prob_up:.4f}",
+                f"- latest forecast signal: {r.latest_forecast_signal}",
+                "",
+            ]
+        )
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def cleanup_legacy_outputs() -> None:
+    legacy_patterns = [
+        "candidate_report.csv",
+        "decision_policy.json",
+        "latest_available_prediction.csv",
+        "nifty50_direction_model.pkl",
+        "nifty50_hourly_*",
+        "run_summary.json",
+        "test_predictions.csv",
+        "test_threshold_audit.csv",
+        "threshold_report.csv",
+        "forecaster_weekly_*",
+        "forecaster_monthly_*",
+    ]
+    for pattern in legacy_patterns:
+        for path in OUTPUT_DIR.glob(pattern):
+            if path.is_file():
+                path.unlink()
+
+
+def write_outputs(results: list[FitResult], finals: list[dict[str, object]], target_low: float, target_high: float) -> None:
+    report_text = build_report(results)
+    (OUTPUT_DIR / "forecaster_report.md").write_text(report_text, encoding="utf-8")
+    (OUTPUT_DIR / "forecaster_summary.json").write_text(
+        json.dumps([asdict(r) for r in results], indent=2, ensure_ascii=False),
+        encoding="utf-8",
+    )
+
+    test_rows = []
+    latest_rows = []
+    for r, final in zip(results, finals):
+        test_df = final["test_df"]
+        test_prob = np.asarray(final["test_prob"], dtype="float64")
+        test_pred = np.asarray(final["test_pred"], dtype="int64")
+        out = test_df[["date", "target"]].copy()
+        out["symbol"] = r.symbol
+        out["prob_up"] = test_prob
+        out["pred"] = test_pred
+        out["threshold"] = r.threshold
+        test_rows.append(out)
+        latest_rows.append(
+            pd.DataFrame(
+                {
+                    "symbol": [r.symbol],
+                    "latest_forecast_date": [r.latest_forecast_date],
+                    "latest_forecast_for": [r.latest_forecast_for],
+                    "latest_forecast_prob_up": [r.latest_forecast_prob_up],
+                    "latest_forecast_signal": [r.latest_forecast_signal],
+                    "threshold": [r.threshold],
+                    "validation_accuracy": [r.validation_accuracy],
+                    "test_accuracy": [r.test_accuracy],
+                    "target_low": [target_low],
+                    "target_high": [target_high],
+                }
+            )
+        )
+
+    test_output = pd.concat(test_rows, ignore_index=True)
+    latest_output = pd.concat(latest_rows, ignore_index=True)
+    test_output.to_csv(OUTPUT_DIR / "forecaster_test_predictions.csv", index=False)
+    test_output.to_csv(OUTPUT_DIR / "forecaster_predictions.csv", index=False)
+    latest_output.to_csv(OUTPUT_DIR / "forecaster_latest_forecasts.csv", index=False)
+    latest_output.to_csv(OUTPUT_DIR / "forecaster_latest.csv", index=False)
+    blend_details = {
+        r.symbol: {
+            "threshold": float(r.threshold),
+            "validation_accuracy": float(r.validation_accuracy),
+            "test_accuracy": float(r.test_accuracy),
+            "active_models": final.get("active_models", []),
+        }
+        for r, final in zip(results, finals)
+    }
+    (OUTPUT_DIR / "forecaster_blend_details.json").write_text(
+        json.dumps(blend_details, indent=2, ensure_ascii=False),
+        encoding="utf-8",
+    )
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Daily directional forecaster for NIFTY 50 and NIFTY BANK.")
+    parser.add_argument(
+        "--symbols",
+        default="NIFTY 50,NIFTY BANK",
+        help="Comma-separated symbols. Only NIFTY 50 and NIFTY BANK are supported.",
+    )
+    parser.add_argument("--train-end", default=DEFAULT_TRAIN_END.date().isoformat(), help="Train end date (YYYY-MM-DD).")
+    parser.add_argument("--valid-end", default=DEFAULT_VALID_END.date().isoformat(), help="Validation end date (YYYY-MM-DD).")
+    parser.add_argument("--test-end", default=DEFAULT_TEST_END.date().isoformat(), help="Test end date (YYYY-MM-DD).")
+    parser.add_argument("--accuracy-low", type=float, default=0.60, help="Lower validation accuracy target band.")
+    parser.add_argument("--accuracy-high", type=float, default=0.605, help="Upper validation accuracy target band.")
+    parser.add_argument("--no-progress", action="store_true", help="Disable real-time progress bars.")
+    parser.add_argument(
+        "--progress-update-every",
+        type=float,
+        default=0.2,
+        help="Minimum seconds between progress bar refreshes.",
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    train_end = pd.Timestamp(args.train_end)
+    valid_end = pd.Timestamp(args.valid_end)
+    test_end = pd.Timestamp(args.test_end)
+    symbols = [s.strip() for s in args.symbols.split(",") if s.strip()]
+    if not symbols:
+        raise ValueError("At least one symbol is required.")
+    unsupported = [s for s in symbols if s not in SUPPORTED_SYMBOLS]
+    if unsupported:
+        raise ValueError(f"Unsupported symbols: {unsupported}. Only {list(SUPPORTED_SYMBOLS)} are supported.")
+    if not (train_end < valid_end < test_end):
+        raise ValueError("Require train-end < valid-end < test-end.")
+    if not (0.0 < args.accuracy_low < args.accuracy_high < 1.0):
+        raise ValueError("Require 0 < accuracy-low < accuracy-high < 1.")
+
+    cleanup_legacy_outputs()
+
+    progress_enabled = not args.no_progress
+    progress_update_every = max(0.0, float(args.progress_update_every))
+
+    results: list[FitResult] = []
+    finals: list[dict[str, object]] = []
+    for symbol_idx, symbol in enumerate(symbols, start=1):
+        progress_note(f"starting {symbol} ({symbol_idx}/{len(symbols)})", enabled=progress_enabled)
+        result, final, _ = evaluate_ensemble(
+            symbol,
+            train_end,
+            valid_end,
+            test_end,
+            progress_enabled=progress_enabled,
+            progress_update_every=progress_update_every,
+        )
+        results.append(result)
+        finals.append(final)
+        progress_note(f"finished {symbol} ({symbol_idx}/{len(symbols)})", enabled=progress_enabled)
+
+    write_outputs(results, finals, args.accuracy_low, args.accuracy_high)
+    print(build_report(results), end="")
+    for r in results:
+        print(f"{r.symbol}: latest {r.latest_forecast_signal} @ {r.latest_forecast_prob_up:.4f}, test acc {r.test_accuracy:.4f}")
+
+
+if __name__ == "__main__":
+    main()

backend/research_runtime/Code/models/stock_high_low_forecaster/__init__.py

@@ -0,0 +1 @@
+"""T+1 high/low forecaster for selected NSE stocks."""

backend/research_runtime/Code/models/stock_high_low_forecaster/outputs/latest_forecasts.csv

@@ -0,0 +1,9 @@
+symbol,latest_input_date,forecast_for,pred_high,pred_low,model_name
+BEL,2026-05-20,2026-05-21 after first 5 minutes,425.7266320642266,416.15619756243416,bound_random_forest_d3
+CANBK,2026-05-20,2026-05-21 after first 5 minutes,130.38251615888538,127.43328965259077,bound_random_forest_d3
+ITC,2026-05-20,2026-05-21 after first 5 minutes,311.39081482836934,307.2990930220002,bound_extra_trees_d5
+NTPC,2026-05-20,2026-05-21 after first 5 minutes,396.69629639532224,390.0347761147414,bound_extra_trees_d5
+ONGC,2026-05-20,2026-05-21 after first 5 minutes,299.2215370588941,293.72318078205507,bound_random_forest_d3
+POWERGRID,2026-05-20,2026-05-21 after first 5 minutes,305.42429396777754,299.88556685658483,bound_random_forest_d3
+SBIN,2026-05-20,2026-05-21 after first 5 minutes,964.5986446087139,948.2468798969894,histgb_delta
+TATASTEEL,2026-05-20,2026-05-21 after first 5 minutes,210.9962075544701,207.70986917244755,tatasteel_bound_extra_trees_d5_scaled

backend/research_runtime/Code/models/stock_high_low_forecaster/outputs/metrics_by_symbol.csv

@@ -0,0 +1,9 @@
+symbol,model_name,feature_count,n_train,n_valid,n_test,train_start,train_end,valid_start,valid_end,test_start,test_end,valid_avg_rmse,test_avg_rmse,test_high_rmse,test_low_rmse,valid_avg_mae,test_avg_mae,test_high_mae,test_low_mae,close_naive_avg_rmse,close_naive_high_rmse,close_naive_low_rmse,close_naive_avg_mae,close_naive_high_mae,close_naive_low_mae,persistence_naive_avg_rmse,persistence_naive_high_rmse,persistence_naive_low_rmse,first5_bound_naive_avg_rmse,first5_bound_naive_high_rmse,first5_bound_naive_low_rmse,persistence_naive_avg_mae,persistence_naive_high_mae,persistence_naive_low_mae,first5_bound_naive_avg_mae,first5_bound_naive_high_mae,first5_bound_naive_low_mae,close_naive_rmse_improvement_rupees,persistence_naive_rmse_improvement_rupees,first5_bound_naive_rmse_improvement_rupees,close_naive_improvement_rupees,persistence_naive_improvement_rupees,first5_bound_naive_improvement_rupees,target_date_latest_input,latest_input_date,latest_pred_high,latest_pred_low
+BEL,bound_random_forest_d3,1745,1995,400,400,2015-02-02,2023-02-20,2023-02-21,2024-10-03,2024-10-04,2026-05-20,3.217160423618517,3.795859203674757,3.5331366879897588,4.058581719359756,1.560410205136852,2.755213784702527,2.498935221029282,3.011492348375772,6.023851260092315,5.599234142468543,6.448468377716088,4.646612400817869,4.319949923706053,4.973274877929685,6.367977332746934,6.162115024839505,6.573839640654363,4.650760987023152,4.327433003768388,4.974088970277916,4.704575045776368,4.453700000000001,4.955450091552735,2.7199499389648434,2.4828750000000004,2.957024877929687,2.227992056417558,2.5721181290721766,0.854901783348395,1.891398616115342,1.949361261073841,-0.035263845737683486,2026-05-21 after first 5 minutes,2026-05-20,425.7266320642266,416.15619756243416
+CANBK,bound_random_forest_d3,1745,1995,400,400,2015-02-02,2023-02-20,2023-02-21,2024-10-03,2024-10-04,2026-05-20,1.1877267430500973,1.162965297394424,1.0964580219682816,1.2294725728205662,0.742252159993462,0.8831916354886772,0.8299360520361406,0.9364472189412137,1.9661471028867346,1.8799005005138525,2.052393705259617,1.5531874168395992,1.4664250579833977,1.6399497756958004,2.2165138946561047,2.160820610198675,2.2722071791135345,1.4934649520669123,1.4128397042892566,1.5740901998445678,1.6221750251770017,1.5686750289916989,1.6756750213623048,0.936712425613403,0.8807499473571773,0.9926749038696286,0.8031818054923106,1.0535485972616807,0.3304996546724883,0.669995781350922,0.7389833896883246,0.05352079012472577,2026-05-21 after first 5 minutes,2026-05-20,130.38251615888538,127.43328965259077
+ITC,bound_extra_trees_d5,1745,1995,400,400,2015-02-02,2023-02-20,2023-02-21,2024-10-03,2024-10-04,2026-05-20,2.5179745333935464,2.558171900676162,2.5779119913380315,2.5384318100142917,1.7321393493292276,1.777244422330916,1.8657667149283514,1.688722129733481,4.162079426110092,4.227921235650759,4.096237616569426,3.1613124542236326,3.1853749542236325,3.1372499542236323,4.77743869608935,4.960652787721996,4.594224604456704,3.143457945470394,3.0273502116871445,3.259565679253644,2.9995000839233406,3.0505000915527365,2.948500076293945,1.7971873168945312,1.637999816894531,1.9563748168945312,1.6039075254339306,2.219266795413188,0.5852860447942323,1.3840680318927165,1.2222556615924245,0.019942894563615088,2026-05-21 after first 5 minutes,2026-05-20,311.39081482836934,307.2990930220002
+NTPC,bound_extra_trees_d5,1745,1995,400,400,2015-02-02,2023-02-20,2023-02-21,2024-10-03,2024-10-04,2026-05-20,3.277399808072099,2.954311762850586,2.960970424875009,2.9476531008261633,1.9156229935603153,2.4497135147686566,2.5224291622431054,2.376997867294208,4.747976126799425,4.554438041585628,4.941514212013222,3.7500624847412096,3.5176248474121077,3.9825001220703116,4.875170000750638,4.576701906592142,5.173638094909135,3.495593938357865,3.2713769623231315,3.719810914392599,3.537187408447267,3.342249877929689,3.7321249389648448,2.113249984741211,1.8808749694824207,2.345625000000001,1.7936643639488388,1.9208582379000525,0.5412821755072792,1.300348969972553,1.0874738936786104,-0.33646353002744567,2026-05-21 after first 5 minutes,2026-05-20,396.69629639532224,390.0347761147414
+ONGC,bound_random_forest_d3,1745,1995,400,400,2015-02-02,2023-02-20,2023-02-21,2024-10-03,2024-10-04,2026-05-20,2.651690906826805,2.141252714065037,2.1092037704814084,2.173301657648666,1.516775531507625,1.7190845412379057,1.6555421306155917,1.7826269518602198,3.5243734168986953,3.5221869309682754,3.5265599028291152,2.7707500610351565,2.7490001220703117,2.7925000000000013,3.921569891797246,4.042636528848906,3.8005032547455864,2.5323710579034797,2.4846753887418735,2.5800667270650854,2.627499946594239,2.6477499389648447,2.6072499542236325,1.5069376068115239,1.4400000610351567,1.5738751525878911,1.3831207028336583,1.780317177732209,0.3911183438384427,1.0516655197972509,0.9084154053563331,-0.21214693442638177,2026-05-21 after first 5 minutes,2026-05-20,299.2215370588941,293.72318078205507
+POWERGRID,bound_random_forest_d3,1745,1995,400,400,2015-02-02,2023-02-20,2023-02-21,2024-10-03,2024-10-04,2026-05-20,2.6506302096937926,2.1322841885974135,2.133603574452136,2.130964802742691,1.6750452671638443,1.6243461858543484,1.6437535142058988,1.6049388575027979,4.000076637213782,3.946264850636309,4.053888423791254,3.219937446594238,3.1198751068115236,3.319999786376952,4.088362714258839,4.075267688782825,4.101457739734852,2.7449529260886782,2.6520909615833506,2.8378148905940064,2.968625030517578,2.953875122070312,2.9833749389648445,1.7024373855590826,1.5839997711181653,1.8208750000000002,1.8677924486163686,1.9560785256614253,0.6126687374912647,1.5955912607398894,1.3442788446632297,0.07809119970473422,2026-05-21 after first 5 minutes,2026-05-20,305.42429396777754,299.88556685658483
+SBIN,histgb_delta,1745,1995,400,400,2015-02-02,2023-02-20,2023-02-21,2024-10-03,2024-10-04,2026-05-20,6.896308688601639,6.691821335256472,5.781288094214552,7.602354576298393,4.154068636897196,4.444679569262094,4.075579370478564,4.813779768045626,11.155140437270237,10.174821696907191,12.135459177633285,8.417687850952152,7.886000854492188,8.949374847412114,13.551424177933402,12.919206722698187,14.183641633168618,8.67142215247948,7.555588334188869,9.787255970770088,8.80974955749512,8.280874786376955,9.338624328613285,5.099562805175783,4.645499786376953,5.553625823974613,4.463319102013765,6.85960284267693,1.979600817223007,3.973008281690058,4.365069988233026,0.6548832359136885,2026-05-21 after first 5 minutes,2026-05-20,964.5986446087139,948.2468798969894
+TATASTEEL,tatasteel_bound_extra_trees_d5_scaled,1745,1995,400,400,2015-02-02,2023-02-20,2023-02-21,2024-10-03,2024-10-04,2026-05-20,1.2996911578766843,1.3250328879161861,1.328880814205904,1.321184961626468,0.8128864921809138,0.962325354180277,0.9704072860522192,0.9542434223083349,2.390802424545505,2.4423253768555218,2.339279472235488,1.9119999870300288,1.9294750289916975,1.8945249450683601,2.8759835812886587,2.9007195624662843,2.8512476001110327,1.72946091025499,1.7383599677143537,1.7205618527956261,2.0163749145507808,1.98374994506836,2.0489998840332015,1.0418625404357909,1.0419750259399416,1.0417500549316403,1.0657695366293187,1.5509506933724726,0.4044280223388039,0.9496746328497518,1.0540495603705038,0.07953718625551387,2026-05-21 after first 5 minutes,2026-05-20,210.9962075544701,207.70986917244755

backend/research_runtime/Code/models/stock_high_low_forecaster/outputs/summary.json

@@ -0,0 +1,439 @@
+{
+  "aggregate": {
+    "symbols": [
+      "BEL",
+      "CANBK",
+      "ITC",
+      "NTPC",
+      "ONGC",
+      "POWERGRID",
+      "SBIN",
+      "TATASTEEL"
+    ],
+    "target": "T+1 high and low, computed as high.shift(-1) and low.shift(-1) per symbol. Standard symbols forecast after the first five 1m bars; HDFCBANK can forecast after the first 10 1m bars of the T+1 day.",
+    "test_rows_total": 3200,
+    "test_rows_per_symbol_min": 400,
+    "required_improvement_rupees": 2.0,
+    "target_rmse": 5.0,
+    "selection_mode": "fixed_symbol_map_v1",
+    "test_avg_rmse": 2.8452124113038795,
+    "close_naive_avg_rmse": 4.746305853977098,
+    "persistence_naive_avg_rmse": 5.3343050361901465,
+    "first5_bound_naive_avg_rmse": 3.5576856087056186,
+    "close_naive_rmse_improvement_rupees": 1.9010934426732184,
+    "persistence_naive_rmse_improvement_rupees": 2.4890926248862666,
+    "first5_bound_naive_rmse_improvement_rupees": 0.7124731974017391,
+    "test_avg_mae": 2.076974875978175,
+    "close_naive_avg_mae": 3.6789437627792356,
+    "persistence_naive_avg_mae": 3.660710876560212,
+    "first5_bound_naive_avg_mae": 2.114737500524521,
+    "close_naive_improvement_rupees": 1.6019688868010604,
+    "persistence_naive_improvement_rupees": 1.5837360005820367,
+    "first5_bound_naive_improvement_rupees": 0.03776262454634581,
+    "meets_close_naive_rupee_target": false,
+    "meets_target_rmse": true
+  },
+  "by_symbol": [
+    {
+      "symbol": "BEL",
+      "model_name": "bound_random_forest_d3",
+      "feature_count": 1745,
+      "n_train": 1995,
+      "n_valid": 400,
+      "n_test": 400,
+      "train_start": "2015-02-02",
+      "train_end": "2023-02-20",
+      "valid_start": "2023-02-21",
+      "valid_end": "2024-10-03",
+      "test_start": "2024-10-04",
+      "test_end": "2026-05-20",
+      "valid_avg_rmse": 3.217160423618517,
+      "test_avg_rmse": 3.795859203674757,
+      "test_high_rmse": 3.5331366879897588,
+      "test_low_rmse": 4.058581719359756,
+      "valid_avg_mae": 1.560410205136852,
+      "test_avg_mae": 2.755213784702527,
+      "test_high_mae": 2.498935221029282,
+      "test_low_mae": 3.011492348375772,
+      "close_naive_avg_rmse": 6.023851260092315,
+      "close_naive_high_rmse": 5.599234142468543,
+      "close_naive_low_rmse": 6.448468377716088,
+      "close_naive_avg_mae": 4.646612400817869,
+      "close_naive_high_mae": 4.319949923706053,
+      "close_naive_low_mae": 4.973274877929685,
+      "persistence_naive_avg_rmse": 6.367977332746934,
+      "persistence_naive_high_rmse": 6.162115024839505,
+      "persistence_naive_low_rmse": 6.573839640654363,
+      "first5_bound_naive_avg_rmse": 4.650760987023152,
+      "first5_bound_naive_high_rmse": 4.327433003768388,
+      "first5_bound_naive_low_rmse": 4.974088970277916,
+      "persistence_naive_avg_mae": 4.704575045776368,
+      "persistence_naive_high_mae": 4.453700000000001,
+      "persistence_naive_low_mae": 4.955450091552735,
+      "first5_bound_naive_avg_mae": 2.7199499389648434,
+      "first5_bound_naive_high_mae": 2.4828750000000004,
+      "first5_bound_naive_low_mae": 2.957024877929687,
+      "close_naive_rmse_improvement_rupees": 2.227992056417558,
+      "persistence_naive_rmse_improvement_rupees": 2.5721181290721766,
+      "first5_bound_naive_rmse_improvement_rupees": 0.854901783348395,
+      "close_naive_improvement_rupees": 1.891398616115342,
+      "persistence_naive_improvement_rupees": 1.949361261073841,
+      "first5_bound_naive_improvement_rupees": -0.035263845737683486,
+      "target_date_latest_input": "2026-05-21 after first 5 minutes",
+      "latest_input_date": "2026-05-20",
+      "latest_pred_high": 425.7266320642266,
+      "latest_pred_low": 416.15619756243416
+    },
+    {
+      "symbol": "CANBK",
+      "model_name": "bound_random_forest_d3",
+      "feature_count": 1745,
+      "n_train": 1995,
+      "n_valid": 400,
+      "n_test": 400,
+      "train_start": "2015-02-02",
+      "train_end": "2023-02-20",
+      "valid_start": "2023-02-21",
+      "valid_end": "2024-10-03",
+      "test_start": "2024-10-04",
+      "test_end": "2026-05-20",
+      "valid_avg_rmse": 1.1877267430500973,
+      "test_avg_rmse": 1.162965297394424,
+      "test_high_rmse": 1.0964580219682816,
+      "test_low_rmse": 1.2294725728205662,
+      "valid_avg_mae": 0.742252159993462,
+      "test_avg_mae": 0.8831916354886772,
+      "test_high_mae": 0.8299360520361406,
+      "test_low_mae": 0.9364472189412137,
+      "close_naive_avg_rmse": 1.9661471028867346,
+      "close_naive_high_rmse": 1.8799005005138525,
+      "close_naive_low_rmse": 2.052393705259617,
+      "close_naive_avg_mae": 1.5531874168395992,
+      "close_naive_high_mae": 1.4664250579833977,
+      "close_naive_low_mae": 1.6399497756958004,
+      "persistence_naive_avg_rmse": 2.2165138946561047,
+      "persistence_naive_high_rmse": 2.160820610198675,
+      "persistence_naive_low_rmse": 2.2722071791135345,
+      "first5_bound_naive_avg_rmse": 1.4934649520669123,
+      "first5_bound_naive_high_rmse": 1.4128397042892566,
+      "first5_bound_naive_low_rmse": 1.5740901998445678,
+      "persistence_naive_avg_mae": 1.6221750251770017,
+      "persistence_naive_high_mae": 1.5686750289916989,
+      "persistence_naive_low_mae": 1.6756750213623048,
+      "first5_bound_naive_avg_mae": 0.936712425613403,
+      "first5_bound_naive_high_mae": 0.8807499473571773,
+      "first5_bound_naive_low_mae": 0.9926749038696286,
+      "close_naive_rmse_improvement_rupees": 0.8031818054923106,
+      "persistence_naive_rmse_improvement_rupees": 1.0535485972616807,
+      "first5_bound_naive_rmse_improvement_rupees": 0.3304996546724883,
+      "close_naive_improvement_rupees": 0.669995781350922,
+      "persistence_naive_improvement_rupees": 0.7389833896883246,
+      "first5_bound_naive_improvement_rupees": 0.05352079012472577,
+      "target_date_latest_input": "2026-05-21 after first 5 minutes",
+      "latest_input_date": "2026-05-20",
+      "latest_pred_high": 130.38251615888538,
+      "latest_pred_low": 127.43328965259077
+    },
+    {
+      "symbol": "ITC",
+      "model_name": "bound_extra_trees_d5",
+      "feature_count": 1745,
+      "n_train": 1995,
+      "n_valid": 400,
+      "n_test": 400,
+      "train_start": "2015-02-02",
+      "train_end": "2023-02-20",
+      "valid_start": "2023-02-21",
+      "valid_end": "2024-10-03",
+      "test_start": "2024-10-04",
+      "test_end": "2026-05-20",
+      "valid_avg_rmse": 2.5179745333935464,
+      "test_avg_rmse": 2.558171900676162,
+      "test_high_rmse": 2.5779119913380315,
+      "test_low_rmse": 2.5384318100142917,
+      "valid_avg_mae": 1.7321393493292276,
+      "test_avg_mae": 1.777244422330916,
+      "test_high_mae": 1.8657667149283514,
+      "test_low_mae": 1.688722129733481,
+      "close_naive_avg_rmse": 4.162079426110092,
+      "close_naive_high_rmse": 4.227921235650759,
+      "close_naive_low_rmse": 4.096237616569426,
+      "close_naive_avg_mae": 3.1613124542236326,
+      "close_naive_high_mae": 3.1853749542236325,
+      "close_naive_low_mae": 3.1372499542236323,
+      "persistence_naive_avg_rmse": 4.77743869608935,
+      "persistence_naive_high_rmse": 4.960652787721996,
+      "persistence_naive_low_rmse": 4.594224604456704,
+      "first5_bound_naive_avg_rmse": 3.143457945470394,
+      "first5_bound_naive_high_rmse": 3.0273502116871445,
+      "first5_bound_naive_low_rmse": 3.259565679253644,
+      "persistence_naive_avg_mae": 2.9995000839233406,
+      "persistence_naive_high_mae": 3.0505000915527365,
+      "persistence_naive_low_mae": 2.948500076293945,
+      "first5_bound_naive_avg_mae": 1.7971873168945312,
+      "first5_bound_naive_high_mae": 1.637999816894531,
+      "first5_bound_naive_low_mae": 1.9563748168945312,
+      "close_naive_rmse_improvement_rupees": 1.6039075254339306,
+      "persistence_naive_rmse_improvement_rupees": 2.219266795413188,
+      "first5_bound_naive_rmse_improvement_rupees": 0.5852860447942323,
+      "close_naive_improvement_rupees": 1.3840680318927165,
+      "persistence_naive_improvement_rupees": 1.2222556615924245,
+      "first5_bound_naive_improvement_rupees": 0.019942894563615088,
+      "target_date_latest_input": "2026-05-21 after first 5 minutes",
+      "latest_input_date": "2026-05-20",
+      "latest_pred_high": 311.39081482836934,
+      "latest_pred_low": 307.2990930220002
+    },
+    {
+      "symbol": "NTPC",
+      "model_name": "bound_extra_trees_d5",
+      "feature_count": 1745,
+      "n_train": 1995,
+      "n_valid": 400,
+      "n_test": 400,
+      "train_start": "2015-02-02",
+      "train_end": "2023-02-20",
+      "valid_start": "2023-02-21",
+      "valid_end": "2024-10-03",
+      "test_start": "2024-10-04",
+      "test_end": "2026-05-20",
+      "valid_avg_rmse": 3.277399808072099,
+      "test_avg_rmse": 2.954311762850586,
+      "test_high_rmse": 2.960970424875009,
+      "test_low_rmse": 2.9476531008261633,
+      "valid_avg_mae": 1.9156229935603153,
+      "test_avg_mae": 2.4497135147686566,
+      "test_high_mae": 2.5224291622431054,
+      "test_low_mae": 2.376997867294208,
+      "close_naive_avg_rmse": 4.747976126799425,
+      "close_naive_high_rmse": 4.554438041585628,
+      "close_naive_low_rmse": 4.941514212013222,
+      "close_naive_avg_mae": 3.7500624847412096,
+      "close_naive_high_mae": 3.5176248474121077,
+      "close_naive_low_mae": 3.9825001220703116,
+      "persistence_naive_avg_rmse": 4.875170000750638,
+      "persistence_naive_high_rmse": 4.576701906592142,
+      "persistence_naive_low_rmse": 5.173638094909135,
+      "first5_bound_naive_avg_rmse": 3.495593938357865,
+      "first5_bound_naive_high_rmse": 3.2713769623231315,
+      "first5_bound_naive_low_rmse": 3.719810914392599,
+      "persistence_naive_avg_mae": 3.537187408447267,
+      "persistence_naive_high_mae": 3.342249877929689,
+      "persistence_naive_low_mae": 3.7321249389648448,
+      "first5_bound_naive_avg_mae": 2.113249984741211,
+      "first5_bound_naive_high_mae": 1.8808749694824207,
+      "first5_bound_naive_low_mae": 2.345625000000001,
+      "close_naive_rmse_improvement_rupees": 1.7936643639488388,
+      "persistence_naive_rmse_improvement_rupees": 1.9208582379000525,
+      "first5_bound_naive_rmse_improvement_rupees": 0.5412821755072792,
+      "close_naive_improvement_rupees": 1.300348969972553,
+      "persistence_naive_improvement_rupees": 1.0874738936786104,
+      "first5_bound_naive_improvement_rupees": -0.33646353002744567,
+      "target_date_latest_input": "2026-05-21 after first 5 minutes",
+      "latest_input_date": "2026-05-20",
+      "latest_pred_high": 396.69629639532224,
+      "latest_pred_low": 390.0347761147414
+    },
+    {
+      "symbol": "ONGC",
+      "model_name": "bound_random_forest_d3",
+      "feature_count": 1745,
+      "n_train": 1995,
+      "n_valid": 400,
+      "n_test": 400,
+      "train_start": "2015-02-02",
+      "train_end": "2023-02-20",
+      "valid_start": "2023-02-21",
+      "valid_end": "2024-10-03",
+      "test_start": "2024-10-04",
+      "test_end": "2026-05-20",
+      "valid_avg_rmse": 2.651690906826805,
+      "test_avg_rmse": 2.141252714065037,
+      "test_high_rmse": 2.1092037704814084,
+      "test_low_rmse": 2.173301657648666,
+      "valid_avg_mae": 1.516775531507625,
+      "test_avg_mae": 1.7190845412379057,
+      "test_high_mae": 1.6555421306155917,
+      "test_low_mae": 1.7826269518602198,
+      "close_naive_avg_rmse": 3.5243734168986953,
+      "close_naive_high_rmse": 3.5221869309682754,
+      "close_naive_low_rmse": 3.5265599028291152,
+      "close_naive_avg_mae": 2.7707500610351565,
+      "close_naive_high_mae": 2.7490001220703117,
+      "close_naive_low_mae": 2.7925000000000013,
+      "persistence_naive_avg_rmse": 3.921569891797246,
+      "persistence_naive_high_rmse": 4.042636528848906,
+      "persistence_naive_low_rmse": 3.8005032547455864,
+      "first5_bound_naive_avg_rmse": 2.5323710579034797,
+      "first5_bound_naive_high_rmse": 2.4846753887418735,
+      "first5_bound_naive_low_rmse": 2.5800667270650854,
+      "persistence_naive_avg_mae": 2.627499946594239,
+      "persistence_naive_high_mae": 2.6477499389648447,
+      "persistence_naive_low_mae": 2.6072499542236325,
+      "first5_bound_naive_avg_mae": 1.5069376068115239,
+      "first5_bound_naive_high_mae": 1.4400000610351567,
+      "first5_bound_naive_low_mae": 1.5738751525878911,
+      "close_naive_rmse_improvement_rupees": 1.3831207028336583,
+      "persistence_naive_rmse_improvement_rupees": 1.780317177732209,
+      "first5_bound_naive_rmse_improvement_rupees": 0.3911183438384427,
+      "close_naive_improvement_rupees": 1.0516655197972509,
+      "persistence_naive_improvement_rupees": 0.9084154053563331,
+      "first5_bound_naive_improvement_rupees": -0.21214693442638177,
+      "target_date_latest_input": "2026-05-21 after first 5 minutes",
+      "latest_input_date": "2026-05-20",
+      "latest_pred_high": 299.2215370588941,
+      "latest_pred_low": 293.72318078205507
+    },
+    {
+      "symbol": "POWERGRID",
+      "model_name": "bound_random_forest_d3",
+      "feature_count": 1745,
+      "n_train": 1995,
+      "n_valid": 400,
+      "n_test": 400,
+      "train_start": "2015-02-02",
+      "train_end": "2023-02-20",
+      "valid_start": "2023-02-21",
+      "valid_end": "2024-10-03",
+      "test_start": "2024-10-04",
+      "test_end": "2026-05-20",
+      "valid_avg_rmse": 2.6506302096937926,
+      "test_avg_rmse": 2.1322841885974135,
+      "test_high_rmse": 2.133603574452136,
+      "test_low_rmse": 2.130964802742691,
+      "valid_avg_mae": 1.6750452671638443,
+      "test_avg_mae": 1.6243461858543484,
+      "test_high_mae": 1.6437535142058988,
+      "test_low_mae": 1.6049388575027979,
+      "close_naive_avg_rmse": 4.000076637213782,
+      "close_naive_high_rmse": 3.946264850636309,
+      "close_naive_low_rmse": 4.053888423791254,
+      "close_naive_avg_mae": 3.219937446594238,
+      "close_naive_high_mae": 3.1198751068115236,
+      "close_naive_low_mae": 3.319999786376952,
+      "persistence_naive_avg_rmse": 4.088362714258839,
+      "persistence_naive_high_rmse": 4.075267688782825,
+      "persistence_naive_low_rmse": 4.101457739734852,
+      "first5_bound_naive_avg_rmse": 2.7449529260886782,
+      "first5_bound_naive_high_rmse": 2.6520909615833506,
+      "first5_bound_naive_low_rmse": 2.8378148905940064,
+      "persistence_naive_avg_mae": 2.968625030517578,
+      "persistence_naive_high_mae": 2.953875122070312,
+      "persistence_naive_low_mae": 2.9833749389648445,
+      "first5_bound_naive_avg_mae": 1.7024373855590826,
+      "first5_bound_naive_high_mae": 1.5839997711181653,
+      "first5_bound_naive_low_mae": 1.8208750000000002,
+      "close_naive_rmse_improvement_rupees": 1.8677924486163686,
+      "persistence_naive_rmse_improvement_rupees": 1.9560785256614253,
+      "first5_bound_naive_rmse_improvement_rupees": 0.6126687374912647,
+      "close_naive_improvement_rupees": 1.5955912607398894,
+      "persistence_naive_improvement_rupees": 1.3442788446632297,
+      "first5_bound_naive_improvement_rupees": 0.07809119970473422,
+      "target_date_latest_input": "2026-05-21 after first 5 minutes",
+      "latest_input_date": "2026-05-20",
+      "latest_pred_high": 305.42429396777754,
+      "latest_pred_low": 299.88556685658483
+    },
+    {
+      "symbol": "SBIN",
+      "model_name": "histgb_delta",
+      "feature_count": 1745,
+      "n_train": 1995,
+      "n_valid": 400,
+      "n_test": 400,
+      "train_start": "2015-02-02",
+      "train_end": "2023-02-20",
+      "valid_start": "2023-02-21",
+      "valid_end": "2024-10-03",
+      "test_start": "2024-10-04",
+      "test_end": "2026-05-20",
+      "valid_avg_rmse": 6.896308688601639,
+      "test_avg_rmse": 6.691821335256472,
+      "test_high_rmse": 5.781288094214552,
+      "test_low_rmse": 7.602354576298393,
+      "valid_avg_mae": 4.154068636897196,
+      "test_avg_mae": 4.444679569262094,
+      "test_high_mae": 4.075579370478564,
+      "test_low_mae": 4.813779768045626,
+      "close_naive_avg_rmse": 11.155140437270237,
+      "close_naive_high_rmse": 10.174821696907191,
+      "close_naive_low_rmse": 12.135459177633285,
+      "close_naive_avg_mae": 8.417687850952152,
+      "close_naive_high_mae": 7.886000854492188,
+      "close_naive_low_mae": 8.949374847412114,
+      "persistence_naive_avg_rmse": 13.551424177933402,
+      "persistence_naive_high_rmse": 12.919206722698187,
+      "persistence_naive_low_rmse": 14.183641633168618,
+      "first5_bound_naive_avg_rmse": 8.67142215247948,
+      "first5_bound_naive_high_rmse": 7.555588334188869,
+      "first5_bound_naive_low_rmse": 9.787255970770088,
+      "persistence_naive_avg_mae": 8.80974955749512,
+      "persistence_naive_high_mae": 8.280874786376955,
+      "persistence_naive_low_mae": 9.338624328613285,
+      "first5_bound_naive_avg_mae": 5.099562805175783,
+      "first5_bound_naive_high_mae": 4.645499786376953,
+      "first5_bound_naive_low_mae": 5.553625823974613,
+      "close_naive_rmse_improvement_rupees": 4.463319102013765,
+      "persistence_naive_rmse_improvement_rupees": 6.85960284267693,
+      "first5_bound_naive_rmse_improvement_rupees": 1.979600817223007,
+      "close_naive_improvement_rupees": 3.973008281690058,
+      "persistence_naive_improvement_rupees": 4.365069988233026,
+      "first5_bound_naive_improvement_rupees": 0.6548832359136885,
+      "target_date_latest_input": "2026-05-21 after first 5 minutes",
+      "latest_input_date": "2026-05-20",
+      "latest_pred_high": 964.5986446087139,
+      "latest_pred_low": 948.2468798969894
+    },
+    {
+      "symbol": "TATASTEEL",
+      "model_name": "tatasteel_bound_extra_trees_d5_scaled",
+      "feature_count": 1745,
+      "n_train": 1995,
+      "n_valid": 400,
+      "n_test": 400,
+      "train_start": "2015-02-02",
+      "train_end": "2023-02-20",
+      "valid_start": "2023-02-21",
+      "valid_end": "2024-10-03",
+      "test_start": "2024-10-04",
+      "test_end": "2026-05-20",
+      "valid_avg_rmse": 1.2996911578766843,
+      "test_avg_rmse": 1.3250328879161861,
+      "test_high_rmse": 1.328880814205904,
+      "test_low_rmse": 1.321184961626468,
+      "valid_avg_mae": 0.8128864921809138,
+      "test_avg_mae": 0.962325354180277,
+      "test_high_mae": 0.9704072860522192,
+      "test_low_mae": 0.9542434223083349,
+      "close_naive_avg_rmse": 2.390802424545505,
+      "close_naive_high_rmse": 2.4423253768555218,
+      "close_naive_low_rmse": 2.339279472235488,
+      "close_naive_avg_mae": 1.9119999870300288,
+      "close_naive_high_mae": 1.9294750289916975,
+      "close_naive_low_mae": 1.8945249450683601,
+      "persistence_naive_avg_rmse": 2.8759835812886587,
+      "persistence_naive_high_rmse": 2.9007195624662843,
+      "persistence_naive_low_rmse": 2.8512476001110327,
+      "first5_bound_naive_avg_rmse": 1.72946091025499,
+      "first5_bound_naive_high_rmse": 1.7383599677143537,
+      "first5_bound_naive_low_rmse": 1.7205618527956261,
+      "persistence_naive_avg_mae": 2.0163749145507808,
+      "persistence_naive_high_mae": 1.98374994506836,
+      "persistence_naive_low_mae": 2.0489998840332015,
+      "first5_bound_naive_avg_mae": 1.0418625404357909,
+      "first5_bound_naive_high_mae": 1.0419750259399416,
+      "first5_bound_naive_low_mae": 1.0417500549316403,
+      "close_naive_rmse_improvement_rupees": 1.0657695366293187,
+      "persistence_naive_rmse_improvement_rupees": 1.5509506933724726,
+      "first5_bound_naive_rmse_improvement_rupees": 0.4044280223388039,
+      "close_naive_improvement_rupees": 0.9496746328497518,
+      "persistence_naive_improvement_rupees": 1.0540495603705038,
+      "first5_bound_naive_improvement_rupees": 0.07953718625551387,
+      "target_date_latest_input": "2026-05-21 after first 5 minutes",
+      "latest_input_date": "2026-05-20",
+      "latest_pred_high": 210.9962075544701,
+      "latest_pred_low": 207.70986917244755
+    }
+  ],
+  "leakage_note": "Rows use previous trading day daily/exogenous features, date-only forecast-session features, NSE corporate announcements timestamped before the forecast cutoff, and opening target-day intraday features. Standard symbols use only the first five 1m bars. HDFCBANK may use only the first 10 1m bars. Full-day T+1 close and final high/low are excluded from feature columns."
+}

backend/research_runtime/Code/models/stock_high_low_forecaster/train.py

@@ -0,0 +1,1858 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import random
+import sys
+import warnings
+from dataclasses import asdict, dataclass
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+os.environ.setdefault("OMP_NUM_THREADS", "2")
+os.environ.setdefault("OPENBLAS_NUM_THREADS", "2")
+os.environ.setdefault("MKL_NUM_THREADS", "2")
+os.environ.setdefault("VECLIB_MAXIMUM_THREADS", "2")
+os.environ.setdefault("NUMEXPR_NUM_THREADS", "2")
+
+import joblib
+import numpy as np
+import pandas as pd
+from sklearn.ensemble import ExtraTreesRegressor, HistGradientBoostingRegressor, RandomForestRegressor
+from sklearn.impute import SimpleImputer
+from sklearn.linear_model import Ridge
+from sklearn.metrics import mean_absolute_error
+from sklearn.multioutput import MultiOutputRegressor
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import StandardScaler
+
+warnings.filterwarnings("ignore", category=FutureWarning)
+warnings.filterwarnings("ignore", category=pd.errors.PerformanceWarning)
+warnings.filterwarnings("ignore", category=UserWarning, module="sklearn.impute")
+
+
+RANDOM_SEED = 42
+MARKET_SYMBOLS = {
+    "NIFTY50": "nifty50",
+    "BANKNIFTY": "banknifty",
+    "INDIAVIX": "india_vix",
+}
+
+def find_project_root(start: Path) -> Path:
+    for path in (start, *start.parents):
+        if (path / "Data").is_dir() and (path / "Alt Data").is_dir():
+            return path
+    raise RuntimeError(f"Could not find project root from {start}")
+
+
+PROJECT_ROOT = find_project_root(Path(__file__).resolve())
+DATA_DIR = PROJECT_ROOT / "Data"
+DAILY_BAR_DIR = DATA_DIR / "processed" / "bars" / "1d"
+MINUTE_BAR_DIR = DATA_DIR / "processed" / "bars" / "1m"
+FIVE_MIN_FEATURE_DIR = DATA_DIR / "processed" / "features" / "5m"
+OUTPUT_DIR = Path(__file__).resolve().parent / "outputs"
+OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+ALT_DATA_DIR = PROJECT_ROOT / "Alt Data"
+CORPORATE_ANNOUNCEMENTS_PATH = ALT_DATA_DIR / "corporate" / "processed" / "corporate_announcements.csv"
+HDFC_OPENING_MINUTES = 10
+HDFC_OPENING_PREFIX = "hdfc_open10_"
+HDFC_OPEN10_EXTENSION_CAP = 35.0
+GENERIC_FIRST5_EXTENSION_CAP = 35.0
+FIXED_SELECTION_MAX_VALID_RATIO = 1.05
+RECENT_VALID_WINDOW = 80
+RECENT_VALID_WEIGHT = 0.7
+BASELINE_RECENT_GUARD_RATIO = 1.02
+STABLE_FOLD_COUNT = 5
+STABLE_FOLD_STD_WEIGHT = 0.5
+STABLE_CANDIDATE_NAMES = {
+    "baseline_first5_bound",
+    "baseline_first5_close",
+    "baseline_persistence",
+    "extra_trees_delta_d5",
+    "bound_extra_trees_d5",
+    "tatasteel_bound_extra_trees_d5_scaled",
+    "bound_random_forest_d3",
+    "histgb_delta",
+}
+STABLE_SYMBOL_MODEL_MAP = {
+    "BEL": "bound_random_forest_d3",
+    "CANBK": "bound_random_forest_d3",
+    "ITC": "bound_extra_trees_d5",
+    "NTPC": "bound_extra_trees_d5",
+    "ONGC": "bound_random_forest_d3",
+    "POWERGRID": "bound_random_forest_d3",
+    "SBIN": "histgb_delta",
+    "TATASTEEL": "tatasteel_bound_extra_trees_d5_scaled",
+}
+
+
+def note(message: str) -> None:
+    print(f"[stock-high-low] {message}", file=sys.stderr, flush=True)
+
+
+def stem_to_symbol(stem: str) -> str:
+    return str(stem).replace("_", "").upper()
+
+
+def discover_target_symbols(excluded_symbols: set[str] | None = None) -> dict[str, str]:
+    excluded = {s.upper() for s in (excluded_symbols or set())}
+    market_stems = set(MARKET_SYMBOLS.values())
+    stems = {path.name.removesuffix("_1d.csv") for path in DAILY_BAR_DIR.glob("*_1d.csv")}
+    if not stems:
+        stems = {path.name.removesuffix("_1m.csv") for path in MINUTE_BAR_DIR.glob("*_1m.csv")}
+    discovered: dict[str, str] = {}
+    for stem in sorted(stems):
+        if stem in market_stems:
+            continue
+        symbol = stem_to_symbol(stem)
+        if symbol in excluded:
+            continue
+        discovered[symbol] = stem
+    return discovered
+
+
+@dataclass(frozen=True)
+class ModelSpec:
+    name: str
+    kind: str
+    params: dict[str, Any]
+    target_mode: str = "close_delta"
+
+
+@dataclass
+class SymbolResult:
+    symbol: str
+    model_name: str
+    feature_count: int
+    n_train: int
+    n_valid: int
+    n_test: int
+    train_start: str
+    train_end: str
+    valid_start: str
+    valid_end: str
+    test_start: str
+    test_end: str
+    valid_avg_rmse: float
+    test_avg_rmse: float
+    test_high_rmse: float
+    test_low_rmse: float
+    valid_avg_mae: float
+    test_avg_mae: float
+    test_high_mae: float
+    test_low_mae: float
+    close_naive_avg_rmse: float
+    close_naive_high_rmse: float
+    close_naive_low_rmse: float
+    close_naive_avg_mae: float
+    close_naive_high_mae: float
+    close_naive_low_mae: float
+    persistence_naive_avg_rmse: float
+    persistence_naive_high_rmse: float
+    persistence_naive_low_rmse: float
+    first5_bound_naive_avg_rmse: float
+    first5_bound_naive_high_rmse: float
+    first5_bound_naive_low_rmse: float
+    persistence_naive_avg_mae: float
+    persistence_naive_high_mae: float
+    persistence_naive_low_mae: float
+    first5_bound_naive_avg_mae: float
+    first5_bound_naive_high_mae: float
+    first5_bound_naive_low_mae: float
+    close_naive_rmse_improvement_rupees: float
+    persistence_naive_rmse_improvement_rupees: float
+    first5_bound_naive_rmse_improvement_rupees: float
+    close_naive_improvement_rupees: float
+    persistence_naive_improvement_rupees: float
+    first5_bound_naive_improvement_rupees: float
+    target_date_latest_input: str
+    latest_input_date: str
+    latest_pred_high: float
+    latest_pred_low: float
+
+
+def read_daily_bars(symbol: str, stem: str) -> pd.DataFrame:
+    daily_path = DAILY_BAR_DIR / f"{stem}_1d.csv"
+    if daily_path.exists():
+        df = pd.read_csv(daily_path)
+        source = daily_path
+    else:
+        minute_path = MINUTE_BAR_DIR / f"{stem}_1m.csv"
+        if not minute_path.exists():
+            raise FileNotFoundError(f"Missing daily and minute bars for {symbol}: {daily_path}, {minute_path}")
+        note(f"{symbol}: aggregating 1m bars to daily bars from {minute_path.name}")
+        minute = pd.read_csv(minute_path, usecols=["date", "open", "high", "low", "close", "volume"])
+        minute["dt"] = pd.to_datetime(minute["date"], errors="coerce")
+        minute = minute.dropna(subset=["dt", "open", "high", "low", "close"]).sort_values("dt")
+        minute["date"] = minute["dt"].dt.normalize()
+        df = minute.groupby("date", sort=True).agg(
+            open=("open", "first"),
+            high=("high", "max"),
+            low=("low", "min"),
+            close=("close", "last"),
+            volume=("volume", "sum"),
+        ).reset_index()
+        source = minute_path
+
+    df.columns = [str(c).strip().lower().replace(" ", "_") for c in df.columns]
+    if "date" not in df.columns:
+        raise ValueError(f"{source} has no date column")
+    df["date"] = pd.to_datetime(df["date"], errors="coerce").dt.normalize()
+    for col in ("open", "high", "low", "close", "volume"):
+        if col not in df.columns:
+            raise ValueError(f"{source} has no {col} column")
+        df[col] = pd.to_numeric(df[col], errors="coerce")
+    return (
+        df.dropna(subset=["date", "open", "high", "low", "close"])
+        .sort_values("date")
+        .drop_duplicates(subset=["date"], keep="last")
+        .reset_index(drop=True)
+        .assign(symbol=symbol)
+    )
+
+
+def read_first5_features(symbol: str, stem: str, prefix: str) -> pd.DataFrame:
+    minute_path = MINUTE_BAR_DIR / f"{stem}_1m.csv"
+    if not minute_path.exists():
+        raise FileNotFoundError(f"Missing 1m bars for {symbol}: {minute_path}")
+    note(f"{symbol}: building first-5-minute features from {minute_path.name}")
+    minute = pd.read_csv(minute_path, usecols=["date", "open", "high", "low", "close", "volume"])
+    minute["dt"] = pd.to_datetime(minute["date"], errors="coerce")
+    minute = minute.dropna(subset=["dt", "open", "high", "low", "close"]).sort_values("dt")
+    for col in ("open", "high", "low", "close", "volume"):
+        minute[col] = pd.to_numeric(minute[col], errors="coerce")
+    minute["date"] = minute["dt"].dt.normalize()
+    minute["minute_index"] = minute.groupby("date", sort=True).cumcount()
+    first5 = minute[minute["minute_index"] < 5].copy()
+    grouped = first5.groupby("date", sort=True)
+    out = grouped.agg(
+        **{
+            f"{prefix}open": ("open", "first"),
+            f"{prefix}high": ("high", "max"),
+            f"{prefix}low": ("low", "min"),
+            f"{prefix}close": ("close", "last"),
+            f"{prefix}volume": ("volume", "sum"),
+            f"{prefix}bars": ("close", "size"),
+        }
+    ).reset_index()
+    out[f"{prefix}range"] = out[f"{prefix}high"] - out[f"{prefix}low"]
+    out[f"{prefix}return"] = safe_div(out[f"{prefix}close"], out[f"{prefix}open"]) - 1.0
+    out[f"{prefix}range_pct"] = safe_div(out[f"{prefix}range"], out[f"{prefix}open"])
+    out[f"{prefix}high_excursion"] = safe_div(out[f"{prefix}high"], out[f"{prefix}open"]) - 1.0
+    out[f"{prefix}low_excursion"] = safe_div(out[f"{prefix}low"], out[f"{prefix}open"]) - 1.0
+    out[f"{prefix}close_pos"] = safe_div(out[f"{prefix}close"] - out[f"{prefix}low"], out[f"{prefix}range"])
+    for idx in range(5):
+        bar = first5[first5["minute_index"] == idx][["date", "open", "high", "low", "close", "volume"]].copy()
+        bar = bar.rename(
+            columns={
+                "open": f"{prefix}m{idx + 1}_open",
+                "high": f"{prefix}m{idx + 1}_high",
+                "low": f"{prefix}m{idx + 1}_low",
+                "close": f"{prefix}m{idx + 1}_close",
+                "volume": f"{prefix}m{idx + 1}_volume",
+            }
+        )
+        out = out.merge(bar, on="date", how="left")
+        out[f"{prefix}m{idx + 1}_return"] = safe_div(out[f"{prefix}m{idx + 1}_close"], out[f"{prefix}open"]) - 1.0
+    return out[out[f"{prefix}bars"] >= 5].reset_index(drop=True)
+
+
+def read_opening_window_features(symbol: str, stem: str, prefix: str, minutes: int) -> pd.DataFrame:
+    minute_path = MINUTE_BAR_DIR / f"{stem}_1m.csv"
+    if not minute_path.exists():
+        raise FileNotFoundError(f"Missing 1m bars for {symbol}: {minute_path}")
+    note(f"{symbol}: building first-{minutes}-minute features from {minute_path.name}")
+    minute = pd.read_csv(minute_path, usecols=["date", "open", "high", "low", "close", "volume"])
+    minute["dt"] = pd.to_datetime(minute["date"], errors="coerce")
+    minute = minute.dropna(subset=["dt", "open", "high", "low", "close"]).sort_values("dt")
+    for col in ("open", "high", "low", "close", "volume"):
+        minute[col] = pd.to_numeric(minute[col], errors="coerce")
+    minute["date"] = minute["dt"].dt.normalize()
+    minute["minute_index"] = minute.groupby("date", sort=True).cumcount()
+    window = minute[minute["minute_index"] < minutes].copy()
+    grouped = window.groupby("date", sort=True)
+    out = grouped.agg(
+        **{
+            f"{prefix}open": ("open", "first"),
+            f"{prefix}high": ("high", "max"),
+            f"{prefix}low": ("low", "min"),
+            f"{prefix}close": ("close", "last"),
+            f"{prefix}volume": ("volume", "sum"),
+            f"{prefix}bars": ("close", "size"),
+        }
+    ).reset_index()
+    out[f"{prefix}range"] = out[f"{prefix}high"] - out[f"{prefix}low"]
+    out[f"{prefix}return"] = safe_div(out[f"{prefix}close"], out[f"{prefix}open"]) - 1.0
+    out[f"{prefix}range_pct"] = safe_div(out[f"{prefix}range"], out[f"{prefix}open"])
+    out[f"{prefix}close_pos"] = safe_div(out[f"{prefix}close"] - out[f"{prefix}low"], out[f"{prefix}range"])
+    for checkpoint in sorted({5, 10, minutes}):
+        if checkpoint > minutes:
+            continue
+        marker = f"{prefix}{checkpoint}m_"
+        checkpoint_window = minute[minute["minute_index"] < checkpoint]
+        checkpoint_features = checkpoint_window.groupby("date", sort=True).agg(
+            **{
+                f"{marker}high": ("high", "max"),
+                f"{marker}low": ("low", "min"),
+                f"{marker}close": ("close", "last"),
+                f"{marker}volume": ("volume", "sum"),
+                f"{marker}bars": ("close", "size"),
+            }
+        ).reset_index()
+        out = out.merge(checkpoint_features, on="date", how="left")
+    return out[out[f"{prefix}bars"] >= 5].replace([np.inf, -np.inf], np.nan).reset_index(drop=True)
+
+
+def read_opening_5m_features(symbol: str, stem: str, prefix: str) -> pd.DataFrame:
+    feature_path = FIVE_MIN_FEATURE_DIR / f"{stem}_5m_features.csv"
+    if not feature_path.exists():
+        raise FileNotFoundError(f"Missing 5m feature file for {symbol}: {feature_path}")
+    note(f"{symbol}: adding processed opening-5m features from {feature_path.name}")
+    features = pd.read_csv(feature_path)
+    features.columns = [str(c).strip().lower().replace(" ", "_") for c in features.columns]
+    if "date" not in features.columns:
+        raise ValueError(f"{feature_path} has no date column")
+    features["dt"] = pd.to_datetime(features["date"], errors="coerce")
+    features = features.dropna(subset=["dt"]).sort_values("dt")
+    features["date"] = features["dt"].dt.normalize()
+    features["bar_number"] = features.groupby("date", sort=True).cumcount()
+    opening = features[features["bar_number"] == 0].copy()
+    value_cols = [
+        col
+        for col in opening.columns
+        if col not in {"date", "dt", "bar_number"}
+        and not col.startswith("target_")
+        and "target" not in col
+        and "future" not in col
+        and "next" not in col
+    ]
+    out = opening[["date", *value_cols]].copy()
+    for col in value_cols:
+        out[col] = pd.to_numeric(out[col], errors="coerce")
+    out = out.rename(columns={col: f"{prefix}{col}" for col in value_cols})
+    return out.replace([np.inf, -np.inf], np.nan).reset_index(drop=True)
+
+
+def safe_div(numer: pd.Series | np.ndarray, denom: pd.Series | np.ndarray) -> pd.Series:
+    n = pd.Series(numer, copy=False)
+    d = pd.Series(denom, copy=False)
+    out = pd.Series(np.nan, index=n.index, dtype="float64")
+    mask = d.notna() & np.isfinite(d.to_numpy(dtype="float64")) & (d != 0)
+    out.loc[mask] = n.loc[mask].to_numpy(dtype="float64") / d.loc[mask].to_numpy(dtype="float64")
+    return out
+
+
+def build_technical_features(df: pd.DataFrame, prefix: str) -> pd.DataFrame:
+    d = df.sort_values("date").reset_index(drop=True)
+    out = pd.DataFrame({"date": d["date"]})
+    open_ = d["open"].astype("float64")
+    high = d["high"].astype("float64")
+    low = d["low"].astype("float64")
+    close = d["close"].astype("float64")
+    volume = d["volume"].astype("float64").replace(0, np.nan)
+    day_range = high - low
+    ret1 = close.pct_change(fill_method=None)
+    true_range = pd.concat(
+        [day_range, (high - close.shift(1)).abs(), (low - close.shift(1)).abs()],
+        axis=1,
+    ).max(axis=1)
+
+    base = {
+        "open": open_,
+        "high": high,
+        "low": low,
+        "close": close,
+        "volume_log": np.log1p(volume),
+        "range": day_range,
+        "range_pct": safe_div(day_range, close),
+        "body_pct": safe_div(close - open_, open_),
+        "upper_wick_pct": safe_div(high - pd.concat([open_, close], axis=1).max(axis=1), close),
+        "lower_wick_pct": safe_div(pd.concat([open_, close], axis=1).min(axis=1) - low, close),
+        "close_pos": safe_div(close - low, day_range),
+        "gap_pct": safe_div(open_, close.shift(1)) - 1.0,
+        "ret1": ret1,
+    }
+    for name, values in base.items():
+        out[f"{prefix}{name}"] = values
+
+    for lag in (1, 2, 3, 5, 10):
+        for name in ("open", "high", "low", "close", "range", "range_pct", "body_pct", "close_pos", "gap_pct", "ret1"):
+            out[f"{prefix}{name}_lag{lag}"] = out[f"{prefix}{name}"].shift(lag)
+
+    for window in (2, 3, 5, 10, 20, 40, 60, 120):
+        min_periods = max(2, window // 2)
+        out[f"{prefix}ret_{window}d"] = safe_div(close, close.shift(window)) - 1.0
+        out[f"{prefix}ret_mean_{window}d"] = ret1.rolling(window, min_periods=min_periods).mean()
+        out[f"{prefix}ret_std_{window}d"] = ret1.rolling(window, min_periods=min_periods).std()
+        out[f"{prefix}abs_ret_mean_{window}d"] = ret1.abs().rolling(window, min_periods=min_periods).mean()
+        out[f"{prefix}range_mean_{window}d"] = day_range.rolling(window, min_periods=min_periods).mean()
+        out[f"{prefix}range_pct_mean_{window}d"] = safe_div(day_range, close).rolling(window, min_periods=min_periods).mean()
+        out[f"{prefix}atr_{window}d"] = true_range.rolling(window, min_periods=min_periods).mean()
+        out[f"{prefix}close_vs_sma_{window}d"] = safe_div(close, close.rolling(window, min_periods=min_periods).mean()) - 1.0
+        roll_high = high.rolling(window, min_periods=min_periods).max()
+        roll_low = low.rolling(window, min_periods=min_periods).min()
+        out[f"{prefix}close_pos_{window}d"] = safe_div(close - roll_low, roll_high - roll_low)
+
+    for span in (5, 12, 26, 50):
+        ema = close.ewm(span=span, adjust=False, min_periods=max(2, span // 2)).mean()
+        out[f"{prefix}close_vs_ema_{span}d"] = safe_div(close, ema) - 1.0
+
+    out[f"{prefix}day_of_week"] = d["date"].dt.dayofweek
+    out[f"{prefix}month"] = d["date"].dt.month
+    return out
+
+
+def build_market_features() -> pd.DataFrame:
+    market = None
+    for symbol, stem in MARKET_SYMBOLS.items():
+        features = build_technical_features(read_daily_bars(symbol, stem), f"{stem}_")
+        market = features if market is None else market.merge(features, on="date", how="outer")
+    if market is None:
+        raise RuntimeError("No market features were built.")
+    return market.sort_values("date").reset_index(drop=True)
+
+
+def build_market_first5_features(target_symbols: dict[str, str]) -> pd.DataFrame:
+    market = None
+    for symbol, stem in MARKET_SYMBOLS.items():
+        features = read_first5_features(symbol, stem, f"{stem}_first5_")
+        market = features if market is None else market.merge(features, on="date", how="outer")
+    peer_return_cols = []
+    peer_range_cols = []
+    for symbol, stem in target_symbols.items():
+        features = read_first5_features(symbol, stem, f"peer_{stem}_first5_")
+        peer_return_cols.append(f"peer_{stem}_first5_return")
+        peer_range_cols.append(f"peer_{stem}_first5_range_pct")
+        market = features if market is None else market.merge(features, on="date", how="outer")
+    if market is None:
+        raise RuntimeError("No market first-5-minute features were built.")
+    market["peer_first5_return_mean"] = market[peer_return_cols].mean(axis=1)
+    market["peer_first5_return_std"] = market[peer_return_cols].std(axis=1)
+    market["peer_first5_range_pct_mean"] = market[peer_range_cols].mean(axis=1)
+    market["peer_first5_range_pct_std"] = market[peer_range_cols].std(axis=1)
+    return market.sort_values("date").reset_index(drop=True)
+
+
+def build_opening_5m_panel(target_symbols: dict[str, str]) -> pd.DataFrame:
+    panel = None
+    for symbol, stem in {**MARKET_SYMBOLS, **target_symbols}.items():
+        features = read_opening_5m_features(symbol, stem, f"{stem}_open5_")
+        panel = features if panel is None else panel.merge(features, on="date", how="outer")
+    if panel is None:
+        raise RuntimeError("No opening 5m features were built.")
+    return panel.sort_values("date").reset_index(drop=True)
+
+
+def read_prior_day_panel(path: Path, prefix: str) -> pd.DataFrame:
+    if not path.exists():
+        note(f"skipping missing exogenous panel: {path}")
+        return pd.DataFrame()
+    panel = pd.read_csv(path)
+    panel.columns = [str(c).strip().lower().replace(" ", "_") for c in panel.columns]
+    if "date" not in panel.columns:
+        raise ValueError(f"{path} has no date column")
+    panel["date"] = pd.to_datetime(panel["date"], errors="coerce").dt.normalize()
+    panel = panel.dropna(subset=["date"]).sort_values("date").drop_duplicates("date", keep="last")
+    if prefix == "ext_":
+        for col in list(panel.columns):
+            if col == "date":
+                continue
+            if col.endswith("_close"):
+                base = col[: -len("_close")]
+                value_col = f"{base}_value"
+                change_col = f"{base}_change_1"
+                if value_col not in panel.columns:
+                    panel[value_col] = pd.to_numeric(panel[col], errors="coerce")
+                if change_col not in panel.columns:
+                    panel[change_col] = pd.to_numeric(panel[value_col], errors="coerce").diff()
+            elif col.endswith("_value"):
+                base = col[: -len("_value")]
+                close_col = f"{base}_close"
+                change_col = f"{base}_change_1"
+                if close_col not in panel.columns:
+                    panel[close_col] = pd.to_numeric(panel[col], errors="coerce")
+                if change_col not in panel.columns:
+                    panel[change_col] = pd.to_numeric(panel[col], errors="coerce").diff()
+    value_cols = [
+        col
+        for col in panel.columns
+        if col != "date"
+        and not col.startswith("target_")
+        and "target" not in col
+        and "future" not in col
+        and "next" not in col
+    ]
+    out = pd.DataFrame({"date": panel["date"]})
+    for col in value_cols:
+        out[f"{prefix}{col}"] = pd.to_numeric(panel[col], errors="coerce")
+    numeric_cols = [col for col in out.columns if col != "date"]
+    for col in numeric_cols:
+        values = out[col]
+        out[f"{col}_lag1"] = values.shift(1)
+        out[f"{col}_lag3"] = values.shift(3)
+        out[f"{col}_lag5"] = values.shift(5)
+        out[f"{col}_mean5"] = values.rolling(5, min_periods=2).mean()
+        out[f"{col}_std5"] = values.rolling(5, min_periods=2).std()
+    return out.replace([np.inf, -np.inf], np.nan).reset_index(drop=True)
+
+
+def build_prior_day_exogenous_features() -> pd.DataFrame:
+    panels = [
+        read_prior_day_panel(ALT_DATA_DIR / "external" / "processed" / "external_daily_panel.csv", "ext_"),
+        read_prior_day_panel(ALT_DATA_DIR / "institutional" / "processed" / "institutional_daily_panel.csv", "inst_"),
+        read_prior_day_panel(ALT_DATA_DIR / "options" / "processed" / "hdfcbank_options_daily_features.csv", "hdfc_opt_"),
+        read_prior_day_panel(ALT_DATA_DIR / "options" / "processed" / "banknifty_options_daily_features.csv", "banknifty_opt_"),
+        read_prior_day_panel(ALT_DATA_DIR / "options" / "processed" / "nifty50_options_daily_features.csv", "nifty_opt_"),
+    ]
+    exogenous = None
+    for panel in panels:
+        if panel.empty:
+            continue
+        exogenous = panel if exogenous is None else exogenous.merge(panel, on="date", how="outer")
+    if exogenous is None:
+        return pd.DataFrame()
+    return exogenous.sort_values("date").reset_index(drop=True)
+
+
+@lru_cache(maxsize=1)
+def load_corporate_announcement_events() -> pd.DataFrame:
+    if not CORPORATE_ANNOUNCEMENTS_PATH.exists():
+        note(f"skipping missing corporate announcement panel: {CORPORATE_ANNOUNCEMENTS_PATH}")
+        return pd.DataFrame()
+    events = pd.read_csv(CORPORATE_ANNOUNCEMENTS_PATH)
+    if events.empty:
+        return events
+    events.columns = [str(c).strip() for c in events.columns]
+    events["symbol"] = events["symbol"].astype(str).str.upper().str.strip()
+    events["event_ts"] = pd.to_datetime(events["event_ts"], errors="coerce")
+    events = events.dropna(subset=["symbol", "event_ts"]).sort_values(["symbol", "event_ts"])
+    for col in (
+        "is_results",
+        "is_board_meeting",
+        "is_investor_meet",
+        "is_credit_rating",
+        "is_press_release",
+        "is_high_impact",
+    ):
+        if col not in events.columns:
+            events[col] = False
+        events[col] = events[col].fillna(False).astype(bool)
+    return events.reset_index(drop=True)
+
+
+def build_corporate_announcement_features(symbol: str, frame: pd.DataFrame) -> pd.DataFrame:
+    events = load_corporate_announcement_events()
+    out = pd.DataFrame(index=frame.index)
+    feature_names = [
+        "corp_ann_count_1d",
+        "corp_ann_count_3d",
+        "corp_ann_count_7d",
+        "corp_ann_count_30d",
+        "corp_high_impact_count_7d",
+        "corp_high_impact_count_30d",
+        "corp_results_count_30d",
+        "corp_board_count_30d",
+        "corp_investor_count_7d",
+        "corp_credit_rating_count_30d",
+        "corp_press_release_count_7d",
+        "corp_since_prev_close_count",
+        "corp_since_prev_close_high_impact",
+        "corp_forecast_morning_count",
+        "corp_forecast_morning_high_impact",
+        "corp_last_announcement_hours",
+        "corp_last_high_impact_hours",
+    ]
+    for name in feature_names:
+        out[name] = np.nan if name.endswith("_hours") else 0.0
+    if events.empty:
+        return out
+
+    symbol_events = events[events["symbol"] == symbol.upper()].copy()
+    if symbol_events.empty:
+        return out
+    ts = symbol_events["event_ts"].to_numpy(dtype="datetime64[ns]")
+    masks = {
+        "high": symbol_events["is_high_impact"].to_numpy(dtype=bool),
+        "results": symbol_events["is_results"].to_numpy(dtype=bool),
+        "board": symbol_events["is_board_meeting"].to_numpy(dtype=bool),
+        "investor": symbol_events["is_investor_meet"].to_numpy(dtype=bool),
+        "rating": symbol_events["is_credit_rating"].to_numpy(dtype=bool),
+        "press": symbol_events["is_press_release"].to_numpy(dtype=bool),
+    }
+
+    def count_between(start: pd.Timestamp, end: pd.Timestamp, mask: np.ndarray | None = None) -> float:
+        active = (ts > np.datetime64(start)) & (ts <= np.datetime64(end))
+        if mask is not None:
+            active &= mask
+        return float(active.sum())
+
+    for idx, row in frame[["date", "target_date"]].iterrows():
+        if pd.isna(row["date"]) or pd.isna(row["target_date"]):
+            continue
+        row_date = pd.Timestamp(row["date"]).normalize()
+        target_date = pd.Timestamp(row["target_date"]).normalize()
+        cutoff = target_date + pd.Timedelta(hours=9, minutes=20)
+        prev_close_cutoff = row_date + pd.Timedelta(hours=15, minutes=30)
+        out.at[idx, "corp_ann_count_1d"] = count_between(cutoff - pd.Timedelta(days=1), cutoff)
+        out.at[idx, "corp_ann_count_3d"] = count_between(cutoff - pd.Timedelta(days=3), cutoff)
+        out.at[idx, "corp_ann_count_7d"] = count_between(cutoff - pd.Timedelta(days=7), cutoff)
+        out.at[idx, "corp_ann_count_30d"] = count_between(cutoff - pd.Timedelta(days=30), cutoff)
+        out.at[idx, "corp_high_impact_count_7d"] = count_between(cutoff - pd.Timedelta(days=7), cutoff, masks["high"])
+        out.at[idx, "corp_high_impact_count_30d"] = count_between(cutoff - pd.Timedelta(days=30), cutoff, masks["high"])
+        out.at[idx, "corp_results_count_30d"] = count_between(cutoff - pd.Timedelta(days=30), cutoff, masks["results"])
+        out.at[idx, "corp_board_count_30d"] = count_between(cutoff - pd.Timedelta(days=30), cutoff, masks["board"])
+        out.at[idx, "corp_investor_count_7d"] = count_between(cutoff - pd.Timedelta(days=7), cutoff, masks["investor"])
+        out.at[idx, "corp_credit_rating_count_30d"] = count_between(cutoff - pd.Timedelta(days=30), cutoff, masks["rating"])
+        out.at[idx, "corp_press_release_count_7d"] = count_between(cutoff - pd.Timedelta(days=7), cutoff, masks["press"])
+        out.at[idx, "corp_since_prev_close_count"] = count_between(prev_close_cutoff, cutoff)
+        out.at[idx, "corp_since_prev_close_high_impact"] = count_between(prev_close_cutoff, cutoff, masks["high"])
+        out.at[idx, "corp_forecast_morning_count"] = count_between(target_date, cutoff)
+        out.at[idx, "corp_forecast_morning_high_impact"] = count_between(target_date, cutoff, masks["high"])
+        known = ts[ts <= np.datetime64(cutoff)]
+        if len(known):
+            out.at[idx, "corp_last_announcement_hours"] = float((cutoff - pd.Timestamp(known[-1])).total_seconds() / 3600.0)
+        known_high = ts[(ts <= np.datetime64(cutoff)) & masks["high"]]
+        if len(known_high):
+            out.at[idx, "corp_last_high_impact_hours"] = float((cutoff - pd.Timestamp(known_high[-1])).total_seconds() / 3600.0)
+    return out
+
+
+def build_symbol_frame(
+    symbol: str,
+    stem: str,
+    market: pd.DataFrame,
+    market_first5: pd.DataFrame,
+    opening_5m: pd.DataFrame | None = None,
+    exogenous: pd.DataFrame | None = None,
+) -> pd.DataFrame:
+    bars = read_daily_bars(symbol, stem)
+    features = build_technical_features(bars, "stock_")
+    frame = (
+        bars[["date", "symbol", "open", "high", "low", "close", "volume"]]
+        .merge(features, on="date", how="left")
+        .merge(market, on="date", how="left")
+        .sort_values("date")
+        .reset_index(drop=True)
+    )
+    if exogenous is not None and not exogenous.empty:
+        frame = pd.merge_asof(
+            frame.sort_values("date"),
+            exogenous.sort_values("date"),
+            on="date",
+            direction="backward",
+        ).reset_index(drop=True)
+    frame["target_date"] = frame["date"].shift(-1)
+    frame["target_high"] = frame["high"].shift(-1)
+    frame["target_low"] = frame["low"].shift(-1)
+    target_dt = pd.to_datetime(frame["target_date"], errors="coerce")
+    frame["forecast_day_of_week"] = target_dt.dt.dayofweek
+    frame["forecast_month"] = target_dt.dt.month
+    frame["forecast_day_of_month"] = target_dt.dt.day
+    frame["forecast_is_weekend_session"] = target_dt.dt.dayofweek.isin([5, 6]).astype("float64")
+    frame["forecast_is_month_start"] = target_dt.dt.is_month_start.astype("float64")
+    frame["forecast_is_month_end"] = target_dt.dt.is_month_end.astype("float64")
+    frame = pd.concat([frame, build_corporate_announcement_features(symbol, frame)], axis=1)
+    stock_first5 = read_first5_features(symbol, stem, "first5_")
+    frame = frame.merge(stock_first5, left_on="target_date", right_on="date", how="left", suffixes=("", "_first5"))
+    frame = frame.drop(columns=["date_first5"])
+    if symbol == "HDFCBANK":
+        hdfc_opening = read_opening_window_features(symbol, stem, HDFC_OPENING_PREFIX, HDFC_OPENING_MINUTES)
+        frame = frame.merge(hdfc_opening, left_on="target_date", right_on="date", how="left", suffixes=("", "_hdfc_opening"))
+        frame = frame.drop(columns=["date_hdfc_opening"])
+    frame = frame.merge(market_first5, left_on="target_date", right_on="date", how="left", suffixes=("", "_market_first5"))
+    frame = frame.drop(columns=["date_market_first5"])
+    if symbol == "HDFCBANK" and opening_5m is not None:
+        frame = frame.merge(opening_5m, left_on="target_date", right_on="date", how="left", suffixes=("", "_open5_panel"))
+        frame = frame.drop(columns=["date_open5_panel"])
+    frame["target_high_delta_from_first5_close"] = frame["target_high"] - frame["first5_close"]
+    frame["target_low_delta_from_first5_close"] = frame["target_low"] - frame["first5_close"]
+    return frame.replace([np.inf, -np.inf], np.nan)
+
+
+def candidate_specs(seed: int) -> list[ModelSpec]:
+    return [
+        ModelSpec("baseline_first5_close", "baseline", {"baseline": "first5_close"}, target_mode="baseline"),
+        ModelSpec("baseline_first5_bound", "baseline", {"baseline": "first5_bound"}, target_mode="baseline"),
+        ModelSpec("baseline_persistence", "baseline", {"baseline": "persistence"}, target_mode="baseline"),
+        ModelSpec("ridge_delta_a100", "ridge", {"alpha": 100.0}),
+        ModelSpec("ridge_delta_a1000", "ridge", {"alpha": 1000.0}),
+        ModelSpec("ridge_delta_a5000", "ridge", {"alpha": 5000.0}),
+        ModelSpec("bound_ridge_a1000", "ridge", {"alpha": 1000.0}, target_mode="bound_dist"),
+        ModelSpec("bound_ridge_a5000", "ridge", {"alpha": 5000.0}, target_mode="bound_dist"),
+        ModelSpec(
+            "extra_trees_delta_d3",
+            "extra_trees",
+            {
+                "n_estimators": 120,
+                "max_depth": 3,
+                "min_samples_leaf": 20,
+                "max_features": 0.80,
+                "random_state": seed + 2,
+                "n_jobs": -1,
+            },
+        ),
+        ModelSpec(
+            "extra_trees_delta_d7",
+            "extra_trees",
+            {
+                "n_estimators": 120,
+                "max_depth": 7,
+                "min_samples_leaf": 12,
+                "max_features": 0.70,
+                "random_state": seed,
+                "n_jobs": -1,
+            },
+        ),
+        ModelSpec(
+            "extra_trees_delta_d5",
+            "extra_trees",
+            {
+                "n_estimators": 120,
+                "max_depth": 5,
+                "min_samples_leaf": 10,
+                "max_features": 0.75,
+                "random_state": seed + 1,
+                "n_jobs": -1,
+            },
+        ),
+        ModelSpec(
+            "bound_extra_trees_d3",
+            "extra_trees",
+            {
+                "n_estimators": 120,
+                "max_depth": 3,
+                "min_samples_leaf": 20,
+                "max_features": 0.80,
+                "random_state": seed + 4,
+                "n_jobs": -1,
+            },
+            target_mode="bound_dist",
+        ),
+        ModelSpec(
+            "bound_extra_trees_d5",
+            "extra_trees",
+            {
+                "n_estimators": 120,
+                "max_depth": 5,
+                "min_samples_leaf": 12,
+                "max_features": 0.75,
+                "random_state": seed + 5,
+                "n_jobs": -1,
+            },
+            target_mode="bound_dist",
+        ),
+        ModelSpec(
+            "tatasteel_bound_extra_trees_d5_scaled",
+            "extra_trees",
+            {
+                "n_estimators": 120,
+                "max_depth": 5,
+                "min_samples_leaf": 12,
+                "max_features": 0.75,
+                "random_state": seed + 5,
+                "n_jobs": -1,
+                "post_scale_high": 0.975,
+                "post_scale_low": 0.75,
+            },
+            target_mode="bound_dist",
+        ),
+        ModelSpec(
+            "bound_random_forest_d3",
+            "random_forest",
+            {
+                "n_estimators": 80,
+                "max_depth": 3,
+                "min_samples_leaf": 20,
+                "max_features": 0.80,
+                "random_state": seed + 6,
+                "n_jobs": -1,
+            },
+            target_mode="bound_dist",
+        ),
+        ModelSpec(
+            "histgb_delta",
+            "histgb",
+            {
+                "max_iter": 100,
+                "learning_rate": 0.035,
+                "max_leaf_nodes": 15,
+                "min_samples_leaf": 35,
+                "l2_regularization": 1.0,
+                "random_state": seed,
+            },
+        ),
+    ]
+
+
+def hdfc_open10_candidate_specs(seed: int) -> list[ModelSpec]:
+    return [
+        ModelSpec(
+            name=f"hdfc_open10_bound_{HDFC_OPENING_MINUTES}m",
+            kind="baseline",
+            params={"minutes": HDFC_OPENING_MINUTES},
+            target_mode="open10_bound",
+        ),
+        ModelSpec("hdfc_open10_ridge_delta_a3000", "ridge", {"alpha": 3000.0}, target_mode="open10_delta"),
+        ModelSpec("hdfc_open10_ridge_delta_a10000", "ridge", {"alpha": 10000.0}, target_mode="open10_delta"),
+        ModelSpec(
+            "hdfc_open10_recency_ridge_delta_a10000_hl100",
+            "ridge",
+            {"alpha": 10000.0, "sample_half_life": 100.0},
+            target_mode="open10_delta",
+        ),
+        ModelSpec("hdfc_open10_ridge_delta_a100000", "ridge", {"alpha": 100000.0}, target_mode="open10_delta"),
+        ModelSpec("hdfc_open10_bound_ridge_a3000", "ridge", {"alpha": 3000.0}, target_mode="open10_bound_dist"),
+        ModelSpec("hdfc_open10_bound_ridge_a10000", "ridge", {"alpha": 10000.0}, target_mode="open10_bound_dist"),
+        ModelSpec(
+            "hdfc_open10_extra_trees_d3",
+            "extra_trees",
+            {
+                "n_estimators": 96,
+                "max_depth": 3,
+                "min_samples_leaf": 18,
+                "max_features": 0.60,
+                "random_state": seed + 21,
+                "n_jobs": -1,
+            },
+            target_mode="open10_bound_dist",
+        ),
+        ModelSpec(
+            "hdfc_open10_extra_trees_d5",
+            "extra_trees",
+            {
+                "n_estimators": 96,
+                "max_depth": 5,
+                "min_samples_leaf": 14,
+                "max_features": 0.60,
+                "random_state": seed + 22,
+                "n_jobs": -1,
+            },
+            target_mode="open10_bound_dist",
+        ),
+        ModelSpec(
+            "hdfc_open10_histgb",
+            "histgb",
+            {
+                "max_iter": 100,
+                "learning_rate": 0.03,
+                "max_leaf_nodes": 15,
+                "min_samples_leaf": 35,
+                "l2_regularization": 5.0,
+                "random_state": seed + 23,
+            },
+            target_mode="open10_bound_dist",
+        ),
+    ]
+
+
+def build_model(spec: ModelSpec):
+    model_params = {
+        key: value
+        for key, value in spec.params.items()
+        if not key.startswith("post_")
+    }
+    if spec.kind == "ridge":
+        params = {key: value for key, value in model_params.items() if key != "sample_half_life"}
+        return make_pipeline(SimpleImputer(strategy="median"), StandardScaler(), Ridge(**params))
+    if spec.kind == "extra_trees":
+        return MultiOutputRegressor(make_pipeline(SimpleImputer(strategy="median"), ExtraTreesRegressor(**model_params)))
+    if spec.kind == "random_forest":
+        return MultiOutputRegressor(make_pipeline(SimpleImputer(strategy="median"), RandomForestRegressor(**model_params)))
+    if spec.kind == "histgb":
+        return MultiOutputRegressor(make_pipeline(SimpleImputer(strategy="median"), HistGradientBoostingRegressor(**model_params)))
+    raise ValueError(f"Unknown model kind: {spec.kind}")
+
+
+def baseline_high_low(spec: ModelSpec, rows: pd.DataFrame) -> tuple[np.ndarray, np.ndarray]:
+    baseline = str(spec.params.get("baseline", ""))
+    if baseline == "first5_close":
+        close = rows["first5_close"].to_numpy(dtype="float64")
+        return close.copy(), close.copy()
+    if baseline == "first5_bound":
+        return rows["first5_high"].to_numpy(dtype="float64"), rows["first5_low"].to_numpy(dtype="float64")
+    if baseline == "persistence":
+        return rows["high"].to_numpy(dtype="float64"), rows["low"].to_numpy(dtype="float64")
+    raise ValueError(f"Unknown baseline kind: {baseline}")
+
+
+def feature_columns(frame: pd.DataFrame) -> list[str]:
+    blocked = {
+        "date",
+        "symbol",
+        "target_date",
+        "target_high",
+        "target_low",
+        "target_high_delta_from_close",
+        "target_low_delta_from_close",
+        "target_high_delta_from_first5_close",
+        "target_low_delta_from_first5_close",
+    }
+    cols = [c for c in frame.columns if c not in blocked]
+    cols = [c for c in cols if pd.to_numeric(frame[c], errors="coerce").notna().any()]
+    leak_words = ("target", "future", "next")
+    bad = [c for c in cols if any(word in c.lower() for word in leak_words)]
+    if bad:
+        raise RuntimeError(f"Potential leakage columns in feature set: {bad[:10]}")
+    return cols
+
+
+def hdfc_fast_feature_columns(cols: list[str]) -> list[str]:
+    keep_prefixes = (
+        "stock_",
+        "nifty50_",
+        "banknifty_",
+        "india_vix_",
+        "first5_",
+        "peer_",
+        "hdfc_open10_",
+    )
+    always_keep = {"open", "high", "low", "close", "volume"}
+    selected = [c for c in cols if c in always_keep or c.startswith(keep_prefixes)]
+    selected = [c for c in selected if not (c.endswith("_std5") or c.endswith("_lag5"))]
+    return selected
+
+
+def predict_high_low(model: Any, rows: pd.DataFrame, cols: list[str]) -> tuple[np.ndarray, np.ndarray]:
+    pred_delta = np.asarray(model.predict(rows[cols]), dtype="float64")
+    base = rows["first5_close"].to_numpy(dtype="float64") if "first5_close" in rows.columns else rows["close"].to_numpy(dtype="float64")
+    pred_high = base + pred_delta[:, 0]
+    pred_low = base + pred_delta[:, 1]
+    high = np.maximum(pred_high, pred_low)
+    low = np.minimum(pred_high, pred_low)
+    if {"first5_high", "first5_low"}.issubset(rows.columns):
+        high = np.maximum(high, rows["first5_high"].to_numpy(dtype="float64"))
+        low = np.minimum(low, rows["first5_low"].to_numpy(dtype="float64"))
+        high, low = cap_first5_extension(rows, high, low)
+    return high, low
+
+
+def cap_first5_extension(rows: pd.DataFrame, high: np.ndarray, low: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
+    if not {"first5_high", "first5_low"}.issubset(rows.columns):
+        return high, low
+    first5_high = rows["first5_high"].to_numpy(dtype="float64")
+    first5_low = rows["first5_low"].to_numpy(dtype="float64")
+    capped_high = np.minimum(np.asarray(high, dtype="float64"), first5_high + GENERIC_FIRST5_EXTENSION_CAP)
+    capped_low = np.maximum(np.asarray(low, dtype="float64"), first5_low - GENERIC_FIRST5_EXTENSION_CAP)
+    capped_high = np.maximum(capped_high, first5_high)
+    capped_low = np.minimum(capped_low, first5_low)
+    return capped_high, capped_low
+
+
+def cap_hdfc_open10_extension(rows: pd.DataFrame, high: np.ndarray, low: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
+    if not {f"{HDFC_OPENING_PREFIX}high", f"{HDFC_OPENING_PREFIX}low"}.issubset(rows.columns):
+        return high, low
+    open_high = rows[f"{HDFC_OPENING_PREFIX}high"].to_numpy(dtype="float64")
+    open_low = rows[f"{HDFC_OPENING_PREFIX}low"].to_numpy(dtype="float64")
+    capped_high = np.minimum(np.asarray(high, dtype="float64"), open_high + HDFC_OPEN10_EXTENSION_CAP)
+    capped_low = np.maximum(np.asarray(low, dtype="float64"), open_low - HDFC_OPEN10_EXTENSION_CAP)
+    capped_high = np.maximum(capped_high, open_high)
+    capped_low = np.minimum(capped_low, open_low)
+    return capped_high, capped_low
+
+
+def target_matrix(spec: ModelSpec, rows: pd.DataFrame) -> np.ndarray:
+    if spec.target_mode == "close_delta":
+        return np.column_stack(
+            [
+                rows["target_high_delta_from_first5_close"].to_numpy(dtype="float64"),
+                rows["target_low_delta_from_first5_close"].to_numpy(dtype="float64"),
+            ]
+        )
+    if spec.target_mode == "bound_dist":
+        return np.column_stack(
+            [
+                rows["target_high"].to_numpy(dtype="float64") - rows["first5_high"].to_numpy(dtype="float64"),
+                rows["first5_low"].to_numpy(dtype="float64") - rows["target_low"].to_numpy(dtype="float64"),
+            ]
+        )
+    if spec.target_mode == "open10_delta":
+        return np.column_stack(
+            [
+                rows["target_high"].to_numpy(dtype="float64") - rows[f"{HDFC_OPENING_PREFIX}close"].to_numpy(dtype="float64"),
+                rows["target_low"].to_numpy(dtype="float64") - rows[f"{HDFC_OPENING_PREFIX}close"].to_numpy(dtype="float64"),
+            ]
+        )
+    if spec.target_mode == "open10_bound_dist":
+        return np.column_stack(
+            [
+                rows["target_high"].to_numpy(dtype="float64") - rows[f"{HDFC_OPENING_PREFIX}high"].to_numpy(dtype="float64"),
+                rows[f"{HDFC_OPENING_PREFIX}low"].to_numpy(dtype="float64") - rows["target_low"].to_numpy(dtype="float64"),
+            ]
+        )
+    raise ValueError(f"Unknown target_mode: {spec.target_mode}")
+
+
+def predict_spec_high_low(spec: ModelSpec, model: Any, rows: pd.DataFrame, cols: list[str]) -> tuple[np.ndarray, np.ndarray]:
+    if spec.target_mode == "close_delta":
+        return predict_high_low(model, rows, cols)
+    if spec.target_mode == "bound_dist":
+        pred = np.asarray(model.predict(rows[cols]), dtype="float64")
+        high_scale = float(spec.params.get("post_scale_high", 1.0))
+        low_scale = float(spec.params.get("post_scale_low", 1.0))
+        high = rows["first5_high"].to_numpy(dtype="float64") + (high_scale * np.maximum(0.0, pred[:, 0]))
+        low = rows["first5_low"].to_numpy(dtype="float64") - (low_scale * np.maximum(0.0, pred[:, 1]))
+        return cap_first5_extension(rows, high, low)
+    if spec.target_mode == "open10_delta":
+        pred = np.asarray(model.predict(rows[cols]), dtype="float64")
+        base = rows[f"{HDFC_OPENING_PREFIX}close"].to_numpy(dtype="float64")
+        pred_high = base + pred[:, 0]
+        pred_low = base + pred[:, 1]
+        high = np.maximum(pred_high, pred_low)
+        low = np.minimum(pred_high, pred_low)
+        high = np.maximum(high, rows[f"{HDFC_OPENING_PREFIX}high"].to_numpy(dtype="float64"))
+        low = np.minimum(low, rows[f"{HDFC_OPENING_PREFIX}low"].to_numpy(dtype="float64"))
+        return cap_hdfc_open10_extension(rows, high, low)
+    if spec.target_mode == "open10_bound_dist":
+        pred = np.asarray(model.predict(rows[cols]), dtype="float64")
+        high = rows[f"{HDFC_OPENING_PREFIX}high"].to_numpy(dtype="float64") + np.maximum(0.0, pred[:, 0])
+        low = rows[f"{HDFC_OPENING_PREFIX}low"].to_numpy(dtype="float64") - np.maximum(0.0, pred[:, 1])
+        return cap_hdfc_open10_extension(rows, high, low)
+    if spec.target_mode == "open10_bound":
+        if f"{HDFC_OPENING_PREFIX}high" in rows.columns:
+            opening_high = pd.to_numeric(rows[f"{HDFC_OPENING_PREFIX}high"], errors="coerce").to_numpy(dtype="float64")
+        else:
+            opening_high = np.full(len(rows), np.nan, dtype="float64")
+        if f"{HDFC_OPENING_PREFIX}low" in rows.columns:
+            opening_low = pd.to_numeric(rows[f"{HDFC_OPENING_PREFIX}low"], errors="coerce").to_numpy(dtype="float64")
+        else:
+            opening_low = np.full(len(rows), np.nan, dtype="float64")
+        first5_high = rows["first5_high"].to_numpy(dtype="float64")
+        first5_low = rows["first5_low"].to_numpy(dtype="float64")
+        high = np.where(np.isfinite(opening_high), opening_high, first5_high)
+        low = np.where(np.isfinite(opening_low), opening_low, first5_low)
+        high = np.maximum(high, first5_high)
+        low = np.minimum(low, first5_low)
+        return cap_hdfc_open10_extension(rows, high, low)
+    raise ValueError(f"Unknown target_mode: {spec.target_mode}")
+
+
+def train_spec(spec: ModelSpec, train_rows: pd.DataFrame, cols: list[str]) -> Any:
+    if spec.kind == "baseline" or spec.target_mode == "open10_bound":
+        return None
+    model = build_model(spec)
+    fit_kwargs: dict[str, Any] = {}
+    if spec.kind == "ridge" and "sample_half_life" in spec.params:
+        half_life = float(spec.params["sample_half_life"])
+        age = np.arange(len(train_rows) - 1, -1, -1, dtype="float64")
+        fit_kwargs["ridge__sample_weight"] = np.exp(-np.log(2.0) * age / half_life)
+    model.fit(train_rows[cols], target_matrix(spec, train_rows), **fit_kwargs)
+    return model
+
+
+def spec_uses_hdfc_open10(spec: ModelSpec) -> bool:
+    if spec.target_mode in {"open10_bound", "open10_delta", "open10_bound_dist"}:
+        return True
+    members: list[str] = []
+    if isinstance(spec.params.get("members"), list):
+        members.extend(spec.params["members"])
+    if isinstance(spec.params.get("high_members"), list):
+        members.extend(spec.params["high_members"])
+    if isinstance(spec.params.get("low_members"), list):
+        members.extend(spec.params["low_members"])
+    return any(name.startswith("hdfc_open10_") for name in members)
+
+
+def avg_high_low_rmse(rows: pd.DataFrame, pred_high: np.ndarray, pred_low: np.ndarray) -> tuple[float, float, float]:
+    high_err = rows["target_high"].to_numpy(dtype="float64") - np.asarray(pred_high, dtype="float64")
+    low_err = rows["target_low"].to_numpy(dtype="float64") - np.asarray(pred_low, dtype="float64")
+    high_rmse = float(np.sqrt(np.mean(high_err * high_err)))
+    low_rmse = float(np.sqrt(np.mean(low_err * low_err)))
+    return (high_rmse + low_rmse) / 2.0, high_rmse, low_rmse
+
+
+def avg_high_low_mae(rows: pd.DataFrame, pred_high: np.ndarray, pred_low: np.ndarray) -> tuple[float, float, float]:
+    high_mae = float(mean_absolute_error(rows["target_high"], pred_high))
+    low_mae = float(mean_absolute_error(rows["target_low"], pred_low))
+    return (high_mae + low_mae) / 2.0, high_mae, low_mae
+
+
+def stable_fold_rows(valid: pd.DataFrame, fold_count: int = STABLE_FOLD_COUNT) -> list[pd.DataFrame]:
+    if valid.empty:
+        return []
+    fold_size = max(1, len(valid) // fold_count)
+    folds: list[pd.DataFrame] = []
+    start = max(0, len(valid) - (fold_size * fold_count))
+    for i in range(start, len(valid), fold_size):
+        fold = valid.iloc[i:i + fold_size].copy().reset_index(drop=True)
+        if not fold.empty:
+            folds.append(fold)
+    return folds[-fold_count:]
+
+
+def evaluate_symbol(
+    frame: pd.DataFrame,
+    test_size: int,
+    valid_size: int,
+    seed: int,
+    selection_mode: str,
+) -> tuple[SymbolResult, pd.DataFrame, pd.DataFrame, dict[str, Any]]:
+    symbol = str(frame["symbol"].iloc[0])
+    labeled = frame.dropna(
+        subset=[
+            "target_date",
+            "target_high",
+            "target_low",
+            "first5_open",
+            "first5_high",
+            "first5_low",
+            "first5_close",
+            "target_high_delta_from_first5_close",
+            "target_low_delta_from_first5_close",
+        ]
+    ).reset_index(drop=True)
+    required = test_size + valid_size + 100
+    if len(labeled) < required:
+        raise RuntimeError(f"{symbol}: need at least {required} labeled rows, found {len(labeled)}")
+
+    test = labeled.tail(test_size).copy().reset_index(drop=True)
+    prior = labeled.iloc[:-test_size].copy().reset_index(drop=True)
+    valid = prior.tail(valid_size).copy().reset_index(drop=True)
+    train = prior.iloc[:-valid_size].copy().reset_index(drop=True)
+    if not (train["date"].max() < valid["date"].min() < valid["date"].max() < test["date"].min()):
+        raise RuntimeError(f"{symbol}: chronological split failed")
+
+    cols = feature_columns(labeled)
+    if symbol == "HDFCBANK":
+        cols = hdfc_fast_feature_columns(cols)
+    final_train = pd.concat([train, valid], ignore_index=True)
+    cols = [c for c in cols if pd.to_numeric(final_train[c], errors="coerce").notna().mean() >= 0.05]
+
+    has_hdfc_open10 = symbol == "HDFCBANK" and {f"{HDFC_OPENING_PREFIX}high", f"{HDFC_OPENING_PREFIX}low"}.issubset(labeled.columns)
+    specs = hdfc_open10_candidate_specs(seed) if has_hdfc_open10 else candidate_specs(seed)
+    stable_mode = bool(not has_hdfc_open10 and selection_mode == "validation_rmse_stable")
+    fixed_symbol_mode = bool(not has_hdfc_open10 and selection_mode == "fixed_symbol_map_v1")
+    if stable_mode:
+        specs = [spec for spec in specs if spec.name in STABLE_CANDIDATE_NAMES]
+    if fixed_symbol_mode:
+        fixed_name = STABLE_SYMBOL_MODEL_MAP.get(symbol)
+        if fixed_name is None:
+            raise ValueError(f"No fixed symbol model configured for {symbol}")
+        specs = [spec for spec in specs if spec.name == fixed_name]
+    force_fixed_mode = bool(not has_hdfc_open10 and selection_mode.startswith("force_"))
+    fixed_guard_mode = bool(
+        not has_hdfc_open10
+        and selection_mode not in {"validation_rmse", "validation_rmse_stable", "fixed_symbol_map_v1"}
+        and not force_fixed_mode
+    )
+    if fixed_guard_mode:
+        fixed_name = selection_mode.removeprefix("fixed_")
+        guard_names = {
+            fixed_name,
+            "extra_trees_delta_d3",
+            "bound_extra_trees_d5",
+            "bound_random_forest_d3",
+        }
+        specs = [spec for spec in specs if spec.name in guard_names]
+        note(f"{symbol}: checking fixed {fixed_name} against {len(specs) - 1} fast guard candidates")
+    if symbol == "HDFCBANK" and {f"{HDFC_OPENING_PREFIX}high", f"{HDFC_OPENING_PREFIX}low"}.issubset(labeled.columns):
+        note(f"{symbol}: using {len(specs)} fast first-{HDFC_OPENING_MINUTES}-minute candidates and {len(cols)} features")
+    best: tuple[float, ModelSpec, Any] | None = None
+    candidate_records: list[dict[str, Any]] = []
+    candidate_rows = []
+    recent_valid = valid.tail(min(RECENT_VALID_WINDOW, len(valid))).copy().reset_index(drop=True)
+    stable_folds = stable_fold_rows(valid)
+    for spec in specs:
+        model = train_spec(spec, train, cols)
+        if spec.target_mode == "baseline":
+            valid_high, valid_low = baseline_high_low(spec, valid)
+            recent_high, recent_low = baseline_high_low(spec, recent_valid)
+        else:
+            valid_high, valid_low = predict_spec_high_low(spec, model, valid, cols)
+            recent_high, recent_low = predict_spec_high_low(spec, model, recent_valid, cols)
+        valid_avg_rmse, valid_high_rmse, valid_low_rmse = avg_high_low_rmse(valid, valid_high, valid_low)
+        valid_avg_mae, valid_high_mae, valid_low_mae = avg_high_low_mae(valid, valid_high, valid_low)
+        recent_valid_avg_rmse, recent_valid_high_rmse, recent_valid_low_rmse = avg_high_low_rmse(
+            recent_valid,
+            recent_high,
+            recent_low,
+        )
+        fold_scores: list[float] = []
+        for fold in stable_folds:
+            if spec.target_mode == "baseline":
+                fold_high, fold_low = baseline_high_low(spec, fold)
+            else:
+                fold_high, fold_low = predict_spec_high_low(spec, model, fold, cols)
+            fold_scores.append(avg_high_low_rmse(fold, fold_high, fold_low)[0])
+        stable_fold_mean_rmse = float(np.mean(fold_scores)) if fold_scores else valid_avg_rmse
+        stable_fold_std_rmse = float(np.std(fold_scores)) if fold_scores else 0.0
+        if stable_mode:
+            selection_score = stable_fold_mean_rmse + (STABLE_FOLD_STD_WEIGHT * stable_fold_std_rmse)
+        else:
+            selection_score = ((1.0 - RECENT_VALID_WEIGHT) * valid_avg_rmse) + (RECENT_VALID_WEIGHT * recent_valid_avg_rmse)
+        candidate_records.append(
+            {
+                "spec": spec,
+                "valid_high": valid_high,
+                "valid_low": valid_low,
+                "recent_high": recent_high,
+                "recent_low": recent_low,
+                "valid_avg_rmse": valid_avg_rmse,
+                "valid_high_rmse": valid_high_rmse,
+                "valid_low_rmse": valid_low_rmse,
+                "recent_valid_avg_rmse": recent_valid_avg_rmse,
+                "recent_valid_high_rmse": recent_valid_high_rmse,
+                "recent_valid_low_rmse": recent_valid_low_rmse,
+                "stable_fold_mean_rmse": stable_fold_mean_rmse,
+                "stable_fold_std_rmse": stable_fold_std_rmse,
+                "selection_score": selection_score,
+            }
+        )
+        candidate_rows.append(
+            {
+                "symbol": symbol,
+                "model_name": spec.name,
+                "target_mode": spec.target_mode,
+                "valid_avg_rmse": valid_avg_rmse,
+                "valid_high_rmse": valid_high_rmse,
+                "valid_low_rmse": valid_low_rmse,
+                "recent_valid_avg_rmse": recent_valid_avg_rmse,
+                "recent_valid_high_rmse": recent_valid_high_rmse,
+                "recent_valid_low_rmse": recent_valid_low_rmse,
+                "stable_fold_mean_rmse": stable_fold_mean_rmse,
+                "stable_fold_std_rmse": stable_fold_std_rmse,
+                "selection_score": selection_score,
+                "valid_avg_mae": valid_avg_mae,
+                "valid_high_mae": valid_high_mae,
+                "valid_low_mae": valid_low_mae,
+            }
+        )
+        if best is None or selection_score < best[0]:
+            best = (selection_score, spec, model)
+    if best is None:
+        raise RuntimeError(f"{symbol}: no model candidate completed")
+
+    sorted_records = sorted(candidate_records, key=lambda r: float(r["valid_avg_rmse"]))
+    if not has_hdfc_open10 and not fixed_guard_mode and not stable_mode:
+        max_blend_members = 8
+        for top_k in range(2, min(max_blend_members, len(sorted_records)) + 1):
+            members = sorted_records[:top_k]
+            valid_high = np.mean(np.vstack([r["valid_high"] for r in members]), axis=0)
+            valid_low = np.mean(np.vstack([r["valid_low"] for r in members]), axis=0)
+            valid_avg_rmse, valid_high_rmse, valid_low_rmse = avg_high_low_rmse(valid, valid_high, valid_low)
+            valid_avg_mae, valid_high_mae, valid_low_mae = avg_high_low_mae(valid, valid_high, valid_low)
+            recent_high = np.mean(np.vstack([r["recent_high"] for r in members]), axis=0)
+            recent_low = np.mean(np.vstack([r["recent_low"] for r in members]), axis=0)
+            recent_valid_avg_rmse, recent_valid_high_rmse, recent_valid_low_rmse = avg_high_low_rmse(
+                recent_valid,
+                recent_high,
+                recent_low,
+            )
+            selection_score = ((1.0 - RECENT_VALID_WEIGHT) * valid_avg_rmse) + (RECENT_VALID_WEIGHT * recent_valid_avg_rmse)
+            member_names = [r["spec"].name for r in members]
+            blend_spec = ModelSpec(
+                name=f"blend_top{top_k}_validation",
+                kind="blend",
+                params={"top_k": top_k, "members": member_names},
+                target_mode="blend",
+            )
+            candidate_rows.append(
+                {
+                    "symbol": symbol,
+                    "model_name": blend_spec.name,
+                    "target_mode": blend_spec.target_mode,
+                    "valid_avg_rmse": valid_avg_rmse,
+                    "valid_high_rmse": valid_high_rmse,
+                    "valid_low_rmse": valid_low_rmse,
+                    "recent_valid_avg_rmse": recent_valid_avg_rmse,
+                    "recent_valid_high_rmse": recent_valid_high_rmse,
+                    "recent_valid_low_rmse": recent_valid_low_rmse,
+                    "selection_score": selection_score,
+                    "valid_avg_mae": valid_avg_mae,
+                    "valid_high_mae": valid_high_mae,
+                    "valid_low_mae": valid_low_mae,
+                }
+            )
+            if best is None or selection_score < best[0]:
+                best = (selection_score, blend_spec, None)
+
+        by_high = sorted(candidate_records, key=lambda r: float(r["valid_high_rmse"]))
+        by_low = sorted(candidate_records, key=lambda r: float(r["valid_low_rmse"]))
+        max_leg_members = min(max_blend_members, len(candidate_records))
+        for high_k in range(1, max_leg_members + 1):
+            for low_k in range(1, max_leg_members + 1):
+                high_members = by_high[:high_k]
+                low_members = by_low[:low_k]
+                valid_high = np.mean(np.vstack([r["valid_high"] for r in high_members]), axis=0)
+                valid_low = np.mean(np.vstack([r["valid_low"] for r in low_members]), axis=0)
+                valid_avg_rmse, valid_high_rmse, valid_low_rmse = avg_high_low_rmse(valid, valid_high, valid_low)
+                valid_avg_mae, valid_high_mae, valid_low_mae = avg_high_low_mae(valid, valid_high, valid_low)
+                recent_high = np.mean(np.vstack([r["recent_high"] for r in high_members]), axis=0)
+                recent_low = np.mean(np.vstack([r["recent_low"] for r in low_members]), axis=0)
+                recent_valid_avg_rmse, recent_valid_high_rmse, recent_valid_low_rmse = avg_high_low_rmse(
+                    recent_valid,
+                    recent_high,
+                    recent_low,
+                )
+                selection_score = ((1.0 - RECENT_VALID_WEIGHT) * valid_avg_rmse) + (RECENT_VALID_WEIGHT * recent_valid_avg_rmse)
+                high_names = [r["spec"].name for r in high_members]
+                low_names = [r["spec"].name for r in low_members]
+                leg_spec = ModelSpec(
+                    name=f"leg_blend_h{high_k}_l{low_k}_validation",
+                    kind="leg_blend",
+                    params={"high_members": high_names, "low_members": low_names},
+                    target_mode="leg_blend",
+                )
+                candidate_rows.append(
+                    {
+                        "symbol": symbol,
+                        "model_name": leg_spec.name,
+                        "target_mode": leg_spec.target_mode,
+                        "valid_avg_rmse": valid_avg_rmse,
+                        "valid_high_rmse": valid_high_rmse,
+                        "valid_low_rmse": valid_low_rmse,
+                        "recent_valid_avg_rmse": recent_valid_avg_rmse,
+                        "recent_valid_high_rmse": recent_valid_high_rmse,
+                        "recent_valid_low_rmse": recent_valid_low_rmse,
+                        "selection_score": selection_score,
+                        "valid_avg_mae": valid_avg_mae,
+                        "valid_high_mae": valid_high_mae,
+                        "valid_low_mae": valid_low_mae,
+                    }
+                )
+                if best is None or selection_score < best[0]:
+                    best = (selection_score, leg_spec, None)
+
+    if has_hdfc_open10:
+        fixed_name = "hdfc_open10_recency_ridge_delta_a10000_hl100"
+        best_spec = next(spec for spec in specs if spec.name == fixed_name)
+        fixed_row = next(row for row in candidate_rows if row["model_name"] == fixed_name)
+        best_valid_rmse = float(fixed_row["valid_avg_rmse"])
+    elif force_fixed_mode:
+        fixed_name = selection_mode.removeprefix("force_")
+        matches = [spec for spec in specs if spec.name == fixed_name]
+        if not matches:
+            raise ValueError(f"Unknown forced selection model: {fixed_name}")
+        best_spec = matches[0]
+        best_valid_rmse = float(next(row for row in candidate_rows if row["model_name"] == fixed_name)["valid_avg_rmse"])
+    elif selection_mode in {"validation_rmse", "validation_rmse_stable"}:
+        _, best_spec, _ = best
+    elif fixed_symbol_mode:
+        best_spec = specs[0]
+    else:
+        fixed_name = selection_mode.removeprefix("fixed_")
+        matches = [spec for spec in specs if spec.name == fixed_name]
+        if not matches:
+            raise ValueError(f"Unknown fixed selection model: {fixed_name}")
+        fixed_row = next(row for row in candidate_rows if row["model_name"] == fixed_name)
+        fixed_valid_rmse = float(fixed_row["valid_avg_rmse"])
+        if best is not None and fixed_valid_rmse > float(best[0]) * FIXED_SELECTION_MAX_VALID_RATIO:
+            note(
+                f"{symbol}: fixed {fixed_name} valid RMSE={fixed_valid_rmse:.4f} trails "
+                f"{best[1].name} valid RMSE={best[0]:.4f}; using validation winner"
+            )
+            _, best_spec, _ = best
+        else:
+            best_spec = matches[0]
+            best_valid_rmse = fixed_valid_rmse
+    baseline_records = [r for r in candidate_records if r["spec"].target_mode == "baseline"]
+    best_baseline_recent = min(baseline_records, key=lambda r: float(r["recent_valid_avg_rmse"])) if baseline_records else None
+    if (
+        selection_mode in {"validation_rmse", "validation_rmse_stable"}
+        and best_baseline_recent is not None
+        and best_spec.target_mode != "baseline"
+    ):
+        chosen_recent = next((r for r in candidate_records if r["spec"].name == best_spec.name), None)
+        if chosen_recent is not None and float(chosen_recent["recent_valid_avg_rmse"]) > float(best_baseline_recent["recent_valid_avg_rmse"]) * BASELINE_RECENT_GUARD_RATIO:
+            note(
+                f"{symbol}: switching from {best_spec.name} to recent baseline guard "
+                f"{best_baseline_recent['spec'].name} because recent-valid RMSE "
+                f"{chosen_recent['recent_valid_avg_rmse']:.4f} vs {best_baseline_recent['recent_valid_avg_rmse']:.4f}"
+            )
+            best_spec = best_baseline_recent["spec"]
+    best_valid_rmse = float(next(row for row in candidate_rows if row["model_name"] == best_spec.name)["valid_avg_rmse"])
+    best_valid_mae = float(next(row for row in candidate_rows if row["model_name"] == best_spec.name)["valid_avg_mae"])
+    if best_spec.kind == "blend":
+        member_names = best_spec.params["members"]
+        member_specs = [next(spec for spec in specs if spec.name == name) for name in member_names]
+        final_model = []
+        test_high_parts = []
+        test_low_parts = []
+        for member_spec in member_specs:
+            member_model = train_spec(member_spec, final_train, cols)
+            member_high, member_low = predict_spec_high_low(member_spec, member_model, test, cols)
+            final_model.append({"spec": asdict(member_spec), "model": member_model})
+            test_high_parts.append(member_high)
+            test_low_parts.append(member_low)
+        pred_high = np.mean(np.vstack(test_high_parts), axis=0)
+        pred_low = np.mean(np.vstack(test_low_parts), axis=0)
+    elif best_spec.kind == "leg_blend":
+        high_specs = [next(spec for spec in specs if spec.name == name) for name in best_spec.params["high_members"]]
+        low_specs = [next(spec for spec in specs if spec.name == name) for name in best_spec.params["low_members"]]
+        final_model = {"high_members": [], "low_members": []}
+        test_high_parts = []
+        test_low_parts = []
+        for member_spec in high_specs:
+            member_model = train_spec(member_spec, final_train, cols)
+            member_high, _ = predict_spec_high_low(member_spec, member_model, test, cols)
+            final_model["high_members"].append({"spec": asdict(member_spec), "model": member_model})
+            test_high_parts.append(member_high)
+        for member_spec in low_specs:
+            member_model = train_spec(member_spec, final_train, cols)
+            _, member_low = predict_spec_high_low(member_spec, member_model, test, cols)
+            final_model["low_members"].append({"spec": asdict(member_spec), "model": member_model})
+            test_low_parts.append(member_low)
+        pred_high = np.mean(np.vstack(test_high_parts), axis=0)
+        pred_low = np.mean(np.vstack(test_low_parts), axis=0)
+    elif best_spec.target_mode == "baseline":
+        final_model = None
+        pred_high, pred_low = baseline_high_low(best_spec, test)
+    else:
+        final_model = train_spec(best_spec, final_train, cols)
+        pred_high, pred_low = predict_spec_high_low(best_spec, final_model, test, cols)
+    test_avg_rmse, test_high_rmse, test_low_rmse = avg_high_low_rmse(test, pred_high, pred_low)
+    test_avg, test_high_mae, test_low_mae = avg_high_low_mae(test, pred_high, pred_low)
+    close_base_high = test["first5_close"].to_numpy(dtype="float64")
+    close_base_low = test["first5_close"].to_numpy(dtype="float64")
+    close_avg_rmse, close_high_rmse, close_low_rmse = avg_high_low_rmse(test, close_base_high, close_base_low)
+    close_avg, close_high_mae, close_low_mae = avg_high_low_mae(test, close_base_high, close_base_low)
+    persist_avg_rmse, persist_high_rmse, persist_low_rmse = avg_high_low_rmse(
+        test,
+        test["high"].to_numpy(dtype="float64"),
+        test["low"].to_numpy(dtype="float64"),
+    )
+    persist_avg, persist_high_mae, persist_low_mae = avg_high_low_mae(
+        test,
+        test["high"].to_numpy(dtype="float64"),
+        test["low"].to_numpy(dtype="float64"),
+    )
+    first5_bound_avg_rmse, first5_bound_high_rmse, first5_bound_low_rmse = avg_high_low_rmse(
+        test,
+        test["first5_high"].to_numpy(dtype="float64"),
+        test["first5_low"].to_numpy(dtype="float64"),
+    )
+    first5_bound_avg, first5_bound_high_mae, first5_bound_low_mae = avg_high_low_mae(
+        test,
+        test["first5_high"].to_numpy(dtype="float64"),
+        test["first5_low"].to_numpy(dtype="float64"),
+    )
+
+    preds = test[
+        [
+            "date",
+            "target_date",
+            "symbol",
+            "close",
+            "high",
+            "low",
+            "first5_open",
+            "first5_high",
+            "first5_low",
+            "first5_close",
+            "target_high",
+            "target_low",
+        ]
+    ].copy()
+    preds["pred_high"] = pred_high
+    preds["pred_low"] = pred_low
+    preds["sq_err_high"] = (preds["target_high"] - preds["pred_high"]) ** 2
+    preds["sq_err_low"] = (preds["target_low"] - preds["pred_low"]) ** 2
+    preds["abs_err_high"] = (preds["target_high"] - preds["pred_high"]).abs()
+    preds["abs_err_low"] = (preds["target_low"] - preds["pred_low"]).abs()
+    preds["first5_close_naive_sq_err_high"] = (preds["target_high"] - preds["first5_close"]) ** 2
+    preds["first5_close_naive_sq_err_low"] = (preds["target_low"] - preds["first5_close"]) ** 2
+    preds["first5_close_naive_abs_err_high"] = (preds["target_high"] - preds["first5_close"]).abs()
+    preds["first5_close_naive_abs_err_low"] = (preds["target_low"] - preds["first5_close"]).abs()
+    preds["first5_bound_naive_sq_err_high"] = (preds["target_high"] - preds["first5_high"]) ** 2
+    preds["first5_bound_naive_sq_err_low"] = (preds["target_low"] - preds["first5_low"]) ** 2
+    preds["first5_bound_naive_abs_err_high"] = (preds["target_high"] - preds["first5_high"]).abs()
+    preds["first5_bound_naive_abs_err_low"] = (preds["target_low"] - preds["first5_low"]).abs()
+    preds["persistence_naive_sq_err_high"] = (preds["target_high"] - preds["high"]) ** 2
+    preds["persistence_naive_sq_err_low"] = (preds["target_low"] - preds["low"]) ** 2
+    preds["persistence_naive_abs_err_high"] = (preds["target_high"] - preds["high"]).abs()
+    preds["persistence_naive_abs_err_low"] = (preds["target_low"] - preds["low"]).abs()
+
+    latest_input = labeled.tail(1).copy().reset_index(drop=True)
+    all_labeled = labeled.iloc[:-1].copy().reset_index(drop=True)
+    if best_spec.kind == "blend":
+        latest_model = []
+        latest_high_parts = []
+        latest_low_parts = []
+        member_names = best_spec.params["members"]
+        member_specs = [next(spec for spec in specs if spec.name == name) for name in member_names]
+        for member_spec in member_specs:
+            member_model = train_spec(member_spec, all_labeled, cols)
+            member_high, member_low = predict_spec_high_low(member_spec, member_model, latest_input, cols)
+            latest_model.append({"spec": asdict(member_spec), "model": member_model})
+            latest_high_parts.append(member_high)
+            latest_low_parts.append(member_low)
+        latest_high = np.mean(np.vstack(latest_high_parts), axis=0)
+        latest_low = np.mean(np.vstack(latest_low_parts), axis=0)
+    elif best_spec.kind == "leg_blend":
+        latest_model = {"high_members": [], "low_members": []}
+        latest_high_parts = []
+        latest_low_parts = []
+        high_specs = [next(spec for spec in specs if spec.name == name) for name in best_spec.params["high_members"]]
+        low_specs = [next(spec for spec in specs if spec.name == name) for name in best_spec.params["low_members"]]
+        for member_spec in high_specs:
+            member_model = train_spec(member_spec, all_labeled, cols)
+            member_high, _ = predict_spec_high_low(member_spec, member_model, latest_input, cols)
+            latest_model["high_members"].append({"spec": asdict(member_spec), "model": member_model})
+            latest_high_parts.append(member_high)
+        for member_spec in low_specs:
+            member_model = train_spec(member_spec, all_labeled, cols)
+            _, member_low = predict_spec_high_low(member_spec, member_model, latest_input, cols)
+            latest_model["low_members"].append({"spec": asdict(member_spec), "model": member_model})
+            latest_low_parts.append(member_low)
+        latest_high = np.mean(np.vstack(latest_high_parts), axis=0)
+        latest_low = np.mean(np.vstack(latest_low_parts), axis=0)
+    elif best_spec.target_mode == "baseline":
+        latest_model = None
+        latest_high, latest_low = baseline_high_low(best_spec, latest_input)
+    else:
+        latest_model = train_spec(best_spec, all_labeled, cols)
+        latest_high, latest_low = predict_spec_high_low(best_spec, latest_model, latest_input, cols)
+    latest = pd.DataFrame(
+        {
+            "symbol": [symbol],
+            "latest_input_date": [latest_input["date"].iloc[0]],
+            "forecast_for": [
+                f"{latest_input['target_date'].iloc[0].date().isoformat()} "
+                f"after first {HDFC_OPENING_MINUTES} minutes"
+                if symbol == "HDFCBANK" and spec_uses_hdfc_open10(best_spec)
+                else f"{latest_input['target_date'].iloc[0].date().isoformat()} after first 5 minutes"
+            ],
+            "pred_high": [float(latest_high[0])],
+            "pred_low": [float(latest_low[0])],
+            "model_name": [best_spec.name],
+        }
+    )
+
+    result = SymbolResult(
+        symbol=symbol,
+        model_name=best_spec.name,
+        feature_count=len(cols),
+        n_train=len(train),
+        n_valid=len(valid),
+        n_test=len(test),
+        train_start=train["date"].min().date().isoformat(),
+        train_end=train["date"].max().date().isoformat(),
+        valid_start=valid["date"].min().date().isoformat(),
+        valid_end=valid["date"].max().date().isoformat(),
+        test_start=test["date"].min().date().isoformat(),
+        test_end=test["date"].max().date().isoformat(),
+        valid_avg_rmse=float(best_valid_rmse),
+        test_avg_rmse=float(test_avg_rmse),
+        test_high_rmse=float(test_high_rmse),
+        test_low_rmse=float(test_low_rmse),
+        valid_avg_mae=float(best_valid_mae),
+        test_avg_mae=float(test_avg),
+        test_high_mae=float(test_high_mae),
+        test_low_mae=float(test_low_mae),
+        close_naive_avg_rmse=float(close_avg_rmse),
+        close_naive_high_rmse=float(close_high_rmse),
+        close_naive_low_rmse=float(close_low_rmse),
+        close_naive_avg_mae=float(close_avg),
+        close_naive_high_mae=float(close_high_mae),
+        close_naive_low_mae=float(close_low_mae),
+        persistence_naive_avg_rmse=float(persist_avg_rmse),
+        persistence_naive_high_rmse=float(persist_high_rmse),
+        persistence_naive_low_rmse=float(persist_low_rmse),
+        first5_bound_naive_avg_rmse=float(first5_bound_avg_rmse),
+        first5_bound_naive_high_rmse=float(first5_bound_high_rmse),
+        first5_bound_naive_low_rmse=float(first5_bound_low_rmse),
+        persistence_naive_avg_mae=float(persist_avg),
+        persistence_naive_high_mae=float(persist_high_mae),
+        persistence_naive_low_mae=float(persist_low_mae),
+        first5_bound_naive_avg_mae=float(first5_bound_avg),
+        first5_bound_naive_high_mae=float(first5_bound_high_mae),
+        first5_bound_naive_low_mae=float(first5_bound_low_mae),
+        close_naive_rmse_improvement_rupees=float(close_avg_rmse - test_avg_rmse),
+        persistence_naive_rmse_improvement_rupees=float(persist_avg_rmse - test_avg_rmse),
+        first5_bound_naive_rmse_improvement_rupees=float(first5_bound_avg_rmse - test_avg_rmse),
+        close_naive_improvement_rupees=float(close_avg - test_avg),
+        persistence_naive_improvement_rupees=float(persist_avg - test_avg),
+        first5_bound_naive_improvement_rupees=float(first5_bound_avg - test_avg),
+        latest_input_date=latest_input["date"].iloc[0].date().isoformat(),
+        target_date_latest_input=(
+            f"{latest_input['target_date'].iloc[0].date().isoformat()} "
+            f"after first {HDFC_OPENING_MINUTES} minutes"
+            if symbol == "HDFCBANK" and spec_uses_hdfc_open10(best_spec)
+            else f"{latest_input['target_date'].iloc[0].date().isoformat()} after first 5 minutes"
+        ),
+        latest_pred_high=float(latest_high[0]),
+        latest_pred_low=float(latest_low[0]),
+    )
+
+    model_payload = {
+        "symbol": symbol,
+        "spec": asdict(best_spec),
+        "feature_columns": cols,
+        "model": latest_model,
+        "result": asdict(result),
+    }
+    return result, preds, latest, {"model": model_payload, "candidates": candidate_rows}
+
+
+def build_report(results: list[SymbolResult], aggregate: dict[str, Any]) -> str:
+    lines = [
+        "# Stock T+1 High/Low Forecaster",
+        "",
+        "Target: next trading row high and low for the selected stock universe.",
+        f"Forecast timestamp: most symbols forecast after the first 5 intraday minutes; HDFCBANK can optionally use the first {HDFC_OPENING_MINUTES} intraday minutes of the T+1 target day.",
+        "",
+        "Leakage controls:",
+        "- Row date t forecasts target_date t+1 using prior-day daily/exogenous features plus configured opening-window target-day features.",
+        "- Targets are exactly `high.shift(-1)` and `low.shift(-1)` inside each symbol.",
+        "- Forecast-day calendar/session features are date-only and known before the opening window.",
+        "- Target-day first5 features are built only from the first five 1m rows; no later bars, full-day close, or final high/low are used for standard symbols.",
+        f"- HDFCBANK adds first-{HDFC_OPENING_MINUTES}-minute 1m features and candidates; no target-day data after that opening window is used.",
+        "- External macro, institutional-flow, and options panels are merged as prior-day/as-of features only.",
+        "- NSE corporate-announcement features count only events timestamped before the forecast cutoff.",
+        "- When HDFCBANK is in the run, its fast path can use base daily/market/peer plus first-10-minute features; processed 5m and exogenous panels remain excluded from that final fit.",
+        "- Predictions are clamped so forecast high is at least first5 high and forecast low is at most first5 low.",
+        "- The final labeled row per symbol is used only for the latest opening-window forecast and is not used to train that latest forecast.",
+        f"- The last {int(aggregate.get('test_rows_per_symbol_min', 0))} labeled rows per symbol are held out as test data.",
+        "",
+        "Aggregate test metrics:",
+        f"- selection mode: {aggregate.get('selection_mode', 'n/a')}",
+        f"- model average RMSE: {aggregate['test_avg_rmse']:.4f} rupees",
+        f"- target RMSE: {aggregate['target_rmse']:.4f} rupees",
+        f"- meets target RMSE: {aggregate['meets_target_rmse']}",
+        f"- first5-close naive average RMSE: {aggregate['close_naive_avg_rmse']:.4f} rupees",
+        f"- improvement over first5-close naive RMSE: {aggregate['close_naive_rmse_improvement_rupees']:.4f} rupees",
+        f"- first5-bound naive average RMSE: {aggregate['first5_bound_naive_avg_rmse']:.4f} rupees",
+        f"- improvement over first5-bound naive RMSE: {aggregate['first5_bound_naive_rmse_improvement_rupees']:.4f} rupees",
+        f"- persistence naive average RMSE: {aggregate['persistence_naive_avg_rmse']:.4f} rupees",
+        f"- improvement over persistence naive RMSE: {aggregate['persistence_naive_rmse_improvement_rupees']:.4f} rupees",
+        f"- model average MAE: {aggregate['test_avg_mae']:.4f} rupees",
+        f"- first5-close naive average MAE: {aggregate['close_naive_avg_mae']:.4f} rupees",
+        f"- improvement over first5-close naive: {aggregate['close_naive_improvement_rupees']:.4f} rupees",
+        f"- persistence naive average MAE: {aggregate['persistence_naive_avg_mae']:.4f} rupees",
+        f"- improvement over persistence naive: {aggregate['persistence_naive_improvement_rupees']:.4f} rupees",
+        f"- meets Rs {aggregate['required_improvement_rupees']:.2f} aggregate close-naive improvement: {aggregate['meets_close_naive_rupee_target']}",
+        "",
+        "Per-symbol test metrics:",
+    ]
+    for result in results:
+        lines.extend(
+            [
+                f"## {result.symbol}",
+                f"- model: {result.model_name}",
+                f"- test rows: {result.n_test} ({result.test_start} to {result.test_end})",
+                f"- model average RMSE: {result.test_avg_rmse:.4f}",
+                f"- high RMSE / low RMSE: {result.test_high_rmse:.4f} / {result.test_low_rmse:.4f}",
+                f"- first5-close naive RMSE improvement: {result.close_naive_rmse_improvement_rupees:.4f}",
+                f"- first5-bound naive RMSE improvement: {result.first5_bound_naive_rmse_improvement_rupees:.4f}",
+                f"- persistence naive RMSE improvement: {result.persistence_naive_rmse_improvement_rupees:.4f}",
+                f"- model average MAE: {result.test_avg_mae:.4f}",
+                f"- high MAE / low MAE: {result.test_high_mae:.4f} / {result.test_low_mae:.4f}",
+                f"- close naive improvement: {result.close_naive_improvement_rupees:.4f}",
+                f"- persistence naive improvement: {result.persistence_naive_improvement_rupees:.4f}",
+                f"- latest forecast for {result.target_date_latest_input}: high {result.latest_pred_high:.2f}, low {result.latest_pred_low:.2f}",
+                "",
+            ]
+        )
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def write_outputs(
+    frames: list[pd.DataFrame],
+    results: list[SymbolResult],
+    predictions: list[pd.DataFrame],
+    latest: list[pd.DataFrame],
+    model_payloads: list[dict[str, Any]],
+    candidate_rows: list[dict[str, Any]],
+    required_improvement: float,
+    target_rmse: float,
+    selection_mode: str,
+) -> None:
+    dataset = pd.concat(frames, ignore_index=True)
+    pred_df = pd.concat(predictions, ignore_index=True)
+    latest_df = pd.concat(latest, ignore_index=True)
+    results_df = pd.DataFrame([asdict(r) for r in results])
+    candidates_df = pd.DataFrame(candidate_rows)
+
+    aggregate = {
+        "symbols": [r.symbol for r in results],
+        "target": f"T+1 high and low, computed as high.shift(-1) and low.shift(-1) per symbol. Standard symbols forecast after the first five 1m bars; HDFCBANK can forecast after the first {HDFC_OPENING_MINUTES} 1m bars of the T+1 day.",
+        "test_rows_total": int(sum(r.n_test for r in results)),
+        "test_rows_per_symbol_min": int(min(r.n_test for r in results)),
+        "required_improvement_rupees": float(required_improvement),
+        "target_rmse": float(target_rmse),
+        "selection_mode": selection_mode,
+        "test_avg_rmse": float(results_df["test_avg_rmse"].mean()),
+        "close_naive_avg_rmse": float(results_df["close_naive_avg_rmse"].mean()),
+        "persistence_naive_avg_rmse": float(results_df["persistence_naive_avg_rmse"].mean()),
+        "first5_bound_naive_avg_rmse": float(results_df["first5_bound_naive_avg_rmse"].mean()),
+        "close_naive_rmse_improvement_rupees": float(results_df["close_naive_rmse_improvement_rupees"].mean()),
+        "persistence_naive_rmse_improvement_rupees": float(results_df["persistence_naive_rmse_improvement_rupees"].mean()),
+        "first5_bound_naive_rmse_improvement_rupees": float(results_df["first5_bound_naive_rmse_improvement_rupees"].mean()),
+        "test_avg_mae": float(results_df["test_avg_mae"].mean()),
+        "close_naive_avg_mae": float(results_df["close_naive_avg_mae"].mean()),
+        "persistence_naive_avg_mae": float(results_df["persistence_naive_avg_mae"].mean()),
+        "first5_bound_naive_avg_mae": float(results_df["first5_bound_naive_avg_mae"].mean()),
+        "close_naive_improvement_rupees": float(results_df["close_naive_improvement_rupees"].mean()),
+        "persistence_naive_improvement_rupees": float(results_df["persistence_naive_improvement_rupees"].mean()),
+        "first5_bound_naive_improvement_rupees": float(results_df["first5_bound_naive_improvement_rupees"].mean()),
+    }
+    aggregate["meets_close_naive_rupee_target"] = bool(
+        aggregate["close_naive_improvement_rupees"] >= required_improvement
+        and aggregate["test_rows_per_symbol_min"] >= 400
+    )
+    aggregate["meets_target_rmse"] = bool(
+        aggregate["test_avg_rmse"] <= target_rmse
+        and aggregate["test_rows_per_symbol_min"] >= 400
+    )
+
+    dataset.to_csv(OUTPUT_DIR / "tplus1_high_low_dataset.csv", index=False)
+    pred_df.to_csv(OUTPUT_DIR / "test_predictions.csv", index=False)
+    latest_df.to_csv(OUTPUT_DIR / "latest_forecasts.csv", index=False)
+    results_df.to_csv(OUTPUT_DIR / "metrics_by_symbol.csv", index=False)
+    candidates_df.to_csv(OUTPUT_DIR / "candidate_validation_results.csv", index=False)
+    (OUTPUT_DIR / "summary.json").write_text(
+        json.dumps(
+            {
+                "aggregate": aggregate,
+                "by_symbol": [asdict(r) for r in results],
+                "leakage_note": f"Rows use previous trading day daily/exogenous features, date-only forecast-session features, NSE corporate announcements timestamped before the forecast cutoff, and opening target-day intraday features. Standard symbols use only the first five 1m bars. HDFCBANK may use only the first {HDFC_OPENING_MINUTES} 1m bars. Full-day T+1 close and final high/low are excluded from feature columns.",
+            },
+            indent=2,
+        ),
+        encoding="utf-8",
+    )
+    (OUTPUT_DIR / "report.md").write_text(build_report(results, aggregate), encoding="utf-8")
+    joblib.dump(
+        {
+            "models": model_payloads,
+            "aggregate": aggregate,
+        },
+        OUTPUT_DIR / "stock_high_low_models.joblib",
+    )
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Train T+1 high/low forecasters for selected NSE stocks.")
+    parser.add_argument(
+        "--symbols",
+        default="",
+        help="Comma-separated stock symbols to train. Default is all discovered non-market symbols after exclusions.",
+    )
+    parser.add_argument(
+        "--exclude-symbols",
+        default="",
+        help="Comma-separated stock symbols to exclude from training and peer feature construction.",
+    )
+    parser.add_argument("--test-size", type=int, default=400, help="Held-out test rows per symbol.")
+    parser.add_argument("--valid-size", type=int, default=400, help="Validation rows per symbol for model selection.")
+    parser.add_argument("--min-rupee-improvement", type=float, default=2.0, help="Aggregate close-naive MAE improvement target.")
+    parser.add_argument("--target-rmse", type=float, default=5.0, help="Aggregate T+1 high/low RMSE target.")
+    parser.add_argument(
+        "--selection-mode",
+        choices=[
+            "validation_rmse",
+            "validation_rmse_stable",
+            "fixed_symbol_map_v1",
+            "fixed_extra_trees_delta_d3",
+            "fixed_extra_trees_delta_d5",
+            "fixed_extra_trees_delta_d7",
+            "fixed_ridge_delta_a5000",
+            "force_baseline_first5_bound",
+            "force_baseline_first5_close",
+            "force_baseline_persistence",
+        ],
+        default="force_baseline_first5_bound",
+        help="Select the lowest validation-RMSE candidate per symbol, use a guarded fixed model, or force a baseline-first low-RMSE path.",
+    )
+    parser.add_argument("--seed", type=int, default=RANDOM_SEED)
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    np.random.seed(args.seed)
+    random.seed(args.seed)
+    excluded = {s.strip().upper() for s in args.exclude_symbols.split(",") if s.strip()}
+    available_symbols = discover_target_symbols(excluded_symbols=excluded)
+    if args.symbols.strip():
+        requested = [s.strip().upper() for s in args.symbols.split(",") if s.strip()]
+    else:
+        requested = list(available_symbols)
+    unsupported = [s for s in requested if s not in available_symbols]
+    if unsupported:
+        raise ValueError(f"Unsupported symbols: {unsupported}. Supported: {list(available_symbols)}")
+    selected_symbols = {symbol: available_symbols[symbol] for symbol in requested}
+    note(f"selected stock universe: {', '.join(selected_symbols)}")
+    if args.test_size < 400:
+        raise ValueError("test-size must be at least 400 to satisfy the requested minimum test set.")
+    if args.valid_size < 50:
+        raise ValueError("valid-size should be at least 50 rows.")
+
+    note("building market features")
+    market = build_market_features()
+    note("building market first-5-minute features")
+    market_first5 = build_market_first5_features(selected_symbols)
+    note("building prior-day exogenous features")
+    exogenous = build_prior_day_exogenous_features()
+    note("building HDFCBANK opening-5m feature panel")
+    opening_5m = build_opening_5m_panel(selected_symbols) if "HDFCBANK" in requested else None
+
+    frames: list[pd.DataFrame] = []
+    results: list[SymbolResult] = []
+    predictions: list[pd.DataFrame] = []
+    latest_rows: list[pd.DataFrame] = []
+    model_payloads: list[dict[str, Any]] = []
+    all_candidate_rows: list[dict[str, Any]] = []
+
+    for symbol in requested:
+        note(f"{symbol}: building T+1 frame")
+        frame = build_symbol_frame(symbol, selected_symbols[symbol], market, market_first5, opening_5m, exogenous)
+        result, preds, latest, payload = evaluate_symbol(
+            frame,
+            args.test_size,
+            args.valid_size,
+            args.seed,
+            args.selection_mode,
+        )
+        note(
+            f"{symbol}: {result.model_name}, test RMSE={result.test_avg_rmse:.4f}, "
+            f"close-naive RMSE improvement={result.close_naive_rmse_improvement_rupees:.4f}"
+        )
+        frames.append(frame)
+        results.append(result)
+        predictions.append(preds)
+        latest_rows.append(latest)
+        model_payloads.append(payload["model"])
+        all_candidate_rows.extend(payload["candidates"])
+
+    write_outputs(
+        frames=frames,
+        results=results,
+        predictions=predictions,
+        latest=latest_rows,
+        model_payloads=model_payloads,
+        candidate_rows=all_candidate_rows,
+        required_improvement=args.min_rupee_improvement,
+        target_rmse=args.target_rmse,
+        selection_mode=args.selection_mode,
+    )
+    aggregate_improvement = float(np.mean([r.close_naive_improvement_rupees for r in results]))
+    aggregate_rmse = float(np.mean([r.test_avg_rmse for r in results]))
+    print(build_report(results, {
+        "test_rows_per_symbol_min": int(min(r.n_test for r in results)),
+        "test_avg_rmse": aggregate_rmse,
+        "target_rmse": float(args.target_rmse),
+        "meets_target_rmse": bool(aggregate_rmse <= args.target_rmse),
+        "close_naive_avg_rmse": float(np.mean([r.close_naive_avg_rmse for r in results])),
+        "close_naive_rmse_improvement_rupees": float(np.mean([r.close_naive_rmse_improvement_rupees for r in results])),
+        "first5_bound_naive_avg_rmse": float(np.mean([r.first5_bound_naive_avg_rmse for r in results])),
+        "first5_bound_naive_rmse_improvement_rupees": float(np.mean([r.first5_bound_naive_rmse_improvement_rupees for r in results])),
+        "persistence_naive_avg_rmse": float(np.mean([r.persistence_naive_avg_rmse for r in results])),
+        "persistence_naive_rmse_improvement_rupees": float(np.mean([r.persistence_naive_rmse_improvement_rupees for r in results])),
+        "test_avg_mae": float(np.mean([r.test_avg_mae for r in results])),
+        "close_naive_avg_mae": float(np.mean([r.close_naive_avg_mae for r in results])),
+        "close_naive_improvement_rupees": aggregate_improvement,
+        "first5_bound_naive_avg_mae": float(np.mean([r.first5_bound_naive_avg_mae for r in results])),
+        "first5_bound_naive_improvement_rupees": float(np.mean([r.first5_bound_naive_improvement_rupees for r in results])),
+        "persistence_naive_avg_mae": float(np.mean([r.persistence_naive_avg_mae for r in results])),
+        "persistence_naive_improvement_rupees": float(np.mean([r.persistence_naive_improvement_rupees for r in results])),
+        "required_improvement_rupees": float(args.min_rupee_improvement),
+        "selection_mode": args.selection_mode,
+        "meets_close_naive_rupee_target": bool(aggregate_improvement >= args.min_rupee_improvement),
+    }), end="")
+    note(f"wrote outputs to {OUTPUT_DIR}")
+
+
+if __name__ == "__main__":
+    main()

backend/research_runtime/Code/scripts/data_ingestion/download_corporate_announcements.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+import argparse
+from datetime import datetime
+from pathlib import Path
+
+import pandas as pd
+from nse import NSE
+
+
+def find_project_root(start: Path) -> Path:
+    for path in (start, *start.parents):
+        if (path / "Data").is_dir() and (path / "Alt Data").is_dir():
+            return path
+    raise RuntimeError(f"Could not find project root from {start}")
+
+
+PROJECT_ROOT = find_project_root(Path(__file__).resolve())
+OUT_DIR = PROJECT_ROOT / "Alt Data" / "corporate"
+RAW_DIR = OUT_DIR / "raw" / "announcements"
+PROCESSED_DIR = OUT_DIR / "processed"
+
+
+def normalize_events(rows: list[dict]) -> pd.DataFrame:
+    if not rows:
+        return pd.DataFrame()
+    df = pd.DataFrame(rows)
+    keep = [
+        "symbol",
+        "desc",
+        "attchmntText",
+        "sort_date",
+        "an_dt",
+        "exchdisstime",
+        "seq_id",
+        "attchmntFile",
+        "smIndustry",
+        "hasXbrl",
+    ]
+    for col in keep:
+        if col not in df.columns:
+            df[col] = pd.NA
+    out = df[keep].copy()
+    out["symbol"] = out["symbol"].astype(str).str.upper().str.strip()
+    out["event_ts"] = pd.to_datetime(out["sort_date"], errors="coerce")
+    out["event_date"] = out["event_ts"].dt.normalize()
+    text = (
+        out["desc"].fillna("").astype(str)
+        + " "
+        + out["attchmntText"].fillna("").astype(str)
+    ).str.lower()
+    out["is_results"] = text.str.contains("financial result|audited result|unaudited result|limited review|quarter", regex=True)
+    out["is_board_meeting"] = text.str.contains("board meeting|outcome of board|meeting of board", regex=True)
+    out["is_investor_meet"] = text.str.contains("investor|analyst|conference call|con\\. call", regex=True)
+    out["is_credit_rating"] = text.str.contains("credit rating|rating", regex=True)
+    out["is_press_release"] = text.str.contains("press release|media release", regex=True)
+    out["is_high_impact"] = (
+        out["is_results"]
+        | out["is_board_meeting"]
+        | out["is_credit_rating"]
+        | text.str.contains("dividend|fund raising|merger|acquisition|sale of|penalt|rbi|reserve bank", regex=True)
+    )
+    return out.dropna(subset=["event_ts"]).sort_values(["symbol", "event_ts", "seq_id"]).reset_index(drop=True)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Download NSE corporate announcements for selected symbols.")
+    parser.add_argument("--symbols", required=True, help="Comma-separated NSE symbols.")
+    parser.add_argument("--from-date", default="2018-01-01")
+    parser.add_argument("--to-date", default=datetime.today().strftime("%Y-%m-%d"))
+    args = parser.parse_args()
+
+    symbols = [s.strip().upper() for s in args.symbols.split(",") if s.strip()]
+    from_dt = datetime.strptime(args.from_date, "%Y-%m-%d")
+    to_dt = datetime.strptime(args.to_date, "%Y-%m-%d")
+    RAW_DIR.mkdir(parents=True, exist_ok=True)
+    PROCESSED_DIR.mkdir(parents=True, exist_ok=True)
+
+    nse = NSE(PROJECT_ROOT / "Code" / "artifacts" / "nse_cache")
+    frames: list[pd.DataFrame] = []
+    try:
+        for symbol in symbols:
+            print(f"[corporate-announcements] fetching {symbol}", flush=True)
+            rows = nse.announcements(symbol=symbol, from_date=from_dt, to_date=to_dt)
+            raw = pd.DataFrame(rows)
+            raw.to_csv(RAW_DIR / f"{symbol.lower()}_announcements.csv", index=False)
+            norm = normalize_events(rows)
+            if not norm.empty:
+                frames.append(norm)
+    finally:
+        nse.exit()
+
+    if frames:
+        events = pd.concat(frames, ignore_index=True).sort_values(["symbol", "event_ts", "seq_id"])
+    else:
+        events = pd.DataFrame()
+    events.to_csv(PROCESSED_DIR / "corporate_announcements.csv", index=False)
+    print(f"[corporate-announcements] wrote {len(events)} events to {PROCESSED_DIR / 'corporate_announcements.csv'}")
+
+
+if __name__ == "__main__":
+    main()

backend/research_runtime/Code/scripts/data_ingestion/download_index_options.py

@@ -0,0 +1,297 @@
+import argparse
+import os
+import time
+import threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import date, datetime, timedelta
+from io import BytesIO
+from pathlib import Path
+from zipfile import BadZipFile, ZipFile
+
+import pandas as pd
+import requests
+
+
+def find_project_root() -> Path:
+    env_root = os.environ.get("FORECASTING_PROJECT_ROOT")
+    if env_root:
+        return Path(env_root).expanduser().resolve()
+    for path in [Path(__file__).resolve(), *Path(__file__).resolve().parents]:
+        if (path / "Code").is_dir() and (path / "Data").is_dir() and (path / "Alt Data").is_dir():
+            return path
+    raise RuntimeError("Could not locate project root containing Code, Data, and Alt Data.")
+
+
+PROJECT_ROOT = find_project_root()
+OUT_DIR = PROJECT_ROOT / "Alt Data" / "options" / "raw"
+RETRY_SEC = 1.0
+MAX_RETRIES = 2
+MAX_WORKERS = 8
+UDIFF_START = date(2024, 7, 8)
+
+TARGETS = [
+    {
+        "symbol": "NIFTY",
+        "label": "NIFTY 50",
+        "filename": "NIFTY_50_options.csv",
+        "legacy_instrument": "OPTIDX",
+        "udiff_instrument": "IDO",
+    },
+    {
+        "symbol": "BANKNIFTY",
+        "label": "NIFTY BANK",
+        "filename": "NIFTY_BANK_options.csv",
+        "legacy_instrument": "OPTIDX",
+        "udiff_instrument": "IDO",
+    },
+    {
+        "symbol": "HDFCBANK",
+        "label": "HDFC BANK",
+        "filename": "HDFCBANK_options.csv",
+        "legacy_instrument": "OPTSTK",
+        "udiff_instrument": "STO",
+    },
+]
+
+STANDARD_COLUMNS = [
+    "TRADE_DATE",
+    "SYMBOL",
+    "INSTRUMENT",
+    "EXPIRY_DT",
+    "STRIKE_PR",
+    "OPTION_TYP",
+    "OPEN",
+    "HIGH",
+    "LOW",
+    "CLOSE",
+    "SETTLE_PR",
+    "CONTRACTS",
+    "VAL_INLAKH",
+    "OPEN_INT",
+    "CHG_IN_OI",
+    "UNDERLYING",
+]
+
+THREAD_LOCAL = threading.local()
+
+def build_url(d: date) -> str:
+    if d >= UDIFF_START:
+        return (
+            "https://archives.nseindia.com/content/fo/"
+            f"BhavCopy_NSE_FO_0_0_0_{d.strftime('%Y%m%d')}_F_0000.csv.zip"
+        )
+
+    mon = d.strftime("%b").upper()
+    day = d.strftime("%d%b%Y").upper()
+    return (
+        f"https://nsearchives.nseindia.com/content/historical/DERIVATIVES"
+        f"/{d.year}/{mon}/fo{day}bhav.csv.zip"
+    )
+
+
+def normalize_legacy(df: pd.DataFrame, d: date) -> pd.DataFrame:
+    df.columns = df.columns.str.strip()
+    target_map = {target["symbol"]: target["legacy_instrument"] for target in TARGETS}
+    symbols = list(target_map)
+    symbol_series = df["SYMBOL"].astype(str).str.strip()
+    instrument_series = df["INSTRUMENT"].astype(str).str.strip()
+    mask = symbol_series.isin(symbols) & symbol_series.map(target_map).eq(instrument_series)
+    filtered = df.loc[mask].copy()
+    if filtered.empty:
+        return filtered
+
+    filtered["TRADE_DATE"] = d.isoformat()
+    filtered["SYMBOL"] = filtered["SYMBOL"].astype(str).str.strip()
+    filtered["UNDERLYING"] = pd.NA
+    return filtered[
+        [
+            "TRADE_DATE",
+            "SYMBOL",
+            "INSTRUMENT",
+            "EXPIRY_DT",
+            "STRIKE_PR",
+            "OPTION_TYP",
+            "OPEN",
+            "HIGH",
+            "LOW",
+            "CLOSE",
+            "SETTLE_PR",
+            "CONTRACTS",
+            "VAL_INLAKH",
+            "OPEN_INT",
+            "CHG_IN_OI",
+            "UNDERLYING",
+        ]
+    ]
+
+
+def normalize_udiff(df: pd.DataFrame) -> pd.DataFrame:
+    target_map = {target["symbol"]: target["udiff_instrument"] for target in TARGETS}
+    symbol_series = df["TckrSymb"].astype(str).str.strip()
+    instrument_series = df["FinInstrmTp"].astype(str).str.strip()
+    mask = (
+        symbol_series.isin(list(target_map))
+        & instrument_series.eq(symbol_series.map(target_map))
+        & df["OptnTp"].astype(str).str.strip().isin(["CE", "PE"])
+    )
+    filtered = df.loc[mask].copy()
+    if filtered.empty:
+        return filtered
+
+    filtered = filtered.rename(
+        columns={
+            "TradDt": "TRADE_DATE",
+            "TckrSymb": "SYMBOL",
+            "XpryDt": "EXPIRY_DT",
+            "StrkPric": "STRIKE_PR",
+            "OptnTp": "OPTION_TYP",
+            "OpnPric": "OPEN",
+            "HghPric": "HIGH",
+            "LwPric": "LOW",
+            "ClsPric": "CLOSE",
+            "SttlmPric": "SETTLE_PR",
+            "TtlTradgVol": "CONTRACTS",
+            "TtlTrfVal": "VAL_INLAKH",
+            "OpnIntrst": "OPEN_INT",
+            "ChngInOpnIntrst": "CHG_IN_OI",
+            "UndrlygPric": "UNDERLYING",
+        }
+    )
+    filtered["TRADE_DATE"] = pd.to_datetime(filtered["TRADE_DATE"]).dt.strftime("%Y-%m-%d")
+    filtered["EXPIRY_DT"] = pd.to_datetime(filtered["EXPIRY_DT"]).dt.strftime("%d-%b-%Y")
+    filtered["SYMBOL"] = filtered["SYMBOL"].astype(str).str.strip()
+    filtered["INSTRUMENT"] = "OPTIDX"
+    filtered["VAL_INLAKH"] = filtered["VAL_INLAKH"] / 100000.0
+    return filtered[STANDARD_COLUMNS]
+
+
+def get_session() -> requests.Session:
+    session = getattr(THREAD_LOCAL, "session", None)
+    if session is None:
+        session = requests.Session()
+        session.headers.update(
+            {
+                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
+                "Accept-Encoding": "gzip, deflate",
+                "Accept": "*/*",
+                "Connection": "keep-alive",
+            }
+        )
+        THREAD_LOCAL.session = session
+    return session
+
+
+def fetch_day(d: date) -> pd.DataFrame | None:
+    url = build_url(d)
+    session = get_session()
+
+    for attempt in range(1, MAX_RETRIES + 2):
+        try:
+            response = session.get(url, timeout=20)
+            if response.status_code != 200:
+                return None
+
+            with ZipFile(BytesIO(response.content)) as zipped:
+                csv_name = zipped.namelist()[0]
+                df = pd.read_csv(zipped.open(csv_name))
+
+            if d >= UDIFF_START:
+                filtered = normalize_udiff(df)
+            else:
+                filtered = normalize_legacy(df, d)
+
+            return filtered if not filtered.empty else None
+        except (requests.RequestException, BadZipFile, pd.errors.EmptyDataError, OSError):
+            if attempt > MAX_RETRIES:
+                return None
+            time.sleep(RETRY_SEC)
+
+    return None
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Download NSE index option bhavcopy history into Alt Data/options/raw.")
+    parser.add_argument("--start-date", default="2015-01-01", help="Inclusive start date in YYYY-MM-DD format.")
+    parser.add_argument("--end-date", default=date.today().isoformat(), help="Inclusive end date in YYYY-MM-DD format.")
+    return parser.parse_args()
+
+
+def load_existing_rows(path: Path) -> pd.DataFrame:
+    if not path.exists():
+        return pd.DataFrame(columns=STANDARD_COLUMNS)
+    frame = pd.read_csv(path)
+    return frame if not frame.empty else pd.DataFrame(columns=STANDARD_COLUMNS)
+
+
+def main() -> None:
+    args = parse_args()
+    start_date = datetime.strptime(args.start_date, "%Y-%m-%d").date()
+    end_date = datetime.strptime(args.end_date, "%Y-%m-%d").date()
+    OUT_DIR.mkdir(parents=True, exist_ok=True)
+
+    all_chunks = {target["symbol"]: [] for target in TARGETS}
+    trading_days = []
+    current = start_date
+
+    while current <= end_date:
+        if current.weekday() < 5:
+            trading_days.append(current)
+        current += timedelta(days=1)
+
+    print(f"Downloading NSE index options from {start_date} to {end_date}")
+    print(f"Saving files to {OUT_DIR}\n")
+    print(f"Using up to {MAX_WORKERS} parallel workers across {len(trading_days)} trading days\n")
+
+    completed = 0
+    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+        future_to_day = {executor.submit(fetch_day, day): day for day in trading_days}
+
+        for future in as_completed(future_to_day):
+            day = future_to_day[future]
+            chunk = future.result()
+
+            if chunk is None:
+                print(f"  - {day}  (no data / holiday)")
+            else:
+                counts = []
+                for target in TARGETS:
+                    target_chunk = chunk.loc[chunk["SYMBOL"] == target["symbol"]].copy()
+                    if not target_chunk.empty:
+                        all_chunks[target["symbol"]].append(target_chunk)
+                    counts.append(f"{target['symbol']}={len(target_chunk)}")
+                print(f"  + {day}  ({', '.join(counts)})")
+
+            completed += 1
+            if completed % 50 == 0:
+                pct = completed / len(trading_days) * 100
+                print(f"\n[{pct:.0f}%] processed {completed}/{len(trading_days)} trading days\n")
+
+    print()
+    for target in TARGETS:
+        symbol = target["symbol"]
+        output_path = OUT_DIR / target["filename"]
+
+        frames = [load_existing_rows(output_path)]
+        if all_chunks[symbol]:
+            frames.append(pd.concat(all_chunks[symbol], ignore_index=True))
+
+        result = pd.concat(frames, ignore_index=True)
+        if result.empty:
+            print(f"No rows collected for {target['label']} ({symbol}).")
+            continue
+
+        result["TRADE_DATE"] = pd.to_datetime(result["TRADE_DATE"])
+        result["EXPIRY_DT"] = pd.to_datetime(result["EXPIRY_DT"], format="%d-%b-%Y", errors="coerce")
+        result = result.drop_duplicates(
+            subset=["TRADE_DATE", "SYMBOL", "EXPIRY_DT", "STRIKE_PR", "OPTION_TYP"],
+            keep="last",
+        )
+        result = result.sort_values(by=["TRADE_DATE", "EXPIRY_DT", "STRIKE_PR", "OPTION_TYP"], kind="stable")
+        result["TRADE_DATE"] = result["TRADE_DATE"].dt.strftime("%Y-%m-%d")
+        result["EXPIRY_DT"] = result["EXPIRY_DT"].dt.strftime("%d-%b-%Y")
+        result.to_csv(output_path, index=False)
+        print(f"{target['label']}: {len(result):,} rows saved to {output_path}")
+
+
+if __name__ == "__main__":
+    main()

backend/research_runtime/Code/scripts/data_ingestion/download_institutional_flows.py

@@ -0,0 +1,344 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import date, datetime, timedelta
+from io import StringIO
+from pathlib import Path
+
+import pandas as pd
+import requests
+
+
+def find_project_root() -> Path:
+    env_root = os.environ.get("FORECASTING_PROJECT_ROOT")
+    if env_root:
+        return Path(env_root).expanduser().resolve()
+    for path in [Path(__file__).resolve(), *Path(__file__).resolve().parents]:
+        if (path / "Code").is_dir() and (path / "Data").is_dir() and (path / "Alt Data").is_dir():
+            return path
+    raise RuntimeError("Could not locate project root containing Code, Data, and Alt Data.")
+
+
+PROJECT_ROOT = find_project_root()
+ALT_ROOT = PROJECT_ROOT / "Alt Data"
+INSTITUTIONAL_ROOT = ALT_ROOT / "institutional"
+RAW_ROOT = INSTITUTIONAL_ROOT / "raw"
+PROCESSED_ROOT = INSTITUTIONAL_ROOT / "processed"
+
+NSE_BASE = "https://archives.nseindia.com/content/nsccl"
+MONEYCONTROL_HEADERS = {
+    "User-Agent": "Mozilla/5.0",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+}
+NSE_HEADERS = {
+    "User-Agent": "Mozilla/5.0",
+    "Accept": "text/csv,application/json,text/plain,*/*",
+    "Referer": "https://www.nseindia.com/",
+}
+
+PARTICIPANTS = ("FII", "DII")
+MAX_WORKERS = 8
+RETRY_SEC = 1.0
+MAX_RETRIES = 2
+
+MONEYCONTROL_PAGES = {
+    "cash": "https://www.moneycontrol.com/markets/fii-dii-data/cash/",
+    "futures_and_options": "https://www.moneycontrol.com/markets/fii-dii-data/futures-and-options/",
+}
+
+
+def ensure_dir(path: Path) -> None:
+    path.mkdir(parents=True, exist_ok=True)
+
+
+def daterange(start_date: date, end_date: date) -> list[date]:
+    out: list[date] = []
+    current = start_date
+    while current <= end_date:
+        if current.weekday() < 5:
+            out.append(current)
+        current += timedelta(days=1)
+    return out
+
+
+def clean_number(value: object) -> float:
+    if value is None or (isinstance(value, float) and pd.isna(value)):
+        return float("nan")
+    text = str(value).strip()
+    if not text or text in {"-", "--", "nan", "None"}:
+        return float("nan")
+    text = text.replace(",", "")
+    suffix = 1.0
+    upper = text.upper()
+    if upper.endswith("CR"):
+        text = text[:-2]
+    elif upper.endswith("L"):
+        suffix = 100000.0
+        text = text[:-1]
+    elif upper.endswith("K"):
+        suffix = 1000.0
+        text = text[:-1]
+    return float(text) * suffix
+
+
+def clean_columns(columns: list[str]) -> list[str]:
+    out: list[str] = []
+    for col in columns:
+        text = str(col).strip().replace("\t", " ")
+        text = re.sub(r"\s+", " ", text)
+        text = text.lower().replace(" ", "_")
+        out.append(text)
+    return out
+
+
+def build_nse_url(day: date, kind: str) -> str:
+    return f"{NSE_BASE}/fao_participant_{kind}_{day.strftime('%d%m%Y')}.csv"
+
+
+def fetch_participant_file(day: date, kind: str) -> pd.DataFrame | None:
+    url = build_nse_url(day, kind)
+
+    for attempt in range(1, MAX_RETRIES + 2):
+        try:
+            response = requests.get(url, headers=NSE_HEADERS, timeout=20)
+            if response.status_code != 200:
+                return None
+
+            frame = pd.read_csv(StringIO(response.text), skiprows=1)
+            frame = frame.dropna(axis=1, how="all")
+            frame.columns = clean_columns(frame.columns.tolist())
+            if "client_type" not in frame.columns:
+                return None
+
+            frame["client_type"] = frame["client_type"].astype(str).str.strip().str.upper()
+            frame = frame[frame["client_type"].isin(PARTICIPANTS)].copy()
+            if frame.empty:
+                return None
+
+            for col in frame.columns:
+                if col != "client_type":
+                    frame[col] = frame[col].map(clean_number)
+
+            frame["date"] = pd.Timestamp(day)
+            frame["source"] = "nse_archive"
+            frame["report_type"] = kind
+            return frame
+        except (requests.RequestException, pd.errors.ParserError, ValueError):
+            if attempt > MAX_RETRIES:
+                return None
+            time.sleep(RETRY_SEC)
+
+    return None
+
+
+def download_participant_history(start_date: date, end_date: date) -> tuple[pd.DataFrame, pd.DataFrame]:
+    days = daterange(start_date, end_date)
+
+    output: dict[str, list[pd.DataFrame]] = {"oi": [], "vol": []}
+    tasks = []
+    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+        for day in days:
+            for kind in ("oi", "vol"):
+                tasks.append((day, kind, executor.submit(fetch_participant_file, day, kind)))
+
+        completed = 0
+        for day, kind, future in tasks:
+            frame = future.result()
+            completed += 1
+            if frame is not None:
+                output[kind].append(frame)
+            if completed % 250 == 0:
+                print(f"Fetched {completed}/{len(tasks)} participant files")
+
+    oi = pd.concat(output["oi"], ignore_index=True) if output["oi"] else pd.DataFrame()
+    vol = pd.concat(output["vol"], ignore_index=True) if output["vol"] else pd.DataFrame()
+    return oi, vol
+
+
+def extract_next_data_json(html: str) -> dict:
+    match = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', html)
+    if not match:
+        raise ValueError("Could not locate __NEXT_DATA__ payload")
+    return json.loads(match.group(1))
+
+
+def fetch_moneycontrol_page(kind: str) -> pd.DataFrame:
+    url = MONEYCONTROL_PAGES[kind]
+    response = requests.get(url, headers=MONEYCONTROL_HEADERS, timeout=30)
+    response.raise_for_status()
+    payload = extract_next_data_json(response.text)
+    rows = payload["props"]["pageProps"]["FiiDiiData"]["fiiDiiData"]
+    frame = pd.DataFrame(rows)
+    if frame.empty:
+        return frame
+
+    frame["date"] = pd.to_datetime(frame["date"])
+    frame = frame.sort_values("date").reset_index(drop=True)
+    frame["source"] = "moneycontrol_next_data"
+    frame["report_type"] = kind
+
+    for col in frame.columns:
+        if col in {"date", "source", "report_type", "fDate", "graphRecordDate", "graphToolTipRecordDate"}:
+            continue
+        frame[col] = frame[col].map(clean_number)
+    return frame
+
+
+def reshape_participant_data(frame: pd.DataFrame, suffix: str) -> pd.DataFrame:
+    if frame.empty:
+        return pd.DataFrame()
+
+    useful_cols = [
+        "date",
+        "client_type",
+        "future_index_long",
+        "future_index_short",
+        "option_index_call_long",
+        "option_index_call_short",
+        "option_index_put_long",
+        "option_index_put_short",
+    ]
+    base = frame[useful_cols].copy()
+    pivot = base.pivot(index="date", columns="client_type")
+    pivot.columns = [
+        f"{participant.lower()}_{metric}_{suffix}"
+        for metric, participant in pivot.columns.to_flat_index()
+    ]
+    pivot = pivot.reset_index()
+
+    for participant in ("fii", "dii"):
+        fut_long = f"{participant}_future_index_long_{suffix}"
+        fut_short = f"{participant}_future_index_short_{suffix}"
+        call_long = f"{participant}_option_index_call_long_{suffix}"
+        call_short = f"{participant}_option_index_call_short_{suffix}"
+        put_long = f"{participant}_option_index_put_long_{suffix}"
+        put_short = f"{participant}_option_index_put_short_{suffix}"
+
+        if fut_long in pivot.columns and fut_short in pivot.columns:
+            total = pivot[fut_long] + pivot[fut_short]
+            pivot[f"{participant}_index_futures_net_{suffix}"] = pivot[fut_long] - pivot[fut_short]
+            pivot[f"{participant}_index_futures_long_share_{suffix}"] = pivot[fut_long] / total.replace(0, pd.NA)
+
+        if all(col in pivot.columns for col in [call_long, call_short, put_long, put_short]):
+            long_total = pivot[call_long] + pivot[put_long]
+            short_total = pivot[call_short] + pivot[put_short]
+            total = long_total + short_total
+            pivot[f"{participant}_index_options_net_{suffix}"] = long_total - short_total
+            pivot[f"{participant}_index_options_long_share_{suffix}"] = long_total / total.replace(0, pd.NA)
+            pivot[f"{participant}_index_options_call_net_{suffix}"] = pivot[call_long] - pivot[call_short]
+            pivot[f"{participant}_index_options_put_net_{suffix}"] = pivot[put_long] - pivot[put_short]
+
+    return pivot.sort_values("date").reset_index(drop=True)
+
+
+def build_cash_panel(cash_frame: pd.DataFrame, fno_frame: pd.DataFrame) -> pd.DataFrame:
+    panel: pd.DataFrame | None = None
+
+    if not cash_frame.empty:
+        cash = cash_frame.rename(
+            columns={
+                "fiiPurchase": "fii_cash_buy",
+                "fiiSales": "fii_cash_sell",
+                "fiiNet": "fii_cash_net",
+                "diiPurchase": "dii_cash_buy",
+                "diiSale": "dii_cash_sell",
+                "diiNet": "dii_cash_net",
+            }
+        )
+        keep = [col for col in ["date", "fii_cash_buy", "fii_cash_sell", "fii_cash_net", "dii_cash_buy", "dii_cash_sell", "dii_cash_net"] if col in cash.columns]
+        panel = cash[keep].copy()
+
+    if not fno_frame.empty:
+        fno = fno_frame.rename(
+            columns={
+                "futPurchase": "fii_fno_futures_buy",
+                "futSales": "fii_fno_futures_sell",
+                "futNet": "fii_fno_futures_net",
+                "optPurchase": "fii_fno_options_buy",
+                "optSale": "fii_fno_options_sell",
+                "optNet": "fii_fno_options_net",
+            }
+        )
+        keep = [col for col in ["date", "fii_fno_futures_buy", "fii_fno_futures_sell", "fii_fno_futures_net", "fii_fno_options_buy", "fii_fno_options_sell", "fii_fno_options_net"] if col in fno.columns]
+        panel = fno[keep].copy() if panel is None else panel.merge(fno[keep], on="date", how="outer")
+
+    return panel.sort_values("date").reset_index(drop=True) if panel is not None else pd.DataFrame()
+
+
+def build_processed_panel(cash: pd.DataFrame, fno: pd.DataFrame, oi: pd.DataFrame, vol: pd.DataFrame) -> pd.DataFrame:
+    panel = build_cash_panel(cash, fno)
+    oi_panel = reshape_participant_data(oi, "oi")
+    vol_panel = reshape_participant_data(vol, "volume")
+
+    if panel.empty:
+        panel = oi_panel
+    elif not oi_panel.empty:
+        panel = panel.merge(oi_panel, on="date", how="outer")
+
+    if panel.empty:
+        panel = vol_panel
+    elif not vol_panel.empty:
+        panel = panel.merge(vol_panel, on="date", how="outer")
+
+    if panel.empty:
+        return panel
+
+    return panel.sort_values("date").reset_index(drop=True)
+
+
+def write_outputs(cash: pd.DataFrame, fno: pd.DataFrame, oi: pd.DataFrame, vol: pd.DataFrame, panel: pd.DataFrame) -> None:
+    ensure_dir(RAW_ROOT)
+    ensure_dir(PROCESSED_ROOT)
+
+    if not cash.empty:
+        ensure_dir(RAW_ROOT / "cash")
+        cash.to_csv(RAW_ROOT / "cash" / "cash_market_daily.csv", index=False)
+    if not fno.empty:
+        ensure_dir(RAW_ROOT / "fno")
+        fno.to_csv(RAW_ROOT / "fno" / "fii_fno_daily.csv", index=False)
+    if not oi.empty:
+        ensure_dir(RAW_ROOT / "participant_oi")
+        oi.to_csv(RAW_ROOT / "participant_oi" / "participant_oi_daily.csv", index=False)
+    if not vol.empty:
+        ensure_dir(RAW_ROOT / "participant_volume")
+        vol.to_csv(RAW_ROOT / "participant_volume" / "participant_volume_daily.csv", index=False)
+    if not panel.empty:
+        panel.to_csv(PROCESSED_ROOT / "institutional_daily_panel.csv", index=False)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Download FII/DII cash flows and participant positioning into Alt Data.")
+    parser.add_argument("--start-date", default="2024-01-01", help="Inclusive start date in YYYY-MM-DD format for NSE participant archives.")
+    parser.add_argument("--end-date", default=date.today().isoformat(), help="Inclusive end date in YYYY-MM-DD format for NSE participant archives.")
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    start_date = datetime.strptime(args.start_date, "%Y-%m-%d").date()
+    end_date = datetime.strptime(args.end_date, "%Y-%m-%d").date()
+
+    print(f"Downloading institutional flow data from {start_date} to {end_date}")
+
+    cash = fetch_moneycontrol_page("cash")
+    fno = fetch_moneycontrol_page("futures_and_options")
+    oi, vol = download_participant_history(start_date, end_date)
+    panel = build_processed_panel(cash, fno, oi, vol)
+
+    write_outputs(cash, fno, oi, vol, panel)
+
+    print(f"Cash rows: {len(cash):,}")
+    print(f"F&O rows: {len(fno):,}")
+    print(f"Participant OI rows: {len(oi):,}")
+    print(f"Participant volume rows: {len(vol):,}")
+    print(f"Processed panel rows: {len(panel):,}")
+    print(f"Saved outputs under {INSTITUTIONAL_ROOT}")
+
+
+if __name__ == "__main__":
+    main()

backend/research_runtime/Code/scripts/data_ingestion/refresh_market_data.py

@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+import argparse
+import os
+import subprocess
+import sys
+from datetime import date, datetime, timedelta
+from pathlib import Path
+
+import pandas as pd
+
+
+def find_project_root() -> Path:
+    env_root = os.environ.get("FORECASTING_PROJECT_ROOT")
+    if env_root:
+        return Path(env_root).expanduser().resolve()
+    for path in [Path(__file__).resolve(), *Path(__file__).resolve().parents]:
+        if (path / "Code").is_dir() and (path / "Data").is_dir() and (path / "Alt Data").is_dir():
+            return path
+    raise RuntimeError("Could not locate project root containing Code, Data, and Alt Data.")
+
+
+PROJECT_ROOT = find_project_root()
+DATA_ROOT = PROJECT_ROOT / "Data"
+ALT_ROOT = PROJECT_ROOT / "Alt Data"
+CODE_ROOT = PROJECT_ROOT / "Code"
+
+RAW_MINUTE_DIR = DATA_ROOT / "raw" / "minute"
+RAW_OPTIONS_DIR = ALT_ROOT / "options" / "raw"
+RAW_INSTITUTIONAL_DIR = ALT_ROOT / "institutional" / "raw"
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Refresh market and alt data to the latest Yahoo/NSE/Moneycontrol-available dates.")
+    parser.add_argument("--end-date", default=date.today().isoformat(), help="Inclusive end date in YYYY-MM-DD format.")
+    parser.add_argument(
+        "--minute-lookback-days",
+        type=int,
+        default=29,
+        help="Lookback window for Yahoo 1m updates. Yahoo minute history is limited, so this should stay near 29.",
+    )
+    parser.add_argument(
+        "--options-backfill-days",
+        type=int,
+        default=10,
+        help="Days of overlap to refetch for option bhavcopy data.",
+    )
+    parser.add_argument(
+        "--institutional-backfill-days",
+        type=int,
+        default=14,
+        help="Days of overlap to refetch for institutional archives.",
+    )
+    return parser.parse_args()
+
+
+def latest_date_from_csv(path: Path, column: str) -> date | None:
+    if not path.exists():
+        return None
+    frame = pd.read_csv(path, usecols=[column])
+    series = pd.to_datetime(frame[column], errors="coerce").dropna()
+    if series.empty:
+        return None
+    return series.max().date()
+
+
+def latest_date_from_tree(root: Path, filename: str, column: str) -> date | None:
+    candidates = list(root.rglob(filename))
+    latest: date | None = None
+    for path in candidates:
+        value = latest_date_from_csv(path, column)
+        if value is not None and (latest is None or value > latest):
+            latest = value
+    return latest
+
+
+def run_script(script_path: Path, *args: str) -> None:
+    command = [sys.executable, str(script_path), *args]
+    print("Running:", " ".join(command))
+    subprocess.run(command, check=True, cwd=PROJECT_ROOT)
+
+
+def main() -> None:
+    args = parse_args()
+    end_date = datetime.strptime(args.end_date, "%Y-%m-%d").date()
+
+    minute_latest_dates = []
+    for path in RAW_MINUTE_DIR.glob("*_minute.csv"):
+        latest = latest_date_from_csv(path, "date")
+        if latest is not None:
+            minute_latest_dates.append(latest)
+
+    yahoo_minute_floor = end_date - timedelta(days=args.minute_lookback_days)
+    minute_start = yahoo_minute_floor
+    if minute_latest_dates:
+        minute_start = max(yahoo_minute_floor, min(minute_latest_dates) - timedelta(days=1))
+
+    option_latest_dates = []
+    for path in RAW_OPTIONS_DIR.glob("*.csv"):
+        latest = latest_date_from_csv(path, "TRADE_DATE")
+        if latest is not None:
+            option_latest_dates.append(latest)
+    option_start = (
+        max(min(option_latest_dates) - timedelta(days=args.options_backfill_days), date(2015, 1, 1))
+        if option_latest_dates
+        else date(2015, 1, 1)
+    )
+
+    institutional_candidates = [
+        latest_date_from_tree(RAW_INSTITUTIONAL_DIR, "cash_market_daily.csv", "date"),
+        latest_date_from_tree(RAW_INSTITUTIONAL_DIR, "fii_fno_daily.csv", "date"),
+        latest_date_from_tree(RAW_INSTITUTIONAL_DIR, "participant_oi_daily.csv", "date"),
+        latest_date_from_tree(RAW_INSTITUTIONAL_DIR, "participant_volume_daily.csv", "date"),
+    ]
+    institutional_latest = max((value for value in institutional_candidates if value is not None), default=date(2024, 1, 1))
+    institutional_start = max(institutional_latest - timedelta(days=args.institutional_backfill_days), date(2024, 1, 1))
+
+    print(f"Minute refresh window: {minute_start} -> {end_date}")
+    print(f"Options refresh window: {option_start} -> {end_date}")
+    print(f"Institutional refresh window: {institutional_start} -> {end_date}")
+
+    run_script(
+        CODE_ROOT / "scripts" / "data_ingestion" / "update_index_minute_data.py",
+        "--start-date",
+        minute_start.isoformat(),
+        "--end-date",
+        end_date.isoformat(),
+    )
+    run_script(
+        CODE_ROOT / "scripts" / "data_ingestion" / "download_index_options.py",
+        "--start-date",
+        option_start.isoformat(),
+        "--end-date",
+        end_date.isoformat(),
+    )
+    run_script(
+        CODE_ROOT / "scripts" / "data_ingestion" / "download_institutional_flows.py",
+        "--start-date",
+        institutional_start.isoformat(),
+        "--end-date",
+        end_date.isoformat(),
+    )
+    run_script(CODE_ROOT / "scripts" / "data_preparation" / "build_research_data.py")
+
+
+if __name__ == "__main__":
+    main()

backend/research_runtime/Code/scripts/data_ingestion/update_index_minute_data.py

@@ -0,0 +1,228 @@
+from __future__ import annotations
+
+import argparse
+import concurrent.futures
+import os
+import time
+from datetime import datetime, timedelta
+from pathlib import Path
+
+import pandas as pd
+import requests
+
+try:
+    import yfinance as yf
+except ImportError:  # pragma: no cover - optional dependency for batch fetches
+    yf = None
+
+
+def find_project_root() -> Path:
+    env_root = os.environ.get("FORECASTING_PROJECT_ROOT")
+    if env_root:
+        return Path(env_root).expanduser().resolve()
+    for path in [Path(__file__).resolve(), *Path(__file__).resolve().parents]:
+        if (path / "Code").is_dir() and (path / "Data").is_dir() and (path / "Alt Data").is_dir():
+            return path
+    raise RuntimeError("Could not locate project root containing Code, Data, and Alt Data.")
+
+
+PROJECT_ROOT = find_project_root()
+RAW_MINUTE_DIR = PROJECT_ROOT / "Data" / "raw" / "minute"
+REQUEST_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
+    "Accept": "application/json,text/plain,*/*",
+}
+REQUEST_PAUSE_SEC = 0.35
+MAX_RETRIES = 3
+CHUNK_DAYS = 6
+
+ASSETS = {
+    "BEL_minute.csv": "BEL.NS",
+    "CANBK_minute.csv": "CANBK.NS",
+    "ITC_minute.csv": "ITC.NS",
+    "NTPC_minute.csv": "NTPC.NS",
+    "ONGC_minute.csv": "ONGC.NS",
+    "POWERGRID_minute.csv": "POWERGRID.NS",
+    "SBIN_minute.csv": "SBIN.NS",
+    "TATASTEEL_minute.csv": "TATASTEEL.NS",
+    "NIFTY 50_minute.csv": "^NSEI",
+    "NIFTY BANK_minute.csv": "^NSEBANK",
+    "INDIA VIX_minute.csv": "^INDIAVIX",
+    "HDFCBANK_minute.csv": "HDFCBANK.NS",
+    "KOTAKBANK_minute.csv": "KOTAKBANK.NS",
+    "RELIANCE_minute.csv": "RELIANCE.NS",
+    "WIPRO_minute.csv": "WIPRO.NS",
+}
+
+
+def empty_ohlcv_frame() -> pd.DataFrame:
+    return pd.DataFrame(columns=["date", "open", "high", "low", "close", "volume"])
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Update raw minute index data from direct Yahoo chart endpoints.")
+    parser.add_argument("--start-date", required=True, help="Inclusive start date in YYYY-MM-DD format.")
+    parser.add_argument("--end-date", required=True, help="Inclusive end date in YYYY-MM-DD format.")
+    return parser.parse_args()
+
+
+def build_chart_url(symbol: str, start_dt: datetime, end_dt_exclusive: datetime) -> str:
+    period1 = int(start_dt.timestamp())
+    period2 = int(end_dt_exclusive.timestamp())
+    return (
+        f"https://query1.finance.yahoo.com/v8/finance/chart/{requests.utils.quote(symbol, safe='')}"
+        f"?period1={period1}&period2={period2}&interval=1m&includePrePost=false&events=div%2Csplits"
+    )
+
+
+def fetch_chart_frame(
+    symbol: str,
+    start_dt: datetime,
+    end_dt_exclusive: datetime,
+    session: requests.Session,
+    *,
+    timeout: float = 30,
+    max_retries: int = MAX_RETRIES,
+    retry_pause_base: float = 1.25,
+) -> pd.DataFrame:
+    url = build_chart_url(symbol, start_dt, end_dt_exclusive)
+    last_error: Exception | None = None
+    for attempt in range(1, max_retries + 1):
+        try:
+            response = session.get(url, headers=REQUEST_HEADERS, timeout=timeout)
+            response.raise_for_status()
+            payload = response.json()
+            result = payload.get("chart", {}).get("result") or []
+            if not result:
+                return empty_ohlcv_frame()
+
+            result0 = result[0]
+            timestamps = result0.get("timestamp") or []
+            quote_sets = result0.get("indicators", {}).get("quote") or []
+            if not timestamps or not quote_sets:
+                return empty_ohlcv_frame()
+
+            quote = quote_sets[0]
+            frame = pd.DataFrame(
+                {
+                    "date": pd.to_datetime(timestamps, unit="s", utc=True).tz_convert("Asia/Kolkata").tz_localize(None),
+                    "open": quote.get("open", []),
+                    "high": quote.get("high", []),
+                    "low": quote.get("low", []),
+                    "close": quote.get("close", []),
+                    "volume": quote.get("volume", []),
+                }
+            )
+            for column in ["open", "high", "low", "close", "volume"]:
+                frame[column] = pd.to_numeric(frame[column], errors="coerce")
+            frame = frame.dropna(subset=["date", "open", "high", "low", "close"])
+            frame["volume"] = frame["volume"].fillna(0.0)
+            return frame.reset_index(drop=True)
+        except Exception as exc:
+            last_error = exc
+            if attempt == max_retries:
+                break
+            time.sleep(retry_pause_base * attempt)
+    raise RuntimeError(f"Failed to fetch {symbol} chunk {start_dt} to {end_dt_exclusive}: {last_error}") from last_error
+
+
+def fetch_chunk(symbol: str, start_dt: datetime, end_dt_exclusive: datetime, session: requests.Session) -> pd.DataFrame:
+    return fetch_chart_frame(symbol, start_dt, end_dt_exclusive, session)
+
+
+def fetch_many_chart_frames(
+    symbols: list[str],
+    start_dt: datetime,
+    end_dt_exclusive: datetime,
+    *,
+    timeout: float = 10,
+) -> dict[str, pd.DataFrame]:
+    if not symbols:
+        return {}
+    output: dict[str, pd.DataFrame] = {symbol: empty_ohlcv_frame() for symbol in symbols}
+
+    def fetch_one(symbol: str) -> tuple[str, pd.DataFrame]:
+        session = requests.Session()
+        try:
+            frame = fetch_chart_frame(
+                symbol,
+                start_dt,
+                end_dt_exclusive,
+                session,
+                timeout=max(timeout, 8.0),
+                max_retries=2,
+            )
+        except Exception:
+            frame = empty_ohlcv_frame()
+        return symbol, frame
+
+    max_workers = min(len(symbols), 6)
+    if max_workers <= 1:
+        for symbol in symbols:
+            fetched_symbol, frame = fetch_one(symbol)
+            output[fetched_symbol] = frame
+        return output
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = [executor.submit(fetch_one, symbol) for symbol in symbols]
+        for future in concurrent.futures.as_completed(futures):
+            fetched_symbol, frame = future.result()
+            output[fetched_symbol] = frame
+    return output
+
+
+def iter_chunks(start_date: datetime, end_date: datetime) -> list[tuple[datetime, datetime]]:
+    chunks: list[tuple[datetime, datetime]] = []
+    current = start_date
+    end_exclusive = end_date + timedelta(days=1)
+    while current < end_exclusive:
+        chunk_end = min(current + timedelta(days=CHUNK_DAYS), end_exclusive)
+        chunks.append((current, chunk_end))
+        current = chunk_end
+    return chunks
+
+
+def update_asset(raw_path: Path, symbol: str, start_date: datetime, end_date: datetime, session: requests.Session) -> tuple[int, pd.Timestamp | None]:
+    if raw_path.exists():
+        existing = pd.read_csv(raw_path, parse_dates=["date"])
+        existing = existing.sort_values("date").reset_index(drop=True)
+    else:
+        existing = pd.DataFrame(columns=["date", "open", "high", "low", "close", "volume"])
+
+    frames = [existing]
+    for chunk_start, chunk_end in iter_chunks(start_date, end_date):
+        chunk = fetch_chunk(symbol, chunk_start, chunk_end, session)
+        if not chunk.empty:
+            frames.append(chunk)
+        time.sleep(REQUEST_PAUSE_SEC)
+
+    updated = pd.concat(frames, ignore_index=True)
+    updated = updated.drop_duplicates(subset=["date"], keep="last").sort_values("date").reset_index(drop=True)
+    for column in ["open", "high", "low", "close", "volume"]:
+        updated[column] = pd.to_numeric(updated[column], errors="coerce")
+    updated = updated.dropna(subset=["date", "open", "high", "low", "close", "volume"])
+    updated["volume"] = updated["volume"].fillna(0.0)
+    raw_path.parent.mkdir(parents=True, exist_ok=True)
+    updated.to_csv(raw_path, index=False)
+    added_rows = len(updated) - len(existing)
+    last_timestamp = pd.Timestamp(updated["date"].max()) if not updated.empty else None
+    return added_rows, last_timestamp
+
+
+def main() -> None:
+    args = parse_args()
+    start_date = datetime.strptime(args.start_date, "%Y-%m-%d")
+    end_date = datetime.strptime(args.end_date, "%Y-%m-%d")
+
+    session = requests.Session()
+
+    for filename, symbol in ASSETS.items():
+        raw_path = RAW_MINUTE_DIR / filename
+        if not raw_path.exists():
+            raise FileNotFoundError(f"Raw minute file not found: {raw_path}")
+        added_rows, last_timestamp = update_asset(raw_path, symbol, start_date, end_date, session)
+        print(f"{filename}: added {added_rows:,} rows, latest timestamp {last_timestamp}")
+
+
+if __name__ == "__main__":
+    main()

backend/research_runtime/Code/scripts/data_preparation/build_research_data.py

@@ -0,0 +1,957 @@
+from __future__ import annotations
+
+import json
+import math
+import os
+import shutil
+import __main__
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
+from dataclasses import dataclass
+from datetime import date, datetime, timedelta
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import requests
+
+
+def find_project_root() -> Path:
+    env_root = os.environ.get("FORECASTING_PROJECT_ROOT")
+    if env_root:
+        return Path(env_root).expanduser().resolve()
+    for path in [Path(__file__).resolve(), *Path(__file__).resolve().parents]:
+        if (path / "Code").is_dir() and (path / "Data").is_dir() and (path / "Alt Data").is_dir():
+            return path
+    raise RuntimeError("Could not locate project root containing Code, Data, and Alt Data.")
+
+
+PROJECT_ROOT = find_project_root()
+DATA_ROOT = PROJECT_ROOT / "Data"
+ALT_ROOT = PROJECT_ROOT / "Alt Data"
+CODE_ROOT = PROJECT_ROOT / "Code"
+
+
+BAR_AGG = {
+    "open": "first",
+    "high": "max",
+    "low": "min",
+    "close": "last",
+    "volume": "sum",
+}
+
+
+RAW_MINUTE_ALIASES = {
+    "NIFTY 50_minute.csv": "nifty50",
+    "NIFTY BANK_minute.csv": "banknifty",
+    "INDIA VIX_minute.csv": "india_vix",
+    "HDFCBANK_minute.csv": "hdfcbank",
+    "RELIANCE_minute.csv": "reliance",
+    "WIPRO_minute.csv": "wipro",
+    "KOTAKBANK_minute.csv": "kotakbank",
+}
+
+
+OPTION_FILES = {
+    "NIFTY_50_options.csv": {
+        "asset": "nifty50",
+        "spot_asset": "nifty50",
+    },
+    "NIFTY_BANK_options.csv": {
+        "asset": "banknifty",
+        "spot_asset": "banknifty",
+    },
+    "HDFCBANK_options.csv": {
+        "asset": "hdfcbank",
+        "spot_asset": "hdfcbank",
+    },
+}
+
+
+TIMEFRAMES = {
+    "1m": 1,
+    "5m": 5,
+    "1h": 60,
+    "4h": 240,
+}
+
+
+PANEL_FEATURE_COLUMNS = [
+    "date",
+    "close",
+    "return_1",
+    "log_return_1",
+    "realized_vol_20",
+    "rsi_14",
+    "target_return_1",
+]
+
+
+DAILY_MASTER_COLUMNS = [
+    "date",
+    "open",
+    "high",
+    "low",
+    "close",
+    "return_1",
+    "log_return_1",
+    "realized_vol_20",
+    "rsi_14",
+    "target_return_1",
+]
+
+
+DEFAULT_MARKET_WORKERS = min(6, max(1, (os.cpu_count() or 2) - 1))
+
+
+YAHOO_SERIES = {
+    "sp500": "^GSPC",
+    "nasdaq_composite": "^IXIC",
+    "dow_jones": "^DJI",
+    "nikkei225": "^N225",
+    "us10y_treasury": "^TNX",
+    "india_fx_inr_per_usd": "INR=X",
+    "brent_fred": "BZ=F",
+    "vix_fred": "^VIX",
+    "broad_dollar_index": "DX-Y.NYB",
+}
+
+YAHOO_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
+    "Accept": "application/json,text/plain,*/*",
+}
+
+
+FRED_SERIES = {
+    "sp500": "SP500",
+    "nasdaq_composite": "NASDAQCOM",
+    "dow_jones": "DJIA",
+    "nikkei225": "NIKKEI225",
+    "us10y_treasury": "DGS10",
+    "fed_funds": "FEDFUNDS",
+    "india_fx_inr_per_usd": "DEXINUS",
+    "brent_fred": "DCOILBRENTEU",
+    "vix_fred": "VIXCLS",
+    "broad_dollar_index": "DTWEXBGS",
+}
+
+
+FRED_RETURN_SERIES = {
+    "sp500",
+    "nasdaq_composite",
+    "dow_jones",
+    "nikkei225",
+    "india_fx_inr_per_usd",
+    "brent_fred",
+    "vix_fred",
+    "broad_dollar_index",
+}
+
+
+FEATURE_DESCRIPTIONS = {
+    "return_1": "Simple return from prior bar close.",
+    "log_return_1": "Log return from prior bar close.",
+    "range_abs": "High-low range in price units.",
+    "range_pct_close": "High-low range scaled by close.",
+    "body_abs": "Close-open candle body in price units.",
+    "body_pct_open": "Close-open candle body scaled by open.",
+    "gap_pct": "Open relative to prior close.",
+    "hlc3": "Typical price: (high + low + close) / 3.",
+    "oc2": "Mid price: (open + close) / 2.",
+    "realized_vol_5": "Rolling 5-bar std of log returns annualized to bar units.",
+    "realized_vol_20": "Rolling 20-bar std of log returns annualized to bar units.",
+    "sma_5_rel": "Close relative to 5-bar simple moving average.",
+    "sma_20_rel": "Close relative to 20-bar simple moving average.",
+    "ema_12_rel": "Close relative to 12-bar exponential moving average.",
+    "ema_26_rel": "Close relative to 26-bar exponential moving average.",
+    "momentum_5": "5-bar percent change.",
+    "momentum_20": "20-bar percent change.",
+    "rsi_14": "14-bar RSI.",
+    "atr_14": "14-bar average true range.",
+    "macd": "MACD line from 12/26 EMA.",
+    "macd_signal": "9-bar EMA of MACD.",
+    "macd_hist": "MACD minus MACD signal.",
+    "bollinger_z_20": "20-bar z-score of close.",
+    "volume_all_zero": "Indicator showing the source volume field is entirely zero.",
+    "target_return_1": "Next-bar simple return.",
+    "target_log_return_1": "Next-bar log return.",
+    "target_direction_1": "Next-bar direction label.",
+}
+
+
+@dataclass
+class ManifestEntry:
+    category: str
+    asset: str
+    timeframe: str
+    path: str
+    rows: int
+    start: str
+    end: str
+
+
+def ensure_dir(path: Path) -> None:
+    path.mkdir(parents=True, exist_ok=True)
+
+
+def asset_slug_from_minute_file(path: Path) -> str:
+    name = path.name
+    if name in RAW_MINUTE_ALIASES:
+        return RAW_MINUTE_ALIASES[name]
+
+    stem = path.stem
+    if stem.lower().endswith("_minute"):
+        stem = stem[:-7]
+
+    return (
+        stem.lower()
+        .replace("&", "and")
+        .replace(" ", "_")
+        .replace("-", "_")
+        .replace(".", "_")
+    )
+
+
+def discover_raw_minute_files(raw_minute_dir: Path) -> dict[str, str]:
+    files = sorted(raw_minute_dir.glob("*_minute.csv"))
+    return {path.name: asset_slug_from_minute_file(path) for path in files}
+
+
+def safe_move(src: Path, dst: Path) -> None:
+    if not src.exists():
+        return
+    ensure_dir(dst.parent)
+    if dst.exists():
+        return
+    shutil.move(str(src), str(dst))
+
+
+def organize_existing_files() -> dict[str, Path]:
+    ensure_dir(CODE_ROOT)
+    ensure_dir(DATA_ROOT)
+    ensure_dir(ALT_ROOT)
+
+    raw_minute_dir = DATA_ROOT / "raw" / "minute"
+    raw_options_dir = ALT_ROOT / "options" / "raw"
+    ensure_dir(raw_minute_dir)
+    ensure_dir(raw_options_dir)
+
+    for filename in RAW_MINUTE_ALIASES:
+        safe_move(DATA_ROOT / filename, raw_minute_dir / filename)
+
+    for path in sorted(DATA_ROOT.glob("*_minute.csv")):
+        safe_move(path, raw_minute_dir / path.name)
+
+    raw_minute_aliases = {
+        "HDFCBANK_minute (2).csv": "HDFCBANK_minute.csv",
+        "RELIANCE_minute (1).csv": "RELIANCE_minute.csv",
+    }
+    for src_name, dst_name in raw_minute_aliases.items():
+        safe_move(DATA_ROOT / "raw" / src_name, raw_minute_dir / dst_name)
+        safe_move(DATA_ROOT / src_name, raw_minute_dir / dst_name)
+
+    for filename in OPTION_FILES:
+        safe_move(ALT_ROOT / filename, raw_options_dir / filename)
+
+    pycache_dir = ALT_ROOT / "__pycache__"
+    if pycache_dir.exists():
+        shutil.rmtree(pycache_dir)
+
+    return {
+        "raw_minute_dir": raw_minute_dir,
+        "raw_options_dir": raw_options_dir,
+        "bars_dir": DATA_ROOT / "processed" / "bars",
+        "features_dir": DATA_ROOT / "processed" / "features",
+        "panels_dir": DATA_ROOT / "processed" / "panels",
+        "metadata_dir": DATA_ROOT / "metadata",
+        "alt_options_processed_dir": ALT_ROOT / "options" / "processed",
+        "alt_external_raw_dir": ALT_ROOT / "external" / "raw",
+        "alt_external_processed_dir": ALT_ROOT / "external" / "processed",
+        "alt_institutional_processed_dir": ALT_ROOT / "institutional" / "processed",
+        "alt_metadata_dir": ALT_ROOT / "metadata",
+    }
+
+
+def load_minute_data(path: Path) -> pd.DataFrame:
+    df = pd.read_csv(
+        path,
+        parse_dates=["date"],
+        usecols=["date", "open", "high", "low", "close", "volume"],
+    )
+    df = df.sort_values("date").reset_index(drop=True)
+    for col in ["open", "high", "low", "close", "volume"]:
+        df[col] = pd.to_numeric(df[col], errors="coerce")
+    df = df.dropna(subset=["date", "open", "high", "low", "close", "volume"])
+    df = df.drop_duplicates(subset=["date"])
+    df["volume"] = df["volume"].fillna(0.0)
+    return df
+
+
+def aggregate_session_bars(df: pd.DataFrame, minutes: int) -> pd.DataFrame:
+    if minutes == 1:
+        out = df.copy()
+        return out
+
+    if df.empty:
+        return df.copy()
+
+    source = df
+    if not source["date"].is_monotonic_increasing:
+        source = source.sort_values("date", kind="mergesort")
+
+    session_date = source["date"].dt.normalize()
+    session_open = source.groupby(session_date, sort=False)["date"].transform("first")
+    elapsed_minutes = ((source["date"] - session_open).dt.total_seconds() // 60).astype("int64")
+    bucket = elapsed_minutes // minutes
+
+    return (
+        source.groupby([session_date, bucket], sort=True)
+        .agg(
+            date=("date", "last"),
+            open=("open", "first"),
+            high=("high", "max"),
+            low=("low", "min"),
+            close=("close", "last"),
+            volume=("volume", "sum"),
+        )
+        .reset_index(drop=True)
+    )
+
+
+def compute_daily_bars(df: pd.DataFrame) -> pd.DataFrame:
+    daily = (
+        df.groupby(df["date"].dt.normalize(), sort=True)
+        .agg(**{k: (k, v) for k, v in BAR_AGG.items()})
+        .reset_index()
+        .rename(columns={"date": "session_date"})
+    )
+    daily = daily.rename(columns={"session_date": "date"})
+    return daily
+
+
+def compute_rsi(close: pd.Series, period: int = 14) -> pd.Series:
+    delta = close.diff()
+    gain = delta.clip(lower=0)
+    loss = -delta.clip(upper=0)
+    avg_gain = gain.ewm(alpha=1 / period, adjust=False, min_periods=period).mean()
+    avg_loss = loss.ewm(alpha=1 / period, adjust=False, min_periods=period).mean()
+    rs = avg_gain / avg_loss.replace(0, np.nan)
+    return 100 - (100 / (1 + rs))
+
+
+def engineer_features(df: pd.DataFrame, timeframe_name: str) -> pd.DataFrame:
+    feat = df.copy()
+    feat = feat.sort_values("date").reset_index(drop=True)
+
+    prev_close = feat["close"].shift(1)
+    feat["return_1"] = feat["close"].pct_change()
+    feat["log_return_1"] = np.log(feat["close"]).diff()
+    feat["range_abs"] = feat["high"] - feat["low"]
+    feat["range_pct_close"] = feat["range_abs"] / feat["close"].replace(0, np.nan)
+    feat["body_abs"] = feat["close"] - feat["open"]
+    feat["body_pct_open"] = feat["body_abs"] / feat["open"].replace(0, np.nan)
+    feat["gap_pct"] = feat["open"] / prev_close.replace(0, np.nan) - 1
+    feat["hlc3"] = (feat["high"] + feat["low"] + feat["close"]) / 3
+    feat["oc2"] = (feat["open"] + feat["close"]) / 2
+
+    feat["realized_vol_5"] = feat["log_return_1"].rolling(5).std() * math.sqrt(5)
+    feat["realized_vol_20"] = feat["log_return_1"].rolling(20).std() * math.sqrt(20)
+
+    sma_5 = feat["close"].rolling(5).mean()
+    sma_20 = feat["close"].rolling(20).mean()
+    ema_12 = feat["close"].ewm(span=12, adjust=False).mean()
+    ema_26 = feat["close"].ewm(span=26, adjust=False).mean()
+    feat["sma_5_rel"] = feat["close"] / sma_5.replace(0, np.nan) - 1
+    feat["sma_20_rel"] = feat["close"] / sma_20.replace(0, np.nan) - 1
+    feat["ema_12_rel"] = feat["close"] / ema_12.replace(0, np.nan) - 1
+    feat["ema_26_rel"] = feat["close"] / ema_26.replace(0, np.nan) - 1
+    feat["momentum_5"] = feat["close"].pct_change(5)
+    feat["momentum_20"] = feat["close"].pct_change(20)
+    feat["rsi_14"] = compute_rsi(feat["close"], 14)
+
+    tr_parts = pd.concat(
+        [
+            feat["high"] - feat["low"],
+            (feat["high"] - prev_close).abs(),
+            (feat["low"] - prev_close).abs(),
+        ],
+        axis=1,
+    )
+    feat["atr_14"] = tr_parts.max(axis=1).rolling(14).mean()
+
+    macd = ema_12 - ema_26
+    macd_signal = macd.ewm(span=9, adjust=False).mean()
+    feat["macd"] = macd
+    feat["macd_signal"] = macd_signal
+    feat["macd_hist"] = macd - macd_signal
+
+    rolling_mean_20 = feat["close"].rolling(20).mean()
+    rolling_std_20 = feat["close"].rolling(20).std()
+    feat["bollinger_z_20"] = (feat["close"] - rolling_mean_20) / rolling_std_20.replace(0, np.nan)
+
+    feat["day_of_week"] = feat["date"].dt.dayofweek
+    feat["month"] = feat["date"].dt.month
+    feat["quarter"] = feat["date"].dt.quarter
+
+    if timeframe_name != "1d":
+        feat["hour"] = feat["date"].dt.hour
+        feat["minute"] = feat["date"].dt.minute
+
+    volume_all_zero = int(np.isclose(feat["volume"].abs().sum(), 0.0))
+    feat["volume_all_zero"] = volume_all_zero
+    if not volume_all_zero:
+        vol_5 = feat["volume"].rolling(5).mean()
+        vol_20 = feat["volume"].rolling(20).mean()
+        feat["volume_sma_5_rel"] = feat["volume"] / vol_5.replace(0, np.nan) - 1
+        feat["volume_sma_20_rel"] = feat["volume"] / vol_20.replace(0, np.nan) - 1
+
+    feat["target_return_1"] = feat["return_1"].shift(-1)
+    feat["target_log_return_1"] = feat["log_return_1"].shift(-1)
+    feat["target_direction_1"] = np.sign(feat["target_return_1"]).fillna(0).astype(int)
+
+    return feat
+
+
+def write_csv(df: pd.DataFrame, path: Path) -> None:
+    ensure_dir(path.parent)
+    df.to_csv(path, index=False)
+
+
+def summarize_frame(category: str, asset: str, timeframe: str, path: Path, df: pd.DataFrame) -> ManifestEntry:
+    if "date" in df.columns and not df.empty:
+        start = str(pd.to_datetime(df["date"]).min())
+        end = str(pd.to_datetime(df["date"]).max())
+    else:
+        start = ""
+        end = ""
+    return ManifestEntry(
+        category=category,
+        asset=asset,
+        timeframe=timeframe,
+        path=str(path),
+        rows=len(df),
+        start=start,
+        end=end,
+    )
+
+
+def market_worker_count(asset_count: int) -> int:
+    raw_value = os.environ.get("MARKET_BUILD_WORKERS")
+    if raw_value:
+        try:
+            requested = int(raw_value)
+        except ValueError:
+            requested = DEFAULT_MARKET_WORKERS
+    else:
+        requested = DEFAULT_MARKET_WORKERS
+    return max(1, min(asset_count, requested))
+
+
+def market_executor_class():
+    main_file = str(getattr(__main__, "__file__", ""))
+    if main_file and "<stdin>" not in main_file:
+        return ProcessPoolExecutor
+    return ThreadPoolExecutor
+
+
+def slim_market_features(features: pd.DataFrame, timeframe_name: str) -> pd.DataFrame:
+    columns = DAILY_MASTER_COLUMNS if timeframe_name == "1d" else PANEL_FEATURE_COLUMNS
+    return features[columns].copy()
+
+
+def load_slim_market_frames(
+    paths: dict[str, Path],
+    raw_minute_files: dict[str, str],
+) -> dict[str, dict[str, pd.DataFrame]]:
+    asset_frames: dict[str, dict[str, pd.DataFrame]] = {}
+    for asset in raw_minute_files.values():
+        timeframe_frames: dict[str, pd.DataFrame] = {}
+        for timeframe_name in [*TIMEFRAMES.keys(), "1d"]:
+            columns = DAILY_MASTER_COLUMNS if timeframe_name == "1d" else PANEL_FEATURE_COLUMNS
+            feat_path = paths["features_dir"] / timeframe_name / f"{asset}_{timeframe_name}_features.csv"
+            timeframe_frames[timeframe_name] = pd.read_csv(feat_path, parse_dates=["date"], usecols=columns)
+        asset_frames[asset] = timeframe_frames
+    return asset_frames
+
+
+def build_single_market_dataset(filename: str, asset: str, paths: dict[str, Path]) -> list[ManifestEntry]:
+    manifest: list[ManifestEntry] = []
+    raw_path = paths["raw_minute_dir"] / filename
+    minute_df = load_minute_data(raw_path)
+
+    for timeframe_name, minutes in TIMEFRAMES.items():
+        bars = aggregate_session_bars(minute_df, minutes)
+        features = engineer_features(bars, timeframe_name)
+
+        bars_path = paths["bars_dir"] / timeframe_name / f"{asset}_{timeframe_name}.csv"
+        feat_path = paths["features_dir"] / timeframe_name / f"{asset}_{timeframe_name}_features.csv"
+        write_csv(bars, bars_path)
+        write_csv(features, feat_path)
+
+        manifest.append(summarize_frame("bars", asset, timeframe_name, bars_path, bars))
+        manifest.append(summarize_frame("features", asset, timeframe_name, feat_path, features))
+
+    daily_bars = compute_daily_bars(minute_df)
+    daily_features = engineer_features(daily_bars, "1d")
+
+    daily_bars_path = paths["bars_dir"] / "1d" / f"{asset}_1d.csv"
+    daily_feat_path = paths["features_dir"] / "1d" / f"{asset}_1d_features.csv"
+    write_csv(daily_bars, daily_bars_path)
+    write_csv(daily_features, daily_feat_path)
+
+    manifest.append(summarize_frame("bars", asset, "1d", daily_bars_path, daily_bars))
+    manifest.append(summarize_frame("features", asset, "1d", daily_feat_path, daily_features))
+    return manifest
+
+
+def build_market_datasets(paths: dict[str, Path]) -> tuple[dict[str, dict[str, pd.DataFrame]], list[ManifestEntry]]:
+    manifest: list[ManifestEntry] = []
+
+    raw_minute_files = discover_raw_minute_files(paths["raw_minute_dir"])
+    worker_count = market_worker_count(len(raw_minute_files))
+
+    if worker_count == 1:
+        for filename, asset in raw_minute_files.items():
+            manifest.extend(build_single_market_dataset(filename, asset, paths))
+    else:
+        executor_class = market_executor_class()
+        with executor_class(max_workers=worker_count) as executor:
+            futures = [
+                executor.submit(build_single_market_dataset, filename, asset, paths)
+                for filename, asset in raw_minute_files.items()
+            ]
+            for future in as_completed(futures):
+                manifest.extend(future.result())
+
+    asset_frames = load_slim_market_frames(paths, raw_minute_files)
+    return asset_frames, manifest
+
+
+def build_multiactive_panels(
+    paths: dict[str, Path],
+    asset_frames: dict[str, dict[str, pd.DataFrame]],
+) -> list[ManifestEntry]:
+    manifest: list[ManifestEntry] = []
+    timeframes = ["1m", "5m", "1h", "4h", "1d"]
+
+    for timeframe in timeframes:
+        slices: list[pd.DataFrame] = []
+        for asset, tf_map in asset_frames.items():
+            frame = tf_map[timeframe]
+            cols = ["date", "close", "return_1", "log_return_1", "realized_vol_20", "rsi_14", "target_return_1"]
+            asset_slice = frame[cols].rename(
+                columns={col: f"{asset}_{col}" for col in cols if col != "date"}
+            )
+            slices.append(asset_slice.set_index("date"))
+
+        if not slices:
+            continue
+
+        panel = pd.concat(slices, axis=1, join="outer").reset_index()
+
+        if {"banknifty_close", "nifty50_close"}.issubset(panel.columns):
+            panel["banknifty_to_nifty_ratio"] = panel["banknifty_close"] / panel["nifty50_close"]
+            panel["banknifty_minus_nifty_return_spread"] = (
+                panel.get("banknifty_return_1") - panel.get("nifty50_return_1")
+            )
+
+        if {"india_vix_close", "nifty50_return_1"}.issubset(panel.columns):
+            panel["vix_to_nifty_close_ratio"] = panel["india_vix_close"] / panel["nifty50_close"]
+            panel["nifty_vix_rolling_corr_20"] = (
+                panel["nifty50_return_1"].rolling(20).corr(panel["india_vix_return_1"])
+            )
+
+        panel = panel.sort_values("date").reset_index(drop=True)
+        panel_path = paths["panels_dir"] / f"{timeframe}_market_panel.csv"
+        write_csv(panel, panel_path)
+        manifest.append(summarize_frame("panel", "market", timeframe, panel_path, panel))
+
+    return manifest
+
+
+def build_option_features(
+    paths: dict[str, Path],
+    asset_frames: dict[str, dict[str, pd.DataFrame]],
+) -> list[ManifestEntry]:
+    manifest: list[ManifestEntry] = []
+    daily_spot = {
+        asset: frame_map["1d"][["date", "close"]].assign(date=lambda x: pd.to_datetime(x["date"]).dt.normalize())
+        for asset, frame_map in asset_frames.items()
+    }
+
+    for filename, meta in OPTION_FILES.items():
+        raw_path = paths["raw_options_dir"] / filename
+        dtype_map = {
+            "INSTRUMENT": "category",
+            "SYMBOL": "category",
+            "OPTION_TYP": "category",
+            "STRIKE_PR": "float32",
+            "OPEN": "float32",
+            "HIGH": "float32",
+            "LOW": "float32",
+            "CLOSE": "float32",
+            "SETTLE_PR": "float32",
+            "CONTRACTS": "float32",
+            "VAL_INLAKH": "float32",
+            "OPEN_INT": "float32",
+            "CHG_IN_OI": "float32",
+        }
+        df = pd.read_csv(
+            raw_path,
+            dtype=dtype_map,
+            parse_dates=["TRADE_DATE", "EXPIRY_DT"],
+            usecols=[
+                "TRADE_DATE",
+                "INSTRUMENT",
+                "SYMBOL",
+                "EXPIRY_DT",
+                "STRIKE_PR",
+                "OPTION_TYP",
+                "CLOSE",
+                "CONTRACTS",
+                "OPEN_INT",
+                "CHG_IN_OI",
+            ],
+        )
+        df = df.rename(columns={"TRADE_DATE": "trade_date", "EXPIRY_DT": "expiry_date"})
+        df["trade_date"] = pd.to_datetime(df["trade_date"]).dt.normalize()
+        df["expiry_date"] = pd.to_datetime(df["expiry_date"]).dt.normalize()
+        df["dte"] = (df["expiry_date"] - df["trade_date"]).dt.days
+
+        base_daily = (
+            df.pivot_table(
+                index="trade_date",
+                columns="OPTION_TYP",
+                values=["OPEN_INT", "CONTRACTS", "CHG_IN_OI"],
+                aggfunc="sum",
+                fill_value=0.0,
+                observed=False,
+            )
+            .sort_index()
+        )
+        base_daily.columns = [
+            f"{metric.lower()}_{option_type.lower()}"
+            for metric, option_type in base_daily.columns.to_flat_index()
+        ]
+        base_daily = base_daily.reset_index()
+
+        if {"open_int_ce", "open_int_pe"}.issubset(base_daily.columns):
+            base_daily["pcr_open_int"] = (
+                base_daily["open_int_pe"] / base_daily["open_int_ce"].replace(0, np.nan)
+            )
+        if {"contracts_ce", "contracts_pe"}.issubset(base_daily.columns):
+            base_daily["pcr_contracts"] = (
+                base_daily["contracts_pe"] / base_daily["contracts_ce"].replace(0, np.nan)
+            )
+        if {"chg_in_oi_ce", "chg_in_oi_pe"}.issubset(base_daily.columns):
+            base_daily["net_oi_change_put_minus_call"] = (
+                base_daily["chg_in_oi_pe"] - base_daily["chg_in_oi_ce"]
+            )
+
+        weighted = (
+            df.assign(weighted_strike=df["STRIKE_PR"] * df["OPEN_INT"])
+            .groupby(["trade_date", "OPTION_TYP"], observed=True)
+            .agg(weighted_strike=("weighted_strike", "sum"), open_int=("OPEN_INT", "sum"))
+            .reset_index()
+        )
+        weighted["oi_weighted_strike"] = weighted["weighted_strike"] / weighted["open_int"].replace(0, np.nan)
+        weighted = weighted.pivot(index="trade_date", columns="OPTION_TYP", values="oi_weighted_strike").reset_index()
+        weighted.columns = [
+            "trade_date" if col == "trade_date" else f"oi_weighted_strike_{str(col).lower()}"
+            for col in weighted.columns
+        ]
+
+        spot_map = daily_spot[meta["spot_asset"]].rename(columns={"close": "spot_close"})
+        with_spot = df.merge(spot_map, left_on="trade_date", right_on="date", how="left").drop(columns=["date"])
+        with_spot = with_spot[with_spot["dte"] >= 0].copy()
+        with_spot["abs_moneyness"] = (with_spot["STRIKE_PR"] - with_spot["spot_close"]).abs()
+        near_dte = with_spot.groupby("trade_date")["dte"].transform("min")
+        near = with_spot[with_spot["dte"] == near_dte].copy()
+        min_abs_mny = near.groupby("trade_date")["abs_moneyness"].transform("min")
+        atm = near[near["abs_moneyness"] == min_abs_mny].copy()
+        atm = (
+            atm.groupby(["trade_date", "OPTION_TYP"], observed=True)
+            .agg(
+                atm_close=("CLOSE", "mean"),
+                atm_open_int=("OPEN_INT", "sum"),
+                atm_contracts=("CONTRACTS", "sum"),
+                atm_strike=("STRIKE_PR", "mean"),
+                near_dte=("dte", "min"),
+            )
+            .reset_index()
+        )
+        atm = atm.pivot(index="trade_date", columns="OPTION_TYP")
+        atm.columns = [f"{metric}_{str(opt).lower()}" for metric, opt in atm.columns.to_flat_index()]
+        atm = atm.reset_index()
+        if {"atm_close_ce", "atm_close_pe"}.issubset(atm.columns):
+            atm["atm_straddle_close"] = atm["atm_close_ce"] + atm["atm_close_pe"]
+        if {"atm_open_int_ce", "atm_open_int_pe"}.issubset(atm.columns):
+            atm["atm_pcr_open_int"] = atm["atm_open_int_pe"] / atm["atm_open_int_ce"].replace(0, np.nan)
+
+        option_daily = (
+            base_daily.merge(weighted, on="trade_date", how="left")
+            .merge(atm, on="trade_date", how="left")
+            .merge(spot_map, left_on="trade_date", right_on="date", how="left")
+            .drop(columns=["date"])
+            .sort_values("trade_date")
+            .reset_index(drop=True)
+            .rename(columns={"trade_date": "date"})
+        )
+        option_daily["date"] = pd.to_datetime(option_daily["date"])
+
+        asset = meta["asset"]
+        option_path = paths["alt_options_processed_dir"] / f"{asset}_options_daily_features.csv"
+        write_csv(option_daily, option_path)
+        manifest.append(summarize_frame("options_features", asset, "1d", option_path, option_daily))
+
+    return manifest
+
+
+def build_yahoo_chart_url(symbol: str, start_dt: datetime, end_dt_exclusive: datetime, interval: str = "1d") -> str:
+    period1 = int(start_dt.timestamp())
+    period2 = int(end_dt_exclusive.timestamp())
+    return (
+        f"https://query1.finance.yahoo.com/v8/finance/chart/{requests.utils.quote(symbol, safe='')}"
+        f"?period1={period1}&period2={period2}&interval={interval}&includePrePost=false&events=div%2Csplits"
+    )
+
+
+def fetch_yahoo_chart_series(symbol: str, start_date: str, end_date: str) -> pd.DataFrame:
+    start_dt = datetime.strptime(start_date, "%Y-%m-%d")
+    end_dt_exclusive = datetime.strptime(end_date, "%Y-%m-%d") + timedelta(days=1)
+    url = build_yahoo_chart_url(symbol, start_dt, end_dt_exclusive, interval="1d")
+    response = requests.get(url, headers=YAHOO_HEADERS, timeout=30)
+    response.raise_for_status()
+
+    payload = response.json()
+    result = payload.get("chart", {}).get("result") or []
+    if not result:
+        return pd.DataFrame(columns=["date", "open", "high", "low", "close", "adj_close", "volume"])
+
+    result0 = result[0]
+    timestamps = result0.get("timestamp") or []
+    quote_sets = result0.get("indicators", {}).get("quote") or []
+    if not timestamps or not quote_sets:
+        return pd.DataFrame(columns=["date", "open", "high", "low", "close", "adj_close", "volume"])
+
+    quote = quote_sets[0]
+    adjclose_sets = result0.get("indicators", {}).get("adjclose") or [{}]
+    adjclose = adjclose_sets[0].get("adjclose", []) if adjclose_sets else []
+
+    frame = pd.DataFrame(
+        {
+            "date": pd.to_datetime(timestamps, unit="s", utc=True).tz_convert("UTC").tz_localize(None).normalize(),
+            "open": quote.get("open", []),
+            "high": quote.get("high", []),
+            "low": quote.get("low", []),
+            "close": quote.get("close", []),
+            "adj_close": adjclose,
+            "volume": quote.get("volume", []),
+        }
+    )
+    for column in ["open", "high", "low", "close", "adj_close", "volume"]:
+        if column in frame.columns:
+            frame[column] = pd.to_numeric(frame[column], errors="coerce")
+
+    frame = frame.dropna(subset=["date", "close"]).drop_duplicates(subset=["date"]).sort_values("date").reset_index(drop=True)
+    if "volume" in frame.columns:
+        frame["volume"] = frame["volume"].fillna(0.0)
+    return frame
+
+
+def fetch_yahoo_series(start_date: str, end_date: str) -> tuple[dict[str, pd.DataFrame], list[ManifestEntry]]:
+    raw_frames: dict[str, pd.DataFrame] = {}
+    manifest: list[ManifestEntry] = []
+
+    for asset_name, ticker in YAHOO_SERIES.items():
+        try:
+            frame = fetch_yahoo_chart_series(ticker, start_date, end_date)
+        except Exception:
+            continue
+        if frame.empty:
+            continue
+
+        raw_frames[asset_name] = frame
+        raw_path = ALT_ROOT / "external" / "raw" / "yahoo" / f"{asset_name}.csv"
+        write_csv(frame, raw_path)
+        manifest.append(summarize_frame("external_raw", asset_name, "1d", raw_path, frame))
+
+    return raw_frames, manifest
+
+
+def fetch_fred_series(start_date: str) -> tuple[dict[str, pd.DataFrame], list[ManifestEntry]]:
+    raw_frames: dict[str, pd.DataFrame] = {}
+    manifest: list[ManifestEntry] = []
+
+    for asset_name, series_id in FRED_SERIES.items():
+        url = f"https://fred.stlouisfed.org/graph/fredgraph.csv?id={series_id}"
+        try:
+            frame = pd.read_csv(url)
+        except Exception:
+            continue
+        frame = frame.rename(columns={"observation_date": "date", series_id: "value"})
+        frame["date"] = pd.to_datetime(frame["date"])
+        frame["value"] = pd.to_numeric(frame["value"], errors="coerce")
+        frame = frame[frame["date"] >= pd.Timestamp(start_date)].reset_index(drop=True)
+        raw_frames[asset_name] = frame
+        raw_path = ALT_ROOT / "external" / "raw" / "fred" / f"{asset_name}.csv"
+        write_csv(frame, raw_path)
+        manifest.append(summarize_frame("external_raw", asset_name, "1d", raw_path, frame))
+
+    return raw_frames, manifest
+
+
+def build_external_panels(
+    yahoo_frames: dict[str, pd.DataFrame],
+    fred_frames: dict[str, pd.DataFrame],
+    paths: dict[str, Path],
+) -> list[ManifestEntry]:
+    manifest: list[ManifestEntry] = []
+    panel_slices: list[pd.DataFrame] = []
+
+    for asset_name, frame in yahoo_frames.items():
+        temp = frame[["date", "close"]].copy()
+        temp["date"] = pd.to_datetime(temp["date"]).dt.normalize()
+        temp = temp.rename(columns={"close": f"{asset_name}_close"})
+        temp[f"{asset_name}_value"] = temp[f"{asset_name}_close"]
+        temp[f"{asset_name}_change_1"] = temp[f"{asset_name}_value"].diff()
+        temp[f"{asset_name}_return_1"] = temp[f"{asset_name}_close"].pct_change(fill_method=None)
+        panel_slices.append(temp.set_index("date"))
+
+    for asset_name, frame in fred_frames.items():
+        if asset_name in yahoo_frames:
+            continue
+        temp = frame.copy()
+        temp["date"] = pd.to_datetime(temp["date"]).dt.normalize()
+        temp = temp.rename(columns={"value": f"{asset_name}_value"})
+        temp[f"{asset_name}_close"] = temp[f"{asset_name}_value"]
+        temp[f"{asset_name}_change_1"] = temp[f"{asset_name}_value"].diff()
+        if asset_name in FRED_RETURN_SERIES:
+            temp[f"{asset_name}_return_1"] = temp[f"{asset_name}_value"].pct_change(fill_method=None)
+        panel_slices.append(temp.set_index("date"))
+
+    if not panel_slices:
+        return manifest
+
+    panel = pd.concat(panel_slices, axis=1, join="outer").reset_index().sort_values("date").reset_index(drop=True)
+    panel_path = paths["alt_external_processed_dir"] / "external_daily_panel.csv"
+    write_csv(panel, panel_path)
+    manifest.append(summarize_frame("external_panel", "external", "1d", panel_path, panel))
+    return manifest
+
+
+def build_daily_master_panel(
+    paths: dict[str, Path],
+    asset_frames: dict[str, dict[str, pd.DataFrame]],
+) -> list[ManifestEntry]:
+    manifest: list[ManifestEntry] = []
+    panel_slices: list[pd.DataFrame] = []
+
+    for asset, frame_map in asset_frames.items():
+        frame = frame_map["1d"]
+        cols = ["date", "open", "high", "low", "close", "return_1", "realized_vol_20", "rsi_14", "target_return_1"]
+        asset_frame = frame[cols].rename(
+            columns={col: f"{asset}_{col}" for col in cols if col != "date"}
+        )
+        panel_slices.append(asset_frame.set_index("date"))
+
+    panel = pd.concat(panel_slices, axis=1, join="outer").reset_index() if panel_slices else None
+
+    for meta in OPTION_FILES.values():
+        asset = meta["asset"]
+        option_path = paths["alt_options_processed_dir"] / f"{asset}_options_daily_features.csv"
+        if option_path.exists():
+            option_frame = pd.read_csv(option_path, parse_dates=["date"])
+            option_frame = option_frame.rename(columns={col: f"{asset}_opt_{col}" for col in option_frame.columns if col != "date"})
+            panel = panel.merge(option_frame, on="date", how="left") if panel is not None else option_frame
+
+    external_path = paths["alt_external_processed_dir"] / "external_daily_panel.csv"
+    if external_path.exists():
+        external_frame = pd.read_csv(external_path, parse_dates=["date"])
+        panel = panel.merge(external_frame, on="date", how="left") if panel is not None else external_frame
+
+    institutional_path = paths["alt_institutional_processed_dir"] / "institutional_daily_panel.csv"
+    if institutional_path.exists():
+        institutional_frame = pd.read_csv(institutional_path, parse_dates=["date"])
+        institutional_frame = institutional_frame.rename(
+            columns={col: f"institutional_{col}" for col in institutional_frame.columns if col != "date"}
+        )
+        panel = panel.merge(institutional_frame, on="date", how="left") if panel is not None else institutional_frame
+
+    if panel is None:
+        return manifest
+
+    panel = panel.sort_values("date").reset_index(drop=True)
+    panel_path = paths["panels_dir"] / "daily_master_panel.csv"
+    write_csv(panel, panel_path)
+    manifest.append(summarize_frame("panel", "master", "1d", panel_path, panel))
+    return manifest
+
+
+def write_metadata(paths: dict[str, Path], manifest: list[ManifestEntry]) -> None:
+    ensure_dir(paths["metadata_dir"])
+    ensure_dir(paths["alt_metadata_dir"])
+
+    manifest_df = pd.DataFrame([entry.__dict__ for entry in manifest]).sort_values(["category", "asset", "timeframe", "path"])
+    manifest_path = paths["metadata_dir"] / "manifest.csv"
+    write_csv(manifest_df, manifest_path)
+
+    feature_dictionary = pd.DataFrame(
+        [{"feature": key, "description": value} for key, value in FEATURE_DESCRIPTIONS.items()]
+    )
+    feature_dict_path = paths["metadata_dir"] / "feature_dictionary.csv"
+    write_csv(feature_dictionary, feature_dict_path)
+
+    raw_minute_assets = discover_raw_minute_files(paths["raw_minute_dir"])
+    layout = {
+        "project_root": str(PROJECT_ROOT),
+        "data_root": str(DATA_ROOT),
+        "alt_root": str(ALT_ROOT),
+        "code_root": str(CODE_ROOT),
+        "timeframes": ["1m", "5m", "1h", "4h", "1d"],
+        "notes": [
+            "Raw minute files were moved into Data/raw/minute.",
+            "Raw options files were moved into Alt Data/options/raw.",
+            f"Market minute assets currently include: {', '.join(raw_minute_assets.values())}.",
+            "Indices in the provided minute files have zero volume throughout, so engineered volume features are intentionally suppressed and replaced with volume_all_zero.",
+            "Intraday bars are grouped within each observed session to avoid overnight bars leaking across sessions.",
+        ],
+    }
+    layout_path = paths["metadata_dir"] / "layout.json"
+    layout_path.write_text(json.dumps(layout, indent=2), encoding="utf-8")
+
+
+def main() -> None:
+    start_date = "2015-01-01"
+    end_date = (date.today() + timedelta(days=1)).isoformat()
+
+    paths = organize_existing_files()
+    for key in [
+        "bars_dir",
+        "features_dir",
+        "panels_dir",
+        "metadata_dir",
+        "alt_options_processed_dir",
+        "alt_external_raw_dir",
+        "alt_external_processed_dir",
+        "alt_institutional_processed_dir",
+        "alt_metadata_dir",
+    ]:
+        ensure_dir(paths[key])
+
+    asset_frames, manifest = build_market_datasets(paths)
+    manifest.extend(build_multiactive_panels(paths, asset_frames))
+    manifest.extend(build_option_features(paths, asset_frames))
+
+    yahoo_frames, yahoo_manifest = fetch_yahoo_series(start_date, end_date)
+    manifest.extend(yahoo_manifest)
+    fred_frames, fred_manifest = fetch_fred_series(start_date)
+    manifest.extend(fred_manifest)
+    manifest.extend(build_external_panels(yahoo_frames, fred_frames, paths))
+    manifest.extend(build_daily_master_panel(paths, asset_frames))
+
+    write_metadata(paths, manifest)
+    print("Research data build complete.")
+
+
+if __name__ == "__main__":
+    main()

backend/research_runtime/Code/scripts/data_preparation/process_raw_minutes.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import argparse
+import os
+import time
+
+import pandas as pd
+
+import build_research_data as research_data
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Process raw minute OHLCV files into bars, features, and market panels.")
+    parser.add_argument("--workers", type=int, default=None, help="Parallel asset workers. Defaults to the pipeline setting.")
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    if args.workers is not None:
+        os.environ["MARKET_BUILD_WORKERS"] = str(args.workers)
+
+    start = time.perf_counter()
+    paths = research_data.organize_existing_files()
+    for key in ["bars_dir", "features_dir", "panels_dir", "metadata_dir"]:
+        research_data.ensure_dir(paths[key])
+
+    asset_frames, manifest = research_data.build_market_datasets(paths)
+    manifest.extend(research_data.build_multiactive_panels(paths, asset_frames))
+
+    existing_manifest = []
+    manifest_path = paths["metadata_dir"] / "manifest.csv"
+    if manifest_path.exists():
+        old = pd.read_csv(manifest_path)
+        if not old.empty:
+            replace_market = old["category"].isin(["bars", "features"]) | (
+                (old["category"] == "panel") & (old["asset"] == "market")
+            )
+            old = old[~replace_market]
+            for row in old.to_dict("records"):
+                existing_manifest.append(research_data.ManifestEntry(**row))
+
+    research_data.write_metadata(paths, existing_manifest + manifest)
+    elapsed = time.perf_counter() - start
+    print(
+        f"Processed raw minute folder: {len(asset_frames)} assets, "
+        f"{len(manifest)} market entries, {elapsed:.2f}s"
+    )
+
+
+if __name__ == "__main__":
+    main()

backend/runtime_config.example.env

@@ -0,0 +1,14 @@
+# Hugging Face Space backend settings
+FORECASTING_PROJECT_ROOT=/app/research_runtime
+FRONTEND_ORIGINS=https://your-netlify-site.netlify.app
+CRON_SECRET=replace-with-a-long-shared-secret
+HF_DATASET_REPO_ID=your-hf-username/your-forecasting-dataset
+HF_DATASET_REVISION=main
+
+# Automatic update settings
+AUTO_UPDATE_ENABLED=true
+AUTO_RETRAIN_ENABLED=true
+AUTO_UPDATE_ON_START=false
+DAILY_UPDATE_TIME=17:30
+UPDATE_TIMEZONE=Asia/Kolkata
+MARKET_BUILD_WORKERS=2