Upload 20 files

.dockerignore

@@ -0,0 +1,5 @@
+__pycache__/
+*.pyc
+.venv/
+venv/
+.env

Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.11-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+EXPOSE 7860
+
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860", "--ws", "none"]

README.md

@@ -1,11 +1,36 @@
 ---
-title: PREDICTIONSITE
-emoji: 🦀
-colorFrom: gray
-colorTo: red
+title: NIFTY 50 Forecaster Backend
+emoji: 📈
+colorFrom: green
+colorTo: blue
 sdk: docker
-pinned: false
-short_description: PREDICTS STUFF, THATS IT
+app_port: 7860
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# NIFTY 50 Forecaster Backend
+
+FastAPI Hugging Face Docker Space for the NIFTY 50 first-five-minute direction forecaster.
+
+## Endpoints
+
+- `GET /health`
+- `GET /dashboard`
+- `GET /prediction/latest`
+- `POST /prediction/refresh-first5`
+- `POST /data/refresh-daily`
+- `GET /cron/keepalive`
+
+## Data
+
+Parquet files live in `data/`:
+
+- `nifty50_1m.parquet`
+- `nifty50_1d.parquet`
+- `opening_direction_training_dataset.parquet`
+- `test_predictions.parquet`
+
+## Runtime
+
+The API starts a daily background refresh loop. It wakes after `09:20 Asia/Kolkata`, fetches Yahoo Finance `^NSEI` 1-minute candles for the `09:15-09:19` opening window, appends them to Parquet, and writes the latest prediction.
+
+Netlify also pings `/cron/keepalive` every 10 minutes through its scheduled function.

__init__.py

@@ -0,0 +1,2 @@
+"""NIFTY Project backend package."""
+

app.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import asyncio
+from datetime import date
+
+import sys
+from pathlib import Path
+
+from fastapi import FastAPI, Query
+from fastapi.middleware.cors import CORSMiddleware
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from nifty_backend.runtime import dashboard_payload, latest_saved_prediction, refresh_daily_data, refresh_first5_prediction, seconds_until_next_ist_run
+
+
+app = FastAPI(title="NIFTY 50 Forecaster Backend")
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=False,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+async def daily_ist_refresh_loop() -> None:
+    while True:
+        await asyncio.sleep(seconds_until_next_ist_run())
+        try:
+            await asyncio.to_thread(refresh_first5_prediction)
+        except Exception as exc:
+            print(f"[scheduler] first5 refresh failed: {exc}", flush=True)
+        try:
+            await asyncio.to_thread(refresh_daily_data)
+        except Exception as exc:
+            print(f"[scheduler] daily refresh failed: {exc}", flush=True)
+
+
+@app.on_event("startup")
+async def start_scheduler() -> None:
+    asyncio.create_task(daily_ist_refresh_loop())
+
+
+@app.get("/health")
+def health() -> dict[str, str]:
+    return {"status": "ok"}
+
+
+@app.get("/")
+def root() -> dict[str, str]:
+    return {"service": "NIFTY 50 Forecaster Backend", "status": "ok"}
+
+
+@app.get("/dashboard")
+def dashboard() -> dict:
+    return dashboard_payload()
+
+
+@app.get("/cron/keepalive")
+def cron_keepalive() -> dict:
+    return {"status": "awake", "latest": latest_saved_prediction()}
+
+
+@app.get("/prediction/latest")
+def prediction_latest() -> dict:
+    return latest_saved_prediction()
+
+
+@app.post("/prediction/refresh-first5")
+def prediction_refresh_first5(
+    session_date: date | None = Query(default=None, description="Optional YYYY-MM-DD session date in IST."),
+) -> dict:
+    prediction = refresh_first5_prediction(session_date=session_date)
+    return prediction.to_dict()
+
+
+@app.post("/data/refresh-daily")
+def data_refresh_daily() -> dict:
+    return refresh_daily_data()

data/nifty50_1d.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cae5a22e18a378933d815b7b22bb413eeb2e196cdb48504fe8136a4549f7b8a7
+size 78101

data/nifty50_1m.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eff0ea13a459412466def2b530b482e84e06346ee4f7203519689baf0adce32c
+size 18555782

data/opening_direction_training_dataset.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c21f3c5e7781b67e9849a6d07c9ec28d3f087b74be7e2472785124ceb35a34f4
+size 4462258

data/test_predictions.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f159a7499394b7882262ffaa6f9b48c0f6ab763d024f373ee19109b301af90c1
+size 14499

models/candidate_results.csv

@@ -0,0 +1,14 @@
+model_name,threshold,validation_accuracy,test_accuracy,validation_auc,test_auc
+blend_extra_trees_tight_logit_overlay,0.425,0.654320987654321,0.6524064171122995,0.6623500611995106,0.6563861499656042
+blend_extra_trees_tight_logit,0.425,0.6444444444444445,0.6310160427807486,0.6623500611995106,0.6563861499656042
+extra_trees_opening,0.514,0.6345679012345679,0.6203208556149733,0.6448470012239902,0.6575326759917449
+extra_trees_opening_tight,0.514,0.6296296296296297,0.6149732620320856,0.6446511627906977,0.6608576014675532
+soft_vote_tree_pack,0.511,0.6296296296296297,0.6042780748663101,0.6448959608323135,0.6621187800963081
+random_forest_opening,0.516,0.6296296296296297,0.5935828877005348,0.6448959608323134,0.6563861499656042
+extra_trees_opening_deep,0.514,0.6271604938271605,0.6042780748663101,0.6432558139534884,0.6671634946113276
+soft_vote_opening,0.556,0.6246913580246913,0.6256684491978609,0.638359853121175,0.6513414354505846
+gradient_boost_opening,0.512,0.6246913580246913,0.5828877005347594,0.640734394124847,0.631506535198349
+soft_vote_all_pack,0.503,0.6172839506172839,0.5989304812834224,0.6431089351285189,0.6598257280440265
+catboost_opening,0.524,0.6098765432098765,0.5882352941176471,0.6180660954712363,0.6319651456088053
+hist_gradient_opening,0.517,0.5925925925925926,0.5133689839572193,0.6042839657282741,0.5806007796376977
+logit_opening,0.386,0.582716049382716,0.5454545454545454,0.6403427172582619,0.6141939922036231

models/latest_prediction.csv

@@ -0,0 +1,2 @@
+input_date,first5_start,first5_end,prediction,prob_up,confidence,threshold,model_name
+2026-05-21,2026-05-21 09:15:00,2026-05-21 09:19:00,UP,0.4379885393062092,0.5129885393062092,0.425,blend_extra_trees_tight_logit_overlay

models/nifty_opening_direction_model.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:faa463488804279181dc6ee63ca62871d0d5817b0cbb23d34e7374d7628ced98
+size 16234249

models/summary.json

@@ -0,0 +1,29 @@
+{
+  "target": "same-day NIFTY 50 close > same-day NIFTY 50 open after first five 1-minute bars",
+  "model_name": "blend_extra_trees_tight_logit_overlay",
+  "threshold": 0.425,
+  "train_rows": 2221,
+  "valid_rows": 405,
+  "test_rows": 187,
+  "train_start": "2015-01-09",
+  "train_end": "2023-12-29",
+  "valid_start": "2024-01-01",
+  "valid_end": "2025-08-14",
+  "test_start": "2025-08-18",
+  "test_end": "2026-05-21",
+  "validation_accuracy": 0.654320987654321,
+  "test_accuracy": 0.6524064171122995,
+  "baseline_test_accuracy": 0.5240641711229946,
+  "validation_auc": 0.6623500611995106,
+  "test_auc": 0.6563861499656042,
+  "validation_log_loss": 0.6563017134616456,
+  "test_log_loss": 0.6607799782446233,
+  "test_brier": 0.2339533732973711,
+  "feature_count": 219,
+  "latest_input_date": "2026-05-21",
+  "latest_first5_start": "2026-05-21 09:15:00",
+  "latest_first5_end": "2026-05-21 09:19:00",
+  "latest_prob_up": 0.4379885393062093,
+  "latest_prediction": "UP",
+  "latest_confidence": 0.5129885393062092
+}

nifty_backend/__init__.py

@@ -0,0 +1,2 @@
+"""Backend runtime for the NIFTY 50 opening-direction forecaster."""
+

nifty_backend/runtime.py

@@ -0,0 +1,423 @@
+from __future__ import annotations
+
+import json
+import sys
+from dataclasses import dataclass
+from datetime import date, datetime, time, timedelta
+from pathlib import Path
+from typing import Any
+from zoneinfo import ZoneInfo
+
+import joblib
+import numpy as np
+import pandas as pd
+import yfinance as yf
+
+
+IST = ZoneInfo("Asia/Kolkata")
+YAHOO_NIFTY_SYMBOL = "^NSEI"
+BACKEND_ROOT = Path(__file__).resolve().parents[1]
+DATA_DIR = BACKEND_ROOT / "data"
+MODEL_DIR = BACKEND_ROOT / "models"
+OPENING_DATASET_PATH = DATA_DIR / "opening_direction_training_dataset.parquet"
+NIFTY_1M_PATH = DATA_DIR / "nifty50_1m.parquet"
+NIFTY_1D_PATH = DATA_DIR / "nifty50_1d.parquet"
+MODEL_PATH = MODEL_DIR / "nifty_opening_direction_model.joblib"
+LATEST_PATH = MODEL_DIR / "latest_prediction.csv"
+TEST_PREDICTIONS_PATH = DATA_DIR / "test_predictions.parquet"
+
+DECISION_OVERLAYS = [
+    {
+        "name": "fifth_minute_momentum_flip",
+        "feature": "m5_ret_1m",
+        "op": ">=",
+        "value": 0.0005085411885759201,
+    },
+    {
+        "name": "vix_stretch_flip",
+        "feature": "india_vix_close_vs_sma_20",
+        "op": ">=",
+        "value": 0.24641908937959742,
+    },
+]
+
+
+class ProbabilityBlend:
+    def __init__(self, models: list[Any], weights: np.ndarray):
+        self.models = models
+        self.weights = np.asarray(weights, dtype="float64")
+        self.weights = self.weights / self.weights.sum()
+
+    def predict_proba(self, x: pd.DataFrame) -> np.ndarray:
+        probs = np.column_stack([predict_proba_up(model, x) for model in self.models])
+        prob_up = probs @ self.weights
+        return np.column_stack([1.0 - prob_up, prob_up])
+
+
+@dataclass(frozen=True)
+class Prediction:
+    input_date: str
+    first5_start: str
+    first5_end: str
+    prediction: str
+    prob_up: float
+    confidence: float
+    threshold: float
+    model_name: str
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "input_date": self.input_date,
+            "first5_start": self.first5_start,
+            "first5_end": self.first5_end,
+            "prediction": self.prediction,
+            "prob_up": self.prob_up,
+            "confidence": self.confidence,
+            "threshold": self.threshold,
+            "model_name": self.model_name,
+        }
+
+
+def predict_proba_up(model: Any, x: pd.DataFrame) -> np.ndarray:
+    return np.asarray(model.predict_proba(x)[:, 1], dtype="float64")
+
+
+def safe_div(numer: pd.Series | np.ndarray, denom: pd.Series | np.ndarray) -> pd.Series:
+    n = pd.Series(numer, copy=False)
+    d = pd.Series(denom, copy=False)
+    out = pd.Series(np.nan, index=n.index, dtype="float64")
+    mask = d.notna() & np.isfinite(d.to_numpy(dtype="float64")) & (d != 0)
+    out.loc[mask] = n.loc[mask].to_numpy(dtype="float64") / d.loc[mask].to_numpy(dtype="float64")
+    return out
+
+
+def load_model() -> dict[str, Any]:
+    # Existing artifact was trained as a script, so its custom blend class
+    # resolves through __main__ when unpickled.
+    sys.modules["__main__"].ProbabilityBlend = ProbabilityBlend
+    sys.modules["__main__"].predict_proba_up = predict_proba_up
+    payload = joblib.load(MODEL_PATH)
+    payload.setdefault("decision_overlays", DECISION_OVERLAYS)
+    payload.setdefault("model_name", "nifty_opening_direction_model")
+    return payload
+
+
+def overlay_mask(frame: pd.DataFrame, overlay: dict[str, object]) -> np.ndarray:
+    feature = str(overlay["feature"])
+    if feature not in frame.columns:
+        return np.zeros(len(frame), dtype=bool)
+    series = pd.to_numeric(frame[feature], errors="coerce")
+    value = float(overlay["value"])
+    if overlay["op"] == ">=":
+        return (series >= value).fillna(False).to_numpy(dtype=bool)
+    if overlay["op"] == "<=":
+        return (series <= value).fillna(False).to_numpy(dtype=bool)
+    raise ValueError(f"Unsupported overlay op: {overlay['op']}")
+
+
+def apply_decision_overlays(pred: np.ndarray, frame: pd.DataFrame, overlays: list[dict[str, object]]) -> np.ndarray:
+    adjusted = np.asarray(pred, dtype="int64").copy()
+    for overlay in overlays:
+        mask = overlay_mask(frame, overlay)
+        adjusted[mask] = 1 - adjusted[mask]
+    return adjusted
+
+
+def directional_confidence(prob_up: np.ndarray, pred: np.ndarray, threshold: float) -> np.ndarray:
+    prob_up = np.asarray(prob_up, dtype="float64")
+    pred = np.asarray(pred, dtype="int64")
+    base_side_prob = np.where(pred == 1, prob_up, 1.0 - prob_up)
+    threshold_distance = np.abs(prob_up - float(threshold))
+    return np.clip(0.50 + threshold_distance, base_side_prob, 0.99)
+
+
+def read_training_dataset() -> pd.DataFrame:
+    df = pd.read_parquet(OPENING_DATASET_PATH)
+    for col in ("date", "first5_start", "first5_end"):
+        if col in df.columns:
+            df[col] = pd.to_datetime(df[col], errors="coerce")
+    return df.sort_values("date").reset_index(drop=True)
+
+
+def normalize_yahoo_frame(df: pd.DataFrame) -> pd.DataFrame:
+    if df.empty:
+        return pd.DataFrame(columns=["date", "open", "high", "low", "close", "volume"])
+    if isinstance(df.columns, pd.MultiIndex):
+        df.columns = [str(c[0]).lower() for c in df.columns]
+    else:
+        df.columns = [str(c).lower().replace(" ", "_") for c in df.columns]
+    df = df.reset_index()
+    date_col = next((c for c in df.columns if c.lower() in {"datetime", "date"}), df.columns[0])
+    df["date"] = pd.to_datetime(df[date_col], errors="coerce")
+    if df["date"].dt.tz is None:
+        df["date"] = df["date"].dt.tz_localize("UTC").dt.tz_convert(IST)
+    else:
+        df["date"] = df["date"].dt.tz_convert(IST)
+    rename = {
+        "open": "open",
+        "high": "high",
+        "low": "low",
+        "close": "close",
+        "adj_close": "close",
+        "volume": "volume",
+    }
+    out = pd.DataFrame({"date": df["date"].dt.tz_localize(None)})
+    for src, dst in rename.items():
+        if src in df.columns and dst not in out.columns:
+            out[dst] = pd.to_numeric(df[src], errors="coerce")
+    return out.dropna(subset=["date", "open", "high", "low", "close"]).sort_values("date")
+
+
+def fetch_yahoo_minutes(period: str = "5d") -> pd.DataFrame:
+    raw = yf.download(YAHOO_NIFTY_SYMBOL, period=period, interval="1m", progress=False, prepost=False, auto_adjust=False)
+    return normalize_yahoo_frame(raw)
+
+
+def fetch_yahoo_daily(period: str = "1mo") -> pd.DataFrame:
+    raw = yf.download(YAHOO_NIFTY_SYMBOL, period=period, interval="1d", progress=False, prepost=False, auto_adjust=False)
+    out = normalize_yahoo_frame(raw)
+    out["date"] = pd.to_datetime(out["date"], errors="coerce").dt.normalize()
+    return out.drop_duplicates("date", keep="last")
+
+
+def append_parquet_rows(path: Path, new_rows: pd.DataFrame, subset: list[str]) -> pd.DataFrame:
+    if path.exists():
+        existing = pd.read_parquet(path)
+        combined = pd.concat([existing, new_rows], ignore_index=True)
+    else:
+        combined = new_rows.copy()
+    combined = combined.drop_duplicates(subset=subset, keep="last").sort_values(subset).reset_index(drop=True)
+    combined.to_parquet(path, index=False, compression="zstd")
+    return combined
+
+
+def first5_features_from_minutes(minutes: pd.DataFrame, session_date: date | None = None) -> pd.DataFrame:
+    if minutes.empty:
+        raise RuntimeError("Yahoo returned no minute bars.")
+    bars = minutes.copy()
+    bars["dt"] = pd.to_datetime(bars["date"], errors="coerce")
+    bars["session_date"] = bars["dt"].dt.normalize()
+    if session_date is None:
+        session_ts = bars["session_date"].max()
+    else:
+        session_ts = pd.Timestamp(session_date).normalize()
+    day = bars[bars["session_date"] == session_ts].sort_values("dt").copy()
+    start_dt = pd.Timestamp.combine(session_ts.date(), time(9, 15))
+    end_dt = pd.Timestamp.combine(session_ts.date(), time(9, 19))
+    first5 = day[(day["dt"] >= start_dt) & (day["dt"] <= end_dt)].head(5).copy()
+    if len(first5) < 5:
+        raise RuntimeError(f"Need 5 opening bars for {session_ts.date()}, got {len(first5)}.")
+    first5["minute_index"] = np.arange(len(first5))
+    first5["ret_1m"] = first5["close"].pct_change(fill_method=None)
+    first5["range_pct_1m"] = safe_div(first5["high"] - first5["low"], first5["open"])
+    first5["body_pct_1m"] = safe_div(first5["close"] - first5["open"], first5["open"])
+    row = {
+        "date": session_ts,
+        "first5_start": first5["dt"].iloc[0],
+        "first5_end": first5["dt"].iloc[-1],
+        "first5_open": first5["open"].iloc[0],
+        "first5_high": first5["high"].max(),
+        "first5_low": first5["low"].min(),
+        "first5_close": first5["close"].iloc[-1],
+        "first5_volume": first5["volume"].sum() if "volume" in first5 else 0.0,
+        "first5_bars": len(first5),
+        "first5_last_1m_ret": first5["ret_1m"].iloc[-1],
+        "first5_ret_std": first5["ret_1m"].std(),
+    }
+    row["first5_return"] = (row["first5_close"] - row["first5_open"]) / row["first5_open"]
+    row["first5_range_pct"] = (row["first5_high"] - row["first5_low"]) / row["first5_open"]
+    first5_range = row["first5_high"] - row["first5_low"]
+    row["first5_body_to_range"] = (row["first5_close"] - row["first5_open"]) / first5_range if first5_range else np.nan
+    row["first5_close_location"] = (row["first5_close"] - row["first5_low"]) / first5_range if first5_range else np.nan
+    for idx, (_, candle) in enumerate(first5.iterrows(), start=1):
+        for field in ("open", "high", "low", "close", "ret_1m", "range_pct_1m", "body_pct_1m"):
+            row[f"m{idx}_{field}"] = candle[field]
+        row[f"m{idx}_close_vs_first5_open"] = (candle["close"] - row["first5_open"]) / row["first5_open"]
+        row[f"m{idx}_range_share"] = (candle["high"] - candle["low"]) / first5_range if first5_range else np.nan
+    row["first5_return_accel"] = row["m5_ret_1m"] - row["m2_ret_1m"]
+    row["first5_last2_return"] = (row["m5_close"] - row["m4_open"]) / row["m4_open"]
+    row["first5_first2_return"] = (row["m2_close"] - row["m1_open"]) / row["m1_open"]
+    row["first5_reversal"] = np.sign(row["first5_first2_return"]) * -np.sign(row["first5_last2_return"])
+    row["dow"] = session_ts.dayofweek
+    row["dom"] = session_ts.day
+    row["month"] = session_ts.month
+    return pd.DataFrame([row])
+
+
+def build_model_row(first5_row: pd.DataFrame) -> pd.DataFrame:
+    dataset = read_training_dataset()
+    latest_context = dataset.iloc[[-1]].copy()
+    output = latest_context.copy()
+    for col in first5_row.columns:
+        output[col] = first5_row[col].iloc[0]
+    if {"first5_open", "nifty_close"}.issubset(output.columns):
+        output["first5_gap_from_prev_close"] = (output["first5_open"] - output["nifty_close"]) / output["nifty_close"]
+        output["first5_close_vs_prev_close"] = (output["first5_close"] - output["nifty_close"]) / output["nifty_close"]
+    if {"first5_range_pct", "nifty_range_pct"}.issubset(output.columns):
+        output["first5_range_vs_prev_range"] = output["first5_range_pct"] / output["nifty_range_pct"]
+    if {"first5_return", "nifty_ret_1"}.issubset(output.columns):
+        output["first5_return_x_prev_ret"] = output["first5_return"] * output["nifty_ret_1"]
+        output["gap_x_prev_ret"] = output["first5_gap_from_prev_close"] * output["nifty_ret_1"]
+    if {"first5_return", "banknifty_ret_1"}.issubset(output.columns):
+        output["first5_return_x_bank_ret_1"] = output["first5_return"] * output["banknifty_ret_1"]
+    if {"first5_range_pct", "india_vix_ret_1"}.issubset(output.columns):
+        output["first5_range_x_vix_ret_1"] = output["first5_range_pct"] * output["india_vix_ret_1"]
+    output["target"] = np.nan
+    output["day_return"] = np.nan
+    return output
+
+
+def predict_row(row: pd.DataFrame) -> Prediction:
+    payload = load_model()
+    model = payload["model"]
+    features = payload["features"]
+    threshold = float(payload["threshold"])
+    missing = [c for c in features if c not in row.columns]
+    if missing:
+        raise RuntimeError(f"Feature row is missing {len(missing)} features; first missing: {missing[:5]}")
+    prob_up = predict_proba_up(model, row[features])
+    raw_pred = (prob_up >= threshold).astype("int64")
+    pred = apply_decision_overlays(raw_pred, row, payload.get("decision_overlays", DECISION_OVERLAYS))
+    confidence = directional_confidence(prob_up, pred, threshold)
+    prediction = Prediction(
+        input_date=pd.to_datetime(row["date"].iloc[0]).date().isoformat(),
+        first5_start=str(pd.to_datetime(row["first5_start"].iloc[0])),
+        first5_end=str(pd.to_datetime(row["first5_end"].iloc[0])),
+        prediction="UP" if int(pred[0]) == 1 else "DOWN",
+        prob_up=float(prob_up[0]),
+        confidence=float(confidence[0]),
+        threshold=threshold,
+        model_name=str(payload.get("model_name", "nifty_opening_direction_model")),
+    )
+    pd.DataFrame([prediction.to_dict()]).to_csv(LATEST_PATH, index=False)
+    return prediction
+
+
+def latest_saved_prediction() -> dict[str, Any]:
+    if LATEST_PATH.exists():
+        return pd.read_csv(LATEST_PATH).iloc[-1].to_dict()
+    summary_path = MODEL_DIR / "summary.json"
+    if summary_path.exists():
+        return json.loads(summary_path.read_text(encoding="utf-8"))
+    raise FileNotFoundError("No latest prediction is available yet.")
+
+
+def _json_ready_frame(df: pd.DataFrame, limit: int | None = None) -> list[dict[str, Any]]:
+    out = df.copy()
+    if limit is not None:
+        out = out.tail(limit)
+    for col in out.columns:
+        if pd.api.types.is_datetime64_any_dtype(out[col]):
+            out[col] = out[col].dt.strftime("%Y-%m-%d %H:%M:%S")
+    out = out.replace({np.nan: None})
+    return out.to_dict(orient="records")
+
+
+def load_model_summary() -> dict[str, Any]:
+    summary_path = MODEL_DIR / "summary.json"
+    if not summary_path.exists():
+        return {}
+    return json.loads(summary_path.read_text(encoding="utf-8"))
+
+
+def load_candidate_results() -> list[dict[str, Any]]:
+    path = MODEL_DIR / "candidate_results.csv"
+    if not path.exists():
+        return []
+    return _json_ready_frame(pd.read_csv(path).head(12))
+
+
+def load_test_predictions() -> pd.DataFrame:
+    if not TEST_PREDICTIONS_PATH.exists():
+        return pd.DataFrame()
+    df = pd.read_parquet(TEST_PREDICTIONS_PATH)
+    df["date"] = pd.to_datetime(df["date"], errors="coerce")
+    return df.sort_values("date").reset_index(drop=True)
+
+
+def dashboard_payload() -> dict[str, Any]:
+    summary = load_model_summary()
+    latest = latest_saved_prediction()
+    test = load_test_predictions()
+    daily = pd.read_parquet(NIFTY_1D_PATH)
+    daily["date"] = pd.to_datetime(daily["date"], errors="coerce")
+    daily = daily.sort_values("date").tail(180)
+    dataset = read_training_dataset()
+    opening = dataset[["date", "first5_return", "first5_range_pct", "first5_close_location"]].tail(120).copy()
+
+    if not test.empty:
+        recent_predictions = test.tail(40).copy()
+        recent_accuracy = float(recent_predictions["correct"].mean())
+        direction_mix = test.groupby("prediction")["correct"].agg(["count", "mean"]).reset_index()
+        monthly = (
+            test.assign(month=test["date"].dt.strftime("%Y-%m"))
+            .groupby("month", as_index=False)["correct"]
+            .mean()
+            .rename(columns={"correct": "accuracy"})
+        )
+    else:
+        recent_predictions = pd.DataFrame()
+        recent_accuracy = None
+        direction_mix = pd.DataFrame()
+        monthly = pd.DataFrame()
+
+    metrics = {
+        "validation_accuracy": summary.get("validation_accuracy"),
+        "test_accuracy": summary.get("test_accuracy"),
+        "baseline_test_accuracy": summary.get("baseline_test_accuracy"),
+        "validation_auc": summary.get("validation_auc"),
+        "test_auc": summary.get("test_auc"),
+        "test_brier": summary.get("test_brier"),
+        "feature_count": summary.get("feature_count"),
+        "recent_accuracy": recent_accuracy,
+    }
+    return {
+        "latest": latest,
+        "metrics": metrics,
+        "summary": summary,
+        "candidates": load_candidate_results(),
+        "charts": {
+            "daily_close": _json_ready_frame(daily[["date", "open", "high", "low", "close"]]),
+            "opening_features": _json_ready_frame(opening),
+            "monthly_accuracy": _json_ready_frame(monthly),
+            "direction_mix": _json_ready_frame(direction_mix),
+            "recent_predictions": _json_ready_frame(recent_predictions),
+        },
+        "data_status": {
+            "nifty_1m_rows": int(len(pd.read_parquet(NIFTY_1M_PATH, columns=["date"]))),
+            "nifty_1d_rows": int(len(pd.read_parquet(NIFTY_1D_PATH, columns=["date"]))),
+            "training_rows": int(len(dataset)),
+            "test_prediction_rows": int(len(test)),
+            "latest_daily_date": pd.to_datetime(daily["date"]).max().date().isoformat(),
+        },
+    }
+
+
+def refresh_first5_prediction(session_date: date | None = None) -> Prediction:
+    minutes = fetch_yahoo_minutes(period="5d")
+    append_parquet_rows(NIFTY_1M_PATH, minutes, ["date"])
+    first5 = first5_features_from_minutes(minutes, session_date=session_date)
+    row = build_model_row(first5)
+    dataset = read_training_dataset()
+    merged = pd.concat([dataset, row], ignore_index=True)
+    merged = merged.drop_duplicates(subset=["date"], keep="last").sort_values("date").reset_index(drop=True)
+    merged.to_parquet(OPENING_DATASET_PATH, index=False, compression="zstd")
+    return predict_row(row)
+
+
+def refresh_daily_data() -> dict[str, Any]:
+    daily = fetch_yahoo_daily(period="1mo")
+    combined = append_parquet_rows(NIFTY_1D_PATH, daily, ["date"])
+    return {
+        "rows": int(len(combined)),
+        "latest_date": pd.to_datetime(combined["date"]).max().date().isoformat(),
+        "path": str(NIFTY_1D_PATH),
+    }
+
+
+def seconds_until_next_ist_run(run_time: time = time(9, 20)) -> float:
+    now = datetime.now(IST)
+    target = datetime.combine(now.date(), run_time, tzinfo=IST)
+    if now >= target:
+        target += timedelta(days=1)
+    return max(1.0, (target - now).total_seconds())

requirements.txt

@@ -0,0 +1,9 @@
+pandas
+pyarrow
+yfinance
+fastapi
+uvicorn
+joblib
+numpy
+scikit-learn
+catboost

scripts/refresh_daily_data.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from nifty_backend.runtime import refresh_daily_data
+
+
+def main() -> None:
+    print(refresh_daily_data())
+
+
+if __name__ == "__main__":
+    main()

scripts/refresh_first5_prediction.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+import argparse
+import sys
+from datetime import date
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from nifty_backend.runtime import refresh_first5_prediction
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Fetch Yahoo Finance first five NIFTY minutes and refresh prediction.")
+    parser.add_argument("--date", default=None, help="Optional IST session date, YYYY-MM-DD.")
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    session_date = date.fromisoformat(args.date) if args.date else None
+    prediction = refresh_first5_prediction(session_date=session_date)
+    print(prediction.to_dict())
+
+
+if __name__ == "__main__":
+    main()

scripts/retrain_opening_model.py

@@ -0,0 +1,182 @@
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import joblib
+import numpy as np
+import pandas as pd
+from sklearn.ensemble import ExtraTreesClassifier
+from sklearn.impute import SimpleImputer
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import accuracy_score, brier_score_loss, log_loss, roc_auc_score
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import StandardScaler
+
+from nifty_backend.runtime import (
+    DECISION_OVERLAYS,
+    MODEL_DIR,
+    MODEL_PATH,
+    OPENING_DATASET_PATH,
+    ProbabilityBlend,
+    apply_decision_overlays,
+    directional_confidence,
+    predict_proba_up,
+)
+
+
+DEFAULT_TRAIN_END = pd.Timestamp("2023-12-31")
+DEFAULT_VALID_END = pd.Timestamp("2025-08-17")
+RANDOM_SEED = 42
+
+
+@dataclass
+class RetrainSummary:
+    model_name: str
+    threshold: float
+    train_rows: int
+    valid_rows: int
+    test_rows: int
+    validation_accuracy: float
+    test_accuracy: float
+    validation_auc: float
+    test_auc: float
+    test_brier: float
+    latest_prediction: str
+    latest_prob_up: float
+    latest_confidence: float
+    feature_count: int
+
+
+def feature_columns(frame: pd.DataFrame) -> list[str]:
+    excluded = {
+        "date",
+        "first5_start",
+        "first5_end",
+        "target",
+        "day_open",
+        "day_high",
+        "day_low",
+        "day_close",
+        "day_volume",
+        "day_return",
+    }
+    cols = []
+    for col in frame.columns:
+        if col in excluded:
+            continue
+        if pd.api.types.is_numeric_dtype(frame[col]) and frame[col].notna().mean() >= 0.40:
+            if frame[col].nunique(dropna=True) > 1:
+                cols.append(col)
+    return cols
+
+
+def best_threshold(y_true: np.ndarray, prob_up: np.ndarray) -> tuple[float, float]:
+    thresholds = np.linspace(0.35, 0.65, 301)
+    scores = ((prob_up[:, None] >= thresholds[None, :]) == y_true[:, None]).mean(axis=0)
+    idx = int(np.argmax(scores))
+    return float(thresholds[idx]), float(scores[idx])
+
+
+def score_auc(y_true: np.ndarray, prob_up: np.ndarray) -> float:
+    if len(np.unique(y_true)) < 2:
+        return float("nan")
+    return float(roc_auc_score(y_true, prob_up))
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Retrain the compact NIFTY opening-direction model from Parquet data.")
+    parser.add_argument("--train-end", default=DEFAULT_TRAIN_END.date().isoformat())
+    parser.add_argument("--valid-end", default=DEFAULT_VALID_END.date().isoformat())
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    train_end = pd.Timestamp(args.train_end)
+    valid_end = pd.Timestamp(args.valid_end)
+    frame = pd.read_parquet(OPENING_DATASET_PATH)
+    frame["date"] = pd.to_datetime(frame["date"], errors="coerce")
+    model_frame = frame.dropna(subset=["target"]).sort_values("date").reset_index(drop=True)
+    features = feature_columns(model_frame)
+    train_df = model_frame[model_frame["date"] <= train_end]
+    valid_df = model_frame[(model_frame["date"] > train_end) & (model_frame["date"] <= valid_end)]
+    test_df = model_frame[model_frame["date"] > valid_end]
+    if train_df.empty or valid_df.empty or test_df.empty:
+        raise RuntimeError("Training, validation, and test windows must all contain rows.")
+
+    x_train = train_df[features]
+    y_train = train_df["target"].to_numpy(dtype="int64")
+    x_valid = valid_df[features]
+    y_valid = valid_df["target"].to_numpy(dtype="int64")
+    x_test = test_df[features]
+    y_test = test_df["target"].to_numpy(dtype="int64")
+
+    extra_trees = make_pipeline(
+        SimpleImputer(strategy="median"),
+        ExtraTreesClassifier(
+            n_estimators=800,
+            max_depth=4,
+            min_samples_leaf=28,
+            max_features=0.60,
+            class_weight="balanced_subsample",
+            random_state=RANDOM_SEED + 13,
+            n_jobs=-1,
+        ),
+    )
+    logit = make_pipeline(
+        SimpleImputer(strategy="median"),
+        StandardScaler(),
+        LogisticRegression(C=0.25, class_weight="balanced", max_iter=2000, random_state=RANDOM_SEED),
+    )
+    extra_trees.fit(x_train, y_train)
+    logit.fit(x_train, y_train)
+    model = ProbabilityBlend([extra_trees, logit], np.array([0.75, 0.25]))
+    valid_prob = predict_proba_up(model, x_valid)
+    test_prob = predict_proba_up(model, x_test)
+    threshold, _ = best_threshold(y_valid, valid_prob)
+    valid_pred = apply_decision_overlays((valid_prob >= threshold).astype("int64"), valid_df, DECISION_OVERLAYS)
+    test_pred = apply_decision_overlays((test_prob >= threshold).astype("int64"), test_df, DECISION_OVERLAYS)
+    latest = frame.iloc[[-1]].copy()
+    latest_prob = predict_proba_up(model, latest[features])
+    latest_pred = apply_decision_overlays((latest_prob >= threshold).astype("int64"), latest, DECISION_OVERLAYS)
+    latest_conf = directional_confidence(latest_prob, latest_pred, threshold)
+
+    payload = {
+        "model": model,
+        "features": features,
+        "threshold": threshold,
+        "target": "same-day NIFTY 50 close > same-day NIFTY 50 open after first five 1-minute bars",
+        "model_name": "compact_extra_trees_logit_overlay",
+        "decision_overlays": DECISION_OVERLAYS,
+    }
+    joblib.dump(payload, MODEL_PATH)
+
+    summary = RetrainSummary(
+        model_name=payload["model_name"],
+        threshold=float(threshold),
+        train_rows=int(len(train_df)),
+        valid_rows=int(len(valid_df)),
+        test_rows=int(len(test_df)),
+        validation_accuracy=float(accuracy_score(y_valid, valid_pred)),
+        test_accuracy=float(accuracy_score(y_test, test_pred)),
+        validation_auc=score_auc(y_valid, valid_prob),
+        test_auc=score_auc(y_test, test_prob),
+        test_brier=float(brier_score_loss(y_test, np.clip(test_prob, 1e-6, 1 - 1e-6))),
+        latest_prediction="UP" if int(latest_pred[0]) == 1 else "DOWN",
+        latest_prob_up=float(latest_prob[0]),
+        latest_confidence=float(latest_conf[0]),
+        feature_count=int(len(features)),
+    )
+    (MODEL_DIR / "summary.json").write_text(json.dumps(asdict(summary), indent=2), encoding="utf-8")
+    pd.DataFrame([asdict(summary)]).to_csv(MODEL_DIR / "retrain_summary.csv", index=False)
+    print(asdict(summary))
+
+
+if __name__ == "__main__":
+    main()

scripts/run_ist_scheduler.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+import time
+import sys
+from datetime import datetime
+from pathlib import Path
+from zoneinfo import ZoneInfo
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from nifty_backend.runtime import refresh_daily_data, refresh_first5_prediction, seconds_until_next_ist_run
+
+
+IST = ZoneInfo("Asia/Kolkata")
+
+
+def main() -> None:
+    print("[scheduler] NIFTY first-five-minute scheduler started.")
+    print("[scheduler] Runs the opening prediction after 09:20 IST so the 09:15-09:19 candles are complete.")
+    while True:
+        sleep_for = seconds_until_next_ist_run()
+        target = datetime.now(IST).timestamp() + sleep_for
+        print(f"[scheduler] sleeping {sleep_for / 60:.1f} minutes; next wake timestamp={target:.0f}")
+        time.sleep(sleep_for)
+        try:
+            prediction = refresh_first5_prediction()
+            print(f"[scheduler] first5 prediction refreshed: {prediction.to_dict()}")
+        except Exception as exc:
+            print(f"[scheduler] first5 refresh failed: {exc}")
+        try:
+            info = refresh_daily_data()
+            print(f"[scheduler] daily data refreshed: {info}")
+        except Exception as exc:
+            print(f"[scheduler] daily refresh failed: {exc}")
+
+
+if __name__ == "__main__":
+    main()