Delete backend

backend/.dockerignore

@@ -1,5 +0,0 @@
-__pycache__/
-*.pyc
-.venv/
-venv/
-.env

backend/Dockerfile

@@ -1,15 +0,0 @@
-FROM python:3.11-slim
-
-ENV PYTHONDONTWRITEBYTECODE=1
-ENV PYTHONUNBUFFERED=1
-
-WORKDIR /app
-
-COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
-
-COPY . .
-
-EXPOSE 7860
-
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860", "--ws", "none"]

backend/README.md

@@ -1,36 +0,0 @@
----
-title: NIFTY 50 Forecaster Backend
-emoji: 📈
-colorFrom: green
-colorTo: blue
-sdk: docker
-app_port: 7860
----
-
-# NIFTY 50 Forecaster Backend
-
-FastAPI Hugging Face Docker Space for the NIFTY 50 first-five-minute direction forecaster.
-
-## Endpoints
-
-- `GET /health`
-- `GET /dashboard`
-- `GET /prediction/latest`
-- `POST /prediction/refresh-first5`
-- `POST /data/refresh-daily`
-- `GET /cron/keepalive`
-
-## Data
-
-Parquet files live in `data/`:
-
-- `nifty50_1m.parquet`
-- `nifty50_1d.parquet`
-- `opening_direction_training_dataset.parquet`
-- `test_predictions.parquet`
-
-## Runtime
-
-The API starts a daily background refresh loop. It wakes after `09:20 Asia/Kolkata`, fetches Yahoo Finance `^NSEI` 1-minute candles for the `09:15-09:19` opening window, appends them to Parquet, and writes the latest prediction.
-
-Netlify also pings `/cron/keepalive` every 10 minutes through its scheduled function.

backend/__init__.py

@@ -1,2 +0,0 @@
-"""NIFTY Project backend package."""
-

backend/app.py

@@ -1,79 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-from datetime import date
-
-import sys
-from pathlib import Path
-
-from fastapi import FastAPI, Query
-from fastapi.middleware.cors import CORSMiddleware
-
-sys.path.insert(0, str(Path(__file__).resolve().parent))
-from nifty_backend.runtime import dashboard_payload, latest_saved_prediction, refresh_daily_data, refresh_first5_prediction, seconds_until_next_ist_run
-
-
-app = FastAPI(title="NIFTY 50 Forecaster Backend")
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_credentials=False,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
-
-
-async def daily_ist_refresh_loop() -> None:
-    while True:
-        await asyncio.sleep(seconds_until_next_ist_run())
-        try:
-            await asyncio.to_thread(refresh_first5_prediction)
-        except Exception as exc:
-            print(f"[scheduler] first5 refresh failed: {exc}", flush=True)
-        try:
-            await asyncio.to_thread(refresh_daily_data)
-        except Exception as exc:
-            print(f"[scheduler] daily refresh failed: {exc}", flush=True)
-
-
-@app.on_event("startup")
-async def start_scheduler() -> None:
-    asyncio.create_task(daily_ist_refresh_loop())
-
-
-@app.get("/health")
-def health() -> dict[str, str]:
-    return {"status": "ok"}
-
-
-@app.get("/")
-def root() -> dict[str, str]:
-    return {"service": "NIFTY 50 Forecaster Backend", "status": "ok"}
-
-
-@app.get("/dashboard")
-def dashboard() -> dict:
-    return dashboard_payload()
-
-
-@app.get("/cron/keepalive")
-def cron_keepalive() -> dict:
-    return {"status": "awake", "latest": latest_saved_prediction()}
-
-
-@app.get("/prediction/latest")
-def prediction_latest() -> dict:
-    return latest_saved_prediction()
-
-
-@app.post("/prediction/refresh-first5")
-def prediction_refresh_first5(
-    session_date: date | None = Query(default=None, description="Optional YYYY-MM-DD session date in IST."),
-) -> dict:
-    prediction = refresh_first5_prediction(session_date=session_date)
-    return prediction.to_dict()
-
-
-@app.post("/data/refresh-daily")
-def data_refresh_daily() -> dict:
-    return refresh_daily_data()

backend/data/nifty50_1d.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:cae5a22e18a378933d815b7b22bb413eeb2e196cdb48504fe8136a4549f7b8a7
-size 78101

backend/data/nifty50_1m.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:eff0ea13a459412466def2b530b482e84e06346ee4f7203519689baf0adce32c
-size 18555782

backend/data/opening_direction_training_dataset.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c21f3c5e7781b67e9849a6d07c9ec28d3f087b74be7e2472785124ceb35a34f4
-size 4462258

backend/data/test_predictions.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f159a7499394b7882262ffaa6f9b48c0f6ab763d024f373ee19109b301af90c1
-size 14499

backend/models/candidate_results.csv

@@ -1,14 +0,0 @@
-model_name,threshold,validation_accuracy,test_accuracy,validation_auc,test_auc
-blend_extra_trees_tight_logit_overlay,0.425,0.654320987654321,0.6524064171122995,0.6623500611995106,0.6563861499656042
-blend_extra_trees_tight_logit,0.425,0.6444444444444445,0.6310160427807486,0.6623500611995106,0.6563861499656042
-extra_trees_opening,0.514,0.6345679012345679,0.6203208556149733,0.6448470012239902,0.6575326759917449
-extra_trees_opening_tight,0.514,0.6296296296296297,0.6149732620320856,0.6446511627906977,0.6608576014675532
-soft_vote_tree_pack,0.511,0.6296296296296297,0.6042780748663101,0.6448959608323135,0.6621187800963081
-random_forest_opening,0.516,0.6296296296296297,0.5935828877005348,0.6448959608323134,0.6563861499656042
-extra_trees_opening_deep,0.514,0.6271604938271605,0.6042780748663101,0.6432558139534884,0.6671634946113276
-soft_vote_opening,0.556,0.6246913580246913,0.6256684491978609,0.638359853121175,0.6513414354505846
-gradient_boost_opening,0.512,0.6246913580246913,0.5828877005347594,0.640734394124847,0.631506535198349
-soft_vote_all_pack,0.503,0.6172839506172839,0.5989304812834224,0.6431089351285189,0.6598257280440265
-catboost_opening,0.524,0.6098765432098765,0.5882352941176471,0.6180660954712363,0.6319651456088053
-hist_gradient_opening,0.517,0.5925925925925926,0.5133689839572193,0.6042839657282741,0.5806007796376977
-logit_opening,0.386,0.582716049382716,0.5454545454545454,0.6403427172582619,0.6141939922036231

backend/models/latest_prediction.csv

@@ -1,2 +0,0 @@
-input_date,first5_start,first5_end,prediction,prob_up,confidence,threshold,model_name
-2026-05-21,2026-05-21 09:15:00,2026-05-21 09:19:00,UP,0.4379885393062092,0.5129885393062092,0.425,blend_extra_trees_tight_logit_overlay

backend/models/nifty_opening_direction_model.joblib

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:faa463488804279181dc6ee63ca62871d0d5817b0cbb23d34e7374d7628ced98
-size 16234249

backend/models/summary.json

@@ -1,29 +0,0 @@
-{
-  "target": "same-day NIFTY 50 close > same-day NIFTY 50 open after first five 1-minute bars",
-  "model_name": "blend_extra_trees_tight_logit_overlay",
-  "threshold": 0.425,
-  "train_rows": 2221,
-  "valid_rows": 405,
-  "test_rows": 187,
-  "train_start": "2015-01-09",
-  "train_end": "2023-12-29",
-  "valid_start": "2024-01-01",
-  "valid_end": "2025-08-14",
-  "test_start": "2025-08-18",
-  "test_end": "2026-05-21",
-  "validation_accuracy": 0.654320987654321,
-  "test_accuracy": 0.6524064171122995,
-  "baseline_test_accuracy": 0.5240641711229946,
-  "validation_auc": 0.6623500611995106,
-  "test_auc": 0.6563861499656042,
-  "validation_log_loss": 0.6563017134616456,
-  "test_log_loss": 0.6607799782446233,
-  "test_brier": 0.2339533732973711,
-  "feature_count": 219,
-  "latest_input_date": "2026-05-21",
-  "latest_first5_start": "2026-05-21 09:15:00",
-  "latest_first5_end": "2026-05-21 09:19:00",
-  "latest_prob_up": 0.4379885393062093,
-  "latest_prediction": "UP",
-  "latest_confidence": 0.5129885393062092
-}

backend/nifty_backend/__init__.py

@@ -1,2 +0,0 @@
-"""Backend runtime for the NIFTY 50 opening-direction forecaster."""
-

backend/nifty_backend/runtime.py

@@ -1,423 +0,0 @@
-from __future__ import annotations
-
-import json
-import sys
-from dataclasses import dataclass
-from datetime import date, datetime, time, timedelta
-from pathlib import Path
-from typing import Any
-from zoneinfo import ZoneInfo
-
-import joblib
-import numpy as np
-import pandas as pd
-import yfinance as yf
-
-
-IST = ZoneInfo("Asia/Kolkata")
-YAHOO_NIFTY_SYMBOL = "^NSEI"
-BACKEND_ROOT = Path(__file__).resolve().parents[1]
-DATA_DIR = BACKEND_ROOT / "data"
-MODEL_DIR = BACKEND_ROOT / "models"
-OPENING_DATASET_PATH = DATA_DIR / "opening_direction_training_dataset.parquet"
-NIFTY_1M_PATH = DATA_DIR / "nifty50_1m.parquet"
-NIFTY_1D_PATH = DATA_DIR / "nifty50_1d.parquet"
-MODEL_PATH = MODEL_DIR / "nifty_opening_direction_model.joblib"
-LATEST_PATH = MODEL_DIR / "latest_prediction.csv"
-TEST_PREDICTIONS_PATH = DATA_DIR / "test_predictions.parquet"
-
-DECISION_OVERLAYS = [
-    {
-        "name": "fifth_minute_momentum_flip",
-        "feature": "m5_ret_1m",
-        "op": ">=",
-        "value": 0.0005085411885759201,
-    },
-    {
-        "name": "vix_stretch_flip",
-        "feature": "india_vix_close_vs_sma_20",
-        "op": ">=",
-        "value": 0.24641908937959742,
-    },
-]
-
-
-class ProbabilityBlend:
-    def __init__(self, models: list[Any], weights: np.ndarray):
-        self.models = models
-        self.weights = np.asarray(weights, dtype="float64")
-        self.weights = self.weights / self.weights.sum()
-
-    def predict_proba(self, x: pd.DataFrame) -> np.ndarray:
-        probs = np.column_stack([predict_proba_up(model, x) for model in self.models])
-        prob_up = probs @ self.weights
-        return np.column_stack([1.0 - prob_up, prob_up])
-
-
-@dataclass(frozen=True)
-class Prediction:
-    input_date: str
-    first5_start: str
-    first5_end: str
-    prediction: str
-    prob_up: float
-    confidence: float
-    threshold: float
-    model_name: str
-
-    def to_dict(self) -> dict[str, Any]:
-        return {
-            "input_date": self.input_date,
-            "first5_start": self.first5_start,
-            "first5_end": self.first5_end,
-            "prediction": self.prediction,
-            "prob_up": self.prob_up,
-            "confidence": self.confidence,
-            "threshold": self.threshold,
-            "model_name": self.model_name,
-        }
-
-
-def predict_proba_up(model: Any, x: pd.DataFrame) -> np.ndarray:
-    return np.asarray(model.predict_proba(x)[:, 1], dtype="float64")
-
-
-def safe_div(numer: pd.Series | np.ndarray, denom: pd.Series | np.ndarray) -> pd.Series:
-    n = pd.Series(numer, copy=False)
-    d = pd.Series(denom, copy=False)
-    out = pd.Series(np.nan, index=n.index, dtype="float64")
-    mask = d.notna() & np.isfinite(d.to_numpy(dtype="float64")) & (d != 0)
-    out.loc[mask] = n.loc[mask].to_numpy(dtype="float64") / d.loc[mask].to_numpy(dtype="float64")
-    return out
-
-
-def load_model() -> dict[str, Any]:
-    # Existing artifact was trained as a script, so its custom blend class
-    # resolves through __main__ when unpickled.
-    sys.modules["__main__"].ProbabilityBlend = ProbabilityBlend
-    sys.modules["__main__"].predict_proba_up = predict_proba_up
-    payload = joblib.load(MODEL_PATH)
-    payload.setdefault("decision_overlays", DECISION_OVERLAYS)
-    payload.setdefault("model_name", "nifty_opening_direction_model")
-    return payload
-
-
-def overlay_mask(frame: pd.DataFrame, overlay: dict[str, object]) -> np.ndarray:
-    feature = str(overlay["feature"])
-    if feature not in frame.columns:
-        return np.zeros(len(frame), dtype=bool)
-    series = pd.to_numeric(frame[feature], errors="coerce")
-    value = float(overlay["value"])
-    if overlay["op"] == ">=":
-        return (series >= value).fillna(False).to_numpy(dtype=bool)
-    if overlay["op"] == "<=":
-        return (series <= value).fillna(False).to_numpy(dtype=bool)
-    raise ValueError(f"Unsupported overlay op: {overlay['op']}")
-
-
-def apply_decision_overlays(pred: np.ndarray, frame: pd.DataFrame, overlays: list[dict[str, object]]) -> np.ndarray:
-    adjusted = np.asarray(pred, dtype="int64").copy()
-    for overlay in overlays:
-        mask = overlay_mask(frame, overlay)
-        adjusted[mask] = 1 - adjusted[mask]
-    return adjusted
-
-
-def directional_confidence(prob_up: np.ndarray, pred: np.ndarray, threshold: float) -> np.ndarray:
-    prob_up = np.asarray(prob_up, dtype="float64")
-    pred = np.asarray(pred, dtype="int64")
-    base_side_prob = np.where(pred == 1, prob_up, 1.0 - prob_up)
-    threshold_distance = np.abs(prob_up - float(threshold))
-    return np.clip(0.50 + threshold_distance, base_side_prob, 0.99)
-
-
-def read_training_dataset() -> pd.DataFrame:
-    df = pd.read_parquet(OPENING_DATASET_PATH)
-    for col in ("date", "first5_start", "first5_end"):
-        if col in df.columns:
-            df[col] = pd.to_datetime(df[col], errors="coerce")
-    return df.sort_values("date").reset_index(drop=True)
-
-
-def normalize_yahoo_frame(df: pd.DataFrame) -> pd.DataFrame:
-    if df.empty:
-        return pd.DataFrame(columns=["date", "open", "high", "low", "close", "volume"])
-    if isinstance(df.columns, pd.MultiIndex):
-        df.columns = [str(c[0]).lower() for c in df.columns]
-    else:
-        df.columns = [str(c).lower().replace(" ", "_") for c in df.columns]
-    df = df.reset_index()
-    date_col = next((c for c in df.columns if c.lower() in {"datetime", "date"}), df.columns[0])
-    df["date"] = pd.to_datetime(df[date_col], errors="coerce")
-    if df["date"].dt.tz is None:
-        df["date"] = df["date"].dt.tz_localize("UTC").dt.tz_convert(IST)
-    else:
-        df["date"] = df["date"].dt.tz_convert(IST)
-    rename = {
-        "open": "open",
-        "high": "high",
-        "low": "low",
-        "close": "close",
-        "adj_close": "close",
-        "volume": "volume",
-    }
-    out = pd.DataFrame({"date": df["date"].dt.tz_localize(None)})
-    for src, dst in rename.items():
-        if src in df.columns and dst not in out.columns:
-            out[dst] = pd.to_numeric(df[src], errors="coerce")
-    return out.dropna(subset=["date", "open", "high", "low", "close"]).sort_values("date")
-
-
-def fetch_yahoo_minutes(period: str = "5d") -> pd.DataFrame:
-    raw = yf.download(YAHOO_NIFTY_SYMBOL, period=period, interval="1m", progress=False, prepost=False, auto_adjust=False)
-    return normalize_yahoo_frame(raw)
-
-
-def fetch_yahoo_daily(period: str = "1mo") -> pd.DataFrame:
-    raw = yf.download(YAHOO_NIFTY_SYMBOL, period=period, interval="1d", progress=False, prepost=False, auto_adjust=False)
-    out = normalize_yahoo_frame(raw)
-    out["date"] = pd.to_datetime(out["date"], errors="coerce").dt.normalize()
-    return out.drop_duplicates("date", keep="last")
-
-
-def append_parquet_rows(path: Path, new_rows: pd.DataFrame, subset: list[str]) -> pd.DataFrame:
-    if path.exists():
-        existing = pd.read_parquet(path)
-        combined = pd.concat([existing, new_rows], ignore_index=True)
-    else:
-        combined = new_rows.copy()
-    combined = combined.drop_duplicates(subset=subset, keep="last").sort_values(subset).reset_index(drop=True)
-    combined.to_parquet(path, index=False, compression="zstd")
-    return combined
-
-
-def first5_features_from_minutes(minutes: pd.DataFrame, session_date: date | None = None) -> pd.DataFrame:
-    if minutes.empty:
-        raise RuntimeError("Yahoo returned no minute bars.")
-    bars = minutes.copy()
-    bars["dt"] = pd.to_datetime(bars["date"], errors="coerce")
-    bars["session_date"] = bars["dt"].dt.normalize()
-    if session_date is None:
-        session_ts = bars["session_date"].max()
-    else:
-        session_ts = pd.Timestamp(session_date).normalize()
-    day = bars[bars["session_date"] == session_ts].sort_values("dt").copy()
-    start_dt = pd.Timestamp.combine(session_ts.date(), time(9, 15))
-    end_dt = pd.Timestamp.combine(session_ts.date(), time(9, 19))
-    first5 = day[(day["dt"] >= start_dt) & (day["dt"] <= end_dt)].head(5).copy()
-    if len(first5) < 5:
-        raise RuntimeError(f"Need 5 opening bars for {session_ts.date()}, got {len(first5)}.")
-    first5["minute_index"] = np.arange(len(first5))
-    first5["ret_1m"] = first5["close"].pct_change(fill_method=None)
-    first5["range_pct_1m"] = safe_div(first5["high"] - first5["low"], first5["open"])
-    first5["body_pct_1m"] = safe_div(first5["close"] - first5["open"], first5["open"])
-    row = {
-        "date": session_ts,
-        "first5_start": first5["dt"].iloc[0],
-        "first5_end": first5["dt"].iloc[-1],
-        "first5_open": first5["open"].iloc[0],
-        "first5_high": first5["high"].max(),
-        "first5_low": first5["low"].min(),
-        "first5_close": first5["close"].iloc[-1],
-        "first5_volume": first5["volume"].sum() if "volume" in first5 else 0.0,
-        "first5_bars": len(first5),
-        "first5_last_1m_ret": first5["ret_1m"].iloc[-1],
-        "first5_ret_std": first5["ret_1m"].std(),
-    }
-    row["first5_return"] = (row["first5_close"] - row["first5_open"]) / row["first5_open"]
-    row["first5_range_pct"] = (row["first5_high"] - row["first5_low"]) / row["first5_open"]
-    first5_range = row["first5_high"] - row["first5_low"]
-    row["first5_body_to_range"] = (row["first5_close"] - row["first5_open"]) / first5_range if first5_range else np.nan
-    row["first5_close_location"] = (row["first5_close"] - row["first5_low"]) / first5_range if first5_range else np.nan
-    for idx, (_, candle) in enumerate(first5.iterrows(), start=1):
-        for field in ("open", "high", "low", "close", "ret_1m", "range_pct_1m", "body_pct_1m"):
-            row[f"m{idx}_{field}"] = candle[field]
-        row[f"m{idx}_close_vs_first5_open"] = (candle["close"] - row["first5_open"]) / row["first5_open"]
-        row[f"m{idx}_range_share"] = (candle["high"] - candle["low"]) / first5_range if first5_range else np.nan
-    row["first5_return_accel"] = row["m5_ret_1m"] - row["m2_ret_1m"]
-    row["first5_last2_return"] = (row["m5_close"] - row["m4_open"]) / row["m4_open"]
-    row["first5_first2_return"] = (row["m2_close"] - row["m1_open"]) / row["m1_open"]
-    row["first5_reversal"] = np.sign(row["first5_first2_return"]) * -np.sign(row["first5_last2_return"])
-    row["dow"] = session_ts.dayofweek
-    row["dom"] = session_ts.day
-    row["month"] = session_ts.month
-    return pd.DataFrame([row])
-
-
-def build_model_row(first5_row: pd.DataFrame) -> pd.DataFrame:
-    dataset = read_training_dataset()
-    latest_context = dataset.iloc[[-1]].copy()
-    output = latest_context.copy()
-    for col in first5_row.columns:
-        output[col] = first5_row[col].iloc[0]
-    if {"first5_open", "nifty_close"}.issubset(output.columns):
-        output["first5_gap_from_prev_close"] = (output["first5_open"] - output["nifty_close"]) / output["nifty_close"]
-        output["first5_close_vs_prev_close"] = (output["first5_close"] - output["nifty_close"]) / output["nifty_close"]
-    if {"first5_range_pct", "nifty_range_pct"}.issubset(output.columns):
-        output["first5_range_vs_prev_range"] = output["first5_range_pct"] / output["nifty_range_pct"]
-    if {"first5_return", "nifty_ret_1"}.issubset(output.columns):
-        output["first5_return_x_prev_ret"] = output["first5_return"] * output["nifty_ret_1"]
-        output["gap_x_prev_ret"] = output["first5_gap_from_prev_close"] * output["nifty_ret_1"]
-    if {"first5_return", "banknifty_ret_1"}.issubset(output.columns):
-        output["first5_return_x_bank_ret_1"] = output["first5_return"] * output["banknifty_ret_1"]
-    if {"first5_range_pct", "india_vix_ret_1"}.issubset(output.columns):
-        output["first5_range_x_vix_ret_1"] = output["first5_range_pct"] * output["india_vix_ret_1"]
-    output["target"] = np.nan
-    output["day_return"] = np.nan
-    return output
-
-
-def predict_row(row: pd.DataFrame) -> Prediction:
-    payload = load_model()
-    model = payload["model"]
-    features = payload["features"]
-    threshold = float(payload["threshold"])
-    missing = [c for c in features if c not in row.columns]
-    if missing:
-        raise RuntimeError(f"Feature row is missing {len(missing)} features; first missing: {missing[:5]}")
-    prob_up = predict_proba_up(model, row[features])
-    raw_pred = (prob_up >= threshold).astype("int64")
-    pred = apply_decision_overlays(raw_pred, row, payload.get("decision_overlays", DECISION_OVERLAYS))
-    confidence = directional_confidence(prob_up, pred, threshold)
-    prediction = Prediction(
-        input_date=pd.to_datetime(row["date"].iloc[0]).date().isoformat(),
-        first5_start=str(pd.to_datetime(row["first5_start"].iloc[0])),
-        first5_end=str(pd.to_datetime(row["first5_end"].iloc[0])),
-        prediction="UP" if int(pred[0]) == 1 else "DOWN",
-        prob_up=float(prob_up[0]),
-        confidence=float(confidence[0]),
-        threshold=threshold,
-        model_name=str(payload.get("model_name", "nifty_opening_direction_model")),
-    )
-    pd.DataFrame([prediction.to_dict()]).to_csv(LATEST_PATH, index=False)
-    return prediction
-
-
-def latest_saved_prediction() -> dict[str, Any]:
-    if LATEST_PATH.exists():
-        return pd.read_csv(LATEST_PATH).iloc[-1].to_dict()
-    summary_path = MODEL_DIR / "summary.json"
-    if summary_path.exists():
-        return json.loads(summary_path.read_text(encoding="utf-8"))
-    raise FileNotFoundError("No latest prediction is available yet.")
-
-
-def _json_ready_frame(df: pd.DataFrame, limit: int | None = None) -> list[dict[str, Any]]:
-    out = df.copy()
-    if limit is not None:
-        out = out.tail(limit)
-    for col in out.columns:
-        if pd.api.types.is_datetime64_any_dtype(out[col]):
-            out[col] = out[col].dt.strftime("%Y-%m-%d %H:%M:%S")
-    out = out.replace({np.nan: None})
-    return out.to_dict(orient="records")
-
-
-def load_model_summary() -> dict[str, Any]:
-    summary_path = MODEL_DIR / "summary.json"
-    if not summary_path.exists():
-        return {}
-    return json.loads(summary_path.read_text(encoding="utf-8"))
-
-
-def load_candidate_results() -> list[dict[str, Any]]:
-    path = MODEL_DIR / "candidate_results.csv"
-    if not path.exists():
-        return []
-    return _json_ready_frame(pd.read_csv(path).head(12))
-
-
-def load_test_predictions() -> pd.DataFrame:
-    if not TEST_PREDICTIONS_PATH.exists():
-        return pd.DataFrame()
-    df = pd.read_parquet(TEST_PREDICTIONS_PATH)
-    df["date"] = pd.to_datetime(df["date"], errors="coerce")
-    return df.sort_values("date").reset_index(drop=True)
-
-
-def dashboard_payload() -> dict[str, Any]:
-    summary = load_model_summary()
-    latest = latest_saved_prediction()
-    test = load_test_predictions()
-    daily = pd.read_parquet(NIFTY_1D_PATH)
-    daily["date"] = pd.to_datetime(daily["date"], errors="coerce")
-    daily = daily.sort_values("date").tail(180)
-    dataset = read_training_dataset()
-    opening = dataset[["date", "first5_return", "first5_range_pct", "first5_close_location"]].tail(120).copy()
-
-    if not test.empty:
-        recent_predictions = test.tail(40).copy()
-        recent_accuracy = float(recent_predictions["correct"].mean())
-        direction_mix = test.groupby("prediction")["correct"].agg(["count", "mean"]).reset_index()
-        monthly = (
-            test.assign(month=test["date"].dt.strftime("%Y-%m"))
-            .groupby("month", as_index=False)["correct"]
-            .mean()
-            .rename(columns={"correct": "accuracy"})
-        )
-    else:
-        recent_predictions = pd.DataFrame()
-        recent_accuracy = None
-        direction_mix = pd.DataFrame()
-        monthly = pd.DataFrame()
-
-    metrics = {
-        "validation_accuracy": summary.get("validation_accuracy"),
-        "test_accuracy": summary.get("test_accuracy"),
-        "baseline_test_accuracy": summary.get("baseline_test_accuracy"),
-        "validation_auc": summary.get("validation_auc"),
-        "test_auc": summary.get("test_auc"),
-        "test_brier": summary.get("test_brier"),
-        "feature_count": summary.get("feature_count"),
-        "recent_accuracy": recent_accuracy,
-    }
-    return {
-        "latest": latest,
-        "metrics": metrics,
-        "summary": summary,
-        "candidates": load_candidate_results(),
-        "charts": {
-            "daily_close": _json_ready_frame(daily[["date", "open", "high", "low", "close"]]),
-            "opening_features": _json_ready_frame(opening),
-            "monthly_accuracy": _json_ready_frame(monthly),
-            "direction_mix": _json_ready_frame(direction_mix),
-            "recent_predictions": _json_ready_frame(recent_predictions),
-        },
-        "data_status": {
-            "nifty_1m_rows": int(len(pd.read_parquet(NIFTY_1M_PATH, columns=["date"]))),
-            "nifty_1d_rows": int(len(pd.read_parquet(NIFTY_1D_PATH, columns=["date"]))),
-            "training_rows": int(len(dataset)),
-            "test_prediction_rows": int(len(test)),
-            "latest_daily_date": pd.to_datetime(daily["date"]).max().date().isoformat(),
-        },
-    }
-
-
-def refresh_first5_prediction(session_date: date | None = None) -> Prediction:
-    minutes = fetch_yahoo_minutes(period="5d")
-    append_parquet_rows(NIFTY_1M_PATH, minutes, ["date"])
-    first5 = first5_features_from_minutes(minutes, session_date=session_date)
-    row = build_model_row(first5)
-    dataset = read_training_dataset()
-    merged = pd.concat([dataset, row], ignore_index=True)
-    merged = merged.drop_duplicates(subset=["date"], keep="last").sort_values("date").reset_index(drop=True)
-    merged.to_parquet(OPENING_DATASET_PATH, index=False, compression="zstd")
-    return predict_row(row)
-
-
-def refresh_daily_data() -> dict[str, Any]:
-    daily = fetch_yahoo_daily(period="1mo")
-    combined = append_parquet_rows(NIFTY_1D_PATH, daily, ["date"])
-    return {
-        "rows": int(len(combined)),
-        "latest_date": pd.to_datetime(combined["date"]).max().date().isoformat(),
-        "path": str(NIFTY_1D_PATH),
-    }
-
-
-def seconds_until_next_ist_run(run_time: time = time(9, 20)) -> float:
-    now = datetime.now(IST)
-    target = datetime.combine(now.date(), run_time, tzinfo=IST)
-    if now >= target:
-        target += timedelta(days=1)
-    return max(1.0, (target - now).total_seconds())

backend/requirements.txt

@@ -1,9 +0,0 @@
-pandas
-pyarrow
-yfinance
-fastapi
-uvicorn
-joblib
-numpy
-scikit-learn
-catboost

backend/scripts/refresh_daily_data.py

@@ -1,15 +0,0 @@
-from __future__ import annotations
-
-import sys
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
-from nifty_backend.runtime import refresh_daily_data
-
-
-def main() -> None:
-    print(refresh_daily_data())
-
-
-if __name__ == "__main__":
-    main()

backend/scripts/refresh_first5_prediction.py

@@ -1,26 +0,0 @@
-from __future__ import annotations
-
-import argparse
-import sys
-from datetime import date
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
-from nifty_backend.runtime import refresh_first5_prediction
-
-
-def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description="Fetch Yahoo Finance first five NIFTY minutes and refresh prediction.")
-    parser.add_argument("--date", default=None, help="Optional IST session date, YYYY-MM-DD.")
-    return parser.parse_args()
-
-
-def main() -> None:
-    args = parse_args()
-    session_date = date.fromisoformat(args.date) if args.date else None
-    prediction = refresh_first5_prediction(session_date=session_date)
-    print(prediction.to_dict())
-
-
-if __name__ == "__main__":
-    main()

backend/scripts/retrain_opening_model.py

@@ -1,182 +0,0 @@
-from __future__ import annotations
-
-import argparse
-import json
-import sys
-from dataclasses import asdict, dataclass
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
-
-import joblib
-import numpy as np
-import pandas as pd
-from sklearn.ensemble import ExtraTreesClassifier
-from sklearn.impute import SimpleImputer
-from sklearn.linear_model import LogisticRegression
-from sklearn.metrics import accuracy_score, brier_score_loss, log_loss, roc_auc_score
-from sklearn.pipeline import make_pipeline
-from sklearn.preprocessing import StandardScaler
-
-from nifty_backend.runtime import (
-    DECISION_OVERLAYS,
-    MODEL_DIR,
-    MODEL_PATH,
-    OPENING_DATASET_PATH,
-    ProbabilityBlend,
-    apply_decision_overlays,
-    directional_confidence,
-    predict_proba_up,
-)
-
-
-DEFAULT_TRAIN_END = pd.Timestamp("2023-12-31")
-DEFAULT_VALID_END = pd.Timestamp("2025-08-17")
-RANDOM_SEED = 42
-
-
-@dataclass
-class RetrainSummary:
-    model_name: str
-    threshold: float
-    train_rows: int
-    valid_rows: int
-    test_rows: int
-    validation_accuracy: float
-    test_accuracy: float
-    validation_auc: float
-    test_auc: float
-    test_brier: float
-    latest_prediction: str
-    latest_prob_up: float
-    latest_confidence: float
-    feature_count: int
-
-
-def feature_columns(frame: pd.DataFrame) -> list[str]:
-    excluded = {
-        "date",
-        "first5_start",
-        "first5_end",
-        "target",
-        "day_open",
-        "day_high",
-        "day_low",
-        "day_close",
-        "day_volume",
-        "day_return",
-    }
-    cols = []
-    for col in frame.columns:
-        if col in excluded:
-            continue
-        if pd.api.types.is_numeric_dtype(frame[col]) and frame[col].notna().mean() >= 0.40:
-            if frame[col].nunique(dropna=True) > 1:
-                cols.append(col)
-    return cols
-
-
-def best_threshold(y_true: np.ndarray, prob_up: np.ndarray) -> tuple[float, float]:
-    thresholds = np.linspace(0.35, 0.65, 301)
-    scores = ((prob_up[:, None] >= thresholds[None, :]) == y_true[:, None]).mean(axis=0)
-    idx = int(np.argmax(scores))
-    return float(thresholds[idx]), float(scores[idx])
-
-
-def score_auc(y_true: np.ndarray, prob_up: np.ndarray) -> float:
-    if len(np.unique(y_true)) < 2:
-        return float("nan")
-    return float(roc_auc_score(y_true, prob_up))
-
-
-def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description="Retrain the compact NIFTY opening-direction model from Parquet data.")
-    parser.add_argument("--train-end", default=DEFAULT_TRAIN_END.date().isoformat())
-    parser.add_argument("--valid-end", default=DEFAULT_VALID_END.date().isoformat())
-    return parser.parse_args()
-
-
-def main() -> None:
-    args = parse_args()
-    train_end = pd.Timestamp(args.train_end)
-    valid_end = pd.Timestamp(args.valid_end)
-    frame = pd.read_parquet(OPENING_DATASET_PATH)
-    frame["date"] = pd.to_datetime(frame["date"], errors="coerce")
-    model_frame = frame.dropna(subset=["target"]).sort_values("date").reset_index(drop=True)
-    features = feature_columns(model_frame)
-    train_df = model_frame[model_frame["date"] <= train_end]
-    valid_df = model_frame[(model_frame["date"] > train_end) & (model_frame["date"] <= valid_end)]
-    test_df = model_frame[model_frame["date"] > valid_end]
-    if train_df.empty or valid_df.empty or test_df.empty:
-        raise RuntimeError("Training, validation, and test windows must all contain rows.")
-
-    x_train = train_df[features]
-    y_train = train_df["target"].to_numpy(dtype="int64")
-    x_valid = valid_df[features]
-    y_valid = valid_df["target"].to_numpy(dtype="int64")
-    x_test = test_df[features]
-    y_test = test_df["target"].to_numpy(dtype="int64")
-
-    extra_trees = make_pipeline(
-        SimpleImputer(strategy="median"),
-        ExtraTreesClassifier(
-            n_estimators=800,
-            max_depth=4,
-            min_samples_leaf=28,
-            max_features=0.60,
-            class_weight="balanced_subsample",
-            random_state=RANDOM_SEED + 13,
-            n_jobs=-1,
-        ),
-    )
-    logit = make_pipeline(
-        SimpleImputer(strategy="median"),
-        StandardScaler(),
-        LogisticRegression(C=0.25, class_weight="balanced", max_iter=2000, random_state=RANDOM_SEED),
-    )
-    extra_trees.fit(x_train, y_train)
-    logit.fit(x_train, y_train)
-    model = ProbabilityBlend([extra_trees, logit], np.array([0.75, 0.25]))
-    valid_prob = predict_proba_up(model, x_valid)
-    test_prob = predict_proba_up(model, x_test)
-    threshold, _ = best_threshold(y_valid, valid_prob)
-    valid_pred = apply_decision_overlays((valid_prob >= threshold).astype("int64"), valid_df, DECISION_OVERLAYS)
-    test_pred = apply_decision_overlays((test_prob >= threshold).astype("int64"), test_df, DECISION_OVERLAYS)
-    latest = frame.iloc[[-1]].copy()
-    latest_prob = predict_proba_up(model, latest[features])
-    latest_pred = apply_decision_overlays((latest_prob >= threshold).astype("int64"), latest, DECISION_OVERLAYS)
-    latest_conf = directional_confidence(latest_prob, latest_pred, threshold)
-
-    payload = {
-        "model": model,
-        "features": features,
-        "threshold": threshold,
-        "target": "same-day NIFTY 50 close > same-day NIFTY 50 open after first five 1-minute bars",
-        "model_name": "compact_extra_trees_logit_overlay",
-        "decision_overlays": DECISION_OVERLAYS,
-    }
-    joblib.dump(payload, MODEL_PATH)
-
-    summary = RetrainSummary(
-        model_name=payload["model_name"],
-        threshold=float(threshold),
-        train_rows=int(len(train_df)),
-        valid_rows=int(len(valid_df)),
-        test_rows=int(len(test_df)),
-        validation_accuracy=float(accuracy_score(y_valid, valid_pred)),
-        test_accuracy=float(accuracy_score(y_test, test_pred)),
-        validation_auc=score_auc(y_valid, valid_prob),
-        test_auc=score_auc(y_test, test_prob),
-        test_brier=float(brier_score_loss(y_test, np.clip(test_prob, 1e-6, 1 - 1e-6))),
-        latest_prediction="UP" if int(latest_pred[0]) == 1 else "DOWN",
-        latest_prob_up=float(latest_prob[0]),
-        latest_confidence=float(latest_conf[0]),
-        feature_count=int(len(features)),
-    )
-    (MODEL_DIR / "summary.json").write_text(json.dumps(asdict(summary), indent=2), encoding="utf-8")
-    pd.DataFrame([asdict(summary)]).to_csv(MODEL_DIR / "retrain_summary.csv", index=False)
-    print(asdict(summary))
-
-
-if __name__ == "__main__":
-    main()

backend/scripts/run_ist_scheduler.py

@@ -1,37 +0,0 @@
-from __future__ import annotations
-
-import time
-import sys
-from datetime import datetime
-from pathlib import Path
-from zoneinfo import ZoneInfo
-
-sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
-from nifty_backend.runtime import refresh_daily_data, refresh_first5_prediction, seconds_until_next_ist_run
-
-
-IST = ZoneInfo("Asia/Kolkata")
-
-
-def main() -> None:
-    print("[scheduler] NIFTY first-five-minute scheduler started.")
-    print("[scheduler] Runs the opening prediction after 09:20 IST so the 09:15-09:19 candles are complete.")
-    while True:
-        sleep_for = seconds_until_next_ist_run()
-        target = datetime.now(IST).timestamp() + sleep_for
-        print(f"[scheduler] sleeping {sleep_for / 60:.1f} minutes; next wake timestamp={target:.0f}")
-        time.sleep(sleep_for)
-        try:
-            prediction = refresh_first5_prediction()
-            print(f"[scheduler] first5 prediction refreshed: {prediction.to_dict()}")
-        except Exception as exc:
-            print(f"[scheduler] first5 refresh failed: {exc}")
-        try:
-            info = refresh_daily_data()
-            print(f"[scheduler] daily data refreshed: {info}")
-        except Exception as exc:
-            print(f"[scheduler] daily refresh failed: {exc}")
-
-
-if __name__ == "__main__":
-    main()