Upload 7 files

.dockerignore

@@ -0,0 +1,18 @@
+.git
+.gitignore
+__pycache__/
+*.py[cod]
+.space_state/
+.env
+*.env
+!*.example.env
+research_runtime/Code/artifacts/
+research_runtime/Code/docs/
+research_runtime/Code/scripts/backtesting/
+research_runtime/Code/scripts/tuning/
+research_runtime/Code/models/**/outputs/*dataset*.csv
+research_runtime/Code/models/**/outputs/test_predictions.csv
+research_runtime/Code/models/**/outputs/*predictions.csv
+research_runtime/Code/models/**/outputs/*.joblib
+research_runtime/Data/
+research_runtime/Alt Data/

.gitattributes

@@ -1,35 +1,5 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
+*.csv filter=lfs diff=lfs merge=lfs -text
 *.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -0,0 +1,23 @@
+FROM python:3.11-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PORT=7860 \
+    FORECASTING_PROJECT_ROOT=/app/research_runtime
+
+WORKDIR /app
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends build-essential curl git libgomp1 \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+RUN pip install --upgrade pip \
+    && pip install -r requirements.txt
+
+COPY . .
+
+EXPOSE 7860
+
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860", "--ws", "none"]

README.md

@@ -1,11 +1,128 @@
 ---
-title: Prediction Site
-emoji: 🐨
-colorFrom: pink
-colorTo: blue
+title: Trading Forecasting Backend
+colorFrom: blue
+colorTo: green
 sdk: docker
+app_port: 7860
 pinned: false
-short_description: backend
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Trading Forecasting Backend
+
+This folder is now a standalone Hugging Face Docker Space backend. Upload the contents of this `backend` folder to a Hugging Face Space repository, upload the separate `dataset` folder to a Hugging Face Dataset repository, and deploy the separate `frontend` folder to Netlify.
+
+The backend contains the quantitative model code, training scripts, model outputs, primary market data, and alternative data from the forecasting research workspace.
+
+## Hugging Face Space Setup
+
+Create a new Hugging Face Space with Docker SDK, then upload this backend folder as the Space root.
+
+Required Space variables/secrets:
+
+- `FRONTEND_ORIGINS`: your Netlify URL, for example `https://your-site.netlify.app`.
+- `CRON_SECRET`: a long shared secret. Use the same value in Netlify.
+- `HF_DATASET_REPO_ID`: your Hugging Face Dataset repo id, for example `your-username/your-forecasting-dataset`.
+
+Useful optional settings:
+
+- `AUTO_UPDATE_ENABLED=true`
+- `AUTO_RETRAIN_ENABLED=true`
+- `AUTO_UPDATE_ON_START=false`
+- `DATASET_SYNC_ON_START=true`
+- `HF_DATASET_REVISION=main`
+- `DAILY_UPDATE_TIME=17:30`
+- `UPDATE_TIMEZONE=Asia/Kolkata`
+- `MARKET_BUILD_WORKERS=2`
+
+The app listens on port `7860` and exposes Swagger docs at `/docs`.
+
+## API Routes
+
+- `GET /health` - Space health, file checks, latest data date, and update status.
+- `GET /api/status` - same as health, for frontend polling.
+- `GET /api/forecast/latest` - latest stock high/low, first-extrema, and Nifty forecasts.
+- `GET /api/models/summaries` - model summary JSONs.
+- `GET /api/data/catalog` - searchable data manifest.
+- `GET /api/data/sample?category=bars&asset=nifty50&timeframe=1d` - small sample from a manifest dataset.
+- `POST /api/cron/tick` - Netlify scheduled ping endpoint; starts an update only when due.
+- `POST /api/update/start` - manual update trigger. Send `x-admin-secret` if `CRON_SECRET` or `ADMIN_SECRET` is set.
+- `POST /api/dataset/sync` - manually sync the Hugging Face Dataset repo into the Space runtime.
+
+## Netlify Keep-Awake Cron
+
+The `frontend` folder now includes:
+
+- `frontend/netlify.toml`
+- `frontend/netlify/functions/keep-space-awake.mjs`
+
+On Netlify, set these environment variables:
+
+- `HUGGING_FACE_SPACE_URL=https://YOUR-HF-USERNAME-YOUR-SPACE.hf.space`
+- `CRON_SECRET=<same value as the Space CRON_SECRET>`
+
+The scheduled function runs every 10 minutes and calls `/api/cron/tick`. This keeps the Space warm and lets the backend start its daily update/retrain job after the configured market-close time.
+
+## Layout
+
+- `app.py` - FastAPI backend app for Hugging Face Spaces.
+- `Dockerfile` - Docker Space runtime setup.
+- `requirements.txt` - Python dependencies.
+- `research_runtime/Code/models/` - trainable model packages and the small latest forecast/summary outputs needed by the API.
+- `research_runtime/Code/scripts/data_ingestion/` - data refresh scripts used by update jobs.
+- `research_runtime/Code/scripts/data_preparation/` - research data rebuild scripts used by update jobs.
+
+`research_runtime/Data/` and `research_runtime/Alt Data/` are intentionally not bundled in the Space repo anymore. They now live in the separate Hugging Face Dataset repo and are downloaded into `research_runtime/` by the backend when `HF_DATASET_REPO_ID` is set.
+
+## Main Model Outputs To Wire First
+
+- Stock high/low forecasts: `research_runtime/Code/models/stock_high_low_forecaster/outputs/latest_forecasts.csv`
+- Stock high/low metrics: `research_runtime/Code/models/stock_high_low_forecaster/outputs/metrics_by_symbol.csv`
+- First-extrema forecasts: `research_runtime/Code/models/first_extrema_forecaster/outputs/latest_forecasts.csv`
+- Nifty forecasts: `research_runtime/Code/models/nifty_forecaster/outputs/forecaster_latest_forecasts.csv`
+- Nifty summary: `research_runtime/Code/models/nifty_forecaster/outputs/forecaster_summary.json`
+
+## Training Entrypoints
+
+Run these from `backend/research_runtime` so project-relative paths resolve correctly:
+
+```powershell
+python Code\models\stock_high_low_forecaster\train.py
+python Code\models\first_extrema_forecaster\train.py
+python Code\models\nifty_forecaster\train.py
+```
+
+## Data Labels
+
+These live in the separate Dataset repo:
+
+- Raw minute OHLCV: `Data/raw/minute/*_minute.csv`
+- Processed bars: `Data/processed/bars/{1m,5m,1h,4h,1d}/*.csv`
+- Processed features: `Data/processed/features/{1m,5m,1h,4h,1d}/*.csv`
+- Market panels: `Data/processed/panels/*_market_panel.csv`
+- Master daily panel: `Data/processed/panels/daily_master_panel.csv`
+- Data manifest: `Data/metadata/manifest.csv`
+- Feature dictionary: `Data/metadata/feature_dictionary.csv`
+- Options features: `Alt Data/options/processed/*_options_daily_features.csv`
+- Institutional panel: `Alt Data/institutional/processed/institutional_daily_panel.csv`
+- External daily panel: `Alt Data/external/processed/external_daily_panel.csv`
+- Corporate events: `Alt Data/corporate/processed/corporate_announcements.csv`
+
+## Frontend Wiring Notes
+
+The current frontend is static mock data in `frontend/index.html` and `frontend/script.js`.
+
+- Forecast cards can call `/api/forecast/latest`.
+- Model accuracy and version/date stats can call `/api/models/summaries`.
+- Market Data can call `/api/data/catalog` and `/api/data/sample`.
+
+## Pruned From Backend
+
+- Kotak credential/runtime files.
+- Live-trading scripts and live broker artifacts.
+- Kotak monitor artifacts and cached NSE temp folders.
+- Python `__pycache__` folders.
+- CatBoost generated training-log folder.
+- One-off maintenance/backfill scripts.
+- Backtest artifacts, chart images, old trade reports, test prediction dumps, generated training datasets, and saved model binaries.
+
+`KOTAKBANK` CSV files remain because those are normal market datasets for Kotak Mahindra Bank, not broker-runtime files.

app.py

@@ -0,0 +1,560 @@
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+import threading
+import time
+from datetime import datetime, time as dt_time
+from pathlib import Path
+from typing import Any
+from zoneinfo import ZoneInfo
+
+import pandas as pd
+from fastapi import BackgroundTasks, FastAPI, Header, HTTPException, Query, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse, PlainTextResponse
+from huggingface_hub import snapshot_download
+
+
+BASE_DIR = Path(__file__).resolve().parent
+RESEARCH_ROOT = Path(os.environ.get("FORECASTING_PROJECT_ROOT", BASE_DIR / "research_runtime")).resolve()
+STATE_DIR = Path(os.environ.get("SPACE_STATE_DIR", "/data/forecasting-space-state" if Path("/data").exists() else BASE_DIR / ".space_state"))
+STATUS_PATH = STATE_DIR / "update_status.json"
+DATASET_READY_MARKER = STATE_DIR / "dataset_ready.json"
+
+API_TITLE = "Trading Forecasting Space Backend"
+API_VERSION = "1.0.0"
+DEFAULT_TIMEZONE = os.environ.get("UPDATE_TIMEZONE", "Asia/Kolkata")
+DEFAULT_UPDATE_TIME = os.environ.get("DAILY_UPDATE_TIME", "17:30")
+
+app = FastAPI(title=API_TITLE, version=API_VERSION)
+
+
+def cors_origins() -> list[str]:
+    raw = os.environ.get("FRONTEND_ORIGINS", "*").strip()
+    return ["*"] if raw == "*" else [item.strip() for item in raw.split(",") if item.strip()]
+
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=cors_origins(),
+    allow_credentials=False,
+    allow_methods=["GET", "POST", "OPTIONS"],
+    allow_headers=["*"],
+)
+
+update_lock = threading.Lock()
+worker_thread: threading.Thread | None = None
+dataset_lock = threading.Lock()
+
+
+def now_utc() -> str:
+    return datetime.utcnow().replace(microsecond=0).isoformat() + "Z"
+
+
+def safe_json(value: Any) -> Any:
+    if isinstance(value, dict):
+        return {str(k): safe_json(v) for k, v in value.items()}
+    if isinstance(value, list):
+        return [safe_json(v) for v in value]
+    if not isinstance(value, (tuple, set)):
+        try:
+            if pd.isna(value):
+                return None
+        except Exception:
+            pass
+    if hasattr(value, "item"):
+        try:
+            return safe_json(value.item())
+        except Exception:
+            pass
+    if isinstance(value, Path):
+        return str(value)
+    if isinstance(value, datetime):
+        return value.isoformat()
+    return value
+
+
+def read_json(path: Path, default: Any) -> Any:
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return default
+
+
+def write_json(path: Path, payload: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(safe_json(payload), indent=2), encoding="utf-8")
+
+
+def read_status() -> dict[str, Any]:
+    return read_json(
+        STATUS_PATH,
+        {
+            "state": "idle",
+            "last_started_at": None,
+            "last_finished_at": None,
+            "last_success_at": None,
+            "last_error": None,
+            "last_exit_code": None,
+            "last_log_tail": [],
+        },
+    )
+
+
+def write_status(**updates: Any) -> None:
+    status = read_status()
+    status.update(updates)
+    write_json(STATUS_PATH, status)
+
+
+def require_secret(x_cron_secret: str | None = Header(default=None), x_admin_secret: str | None = Header(default=None)) -> None:
+    expected = os.environ.get("CRON_SECRET") or os.environ.get("ADMIN_SECRET")
+    if not expected:
+        return
+    supplied = x_cron_secret or x_admin_secret
+    if supplied != expected:
+        raise HTTPException(status_code=401, detail="Missing or invalid cron/admin secret.")
+
+
+def csv_rows(path: Path, *, limit: int | None = None, columns: list[str] | None = None) -> list[dict[str, Any]]:
+    if not path.exists():
+        return []
+    try:
+        frame = pd.read_csv(path, usecols=columns)
+    except ValueError:
+        frame = pd.read_csv(path)
+        if columns:
+            frame = frame[[col for col in columns if col in frame.columns]]
+    if limit is not None:
+        frame = frame.head(limit)
+    return safe_json(frame.where(pd.notna(frame), None).to_dict(orient="records"))
+
+
+def model_output_path(*parts: str) -> Path:
+    return RESEARCH_ROOT / "Code" / "models" / Path(*parts)
+
+
+def manifest_path() -> Path:
+    return RESEARCH_ROOT / "Data" / "metadata" / "manifest.csv"
+
+
+def dataset_dirs_present() -> bool:
+    return (RESEARCH_ROOT / "Data").is_dir() and (RESEARCH_ROOT / "Alt Data").is_dir()
+
+
+def dataset_status() -> dict[str, Any]:
+    marker = read_json(DATASET_READY_MARKER, {})
+    return {
+        "ready": dataset_dirs_present(),
+        "repo_id": os.environ.get("HF_DATASET_REPO_ID"),
+        "revision": os.environ.get("HF_DATASET_REVISION", "main"),
+        "data_dir": file_meta(RESEARCH_ROOT / "Data"),
+        "alt_data_dir": file_meta(RESEARCH_ROOT / "Alt Data"),
+        "last_sync": marker,
+    }
+
+
+def ensure_dataset_available(force: bool = False) -> bool:
+    if dataset_dirs_present() and not force:
+        return True
+
+    repo_id = os.environ.get("HF_DATASET_REPO_ID", "").strip()
+    if not repo_id:
+        return dataset_dirs_present()
+
+    with dataset_lock:
+        if dataset_dirs_present() and not force:
+            return True
+
+        STATE_DIR.mkdir(parents=True, exist_ok=True)
+        revision = os.environ.get("HF_DATASET_REVISION", "main")
+        local_dir = Path(os.environ.get("HF_DATASET_LOCAL_DIR", str(RESEARCH_ROOT))).resolve()
+        local_dir.mkdir(parents=True, exist_ok=True)
+
+        snapshot_download(
+            repo_id=repo_id,
+            repo_type="dataset",
+            revision=revision,
+            local_dir=str(local_dir),
+            local_dir_use_symlinks=False,
+            allow_patterns=["Data/**", "Alt Data/**", "README.md"],
+        )
+
+        write_json(
+            DATASET_READY_MARKER,
+            {
+                "repo_id": repo_id,
+                "revision": revision,
+                "synced_at": now_utc(),
+                "local_dir": str(local_dir),
+            },
+        )
+        return dataset_dirs_present()
+
+
+def resolve_dataset_path(value: str) -> Path:
+    raw = str(value)
+    candidate = Path(raw)
+    if candidate.exists():
+        return candidate
+
+    normalized = raw.replace("\\", "/")
+    marker = "research_runtime/"
+    if marker in normalized:
+        suffix = normalized.split(marker, 1)[1]
+        return BASE_DIR / "research_runtime" / Path(*suffix.split("/"))
+
+    relative = Path(*normalized.split("/"))
+    if not relative.is_absolute():
+        return BASE_DIR / relative
+    return candidate
+
+
+def file_meta(path: Path) -> dict[str, Any]:
+    if not path.exists():
+        return {"exists": False, "path": str(path)}
+    stat = path.stat()
+    return {
+        "exists": True,
+        "path": str(path),
+        "bytes": stat.st_size,
+        "modified_at": datetime.utcfromtimestamp(stat.st_mtime).replace(microsecond=0).isoformat() + "Z",
+    }
+
+
+def latest_manifest_end() -> str | None:
+    path = manifest_path()
+    if not path.exists():
+        return None
+    try:
+        frame = pd.read_csv(path, usecols=["end"])
+        dates = pd.to_datetime(frame["end"], errors="coerce").dropna()
+        return str(dates.max()) if not dates.empty else None
+    except Exception:
+        return None
+
+
+def parse_daily_update_time() -> dt_time:
+    hour, minute = DEFAULT_UPDATE_TIME.split(":", 1)
+    return dt_time(int(hour), int(minute))
+
+
+def update_due() -> bool:
+    if os.environ.get("AUTO_UPDATE_ENABLED", "true").lower() not in {"1", "true", "yes", "on"}:
+        return False
+    status = read_status()
+    if status.get("state") == "running":
+        return False
+
+    tz = ZoneInfo(DEFAULT_TIMEZONE)
+    local_now = datetime.now(tz)
+    if local_now.time() < parse_daily_update_time():
+        return False
+
+    last_success = status.get("last_success_at")
+    if not last_success:
+        return True
+    try:
+        last_success_date = datetime.fromisoformat(last_success.replace("Z", "+00:00")).astimezone(tz).date()
+    except ValueError:
+        return True
+    return last_success_date < local_now.date()
+
+
+def build_update_commands(retrain: bool) -> list[list[str]]:
+    commands = [
+        [
+            sys.executable,
+            "Code/scripts/data_ingestion/refresh_market_data.py",
+            "--end-date",
+            datetime.now(ZoneInfo(DEFAULT_TIMEZONE)).date().isoformat(),
+        ]
+    ]
+    if retrain:
+        commands.extend(
+            [
+                [sys.executable, "Code/models/stock_high_low_forecaster/train.py"],
+                [sys.executable, "Code/models/first_extrema_forecaster/train.py", "--rebuild-cache"],
+                [sys.executable, "Code/models/nifty_forecaster/train.py", "--no-progress"],
+            ]
+        )
+    return commands
+
+
+def prune_generated_junk() -> None:
+    patterns = [
+        "Code/artifacts",
+        "Code/models/*/outputs/*dataset*.csv",
+        "Code/models/*/outputs/test_predictions.csv",
+        "Code/models/*/outputs/*_test_predictions.csv",
+        "Code/models/*/outputs/*predictions.csv",
+        "Code/models/*/outputs/*.joblib",
+        "Code/models/*/outputs/report.md",
+        "Code/models/*/outputs/*report.md",
+        "Code/models/*/outputs/candidate*.csv",
+        "Code/models/*/outputs/*candidate*.csv",
+        "Code/models/first_extrema_forecaster/outputs/may7_forecasts.csv",
+        "Code/models/nifty_forecaster/outputs/forecaster_latest.csv",
+        "Code/models/nifty_forecaster/outputs/forecaster_blend_details.json",
+    ]
+    for pattern in patterns:
+        for path in RESEARCH_ROOT.glob(pattern):
+            try:
+                if path.is_dir():
+                    shutil.rmtree(path)
+                elif path.exists():
+                    path.unlink()
+            except OSError:
+                pass
+    for cache_dir in RESEARCH_ROOT.rglob("__pycache__"):
+        try:
+            shutil.rmtree(cache_dir)
+        except OSError:
+            pass
+
+
+def run_update_job(trigger: str = "manual", retrain: bool | None = None) -> None:
+    global worker_thread
+    with update_lock:
+        status = read_status()
+        if status.get("state") == "running":
+            return
+        write_status(
+            state="running",
+            trigger=trigger,
+            last_started_at=now_utc(),
+            last_finished_at=None,
+            last_error=None,
+            last_exit_code=None,
+            last_log_tail=[],
+        )
+
+    if retrain is None:
+        retrain = os.environ.get("AUTO_RETRAIN_ENABLED", "true").lower() in {"1", "true", "yes", "on"}
+
+    env = os.environ.copy()
+    env["FORECASTING_PROJECT_ROOT"] = str(RESEARCH_ROOT)
+    env.setdefault("PYTHONUNBUFFERED", "1")
+    env.setdefault("MARKET_BUILD_WORKERS", "2")
+
+    log_tail: list[str] = []
+    exit_code = 0
+    try:
+        if not ensure_dataset_available():
+            raise RuntimeError("Dataset folders are missing. Set HF_DATASET_REPO_ID to the Hugging Face Dataset repo.")
+        for command in build_update_commands(retrain):
+            log_tail.append("$ " + " ".join(command))
+            process = subprocess.Popen(
+                command,
+                cwd=RESEARCH_ROOT,
+                env=env,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+                bufsize=1,
+            )
+            assert process.stdout is not None
+            for line in process.stdout:
+                line = line.rstrip()
+                if line:
+                    log_tail.append(line)
+                    log_tail = log_tail[-80:]
+            exit_code = process.wait()
+            if exit_code != 0:
+                raise RuntimeError(f"Command failed with exit code {exit_code}: {' '.join(command)}")
+        prune_generated_junk()
+        write_status(
+            state="idle",
+            last_finished_at=now_utc(),
+            last_success_at=now_utc(),
+            last_error=None,
+            last_exit_code=exit_code,
+            last_log_tail=log_tail[-80:],
+        )
+    except Exception as exc:
+        write_status(
+            state="failed",
+            last_finished_at=now_utc(),
+            last_error=str(exc),
+            last_exit_code=exit_code,
+            last_log_tail=log_tail[-80:],
+        )
+
+
+def start_update(trigger: str, retrain: bool | None = None) -> bool:
+    global worker_thread
+    status = read_status()
+    if status.get("state") == "running":
+        return False
+    worker_thread = threading.Thread(target=run_update_job, kwargs={"trigger": trigger, "retrain": retrain}, daemon=True)
+    worker_thread.start()
+    return True
+
+
+def scheduler_loop() -> None:
+    while True:
+        if update_due():
+            start_update("internal_scheduler")
+        time.sleep(300)
+
+
+@app.on_event("startup")
+def startup() -> None:
+    STATE_DIR.mkdir(parents=True, exist_ok=True)
+    prune_generated_junk()
+    if not STATUS_PATH.exists():
+        write_status(state="idle", app_started_at=now_utc())
+    if os.environ.get("DATASET_SYNC_ON_START", "true").lower() in {"1", "true", "yes", "on"}:
+        try:
+            ensure_dataset_available()
+        except Exception as exc:
+            write_status(dataset_sync_error=str(exc), dataset_sync_failed_at=now_utc())
+    threading.Thread(target=scheduler_loop, daemon=True).start()
+    if os.environ.get("AUTO_UPDATE_ON_START", "false").lower() in {"1", "true", "yes", "on"}:
+        start_update("startup")
+
+
+@app.get("/", response_class=PlainTextResponse)
+def root() -> str:
+    return "Trading Forecasting Hugging Face Space backend is running. See /docs for API routes."
+
+
+@app.get("/health")
+def health() -> dict[str, Any]:
+    required = {
+        "research_root": file_meta(RESEARCH_ROOT),
+        "manifest": file_meta(manifest_path()),
+        "stock_latest": file_meta(model_output_path("stock_high_low_forecaster", "outputs", "latest_forecasts.csv")),
+        "extrema_latest": file_meta(model_output_path("first_extrema_forecaster", "outputs", "latest_forecasts.csv")),
+        "nifty_latest": file_meta(model_output_path("nifty_forecaster", "outputs", "forecaster_latest_forecasts.csv")),
+    }
+    ok = all(item["exists"] for item in required.values())
+    return {
+        "ok": ok,
+        "service": API_TITLE,
+        "version": API_VERSION,
+        "checked_at": now_utc(),
+        "latest_manifest_end": latest_manifest_end(),
+        "dataset": dataset_status(),
+        "update_status": read_status(),
+        "files": required,
+    }
+
+
+@app.get("/api/status")
+def api_status() -> dict[str, Any]:
+    return health()
+
+
+@app.get("/api/forecast/latest")
+def latest_forecasts() -> dict[str, Any]:
+    return {
+        "generated_at": now_utc(),
+        "stock_high_low": csv_rows(model_output_path("stock_high_low_forecaster", "outputs", "latest_forecasts.csv")),
+        "first_extrema": csv_rows(
+            model_output_path("first_extrema_forecaster", "outputs", "latest_forecasts.csv"),
+            columns=["date", "symbol", "target", "prob_high_first", "prediction"],
+        ),
+        "nifty_direction": csv_rows(model_output_path("nifty_forecaster", "outputs", "forecaster_latest_forecasts.csv")),
+    }
+
+
+@app.get("/api/models/summaries")
+def model_summaries() -> dict[str, Any]:
+    return safe_json(
+        {
+            "stock_high_low": read_json(model_output_path("stock_high_low_forecaster", "outputs", "summary.json"), {}),
+            "first_extrema": read_json(model_output_path("first_extrema_forecaster", "outputs", "summary.json"), {}),
+            "nifty_direction": read_json(model_output_path("nifty_forecaster", "outputs", "forecaster_summary.json"), []),
+        }
+    )
+
+
+@app.get("/api/data/catalog")
+def data_catalog(
+    category: str | None = None,
+    asset: str | None = None,
+    timeframe: str | None = None,
+    limit: int = Query(default=500, ge=1, le=5000),
+) -> dict[str, Any]:
+    path = manifest_path()
+    if not path.exists():
+        ensure_dataset_available()
+    if not path.exists():
+        return {"count": 0, "items": []}
+    frame = pd.read_csv(path)
+    if category:
+        frame = frame[frame["category"].astype(str).str.lower() == category.lower()]
+    if asset:
+        frame = frame[frame["asset"].astype(str).str.lower() == asset.lower()]
+    if timeframe:
+        frame = frame[frame["timeframe"].astype(str).str.lower() == timeframe.lower()]
+    return {"count": int(len(frame)), "items": safe_json(frame.head(limit).where(pd.notna(frame), None).to_dict(orient="records"))}
+
+
+@app.get("/api/data/sample")
+def data_sample(
+    category: str,
+    asset: str,
+    timeframe: str,
+    limit: int = Query(default=50, ge=1, le=1000),
+) -> dict[str, Any]:
+    path = manifest_path()
+    if not path.exists():
+        ensure_dataset_available()
+    if not path.exists():
+        raise HTTPException(status_code=404, detail="Data manifest not found.")
+    manifest = pd.read_csv(path)
+    matches = manifest[
+        (manifest["category"].astype(str).str.lower() == category.lower())
+        & (manifest["asset"].astype(str).str.lower() == asset.lower())
+        & (manifest["timeframe"].astype(str).str.lower() == timeframe.lower())
+    ]
+    if matches.empty:
+        raise HTTPException(status_code=404, detail="No matching dataset in manifest.")
+    dataset_path = resolve_dataset_path(str(matches.iloc[0]["path"]))
+    if not dataset_path.exists():
+        raise HTTPException(status_code=404, detail=f"Dataset file not found: {dataset_path}")
+    return {
+        "dataset": safe_json(matches.iloc[0].to_dict()),
+        "rows": csv_rows(dataset_path, limit=limit),
+    }
+
+
+@app.api_route("/api/cron/tick", methods=["GET", "POST"])
+async def cron_tick(
+    request: Request,
+    background_tasks: BackgroundTasks,
+    x_cron_secret: str | None = Header(default=None),
+) -> JSONResponse:
+    require_secret(x_cron_secret=x_cron_secret)
+    due = update_due()
+    started = False
+    if due:
+        background_tasks.add_task(start_update, "netlify_cron")
+        started = True
+    return JSONResponse({"ok": True, "checked_at": now_utc(), "update_due": due, "update_start_queued": started, "status": read_status()})
+
+
+@app.post("/api/update/start")
+def manual_update(
+    retrain: bool | None = None,
+    x_admin_secret: str | None = Header(default=None),
+) -> dict[str, Any]:
+    require_secret(x_admin_secret=x_admin_secret)
+    started = start_update("manual_api", retrain=retrain)
+    return {"ok": True, "started": started, "status": read_status()}
+
+
+@app.post("/api/dataset/sync")
+def sync_dataset(
+    force: bool = False,
+    x_admin_secret: str | None = Header(default=None),
+) -> dict[str, Any]:
+    require_secret(x_admin_secret=x_admin_secret)
+    ok = ensure_dataset_available(force=force)
+    return {"ok": ok, "dataset": dataset_status()}

requirements.txt

@@ -0,0 +1,11 @@
+fastapi==0.115.12
+uvicorn[standard]==0.34.2
+pandas==2.2.3
+numpy==2.2.6
+requests==2.32.3
+scikit-learn==1.6.1
+joblib==1.4.2
+xgboost==3.0.1
+catboost==1.2.8
+lightgbm==4.6.0
+huggingface_hub==0.31.4

runtime_config.example.env

@@ -0,0 +1,14 @@
+# Hugging Face Space backend settings
+FORECASTING_PROJECT_ROOT=/app/research_runtime
+FRONTEND_ORIGINS=https://your-netlify-site.netlify.app
+CRON_SECRET=replace-with-a-long-shared-secret
+HF_DATASET_REPO_ID=your-hf-username/your-forecasting-dataset
+HF_DATASET_REVISION=main
+
+# Automatic update settings
+AUTO_UPDATE_ENABLED=true
+AUTO_RETRAIN_ENABLED=true
+AUTO_UPDATE_ON_START=false
+DAILY_UPDATE_TIME=17:30
+UPDATE_TIMEZONE=Asia/Kolkata
+MARKET_BUILD_WORKERS=2