Upload runtime.py

runtime.py

@@ -1243,85 +1243,80 @@ def warm_dashboard_payload_cache() -> None:
     dashboard_payload()
 
 
+def _load_forecaster_predictions_by_target() -> dict[date, dict[str, Any]]:
+    """Index bundled forecaster backtest rows by target session date."""
+    path = DAILY_FORECASTER_PREDICTIONS_PATH
+    if not path.exists():
+        return {}
+    try:
+        frame = pd.read_csv(path)
+    except Exception:
+        return {}
+    if frame.empty:
+        return {}
+    if "symbol" in frame.columns:
+        frame = frame[frame["symbol"].astype(str) == "NIFTY 50"].copy()
+    indexed: dict[date, dict[str, Any]] = {}
+    for _, row in frame.iterrows():
+        target_day = _parse_iso_date(row.get("target_date"))
+        if target_day is None:
+            continue
+        pred_value = row.get("pred")
+        if pd.isna(pred_value) and "raw_pred" in row:
+            pred_value = row.get("raw_pred")
+        try:
+            pred_int = int(pred_value)
+        except Exception:
+            continue
+        prob_up = row.get("prob_up")
+        try:
+            prob_up = float(prob_up) if pd.notna(prob_up) else None
+        except Exception:
+            prob_up = None
+        indexed[target_day] = {
+            "prediction": "UP" if pred_int == 1 else "DOWN",
+            "prob_up": prob_up,
+            "forecast_date": row.get("forecast_date"),
+            "source": "Tomorrow (forecaster)",
+        }
+    return indexed
+
+
+def _rolling_tomorrow_prediction(
+    input_day: date,
+    daily_rows: pd.DataFrame,
+    threshold: float,
+    fallback_prob: float,
+) -> tuple[str, float]:
+    """Simulate the Tomorrow model using only daily bars available through ``input_day``."""
+    history = daily_rows[daily_rows["date"].dt.date <= input_day].copy()
+    prob_up = _tomorrow_probability_from_daily(history, fallback_prob)
+    prediction = "UP" if prob_up >= threshold else "DOWN"
+    return prediction, float(prob_up)
+
+
 def build_prediction_track_record(
     daily: pd.DataFrame,
-    t5_test: pd.DataFrame,
-    tomorrow_test: pd.DataFrame,
-    tomorrow_history: pd.DataFrame,
-    tplus1_test: pd.DataFrame,
-    t5_latest: dict[str, Any],
-    tomorrow_latest: dict[str, Any],
-    tplus1_latest: dict[str, Any],
-    live_accuracy: dict[str, Any] | None = None,
+    sessions: int = 10,
 ) -> list[dict[str, Any]]:
+    """Rolling last-N Tomorrow simulation: predict each session, score vs prior close."""
+    summary = load_tomorrow_summary()
+    artifact = load_tomorrow_model_artifact()
+    threshold = float(artifact.get("threshold", summary.get("threshold", 0.534)))
+    fallback_prob = float(summary.get("latest_forecast_prob_up", 0.49900560447008563))
+    forecaster_by_target = _load_forecaster_predictions_by_target()
+
     daily_rows = daily.copy()
     daily_rows["date"] = pd.to_datetime(daily_rows["date"], errors="coerce").dt.normalize()
     daily_rows = daily_rows.dropna(subset=["date"]).sort_values("date")
     daily_rows = daily_rows[
-        daily_rows["open"].map(lambda value: np.isfinite(float(value)) if pd.notna(value) else False)
-        & daily_rows["close"].map(lambda value: np.isfinite(float(value)) if pd.notna(value) else False)
+        daily_rows["close"].map(lambda value: np.isfinite(float(value)) if pd.notna(value) else False)
     ].copy()
-    daily_rows = daily_rows[daily_rows["open"].astype(float) != 0]
     completed_day = expected_completed_daily_date()
     daily_rows = daily_rows[daily_rows["date"].dt.date <= completed_day]
     if daily_rows.empty:
         return []
 
-    predictions_by_date: dict[str, dict[str, Any]] = {}
-
-    def add_prediction(target_date: Any, prediction: Any, source: str, priority: int, meta: dict[str, Any] | None = None) -> None:
-        day = str(target_date or "")[:10]
-        pred = str(prediction or "").upper()
-        if not day or pred not in {"UP", "DOWN"}:
-            return
-        existing = predictions_by_date.get(day)
-        if existing and existing.get("_priority", 0) >= priority:
-            return
-        predictions_by_date[day] = {
-            "prediction": pred,
-            "source": source,
-            "_priority": priority,
-            **(meta or {}),
-        }
-
-    for _, row in tomorrow_test.iterrows():
-        pred = row.get("prediction")
-        if pd.isna(pred) and "pred" in row:
-            pred = "UP" if int(row.get("pred")) == 1 else "DOWN"
-        add_prediction(row.get("target_date") or row.get("date"), pred, "Tomorrow", 20, {"prob_up": row.get("prob_up")})
-
-    if not tomorrow_history.empty:
-        for _, row in tomorrow_history.iterrows():
-            pred = row.get("prediction")
-            if pd.isna(pred) and "pred" in row:
-                pred = "UP" if int(row.get("pred")) == 1 else "DOWN"
-            target_date = row.get("target_date") or row.get("date") or row.get("input_date")
-            add_prediction(
-                target_date,
-                pred,
-                "Tomorrow",
-                30,
-                {
-                    "prob_up": row.get("prob_up"),
-                    "source": row.get("source"),
-                },
-            )
-
-    add_prediction(tomorrow_latest.get("target_date"), tomorrow_latest.get("prediction"), "Tomorrow", 40, {"prob_up": tomorrow_latest.get("prob_up")})
-
-    ledger = live_accuracy or load_live_accuracy()
-    for entry in ledger.get("tomorrow", {}).get("entries", []):
-        day = str(entry.get("date", ""))[:10]
-        pred = str(entry.get("prediction", "")).upper()
-        if day and pred in {"UP", "DOWN"}:
-            add_prediction(
-                day,
-                pred,
-                "Tomorrow",
-                50,
-                {"source": entry.get("source", "live")},
-            )
-
     closes_by_date = {
         row["date"].date(): float(row["close"])
         for _, row in daily_rows.iterrows()
@@ -1329,31 +1324,38 @@ def build_prediction_track_record(
     }
 
     records: list[dict[str, Any]] = []
-    for _, row in daily_rows.tail(20).iterrows():
-        day = row["date"].date().isoformat()
+    for _, row in daily_rows.tail(sessions).iterrows():
+        target_day = row["date"].date()
         day_close = float(row["close"])
-        actual_move, actual_direction = _tomorrow_actual_outcome(row["date"].date(), day_close, closes_by_date)
+        actual_move, actual_direction = _tomorrow_actual_outcome(target_day, day_close, closes_by_date)
         if actual_direction is None:
             continue
-        pred = predictions_by_date.get(day)
-        prediction = pred.get("prediction") if pred else None
-        if prediction is None:
-            found = _find_tomorrow_prediction_for_target(row["date"].date())
-            if found:
-                prediction = str(found.get("prediction", "")).upper()
-                if prediction not in {"UP", "DOWN"}:
-                    prediction = None
-                else:
-                    pred = {"prediction": prediction, "source": found.get("source", "Tomorrow"), "prob_up": found.get("prob_up")}
+
+        input_day = previous_trading_day(target_day - timedelta(days=1))
+        cached = forecaster_by_target.get(target_day)
+        if cached and cached.get("prediction") in {"UP", "DOWN"}:
+            prediction = cached["prediction"]
+            prob_up = cached.get("prob_up")
+            source = cached.get("source", "Tomorrow (forecaster)")
+        else:
+            prediction, prob_up = _rolling_tomorrow_prediction(
+                input_day,
+                daily_rows,
+                threshold,
+                fallback_prob,
+            )
+            source = "Tomorrow (rolling)"
+
         records.append(
             {
-                "date": day,
+                "date": target_day.isoformat(),
+                "input_date": input_day.isoformat(),
                 "prediction": prediction,
-                "prediction_source": pred.get("source") if pred else None,
-                "prob_up": pred.get("prob_up") if pred else None,
+                "prediction_source": source,
+                "prob_up": prob_up,
                 "actual_move": actual_move,
                 "actual_direction": actual_direction,
-                "correct": None if prediction is None else prediction == actual_direction,
+                "correct": prediction == actual_direction,
             }
         )
     return records
@@ -1450,20 +1452,7 @@ def _dashboard_payload_cached(key: tuple[tuple[str, int | None, int | None], ...
         "total_test_days": int(tomorrow_summary.get("n_test") or len(tomorrow_test) or 0),
         "models": model_metrics,
     }
-    live_accuracy = load_live_accuracy()
-    ensure_completed_sessions_scored(datetime.now(IST), live_accuracy)
-    live_accuracy = load_live_accuracy()
-    track_record = build_prediction_track_record(
-        daily,
-        t5_test,
-        tomorrow_test,
-        tomorrow_history,
-        tplus1_test,
-        t5_latest,
-        tomorrow_latest,
-        tplus1_latest,
-        live_accuracy=live_accuracy,
-    )
+    track_record = build_prediction_track_record(daily, sessions=10)
     return {
         "latest": t5_latest,
         "tomorrow_latest": tomorrow_latest,