Update app.py

app.py

@@ -1,4 +1,4 @@
-# app.py — DataSynthis_ML_JobTask (ARIMA + optional LSTM, simple UI)
+# app.py — DataSynthis_ML_JobTask (ARIMA + LSTM REQUIRED, simple UI)
 
 import os, json, math, pickle, traceback
 from datetime import datetime, timedelta
@@ -16,10 +16,15 @@ from sklearn.metrics import mean_squared_error, mean_absolute_error, mean_absolu
 # -------------------------
 # Artifact Paths
 # -------------------------
-ARIMA_ORDER_JSON = "arima_order.json"     # preferred way to get (p,d,q)
-ARIMA_PICKLE = "arima_model.pkl"          # optional local archive (not used for inference)
-LSTM_KERAS = "lstm_model.keras"           # optional: pre-trained LSTM (Keras 3)
-LSTM_INFO = "lstm_info.json"              # optional: {"lookback": int, "train_min": float, "train_max": float}
+ARIMA_ORDER_JSON = "arima_order.json"      # required for ARIMA order (p,d,q)
+ARIMA_PICKLE = "arima_model.pkl"           # optional (not used for inference)
+
+# LSTM artifacts (LSTM is REQUIRED in this build)
+LSTM_KERAS = "lstm_model.keras"            # required
+LSTM_INFO_PRIMARY = "lstm_info.json"       # primary metadata location
+LSTM_INFO_ALT = "lstm_model/info.json"     # alternate metadata location
+
+REQUIRE_LSTM = True  # <- make LSTM mandatory
 
 # -------------------------
 # Globals (lazy-load LSTM)
@@ -47,19 +52,14 @@ def fetch_data(ticker: str, start: str, end: str) -> pd.DataFrame:
 
 def load_arima_order(default=(2, 1, 2)):
     """
-    Prefer reading (p,d,q) from arima_order.json; fall back to pickle; else default.
+    Read (p,d,q) from arima_order.json; fallback to pickle; else default.
     """
-    # JSON first (version-proof)
     if os.path.isfile(ARIMA_ORDER_JSON):
-        try:
-            with open(ARIMA_ORDER_JSON, "r") as f:
-                obj = json.load(f)
-            order = tuple(int(x) for x in obj.get("order", list(default)))
-            return order
-        except Exception as e:
-            print(f"[WARN] Failed to read {ARIMA_ORDER_JSON}: {e}")
+        with open(ARIMA_ORDER_JSON, "r") as f:
+            obj = json.load(f)
+        order = tuple(int(x) for x in obj.get("order", list(default)))
+        return order
 
-    # Fallback: try to extract order from pickle (not used for inference otherwise)
     if os.path.isfile(ARIMA_PICKLE):
         try:
             with open(ARIMA_PICKLE, "rb") as f:
@@ -69,7 +69,6 @@ def load_arima_order(default=(2, 1, 2)):
         except Exception as e:
             print(f"[WARN] Failed to read order from {ARIMA_PICKLE}: {e}")
 
-    # Final fallback
     return default
 
 # -------------------------
@@ -78,7 +77,7 @@ def load_arima_order(default=(2, 1, 2)):
 def arima_roll_single_fit(train_s: pd.Series, test_s: pd.Series, order=(2, 1, 2)):
     """
     Fair evaluation: fit once on TRAIN, then roll 1-step ahead through TEST.
-    Uses ARIMAResults.append() to update state; refits if append not available.
+    Uses ARIMAResults.append() to update state; refits if append is unavailable.
     """
     res = ARIMA(train_s, order=order,
                 enforce_stationarity=False, enforce_invertibility=False).fit()
@@ -88,10 +87,8 @@ def arima_roll_single_fit(train_s: pd.Series, test_s: pd.Series, order=(2, 1, 2)
         y_true = test_s.iloc[i]
         trues.append(y_true)
         try:
-            # Fast state update
             res = res.append(test_s.iloc[i:i+1])
         except Exception:
-            # Rare fallback: refit to train + observed test slice
             res = ARIMA(pd.concat([train_s, test_s.iloc[:i+1]]), order=order,
                         enforce_stationarity=False, enforce_invertibility=False).fit()
     return np.array(trues), np.array(preds)
@@ -108,23 +105,57 @@ def arima_future_forecast(all_close: pd.Series, order, steps: int) -> np.ndarray
 # -------------------------
 # LSTM helpers (Keras .keras + simple min-max scaling)
 # -------------------------
-def _maybe_load_lstm() -> bool:
+def _find_lstm_info_path():
+    if os.path.isfile(LSTM_INFO_PRIMARY):
+        return LSTM_INFO_PRIMARY
+    if os.path.isfile(LSTM_INFO_ALT):
+        return LSTM_INFO_ALT
+    return None
+
+def _maybe_load_lstm(required: bool = False) -> bool:
     """
-    Load LSTM and metadata once, if present. Returns True if available.
+    Load LSTM and metadata once. If `required=True`, raise a RuntimeError when missing.
     """
     global _LSTM_MODEL, _LSTM_META
+
     if _LSTM_MODEL is not None and _LSTM_META is not None:
         return True
-    if not (os.path.isfile(LSTM_KERAS) and os.path.isfile(LSTM_INFO)):
+
+    info_path = _find_lstm_info_path()
+    missing_parts = []
+    if not os.path.isfile(LSTM_KERAS):
+        missing_parts.append(LSTM_KERAS)
+    if info_path is None:
+        missing_parts.append("lstm_info.json (or lstm_model/info.json)")
+
+    if missing_parts:
+        msg = ("LSTM artifacts missing: " + ", ".join(missing_parts) +
+               ". Please add them to the repo. (LSTM is required.)")
+        if required:
+            raise RuntimeError(msg)
+        print("[WARN]", msg)
         return False
+
     try:
         import tensorflow as tf  # import lazily
         _LSTM_MODEL = tf.keras.models.load_model(LSTM_KERAS, compile=False)
-        with open(LSTM_INFO, "r") as f:
+        with open(info_path, "r") as f:
             _LSTM_META = json.load(f)
-        assert all(k in _LSTM_META for k in ("lookback", "train_min", "train_max"))
+
+        # Validate keys
+        for k in ("lookback", "train_min", "train_max"):
+            if k not in _LSTM_META:
+                raise RuntimeError(f"Missing key '{k}' in {info_path}")
+
+        # Optional: feature_range
+        if "feature_range" not in _LSTM_META:
+            _LSTM_META["feature_range"] = [0.0, 1.0]
+
         return True
+
     except Exception as e:
+        if required:
+            raise RuntimeError(f"LSTM load failed: {e}")
         print(f"[WARN] LSTM load failed: {e}")
         _LSTM_MODEL, _LSTM_META = None, None
         return False
@@ -151,8 +182,8 @@ def lstm_roll_eval_and_forecast(close_train: pd.Series, close_test: pd.Series, h
       (1) fair 1-step-ahead evaluation across TEST
       (2) recursive forecast for 'horizon' steps
     """
-    if not _maybe_load_lstm():
-        return None, None, None
+    # MUST be loaded; raise if not
+    _maybe_load_lstm(required=True)
 
     import tensorflow as tf  # assured by requirements
     model = _LSTM_MODEL
@@ -164,6 +195,12 @@ def lstm_roll_eval_and_forecast(close_train: pd.Series, close_test: pd.Series, h
     train_vals = close_train.values.astype("float32")
     test_vals  = close_test.values.astype("float32")
 
+    if len(train_vals) < L:
+        raise RuntimeError(
+            f"Not enough history for LSTM lookback L={L}. "
+            f"Increase 'How much history?' so train length ≥ {L}."
+        )
+
     train_s = _scale(train_vals, xmin, xmax)
     test_s  = _scale(test_vals,  xmin, xmax)
 
@@ -173,7 +210,10 @@ def lstm_roll_eval_and_forecast(close_train: pd.Series, close_test: pd.Series, h
 
     test_start_idx = len(train_s) - L
     if test_start_idx < 0:
-        return None, None, None  # not enough history for the lookback
+        raise RuntimeError(
+            f"Not enough history for LSTM lookback L={L}. "
+            f"Increase 'How much history?' so train length ≥ {L}."
+        )
 
     X_test = X_all[test_start_idx : test_start_idx + len(test_s)]
     y_test = y_all[test_start_idx : test_start_idx + len(test_s)]
@@ -193,8 +233,7 @@ def lstm_roll_eval_and_forecast(close_train: pd.Series, close_test: pd.Series, h
         x = window.reshape(1, L, 1)
         yhat_s = model.predict(x, verbose=0).reshape(-1)[0]
         fut.append(_unscale(yhat_s, xmin, xmax))
-        # roll the window (note: keep in scaled space)
-        window = np.concatenate([window[1:], [yhat_s]], axis=0)
+        window = np.concatenate([window[1:], [yhat_s]], axis=0)  # roll in scaled space
 
     return y_true, y_pred, np.asarray(fut)
 
@@ -220,7 +259,7 @@ def make_chart(hist_df: pd.DataFrame,
         h['Date'] = pd.to_datetime(h['Date'])
         h_last = h.iloc[-252:] if len(h) > 252 else h
 
-        fig, ax = plt.subplots(figsize=(9, 4))
+        fig, ax = plt.subplots(figsize=( nine := 9, four := 4))
         ax.plot(h_last['Date'], h_last['Close ($)'], label="Historical Close", linewidth=2)
 
         if arima_fut is not None and not arima_fut.empty:
@@ -250,29 +289,29 @@ def do_forecast(ticker_choice: str, years_back: int, horizon_days: int):
     Returns:
       - Historical table
       - Future ARIMA forecast table
-      - KPI markdown (ARIMA + LSTM if available)
-      - Chart (ARIMA + optional LSTM)
-      - Comparison table (ARIMA vs LSTM) if LSTM present
+      - KPI markdown (ARIMA + LSTM)
+      - Chart (ARIMA + LSTM)
+      - Comparison table (ARIMA vs LSTM)
     """
     try:
+        # Require LSTM to be present & loadable (will raise otherwise)
+        _maybe_load_lstm(required=REQUIRE_LSTM)
+
         ticker = (ticker_choice or "").strip().upper()
         end = datetime.utcnow().date()
         start = end - timedelta(days=int(years_back * 365.25))
 
         df = fetch_data(ticker, start.isoformat(), end.isoformat())
         if len(df) < 80:
-            msg = "Not enough data to evaluate. Increase 'How much history?'."
-            empty = pd.DataFrame()
-            return empty, empty, msg, None, pd.DataFrame()
+            raise RuntimeError("Not enough data to evaluate. Increase 'How much history?'.")
 
-        split_idx = int(len(df) * 0.8)
         order = load_arima_order(default=(2, 1, 2))
+        split_idx = int(len(df) * 0.8)
 
         # ---------- ARIMA ----------
         train_s, test_s = df['Close'][:split_idx], df['Close'][split_idx:]
         y_true_a, y_pred_a = arima_roll_single_fit(train_s, test_s, order=order)
         ar_tbl = metrics_df(y_true_a, y_pred_a)
-
         fut_ar = arima_future_forecast(df['Close'], order, horizon_days)
 
         # Tables for UI (ARIMA)
@@ -284,40 +323,30 @@ def do_forecast(ticker_choice: str, years_back: int, horizon_days: int):
         hist_out = format_prices_df(hist, "date", "close", "Close ($)")
         fut_ar_out = format_prices_df(fut_ar_df, "date", "forecast", "Forecast ($)")
 
-        # ---------- LSTM (optional) ----------
-        lstm_available = _maybe_load_lstm()
-        fut_lstm_out = None
-        comp_tbl = None
-        kpi_extra = ""
-
-        if lstm_available:
-            lstm_true, lstm_pred, fut_lstm = lstm_roll_eval_and_forecast(train_s, test_s, horizon_days)
-            if lstm_true is not None:
-                lstm_tbl = metrics_df(lstm_true, lstm_pred)
-                fut_lstm_df = pd.DataFrame({"date": pd.Index(future_dates).astype(str),
-                                            "forecast": fut_lstm.reshape(-1,)})
-                fut_lstm_out = format_prices_df(fut_lstm_df, "date", "forecast", "Forecast ($)")
-
-                comp_tbl = pd.DataFrame({
-                    "Model": ["ARIMA", "LSTM"],
-                    "RMSE": [ar_tbl["RMSE"][0], lstm_tbl["RMSE"][0]],
-                    "MAE": [ar_tbl["MAE"][0], lstm_tbl["MAE"][0]],
-                    "MAPE (%)": [ar_tbl["MAPE (%)"][0], lstm_tbl["MAPE (%)"][0]],
-                }).round(3)
-
-                kpi_extra = (f"\n**LSTM** | Test RMSE ≈ {lstm_tbl['RMSE'][0]:.2f}, "
-                             f"MAE ≈ {lstm_tbl['MAE'][0]:.2f}, "
-                             f"MAPE ≈ {lstm_tbl['MAPE (%)'][0]:.2f}%")
-
-        if comp_tbl is None:
-            comp_tbl = pd.DataFrame(columns=["Model", "RMSE", "MAE", "MAPE (%)"])
-
-        # KPI text (always show ARIMA; add LSTM if available)
+        # ---------- LSTM (REQUIRED) ----------
+        lstm_true, lstm_pred, fut_lstm = lstm_roll_eval_and_forecast(train_s, test_s, horizon_days)
+        lstm_tbl = metrics_df(lstm_true, lstm_pred)
+
+        fut_lstm_df = pd.DataFrame({"date": pd.Index(future_dates).astype(str),
+                                    "forecast": fut_lstm.reshape(-1,)})
+        fut_lstm_out = format_prices_df(fut_lstm_df, "date", "forecast", "Forecast ($)")
+
+        # Comparison table
+        comp_tbl = pd.DataFrame({
+            "Model": ["ARIMA", "LSTM"],
+            "RMSE": [ar_tbl["RMSE"][0], lstm_tbl["RMSE"][0]],
+            "MAE": [ar_tbl["MAE"][0],  lstm_tbl["MAE"][0]],
+            "MAPE (%)": [ar_tbl["MAPE (%)"][0], lstm_tbl["MAPE (%)"][0]],
+        }).round(3)
+
+        # KPI text
         kpi = (
             f"**ARIMA {order}** | Test RMSE ≈ {ar_tbl['RMSE'][0]:.2f}, "
             f"MAE ≈ {ar_tbl['MAE'][0]:.2f}, MAPE ≈ {ar_tbl['MAPE (%)'][0]:.2f}%  "
-            f"| **Ticker:** {ticker}  |  **Forecast days:** {horizon_days}"
-        ) + kpi_extra
+            f"| **Ticker:** {ticker}  |  **Forecast days:** {horizon_days}\n"
+            f"**LSTM** | Test RMSE ≈ {lstm_tbl['RMSE'][0]:.2f}, "
+            f"MAE ≈ {lstm_tbl['MAE'][0]:.2f}, MAPE ≈ {lstm_tbl['MAPE (%)'][0]:.2f}%"
+        )
 
         # Chart overlay
         fig = make_chart(hist_out, fut_ar_out, fut_lstm_out)
@@ -328,6 +357,7 @@ def do_forecast(ticker_choice: str, years_back: int, horizon_days: int):
         traceback.print_exc()
         err = f"❌ {str(e)}"
         empty = pd.DataFrame()
+        # Return an error message in KPI area so users immediately see what's wrong
         return empty, empty, err, None, pd.DataFrame()
 
 # -------------------------
@@ -338,9 +368,9 @@ POPULAR = ["AAPL", "MSFT", "GOOGL", "AMZN", "TSLA", "NVDA", "META"]
 with gr.Blocks(title="DataSynthis_ML_JobTask", theme=gr.themes.Soft()) as demo:
     gr.Markdown(
         """
-        # DataSynthis_ML_JobTask — Stock Forecasts (Simple & Clear)
-        **What it does:** Downloads recent daily prices, evaluates an ARIMA model fairly,
-        predicts the next trading days, and (if provided) compares with a pre-trained LSTM.  
+        # DataSynthis_ML_JobTask — Stock Forecasts (ARIMA + LSTM)
+        **What it does:** Downloads recent daily prices, evaluates ARIMA and LSTM fairly,
+        and predicts the next trading days.  
         **How to use:** Pick a stock, choose how many years of history and how far ahead to forecast, then press **Run Forecast**.
         """
     )
@@ -366,7 +396,7 @@ with gr.Blocks(title="DataSynthis_ML_JobTask", theme=gr.themes.Soft()) as demo:
         run_btn = gr.Button("Run Forecast", variant="primary")
 
         gr.Markdown("### Results")
-        kpi = gr.Markdown(label="Model summary (lower is better)", value="")
+        kpi = gr.Markdown(label="Model summary", value="")
         chart = gr.Plot(label="Chart (last year + forecast)")
         with gr.Accordion("Tables (click to expand)", open=False):
             hist_df = gr.Dataframe(label="Historical Prices", wrap=True, interactive=False)
@@ -379,15 +409,4 @@ with gr.Blocks(title="DataSynthis_ML_JobTask", theme=gr.themes.Soft()) as demo:
             outputs=[hist_df, fut_df, kpi, chart, comp_df]
         )
 
-    with gr.Tab("Help & Glossary"):
-        gr.Markdown(
-            """
-            ### Notes
-            - **ARIMA (p,d,q)** is read from `arima_order.json` and fit on your *selected history* for future forecasts.
-            - **Fair evaluation:** For both ARIMA and (if present) LSTM, we train on the first 80% of your selected history,
-              then predict **one day ahead** across the remaining 20% (no peeking), and report RMSE/MAE/MAPE.
-            - **LSTM (optional):** If `lstm_model.keras` and `lstm_info.json` are present, we score and overlay its forecast.
-            """
-        )
-
 demo.launch()