Update app.py

app.py

@@ -1,6 +1,6 @@
 # app.py — ST_Min_Horizontal_Stress (σhmin)
-# Full Streamlit app — trains the model inside the app (fixed best params or optional GridSearch).
-# No external model file is required. Users can still download the trained .joblib + meta.json.
+# Self-contained Streamlit app that TRAINS a fixed, optimized ML pipeline in-app.
+# No external model files, no model-source UI. Upload Excel and go.
 
 import io, json, os, base64, math
 from pathlib import Path
@@ -9,7 +9,7 @@ from datetime import datetime
 import streamlit as st
 import pandas as pd
 import numpy as np
-import joblib
+import joblib  # only used to cache pipeline inside session
 
 # Matplotlib for static previews & cross-plot
 import matplotlib
@@ -19,6 +19,9 @@ from matplotlib.ticker import FuncFormatter
 
 import plotly.graph_objects as go
 from sklearn.metrics import mean_squared_error
+from sklearn.ensemble import RandomForestRegressor
+from sklearn.pipeline import Pipeline
+from sklearn.impute import SimpleImputer
 
 # =========================
 # App constants / defaults
@@ -26,13 +29,26 @@ from sklearn.metrics import mean_squared_error
 APP_NAME = "ST_Min_Horizontal_Stress"
 TAGLINE  = "Real-Time Minimum Horizontal Stress Prediction"
 
-FEATURES   = ["Q (gpm)", "SPP (psi)", "ST (kft.lbf)", "WOB (klbf)", "ROP (ft/h)"]
-TARGET     = "σhmin (MPa)"
-PRED_COL   = "σhmin_Pred"
-ACTUAL_COL = None               # If your workbook has a separate actual column, set via meta.json (actual_col)
-TRANSFORM  = "none"             # "none" | "log10" | "ln"
+# -------- Option B (canonical names = what your files already have) --------
+FEATURES   = ["Q (gpm)", "SPP (psi)", "T (kft.lbf)", "WOB (klbf)", "ROP (ft/h)"]
+TARGET     = "MINStress_Actual"
+PRED_COL   = "MINStress_Pred"
+ACTUAL_COL = TARGET
+TRANSFORM  = "none"     # "none" | "log10" | "ln"
 UNITS      = "MPa"
 
+# ---- "Best" (fixed) model setup baked into the code (no grid search UI) ----
+BEST_PARAMS = dict(
+    n_estimators=400,
+    max_depth=None,
+    min_samples_split=2,
+    min_samples_leaf=1,
+    max_features=0.6,
+    bootstrap=True,
+    random_state=42,
+    n_jobs=-1,
+)
+
 # Color / layout
 COLORS = {"pred": "#1f77b4", "actual": "#f2b702", "ref": "#5a5a5a"}
 CROSS_W, CROSS_H = 350, 350
@@ -40,10 +56,7 @@ TRACK_H, TRACK_W = 1000, 500
 FONT_SZ = 13
 BOLD_FONT = "Arial Black, Arial, sans-serif"
 
-STRICT_VERSION_CHECK = True
-
-# Local (optional) — only used for Excel export helper sizing
-MODELS_DIR = Path("models")
+STRICT_VERSION_CHECK = False  # we now train in this env, so no version pin warnings
 
 # =========================
 # Page / CSS
@@ -74,7 +87,7 @@ TABLE_CENTER_CSS = [
 ]
 
 # =========================
-# Password gate
+# Password gate (optional)
 # =========================
 def inline_logo(path="logo.png") -> str:
     try:
@@ -91,8 +104,8 @@ def add_password_gate() -> None:
         required = os.environ.get("APP_PASSWORD", "")
 
     if not required:
-        st.warning("Set APP_PASSWORD in Secrets (or environment) and restart.")
-        st.stop()
+        return  # no password configured
+
     if st.session_state.get("auth_ok", False):
         return
 
@@ -182,18 +195,6 @@ def ensure_cols(df: pd.DataFrame, cols: list[str]) -> bool:
 def _nice_tick0(xmin: float, step: float = 0.1) -> float:
     return step * math.floor(xmin / step) if np.isfinite(xmin) else xmin
 
-# ---------- Aliasing / Normalization ----------
-def _normalize_columns(df: pd.DataFrame, canonical_features: list[str], target_name: str, aliases: dict|None) -> pd.DataFrame:
-    out = df.copy()
-    out.columns = [str(c).strip().replace(" ,", ",").replace(",  ", ", ").replace("  ", " ") for c in out.columns]
-    if not aliases: return out
-    mapping = {}
-    for can, alist in aliases.items():
-        for a in alist:
-            if a in out.columns and can != a:
-                mapping[a] = can
-    return out.rename(columns=mapping)
-
 # ---------- Transform helpers ----------
 def _inv_transform(x: np.ndarray, transform: str) -> np.ndarray:
     t = (transform or "none").lower()
@@ -468,6 +469,23 @@ def preview_tracks(df: pd.DataFrame, cols: list[str]):
     fig.tight_layout()
     return fig
 
+# =========================
+# Fixed training pipeline
+# =========================
+def build_pipeline() -> Pipeline:
+    """
+    Fixed, optimized pipeline:
+      - Numeric imputation (median)
+      - RandomForestRegressor with tuned params (BEST_PARAMS)
+    Trees don't need scaling; robust to feature distributions.
+    """
+    model = RandomForestRegressor(**BEST_PARAMS)
+    pipe = Pipeline(steps=[
+        ("imputer", SimpleImputer(strategy="median")),
+        ("model", model),
+    ])
+    return pipe
+
 # =========================
 # Session state
 # =========================
@@ -479,6 +497,7 @@ st.session_state.setdefault("dev_file_bytes",b"")
 st.session_state.setdefault("dev_file_loaded",False)
 st.session_state.setdefault("dev_preview",False)
 st.session_state.setdefault("show_preview_modal", False)
+st.session_state.setdefault("fitted_model", None)  # cache trained pipeline
 
 # =========================
 # Sidebar branding
@@ -508,144 +527,6 @@ def sticky_header(title, message):
         unsafe_allow_html=True
     )
 
-# ===============================================================
-# TRAIN THE MODEL IN-APP (no external pickle needed)
-# ===============================================================
-from sklearn.ensemble import RandomForestRegressor
-from sklearn.model_selection import train_test_split
-
-# ❶ Set YOUR optimized hyperparameters here
-BEST_PARAMS = {
-    "n_estimators": 300,
-    "max_depth": 22,
-    "max_features": "sqrt",    # or "log2" / float in (0,1]
-    "min_samples_split": 2,
-    "min_samples_leaf": 1,
-    "bootstrap": True,
-    "random_state": 42,
-    "n_jobs": -1
-}
-
-st.sidebar.markdown("### Model source")
-source = st.sidebar.radio(
-    "Choose how to get the model",
-    ["Train now (fixed best params)", "Train with Grid Search (optional)"],
-    help="Avoids uploading big pickles. Deterministic best-params training is recommended."
-)
-
-st.sidebar.markdown("### Training data")
-file_train = st.sidebar.file_uploader("Upload Excel for training (has Train sheet or any sheet)", type=["xlsx","xls"])
-
-def _train_model_fixed(X: pd.DataFrame, y: pd.Series, params: dict) -> RandomForestRegressor:
-    rf = RandomForestRegressor(**params)
-    rf.fit(X, y)
-    return rf
-
-def _download_buttons(model_obj, meta_dict):
-    # model
-    buf_model = io.BytesIO()
-    joblib.dump(model_obj, buf_model)
-    buf_model.seek(0)
-    st.download_button("⬇️ Download trained model (.joblib)", buf_model.getvalue(), "minstress_model.joblib")
-
-    # meta
-    meta_bytes = json.dumps(meta_dict, indent=2).encode("utf-8")
-    st.download_button("⬇️ Download meta (.json)", meta_bytes, "minstress_meta.json")
-
-if not file_train:
-    st.info("Upload a training Excel file in the sidebar to build the model.")
-    st.stop()
-
-# Load train data
-book_train = read_book_bytes(file_train.getvalue())
-sheet_train = next((s for s in book_train if s.lower() in ("train", "training")), list(book_train)[0])
-df_tr0 = _normalize_columns(book_train[sheet_train].copy(), FEATURES, TARGET, None)
-
-# Build X/y
-act_col = ACTUAL_COL if (ACTUAL_COL and ACTUAL_COL in df_tr0.columns) else TARGET
-if not ensure_cols(df_tr0, FEATURES + [act_col]):
-    st.stop()
-
-X_all = _make_X(df_tr0, FEATURES).copy()
-y_all = pd.to_numeric(df_tr0[act_col], errors="coerce")
-
-# Split for reporting
-tsz = st.sidebar.slider("Validation split for reporting", 0.10, 0.40, 0.20, 0.05)
-seed = st.sidebar.number_input("Random seed", 0, 1_000_000, BEST_PARAMS.get("random_state", 42), step=1)
-Xtr, Xva, ytr, yva = train_test_split(X_all, y_all, test_size=tsz, random_state=seed)
-
-if source == "Train with Grid Search (optional)":
-    from sklearn.model_selection import GridSearchCV
-    st.sidebar.markdown("### Grid Search")
-    n_list    = st.sidebar.multiselect("n_estimators", [100, 200, 300, 400], default=[BEST_PARAMS["n_estimators"]])
-    depth_list= st.sidebar.multiselect("max_depth",  [12, 16, 20, 22, 26], default=[BEST_PARAMS["max_depth"]])
-    maxf_list = st.sidebar.multiselect("max_features", ["sqrt", "log2"], default=[BEST_PARAMS["max_features"]])
-
-    param_grid = {
-        "n_estimators": n_list or [BEST_PARAMS["n_estimators"]],
-        "max_depth": depth_list or [BEST_PARAMS["max_depth"]],
-        "max_features": maxf_list or [BEST_PARAMS["max_features"]],
-        "min_samples_split": [BEST_PARAMS["min_samples_split"]],
-        "min_samples_leaf": [BEST_PARAMS["min_samples_leaf"]],
-        "bootstrap": [BEST_PARAMS["bootstrap"]],
-        "random_state": [seed]
-    }
-    base = RandomForestRegressor(n_jobs=-1)
-    with st.spinner("Running GridSearchCV..."):
-        gs = GridSearchCV(base, param_grid=param_grid, cv=3, n_jobs=-1, refit=True)
-        gs.fit(Xtr, ytr)
-    best = gs.best_estimator_
-    st.success(f"GridSearch done. Best params: {gs.best_params_}")
-
-    # Validation report
-    pred_tr = best.predict(Xtr); pred_va = best.predict(Xva)
-    m_train = {"R": pearson_r(ytr, pred_tr), "RMSE": rmse(ytr, pred_tr), "MAPE%": mape(ytr, pred_tr)}
-    m_valid = {"R": pearson_r(yva, pred_va), "RMSE": rmse(yva, pred_va), "MAPE%": mape(yva, pred_va)}
-    st.write("**Training split metrics**:", m_train)
-    st.write("**Validation split metrics**:", m_valid)
-
-    # Final fit on all data with best params
-    model = RandomForestRegressor(**{**gs.best_params_, "n_jobs": -1, "random_state": seed})
-    model.fit(X_all, y_all)
-else:
-    # Deterministic fixed-params training (recommended)
-    params = {**BEST_PARAMS, "random_state": seed}
-    with st.spinner("Training fixed-params model..."):
-        tmp_model = _train_model_fixed(Xtr, ytr, params)
-        pred_tr = tmp_model.predict(Xtr); pred_va = tmp_model.predict(Xva)
-        m_train = {"R": pearson_r(ytr, pred_tr), "RMSE": rmse(ytr, pred_tr), "MAPE%": mape(ytr, pred_tr)}
-        m_valid = {"R": pearson_r(yva, pred_va), "RMSE": rmse(yva, pred_va), "MAPE%": mape(yva, pred_va)}
-        st.write("**Training split metrics**:", m_train)
-        st.write("**Validation split metrics**:", m_valid)
-
-        model = _train_model_fixed(X_all, y_all, params)
-
-# Create meta + training ranges for OOR checks later
-meta = {
-    "features": FEATURES,
-    "target": TARGET,
-    "pred_col": PRED_COL,
-    "actual_col": ACTUAL_COL,
-    "transform": TRANSFORM,
-    "units": UNITS,
-    "versions": {
-        "numpy": np.__version__,
-        "scikit_learn": __import__("sklearn").__version__
-    },
-    "training": {
-        "n_rows": int(len(X_all)),
-        "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
-        "used_grid_search": (source == "Train with Grid Search (optional)")
-    }
-}
-
-tr_min = X_all.min().to_dict()
-tr_max = X_all.max().to_dict()
-st.session_state.train_ranges = {f: (float(tr_min[f]), float(tr_max[f])) for f in FEATURES}
-
-st.success("Model ready ✓  — proceed to **Case Building**, **Validation**, or **Prediction**.")
-_download_buttons(model, meta)
-
 # =========================
 # INTRO
 # =========================
@@ -654,15 +535,15 @@ if st.session_state.app_step == "intro":
     st.markdown(f"This software is developed by *Smart Thinking AI-Solutions Team* to estimate **Minimum Horizontal Stress** ({UNITS}) from drilling/offset data.")
     st.subheader("How It Works")
     st.markdown(
-        "1) **Upload your data to build the case and preview the model performance.**  \n"
-        "2) Click **Run Model** to compute metrics and plots.  \n"
-        "3) **Proceed to Validation** (with actual) or **Proceed to Prediction** (no actual)."
+        "1) **Upload your Train/Test file** and click **Run Model** to fit the baked-in pipeline.  \n"
+        "2) **Validate** on held-out wells (with actual).  \n"
+        "3) **Predict** on wells without actual."
     )
     if st.button("Start Showcase", type="primary"):
         st.session_state.app_step = "dev"; st.rerun()
 
 # =========================
-# CASE BUILDING (Train/Test) — optional evaluation stage
+# CASE BUILDING (Train/Test)
 # =========================
 def _find_sheet(book, names):
     low2orig = {k.lower(): k for k in book.keys()}
@@ -678,6 +559,8 @@ if st.session_state.app_step == "dev":
         st.session_state.dev_file_name = up.name
         st.session_state.dev_file_loaded = True
         st.session_state.dev_preview = False
+        st.session_state.fitted_model = None  # reset
+
     if st.session_state.dev_file_loaded:
         tmp = read_book_bytes(st.session_state.dev_file_bytes)
         if tmp:
@@ -707,33 +590,50 @@ if st.session_state.app_step == "dev":
             st.markdown('<div class="st-message-box st-error">Workbook must include Train/Training and Test/Testing sheets.</div>', unsafe_allow_html=True)
             st.stop()
 
-        tr0 = _normalize_columns(book[sh_train].copy(), FEATURES, TARGET, None)
-        te0 = _normalize_columns(book[sh_test].copy(),  FEATURES, TARGET, None)
+        tr0 = book[sh_train].copy()
+        te0 = book[sh_test].copy()
 
-        actual_col = ACTUAL_COL if (ACTUAL_COL and ACTUAL_COL in tr0.columns) else TARGET
-        if not (ensure_cols(tr0, FEATURES+[actual_col]) and ensure_cols(te0, FEATURES+[actual_col])):
+        # Ensure columns exist
+        if not (ensure_cols(tr0, FEATURES+[TARGET]) and ensure_cols(te0, FEATURES+[TARGET])):
             st.markdown('<div class="st-message-box st-error">Missing required columns.</div>', unsafe_allow_html=True)
             st.stop()
 
+        # Prepare X,y
+        X_tr = _make_X(tr0, FEATURES)
+        y_tr = pd.to_numeric(tr0[TARGET], errors="coerce")
+        X_te = _make_X(te0, FEATURES)
+        y_te = pd.to_numeric(te0[TARGET], errors="coerce")
+
+        # Drop rows with NA in y
+        mask_tr = np.isfinite(y_tr)
+        X_tr, y_tr = X_tr.loc[mask_tr], y_tr.loc[mask_tr]
+        mask_te = np.isfinite(y_te)
+        X_te, y_te = X_te.loc[mask_te], y_te.loc[mask_te]
+
+        pipe = build_pipeline()
+        pipe.fit(X_tr, y_tr)
+        st.session_state.fitted_model = pipe  # cache
+
+        # Predictions
         tr = tr0.copy(); te = te0.copy()
-        tr[PRED_COL] = _inv_transform(model.predict(_make_X(tr0, FEATURES)), TRANSFORM)
-        te[PRED_COL] = _inv_transform(model.predict(_make_X(te0, FEATURES)), TRANSFORM)
+        tr[PRED_COL] = _inv_transform(pipe.predict(_make_X(tr0, FEATURES)), TRANSFORM)
+        te[PRED_COL] = _inv_transform(pipe.predict(_make_X(te0, FEATURES)), TRANSFORM)
 
         st.session_state.results["Train"] = tr
         st.session_state.results["Test"]  = te
         st.session_state.results["m_train"] = {
-            "R":    pearson_r(tr[actual_col], tr[PRED_COL]),
-            "RMSE": rmse(tr[actual_col], tr[PRED_COL]),
-            "MAPE%": mape(tr[actual_col], tr[PRED_COL]),
+            "R":    pearson_r(tr[TARGET], tr[PRED_COL]),
+            "RMSE": rmse(tr[TARGET], tr[PRED_COL]),
+            "MAPE%": mape(tr[TARGET], tr[PRED_COL]),
         }
         st.session_state.results["m_test"] = {
-            "R":    pearson_r(te[actual_col], te[PRED_COL]),
-            "RMSE": rmse(te[actual_col], te[PRED_COL]),
-            "MAPE%": mape(te[actual_col], te[PRED_COL]),
+            "R":    pearson_r(te[TARGET], te[PRED_COL]),
+            "RMSE": rmse(te[TARGET], te[PRED_COL]),
+            "MAPE%": mape(te[TARGET], te[PRED_COL]),
         }
 
-        tr_min2 = tr[FEATURES].min().to_dict(); tr_max2 = tr[FEATURES].max().to_dict()
-        st.session_state.train_ranges = {f:(float(tr_min2[f]), float(tr_max2[f])) for f in FEATURES}
+        tr_min = tr[FEATURES].min().to_dict(); tr_max = tr[FEATURES].max().to_dict()
+        st.session_state.train_ranges = {f:(float(tr_min[f]), float(tr_max[f])) for f in FEATURES}
         st.markdown('<div class="st-message-box st-success">Case has been built and results are displayed below.</div>', unsafe_allow_html=True)
 
     def _dev_block(df, m):
@@ -753,8 +653,7 @@ if st.session_state.app_step == "dev":
             st.plotly_chart(track_plot(df, include_actual=True),
                             use_container_width=False, config={"displayModeBar": False, "scrollZoom": True})
         with col_cross:
-            act_col2 = ACTUAL_COL if (ACTUAL_COL and ACTUAL_COL in df.columns) else TARGET
-            st.pyplot(cross_plot_static(df[act_col2], df[PRED_COL]), use_container_width=False)
+            st.pyplot(cross_plot_static(df[TARGET], df[PRED_COL]), use_container_width=False)
 
     if "Train" in st.session_state.results or "Test" in st.session_state.results:
         tab1, tab2 = st.tabs(["Training", "Testing"])
@@ -781,20 +680,23 @@ if st.session_state.app_step == "validate":
     if st.sidebar.button("⬅ Back to Case Building", use_container_width=True): st.session_state.app_step="dev"; st.rerun()
     if st.sidebar.button("Proceed to Prediction ▶", use_container_width=True): st.session_state.app_step="predict"; st.rerun()
 
-    sticky_header("Validate the Model", "Upload a dataset with the same **features** and **actual stress** to evaluate performance.")
+    sticky_header("Validate the Model", "Upload a dataset with the same **features** and **MINStress_Actual** to evaluate performance.")
 
     if go_btn and up is not None:
+        if st.session_state.fitted_model is None:
+            st.error("Please train the model first in Case Building.")
+            st.stop()
+
         book = read_book_bytes(up.getvalue())
         names = list(book.keys())
         name = next((s for s in names if s.lower() in ("validation","validate","validation2","val","val2")), names[0])
-        df0 = _normalize_columns(book[name].copy(), FEATURES, TARGET, None)
+        df0 = book[name].copy()
 
-        act_col2 = ACTUAL_COL if (ACTUAL_COL and ACTUAL_COL in df0.columns) else TARGET
-        if not ensure_cols(df0, FEATURES+[act_col2]):
+        if not ensure_cols(df0, FEATURES+[TARGET]):
             st.markdown('<div class="st-message-box st-error">Missing required columns.</div>', unsafe_allow_html=True); st.stop()
 
         df = df0.copy()
-        df[PRED_COL] = _inv_transform(model.predict(_make_X(df0, FEATURES)), TRANSFORM)
+        df[PRED_COL] = _inv_transform(st.session_state.fitted_model.predict(_make_X(df0, FEATURES)), TRANSFORM)
         st.session_state.results["Validate"] = df
 
         ranges = st.session_state.train_ranges; oor_pct = 0.0; tbl=None
@@ -810,9 +712,9 @@ if st.session_state.app_step == "validate":
                 )
 
         st.session_state.results["m_val"] = {
-            "R":     pearson_r(df[act_col2], df[PRED_COL]),
-            "RMSE":  rmse(df[act_col2], df[PRED_COL]),
-            "MAPE%": mape(df[act_col2], df[PRED_COL]),
+            "R":     pearson_r(df[TARGET], df[PRED_COL]),
+            "RMSE":  rmse(df[TARGET], df[PRED_COL]),
+            "MAPE%": mape(df[TARGET], df[PRED_COL]),
         }
         st.session_state.results["sv_val"] = {"n":len(df), "pred_min":float(df[PRED_COL].min()), "pred_max":float(df[PRED_COL].max()), "oor":oor_pct}
         st.session_state.results["oor_tbl"] = tbl
@@ -834,8 +736,7 @@ if st.session_state.app_step == "validate":
             st.plotly_chart(track_plot(st.session_state.results["Validate"], include_actual=True),
                             use_container_width=False, config={"displayModeBar": False, "scrollZoom": True})
         with col_cross:
-            act_col3 = ACTUAL_COL if (ACTUAL_COL and ACTUAL_COL in st.session_state.results["Validate"].columns) else TARGET
-            st.pyplot(cross_plot_static(st.session_state.results["Validate"][act_col3],
+            st.pyplot(cross_plot_static(st.session_state.results["Validate"][TARGET],
                                         st.session_state.results["Validate"][PRED_COL]),
                       use_container_width=False)
 
@@ -863,15 +764,19 @@ if st.session_state.app_step == "predict":
     go_btn = st.sidebar.button("Predict", type="primary", use_container_width=True)
     if st.sidebar.button("⬅ Back to Case Building", use_container_width=True): st.session_state.app_step="dev"; st.rerun()
 
-    sticky_header("Prediction", "Upload a dataset with the feature columns (no actual column).")
+    sticky_header("Prediction", "Upload a dataset with the 5 feature columns (no actual column).")
 
     if go_btn and up is not None:
+        if st.session_state.fitted_model is None:
+            st.error("Please train the model first in Case Building.")
+            st.stop()
+
         book = read_book_bytes(up.getvalue()); name = list(book.keys())[0]
-        df0 = _normalize_columns(book[name].copy(), FEATURES, TARGET, None)
+        df0 = book[name].copy()
         if not ensure_cols(df0, FEATURES):
             st.markdown('<div class="st-message-box st-error">Missing required columns.</div>', unsafe_allow_html=True); st.stop()
         df = df0.copy()
-        df[PRED_COL] = _inv_transform(model.predict(_make_X(df0, FEATURES)), TRANSFORM)
+        df[PRED_COL] = _inv_transform(st.session_state.fitted_model.predict(_make_X(df0, FEATURES)), TRANSFORM)
         st.session_state.results["PredictOnly"] = df
 
         ranges = st.session_state.train_ranges; oor_pct = 0.0
@@ -922,7 +827,7 @@ if st.session_state.show_preview_modal:
             tabs = st.tabs(names)
             for t, name in zip(tabs, names):
                 with t:
-                    df = _normalize_columns(book_to_preview[name], FEATURES, TARGET, None)
+                    df = book_to_preview[name]
                     t1, t2 = st.tabs(["Tracks", "Summary"])
                     with t1:
                         st.pyplot(preview_tracks(df, FEATURES), use_container_width=True)