Update src/streamlit_app.py

src/streamlit_app.py

@@ -261,7 +261,7 @@ def generate_advanced_flatfile(
 
     # clean NaN and infinite
     df.replace([np.inf, -np.inf], np.nan, inplace=True)
-    df.fillna(method="bfill", inplace=True)
+    df.bfill(inplace=True)
     df.fillna(0, inplace=True)
 
     # save CSV & metadata
@@ -847,13 +847,117 @@ with tabs[4]:
                     st.stop()
                 
                 # Meta prediction
-                y_meta_pred = meta.predict(X_meta_val)
+                y_meta_pred = meta.predict(pd.DataFrame(X_meta_val, columns=X_stack.columns))
                 
                 # Final evaluation
                 final_r2 = r2_score(y_val, y_meta_pred)
                 final_rmse = float(np.sqrt(mean_squared_error(y_val, y_meta_pred)))
                 st.success("AutoML + Stacking complete — metrics, artifacts, and SHAP ready.")
                 log(f"Completed stacking. Final R2={final_r2:.4f}, RMSE={final_rmse:.4f}")
+                # ===============================
+                # OPERATOR ADVISORY SYSTEM
+                # ===============================
+                st.markdown("---")
+                st.subheader("Operator Advisory System — Real-Time Shift Recommendations")
+                
+                try:
+                    # Use top base model already identified
+                    top_base = next((b for b in base_models if b["family"] == selected[0]), None)
+                    if top_base and hasattr(top_base["model"], "predict"):
+                        sample_X = X_val.sample(min(300, len(X_val)), random_state=42)
+                        model = top_base["model"]
+                        
+                        # SHAP direction analysis
+                        expl = shap.TreeExplainer(model)
+                        shap_vals = expl.shap_values(sample_X)
+                        mean_abs = np.abs(shap_vals).mean(axis=0)
+                        mean_sign = np.sign(shap_vals).mean(axis=0)
+                        importance = pd.DataFrame({
+                            "Feature": sample_X.columns,
+                            "Mean |SHAP|": mean_abs,
+                            "Mean SHAP Sign": mean_sign
+                        }).sort_values("Mean |SHAP|", ascending=False)
+                        
+                        # Display Top 5 Drivers
+                        st.markdown("### Top 5 Operational Drivers Influencing Target")
+                        st.dataframe(importance.head(5).style.format({"Mean |SHAP|": "{:.3f}", "Mean SHAP Sign": "{:.3f}"}))
+                
+                        # Direction-based recommendations
+                        recommendations = []
+                        for _, row in importance.head(5).iterrows():
+                            f = row["Feature"]
+                            s = row["Mean SHAP Sign"]
+                            if s > 0.05:
+                                recommendations.append(f"Increase `{f}` likely increases `{target}`")
+                            elif s < -0.05:
+                                recommendations.append(f"Decrease `{f}` likely increases `{target}`")
+                            else:
+                                recommendations.append(f" `{f}` is neutral or nonlinear for `{target}`")
+                
+                        st.markdown("###  Suggested Operator Adjustments (Model-Inferred)")
+                        st.write("\n".join(recommendations))
+                
+                        # Delta recommendations vs previous shift
+                        prev_shift = df.tail(200).mean(numeric_only=True)
+                        recommended_shift = prev_shift.copy()
+                        for rec in recommendations:
+                            if "Increase" in rec:
+                                name = rec.split('`')[1]
+                                if name in recommended_shift:
+                                    recommended_shift[name] *= 1.03  # +3%
+                            elif "Decrease" in rec:
+                                name = rec.split('`')[1]
+                                if name in recommended_shift:
+                                    recommended_shift[name] *= 0.97  # -3%
+                
+                        # Delta table
+                        st.markdown("### 🧾 Shift Adjustment Summary (vs Previous 200 Samples)")
+                        deltas = pd.DataFrame({
+                            "Current Avg": prev_shift,
+                            "Suggested": recommended_shift,
+                            "Δ (%)": ((recommended_shift - prev_shift) / prev_shift * 100)
+                        }).loc[[r.split('`')[1] for r in recommendations if '`' in r]].round(2)
+                
+                        st.dataframe(deltas.fillna(0).style.format("{:.2f}"))
+                        log("Operator advisory system executed successfully.")
+                
+                        # Optional: LLM-generated human-friendly summary
+                        st.markdown("### Natural Language Operator Note")
+                        try:
+                            import importlib.util
+                            if importlib.util.find_spec("transformers"):
+                                from transformers import pipeline
+                                tiny_llm_path = os.path.join(LOG_DIR, "cached_tiny_llm")
+                                if os.path.exists(os.path.join(tiny_llm_path, "config.json")):
+                                    from transformers import AutoModelForCausalLM, AutoTokenizer
+                                    model = AutoModelForCausalLM.from_pretrained(tiny_llm_path)
+                                    tokenizer = AutoTokenizer.from_pretrained(tiny_llm_path)
+                                    assistant = pipeline("text-generation", model=model, tokenizer=tokenizer)
+                                else:
+                                    assistant = pipeline("text-generation", model="sshleifer/tiny-gpt2")
+                
+                                llm_prompt = f"""
+                                You are a metallurgical process advisor working in a steel manufacturing unit.
+                                Based on these recommendations:
+                                {recommendations}
+                                and these shift averages:
+                                {deltas.to_dict(orient='index')}
+                                Write a concise 3-line message to the operator suggesting what to adjust this shift.
+                                """
+                                resp = assistant(llm_prompt, max_new_tokens=80, do_sample=True, temperature=0.6)[0]["generated_text"]
+                                st.info(resp)
+                                log("Operator LLM advisory note generated successfully.")
+                            else:
+                                st.warning("Transformers not available — install it for text generation.")
+                        except Exception as e:
+                            st.warning(f"LLM advisory generation skipped: {e}")
+                
+                    else:
+                        st.info("No suitable model found for operator advisory system.")
+                except Exception as e:
+                    st.error(f"Operator advisory system failed: {e}")
+                    log(f"Operator advisory error: {e}")
+
 
                 
                 c1, c2 = st.columns(2)
@@ -896,75 +1000,69 @@ with tabs[4]:
                 st.success(" AutoML + Stacking complete — metrics, artifacts, and SHAP ready.")
 
                 # --- Store AutoML summary for optional LLM advisory ---
-                st.session_state["automl_summary"] = {
-                    "leaderboard": lb[["family", "cv_r2"]].round(4).to_dict(orient="records"),
-                    "final_r2": float(final_r2),
-                    "final_rmse": float(final_rmse),
-                    "target": target,
-                    "use_case": use_case
-                }
+                if "automl_summary" not in st.session_state:
+                    st.session_state["automl_summary"] = {
+                        "leaderboard": lb[["family", "cv_r2"]].round(4).to_dict(orient="records"),
+                        "final_r2": float(final_r2),
+                        "final_rmse": float(final_rmse),
+                        "target": target,
+                        "use_case": use_case
+                    }
+                else:
+                    # Always refresh with latest metrics after each run
+                    st.session_state["automl_summary"].update({
+                        "leaderboard": lb[["family", "cv_r2"]].round(4).to_dict(orient="records"),
+                        "final_r2": float(final_r2),
+                        "final_rmse": float(final_rmse),
+                        "target": target,
+                        "use_case": use_case
+                    })
                 
-                # --- Optional: AI Model Recommendation Assistant ---
+                # Persist SHAP-based recommendations for reuse across reruns
+                if "shap_recommendations" not in st.session_state:
+                    st.session_state["shap_recommendations"] = recommendations
+                else:
+                    st.session_state["shap_recommendations"] = recommendations
+                
+                # --- AI Recommendation Assistant (in-memory safe for Hugging Face) ---
                 st.markdown("---")
-                st.subheader("AI Recommendation Assistant (cached local model)")
-                st.caption("Get quick local AI suggestions without internet — cached inside ./logs")
+                st.subheader("AI Recommendation Assistant (in-memory mode)")
+                st.caption("Generates quick local AI suggestions — no file writes required.")
                 
                 if st.button("Get AI Recommendation (tiny local LLM)", key="ai_reco"):
                     summary = st.session_state.get("automl_summary", {})
-                    st.info("Loading local model... first time may take ~10s.")
-                    try:
-                        import importlib.util, os
-                        from pathlib import Path
+                    if not summary:
+                        st.warning("Please run AutoML first to generate context.")
+                        st.stop()
                 
-                        # Ensure transformers is available
-                        if importlib.util.find_spec("transformers") is None:
-                            st.error("Transformers not installed. Run `pip install transformers`.")
-                        else:
-                            from transformers import pipeline, AutoTokenizer, AutoModelForCausalLM
+                    try:
+                        from transformers import pipeline
                 
-                            MODEL_NAME = "sshleifer/tiny-gpt2"       # very small 6 MB model
-                            MODEL_DIR = Path(LOG_DIR) / "cached_tiny_llm"
-                            os.makedirs(MODEL_DIR, exist_ok=True)
+                        st.info("Loading tiny model in-memory (ephemeral)...")
+                        assistant = pipeline("text-generation", model="sshleifer/tiny-gpt2")
                 
-                            # If model is already cached locally, load from there
-                            if (MODEL_DIR / "config.json").exists():
-                                st.caption("Loading tiny model from local cache...")
-                                model = AutoModelForCausalLM.from_pretrained(MODEL_DIR)
-                                tokenizer = AutoTokenizer.from_pretrained(MODEL_DIR)
-                            else:
-                                st.caption("☁️ Downloading tiny model (once only)...")
-                                model = AutoModelForCausalLM.from_pretrained(MODEL_NAME)
-                                tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
-                                model.save_pretrained(MODEL_DIR)
-                                tokenizer.save_pretrained(MODEL_DIR)
-                                st.success("Cached tiny LLM in ./logs/cached_tiny_llm")
+                        prompt = f"""
+                        You are an ML model tuning assistant.
+                        Given this AutoML summary, provide 3 actionable steps for improvement
+                        if overfitting, underfitting, or data quality issues are suspected.
                 
-                            assistant = pipeline("text-generation", model=model, tokenizer=tokenizer)
+                        Use case: {summary.get('use_case')}
+                        Target: {summary.get('target')}
+                        Final R²: {summary.get('final_r2')}
+                        Final RMSE: {summary.get('final_rmse')}
+                        Leaderboard: {summary.get('leaderboard')}
                 
-                            prompt = f"""
-                You are an ML model tuning assistant.
-                Given this AutoML summary, provide 3 actionable steps for improvement if overfitting,
-                underfitting, or data quality issues are suspected.
+                        Respond in concise numbered steps.
+                        """
                 
-                Use case: {summary.get('use_case')}
-                Target: {summary.get('target')}
-                Final R²: {summary.get('final_r2')}
-                Final RMSE: {summary.get('final_rmse')}
-                Leaderboard: {summary.get('leaderboard')}
+                        result = assistant(prompt, max_new_tokens=90, temperature=0.7, do_sample=True)[0]["generated_text"]
+                        st.success("LLM Recommendation:")
+                        st.markdown(result)
+                        log("Tiny LLM in-memory advisory generated successfully.")
                 
-                Respond in concise numbered steps.
-                """
-                            out = assistant(prompt, max_new_tokens=90, temperature=0.7, do_sample=True)[0]["generated_text"]
-                            st.success("LLM Recommendation:")
-                            st.markdown(out)
-                            log("Tiny LLM recommendation generated successfully.")
                     except Exception as e:
                         st.error(f"LLM generation failed: {e}")
-                        st.info("If the model download failed, rerun once — it will cache afterward.")
-
-
-
-
+                        st.info("Make sure `transformers` is installed in your Space environment.")
     
 # -----  Target & Business Impact tab
 with tabs[5]:
@@ -1042,12 +1140,17 @@ in metallurgical AI modeling. Click any title to open the official paper.
     bib_df = pd.DataFrame(bib_data)
     bib_df["Paper Title"] = bib_df.apply(lambda x: f"[{x['title']}]({x['url']})", axis=1)
 
-    st.dataframe(
-        bib_df[["Paper Title", "authors", "notes"]]
-        .rename(columns={"authors": "Authors / Year", "notes": "Relevance"}),
-        width="stretch",
-        hide_index=True
-    )
+    st.markdown("### Annotated Bibliography — Justification for Target Variables")
+
+    for _, row in bib_df.iterrows():
+        st.markdown(
+            f"**[{row['title']}]({row['url']})**  \n"
+            f"*{row['authors']}*  \n"
+            f" _{row['notes']}_  \n",
+            unsafe_allow_html=True
+        )
+    st.info("Click any paper title above to open it in a new tab.")
+
 
     st.markdown("""
 **Feature ↔ Target Justification**