add shap for bulk preds

src/pages/2_Bulk_Predictions.py

@@ -1,11 +1,17 @@
 import streamlit as st
 import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
 from model_utils import load_model, load_model_ensemble, ensemble_predict
 from preprocess_utils import load_train_features
 from preprocess_utils import preprocess_pipeline as preprocess
-from inference_utils import add_predictions, compute_metrics
+from inference_utils import add_predictions, compute_metrics, st_shap, ensemble_shap
 from sidebar import sidebar
 
+import shap
+import lime
+import lime.lime_tabular
+
 # Initialize sidebar
 sidebar()
 
@@ -31,7 +37,7 @@ if "selected_model" in st.session_state:
         target_col = model_dict.get("target_col", "UNKNOWN")
 
     st.session_state.target_col = target_col
-    st.warning(f"The model selected will only predict the target \"{target_col}\". Please choose a different model if you want to predict a different target.")
+    st.warning(f"The model selected will only predict the target \"{target_col}\". Please choose a different model if you wish to predict a different target.")
 
 st.title("📊 Bulk Patient Predictions")
 
@@ -51,7 +57,7 @@ if uploaded_file:
     
     if st.button("Predict"):
         if "bulk_input_df" not in st.session_state:
-            st.warning("Please preprocess data first.")
+            st.warning("Please preprocess the data first.")
         else:
             df = st.session_state.bulk_input_df
             
@@ -74,10 +80,22 @@ if uploaded_file:
             if ensemble:
                 preds = ensemble_predict(models, X, cat_features)
                 metrics_result_ensemble = compute_metrics(y, preds)
+                shap_values = ensemble_shap(models, X)
             else:
-                # single model prediction
                 preds = model.predict_proba(X)[:, 1]
                 metrics_result_single = compute_metrics(y, preds)
+
+                explainer = shap.TreeExplainer(model)
+                shap_values = explainer(X)
+
+                # Handle multi-class (use class 1)
+                if shap_values.values.ndim == 3:
+                    shap_values = shap.Explanation(
+                        values=shap_values.values[:, :, 1],
+                        base_values=shap_values.base_values[:, 1] if shap_values.base_values.ndim == 2 else shap_values.base_values,
+                        data=X,
+                        feature_names=X.columns
+                    )
             
             st.session_state.targets_df = y
             styled = add_predictions(X.copy(), preds)
@@ -93,6 +111,76 @@ if uploaded_file:
                 for metric, value in metrics_result_ensemble.items():
                     st.write(f"  **{metric}**: {value:.3f}")
 
+            def get_top_features(shap_values_array, feature_names, n=20):
+                import numpy as np
+                import shap
+
+                # If a shap.Explanation was passed, extract .values
+                if isinstance(shap_values_array, shap.Explanation):
+                    shap_values_array = shap_values_array.values
+
+                mean_abs_shap = np.abs(shap_values_array).mean(0)
+                feature_importance = pd.DataFrame({
+                    'feature': feature_names,
+                    'importance': mean_abs_shap
+                })
+                return feature_importance.sort_values('importance', ascending=False)['feature'].tolist()[:n]
+
+
+            with st.expander("Show SHAP Explainability", expanded=True):
+                # Get top 20 features
+                top_features = get_top_features(shap_values, X.columns)
+                
+                # Feature selection widget
+                selected_features = st.multiselect(
+                    "Select features to display in plots",
+                    options=list(X.columns),
+                    default=top_features
+                )
+                
+                if not selected_features:
+                    st.warning("Please select at least one feature to display")
+                else:
+                    # Filter data for selected features
+                    X_selected = X[selected_features]
+                    feature_indices = [list(X.columns).index(f) for f in selected_features]
+
+                    # Slice features directly from the SHAP Explanation
+                    shap_values_selected = shap_values[:, feature_indices]
+                    shap_values_selected.feature_names = selected_features
+                    shap_values_selected.data = X_selected
+
+                    # ---- Beeswarm: overall feature impact ----
+                    st.subheader("SHAP Feature Importance")
+                    plt.figure(figsize=(10, 6))
+                    shap.plots.beeswarm(shap_values_selected, max_display=20, show=False)
+                    st.pyplot(plt.gcf(), bbox_inches='tight')
+                    plt.clf()
+
+                    # ---- Mean absolute SHAP bar chart ----
+                    st.subheader("Mean(|SHAP value|) per Feature")
+                    plt.figure(figsize=(10, 6))
+                    shap.plots.bar(shap_values_selected, max_display=20, show=False)
+                    st.pyplot(plt.gcf(), bbox_inches='tight')
+                    plt.clf()
+
+                    # ---- Dependence plot ----
+                    st.subheader("SHAP Dependence Plot")
+                    feature = st.selectbox("Select main feature", selected_features)
+                    interaction_feature = st.selectbox(
+                        "Select interaction feature (optional)",
+                        ["None"] + selected_features
+                    )
+
+                    plt.figure(figsize=(10, 6))
+                    if interaction_feature == "None":
+                        shap.dependence_plot(feature, shap_values_selected.values, X_selected, show=False)
+                    else:
+                        shap.dependence_plot(feature, shap_values_selected.values, X_selected, interaction_index=interaction_feature, show=False)
+
+                    st.pyplot(plt.gcf(), bbox_inches='tight')
+                    plt.clf()
+
             # Find difference in columns between uploaded data and training features
             missing_features = set(st.session_state.orig_train_cols).union(train_features) - set(df.columns)
             missing_features = set([i if i[-2:] != "_X" else '' for i in missing_features])