Update app.py

app.py

@@ -9,7 +9,15 @@ import matplotlib.pyplot as plt
 import os
 from huggingface_hub import hf_hub_download, HfApi
 import hmac
-
+from sklearn.metrics import (
+    roc_auc_score, accuracy_score,
+    roc_curve, confusion_matrix,
+    precision_score, recall_score, f1_score,
+    balanced_accuracy_score,
+    precision_recall_curve, average_precision_score,
+    brier_score_loss
+)
+from sklearn.calibration import calibration_curve
 
 
 from sklearn.pipeline import Pipeline
@@ -76,14 +84,6 @@ def build_pipeline(num_cols, cat_cols):
 # ============================================================
 # Validation utilities
 # ============================================================
-def validate_schema(df: pd.DataFrame) -> pd.DataFrame:
-    missing = [c for c in FEATURE_COLS + [LABEL_COL] if c not in df.columns]
-    if missing:
-        raise ValueError(
-            f"Missing required columns: {missing}. "
-            f"Excel must contain columns A..Z and AA exactly."
-        )
-    return df[FEATURE_COLS + [LABEL_COL]].copy()
 
 
 def coerce_binary_label(y: pd.Series):
@@ -156,13 +156,42 @@ def train_and_save(df: pd.DataFrame, feature_cols, num_cols, cat_cols):
     proba = pipe.predict_proba(X_test)[:, 1]
     pred = (proba >= 0.5).astype(int)
 
+    # Core probability-based metrics
+    roc_auc = float(roc_auc_score(y_test, proba))
+    
+    # ROC curve coordinates
+    fpr, tpr, roc_thresholds = roc_curve(y_test, proba)
+    
+    # Threshold-based metrics (default at 0.5)
+    cls = compute_classification_metrics(y_test, proba, threshold=0.5)
+    
     metrics = {
-        "roc_auc": float(roc_auc_score(y_test, proba)),
-        "accuracy@0.5": float(accuracy_score(y_test, pred)),
+        "roc_auc": roc_auc,
         "n_train": int(len(X_train)),
         "n_test": int(len(X_test)),
+    
+        # Store threshold metrics at 0.5
+        "threshold@0.5": cls["threshold"],
+        "accuracy@0.5": cls["accuracy"],
+        "balanced_accuracy@0.5": cls["balanced_accuracy"],
+        "precision@0.5": cls["precision"],
+        "recall@0.5": cls["recall"],
+        "f1@0.5": cls["f1"],
+        "sensitivity@0.5": cls["sensitivity"],
+        "specificity@0.5": cls["specificity"],
+        "confusion_matrix@0.5": {
+            "tn": cls["tn"], "fp": cls["fp"], "fn": cls["fn"], "tp": cls["tp"]
+        },
+    
+        # Store ROC curve arrays for later plotting if needed
+        "roc_curve": {
+            "fpr": [float(x) for x in fpr],
+            "tpr": [float(x) for x in tpr],
+            "thresholds": [float(x) for x in roc_thresholds],
+        },
     }
 
+
     joblib.dump(pipe, "model.joblib")
 
     meta = {
@@ -184,6 +213,30 @@ def train_and_save(df: pd.DataFrame, feature_cols, num_cols, cat_cols):
 
     return pipe, meta, X
 
+def compute_classification_metrics(y_true, y_proba, threshold: float = 0.5):
+    y_pred = (y_proba >= threshold).astype(int)
+
+    tn, fp, fn, tp = confusion_matrix(y_true, y_pred, labels=[0, 1]).ravel()
+
+    sensitivity = tp / (tp + fn) if (tp + fn) else 0.0  # recall, TPR
+    specificity = tn / (tn + fp) if (tn + fp) else 0.0  # TNR
+    precision = precision_score(y_true, y_pred, zero_division=0)
+    recall = recall_score(y_true, y_pred, zero_division=0)
+    f1 = f1_score(y_true, y_pred, zero_division=0)
+    acc = accuracy_score(y_true, y_pred)
+    bacc = balanced_accuracy_score(y_true, y_pred)
+
+    return {
+        "threshold": float(threshold),
+        "tn": int(tn), "fp": int(fp), "fn": int(fn), "tp": int(tp),
+        "sensitivity": float(sensitivity),
+        "specificity": float(specificity),
+        "precision": float(precision),
+        "recall": float(recall),
+        "f1": float(f1),
+        "accuracy": float(acc),
+        "balanced_accuracy": float(bacc),
+    }
 
 
 
@@ -402,12 +455,69 @@ with tab_train:
 
                 st.success("Training complete. model.joblib and meta.json created.")
 
+                
+                st.divider()
+                st.subheader("Training performance (test split)")
+                
                 m = meta["metrics"]
+                
+                # Show key metrics at threshold 0.5
                 c1, c2, c3, c4 = st.columns(4)
                 c1.metric("ROC AUC", f"{m['roc_auc']:.3f}")
-                c2.metric("Accuracy", f"{m['accuracy@0.5']:.3f}")
-                c3.metric("Train N", m["n_train"])
-                c4.metric("Test N", m["n_test"])
+                c2.metric("Sensitivity (Recall)", f"{m['sensitivity@0.5']:.3f}")
+                c3.metric("Specificity", f"{m['specificity@0.5']:.3f}")
+                c4.metric("F1", f"{m['f1@0.5']:.3f}")
+                
+                c5, c6, c7, c8 = st.columns(4)
+                c5.metric("Precision", f"{m['precision@0.5']:.3f}")
+                c6.metric("Accuracy", f"{m['accuracy@0.5']:.3f}")
+                c7.metric("Balanced Acc", f"{m['balanced_accuracy@0.5']:.3f}")
+                c8.metric("Test N", m["n_test"])
+                
+                # Confusion matrix display
+                cm = m["confusion_matrix@0.5"]
+                cm_df = pd.DataFrame(
+                    [[cm["tn"], cm["fp"]], [cm["fn"], cm["tp"]]],
+                    index=["Actual 0", "Actual 1"],
+                    columns=["Pred 0", "Pred 1"]
+                )
+                st.markdown("**Confusion Matrix (threshold = 0.5)**")
+                st.dataframe(cm_df)
+                
+                # ROC curve plot (matplotlib)
+                roc = m["roc_curve"]
+                fig = plt.figure()
+                plt.plot(roc["fpr"], roc["tpr"])
+                plt.plot([0, 1], [0, 1])
+                plt.xlabel("False Positive Rate (1 - Specificity)")
+                plt.ylabel("True Positive Rate (Sensitivity)")
+                plt.title(f"ROC Curve (AUC = {m['roc_auc']:.3f})")
+                st.pyplot(fig, clear_figure=True)
+
+                st.divider()
+                st.subheader("Threshold analysis")
+                
+                thr = st.slider("Decision threshold", 0.0, 1.0, 0.5, 0.01)
+                
+                # Recompute threshold-based metrics quickly using stored probabilities
+                # You need y_test and proba in scope. Easiest is to store them in session_state during training.
+                st.session_state.y_test_last = y_test
+                st.session_state.proba_last = proba
+                if "y_test_last" in st.session_state and "proba_last" in st.session_state:
+                    cls = compute_classification_metrics(st.session_state.y_test_last, st.session_state.proba_last, threshold=thr)
+                    st.write({
+                        "Sensitivity": cls["sensitivity"],
+                        "Specificity": cls["specificity"],
+                        "Precision": cls["precision"],
+                        "Recall": cls["recall"],
+                        "F1": cls["f1"],
+                        "Accuracy": cls["accuracy"],
+                        "Balanced Accuracy": cls["balanced_accuracy"],
+                    })
+
+
+
+
 
     # ---------------- PUBLISH (only after training) ----------------