Update app.py

app.py

@@ -1,6 +1,9 @@
 import os, json, io, traceback
 from typing import Any, Dict, List, Optional
 
+import pandas as pd
+from sklearn.impute import SimpleImputer
+from sklearn.preprocessing import StandardScaler
 import numpy as np
 import tensorflow as tf
 from fastapi import FastAPI, Request
@@ -546,9 +549,8 @@ async def predict(req: Request):
             content={"error": str(e), "trace": traceback.format_exc()},
         )
 
-
-    # ============================================================
-# CORAL ORDINAL HELPERS (copied from your training script)
+# ============================================================
+# CORAL ORDINAL HELPERS (from training script)
 # ============================================================
 
 def to_cumulative_targets_tf(y_true_int, K_):
@@ -558,6 +560,7 @@ def to_cumulative_targets_tf(y_true_int, K_):
     T = tf.cast(tf.greater_equal(y[:, None], thresholds[None, :]), tf.float32)
     return T
 
+
 def coral_loss_tf(y_true, logits):
     y_true = tf.reshape(y_true, [-1])
     y_true = tf.cast(y_true, tf.int32)
@@ -565,21 +568,32 @@ def coral_loss_tf(y_true, logits):
     bce = tf.nn.sigmoid_cross_entropy_with_logits(labels=T, logits=logits)
     return tf.reduce_mean(tf.reduce_sum(bce, axis=1))
 
-def coral_probs_from_logits(logits):
-    sig = tf.math.sigmoid(logits)
+
+# ---------- TF helper & numpy wrapper (unified version) ----------
+def _coral_probs_from_logits_tf(logits_tf: tf.Tensor) -> tf.Tensor:
+    """Pure TF CORAL probability transform."""
+    sig = tf.math.sigmoid(logits_tf)
     left  = tf.concat([tf.ones_like(sig[:, :1]), sig], axis=1)
     right = tf.concat([sig, tf.zeros_like(sig[:, :1])], axis=1)
     return tf.clip_by_value(left - right, 1e-12, 1.0)
 
+
+def coral_probs_from_logits(logits_np: np.ndarray) -> np.ndarray:
+    """Numpy wrapper used by decode_logits + SHAP."""
+    logits_tf = tf.convert_to_tensor(logits_np, dtype=tf.float32)
+    return _coral_probs_from_logits_tf(logits_tf).numpy()
+
+
 @tf.function
 def ordinal_accuracy_metric(y_true, y_pred_logits):
     y_true = tf.reshape(y_true, [-1])
     y_true = tf.cast(y_true, tf.int32)
-    probs  = coral_probs_from_logits(y_pred_logits)
+    probs  = _coral_probs_from_logits_tf(y_pred_logits)
     y_pred = tf.argmax(probs, axis=1, output_type=tf.int32)
     return tf.reduce_mean(tf.cast(tf.equal(y_true, y_pred), tf.float32))
 
-    # ============================================================
+
+# ============================================================
 # RECREATE MODEL FROM BEST HYPERPARAMETERS
 # ============================================================
 
@@ -608,7 +622,6 @@ def build_model_from_hparams(hp: dict):
         if drop > 0:
             x = tf.keras.layers.Dropout(drop)(x)
 
-    # CORAL output
     outputs = tf.keras.layers.Dense(len(CLASSES) - 1, activation=None)(x)
 
     model = tf.keras.Model(inputs, outputs)
@@ -620,12 +633,12 @@ def build_model_from_hparams(hp: dict):
     return model
 
 
-    # ============================================================
-# RETRAINING LOGIC
+# ============================================================
+# RETRAINING LOGIC + DATASET MGMT
 # ============================================================
 
-FINGERPRINT_CSV = "fingerprints_db.csv"   # <-- choose file name
-BEST_HP_JSON     = "best_params_and_metrics.json"
+FINGERPRINT_CSV = "fingerprints_db.csv"
+BEST_HP_JSON    = "best_params_and_metrics.json"
 
 
 def load_best_hparams():
@@ -637,11 +650,9 @@ def load_best_hparams():
 def load_fingerprint_dataset():
     df = pd.read_csv(FINGERPRINT_CSV)
 
-    # Must include: company, date, rating, and 21 features
-    y = df["rating"].map({c:i for i,c in enumerate(CLASSES)}).astype("int32").to_numpy()
+    y = df["rating"].map({c: i for i, c in enumerate(CLASSES)}).astype("int32").to_numpy()
     X_raw = df[FEATURES].to_numpy().astype("float32")
 
-    # Fit imputer + scaler from full dataset
     imp = SimpleImputer(strategy="median")
     sc  = StandardScaler()
 
@@ -658,40 +669,57 @@ def retrain_model():
     model_new = build_model_from_hparams(hp)
 
     es = tf.keras.callbacks.EarlyStopping(
-        monitor="loss", patience=15, restore_best_weights=True
+        monitor="loss",
+        patience=15,
+        restore_best_weights=True,
+        verbose=1
     )
 
     model_new.fit(
         X, y,
         epochs=150,
         batch_size=128,
+        callbacks=[es],
         verbose=1,
-        callbacks=[es]
     )
 
-    # Update globals used by /predict
-    global model, IMPUTER, SCALER
+    # Update global model + preprocessors
+    global model, imputer, scaler
     model = model_new
-    IMPUTER = imp
-    SCALER  = sc
+    imputer = imp
+    scaler  = sc
+
+    # Rebuild SHAP explainer to match new model
+    global EXPLAINER
+    if SHAP_AVAILABLE:
+        try:
+            BACKGROUND_Z = np.zeros((50, len(FEATURES)), dtype=np.float32)
+            EXPLAINER = shap.KernelExplainer(model_proba_from_z, BACKGROUND_Z)
+            print("SHAP explainer rebuilt after retrain.")
+        except Exception as e:
+            EXPLAINER = None
+            print("⚠️ Failed to rebuild SHAP explainer:", repr(e))
 
     return True
 
 
-
-    # ============================================================
+# ============================================================
 # API ENDPOINT: APPEND + RETRAIN
 # ============================================================
 
 @app.post("/append_and_retrain")
 def append_and_retrain(payload: dict):
     """
-    payload:
+    payload format:
     {
         "company": "...",
         "date": "2025-Q1",
-        "rating": "Mid-Top",
-        "features": { autosuf_oper: ..., improductiva: ..., ... }
+        "rating": "Mid",
+        "features": {
+            "autosuf_oper": ..., 
+            "improductiva": ...,
+            ...
+        }
     }
     """
 
@@ -700,14 +728,20 @@ def append_and_retrain(payload: dict):
     rating  = payload.get("rating")
     feats   = payload.get("features", {})
 
-    if not company or not rating or len(feats) != len(FEATURES):
-        return {"ok": False, "error": "Invalid payload"}
+    if not company or not date or not rating:
+        return {"ok": False, "error": "Missing company/date/rating"}
 
-    # Append to CSV
-    df_new = pd.DataFrame([{**{"company": company,
-                               "date": date,
-                               "rating": rating},
-                            **feats}])
+    if set(feats.keys()) != set(FEATURES):
+        return {"ok": False, "error": "Features missing or incorrect"}
+
+    # Append row
+    new_row = {
+        "company": company,
+        "date": date,
+        "rating": rating,
+        **feats
+    }
+    df_new = pd.DataFrame([new_row])
 
     if os.path.exists(FINGERPRINT_CSV):
         df = pd.read_csv(FINGERPRINT_CSV)
@@ -717,7 +751,7 @@ def append_and_retrain(payload: dict):
 
     df.to_csv(FINGERPRINT_CSV, index=False)
 
-    # Retrain
+    # Retrain model
     retrain_model()
 
-    return {"ok": True, "message": "Fingerprint added and model retrained"}
+    return {"ok": True, "message": "Fingerprint appended + model retrained"}