Update app.py

app.py

@@ -1,112 +1,108 @@
-import json
 import os
+import json
 from typing import Any, Dict, List
 
 import numpy as np
 import tensorflow as tf
 from fastapi import FastAPI, Request
 from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+import joblib
 
 # ----------------- CONFIG -----------------
-MODEL_PATH = os.getenv("MODEL_PATH", "best_model.keras")
-STATS_PATH = os.getenv("STATS_PATH", "means_std.json")
+# Use your actual filenames here (from your training export)
+MODEL_PATH   = os.getenv("MODEL_PATH", "best_model.h5")      # or "best_model.h5" if that's what you have
+IMPUTER_PATH = os.getenv("IMPUTER_PATH", "imputer.joblib")
+SCALER_PATH  = os.getenv("SCALER_PATH",  "scaler.joblib")
+
+# Class order per training: 0=Top .. 4=Low
 CLASSES = ["Top", "Mid-Top", "Mid", "Mid-Low", "Low"]
-# IMPORTANT: Freeze the exact training order of features:
+
+# EXACT feature order used during training (from your Section 3.1)
 FEATURES: List[str] = [
     "autosuf_oper",
-    "cov_improductiva",
-    "ing_cartera_over_ing_total",
-    "gastos_oper_over_cart",
-    "prov_over_cartera",
-    "_margen_bruto",
-    "equity_over_assets",
-    "rend_cart_over_avg_cart",
-    "_assets",
-    "roa_pre_tax",
-    "cartera_vencida_ratio",
-    "gastos_oper_over_ing_oper",
-    "_cartera_bruta",
-    "grado_absorcion",
-    "_equity",
-    "gastos_fin_over_avg_cart",
     "improductiva",
+    "gastos_fin_over_avg_cart",
+    "_equity",
+    "grado_absorcion",
+    "_cartera_bruta",
+    "gastos_oper_over_ing_oper",
+    "cartera_vencida_ratio",
     "roe_pre_tax",
-    "debt_to_equity",
+    "_assets",
     "_liab",
+    "equity_over_assets",
+    "_margen_bruto",
+    "prov_over_cartera",
+    "gastos_oper_over_cart",
+    "ing_cartera_over_ing_total",
+    "debt_to_equity",
     "prov_gasto_over_cart",
+    "cov_improductiva",
+    "rend_cart_over_avg_cart",
+    "roa_pre_tax",
 ]
 # ------------------------------------------
 
-print("Loading model and stats...")
+print("Loading model / imputer / scaler...")
+# Loss is not needed for inference; compile=False avoids needing custom loss objects.
 model = tf.keras.models.load_model(MODEL_PATH, compile=False)
+imputer = joblib.load(IMPUTER_PATH)   # median imputation from training
+scaler  = joblib.load(SCALER_PATH)    # StandardScaler from training
 
-with open(STATS_PATH, "r") as f:
-    stats: Dict[str, Dict[str, float]] = json.load(f)
-
-# ---- Per-feature transforms used at training (make all 'higher = better') ----
-# If during dataset prep you flipped signs on some “bad” metrics, reflect it here.
-# This set is the typical choice for microfinance health where larger values are worse:
-NEGATE = {
-    "gastos_oper_over_cart",
-    "prov_over_cartera",
-    "cartera_vencida_ratio",
-    "gastos_oper_over_ing_oper",
-    "gastos_fin_over_avg_cart",
-    "improductiva",
-    "debt_to_equity",
-    "prov_gasto_over_cart",
-    # If your training actually negated coverage too (to align “higher=better”),
-    # include the next line. If not, comment it out.
-    # "cov_improductiva",
-}
+print("Model loaded. Feature order:", FEATURES)
 
 def coerce_float(val: Any) -> float:
-    """Coerce numbers from strings with either comma or dot decimal and thousands."""
-    if isinstance(val, (int, float)):
+    """
+    Robust numeric parse:
+      "49.709,14"  -> 49709.14
+      "49,709.14"  -> 49709.14
+      "0,005"      -> 0.005
+      1.23         -> 1.23
+    Raises ValueError on failure.
+    """
+    if isinstance(val, (int, float, np.integer, np.floating)):
         return float(val)
+
     s = str(val).strip()
     if s == "":
         raise ValueError("empty")
+
     s = s.replace(" ", "")
     has_dot = "." in s
     has_comma = "," in s
+
     if has_dot and has_comma:
-        # pick last as decimal
-        if s.rfind(",") > s.rfind("."):
-            s = s.replace(".", "").replace(",", ".")
+        last_dot = s.rfind(".")
+        last_comma = s.rfind(",")
+        if last_comma > last_dot:
+            # decimal is comma, thousands is dot
+            s = s.replace(".", "")
+            s = s.replace(",", ".")
         else:
+            # decimal is dot, thousands is comma
             s = s.replace(",", "")
     elif has_comma and not has_dot:
         s = s.replace(",", ".")
-    return float(s)
-
-def transform_feature(name: str, raw_val: Any) -> float:
-    v = coerce_float(raw_val)
-    if name in NEGATE:
-        return -v
-    return v
+    # else: dots only or pure digits
 
-def zscore(x: float, mean: float, std: float) -> float:
-    if not std:
-        return 0.0
-    return (x - mean) / std
+    return float(s)
 
 def coral_probs_from_logits(logits_np: np.ndarray) -> np.ndarray:
-    """(N, K-1) logits -> (N, K) probabilities (CORAL). Enforce monotonicity."""
-    logits = tf.convert_to_tensor(logits_np, dtype=tf.float32)  # (N, K-1)
-    sig = tf.math.sigmoid(logits)
-    # enforce monotone increasing cumulative (numerical guard)
-    sig_sorted = tf.sort(sig, axis=1)
-    left  = tf.concat([tf.ones_like(sig_sorted[:, :1]), sig_sorted], axis=1)
-    right = tf.concat([sig_sorted, tf.zeros_like(sig_sorted[:, :1])], axis=1)
+    """
+    Decode CORAL logits (N, K-1) -> probabilities (N, K)
+    """
+    logits = tf.convert_to_tensor(logits_np, dtype=tf.float32)
+    sig = tf.math.sigmoid(logits)  # (N, K-1)
+    left  = tf.concat([tf.ones_like(sig[:, :1]), sig], axis=1)
+    right = tf.concat([sig, tf.zeros_like(sig[:, :1])], axis=1)
     probs = tf.clip_by_value(left - right, 1e-12, 1.0)
-    # re-normalize (safety)
-    probs = probs / tf.reduce_sum(probs, axis=1, keepdims=True)
     return probs.numpy()
 
 # ------------- FastAPI app ----------------
-app = FastAPI(title="Static Fingerprint API", version="1.1.0")
+app = FastAPI(title="Static Fingerprint API", version="1.0.0")
 
+# Allow Excel / local tools to call the API
 app.add_middleware(
     CORSMiddleware,
     allow_origins=["*"],
@@ -121,14 +117,13 @@ def root():
 
 @app.get("/health")
 def health():
-    # show the frozen order and which transforms are active
     return {
         "status": "ok",
         "features": FEATURES,
-        "negated_features": sorted(list(NEGATE)),
         "classes": CLASSES,
         "model_file": MODEL_PATH,
-        "stats_file": STATS_PATH,
+        "imputer_file": IMPUTER_PATH,
+        "scaler_file": SCALER_PATH,
     }
 
 @app.post("/echo")
@@ -138,53 +133,68 @@ async def echo(req: Request):
 
 @app.post("/predict")
 async def predict(req: Request):
-    payload = await req.json()
-    if not isinstance(payload, dict):
-        return {"error": "Expected a JSON object mapping feature -> value."}
+    """
+    Body: JSON object mapping feature -> value (raw numbers; median+z will be applied here)
+    Missing features are imputed by the training imputer (median).
+    """
+    try:
+        payload = await req.json()
+    except Exception as e:
+        return JSONResponse(status_code=400, content={"error": f"Invalid JSON: {e}"})
 
-    transformed: Dict[str, float] = {}
-    z_detail: Dict[str, float] = {}
-    missing: List[str] = []
+    if not isinstance(payload, dict):
+        return JSONResponse(status_code=400, content={"error": "Expected a JSON object mapping feature -> value."})
 
-    z_row: List[float] = []
+    # Build raw vector in EXACT training order; use np.nan for missing so imputer handles it
+    x_raw = []
+    missing = []
     for f in FEATURES:
-        mean = float(stats[f]["mean"])
-        std  = float(stats[f]["std"])
         if f in payload:
-            tv = transform_feature(f, payload[f])     # apply the same transform as training
+            try:
+                x_raw.append(coerce_float(payload[f]))
+            except Exception:
+                # treat unparsable as missing -> np.nan (imputer will fill)
+                x_raw.append(np.nan)
+                missing.append(f)
         else:
+            x_raw.append(np.nan)
             missing.append(f)
-            tv = transform_feature(f, 0.0)            # treat missing as 0 before transform
-        transformed[f] = tv
-        zf = zscore(tv, mean, std)
-        z_detail[f] = zf
-        z_row.append(zf)
 
-    X = np.array([z_row], dtype=np.float32)
-    raw = model.predict(X, verbose=0)
+    X_raw = np.array([x_raw], dtype=np.float64)          # (1, 21)
+    X_imp = imputer.transform(X_raw)                      # median imputation
+    X_std = scaler.transform(X_imp).astype(np.float32)    # z-scores as per training
+
+    # Predict
+    raw = model.predict(X_std, verbose=0)
 
-    # Decode: CORAL (K-1) vs softmax (K)
+    # CORAL vs softmax detection (your model is CORAL with 4 logits)
     if raw.ndim == 2 and raw.shape[1] == (len(CLASSES) - 1):
-        decode_mode = "auto_coral_monotone"
         probs = coral_probs_from_logits(raw)[0]
+        decode_mode = "coral"
+    elif raw.ndim == 2 and raw.shape[1] == len(CLASSES):
+        p = raw[0]
+        s = float(np.sum(p))
+        probs = (p / s) if s > 0 else p
+        decode_mode = "softmax"
     else:
-        decode_mode = "softmax_or_logits_norm"
-        probs = raw[0]
-        s = float(np.sum(probs))
-        if s > 0:
-            probs = probs / s
+        # Fallback: try CORAL first
+        probs = coral_probs_from_logits(raw)[0]
+        decode_mode = "auto"
 
     pred_idx = int(np.argmax(probs))
+
+    # Build z-score dict for transparency
+    z_detail = {FEATURES[i]: float(X_std[0, i]) for i in range(len(FEATURES))}
+
     return {
         "input_ok": (len(missing) == 0),
-        "missing": missing,
-        "transformed": transformed,      # post-transform, pre-z (should match training inputs)
-        "z_scores": z_detail,
+        "missing": missing,                    # features that were np.nan (imputed)
+        "z_scores": z_detail,                  # exactly what the model saw
         "probabilities": {CLASSES[i]: float(probs[i]) for i in range(len(CLASSES))},
         "predicted_state": CLASSES[pred_idx],
         "debug": {
             "raw_shape": list(raw.shape),
             "decode_mode": decode_mode,
-            "raw_first_row": [float(x) for x in raw[0].tolist()],
+            "first_row_logits": [float(v) for v in (raw[0].tolist() if raw.ndim == 2 else np.atleast_1d(raw).tolist())],
         },
     }