barely working daemon

app/api/background_drift.py

@@ -13,16 +13,10 @@ REFERENCE_PATH = "models/v1/reference_data.csv"
 PROD_LOG_PATH = "data/production/predictions_log.csv"
 DASHBOARD_JSON = "reports/evidently/drift_report.json"
 
-# Retention policy (VERY IMPORTANT for HF Spaces)
 MAX_ROWS = 5000  # rolling window
-
 os.makedirs(os.path.dirname(DASHBOARD_JSON), exist_ok=True)
 
-
 async def drift_loop(interval_seconds: int = 10):
-    """
-    Continuously compute drift from production inference data.
-    """
     while True:
         try:
             if not os.path.exists(PROD_LOG_PATH):
@@ -30,13 +24,13 @@ async def drift_loop(interval_seconds: int = 10):
                 continue
 
             prod_df = pd.read_csv(PROD_LOG_PATH)
-            
-            # ---- Retention window (prevents infinite growth) ----
+
+            # Retention window
             if len(prod_df) > MAX_ROWS:
                 prod_df = prod_df.tail(MAX_ROWS)
                 prod_df.to_csv(PROD_LOG_PATH, index=False)
 
-            # ---- Keep only rows with all required features ----
+            # Keep only rows with all required features
             missing_features = set(predictor.features) - set(prod_df.columns)
             if missing_features:
                 print(f"Skipping drift check, missing features: {missing_features}")
@@ -50,9 +44,11 @@ async def drift_loop(interval_seconds: int = 10):
 
             reference_df = pd.read_csv(REFERENCE_PATH)
 
-            # ---- FIX: pass reference_df to run_drift_check ----
+            # ---- Run drift on features only ----
             _, drift_dict = run_drift_check(
-                prod_df[predictor.features], reference_df[predictor.features], model_version="v1"
+                prod_df[predictor.features],
+                reference_df[predictor.features],
+                model_version="v1"
             )
 
             dashboard_payload = {
@@ -64,7 +60,6 @@ async def drift_loop(interval_seconds: int = 10):
                 ],
             }
 
-            # Atomic write (prevents frontend race conditions)
             tmp_path = DASHBOARD_JSON + ".tmp"
             with open(tmp_path, "w") as f:
                 json.dump(dashboard_payload, f, indent=2)

app/api/routes.py

@@ -5,7 +5,6 @@ from fastapi.templating import Jinja2Templates
 
 from app.inference.predictor import Predictor
 from app.monitoring.data_loader import load_production_data
-from app.monitoring.drift import run_drift_check
 from app.monitoring.governance import run_governance_checks
 
 import pandas as pd
@@ -21,17 +20,17 @@ predictor = Predictor()
 PROD_LOG = "data/production/predictions_log.csv"
 
 # ------------------------------------------------------------------
-# ENSURE production log exists at server startup (CRITICAL FIX)
+# ENSURE production log exists at server startup
 # ------------------------------------------------------------------
 os.makedirs(os.path.dirname(PROD_LOG), exist_ok=True)
 
 if not os.path.exists(PROD_LOG):
-    # Create empty production log with correct schema
     base_cols = list(predictor.features)
     extra_cols = [
-        "prediction",
-        "probability",
-        "risk_level",
+        "target",            # true label
+        "model_prediction",  # model output
+        "model_probability",
+        "model_risk_level",
         "model_version",
         "timestamp",
     ]
@@ -63,35 +62,23 @@ async def predict_file(background_tasks: BackgroundTasks, file: UploadFile = Fil
             "risk_level": "High" if proba >= 0.75 else "Medium" if proba >= 0.5 else "Low"
         })
 
-    # ---- Drift: immediate for frontend ----
-    reference_df = pd.read_csv("models/v1/reference_data.csv")
-    _, drift_dict = run_drift_check(
-        df[predictor.features],
-        reference_df[predictor.features],
-        "v1",
-    )
-
-    drift_for_chart = []
-    for col, score in drift_dict.items():
-        try:
-            score_value = float(score)
-            if not np.isfinite(score_value):
-                score_value = 0.0
-        except Exception:
-            score_value = 0.0
-        drift_for_chart.append({"column": col, "score": score_value})
-
-    # ---- Append predictions to production log ----
+    # ---- Append predictions to production log (minimal, fast) ----
     df_log = df.copy()
 
-    # ---- FIX: Remove existing prediction/risk/probability/etc columns to avoid extra column issue ----
-    for col in ["prediction", "probability", "risk_level", "model_version", "timestamp"]:
+    # Keep true target if present
+    if "target" in df.columns:
+        df_log["target"] = df["target"]
+    else:
+        df_log["target"] = np.nan
+
+    # Remove any old model prediction columns to prevent duplicates
+    for col in ["model_prediction", "model_probability", "model_risk_level", "model_version", "timestamp"]:
         if col in df_log.columns:
             df_log = df_log.drop(columns=[col])
 
-    df_log["prediction"] = preds
-    df_log["probability"] = probas
-    df_log["risk_level"] = [
+    df_log["model_prediction"] = preds
+    df_log["model_probability"] = probas
+    df_log["model_risk_level"] = [
         "High" if p >= 0.75 else "Medium" if p >= 0.5 else "Low"
         for p in probas
     ]
@@ -100,25 +87,9 @@ async def predict_file(background_tasks: BackgroundTasks, file: UploadFile = Fil
 
     df_log.to_csv(PROD_LOG, mode="a", header=False, index=False)
 
-    # ---- Dashboard JSON ----
-    DASHBOARD_JSON = "reports/evidently/drift_report.json"
-
-    dashboard_payload = {
-        "n_rows": len(results),
-        "results": results,
-        "drift": drift_for_chart,
-    }
-
-    os.makedirs(os.path.dirname(DASHBOARD_JSON), exist_ok=True)
-    tmp_path = DASHBOARD_JSON + ".tmp"
-    with open(tmp_path, "w") as f:
-        json.dump(dashboard_payload, f, indent=2)
-    os.replace(tmp_path, DASHBOARD_JSON)
-
     return JSONResponse({
         "n_rows": len(results),
         "results": results,
-        "drift": drift_for_chart,
     })
 
 
@@ -130,8 +101,10 @@ def health():
 @router.get("/run-drift")
 def run_drift():
     current_df = load_production_data()
-    report_path = run_drift_check(current_df)
-    return {"status": "drift_check_completed", "report_path": report_path}
+    from app.monitoring.drift import run_drift_check
+    reference_df = pd.read_csv("models/v1/reference_data.csv")
+    _, drift_dict = run_drift_check(current_df[predictor.features], reference_df[predictor.features])
+    return {"status": "drift_check_completed", "drift": drift_dict}
 
 
 @router.get("/dashboard")

app/api/traffic_daemon.py

@@ -4,7 +4,6 @@ import pandas as pd
 import random
 import requests
 import os
-import time
 
 API_URL = "http://localhost:8000/predict"
 SOURCE_DATA = "data/processed/current_data.csv"
@@ -13,13 +12,9 @@ MIN_SLEEP = 2
 MAX_SLEEP = 8
 MIN_BATCH = 1
 MAX_BATCH = 5
-STARTUP_DELAY = 10  # seconds – allow FastAPI to fully start
-
+STARTUP_DELAY = 10  # allow server startup
 
 async def traffic_loop():
-    """
-    Continuously generate inference traffic against /predict.
-    """
     await asyncio.sleep(STARTUP_DELAY)
 
     if not os.path.exists(SOURCE_DATA):
@@ -27,20 +22,19 @@ async def traffic_loop():
         return
 
     df = pd.read_csv(SOURCE_DATA)
-
     print("Traffic daemon started.")
 
     while True:
         try:
             batch_size = random.randint(MIN_BATCH, MAX_BATCH)
             sample = df.sample(batch_size)
-
             csv_bytes = sample.to_csv(index=False).encode("utf-8")
 
+            # ---- Increased timeout to avoid ReadTimeout ----
             response = requests.post(
                 API_URL,
                 files={"file": ("sample.csv", csv_bytes, "text/csv")},
-                timeout=10,
+                timeout=60,
             )
 
             if response.status_code != 200:

app/core/model_registry.py

@@ -1 +0,0 @@
-# model loading/versioning

app/db/models.py

@@ -1 +0,0 @@
-# ORM-style tables (optional)

app/db/session.py

@@ -1 +0,0 @@
-# SQLite connection

app/inference/preprocessing.py

@@ -1 +0,0 @@
-# feature handling

app/monitoring/metrics.py

@@ -1 +0,0 @@
-# feature stats extraction

data/production/predictions_log.csv

@@ -1,6 +1,9 @@
-credit_limit,age,pay_delay_sep,pay_delay_aug,bill_amt_sep,bill_amt_aug,pay_amt_sep,pay_amt_aug,prediction,probability,risk_level,model_version,timestamp
-80000.0,26,0,0,40216.0,10400.0,1400.0,10935.0,0,0,0.18062978660671827,Low,v1,2026-01-14 18:43:36.319064+00:00
-200000.0,36,-1,-1,396.0,396.0,396.0,396.0,0,0,0.1170869637407986,Low,v1,2026-01-14 18:43:36.319064+00:00
-210000.0,39,-1,-1,22861.0,6437.0,6536.0,72037.0,0,0,0.057603159126245064,Low,v1,2026-01-14 18:45:57.603850+00:00
-100000.0,28,1,-2,0.0,0.0,0.0,0.0,1,0,0.28137664031455745,Low,v1,2026-01-14 18:45:57.603850+00:00
-290000.0,36,-2,-2,3602.0,6107.0,6107.0,4538.0,0,0,0.04814376167314694,Low,v1,2026-01-14 18:45:57.603850+00:00
+credit_limit,age,pay_delay_sep,pay_delay_aug,bill_amt_sep,bill_amt_aug,pay_amt_sep,pay_amt_aug,target,model_prediction,model_probability,model_risk_level,model_version,timestamp
+70000.0,48,0,0,20744.0,22093.0,2000.0,2000.0,0,0,0.2563046708563498,Low,v1,2026-01-14 19:33:08.201187+00:00
+390000.0,42,0,0,310075.0,184647.0,10021.0,4000.0,0,0,0.07238720996682343,Low,v1,2026-01-14 19:33:08.201187+00:00
+230000.0,50,-2,-2,2789.0,2942.0,2942.0,2520.0,0,0,0.06061841289885345,Low,v1,2026-01-14 19:33:08.201187+00:00
+40000.0,47,2,2,52358.0,54892.0,4000.0,0.0,1,1,0.608097803732095,Medium,v1,2026-01-14 19:35:59.632246+00:00
+140000.0,41,0,0,130138.0,132726.0,4756.0,4912.0,1,0,0.19300589506044843,Low,v1,2026-01-14 19:35:59.632246+00:00
+50000.0,26,-1,-1,5052.0,0.0,0.0,0.0,0,0,0.12125611010883464,Low,v1,2026-01-14 19:37:40.778834+00:00
+400000.0,42,-1,-1,44198.0,10132.0,10132.0,16932.0,0,0,0.06632404342712281,Low,v1,2026-01-14 19:39:17.748870+00:00
+280000.0,45,-2,-2,26573.0,17597.0,18388.0,22302.0,0,0,0.036231471756518835,Low,v1,2026-01-14 19:39:17.748870+00:00

reports/evidently/drift_report.json

@@ -1,25 +1,6 @@
 {
-  "n_rows": 3,
-  "results": [
-    {
-      "row": 0,
-      "probability": 0.0576,
-      "prediction": "No Default",
-      "risk_level": "Low"
-    },
-    {
-      "row": 1,
-      "probability": 0.2814,
-      "prediction": "No Default",
-      "risk_level": "Low"
-    },
-    {
-      "row": 2,
-      "probability": 0.0481,
-      "prediction": "No Default",
-      "risk_level": "Low"
-    }
-  ],
+  "n_rows": 8,
+  "results": [],
   "drift": [
     {
       "column": "dataset",
@@ -27,35 +8,35 @@
     },
     {
       "column": "age",
-      "score": 0.44474387293503476
+      "score": 0.8679104867707123
     },
     {
       "column": "bill_amt_aug",
-      "score": 0.6359128906609168
+      "score": 0.23085843753803348
     },
     {
       "column": "bill_amt_sep",
-      "score": 0.5786372146880444
+      "score": 0.37457217443848057
     },
     {
       "column": "credit_limit",
-      "score": 0.5490100613355632
+      "score": 0.36873847560130574
     },
     {
       "column": "pay_amt_aug",
-      "score": 1.0090700728243183
+      "score": 0.20876449446182804
     },
     {
       "column": "pay_amt_sep",
-      "score": 0.28309439717741114
+      "score": 0.27529913358402724
     },
     {
       "column": "pay_delay_aug",
-      "score": 1.298651499017585
+      "score": 0.32956762578626103
     },
     {
       "column": "pay_delay_sep",
-      "score": 0.7812979392472955
+      "score": 0.4885655407858251
     }
   ]
 }