auto drift

app/api/background_drift.py

@@ -3,55 +3,69 @@ import asyncio
 import pandas as pd
 import os
 import json
-import numpy as np
 
 from app.monitoring.drift import run_drift_check
 from app.inference.predictor import Predictor
 
 predictor = Predictor()
+
 REFERENCE_PATH = "models/v1/reference_data.csv"
-CURRENT_DATA_PATH = "data/production/predictions_log.csv"
+PROD_LOG_PATH = "data/production/predictions_log.csv"
 DASHBOARD_JSON = "reports/evidently/drift_report.json"
 
-# Ensure folder exists and JSON file exists at startup
+# Retention policy (VERY IMPORTANT for HF Spaces)
+MAX_ROWS = 5000  # rolling window
+
 os.makedirs(os.path.dirname(DASHBOARD_JSON), exist_ok=True)
-if not os.path.exists(DASHBOARD_JSON):
-    with open(DASHBOARD_JSON, "w") as f:
-        json.dump({"n_rows": 0, "results": [], "drift": [{"column": feat, "score": 0.0} for feat in predictor.features]}, f, indent=2)
 
-async def drift_loop(interval_seconds: int = 30):
+
+async def drift_loop(interval_seconds: int = 10):
     """
-    Continuously run drift checks and update dashboard JSON.
+    Continuously compute drift from production inference data.
     """
     while True:
         try:
-            current_df = pd.read_csv(CURRENT_DATA_PATH)
+            if not os.path.exists(PROD_LOG_PATH):
+                await asyncio.sleep(interval_seconds)
+                continue
+
+            prod_df = pd.read_csv(PROD_LOG_PATH)
+
+            # ---- Retention window (prevents infinite growth) ----
+            if len(prod_df) > MAX_ROWS:
+                prod_df = prod_df.tail(MAX_ROWS)
+                prod_df.to_csv(PROD_LOG_PATH, index=False)
+
+            # ---- Keep only rows with all required features ----
+            missing_features = set(predictor.features) - set(prod_df.columns)
+            if missing_features:
+                print(f"Skipping drift check, missing features: {missing_features}")
+                await asyncio.sleep(interval_seconds)
+                continue
+
+            prod_df = prod_df.dropna(subset=predictor.features)
+            if prod_df.empty:
+                await asyncio.sleep(interval_seconds)
+                continue
+
             reference_df = pd.read_csv(REFERENCE_PATH)
 
             _, drift_dict = run_drift_check(
-                current_df[predictor.features],
+                prod_df[predictor.features],
                 reference_df[predictor.features],
-                "v1"
+                model_version="v1",
             )
 
-            # Ensure numeric safe drift values
-            drift_for_chart = []
-            for col, score in drift_dict.items():
-                try:
-                    val = float(score)
-                    if not np.isfinite(val):
-                        val = 0.0
-                except Exception:
-                    val = 0.0
-                drift_for_chart.append({"column": col, "score": val})
-
             dashboard_payload = {
-                "n_rows": len(current_df),
-                "results": [],  # predictions not included in background loop
-                "drift": drift_for_chart
+                "n_rows": len(prod_df),
+                "results": [],
+                "drift": [
+                    {"column": col, "score": float(score)}
+                    for col, score in drift_dict.items()
+                ],
             }
 
-            # Atomic write to avoid read/write collision
+            # Atomic write (prevents frontend race conditions)
             tmp_path = DASHBOARD_JSON + ".tmp"
             with open(tmp_path, "w") as f:
                 json.dump(dashboard_payload, f, indent=2)

app/api/routes.py

@@ -60,6 +60,7 @@ async def predict_file(background_tasks: BackgroundTasks, file: UploadFile = Fil
     df_log = df.copy()
     df_log["prediction"] = preds
     df_log["probability"] = probas
+    df_log["risk_level"] = ["High" if p >= 0.75 else "Medium" if p >= 0.5 else "Low" for p in probas]
     df_log["model_version"] = predictor.model_version
     df_log["timestamp"] = pd.Timestamp.utcnow()
 

app/templates/dashboard.html

@@ -14,65 +14,34 @@
         <button type="submit">Run Prediction</button>
     </form>
 
-    <h2>Predictions</h2>
+    <h2>Recent Predictions</h2>
     <div id="predictions"></div>
 
     <h2>Drift Metrics</h2>
     <div id="drift-chart"></div>
 
-<script>
-async function fetchResults(csvFile) {
-    const formData = new FormData();
-    formData.append("file", csvFile);
-
-    const response = await fetch("/predict", { method: "POST", body: formData });
-    const data = await response.json();
-
-    document.getElementById("predictions").innerHTML =
-        `<pre>${JSON.stringify(data.results, null, 2)}</pre>`;
-
-    const driftContainer = document.getElementById("drift-chart");
-    driftContainer.innerHTML = "";
-
-    if (Array.isArray(data.drift)) {
-        const cols = data.drift.map(d => d.column);
-        const scores = data.drift.map(d => {
-            let val = Number(d.score);
-            if (!Number.isFinite(val)) val = 0;
-            return val;
-        });
-
-        Plotly.newPlot(driftContainer, [{
-            x: cols,
-            y: scores,
-            type: "bar"
-        }]);
-    } else {
-        driftContainer.innerHTML =
-            "<p>Drift report scheduled. Open the Evidently HTML report.</p>";
-    }
-}
-
-document.getElementById("upload-form").addEventListener("submit", async (e) => {
-    e.preventDefault();
-    const fileInput = e.target.file.files[0];
-    if (fileInput) {
-        await fetchResults(fileInput);
-    }
-});
-</script>
-<div id="drift-chart"></div>
-
-<script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
 <script>
 async function fetchDashboardData() {
     try {
         const resp = await fetch("/dashboard/data");
         const json = await resp.json();
+
         if(json.status === "ok") {
-            const driftData = json.data.drift;
+            const data = json.data;
+
+            // Update predictions table
+            const predDiv = document.getElementById("predictions");
+            if(Array.isArray(data.results) && data.results.length > 0){
+                predDiv.innerHTML = "<pre>" + JSON.stringify(data.results, null, 2) + "</pre>";
+            } else {
+                predDiv.innerHTML = "<p>No recent predictions.</p>";
+            }
+
+            // Update drift chart
+            const driftContainer = document.getElementById("drift-chart");
+            const driftData = data.drift || [];
             const columns = driftData.map(d => d.column);
-            const scores = driftData.map(d => d.score);
+            const scores = driftData.map(d => Number(d.score));
 
             const trace = {
                 x: columns,
@@ -87,10 +56,12 @@ async function fetchDashboardData() {
                 xaxis: {title: 'Column'}
             };
 
-            Plotly.newPlot('drift-chart', [trace], layout);
+            Plotly.newPlot(driftContainer, [trace], layout);
+
         } else {
             console.warn("Dashboard data not available:", json.message);
         }
+
     } catch(err) {
         console.error("Failed to fetch dashboard data:", err);
     }
@@ -99,6 +70,21 @@ async function fetchDashboardData() {
 // Fetch every 10 seconds
 fetchDashboardData();
 setInterval(fetchDashboardData, 10000);
+
+// File upload handler
+document.getElementById("upload-form").addEventListener("submit", async (e) => {
+    e.preventDefault();
+    const fileInput = e.target.file.files[0];
+    if (fileInput) {
+        const formData = new FormData();
+        formData.append("file", fileInput);
+
+        const response = await fetch("/predict", { method: "POST", body: formData });
+        const data = await response.json();
+        document.getElementById("predictions").innerHTML =
+            `<pre>${JSON.stringify(data.results, null, 2)}</pre>`;
+    }
+});
 </script>
 </body>
 </html>