Add scripts for baseline preparation and drift detection, including Grafana dashboard configuration

monitoring/drift/scripts/prepare_baseline.py

@@ -0,0 +1,107 @@
+"""
+Prepare baseline/reference data for drift detection.
+This script samples representative data from the training set.
+"""
+
+import pickle
+import pandas as pd
+import numpy as np
+from pathlib import Path
+from sklearn.model_selection import train_test_split
+
+# Paths
+PROJECT_ROOT = Path(__file__).parent.parent.parent.parent
+BASELINE_DIR = Path(__file__).parent.parent / "baseline"
+BASELINE_DIR.mkdir(parents=True, exist_ok=True)
+
+
+def load_training_data():
+    """Load the original training dataset."""
+    # Adjust path to your actual data
+    data_path = PROJECT_ROOT / "data" / "train.csv"
+    
+    if not data_path.exists():
+        raise FileNotFoundError(f"Training data not found at {data_path}")
+    
+    df = pd.read_csv(data_path)
+    print(f"Loaded {len(df)} training samples")
+    return df
+
+
+def prepare_baseline(df, sample_size=1000, random_state=42):
+    """
+    Sample representative baseline data.
+    
+    Args:
+        df: Training dataframe
+        sample_size: Number of samples for baseline
+        random_state: Random seed for reproducibility
+    
+    Returns:
+        Baseline dataframe
+    """
+    # Stratified sampling if you have labels
+    if 'label' in df.columns:
+        _, baseline_df = train_test_split(
+            df,
+            test_size=sample_size,
+            random_state=random_state,
+            stratify=df['label']
+        )
+    else:
+        baseline_df = df.sample(n=min(sample_size, len(df)), random_state=random_state)
+    
+    print(f"Sampled {len(baseline_df)} baseline samples")
+    return baseline_df
+
+
+def extract_features(df):
+    """
+    Extract features used for drift detection.
+    Should match the features used by your model.
+    """
+    
+    feature_columns = [col for col in df.columns if col not in ['label', 'id', 'timestamp']]
+    X = df[feature_columns].values
+    
+    print(f"Extracted {X.shape[1]} features from {X.shape[0]} samples")
+    return X
+
+
+def save_baseline(baseline_data, filename="reference_data.pkl"):
+    """Save baseline data to disk."""
+    baseline_path = BASELINE_DIR / filename
+    
+    with open(baseline_path, 'wb') as f:
+        pickle.dump(baseline_data, f)
+    
+    print(f"Baseline saved to {baseline_path}")
+    print(f"   Shape: {baseline_data.shape}")
+    print(f"   Size: {baseline_path.stat().st_size / 1024:.2f} KB")
+
+
+def main():
+    """Main execution."""
+    print("=" * 60)
+    print("Preparing Baseline Data for Drift Detection")
+    print("=" * 60)
+    
+    # Load data
+    df = load_training_data()
+    
+    # Sample baseline
+    baseline_df = prepare_baseline(df, sample_size=1000)
+    
+    # Extract features
+    X_baseline = extract_features(baseline_df)
+    
+    # Save
+    save_baseline(X_baseline)
+    
+    print("\n" + "=" * 60)
+    print("Baseline preparation complete!")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    main()

monitoring/drift/scripts/requirements.txt

@@ -0,0 +1,6 @@
+alibi-detect>=0.11.4
+pandas>=2.0.0
+numpy>=1.24.0
+scikit-learn>=1.3.0
+requests>=2.31.0
+mlflow>=2.8.0

monitoring/drift/scripts/run_drift_check.py

@@ -0,0 +1,210 @@
+"""
+Data Drift Detection using Alibi Detect.
+Detects distribution shifts between baseline and new data.
+"""
+
+import pickle
+import json
+import requests
+import numpy as np
+import pandas as pd
+from pathlib import Path
+from datetime import datetime
+from alibi_detect.cd import KSDrift
+from typing import Dict, Tuple
+
+# Configuration
+PROJECT_ROOT = Path(__file__).parent.parent.parent.parent
+BASELINE_DIR = Path(__file__).parent.parent / "baseline"
+REPORTS_DIR = Path(__file__).parent.parent / "reports"
+REPORTS_DIR.mkdir(parents=True, exist_ok=True)
+
+PUSHGATEWAY_URL = "http://localhost:9091"
+P_VALUE_THRESHOLD = 0.05  # Significance level
+
+
+def load_baseline() -> np.ndarray:
+    """Load reference/baseline data."""
+    baseline_path = BASELINE_DIR / "reference_data.pkl"
+    
+    if not baseline_path.exists():
+        raise FileNotFoundError(
+            f"Baseline data not found at {baseline_path}\n"
+            f"Run `python prepare_baseline.py` first!"
+        )
+    
+    with open(baseline_path, 'rb') as f:
+        X_baseline = pickle.load(f)
+    
+    print(f"Loaded baseline data: {X_baseline.shape}")
+    return X_baseline
+
+
+def load_new_data() -> np.ndarray:
+    """
+    Load new/production data to check for drift.
+    
+    In production, this would fetch from:
+    - Database
+    - S3 bucket
+    - API logs
+    - Data lake
+    
+    For now, simulate or load from file.
+    """
+    
+    # Option 1: Load from file
+    data_path = PROJECT_ROOT / "data" / "test.csv"
+    if data_path.exists():
+        df = pd.read_csv(data_path)
+        # Extract same features as baseline
+        feature_columns = [col for col in df.columns if col not in ['label', 'id', 'timestamp']]
+        X_new = df[feature_columns].values[:500]  # Take 500 samples
+        print(f"Loaded new data from file: {X_new.shape}")
+        return X_new
+    
+    # Option 2: Simulate (for testing)
+    print("Simulating new data (no test file found)")
+    X_baseline = load_baseline()
+    # Add slight shift to simulate drift
+    X_new = X_baseline[:500] + np.random.normal(0, 0.1, (500, X_baseline.shape[1]))
+    return X_new
+
+
+def run_drift_detection(X_baseline: np.ndarray, X_new: np.ndarray) -> Dict:
+    """
+    Run Kolmogorov-Smirnov drift detection.
+    
+    Args:
+        X_baseline: Reference data
+        X_new: New data to check
+    
+    Returns:
+        Drift detection results
+    """
+    print("\n" + "=" * 60)
+    print("Running Drift Detection (Kolmogorov-Smirnov Test)")
+    print("=" * 60)
+    
+    # Initialize detector
+    cd = KSDrift(
+        X_baseline,
+        p_val=P_VALUE_THRESHOLD,
+        alternative='two-sided',
+        correction='bonferroni'  # Multiple testing correction
+    )
+    
+    # Run detection
+    preds = cd.predict(X_new)
+    
+    # Extract results
+    results = {
+        "timestamp": datetime.now().isoformat(),
+        "drift_detected": int(preds['data']['is_drift']),
+        "p_value": float(preds['data']['p_val']),
+        "threshold": P_VALUE_THRESHOLD,
+        "distance": float(preds['data']['distance']),
+        "baseline_samples": X_baseline.shape[0],
+        "new_samples": X_new.shape[0],
+        "num_features": X_baseline.shape[1]
+    }
+    
+    # Print results
+    print(f"\nResults:")
+    print(f"   Drift Detected: {'YES' if results['drift_detected'] else 'NO'}")
+    print(f"   P-Value: {results['p_value']:.6f} (threshold: {P_VALUE_THRESHOLD})")
+    print(f"   Distance: {results['distance']:.6f}")
+    print(f"   Baseline: {X_baseline.shape[0]} samples")
+    print(f"   New Data: {X_new.shape[0]} samples")
+    
+    return results
+
+
+def save_report(results: Dict):
+    """Save drift detection report to file."""
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    report_path = REPORTS_DIR / f"drift_report_{timestamp}.json"
+    
+    with open(report_path, 'w') as f:
+        json.dump(results, f, indent=2)
+    
+    print(f"\nReport saved to: {report_path}")
+
+
+def push_to_prometheus(results: Dict):
+    """
+    Push drift metrics to Prometheus via Pushgateway.
+    
+    This allows Prometheus to scrape short-lived job metrics.
+    """
+    metrics = f"""# TYPE drift_detected gauge
+# HELP drift_detected Whether data drift was detected (1=yes, 0=no)
+drift_detected {results['drift_detected']}
+
+# TYPE drift_p_value gauge
+# HELP drift_p_value P-value from drift detection test
+drift_p_value {results['p_value']}
+
+# TYPE drift_distance gauge
+# HELP drift_distance Statistical distance between distributions
+drift_distance {results['distance']}
+
+# TYPE drift_check_timestamp gauge
+# HELP drift_check_timestamp Unix timestamp of last drift check
+drift_check_timestamp {datetime.now().timestamp()}
+"""
+    
+    try:
+        response = requests.post(
+            f"{PUSHGATEWAY_URL}/metrics/job/drift_detection/instance/hopcroft",
+            data=metrics,
+            headers={'Content-Type': 'text/plain'}
+        )
+        response.raise_for_status()
+        print(f"Metrics pushed to Pushgateway at {PUSHGATEWAY_URL}")
+    except requests.exceptions.RequestException as e:
+        print(f"Failed to push to Pushgateway: {e}")
+        print(f"   Make sure Pushgateway is running: docker compose ps pushgateway")
+
+
+def main():
+    """Main execution."""
+    print("\n" + "=" * 60)
+    print("Hopcroft Data Drift Detection")
+    print("=" * 60)
+    
+    try:
+        # Load data
+        X_baseline = load_baseline()
+        X_new = load_new_data()
+        
+        # Run drift detection
+        results = run_drift_detection(X_baseline, X_new)
+        
+        # Save report
+        save_report(results)
+        
+        # Push to Prometheus
+        push_to_prometheus(results)
+        
+        print("\n" + "=" * 60)
+        print("Drift Detection Complete!")
+        print("=" * 60)
+        
+        if results['drift_detected']:
+            print("\nWARNING: Data drift detected!")
+            print(f"   P-value: {results['p_value']:.6f} < {P_VALUE_THRESHOLD}")
+            return 1
+        else:
+            print("\nNo significant drift detected")
+            return 0
+            
+    except Exception as e:
+        print(f"\nError: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+
+
+if __name__ == "__main__":
+    exit(main())

monitoring/grafana/provisioning/dashboards/hopcroft_dashboard.json

@@ -0,0 +1,358 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "reqps"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true
+      },
+      "pluginVersion": "9.0.0",
+      "targets": [
+        {
+          "expr": "rate(fastapi_requests_total[1m])",
+          "refId": "A"
+        }
+      ],
+      "title": "Request Rate",
+      "type": "gauge",
+      "description": "Number of requests per second handled by the API"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": true
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "ms"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 18,
+        "x": 6,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "right"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "pluginVersion": "9.0.0",
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.95, rate(fastapi_request_duration_seconds_bucket[5m])) * 1000",
+          "legendFormat": "p95",
+          "refId": "A"
+        },
+        {
+          "expr": "histogram_quantile(0.50, rate(fastapi_request_duration_seconds_bucket[5m])) * 1000",
+          "legendFormat": "p50 (median)",
+          "refId": "B"
+        }
+      ],
+      "title": "Request Latency (p50, p95)",
+      "type": "timeseries",
+      "description": "API response time percentiles over time"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "red",
+                  "index": 1,
+                  "text": "No Drift"
+                },
+                "1": {
+                  "color": "green",
+                  "index": 0,
+                  "text": "Drift Detected"
+                }
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        }
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 6,
+        "x": 0,
+        "y": 8
+      },
+      "id": 3,
+      "options": {
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true,
+        "text": {}
+      },
+      "pluginVersion": "9.0.0",
+      "targets": [
+        {
+          "expr": "drift_detected",
+          "refId": "A"
+        }
+      ],
+      "title": "Data Drift Status",
+      "type": "stat",
+      "description": "Current data drift detection status (1 = drift detected, 0 = no drift)"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "decimals": 4,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 0.01
+              },
+              {
+                "color": "red",
+                "value": 0.05
+              }
+            ]
+          },
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 6,
+        "x": 6,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true,
+        "text": {}
+      },
+      "pluginVersion": "9.0.0",
+      "targets": [
+        {
+          "expr": "drift_p_value",
+          "refId": "A"
+        }
+      ],
+      "title": "Drift P-Value",
+      "type": "stat",
+      "description": "Statistical significance of detected drift (lower = more significant)"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "lastNotNull"],
+          "displayMode": "table",
+          "placement": "right"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "pluginVersion": "9.0.0",
+      "targets": [
+        {
+          "expr": "drift_distance",
+          "legendFormat": "Distance",
+          "refId": "A"
+        }
+      ],
+      "title": "Drift Distance Over Time",
+      "type": "timeseries",
+      "description": "Statistical distance between baseline and current data distribution"
+    }
+  ],
+  "refresh": "10s",
+  "schemaVersion": 36,
+  "style": "dark",
+  "tags": ["hopcroft", "ml", "monitoring"],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Hopcroft ML Model Monitoring",
+  "uid": "hopcroft-ml-dashboard",
+  "version": 1,
+  "weekStart": ""
+}

requirements.txt

@@ -65,4 +65,6 @@ pytest-html
 pytest-json-report
 
 # GUI
-streamlit>=1.28.0
+streamlit>=1.28.0
+
+alibi-detect>=0.11.4