feat: add offline drift checks and inference logging

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help install train evaluate test lint typecheck run docker-build docker-run clean
+.PHONY: help install train evaluate drift-check test lint typecheck run docker-build docker-run clean
 
 help:
 	@echo "Rossmann Sales Prediction - Make Commands"
@@ -6,6 +6,7 @@ help:
 	@echo "  make install       Install dependencies"
 	@echo "  make train         Run training pipeline"
 	@echo "  make evaluate      Run holdout and backtesting evaluation"
+	@echo "  make drift-check   Build an offline drift report from inference logs"
 	@echo "  make test          Run tests"
 	@echo "  make lint          Run linting"
 	@echo "  make typecheck     Run type checking"
@@ -23,6 +24,9 @@ train:
 evaluate:
 	python scripts/evaluate_model.py
 
+drift-check:
+	python scripts/check_drift.py
+
 test:
 	python scripts/run_tests.py
 

README.md

@@ -28,7 +28,7 @@ Main ideas:
 src/training/    data loading, feature engineering, split helpers, model training
 src/serving/     FastAPI prediction service
 src/shared/      config, MLflow helper, and request/response schemas
-scripts/         evaluation and test runner
+scripts/         evaluation, drift check, and test runner
 web/             minimal HTML demo page
 reports/metrics/ generated training and evaluation outputs
 tests/           unit tests for training, serving, and split logic
@@ -65,6 +65,14 @@ This writes:
 If `mlflow` is installed, both commands also create local experiment runs under `mlruns/`
 by default. You can override that with `MLFLOW_TRACKING_URI`.
 
+Build an offline drift report from logged inference requests:
+
+```bash
+make drift-check
+```
+
+This writes `reports/metrics/drift_report.json` when inference logs are available.
+
 Start the API demo:
 
 ```bash
@@ -117,6 +125,8 @@ curl -X POST http://localhost:7860/predict \
 ```
 
 The API looks up static store metadata from `store.csv`, so the request stays small.
+Each request also appends one structured JSONL record to `logs/inference_requests.jsonl`
+with timestamp, store id, forecast horizon, model version, and latency.
 
 ## Example Results
 
@@ -154,5 +164,6 @@ The repository also includes `test.csv`, `sample_submission.csv`, and `train_sch
 - saved metrics may become stale if code or data changes
 - MLflow tracking is local and file-based; there is no remote tracking server or registry
 - CI validates the codebase but does not deploy artifacts or publish models
+- Drift checking is offline and based on logged inference requests, not live monitoring
 - the explanation output is only a model contribution view, not a causal interpretation
 - the API assumes the requested store exists in `store.csv`

scripts/check_drift.py

@@ -0,0 +1,104 @@
+# ruff: noqa: E402
+import json
+from pathlib import Path
+import sys
+
+import pandas as pd
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from src.serving.monitoring import DEFAULT_INFERENCE_LOG_PATH
+from src.shared.config import settings
+from src.training.data_loader import clean_data, load_raw_data, load_store_data
+from src.training.features import apply_feature_pipeline, build_feature_matrix
+
+
+def load_recent_inference_rows(log_path: Path) -> pd.DataFrame:
+    if not log_path.exists():
+        return pd.DataFrame()
+
+    records = [json.loads(line) for line in log_path.read_text(encoding="utf-8").splitlines() if line.strip()]
+    if not records:
+        return pd.DataFrame()
+
+    request_df = pd.DataFrame(records)
+    store_df = load_store_data(settings.data.store_path)
+    request_df = request_df.rename(
+        columns={
+            "store": "Store",
+            "start_date": "Date",
+            "promo": "Promo",
+            "state_holiday": "StateHoliday",
+            "school_holiday": "SchoolHoliday",
+        }
+    )
+    merged = request_df.merge(store_df, on="Store", how="left")
+    merged["Open"] = 1
+    for col in ["Promo2", "Promo2SinceWeek", "Promo2SinceYear"]:
+        merged[col] = merged[col].fillna(0).astype(int)
+    return merged
+
+
+def build_drift_report() -> dict[str, object]:
+    train_df = load_raw_data(settings.data.train_path, settings.data.store_path)
+    train_df = clean_data(train_df)
+    train_df = apply_feature_pipeline(
+        train_df,
+        fourier_period=settings.pipeline.fourier_period,
+        fourier_order=settings.pipeline.fourier_order,
+    )
+    train_features = build_feature_matrix(train_df, settings.data.features)
+
+    inference_df = load_recent_inference_rows(DEFAULT_INFERENCE_LOG_PATH)
+    if inference_df.empty:
+        return {
+            "status": "no_inference_logs",
+            "log_path": str(DEFAULT_INFERENCE_LOG_PATH),
+        }
+
+    inference_df = apply_feature_pipeline(
+        inference_df,
+        fourier_period=settings.pipeline.fourier_period,
+        fourier_order=settings.pipeline.fourier_order,
+    )
+    inference_features = build_feature_matrix(inference_df, settings.data.features)
+
+    drift_rows = []
+    for col in inference_features.columns:
+        train_mean = float(train_features[col].mean())
+        inference_mean = float(inference_features[col].mean())
+        train_std = float(train_features[col].std(ddof=0))
+        drift_score = abs(inference_mean - train_mean) / max(train_std, 1e-6)
+        drift_rows.append(
+            {
+                "feature": col,
+                "train_mean": round(train_mean, 4),
+                "inference_mean": round(inference_mean, 4),
+                "train_std": round(train_std, 4),
+                "normalized_mean_shift": round(float(drift_score), 4),
+            }
+        )
+
+    drift_rows.sort(key=lambda row: row["normalized_mean_shift"], reverse=True)
+    return {
+        "status": "ok",
+        "log_path": str(DEFAULT_INFERENCE_LOG_PATH),
+        "num_inference_events": int(len(inference_df)),
+        "top_drift_features": drift_rows[:10],
+    }
+
+
+def main() -> Path:
+    report = build_drift_report()
+    output_path = Path("reports/metrics/drift_report.json")
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    with output_path.open("w", encoding="utf-8") as f:
+        json.dump(report, f, indent=2)
+    print(f"Drift report written to {output_path}")
+    return output_path
+
+
+if __name__ == "__main__":
+    main()

src/serving/api.py

@@ -8,9 +8,15 @@ import logging
 import os
 from pathlib import Path
 import json
+from time import perf_counter
 
 from src.shared.config import DEFAULT_MODEL_METADATA_PATH, DEFAULT_MODEL_PATH, settings
 from src.shared.schemas import PredictionRequest, PredictionResponse, ExplanationItem
+from src.serving.monitoring import (
+    DEFAULT_INFERENCE_LOG_PATH,
+    append_jsonl_record,
+    build_inference_log_entry,
+)
 from src.training.data_loader import load_store_data
 from src.training.features import (
     apply_feature_pipeline,
@@ -82,6 +88,7 @@ def predict(request: PredictionRequest):
         raise HTTPException(status_code=404, detail=f"Store {request.Store} not found in metadata")
 
     try:
+        started_at = perf_counter()
         store_meta = store_lookup[request.Store]
         start_date = pd.to_datetime(request.Date)
         dates = [start_date + pd.Timedelta(days=i) for i in range(request.ForecastDays)]
@@ -137,6 +144,28 @@ def predict(request: PredictionRequest):
                 "sales": sales_val
             })
 
+        latency_ms = (perf_counter() - started_at) * 1000
+        append_jsonl_record(
+            DEFAULT_INFERENCE_LOG_PATH,
+            build_inference_log_entry(
+                store=request.Store,
+                start_date=request.Date,
+                forecast_days=request.ForecastDays,
+                promo=request.Promo,
+                state_holiday=request.StateHoliday,
+                school_holiday=request.SchoolHoliday,
+                model_version=model_version,
+                latency_ms=latency_ms,
+            ),
+        )
+        logger.info(
+            "prediction_complete store=%s horizon=%s model_version=%s latency_ms=%.3f",
+            request.Store,
+            request.ForecastDays,
+            model_version,
+            latency_ms,
+        )
+
         return PredictionResponse(
             Store=request.Store,
             Date=request.Date,

src/serving/monitoring.py

@@ -0,0 +1,39 @@
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+
+from src.shared.config import PROJECT_ROOT
+
+DEFAULT_INFERENCE_LOG_PATH = PROJECT_ROOT / "logs" / "inference_requests.jsonl"
+
+
+def build_inference_log_entry(
+    *,
+    store: int,
+    start_date: str,
+    forecast_days: int,
+    promo: int,
+    state_holiday: str,
+    school_holiday: int,
+    model_version: str,
+    latency_ms: float,
+) -> dict[str, object]:
+    """Builds a single structured inference log event."""
+    return {
+        "timestamp_utc": datetime.now(timezone.utc).isoformat(),
+        "store": store,
+        "start_date": start_date,
+        "forecast_days": forecast_days,
+        "promo": promo,
+        "state_holiday": state_holiday,
+        "school_holiday": school_holiday,
+        "model_version": model_version,
+        "latency_ms": round(latency_ms, 3),
+    }
+
+
+def append_jsonl_record(path: Path, payload: dict[str, object]) -> None:
+    """Appends a JSON line to the configured log file."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a", encoding="utf-8") as f:
+        f.write(json.dumps(payload) + "\n")

tests/test_monitoring.py

@@ -0,0 +1,27 @@
+from src.serving.monitoring import append_jsonl_record, build_inference_log_entry
+
+
+def test_build_inference_log_entry_contains_expected_fields():
+    entry = build_inference_log_entry(
+        store=1,
+        start_date="2015-07-31",
+        forecast_days=7,
+        promo=1,
+        state_holiday="0",
+        school_holiday=1,
+        model_version="v1",
+        latency_ms=12.3456,
+    )
+
+    assert entry["store"] == 1
+    assert entry["forecast_days"] == 7
+    assert entry["model_version"] == "v1"
+    assert entry["latency_ms"] == 12.346
+
+
+def test_append_jsonl_record_writes_one_line(tmp_path):
+    destination = tmp_path / "logs" / "inference.jsonl"
+    append_jsonl_record(destination, {"hello": "world"})
+
+    assert destination.exists()
+    assert destination.read_text(encoding="utf-8").strip() == '{"hello": "world"}'