src/ml/evaluate.py

"""Evaluate the persisted preparation-time pipeline.

Produces error analysis:
- overall metrics
- residuals per time-of-day bucket
- residuals per order type
- distribution of absolute errors

Usage:
    python -m src.ml.evaluate
"""

from __future__ import annotations

import json
import sys
from pathlib import Path

import joblib
import numpy as np
import pandas as pd
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score
from sklearn.model_selection import train_test_split

sys.path.insert(0, str(Path(__file__).resolve().parents[2]))

from src.config import ML_METRICS_PATH, ML_PIPELINE_PATH, MODELS_DIR  # noqa: E402
from src.ml.feature_engineering import FEATURES_CSV, TARGET  # noqa: E402


ERROR_REPORT = MODELS_DIR / "ml_error_report.json"


def main() -> None:
    if not ML_PIPELINE_PATH.exists():
        raise FileNotFoundError("Pipeline not found. Train first with 'python -m src.ml.train'.")

    bundle = joblib.load(ML_PIPELINE_PATH)
    pipe = bundle["pipeline"]
    feature_cols = bundle["feature_cols"]

    df = pd.read_csv(FEATURES_CSV)
    X = df[feature_cols]
    y = df[TARGET]

    _, X_test, _, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
    preds = pipe.predict(X_test)
    residuals = y_test.values - preds

    overall = {
        "MAE": float(mean_absolute_error(y_test, preds)),
        "RMSE": float(np.sqrt(mean_squared_error(y_test, preds))),
        "R2": float(r2_score(y_test, preds)),
        "n_test": int(len(y_test)),
    }
    print(
        f"[evaluate] overall  MAE={overall['MAE']:.2f}  RMSE={overall['RMSE']:.2f}  "
        f"R2={overall['R2']:.3f}  (n={overall['n_test']})"
    )

    # Per time-of-day bucket
    eval_frame = X_test.copy()
    eval_frame["y_true"] = y_test.values
    eval_frame["y_pred"] = preds
    eval_frame["abs_error"] = np.abs(residuals)

    def _bucket(h: float) -> str:
        if pd.isna(h):
            return "unknown"
        h = int(h)
        if 5 <= h < 11:
            return "morning"
        if 11 <= h < 15:
            return "lunch"
        if 15 <= h < 18:
            return "afternoon"
        if 18 <= h < 23:
            return "dinner"
        return "late_night"

    error_by_daypart = {}
    if "hour_of_day" in eval_frame.columns:
        eval_frame["daypart"] = eval_frame["hour_of_day"].apply(_bucket)
        grp = eval_frame.groupby("daypart")["abs_error"].agg(["mean", "count"])
        error_by_daypart = grp.to_dict(orient="index")
        print("[evaluate] MAE per daypart:")
        print(grp.round(2))

    error_by_order = {}
    if "Type_of_order" in eval_frame.columns:
        grp = eval_frame.groupby("Type_of_order")["abs_error"].agg(["mean", "count"])
        error_by_order = grp.to_dict(orient="index")
        print("[evaluate] MAE per order type:")
        print(grp.round(2))

    report = {
        "overall": overall,
        "error_by_daypart": error_by_daypart,
        "error_by_order_type": error_by_order,
        "best_model": bundle.get("best_model"),
    }
    ERROR_REPORT.write_text(json.dumps(report, indent=2, default=str))
    print(f"[evaluate] wrote {ERROR_REPORT}")

    # also refresh metrics file
    if ML_METRICS_PATH.exists():
        prev = json.loads(ML_METRICS_PATH.read_text())
    else:
        prev = {}
    prev["test_overall"] = overall
    ML_METRICS_PATH.write_text(json.dumps(prev, indent=2))


if __name__ == "__main__":
    main()