training/train_recommender.py

"""Recommender training pipeline.

Trains two GradientBoostingClassifier models for recommendation outcome prediction.
Targets: clicked (binary), is_completed (binary) — trained as two separate models.
Features: priority (encoded), ai_confidence, recommendation_type (encoded),
          grade, subject (encoded).
Primary metric: ROC-AUC for clicked.
"""

import logging
from datetime import datetime, timezone

import numpy as np
import pandas as pd
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import roc_auc_score
from sklearn.preprocessing import OrdinalEncoder

from app.core.config import settings
from app.core.exceptions import TrainingError
from training.base_trainer import BaseTrainer, TrainingResult

logger = logging.getLogger(__name__)

FEATURE_COLUMNS = ["priority", "ai_confidence", "recommendation_type", "grade", "subject"]
CATEGORICAL_COLUMNS = ["priority", "recommendation_type", "subject"]
NUMERIC_COLUMNS = ["ai_confidence", "grade"]
TARGET_COLUMNS = ["clicked", "is_completed"]


class RecommenderTrainer(BaseTrainer):
    """GradientBoostingClassifier for recommendation outcome prediction.

    Targets: clicked (binary), is_completed (binary) — trained as two separate models
    Features: priority (encoded), ai_confidence, recommendation_type (encoded),
              grade, subject (encoded)
    Primary metric: ROC-AUC for clicked
    """

    @property
    def model_name(self) -> str:
        return "recommender"

    @property
    def model_version(self) -> str:
        return "recommender_v2_baseline_001"

    @property
    def table_name(self) -> str:
        return "training_recommendation_outcomes"

    def _build_features(
        self, df: pd.DataFrame, encoder: OrdinalEncoder, fit: bool = False
    ) -> np.ndarray:
        """Build feature matrix from DataFrame using OrdinalEncoder.

        Args:
            df: DataFrame with feature columns.
            encoder: OrdinalEncoder instance.
            fit: If True, fit the encoder on the data first.

        Returns:
            Feature matrix as numpy array.
        """
        if fit:
            encoder.fit(df[CATEGORICAL_COLUMNS])

        X_cat = encoder.transform(df[CATEGORICAL_COLUMNS])
        X_num = df[NUMERIC_COLUMNS].values
        return np.hstack([X_num, X_cat])

    def train(self, train_df: pd.DataFrame, val_df: pd.DataFrame) -> dict:
        """Train two GradientBoostingClassifier models for clicked and is_completed.

        Algorithm:
        1. Encode categorical columns (priority, recommendation_type, subject)
           with OrdinalEncoder
        2. Train GBC for 'clicked': GradientBoostingClassifier(
               n_estimators=100, max_depth=4, random_state=seed)
        3. Train GBC for 'is_completed': same hyperparameters
        4. Return {"model": {"clicked": gbc_clicked, "is_completed": gbc_completed},
                   "encoder": ordinal_enc,
                   "feature_columns.json": FEATURE_COLUMNS}
        """
        # Fit OrdinalEncoder on categorical columns
        ordinal_enc = OrdinalEncoder(
            handle_unknown="use_encoded_value",
            unknown_value=-1,
        )

        # Build feature matrix
        X_train = self._build_features(train_df, ordinal_enc, fit=True)

        # Train GBC for 'clicked'
        y_clicked = train_df["clicked"].values.astype(int)
        gbc_clicked = GradientBoostingClassifier(
            n_estimators=100,
            max_depth=4,
            random_state=self._seed,
        )
        gbc_clicked.fit(X_train, y_clicked)

        # Train GBC for 'is_completed'
        y_completed = train_df["is_completed"].values.astype(int)
        gbc_completed = GradientBoostingClassifier(
            n_estimators=100,
            max_depth=4,
            random_state=self._seed,
        )
        gbc_completed.fit(X_train, y_completed)

        logger.info(
            "Recommender trained — %d samples, %d features, "
            "clicked positive rate: %.2f%%, is_completed positive rate: %.2f%%",
            X_train.shape[0],
            X_train.shape[1],
            100.0 * y_clicked.sum() / len(y_clicked),
            100.0 * y_completed.sum() / len(y_completed),
        )

        return {
            "model": {"clicked": gbc_clicked, "is_completed": gbc_completed},
            "encoder": ordinal_enc,
            "feature_columns.json": FEATURE_COLUMNS,
        }

    def _compute_lift_at_10(
        self, y_true: np.ndarray, y_proba: np.ndarray
    ) -> float:
        """Compute lift@10: ratio of positive rate in top-10% predicted vs overall.

        Lift@10 = (positive rate in top 10% by predicted probability) /
                  (overall positive rate)
        """
        n = len(y_true)
        if n == 0:
            return 0.0

        overall_positive_rate = y_true.sum() / n
        if overall_positive_rate == 0.0:
            return 0.0

        # Top 10% by predicted probability
        top_k = max(1, int(np.ceil(n * 0.10)))
        top_indices = np.argsort(y_proba)[::-1][:top_k]
        top_positive_rate = y_true[top_indices].sum() / top_k

        lift = top_positive_rate / overall_positive_rate
        return round(lift, 4)

    def evaluate(self, artifacts: dict, df: pd.DataFrame, split_name: str) -> dict:
        """Evaluate model on a split.

        Computes: ROC-AUC for clicked, ROC-AUC for is_completed,
        lift@10 for each target.
        """
        models = artifacts["model"]
        encoder = artifacts["encoder"]

        # Build feature matrix
        X = self._build_features(df, encoder)

        metrics = {}

        for target in TARGET_COLUMNS:
            model = models[target]
            y_true = df[target].values.astype(int)
            y_proba = model.predict_proba(X)[:, 1]

            # ROC-AUC
            try:
                roc_auc = roc_auc_score(y_true, y_proba)
            except ValueError:
                # Only one class present in y_true
                roc_auc = 0.0

            # Lift@10
            lift_10 = self._compute_lift_at_10(y_true, y_proba)

            metrics[f"roc_auc_{target}"] = round(roc_auc, 4)
            metrics[f"lift_at_10_{target}"] = lift_10

        logger.info(
            "%s metrics — ROC-AUC clicked: %.4f, ROC-AUC is_completed: %.4f, "
            "lift@10 clicked: %.4f, lift@10 is_completed: %.4f",
            split_name,
            metrics["roc_auc_clicked"],
            metrics["roc_auc_is_completed"],
            metrics["lift_at_10_clicked"],
            metrics["lift_at_10_is_completed"],
        )

        return metrics

    def _check_baseline(self, metrics: dict) -> None:
        """Verify ROC-AUC for clicked > 0.50 (above random).

        Raises TrainingError if not met.
        """
        test_metrics = metrics.get("metrics", {}).get("test", {})
        roc_auc_clicked = test_metrics.get("roc_auc_clicked")

        # Fallback to validation metrics if test not available
        if roc_auc_clicked is None:
            val_metrics = metrics.get("metrics", {}).get("validation", {})
            roc_auc_clicked = val_metrics.get("roc_auc_clicked")

        if roc_auc_clicked is None:
            raise TrainingError(
                "Cannot compute baseline: roc_auc_clicked not found in metrics.",
                model_name=self.model_name,
            )

        if roc_auc_clicked <= 0.50:
            raise TrainingError(
                f"ROC-AUC for clicked ({roc_auc_clicked:.4f}) does not exceed "
                f"baseline (0.50). Model fails to predict click engagement.",
                model_name=self.model_name,
            )

        logger.info(
            "Baseline check passed — ROC-AUC clicked %.4f > 0.50", roc_auc_clicked
        )

    def _build_metrics(
        self,
        val_metrics: dict,
        test_metrics: dict,
        train_df: pd.DataFrame,
        val_df: pd.DataFrame,
        test_df: pd.DataFrame,
    ) -> dict:
        """Assemble full metrics.json content."""
        return {
            "model_name": self.model_name,
            "model_version": self.model_version,
            "dataset_version": settings.ai_service_version,
            "trained_at": datetime.now(timezone.utc).isoformat(),
            "seed": self._seed,
            "split_counts": {
                "train": len(train_df),
                "validation": len(val_df),
                "test": len(test_df),
            },
            "metrics": {
                "validation": val_metrics,
                "test": test_metrics,
            },
            "limitations": [
                "Trained on synthetic data only.",
                "Two separate GBC models — no joint optimization of clicked + is_completed.",
                "OrdinalEncoder assumes an ordering for priority/recommendation_type/subject.",
                "Lift@10 depends on the distribution of positive labels in the dataset.",
                "No user-level features (e.g., engagement history) included in baseline.",
            ],
        }

    def _build_training_config(
        self,
        train_df: pd.DataFrame,
        val_df: pd.DataFrame,
        test_df: pd.DataFrame,
    ) -> dict:
        """Build training_config.json with hyperparameters."""
        return {
            "model_name": self.model_name,
            "model_version": self.model_version,
            "dataset_version": settings.ai_service_version,
            "seed": self._seed,
            "split_counts": {
                "train": len(train_df),
                "validation": len(val_df),
                "test": len(test_df),
            },
            "hyperparameters": {
                "n_estimators": 100,
                "max_depth": 4,
                "random_state": self._seed,
                "algorithm": "GradientBoostingClassifier",
                "encoder": "OrdinalEncoder",
            },
            "feature_columns": FEATURE_COLUMNS,
            "categorical_columns": CATEGORICAL_COLUMNS,
            "numeric_columns": NUMERIC_COLUMNS,
            "target_columns": TARGET_COLUMNS,
            "algorithm": "GradientBoostingClassifier",
        }

    def _build_model_card(self, metrics: dict) -> str:
        """Generate model_card.md content."""
        val_metrics = metrics.get("metrics", {}).get("validation", {})
        test_metrics = metrics.get("metrics", {}).get("test", {})

        card = f"""# Model Card: Recommender

## Model Details

- **Model Name:** {self.model_name}
- **Model Version:** {self.model_version}
- **Algorithm:** GradientBoostingClassifier (two models: clicked, is_completed)
- **Framework:** scikit-learn
- **Trained At:** {metrics.get("trained_at", "N/A")}
- **Seed:** {self._seed}

## Intended Use

Predict whether a student will click on a recommendation and whether they will
complete the recommended content. Used in the recommendation engine to rank
content by predicted engagement. Two separate binary classifiers are trained:
one for `clicked` and one for `is_completed`.

## Training Data

- **Source:** training_recommendation_outcomes.csv (synthetic dataset v2)
- **Split Counts:** train={metrics.get("split_counts", {}).get("train", "N/A")}, \
validation={metrics.get("split_counts", {}).get("validation", "N/A")}, \
test={metrics.get("split_counts", {}).get("test", "N/A")}
- **Features:** priority (OrdinalEncoded), ai_confidence (numeric), \
recommendation_type (OrdinalEncoded), grade (numeric), subject (OrdinalEncoded)
- **Targets:** clicked (binary), is_completed (binary)

## Metrics

### Validation Set
- ROC-AUC (clicked): {val_metrics.get("roc_auc_clicked", "N/A")}
- ROC-AUC (is_completed): {val_metrics.get("roc_auc_is_completed", "N/A")}
- Lift@10 (clicked): {val_metrics.get("lift_at_10_clicked", "N/A")}
- Lift@10 (is_completed): {val_metrics.get("lift_at_10_is_completed", "N/A")}

### Test Set
- ROC-AUC (clicked): {test_metrics.get("roc_auc_clicked", "N/A")}
- ROC-AUC (is_completed): {test_metrics.get("roc_auc_is_completed", "N/A")}
- Lift@10 (clicked): {test_metrics.get("lift_at_10_clicked", "N/A")}
- Lift@10 (is_completed): {test_metrics.get("lift_at_10_is_completed", "N/A")}

## Known Limitations

- Trained on synthetic data only — performance on real recommendation data is unknown.
- Two separate GBC models — no joint optimization of clicked + is_completed.
- OrdinalEncoder assumes an ordering for priority/recommendation_type/subject.
- Lift@10 depends on the distribution of positive labels in the dataset.
- No user-level features (e.g., engagement history) included in baseline.
- Limited feature set (5 features); adding student history could improve performance.

## Fallback Behavior

When the model is not loaded or confidence is below threshold, the system
falls back to knowledge-graph weakest-prerequisite + content_catalog filtered
by LO + difficulty, ranked by estimated_mastery_gain.
"""
        return card