src/preprocess.py

"""
Turn engineered feature tables into model-ready matrices.

Gradient boosting (LightGBM) handles categoricals natively, so the default path
keeps `category`/`gender`/`state` as pandas categoricals. For models that need
purely numeric input (autoencoder, GNN, SMOTE), we one-hot / ordinal encode.
"""
from __future__ import annotations

import numpy as np
import pandas as pd

from src import config


def load_features(which: str = "train") -> pd.DataFrame:
    path = config.FEATURES_TRAIN if which == "train" else config.FEATURES_TEST
    if not path.exists():
        raise FileNotFoundError(f"{path} missing — run scripts/run_features.py first.")
    return pd.read_parquet(path)


def split_xy(df: pd.DataFrame, categorical: bool = True):
    """Return (X, y) with categoricals typed for LightGBM if requested."""
    X = df[config.ALL_FEATURES].copy()
    y = df[config.TARGET].astype(int).values
    if categorical:
        for c in config.CATEGORICAL_FEATURES:
            if c in X.columns:
                X[c] = X[c].astype("category")
    return X, y


def time_validation_split(df: pd.DataFrame, valid_fraction: float = config.VALID_FRACTION):
    """
    Hold out the last `valid_fraction` of the (already time-sorted) training
    frame for early stopping. No shuffling — mirrors real deployment where you
    train on the past and validate on the more-recent past.
    """
    n = len(df)
    cut = int(n * (1 - valid_fraction))
    return df.iloc[:cut].copy(), df.iloc[cut:].copy()


def encode_numeric_matrix(df_train: pd.DataFrame, df_other: pd.DataFrame = None):
    """
    Fully-numeric encoding for AE / SMOTE / GNN node features:
    numeric columns passed through (standardised), categoricals one-hot encoded.
    Returns (X_train, [X_other], encoder_state) where encoder_state lets you
    re-apply the exact same transform at inference time.
    """
    from sklearn.preprocessing import StandardScaler

    num = config.NUMERIC_FEATURES
    cat = [c for c in config.CATEGORICAL_FEATURES if c in df_train.columns]

    # One-hot via pandas, aligned columns across splits
    tr_oh = pd.get_dummies(df_train[cat].astype(str), prefix=cat)
    scaler = StandardScaler()
    tr_num = pd.DataFrame(scaler.fit_transform(df_train[num]), columns=num, index=df_train.index)
    X_train = pd.concat([tr_num, tr_oh.set_index(df_train.index)], axis=1)

    state = {"scaler": scaler, "columns": list(X_train.columns), "num": num, "cat": cat}

    if df_other is not None:
        ot_oh = pd.get_dummies(df_other[cat].astype(str), prefix=cat)
        ot_num = pd.DataFrame(scaler.transform(df_other[num]), columns=num, index=df_other.index)
        X_other = pd.concat([ot_num, ot_oh.set_index(df_other.index)], axis=1)
        X_other = X_other.reindex(columns=X_train.columns, fill_value=0.0)
        return X_train.astype(np.float32), X_other.astype(np.float32), state

    return X_train.astype(np.float32), state