"""Heavy aggregation functions extracted from notebooks/EDA.ipynb. Each function reads one auxiliary CSV (bureau, previous_application, POS_CASH_balance, installments_payments, credit_card_balance) and returns a DataFrame indexed by SK_ID_CURR with one row per client. These functions are used only offline by scripts/build_feature_store.py. """ from __future__ import annotations import gc from pathlib import Path import numpy as np import pandas as pd def one_hot_encoder( df: pd.DataFrame, nan_as_category: bool = True ) -> tuple[pd.DataFrame, list[str]]: """One-hot encode all object dtype columns. Returns the encoded df and the list of newly created column names.""" original_columns = list(df.columns) categorical_columns = [col for col in df.columns if df[col].dtype == "object"] df = pd.get_dummies(df, columns=categorical_columns, dummy_na=nan_as_category) new_columns = [c for c in df.columns if c not in original_columns] return df, new_columns def bureau_and_balance( data_dir: Path, num_rows: int | None = None, nan_as_category: bool = True ) -> pd.DataFrame: """Aggregate bureau.csv and bureau_balance.csv per SK_ID_CURR. Produces BURO_*, ACTIVE_*, CLOSED_* columns. """ bureau = pd.read_csv(data_dir / "bureau.csv", nrows=num_rows) bb = pd.read_csv(data_dir / "bureau_balance.csv", nrows=num_rows) bb, bb_cat = one_hot_encoder(bb, nan_as_category) bureau, bureau_cat = one_hot_encoder(bureau, nan_as_category) bb_aggregations: dict[str, list[str]] = {"MONTHS_BALANCE": ["min", "max", "size"]} for col in bb_cat: bb_aggregations[col] = ["mean"] bb_agg = bb.groupby("SK_ID_BUREAU").agg(bb_aggregations) bb_agg.columns = pd.Index( [e[0] + "_" + e[1].upper() for e in bb_agg.columns.tolist()] ) bureau = bureau.join(bb_agg, how="left", on="SK_ID_BUREAU") bureau.drop(["SK_ID_BUREAU"], axis=1, inplace=True) del bb, bb_agg gc.collect() num_aggregations = { "DAYS_CREDIT": ["min", "max", "mean", "var"], "DAYS_CREDIT_ENDDATE": ["min", "max", "mean"], "DAYS_CREDIT_UPDATE": ["mean"], "CREDIT_DAY_OVERDUE": ["max", "mean"], "AMT_CREDIT_MAX_OVERDUE": ["mean"], "AMT_CREDIT_SUM": ["max", "mean", "sum"], "AMT_CREDIT_SUM_DEBT": ["max", "mean", "sum"], "AMT_CREDIT_SUM_OVERDUE": ["mean"], "AMT_CREDIT_SUM_LIMIT": ["mean", "sum"], "AMT_ANNUITY": ["max", "mean"], "CNT_CREDIT_PROLONG": ["sum"], "MONTHS_BALANCE_MIN": ["min"], "MONTHS_BALANCE_MAX": ["max"], "MONTHS_BALANCE_SIZE": ["mean", "sum"], } cat_aggregations: dict[str, list[str]] = {} for cat in bureau_cat: cat_aggregations[cat] = ["mean"] for cat in bb_cat: cat_aggregations[cat + "_MEAN"] = ["mean"] bureau_agg = bureau.groupby("SK_ID_CURR").agg( {**num_aggregations, **cat_aggregations} ) bureau_agg.columns = pd.Index( ["BURO_" + e[0] + "_" + e[1].upper() for e in bureau_agg.columns.tolist()] ) active = bureau[bureau["CREDIT_ACTIVE_Active"] == 1] active_agg = active.groupby("SK_ID_CURR").agg(num_aggregations) active_agg.columns = pd.Index( ["ACTIVE_" + e[0] + "_" + e[1].upper() for e in active_agg.columns.tolist()] ) bureau_agg = bureau_agg.join(active_agg, how="left", on="SK_ID_CURR") del active, active_agg gc.collect() closed = bureau[bureau["CREDIT_ACTIVE_Closed"] == 1] closed_agg = closed.groupby("SK_ID_CURR").agg(num_aggregations) closed_agg.columns = pd.Index( ["CLOSED_" + e[0] + "_" + e[1].upper() for e in closed_agg.columns.tolist()] ) bureau_agg = bureau_agg.join(closed_agg, how="left", on="SK_ID_CURR") del closed, closed_agg, bureau gc.collect() return bureau_agg def previous_applications( data_dir: Path, num_rows: int | None = None, nan_as_category: bool = True ) -> pd.DataFrame: """Aggregate previous_application.csv per SK_ID_CURR. Produces PREV_*, APPROVED_*, REFUSED_* columns. """ prev = pd.read_csv(data_dir / "previous_application.csv", nrows=num_rows) prev, cat_cols = one_hot_encoder(prev, nan_as_category=nan_as_category) for col in [ "DAYS_FIRST_DRAWING", "DAYS_FIRST_DUE", "DAYS_LAST_DUE_1ST_VERSION", "DAYS_LAST_DUE", "DAYS_TERMINATION", ]: prev[col] = prev[col].replace(365243, np.nan) prev["APP_CREDIT_PERC"] = prev["AMT_APPLICATION"] / prev["AMT_CREDIT"] num_aggregations = { "AMT_ANNUITY": ["min", "max", "mean"], "AMT_APPLICATION": ["min", "max", "mean"], "AMT_CREDIT": ["min", "max", "mean"], "APP_CREDIT_PERC": ["min", "max", "mean", "var"], "AMT_DOWN_PAYMENT": ["min", "max", "mean"], "AMT_GOODS_PRICE": ["min", "max", "mean"], "HOUR_APPR_PROCESS_START": ["min", "max", "mean"], "RATE_DOWN_PAYMENT": ["min", "max", "mean"], "DAYS_DECISION": ["min", "max", "mean"], "CNT_PAYMENT": ["mean", "sum"], } cat_aggregations = {cat: ["mean"] for cat in cat_cols} prev_agg = prev.groupby("SK_ID_CURR").agg({**num_aggregations, **cat_aggregations}) prev_agg.columns = pd.Index( ["PREV_" + e[0] + "_" + e[1].upper() for e in prev_agg.columns.tolist()] ) approved = prev[prev["NAME_CONTRACT_STATUS_Approved"] == 1] approved_agg = approved.groupby("SK_ID_CURR").agg(num_aggregations) approved_agg.columns = pd.Index( ["APPROVED_" + e[0] + "_" + e[1].upper() for e in approved_agg.columns.tolist()] ) prev_agg = prev_agg.join(approved_agg, how="left", on="SK_ID_CURR") refused = prev[prev["NAME_CONTRACT_STATUS_Refused"] == 1] refused_agg = refused.groupby("SK_ID_CURR").agg(num_aggregations) refused_agg.columns = pd.Index( ["REFUSED_" + e[0] + "_" + e[1].upper() for e in refused_agg.columns.tolist()] ) prev_agg = prev_agg.join(refused_agg, how="left", on="SK_ID_CURR") del refused, refused_agg, approved, approved_agg, prev gc.collect() return prev_agg def pos_cash( data_dir: Path, num_rows: int | None = None, nan_as_category: bool = True ) -> pd.DataFrame: """Aggregate POS_CASH_balance.csv per SK_ID_CURR. Produces POS_* columns.""" pos = pd.read_csv(data_dir / "POS_CASH_balance.csv", nrows=num_rows) pos, cat_cols = one_hot_encoder(pos, nan_as_category=nan_as_category) aggregations: dict[str, list[str]] = { "MONTHS_BALANCE": ["max", "mean", "size"], "SK_DPD": ["max", "mean"], "SK_DPD_DEF": ["max", "mean"], } for cat in cat_cols: aggregations[cat] = ["mean"] pos_agg = pos.groupby("SK_ID_CURR").agg(aggregations) pos_agg.columns = pd.Index( ["POS_" + e[0] + "_" + e[1].upper() for e in pos_agg.columns.tolist()] ) pos_agg["POS_COUNT"] = pos.groupby("SK_ID_CURR").size() del pos gc.collect() return pos_agg def installments_payments( data_dir: Path, num_rows: int | None = None, nan_as_category: bool = True ) -> pd.DataFrame: """Aggregate installments_payments.csv per SK_ID_CURR. Produces INSTAL_*.""" ins = pd.read_csv(data_dir / "installments_payments.csv", nrows=num_rows) ins, cat_cols = one_hot_encoder(ins, nan_as_category=nan_as_category) ins["PAYMENT_PERC"] = ins["AMT_PAYMENT"] / ins["AMT_INSTALMENT"] ins["PAYMENT_DIFF"] = ins["AMT_INSTALMENT"] - ins["AMT_PAYMENT"] ins["DPD"] = ins["DAYS_ENTRY_PAYMENT"] - ins["DAYS_INSTALMENT"] ins["DBD"] = ins["DAYS_INSTALMENT"] - ins["DAYS_ENTRY_PAYMENT"] ins["DPD"] = ins["DPD"].clip(lower=0) ins["DBD"] = ins["DBD"].clip(lower=0) aggregations: dict[str, list[str]] = { "NUM_INSTALMENT_VERSION": ["nunique"], "DPD": ["max", "mean", "sum"], "DBD": ["max", "mean", "sum"], "PAYMENT_PERC": ["max", "mean", "sum", "var"], "PAYMENT_DIFF": ["max", "mean", "sum", "var"], "AMT_INSTALMENT": ["max", "mean", "sum"], "AMT_PAYMENT": ["min", "max", "mean", "sum"], "DAYS_ENTRY_PAYMENT": ["max", "mean", "sum"], } for cat in cat_cols: aggregations[cat] = ["mean"] ins_agg = ins.groupby("SK_ID_CURR").agg(aggregations) ins_agg.columns = pd.Index( ["INSTAL_" + e[0] + "_" + e[1].upper() for e in ins_agg.columns.tolist()] ) ins_agg["INSTAL_COUNT"] = ins.groupby("SK_ID_CURR").size() del ins gc.collect() return ins_agg def credit_card_balance( data_dir: Path, num_rows: int | None = None, nan_as_category: bool = True ) -> pd.DataFrame: """Aggregate credit_card_balance.csv per SK_ID_CURR. Produces CC_* columns.""" cc = pd.read_csv(data_dir / "credit_card_balance.csv", nrows=num_rows) cc, cat_cols = one_hot_encoder(cc, nan_as_category=nan_as_category) cc.drop(["SK_ID_PREV"], axis=1, inplace=True) num_aggregations = { col: ["min", "max", "mean", "sum", "var"] for col in cc.columns if col not in cat_cols } cat_aggregations = {cat: ["mean"] for cat in cat_cols} cc_agg = cc.groupby("SK_ID_CURR").agg({**num_aggregations, **cat_aggregations}) cc_agg.columns = pd.Index( ["CC_" + e[0] + "_" + e[1].upper() for e in cc_agg.columns.tolist()] ) cc_agg["CC_COUNT"] = cc.groupby("SK_ID_CURR").size() del cc gc.collect() return cc_agg