utils/pre_processing.py

"""Data pre-processing functions."""

import numpy
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder, FunctionTransformer, StandardScaler


def _get_pipeline_replace_one_hot(func, value):
    return Pipeline([ 
        ("replace", FunctionTransformer(
            func, 
            kw_args={"value": value},
            feature_names_out='one-to-one',
        )),
        ("one_hot", OneHotEncoder(),),
    ])

def _replace_values_eq(column, value):
    for desired_value, values_to_replace in value.items():
        column = numpy.where(numpy.isin(column, values_to_replace), desired_value, column)
    return column

def get_pre_processors():
    pre_processor_applicant = ColumnTransformer(
        transformers=[
            (
                "replace_occupation_type_labor",
                _get_pipeline_replace_one_hot(
                    _replace_values_eq, 
                        {
                            "Labor_work": [
                                "Cooking Staff", "Carpenter", "Plumber", "Factory Worker", "Bus Driver"
                            ],
                            "Office_work": [
                                "Business Owners", "Office Worker", "Accountant", "Entrepreneur", "Salesperson"
                            ],
                            "High_tech_work": ["Engineer", "Manager", "Consultant", "Software Developer"],
                        },
                ),
                ['Occupation_type']
            ),
            ('one_hot_others', OneHotEncoder(), ['Housing_type', 'Family_status', 'Education_type', 'Income_type']),
            ('standard_scaler', StandardScaler(), ['Num_children', 'Household_size', 'Total_income', 'Age']),
        ],
        remainder='passthrough',
        verbose_feature_names_out=False,
    )

    pre_processor_bank = ColumnTransformer(
        transformers=[
            ('standard_scaler', StandardScaler(), ['Account_age']),
        ],
        remainder='passthrough',
        verbose_feature_names_out=False,
    )

    pre_processor_credit_bureau = ColumnTransformer(
        transformers=[],
        remainder='passthrough',
        verbose_feature_names_out=False,
    )

    return pre_processor_applicant, pre_processor_bank, pre_processor_credit_bureau