utils.py

import pandas as pd
from sklearn.preprocessing import LabelEncoder, StandardScaler

def preprocess_data(df, target):
    summary = {}
    df = df.copy()

    if target not in df.columns:
        raise ValueError("Target column not found.")

    df = df[df[target].notna()]

    # Handle missing values
    missing = df.isnull().sum()
    summary["Missing Values"] = missing[missing > 0].to_dict()

    for col in df.columns:
        if df[col].isnull().sum() > 0:
            if df[col].dtype == 'object':
                df[col].fillna(df[col].mode()[0], inplace=True)
            else:
                df[col].fillna(df[col].mean(), inplace=True)

    # Encode categorical
    le_dict = {}
    for col in df.select_dtypes(include=['object']).columns:
        le = LabelEncoder()
        df[col] = le.fit_transform(df[col].astype(str))
        le_dict[col] = le.classes_.tolist()
    summary["Encoded Columns"] = le_dict

    # Scale numerical
    num_cols = df.select_dtypes(include=['int64', 'float64']).drop(columns=[target], errors='ignore').columns
    scaler = StandardScaler()
    df[num_cols] = scaler.fit_transform(df[num_cols])
    summary["Scaled Columns"] = num_cols.tolist()

    X = df.drop(columns=[target])
    y = df[target]

    # Detect problem type
    if y.dtype == 'object' or y.nunique() <= 10:
        y = LabelEncoder().fit_transform(y.astype(str))
        problem_type = "classification"
    else:
        problem_type = "regression"

    return X, y, summary, problem_type