Spaces:
Build error
Build error
File size: 1,501 Bytes
88a18a5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | import pandas as pd
from sklearn.preprocessing import LabelEncoder, StandardScaler
def preprocess_data(df, target):
summary = {}
df = df.copy()
if target not in df.columns:
raise ValueError("Target column not found.")
df = df[df[target].notna()]
# Handle missing values
missing = df.isnull().sum()
summary["Missing Values"] = missing[missing > 0].to_dict()
for col in df.columns:
if df[col].isnull().sum() > 0:
if df[col].dtype == 'object':
df[col].fillna(df[col].mode()[0], inplace=True)
else:
df[col].fillna(df[col].mean(), inplace=True)
# Encode categorical
le_dict = {}
for col in df.select_dtypes(include=['object']).columns:
le = LabelEncoder()
df[col] = le.fit_transform(df[col].astype(str))
le_dict[col] = le.classes_.tolist()
summary["Encoded Columns"] = le_dict
# Scale numerical
num_cols = df.select_dtypes(include=['int64', 'float64']).drop(columns=[target], errors='ignore').columns
scaler = StandardScaler()
df[num_cols] = scaler.fit_transform(df[num_cols])
summary["Scaled Columns"] = num_cols.tolist()
X = df.drop(columns=[target])
y = df[target]
# Detect problem type
if y.dtype == 'object' or y.nunique() <= 10:
y = LabelEncoder().fit_transform(y.astype(str))
problem_type = "classification"
else:
problem_type = "regression"
return X, y, summary, problem_type
|