Upload data_cleaning.py

utils/data_cleaning.py

@@ -0,0 +1,31 @@
+import pandas as pd
+import numpy as np
+
+# Function to handle missing values
+def handle_missing_values(df, method="drop"):
+    if method == "drop":
+        df = df.dropna()
+    elif method == "fill":
+        for col in df.columns:
+            if df[col].dtype in ['float64', 'int64']:
+                df[col].fillna(df[col].mean(), inplace=True)
+            else:
+                df[col].fillna(df[col].mode()[0], inplace=True)
+    return df
+
+# Function to remove outliers using the IQR method
+def remove_outliers_iqr(df, column):
+    Q1 = df[column].quantile(0.25)
+    Q3 = df[column].quantile(0.75)
+    IQR = Q3 - Q1
+    lower_bound = Q1 - 1.5 * IQR
+    upper_bound = Q3 + 1.5 * IQR
+    return df[(df[column] >= lower_bound) & (df[column] <= upper_bound)]
+
+# Function to cap extreme values (5th and 95th percentiles)
+def cap_extreme_values(df):
+    for col in df.select_dtypes(include=[np.number]).columns:
+        lower_limit = df[col].quantile(0.05)
+        upper_limit = df[col].quantile(0.95)
+        df[col] = np.clip(df[col], lower_limit, upper_limit)
+    return df