Update app/services/preprocessing.py

app/services/preprocessing.py

@@ -11,36 +11,68 @@ def data_quality(df: pd.DataFrame):
 def standardize_data_types(df: pd.DataFrame) -> pd.DataFrame:
     for col in df.columns:
         if df[col].isin([True, False]).all():
+            continue  # already boolean
+
+        # Handle boolean strings
+        if df[col].dropna().astype(str).isin(["TRUE", "FALSE", "true", "false"]).all():
+            df[col] = df[col].map({
+                "TRUE": True, "FALSE": False,
+                "true": True, "false": False
+            })
             continue
-        if df[col].dtype == 'object' and df[col].str.replace('.', '', 1).str.isnumeric().all():
-            df[col] = pd.to_numeric(df[col], errors='ignore')
+
+        # Try to parse as datetime, if at least 50% parse correctly
         try:
-            df[col] = pd.to_datetime(df[col], errors='coerce')
-            if df[col].notna().sum() == 0:
-                df[col] = df[col].astype(str)
-        except Exception:
+            temp = pd.to_datetime(df[col], errors='coerce')
+            if temp.notna().mean() > 0.5:
+                df[col] = temp
+                continue
+        except:
             pass
+
+        # Try to parse numeric if at least 50% can be converted
         try:
-            if df[col].apply(lambda x: isinstance(x, str) and x.startswith("[") and x.endswith("]")).all():
+            temp = pd.to_numeric(df[col], errors='coerce')
+            if temp.notna().mean() > 0.5:
+                df[col] = temp
+                continue
+        except:
+            pass
+
+        # Convert JSON-like strings
+        try:
+            if df[col].dropna().apply(lambda x: isinstance(x, str) and x.strip().startswith("[") and x.strip().endswith("]")).all():
                 df[col] = df[col].apply(json.loads)
-        except Exception:
+                continue
+        except:
             pass
-        if df[col].dtype == 'object' and df[col].dropna().isin(["TRUE", "FALSE"]).all():
-            df[col] = df[col].map({"TRUE": True, "FALSE": False})
-        if df[col].dtype == 'object':
-            df[col] = df[col].astype(str)
-    df.fillna("", inplace=True)
+
+        # Default: make sure column is string
+        df[col] = df[col].astype(str)
+
     return df
 
+
 def handle_missing_data(df: pd.DataFrame) -> pd.DataFrame:
+    print("Before Imputation (NA Counts):")
+    print(df.isnull().sum())
+
     numeric_col = df.select_dtypes(include=['number']).columns
     if not numeric_col.empty:
-        df[numeric_col] = SimpleImputer(strategy='median').fit_transform(df[numeric_col])
-    categorical_col = df.select_dtypes(include=['object']).columns
+        num_imputer = SimpleImputer(strategy='median')
+        df[numeric_col] = num_imputer.fit_transform(df[numeric_col])
+
+    categorical_col = df.select_dtypes(include=['object', 'category']).columns
     if not categorical_col.empty:
-        df[categorical_col] = SimpleImputer(strategy='most_frequent').fit_transform(df[categorical_col])
+        cat_imputer = SimpleImputer(strategy='most_frequent')
+        df[categorical_col] = cat_imputer.fit_transform(df[categorical_col])
+
+    print("After Imputation (NA Counts):")
+    print(df.isnull().sum())
+
     return df
 
+
 def handle_outliers(df: pd.DataFrame) -> pd.DataFrame:
     numeric_col = df.select_dtypes(include=['number','int64', 'float64']).columns
     if not numeric_col.empty: