Update app/services/preprocessing.py

app/services/preprocessing.py

@@ -9,46 +9,18 @@ def data_quality(df: pd.DataFrame):
     return df
 
 def standardize_data_types(df: pd.DataFrame) -> pd.DataFrame:
+    # Convert string-based dates to datetime, but ignore boolean values
     for col in df.columns:
-        if df[col].isin([True, False]).all():
-            continue  # already boolean
-
-        # Handle boolean strings
-        if df[col].dropna().astype(str).isin(["TRUE", "FALSE", "true", "false"]).all():
-            df[col] = df[col].map({
-                "TRUE": True, "FALSE": False,
-                "true": True, "false": False
-            })
-            continue
-
-        # Try to parse as datetime, if at least 50% parse correctly
-        try:
-            temp = pd.to_datetime(df[col], errors='coerce')
-            if temp.notna().mean() > 0.5:
-                df[col] = temp
-                continue
-        except:
-            pass
-
-        # Try to parse numeric if at least 50% can be converted
-        try:
-            temp = pd.to_numeric(df[col], errors='coerce')
-            if temp.notna().mean() > 0.5:
-                df[col] = temp
-                continue
-        except:
-            pass
-
-        # Convert JSON-like strings
-        try:
-            if df[col].dropna().apply(lambda x: isinstance(x, str) and x.strip().startswith("[") and x.strip().endswith("]")).all():
-                df[col] = df[col].apply(json.loads)
-                continue
-        except:
-            pass
-
-        # Default: make sure column is string
-        df[col] = df[col].astype(str)
+        if df[col].dtype == 'object' and not df[col].isin([True, False]).all():  
+            try:
+                df[col] = pd.to_datetime(df[col], errors='coerce')  # Invalid values become NaT
+            except Exception as e:
+                print(f"Skipping column {col}: {e}")
+
+    # Convert numeric strings to actual numbers
+    for col in df.select_dtypes(include=['object']).columns:
+        if df[col].str.replace('.', '', 1).str.isnumeric().all():
+            df[col] = pd.to_numeric(df[col])
 
     return df