data cleaning before insert to remove infinity numbers

app/mapper.py

@@ -451,6 +451,7 @@ def canonify_df(org_id: str, source_id: str, hours_window: int = 24) -> tuple[pd
         table_name = ensure_canonical_table(duck, df, entity_type)
         
         # 8c) Transactional insert
+                # 8d) Clean and insert data
         if not df.empty:
             table_info = duck.execute(f"PRAGMA table_info('{table_name}')").fetchall()
             table_cols = [str(r[0]) for r in table_info]
@@ -458,6 +459,9 @@ def canonify_df(org_id: str, source_id: str, hours_window: int = 24) -> tuple[pd
             df_to_insert = df[[col for col in df.columns if col in table_cols]]
             
             if not df_to_insert.empty:
+                # 🔧 CRITICAL: Replace NaN/Infinity with None for JSON compliance
+                df_to_insert = df_to_insert.replace([np.inf, -np.inf, np.nan], None)
+                
                 cols_str = ", ".join(df_to_insert.columns)
                 placeholders = ", ".join(["?"] * len(df_to_insert.columns))