shaliz-kong commited on
Commit ·
2fff85c
1
Parent(s): aef3a87
inserted raws verification
Browse files- app/mapper.py +23 -2
app/mapper.py
CHANGED
|
@@ -456,7 +456,7 @@ def canonify_df(org_id: str, source_id: str, hours_window: int = 24) -> tuple[pd
|
|
| 456 |
# Insert data
|
| 457 |
if not df.empty:
|
| 458 |
table_info = duck.execute(f"PRAGMA table_info('{table_name}')").fetchall()
|
| 459 |
-
table_cols = [str(r[
|
| 460 |
|
| 461 |
df_to_insert = df[[col for col in df.columns if col in table_cols]]
|
| 462 |
|
|
@@ -471,7 +471,28 @@ def canonify_df(org_id: str, source_id: str, hours_window: int = 24) -> tuple[pd
|
|
| 471 |
df_to_insert.values.tolist()
|
| 472 |
)
|
| 473 |
rows_inserted = len(df_to_insert)
|
| 474 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 475 |
|
| 476 |
# Mark schema as applied
|
| 477 |
if is_new_schema and version_id:
|
|
|
|
| 456 |
# Insert data
|
| 457 |
if not df.empty:
|
| 458 |
table_info = duck.execute(f"PRAGMA table_info('{table_name}')").fetchall()
|
| 459 |
+
table_cols = [str(r[1]) for r in table_info]
|
| 460 |
|
| 461 |
df_to_insert = df[[col for col in df.columns if col in table_cols]]
|
| 462 |
|
|
|
|
| 471 |
df_to_insert.values.tolist()
|
| 472 |
)
|
| 473 |
rows_inserted = len(df_to_insert)
|
| 474 |
+
# In app/mapper.py, after `rows_inserted = len(df_to_insert)`
|
| 475 |
+
|
| 476 |
+
# ==================== DIAGNOSTIC QUERIES (DISABLE IN PROD) ====================
|
| 477 |
+
print(f"[canonify] 🔬 DIAGNOSTIC: Querying table immediately post-insert...")
|
| 478 |
+
try:
|
| 479 |
+
diagnostic_conn = get_conn(org_id)
|
| 480 |
+
|
| 481 |
+
# Check total row count
|
| 482 |
+
total_rows = diagnostic_conn.execute(f"SELECT COUNT(*) FROM {table_name}").fetchone()[0]
|
| 483 |
+
print(f"[canonify] 🔬 DIAGNOSTIC: Table {table_name} has {total_rows} total rows")
|
| 484 |
+
|
| 485 |
+
# Sample the data
|
| 486 |
+
sample = diagnostic_conn.execute(f"SELECT * FROM {table_name} LIMIT 3").df()
|
| 487 |
+
if not sample.empty:
|
| 488 |
+
print(f"[canonify] 🔬 DIAGNOSTIC: Sample data:\n{sample.to_string()}")
|
| 489 |
+
else:
|
| 490 |
+
print(f"[canonify] 🔬 DIAGNOSTIC: Table exists but SELECT returned ZERO rows")
|
| 491 |
+
|
| 492 |
+
diagnostic_conn.close()
|
| 493 |
+
except Exception as diag_e:
|
| 494 |
+
print(f"[canonify] 🔬 DIAGNOSTIC ERROR: {diag_e}")
|
| 495 |
+
# ============================================================================
|
| 496 |
|
| 497 |
# Mark schema as applied
|
| 498 |
if is_new_schema and version_id:
|