Spaces:
Running
Running
fix: DDL type fallback in insert_rows when DESCRIBE TABLE returns empty
Browse filesIf _get_column_info fails silently (returns {}), convert_value had no type
info and passed string values through unconverted — causing 22018 on every
row for any numeric column.
- Add ddl_types param to insert_rows: {COL_UPPER: DDL_TYPE} from LegitData schema
- convert_value uses ddl_types as fallback when column not in column_info
- Log warning when column_info is empty so future failures are visible
- Pass ddl_types from generate() at the call site
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
- legitdata_bridge.py +20 -7
legitdata_bridge.py
CHANGED
|
@@ -172,20 +172,26 @@ class KeyPairSnowflakeWriter:
|
|
| 172 |
self.connection = None
|
| 173 |
print("Disconnected from Snowflake")
|
| 174 |
|
| 175 |
-
def insert_rows(self, table_name: str, columns: list, rows: list, batch_size: int = 1000) -> int:
|
| 176 |
-
"""Insert rows into a Snowflake table.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 177 |
from decimal import Decimal
|
| 178 |
from datetime import datetime, date
|
| 179 |
-
|
| 180 |
if not rows:
|
| 181 |
return 0
|
| 182 |
-
|
| 183 |
if not self.cursor:
|
| 184 |
raise RuntimeError("Not connected to database")
|
| 185 |
|
| 186 |
# Get column metadata for this table to know constraints
|
| 187 |
column_info = self._get_column_info(table_name)
|
| 188 |
-
|
|
|
|
|
|
|
| 189 |
# Filter out identity columns (let Snowflake auto-generate)
|
| 190 |
filtered_columns = []
|
| 191 |
filtered_indices = []
|
|
@@ -232,6 +238,10 @@ class KeyPairSnowflakeWriter:
|
|
| 232 |
if col_upper in column_info:
|
| 233 |
info = column_info[col_upper]
|
| 234 |
data_type = info.get('type', '')
|
|
|
|
|
|
|
|
|
|
|
|
|
| 235 |
|
| 236 |
# Truncate strings to fit VARCHAR length
|
| 237 |
if 'VARCHAR' in data_type or 'TEXT' in data_type:
|
|
@@ -476,8 +486,11 @@ class DemoLegitGenerator:
|
|
| 476 |
|
| 477 |
if truncate_first:
|
| 478 |
writer.truncate_table(table.name)
|
| 479 |
-
|
| 480 |
-
|
|
|
|
|
|
|
|
|
|
| 481 |
results[table.name] = count
|
| 482 |
|
| 483 |
return results
|
|
|
|
| 172 |
self.connection = None
|
| 173 |
print("Disconnected from Snowflake")
|
| 174 |
|
| 175 |
+
def insert_rows(self, table_name: str, columns: list, rows: list, batch_size: int = 1000, ddl_types: dict = None) -> int:
|
| 176 |
+
"""Insert rows into a Snowflake table.
|
| 177 |
+
|
| 178 |
+
ddl_types: optional dict of {COLUMN_NAME_UPPER: DDL_DATA_TYPE} from the LegitData schema,
|
| 179 |
+
used as fallback when DESCRIBE TABLE (column_info) doesn't have a column.
|
| 180 |
+
"""
|
| 181 |
from decimal import Decimal
|
| 182 |
from datetime import datetime, date
|
| 183 |
+
|
| 184 |
if not rows:
|
| 185 |
return 0
|
| 186 |
+
|
| 187 |
if not self.cursor:
|
| 188 |
raise RuntimeError("Not connected to database")
|
| 189 |
|
| 190 |
# Get column metadata for this table to know constraints
|
| 191 |
column_info = self._get_column_info(table_name)
|
| 192 |
+
if not column_info:
|
| 193 |
+
print(f"WARNING: column_info empty for {table_name} — DESCRIBE TABLE failed, falling back to DDL types")
|
| 194 |
+
|
| 195 |
# Filter out identity columns (let Snowflake auto-generate)
|
| 196 |
filtered_columns = []
|
| 197 |
filtered_indices = []
|
|
|
|
| 238 |
if col_upper in column_info:
|
| 239 |
info = column_info[col_upper]
|
| 240 |
data_type = info.get('type', '')
|
| 241 |
+
elif ddl_types and col_upper in ddl_types:
|
| 242 |
+
# Fallback: use DDL-declared type when DESCRIBE TABLE didn't return this column
|
| 243 |
+
info = {}
|
| 244 |
+
data_type = ddl_types[col_upper].upper()
|
| 245 |
|
| 246 |
# Truncate strings to fit VARCHAR length
|
| 247 |
if 'VARCHAR' in data_type or 'TEXT' in data_type:
|
|
|
|
| 486 |
|
| 487 |
if truncate_first:
|
| 488 |
writer.truncate_table(table.name)
|
| 489 |
+
|
| 490 |
+
# Build DDL type map as fallback for convert_value when DESCRIBE TABLE fails
|
| 491 |
+
ddl_types = {col.name.upper(): (col.data_type or '').upper()
|
| 492 |
+
for col in table.columns}
|
| 493 |
+
count = writer.insert_rows(table.name, columns, list_rows, ddl_types=ddl_types)
|
| 494 |
results[table.name] = count
|
| 495 |
|
| 496 |
return results
|