mikeboone Claude Sonnet 4.6 commited on
Commit
2164672
·
1 Parent(s): d523f1c

fix: DDL type fallback in insert_rows when DESCRIBE TABLE returns empty

Browse files

If _get_column_info fails silently (returns {}), convert_value had no type
info and passed string values through unconverted — causing 22018 on every
row for any numeric column.

- Add ddl_types param to insert_rows: {COL_UPPER: DDL_TYPE} from LegitData schema
- convert_value uses ddl_types as fallback when column not in column_info
- Log warning when column_info is empty so future failures are visible
- Pass ddl_types from generate() at the call site

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Files changed (1) hide show
  1. legitdata_bridge.py +20 -7
legitdata_bridge.py CHANGED
@@ -172,20 +172,26 @@ class KeyPairSnowflakeWriter:
172
  self.connection = None
173
  print("Disconnected from Snowflake")
174
 
175
- def insert_rows(self, table_name: str, columns: list, rows: list, batch_size: int = 1000) -> int:
176
- """Insert rows into a Snowflake table."""
 
 
 
 
177
  from decimal import Decimal
178
  from datetime import datetime, date
179
-
180
  if not rows:
181
  return 0
182
-
183
  if not self.cursor:
184
  raise RuntimeError("Not connected to database")
185
 
186
  # Get column metadata for this table to know constraints
187
  column_info = self._get_column_info(table_name)
188
-
 
 
189
  # Filter out identity columns (let Snowflake auto-generate)
190
  filtered_columns = []
191
  filtered_indices = []
@@ -232,6 +238,10 @@ class KeyPairSnowflakeWriter:
232
  if col_upper in column_info:
233
  info = column_info[col_upper]
234
  data_type = info.get('type', '')
 
 
 
 
235
 
236
  # Truncate strings to fit VARCHAR length
237
  if 'VARCHAR' in data_type or 'TEXT' in data_type:
@@ -476,8 +486,11 @@ class DemoLegitGenerator:
476
 
477
  if truncate_first:
478
  writer.truncate_table(table.name)
479
-
480
- count = writer.insert_rows(table.name, columns, list_rows)
 
 
 
481
  results[table.name] = count
482
 
483
  return results
 
172
  self.connection = None
173
  print("Disconnected from Snowflake")
174
 
175
+ def insert_rows(self, table_name: str, columns: list, rows: list, batch_size: int = 1000, ddl_types: dict = None) -> int:
176
+ """Insert rows into a Snowflake table.
177
+
178
+ ddl_types: optional dict of {COLUMN_NAME_UPPER: DDL_DATA_TYPE} from the LegitData schema,
179
+ used as fallback when DESCRIBE TABLE (column_info) doesn't have a column.
180
+ """
181
  from decimal import Decimal
182
  from datetime import datetime, date
183
+
184
  if not rows:
185
  return 0
186
+
187
  if not self.cursor:
188
  raise RuntimeError("Not connected to database")
189
 
190
  # Get column metadata for this table to know constraints
191
  column_info = self._get_column_info(table_name)
192
+ if not column_info:
193
+ print(f"WARNING: column_info empty for {table_name} — DESCRIBE TABLE failed, falling back to DDL types")
194
+
195
  # Filter out identity columns (let Snowflake auto-generate)
196
  filtered_columns = []
197
  filtered_indices = []
 
238
  if col_upper in column_info:
239
  info = column_info[col_upper]
240
  data_type = info.get('type', '')
241
+ elif ddl_types and col_upper in ddl_types:
242
+ # Fallback: use DDL-declared type when DESCRIBE TABLE didn't return this column
243
+ info = {}
244
+ data_type = ddl_types[col_upper].upper()
245
 
246
  # Truncate strings to fit VARCHAR length
247
  if 'VARCHAR' in data_type or 'TEXT' in data_type:
 
486
 
487
  if truncate_first:
488
  writer.truncate_table(table.name)
489
+
490
+ # Build DDL type map as fallback for convert_value when DESCRIBE TABLE fails
491
+ ddl_types = {col.name.upper(): (col.data_type or '').upper()
492
+ for col in table.columns}
493
+ count = writer.insert_rows(table.name, columns, list_rows, ddl_types=ddl_types)
494
  results[table.name] = count
495
 
496
  return results