Spaces:
Running
Running
fix: align Snowflake loader identifiers with DDL compiler (fixes 'invalid identifier')
Browse filesCREATE TABLE normalized column/table names (upper + non-alnum->_) but the row
loader quoted them raw, so a non-uppercase-safe name (e.g. a Marketing funnel
measure 'Checkout Initiated') was created as CHECKOUT_INITIATED yet INSERTed as
"Checkout Initiated" -> Snowflake 'invalid identifier'. Extract one shared
safe_identifier() and use it in both paths. Names already UPPER_SNAKE are
unaffected (no regression).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
demoprep_app/ddl/from_dataset.py
CHANGED
|
@@ -5,6 +5,22 @@ from __future__ import annotations
|
|
| 5 |
from demoprep_app.dataset.contracts import DatasetBundle, DatasetColumn, DatasetTable
|
| 6 |
|
| 7 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
class DatasetDdlCompiler:
|
| 9 |
"""Derive simple Snowflake DDL from canonical dataset metadata."""
|
| 10 |
|
|
@@ -66,12 +82,7 @@ class DatasetDdlCompiler:
|
|
| 66 |
|
| 67 |
@staticmethod
|
| 68 |
def _safe_ident(identifier: str) -> str:
|
| 69 |
-
|
| 70 |
-
if not safe:
|
| 71 |
-
raise ValueError("Identifier cannot be empty")
|
| 72 |
-
if safe[0].isdigit():
|
| 73 |
-
safe = f"_{safe}"
|
| 74 |
-
return safe
|
| 75 |
|
| 76 |
@staticmethod
|
| 77 |
def _snowflake_type(data_type: str) -> str:
|
|
|
|
| 5 |
from demoprep_app.dataset.contracts import DatasetBundle, DatasetColumn, DatasetTable
|
| 6 |
|
| 7 |
|
| 8 |
+
def safe_identifier(identifier: str) -> str:
|
| 9 |
+
"""Normalize a name into the Snowflake identifier this pipeline creates.
|
| 10 |
+
|
| 11 |
+
Single source of truth: CREATE TABLE (this module) and the row loader
|
| 12 |
+
(integrations/snowflake/dataset_writer.py) MUST agree on this, or the
|
| 13 |
+
INSERT references a column name the table doesn't have and Snowflake
|
| 14 |
+
raises "invalid identifier". Uppercases and replaces any non-alphanumeric
|
| 15 |
+
character with '_' (so 'Checkout Initiated' -> 'CHECKOUT_INITIATED')."""
|
| 16 |
+
safe = "".join(ch if ch.isalnum() or ch == "_" else "_" for ch in identifier.strip().upper())
|
| 17 |
+
if not safe:
|
| 18 |
+
raise ValueError("Identifier cannot be empty")
|
| 19 |
+
if safe[0].isdigit():
|
| 20 |
+
safe = f"_{safe}"
|
| 21 |
+
return safe
|
| 22 |
+
|
| 23 |
+
|
| 24 |
class DatasetDdlCompiler:
|
| 25 |
"""Derive simple Snowflake DDL from canonical dataset metadata."""
|
| 26 |
|
|
|
|
| 82 |
|
| 83 |
@staticmethod
|
| 84 |
def _safe_ident(identifier: str) -> str:
|
| 85 |
+
return safe_identifier(identifier)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 86 |
|
| 87 |
@staticmethod
|
| 88 |
def _snowflake_type(data_type: str) -> str:
|
demoprep_app/integrations/snowflake/dataset_writer.py
CHANGED
|
@@ -51,5 +51,12 @@ def populate_dataset_bundle(
|
|
| 51 |
|
| 52 |
|
| 53 |
def _quote_ident(identifier: str) -> str:
|
| 54 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
return f'"{escaped}"'
|
|
|
|
| 51 |
|
| 52 |
|
| 53 |
def _quote_ident(identifier: str) -> str:
|
| 54 |
+
# Normalize the SAME way the DDL compiler does before quoting, so the
|
| 55 |
+
# INSERT targets the exact columns CREATE TABLE produced. A raw name like
|
| 56 |
+
# 'Checkout Initiated' is created as CHECKOUT_INITIATED by the DDL; quoting
|
| 57 |
+
# the raw name here would look for "Checkout Initiated" and Snowflake would
|
| 58 |
+
# raise "invalid identifier".
|
| 59 |
+
from demoprep_app.ddl.from_dataset import safe_identifier
|
| 60 |
+
|
| 61 |
+
escaped = safe_identifier(identifier).replace('"', '""')
|
| 62 |
return f'"{escaped}"'
|