Commit ·
6d46ba5
1
Parent(s): f07e524
[NOTICKET] refactor(catalog): single SAMPLE_LIMIT constant shared across introspection paths
Browse filesHoist the per-column sample cap into catalog.introspect.base.SAMPLE_LIMIT so the
tabular and DB introspection paths reference one source of truth instead of two
separate literals (tabular head(3) + extractor SAMPLE_LIMIT=3) that can drift.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
src/catalog/introspect/base.py
CHANGED
|
@@ -9,6 +9,11 @@ from abc import ABC, abstractmethod
|
|
| 9 |
|
| 10 |
from ..models import Source
|
| 11 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
|
| 13 |
class BaseIntrospector(ABC):
|
| 14 |
"""Abstract base. Subclasses: DatabaseIntrospector, TabularIntrospector."""
|
|
|
|
| 9 |
|
| 10 |
from ..models import Source
|
| 11 |
|
| 12 |
+
# Max sample values stored per column (down from 5 — token cost: sample values
|
| 13 |
+
# are fed to the planner prompt). Single source of truth for every introspection
|
| 14 |
+
# path (tabular files + DB), so the cap can never drift between them.
|
| 15 |
+
SAMPLE_LIMIT = 3
|
| 16 |
+
|
| 17 |
|
| 18 |
class BaseIntrospector(ABC):
|
| 19 |
"""Abstract base. Subclasses: DatabaseIntrospector, TabularIntrospector."""
|
src/catalog/introspect/tabular.py
CHANGED
|
@@ -26,7 +26,7 @@ from src.middlewares.logging import get_logger
|
|
| 26 |
|
| 27 |
from ..models import Column, ColumnStats, DataType, Source, Table
|
| 28 |
from ..pii_detector import PIIDetector
|
| 29 |
-
from .base import BaseIntrospector
|
| 30 |
|
| 31 |
logger = get_logger("tabular_introspector")
|
| 32 |
|
|
@@ -198,7 +198,7 @@ class TabularIntrospector(BaseIntrospector):
|
|
| 198 |
(document_id, sheet_name, col_name) if sheet_name else (document_id, col_name)
|
| 199 |
)
|
| 200 |
|
| 201 |
-
sample_raw = series.dropna().head(
|
| 202 |
sample_values: list[Any] | None = [_normalize(v) for v in sample_raw] or None
|
| 203 |
|
| 204 |
is_numeric = pd.api.types.is_numeric_dtype(series)
|
|
|
|
| 26 |
|
| 27 |
from ..models import Column, ColumnStats, DataType, Source, Table
|
| 28 |
from ..pii_detector import PIIDetector
|
| 29 |
+
from .base import SAMPLE_LIMIT, BaseIntrospector
|
| 30 |
|
| 31 |
logger = get_logger("tabular_introspector")
|
| 32 |
|
|
|
|
| 198 |
(document_id, sheet_name, col_name) if sheet_name else (document_id, col_name)
|
| 199 |
)
|
| 200 |
|
| 201 |
+
sample_raw = series.dropna().head(SAMPLE_LIMIT).tolist()
|
| 202 |
sample_values: list[Any] | None = [_normalize(v) for v in sample_raw] or None
|
| 203 |
|
| 204 |
is_numeric = pd.api.types.is_numeric_dtype(series)
|
src/pipeline/db_pipeline/extractor.py
CHANGED
|
@@ -12,12 +12,14 @@ import pandas as pd
|
|
| 12 |
from sqlalchemy import Date, DateTime, Float, Integer, Numeric, inspect
|
| 13 |
from sqlalchemy.engine import Engine
|
| 14 |
|
|
|
|
| 15 |
from src.middlewares.logging import get_logger
|
| 16 |
|
| 17 |
logger = get_logger("db_extractor")
|
| 18 |
|
| 19 |
TOP_VALUES_THRESHOLD = 0.05 # show top values if distinct_ratio <= 5%
|
| 20 |
-
|
|
|
|
| 21 |
|
| 22 |
# Dialects with a single-statement CTE that survives `pd.read_sql`. On these we
|
| 23 |
# fold the stats and sample queries into one round-trip per column. MySQL <8 and
|
|
|
|
| 12 |
from sqlalchemy import Date, DateTime, Float, Integer, Numeric, inspect
|
| 13 |
from sqlalchemy.engine import Engine
|
| 14 |
|
| 15 |
+
from src.catalog.introspect.base import SAMPLE_LIMIT
|
| 16 |
from src.middlewares.logging import get_logger
|
| 17 |
|
| 18 |
logger = get_logger("db_extractor")
|
| 19 |
|
| 20 |
TOP_VALUES_THRESHOLD = 0.05 # show top values if distinct_ratio <= 5%
|
| 21 |
+
# SAMPLE_LIMIT (sample N rows per column) is the shared cap defined in
|
| 22 |
+
# catalog.introspect.base — one source of truth for both introspection paths.
|
| 23 |
|
| 24 |
# Dialects with a single-statement CTE that survives `pd.read_sql`. On these we
|
| 25 |
# fold the stats and sample queries into one round-trip per column. MySQL <8 and
|