sofhiaazzhr Claude Opus 4.7 commited on
Commit
6d46ba5
·
1 Parent(s): f07e524

[NOTICKET] refactor(catalog): single SAMPLE_LIMIT constant shared across introspection paths

Browse files

Hoist the per-column sample cap into catalog.introspect.base.SAMPLE_LIMIT so the
tabular and DB introspection paths reference one source of truth instead of two
separate literals (tabular head(3) + extractor SAMPLE_LIMIT=3) that can drift.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

src/catalog/introspect/base.py CHANGED
@@ -9,6 +9,11 @@ from abc import ABC, abstractmethod
9
 
10
  from ..models import Source
11
 
 
 
 
 
 
12
 
13
  class BaseIntrospector(ABC):
14
  """Abstract base. Subclasses: DatabaseIntrospector, TabularIntrospector."""
 
9
 
10
  from ..models import Source
11
 
12
+ # Max sample values stored per column (down from 5 — token cost: sample values
13
+ # are fed to the planner prompt). Single source of truth for every introspection
14
+ # path (tabular files + DB), so the cap can never drift between them.
15
+ SAMPLE_LIMIT = 3
16
+
17
 
18
  class BaseIntrospector(ABC):
19
  """Abstract base. Subclasses: DatabaseIntrospector, TabularIntrospector."""
src/catalog/introspect/tabular.py CHANGED
@@ -26,7 +26,7 @@ from src.middlewares.logging import get_logger
26
 
27
  from ..models import Column, ColumnStats, DataType, Source, Table
28
  from ..pii_detector import PIIDetector
29
- from .base import BaseIntrospector
30
 
31
  logger = get_logger("tabular_introspector")
32
 
@@ -198,7 +198,7 @@ class TabularIntrospector(BaseIntrospector):
198
  (document_id, sheet_name, col_name) if sheet_name else (document_id, col_name)
199
  )
200
 
201
- sample_raw = series.dropna().head(3).tolist()
202
  sample_values: list[Any] | None = [_normalize(v) for v in sample_raw] or None
203
 
204
  is_numeric = pd.api.types.is_numeric_dtype(series)
 
26
 
27
  from ..models import Column, ColumnStats, DataType, Source, Table
28
  from ..pii_detector import PIIDetector
29
+ from .base import SAMPLE_LIMIT, BaseIntrospector
30
 
31
  logger = get_logger("tabular_introspector")
32
 
 
198
  (document_id, sheet_name, col_name) if sheet_name else (document_id, col_name)
199
  )
200
 
201
+ sample_raw = series.dropna().head(SAMPLE_LIMIT).tolist()
202
  sample_values: list[Any] | None = [_normalize(v) for v in sample_raw] or None
203
 
204
  is_numeric = pd.api.types.is_numeric_dtype(series)
src/pipeline/db_pipeline/extractor.py CHANGED
@@ -12,12 +12,14 @@ import pandas as pd
12
  from sqlalchemy import Date, DateTime, Float, Integer, Numeric, inspect
13
  from sqlalchemy.engine import Engine
14
 
 
15
  from src.middlewares.logging import get_logger
16
 
17
  logger = get_logger("db_extractor")
18
 
19
  TOP_VALUES_THRESHOLD = 0.05 # show top values if distinct_ratio <= 5%
20
- SAMPLE_LIMIT = 3 # sample N rows per column (down from 5 token cost)
 
21
 
22
  # Dialects with a single-statement CTE that survives `pd.read_sql`. On these we
23
  # fold the stats and sample queries into one round-trip per column. MySQL <8 and
 
12
  from sqlalchemy import Date, DateTime, Float, Integer, Numeric, inspect
13
  from sqlalchemy.engine import Engine
14
 
15
+ from src.catalog.introspect.base import SAMPLE_LIMIT
16
  from src.middlewares.logging import get_logger
17
 
18
  logger = get_logger("db_extractor")
19
 
20
  TOP_VALUES_THRESHOLD = 0.05 # show top values if distinct_ratio <= 5%
21
+ # SAMPLE_LIMIT (sample N rows per column) is the shared cap defined in
22
+ # catalog.introspect.base — one source of truth for both introspection paths.
23
 
24
  # Dialects with a single-statement CTE that survives `pd.read_sql`. On these we
25
  # fold the stats and sample queries into one round-trip per column. MySQL <8 and