initial structure

scripts/build_initial_catalogs.py

@@ -0,0 +1,16 @@
+"""Backfill catalogs for existing users.
+
+One-off script. For each user that already has registered DB connections or
+uploaded tabular files, run the structured pipeline to build their catalog.
+
+Usage:
+    uv run python scripts/build_initial_catalogs.py [--user-id USER_ID]
+"""
+
+
+def main() -> None:
+    raise NotImplementedError
+
+
+if __name__ == "__main__":
+    main()

scripts/enrich_all_sources.py

@@ -0,0 +1,16 @@
+"""Bulk re-run CatalogEnricher with the current prompt.
+
+For when src/config/prompts/catalog_enricher.md changes and existing
+catalog descriptions need to be regenerated across all users.
+
+Usage:
+    uv run python scripts/enrich_all_sources.py [--user-id USER_ID]
+"""
+
+
+def main() -> None:
+    raise NotImplementedError
+
+
+if __name__ == "__main__":
+    main()

src/agents/intent_router.py

@@ -0,0 +1,24 @@
+"""IntentRouter — classifies a user message and emits source_hint.
+
+Output: needs_search (bool) + source_hint ∈ { chat, unstructured, structured }.
+
+Replaces the previous orchestration.py once the chat endpoint is rewired.
+"""
+
+from dataclasses import dataclass
+from typing import Literal
+
+
+SourceHint = Literal["chat", "unstructured", "structured"]
+
+
+@dataclass
+class IntentRouterDecision:
+    needs_search: bool
+    source_hint: SourceHint
+    rewritten_query: str | None = None
+
+
+class IntentRouter:
+    async def classify(self, message: str) -> IntentRouterDecision:
+        raise NotImplementedError

src/catalog/enricher.py

@@ -0,0 +1,16 @@
+"""CatalogEnricher — runs 1 LLM call per source to generate AI descriptions.
+
+Input: a draft Source produced by an introspector (raw schema, no descriptions).
+Output: the same Source enriched with description fields at source/table/column level.
+
+Prompt: src/config/prompts/catalog_enricher.md
+"""
+
+from .models import Source
+
+
+class CatalogEnricher:
+    """Adds AI-generated descriptions to a freshly introspected source."""
+
+    async def enrich(self, source: Source) -> Source:
+        raise NotImplementedError

src/catalog/introspect/base.py

@@ -0,0 +1,17 @@
+"""BaseIntrospector — contract for source-specific schema readers.
+
+Subclasses produce a draft Source object with raw schema (names + types +
+sample values). The CatalogEnricher then adds descriptions in a separate step.
+"""
+
+from abc import ABC, abstractmethod
+
+from ..models import Source
+
+
+class BaseIntrospector(ABC):
+    """Abstract base. Subclasses: DatabaseIntrospector, TabularIntrospector."""
+
+    @abstractmethod
+    async def introspect(self, location_ref: str) -> Source:
+        ...

src/catalog/introspect/database.py

@@ -0,0 +1,16 @@
+"""Database schema introspection (Postgres / MySQL / Supabase).
+
+Reads information_schema for tables/columns/types, samples ~100 rows per table
+for `sample_values` and basic stats. Does NOT generate descriptions
+(that happens in CatalogEnricher).
+"""
+
+from ..models import Source
+from .base import BaseIntrospector
+
+
+class DatabaseIntrospector(BaseIntrospector):
+    """Connect to user DB → read information_schema → sample 100 rows/table."""
+
+    async def introspect(self, location_ref: str) -> Source:
+        raise NotImplementedError

src/catalog/introspect/tabular.py

@@ -0,0 +1,15 @@
+"""Tabular file schema introspection (Parquet / CSV / XLSX).
+
+Reads file headers + samples ~100 rows. For XLSX, each sheet becomes a Table.
+Files are expected to live in Azure Blob (location_ref like az_blob://...).
+"""
+
+from ..models import Source
+from .base import BaseIntrospector
+
+
+class TabularIntrospector(BaseIntrospector):
+    """Read column names, dtypes, and sample values from Parquet/CSV/XLSX."""
+
+    async def introspect(self, location_ref: str) -> Source:
+        raise NotImplementedError

src/catalog/models.py

@@ -0,0 +1,55 @@
+"""Pydantic models for the per-user data catalog (Cs + Ct).
+
+See ARCHITECTURE.md §6 for the full schema definition.
+"""
+
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+
+SourceType = Literal["schema", "tabular", "unstructured"]
+DataType = Literal["int", "decimal", "string", "datetime", "date", "bool", "json"]
+
+
+class ColumnStats(BaseModel):
+    min: Any | None = None
+    max: Any | None = None
+    distinct_count: int | None = None
+
+
+class Column(BaseModel):
+    column_id: str
+    name: str
+    data_type: DataType
+    description: str
+    nullable: bool
+    pii_flag: bool = False
+    sample_values: list[Any] | None = None
+    stats: ColumnStats | None = None
+
+
+class Table(BaseModel):
+    table_id: str
+    name: str
+    description: str
+    row_count: int | None = None
+    columns: list[Column]
+
+
+class Source(BaseModel):
+    source_id: str
+    source_type: SourceType
+    name: str
+    description: str
+    location_ref: str
+    updated_at: datetime
+    tables: list[Table] = Field(default_factory=list)
+
+
+class Catalog(BaseModel):
+    user_id: str
+    schema_version: str = "1.0"
+    generated_at: datetime
+    sources: list[Source] = Field(default_factory=list)

src/catalog/pii_detector.py

@@ -0,0 +1,16 @@
+"""PII auto-detection for catalog columns.
+
+When pii_flag is set True, sample_values is forced to None so real PII
+never enters LLM prompts.
+
+Patterns live in src/security/pii_patterns.py.
+"""
+
+from .models import Column
+
+
+class PIIDetector:
+    """Marks columns as pii_flag=True when name/values look sensitive."""
+
+    def detect(self, column: Column) -> bool:
+        raise NotImplementedError

src/catalog/reader.py

@@ -0,0 +1,23 @@
+"""CatalogReader — loads + filters catalog by source_hint.
+
+For typical users (≤50 tables), returns the FULL catalog with no slicing.
+Catalog-level search is added later if catalog grows past the limit.
+"""
+
+from typing import Literal
+
+from .models import Catalog
+from .store import CatalogStore
+
+
+SourceHint = Literal["chat", "unstructured", "structured"]
+
+
+class CatalogReader:
+    """Loads the user's catalog and filters by source_hint."""
+
+    def __init__(self, store: CatalogStore) -> None:
+        self._store = store
+
+    async def read(self, user_id: str, source_hint: SourceHint) -> Catalog:
+        raise NotImplementedError

src/catalog/store.py

@@ -0,0 +1,20 @@
+"""CatalogStore — persists per-user catalogs as Postgres jsonb rows.
+
+Storage shape: one row per user in a `catalogs` table with columns
+(user_id PK, data jsonb, schema_version, generated_at, updated_at).
+"""
+
+from .models import Catalog
+
+
+class CatalogStore:
+    """Read/write catalogs keyed by user_id."""
+
+    async def get(self, user_id: str) -> Catalog | None:
+        raise NotImplementedError
+
+    async def upsert(self, catalog: Catalog) -> None:
+        raise NotImplementedError
+
+    async def delete(self, user_id: str) -> None:
+        raise NotImplementedError

src/catalog/validator.py

@@ -0,0 +1,24 @@
+"""CatalogValidator — Pydantic + business-rule validation for a catalog.
+
+Pydantic handles shape; this layer adds invariants that span fields.
+"""
+
+from .models import Catalog
+
+
+class CatalogValidationError(Exception):
+    pass
+
+
+class CatalogValidator:
+    """Validates a Catalog beyond Pydantic schema checks.
+
+    Business rules:
+    - All source_ids unique within a user
+    - All table_ids unique within a source
+    - All column_ids unique within a table
+    - foreign_keys (when added) reference existing tables/columns
+    """
+
+    def validate(self, catalog: Catalog) -> None:
+        raise NotImplementedError

src/config/prompts/catalog_enricher.md

@@ -0,0 +1,21 @@
+# Catalog Enricher Prompt
+
+Takes raw introspected schema (table names, column names, types, sample values)
+and produces AI-generated descriptions for the source, each table, and each column.
+
+One LLM call per source at ingestion time. Used by `src/catalog/enricher.py`.
+
+## System prompt
+
+(to be written)
+
+## Output schema
+
+Strict JSON matching the catalog Pydantic models in `src/catalog/models.py`.
+Validated by `CatalogValidator` before being persisted.
+
+## Notes
+
+- Descriptions should be one-line and factual.
+- Do NOT invent column meanings beyond what the sample values support.
+- For `pii_flag` columns, do NOT include sample values in the description.

src/config/prompts/chatbot_system.md

@@ -0,0 +1,17 @@
+# Chatbot System Prompt
+
+Final answer-formation step. Receives the user question, retrieval context (Cu)
+or query results (from QueryExecutor), and produces the natural-language answer.
+
+Used by `src/agents/chatbot.py`. SSE-streamed back to the user.
+
+## System prompt
+
+(content to be ported from src/config/agents/system_prompt.md, then rewritten
+to remove references to the old retrieval pipeline and reflect catalog-driven flow)
+
+## Notes
+
+- Keep answers grounded in the provided context — no hallucination.
+- For tabular results, format numbers with appropriate units (from `column.unit` when available).
+- Cite sources when applicable (filename + page label).

src/config/prompts/guardrails.md

@@ -0,0 +1,12 @@
+# Guardrails
+
+Safety / refusal / scope-bounding rules applied to all LLM calls.
+
+(content to be ported from src/config/agents/guardrails_prompt.md)
+
+## Scope
+
+- Refuse PII extraction requests
+- Refuse questions outside the user's data scope
+- Refuse code-execution / shell-style requests
+- (more to be added)

src/config/prompts/intent_router.md

@@ -0,0 +1,25 @@
+# Intent Router Prompt
+
+Classifies a user message into:
+- `needs_search`: bool
+- `source_hint`: `chat` | `unstructured` | `structured`
+
+Used by `src/agents/intent_router.py`.
+
+## System prompt
+
+(to be written)
+
+## Output schema
+
+```json
+{
+  "needs_search": true,
+  "source_hint": "structured",
+  "rewritten_query": "..."
+}
+```
+
+## Few-shot examples
+
+(to be written)

src/config/prompts/query_planner.md

@@ -0,0 +1,25 @@
+# Query Planner Prompt
+
+Takes a user question + their data catalog (Cs ∪ Ct).
+Produces a JSON IR that describes the query intent.
+
+See ARCHITECTURE.md §7 for the IR schema. Used by `src/query/planner/service.py`.
+
+## System prompt
+
+(to be written)
+
+## Output schema
+
+Strict JSON matching `src/query/ir/models.py:QueryIR`.
+Validated by `IRValidator` against the catalog before reaching the compiler.
+
+## Few-shot examples
+
+(to be written — 5–10 examples covering filter + groupby + agg + sort + limit)
+
+## Notes
+
+- Reference columns by `column_id`, not `name`.
+- `value_type` must match the column's `data_type`.
+- Only emit operators/aggs from the whitelist (`src/query/ir/operators.py`).

src/models/api/__init__.py

@@ -0,0 +1 @@
+"""API request/response shapes per route family."""

src/models/api/catalog.py

@@ -0,0 +1,14 @@
+"""Request / response models for catalog-related routes (e.g. /knowledge/rebuild)."""
+
+from pydantic import BaseModel
+
+
+class CatalogRebuildRequest(BaseModel):
+    user_id: str
+
+
+class CatalogRebuildResponse(BaseModel):
+    user_id: str
+    sources_enriched: int
+    tables_enriched: int
+    columns_enriched: int

src/models/api/chat.py

@@ -0,0 +1,17 @@
+"""Request / response models for /api/v1/chat/* routes."""
+
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class ChatRequest(BaseModel):
+    user_id: str
+    room_id: str
+    message: str
+
+
+class ChatStreamEvent(BaseModel):
+    """One SSE event. Type values: `sources`, `chunk`, `done`."""
+    event: str
+    data: dict[str, Any]

src/models/api/document.py

@@ -0,0 +1,9 @@
+"""Request / response models for /api/v1/documents/* routes."""
+
+from pydantic import BaseModel
+
+
+class DocumentUploadResponse(BaseModel):
+    document_id: str
+    filename: str
+    status: str  # uploaded | processing | completed | failed

src/pipeline/document_pipeline.py

@@ -0,0 +1,11 @@
+"""DocumentPipeline — extract text, chunk, embed, ingest to PGVector.
+
+For unstructured sources (PDF / DOCX / TXT). Receives the working
+implementation from the previous pipeline/document_pipeline/document_pipeline.py
+during the cleanup phase.
+"""
+
+
+class DocumentPipeline:
+    async def run(self, document_id: str, user_id: str) -> None:
+        raise NotImplementedError

src/pipeline/orchestrator.py

@@ -0,0 +1,11 @@
+"""IngestionOrchestrator — top-level coordinator for ingestion.
+
+Routes uploads / DB connections to the right pipeline:
+- unstructured (pdf/docx/txt) → DocumentPipeline
+- schema or tabular            → StructuredPipeline (which writes to the catalog)
+"""
+
+
+class IngestionOrchestrator:
+    async def ingest(self, source_ref: str, source_type: str, user_id: str) -> None:
+        raise NotImplementedError

src/pipeline/structured_pipeline.py

@@ -0,0 +1,13 @@
+"""StructuredPipeline — runs catalog enrichment for DB / tabular sources.
+
+Steps:
+1. introspect (catalog/introspect/database.py or catalog/introspect/tabular.py)
+2. enrich    (catalog/enricher.py — 1 LLM call per source)
+3. validate  (catalog/validator.py)
+4. write     (catalog/store.py)
+"""
+
+
+class StructuredPipeline:
+    async def run(self, source_ref: str, source_type: str, user_id: str) -> None:
+        raise NotImplementedError

src/pipeline/triggers.py

@@ -0,0 +1,21 @@
+"""Pipeline trigger entry points called from API routes / event handlers.
+
+These thin functions are what the FastAPI routes invoke; they delegate
+to IngestionOrchestrator.
+"""
+
+
+async def on_document_uploaded(document_id: str, user_id: str) -> None:
+    raise NotImplementedError
+
+
+async def on_db_registered(database_client_id: str, user_id: str) -> None:
+    raise NotImplementedError
+
+
+async def on_tabular_uploaded(document_id: str, user_id: str) -> None:
+    raise NotImplementedError
+
+
+async def on_catalog_rebuild_requested(user_id: str) -> None:
+    raise NotImplementedError

src/query/compiler/base.py

@@ -0,0 +1,13 @@
+"""BaseCompiler — contract for IR → executable shape (SQL string or pandas chain)."""
+
+from abc import ABC, abstractmethod
+
+from ..ir.models import QueryIR
+
+
+class BaseCompiler(ABC):
+    """Subclasses: SqlCompiler, PandasCompiler."""
+
+    @abstractmethod
+    def compile(self, ir: QueryIR) -> object:
+        ...

src/query/compiler/pandas.py

@@ -0,0 +1,22 @@
+"""PandasCompiler — IR → callable that runs against a DataFrame.
+
+For tabular sources. The callable encapsulates the chain of operations
+(filter → groupby → agg → sort → limit) so the executor can apply them
+to a DataFrame loaded eagerly or via predicate pushdown / polars lazy scan.
+"""
+
+from typing import Callable
+
+from ...catalog.models import Catalog
+from ..ir.models import QueryIR
+from .base import BaseCompiler
+
+
+class PandasCompiler(BaseCompiler):
+    """Deterministic IR → pandas/polars op chain. No LLM."""
+
+    def __init__(self, catalog: Catalog) -> None:
+        self._catalog = catalog
+
+    def compile(self, ir: QueryIR) -> Callable:
+        raise NotImplementedError

src/query/compiler/sql.py

@@ -0,0 +1,28 @@
+"""SqlCompiler — IR → (SQL string, parameters list).
+
+Identifiers (table, column names) come from the catalog (trusted).
+Values come from IR.filters and are ALWAYS parameterized — never inlined.
+Output is validated by sqlglot before reaching the executor.
+"""
+
+from dataclasses import dataclass
+
+from ...catalog.models import Catalog
+from ..ir.models import QueryIR
+from .base import BaseCompiler
+
+
+@dataclass
+class CompiledSql:
+    sql: str
+    params: list[object]
+
+
+class SqlCompiler(BaseCompiler):
+    """Deterministic IR → SQL. No LLM."""
+
+    def __init__(self, catalog: Catalog) -> None:
+        self._catalog = catalog
+
+    def compile(self, ir: QueryIR) -> CompiledSql:
+        raise NotImplementedError

src/query/executor/base.py

@@ -0,0 +1,26 @@
+"""BaseExecutor + QueryResult — uniform return shape across DB and tabular paths."""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any
+
+from ..ir.models import QueryIR
+
+
+@dataclass
+class QueryResult:
+    source_id: str
+    backend: str  # "sql" | "tabular"
+    rows: list[dict[str, Any]] = field(default_factory=list)
+    row_count: int = 0
+    truncated: bool = False
+    elapsed_ms: int = 0
+    error: str | None = None
+
+
+class BaseExecutor(ABC):
+    """Subclasses: DbExecutor, TabularExecutor."""
+
+    @abstractmethod
+    async def run(self, ir: QueryIR) -> QueryResult:
+        ...

src/query/executor/db.py

@@ -0,0 +1,25 @@
+"""DbExecutor — runs compiled SQL on a user's external DB.
+
+Pipeline:
+  IR → SqlCompiler → SQL string + params
+       ↓
+  sqlglot validation (SELECT-only, whitelist tables/columns, LIMIT enforced)
+       ↓
+  asyncpg / pymysql in read-only transaction with timeout (30s)
+       ↓
+  QueryResult
+"""
+
+from ...catalog.models import Catalog
+from ..compiler.sql import SqlCompiler
+from ..ir.models import QueryIR
+from .base import BaseExecutor, QueryResult
+
+
+class DbExecutor(BaseExecutor):
+    def __init__(self, catalog: Catalog) -> None:
+        self._catalog = catalog
+        self._compiler = SqlCompiler(catalog)
+
+    async def run(self, ir: QueryIR) -> QueryResult:
+        raise NotImplementedError

src/query/executor/dispatcher.py

@@ -0,0 +1,17 @@
+"""Picks DB vs Tabular executor based on the source_type of the IR's source.
+
+This is the only place in the structured query path where the schema/tabular
+distinction matters. Every step before this is source-type-agnostic.
+"""
+
+from ...catalog.models import Catalog
+from ..ir.models import QueryIR
+from .base import BaseExecutor
+
+
+class ExecutorDispatcher:
+    def __init__(self, catalog: Catalog) -> None:
+        self._catalog = catalog
+
+    def pick(self, ir: QueryIR) -> BaseExecutor:
+        raise NotImplementedError

src/query/executor/tabular.py

@@ -0,0 +1,24 @@
+"""TabularExecutor — runs compiled pandas/polars chain on a Parquet file.
+
+Picks engine by file size:
+  ≤ 100 MB    → eager pandas
+  100 MB-1 GB → pyarrow with predicate pushdown
+  > 1 GB      → polars lazy scan
+
+Initial scope ships eager pandas only; the others are added when a real
+file is too big.
+"""
+
+from ...catalog.models import Catalog
+from ..compiler.pandas import PandasCompiler
+from ..ir.models import QueryIR
+from .base import BaseExecutor, QueryResult
+
+
+class TabularExecutor(BaseExecutor):
+    def __init__(self, catalog: Catalog) -> None:
+        self._catalog = catalog
+        self._compiler = PandasCompiler(catalog)
+
+    async def run(self, ir: QueryIR) -> QueryResult:
+        raise NotImplementedError

src/query/ir/models.py

@@ -0,0 +1,60 @@
+"""JSON IR (intermediate representation) Pydantic models.
+
+See ARCHITECTURE.md §7 for the schema.
+
+Initial scope: single-table; filter, group_by, agg, order_by, limit.
+Joins, having, offset, boolean tree filters are deferred to later versions.
+"""
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+
+FilterOp = Literal[
+    "=", "!=", "<", "<=", ">", ">=",
+    "in", "not_in", "is_null", "is_not_null",
+    "like", "between",
+]
+AggFn = Literal["count", "count_distinct", "sum", "avg", "min", "max"]
+ValueType = Literal["int", "decimal", "string", "datetime", "date", "bool"]
+SortDir = Literal["asc", "desc"]
+
+
+class ColumnSelect(BaseModel):
+    kind: Literal["column"] = "column"
+    column_id: str
+    alias: str | None = None
+
+
+class AggSelect(BaseModel):
+    kind: Literal["agg"] = "agg"
+    fn: AggFn
+    column_id: str | None = None
+    alias: str | None = None
+
+
+SelectItem = ColumnSelect | AggSelect
+
+
+class FilterClause(BaseModel):
+    column_id: str
+    op: FilterOp
+    value: Any
+    value_type: ValueType
+
+
+class OrderByClause(BaseModel):
+    column_id: str
+    dir: SortDir = "asc"
+
+
+class QueryIR(BaseModel):
+    ir_version: str = "1.0"
+    source_id: str
+    table_id: str
+    select: list[SelectItem]
+    filters: list[FilterClause] = Field(default_factory=list)
+    group_by: list[str] = Field(default_factory=list)
+    order_by: list[OrderByClause] = Field(default_factory=list)
+    limit: int | None = None

src/query/ir/operators.py

@@ -0,0 +1,17 @@
+"""Whitelisted operators + aggregation functions for IR validation."""
+
+ALLOWED_FILTER_OPS = frozenset({
+    "=", "!=", "<", "<=", ">", ">=",
+    "in", "not_in", "is_null", "is_not_null",
+    "like", "between",
+})
+
+ALLOWED_AGG_FNS = frozenset({
+    "count", "count_distinct", "sum", "avg", "min", "max",
+})
+
+LIMIT_HARD_CAP = 10_000
+
+# Type compatibility: which value_types are valid for each column data_type.
+# To be filled with the explicit matrix when validator.py is implemented.
+TYPE_COMPATIBILITY: dict[str, frozenset[str]] = {}

src/query/ir/validator.py

@@ -0,0 +1,28 @@
+"""IRValidator — checks a QueryIR against a user's catalog.
+
+See ARCHITECTURE.md §7 for the validation rules.
+On failure, the planner is re-prompted with the error context (max 3 retries).
+"""
+
+from ...catalog.models import Catalog
+from .models import QueryIR
+
+
+class IRValidationError(Exception):
+    pass
+
+
+class IRValidator:
+    """Reject IRs that reference unknown sources/tables/columns or use disallowed ops.
+
+    Rules:
+    - source_id exists in catalog for this user
+    - table_id belongs to that source
+    - every column_id exists in that table
+    - every agg.fn and filter.op is whitelisted (see operators.py)
+    - value_type consistent with column.data_type
+    - limit positive int, ≤ hard cap
+    """
+
+    def validate(self, ir: QueryIR, catalog: Catalog) -> None:
+        raise NotImplementedError

src/query/planner/prompt.py

@@ -0,0 +1,12 @@
+"""Builds the planner LLM prompt from question + catalog.
+
+Renders the catalog into a compact textual form that fits the LLM context
+window. For users with ≤50 tables the full catalog goes in verbatim.
+"""
+
+from ...catalog.models import Catalog
+
+
+def build_planner_prompt(question: str, catalog: Catalog) -> str:
+    """Return the full prompt string to feed the planner LLM."""
+    raise NotImplementedError

src/query/planner/service.py

@@ -0,0 +1,15 @@
+"""QueryPlannerService — single LLM call: question + catalog → JSON IR.
+
+Prompt: src/config/prompts/query_planner.md
+Output: a QueryIR ready for the IRValidator.
+"""
+
+from ...catalog.models import Catalog
+from ..ir.models import QueryIR
+
+
+class QueryPlannerService:
+    """Wraps the LLM call with structured-output parsing into QueryIR."""
+
+    async def plan(self, question: str, catalog: Catalog) -> QueryIR:
+        raise NotImplementedError

src/query/service.py

@@ -0,0 +1,15 @@
+"""QueryService — orchestrates plan → validate → compile → execute.
+
+Top-level entry point for catalog-driven structured queries. Wired into
+the chat endpoint when source_hint == "structured".
+"""
+
+from ..catalog.models import Catalog
+from .executor.base import QueryResult
+
+
+class QueryService:
+    """End-to-end runner for a user question against a catalog."""
+
+    async def run(self, user_id: str, question: str, catalog: Catalog) -> QueryResult:
+        raise NotImplementedError

src/retrieval/document.py

@@ -0,0 +1,15 @@
+"""DocumentRetriever — dense similarity over prose chunks (Cu).
+
+For unstructured sources only (PDF / DOCX / TXT). Backed by PGVector with
+collection `document_embeddings`. Methods: MMR, cosine, euclidean, etc.
+
+Receives the working implementation from the previous src/rag/retrievers/document.py
+during the cleanup phase; for now this is a placeholder.
+"""
+
+
+class DocumentRetriever:
+    """Dense retrieval over PGVector chunks for unstructured sources."""
+
+    async def retrieve(self, query: str, user_id: str, k: int = 5) -> list:
+        raise NotImplementedError

src/retrieval/router.py

@@ -0,0 +1,11 @@
+"""Retrieval-side router.
+
+Currently dispatches only the `unstructured` route to DocumentRetriever.
+The `structured` route is owned by query/service.py — not by retrieval.
+The `chat` route bypasses retrieval entirely.
+"""
+
+
+class RetrievalRouter:
+    async def dispatch(self, query: str, user_id: str, source_hint: str) -> list:
+        raise NotImplementedError

src/security/auth.py

@@ -0,0 +1,21 @@
+"""Authentication helpers: password hashing, JWT encode/decode, get_user.
+
+Receives the working implementation from the previous src/users/users.py
+during the cleanup phase.
+"""
+
+
+def hash_password(plaintext: str) -> str:
+    raise NotImplementedError
+
+
+def verify_password(plaintext: str, hashed: str) -> bool:
+    raise NotImplementedError
+
+
+def encode_jwt(payload: dict) -> str:
+    raise NotImplementedError
+
+
+def decode_jwt(token: str) -> dict:
+    raise NotImplementedError

src/security/credentials.py

@@ -0,0 +1,15 @@
+"""Fernet-encrypted credential storage for user-registered DB connections.
+
+Receives the working implementation from the previous
+src/utils/db_credential_encryption.py during the cleanup phase.
+
+Key: settings.dataeyond__db__credential__key (Fernet, kept out of source).
+"""
+
+
+def encrypt_credential(plaintext: str) -> str:
+    raise NotImplementedError
+
+
+def decrypt_credential(ciphertext: str) -> str:
+    raise NotImplementedError

src/security/pii_patterns.py

@@ -0,0 +1,20 @@
+"""Regex patterns and column-name heuristics for PII detection.
+
+Used by catalog/pii_detector.py at ingestion time. Default policy:
+when in doubt, set pii_flag=True. False positives cost nothing; false
+negatives leak data.
+"""
+
+import re
+
+PII_NAME_PATTERNS = frozenset({
+    "email",
+    "phone", "mobile", "telp", "telephone",
+    "ssn", "tin", "passport", "ktp", "nik",
+    "name", "fullname", "first_name", "last_name", "surname",
+    "address", "street", "zipcode", "postal",
+    "birthdate", "dob", "birthday",
+})
+
+EMAIL_REGEX = re.compile(r"^[\w.+-]+@[\w-]+\.[\w.-]+$")
+PHONE_REGEX = re.compile(r"^\+?[\d\s\-()]{7,}$")