[NOTICKET][DB] add endpoints for dbtypes (#6)

- [noticket] add gitignore (c87f27f458854787102eebc110b178c2572ef8a6)
- [NOTICKET]: add document pipeline, simplify document API (fb871f3d7db9c5a9c95a012761c4180a432e7e4b)
- [NOTICKET]: update folder document_pipelines after pipelines (a4cf97ab3890e35831c0043a1d411c17d84f374b)
- [NOTICKET][DB] refactor code to new repo (7f3bb9782bb9b6ace69e6be35377c12757a2f81d)
- [KM-441] add mean and median (9b5933420821187e73bb72a896e28d7adf6d6137)
- [NOTICKET] new metadata format for cleaner code (6b590d94c398ac4966a0e376f2809aa52c5ba022)
- update document (5a69e0eecf80dd807565dbc369374e41450c37fc)
- delete duplicate file (3848d7b262745b758069e3ab8089437ced42a45f)
- edit document for new pipeline (425e021053c4de65e2598b0f5554b7c1044a904c)
- [NOTICKET]: add CSV and XLSX file type (31920c3bc6f9faf9e07095a02fc1faf08adc246e)
- [DB] fix/rename db_pipeline.py (d913315c356d741cc7a5a5b1f8e36ba41397f2ab)
- [NOTICKET][DB] menyesuaikan format struktur db_pipeline sesuai dengan file lain (e13a9017e0f7fd26ef9b8ad5981d6c0799c01661)
- [NOTICKET][DB] pisahin db credential ke folder model. add ingestion endpoint at db_client to use db pipeline. add router db_client di main. (347a73aa5597cb28d1f43940eb60db619e4a10be)
- [NOTICKET]: use tesseract for extract PDF (6b9a13d417f6ffffed61a44567a3acbee822e173)
- [NOTICKET]: add Tesseract and Poppler binaries via Git LFS (0a9101a16c713c3facd8c06279da1f01403306a0)
- [NOTICKET]: update uv.lock (bb79f64b69c3bdcb2c98867c5d4945bf2311394d)
- [NOTICKET][DB] update credential & databaseclient. update settings (0e079550f8524da66df365638167dfe42b70ae62)
- [NOTICKET] update settings (65a5c6b1c1539de5d66e700aa5bef011e3c2b2f6)
- [KM-437][DB] add mysql, sqlserver, bigquery, snowflake connections (43539293eab9328d1d6f22f2f837c0a0085485a5)
- [NOTICKET]: adjusted pyproject.toml for OCR PDF (a00e2ad5306306ff73c30d085f126fab7bd1e78c)
- [NOTICKET]: fix merge conflict (6c8734607da68151ab574d0d186b17ac0d8ec460)
- [NOTICKET][DB] fix mysql pipeline (060c8cc81d4de2e85d2f6b4d3de178307e211ad0)
- [NOTICKET] edit imports (b145c06e354030dc38a550cd53030ba4864870ae)
- [NOTICKET] minor code refactor (52415b6a52f08900d27b548459a15c1113c1b892)
- [NOTICKET] add duplicate check for storing database (d310770ffcbcf75cb6b952f86bc6947ab2c7bb6a)
- [NOTICKET][DB] add supported dbtype for frontend (a531fcc7f53fe56a634ebba3b0fd4ff7ad51a33d)
- Merge branch 'dev_new' into pr/6 (14ec882dd774b73a44ba4521f35c78b04c8c3e73)


Co-authored-by: Sofhia Az-Zahra <sofhiaazzhr@users.noreply.huggingface.co>

src/api/v1/db_client.py

@@ -117,11 +117,122 @@ class DatabaseClientResponse(BaseModel):
     model_config = {"from_attributes": True}
 
 
+# ---------------------------------------------------------------------------
+# Supported DB types registry
+# ---------------------------------------------------------------------------
+
+_DB_TYPES: List[Dict[str, Any]] = [
+    {
+        "db_type": "postgres",
+        "display_name": "PostgreSQL",
+        "logo": "postgres",
+        "status": "active",
+        "message": None,
+        "fields": [
+            {"name": "host", "type": "string", "required": True, "default": None, "description": "Hostname or IP address"},
+            {"name": "port", "type": "integer", "required": False, "default": 5432, "description": "Port number"},
+            {"name": "database", "type": "string", "required": True, "default": None, "description": "Database name"},
+            {"name": "username", "type": "string", "required": True, "default": None, "description": "Database username"},
+            {"name": "password", "type": "string", "required": True, "default": None, "description": "Database password", "sensitive": True},
+            {"name": "ssl_mode", "type": "select", "required": False, "default": "require", "description": "SSL mode", "options": ["disable", "require", "verify-ca", "verify-full"]},
+        ],
+    },
+    {
+        "db_type": "mysql",
+        "display_name": "MySQL",
+        "logo": "mysql",
+        "status": "active",
+        "message": None,
+        "fields": [
+            {"name": "host", "type": "string", "required": True, "default": None, "description": "Hostname or IP address"},
+            {"name": "port", "type": "integer", "required": False, "default": 3306, "description": "Port number"},
+            {"name": "database", "type": "string", "required": True, "default": None, "description": "Database name"},
+            {"name": "username", "type": "string", "required": True, "default": None, "description": "Database username"},
+            {"name": "password", "type": "string", "required": True, "default": None, "description": "Database password", "sensitive": True},
+            {"name": "ssl", "type": "boolean", "required": False, "default": True, "description": "Enable SSL"},
+        ],
+    },
+    {
+        "db_type": "supabase",
+        "display_name": "Supabase",
+        "logo": "supabase",
+        "status": "active",
+        "message": None,
+        "fields": [
+            {"name": "host", "type": "string", "required": True, "default": None, "description": "Supabase database host"},
+            {"name": "port", "type": "integer", "required": False, "default": 5432, "description": "Port number (5432 direct, 6543 pooler)"},
+            {"name": "database", "type": "string", "required": False, "default": "postgres", "description": "Database name"},
+            {"name": "username", "type": "string", "required": True, "default": None, "description": "Database user"},
+            {"name": "password", "type": "string", "required": True, "default": None, "description": "Database password", "sensitive": True},
+            {"name": "ssl_mode", "type": "select", "required": False, "default": "require", "description": "SSL mode", "options": ["require", "verify-ca", "verify-full"]},
+        ],
+    },
+    {
+        "db_type": "sqlserver",
+        "display_name": "SQL Server",
+        "logo": "sqlserver",
+        "status": "inactive",
+        "message": "Coming soon",
+        "fields": [
+            {"name": "host", "type": "string", "required": True, "default": None, "description": "Hostname or IP address"},
+            {"name": "port", "type": "integer", "required": False, "default": 1433, "description": "Port number"},
+            {"name": "database", "type": "string", "required": True, "default": None, "description": "Database name"},
+            {"name": "username", "type": "string", "required": True, "default": None, "description": "Database username"},
+            {"name": "password", "type": "string", "required": True, "default": None, "description": "Database password", "sensitive": True},
+            {"name": "driver", "type": "string", "required": False, "default": None, "description": "ODBC driver name"},
+        ],
+    },
+    {
+        "db_type": "bigquery",
+        "display_name": "BigQuery",
+        "logo": "bigquery",
+        "status": "inactive",
+        "message": "Coming soon",
+        "fields": [
+            {"name": "project_id", "type": "string", "required": True, "default": None, "description": "GCP project ID"},
+            {"name": "dataset_id", "type": "string", "required": True, "default": None, "description": "BigQuery dataset name"},
+            {"name": "location", "type": "string", "required": False, "default": "US", "description": "Dataset location/region"},
+            {"name": "service_account_json", "type": "string", "required": True, "default": None, "description": "GCP Service Account key JSON", "sensitive": True},
+        ],
+    },
+    {
+        "db_type": "snowflake",
+        "display_name": "Snowflake",
+        "logo": "snowflake",
+        "status": "inactive",
+        "message": "Coming soon",
+        "fields": [
+            {"name": "account", "type": "string", "required": True, "default": None, "description": "Snowflake account identifier"},
+            {"name": "warehouse", "type": "string", "required": True, "default": None, "description": "Virtual warehouse name"},
+            {"name": "database", "type": "string", "required": True, "default": None, "description": "Database name"},
+            {"name": "schema", "type": "string", "required": False, "default": "PUBLIC", "description": "Schema name"},
+            {"name": "username", "type": "string", "required": True, "default": None, "description": "Snowflake username"},
+            {"name": "password", "type": "string", "required": True, "default": None, "description": "Snowflake password", "sensitive": True},
+            {"name": "role", "type": "string", "required": False, "default": None, "description": "Snowflake role"},
+        ],
+    },
+]
+
+
 # ---------------------------------------------------------------------------
 # Endpoints
 # ---------------------------------------------------------------------------
 
 
+@router.get(
+    "/database-clients/dbtypes",
+    summary="List supported database types",
+    response_description="All database types supported by DataEyond with their connection parameters.",
+)
+async def list_db_types():
+    """
+    Return every database type DataEyond can connect to, along with the
+    credential fields the frontend should render, a logo filename, and
+    an active/inactive status with an optional message.
+    """
+    return _DB_TYPES
+
+
 @router.post(
     "/database-clients",
     response_model=DatabaseClientResponse,
@@ -335,9 +446,14 @@ async def ingest_database_client(
     if client.user_id != user_id:
         raise HTTPException(status_code=403, detail="Access denied")
 
-    creds = decrypt_credentials_dict(client.credentials)
+    if client.status != "active":
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT,
+            detail="Cannot ingest from an inactive database connection.",
+        )
 
     try:
+        creds = decrypt_credentials_dict(client.credentials)
         with db_pipeline_service.engine_scope(
             db_type=client.db_type,
             credentials=creds,

src/config/settings.py

@@ -63,7 +63,7 @@ class Settings(BaseSettings):
 
     # DB credential encryption (Fernet key for user-registered database creds)
     dataeyond_db_credential_key: str = Field(
-        alias="dataeyond__db__credential__key", default=""
+        alias="dataeyond__db__credential__key"
     )
 
 

src/database_client/database_client_service.py

@@ -16,9 +16,46 @@ from src.utils.db_credential_encryption import (
 logger = get_logger("database_client_service")
 
 
+    # Fields that identify the same physical database per db_type.
+_CONNECTION_IDENTITY_KEYS: dict[str, tuple[str, ...]] = {
+    "postgres": ("host", "port", "database"),
+    "supabase": ("host", "port", "database"),
+    "mysql": ("host", "port", "database"),
+    "sqlserver": ("host", "port", "database"),
+    "bigquery": ("project_id", "dataset_id"),
+    "snowflake": ("account", "warehouse", "database"),
+}
+
+
 class DatabaseClientService:
     """Service for managing user-registered external database connections."""
 
+    async def _find_duplicate(
+        self,
+        db: AsyncSession,
+        user_id: str,
+        db_type: str,
+        credentials: dict,
+    ) -> Optional[DatabaseClient]:
+        """Return an existing client if it points to the same physical database."""
+        identity_keys = _CONNECTION_IDENTITY_KEYS.get(db_type, ())
+        if not identity_keys:
+            return None
+
+        result = await db.execute(
+            select(DatabaseClient).where(
+                DatabaseClient.user_id == user_id,
+                DatabaseClient.db_type == db_type,
+            )
+        )
+        for existing in result.scalars().all():
+            decrypted = decrypt_credentials_dict(existing.credentials)
+            if all(
+                decrypted.get(k) == credentials.get(k) for k in identity_keys
+            ):
+                return existing
+        return None
+
     async def create(
         self,
         db: AsyncSession,
@@ -29,8 +66,17 @@ class DatabaseClientService:
     ) -> DatabaseClient:
         """Register a new database client connection.
 
+        If a connection to the same physical database already exists for this
+        user, the existing record is returned instead of creating a duplicate.
         Credentials are encrypted before being stored.
         """
+        existing = await self._find_duplicate(db, user_id, db_type, credentials)
+        if existing:
+            logger.info(
+                f"Duplicate connection detected, returning existing client {existing.id}"
+            )
+            return existing
+
         client = DatabaseClient(
             id=str(uuid.uuid4()),
             user_id=user_id,

src/pipeline/db_pipeline/db_pipeline_service.py

@@ -128,7 +128,7 @@ class DbPipelineService:
             )
             return create_engine(url)
 
-        raise ValueError(f"Unsupported db_type: {db_type}")
+        raise NotImplementedError(f"Unsupported db_type: {db_type}")
 
     @contextmanager
     def engine_scope(

src/pipeline/db_pipeline/extractor.py

@@ -149,10 +149,10 @@ def profile_column(
             order_by="ORDER BY cnt DESC",
         )
         top = pd.read_sql(top_sql, engine)
-        profile["top_values"] = list(zip(top[col_name].tolist(), top["cnt"].tolist()))
+        profile["top_values"] = list(zip(top.iloc[:, 0].tolist(), top["cnt"].tolist()))
 
     sample = pd.read_sql(_head_query(engine, qc, qt, 5), engine)
-    profile["sample_values"] = sample[col_name].tolist()
+    profile["sample_values"] = sample.iloc[:, 0].tolist()
 
     return profile