Merge branch 'dev_new' into pr/6

src/api/v1/db_client.py

@@ -117,11 +117,122 @@ class DatabaseClientResponse(BaseModel):
     model_config = {"from_attributes": True}
 
 
+# ---------------------------------------------------------------------------
+# Supported DB types registry
+# ---------------------------------------------------------------------------
+
+_DB_TYPES: List[Dict[str, Any]] = [
+    {
+        "db_type": "postgres",
+        "display_name": "PostgreSQL",
+        "logo": "postgres",
+        "status": "active",
+        "message": None,
+        "fields": [
+            {"name": "host", "type": "string", "required": True, "default": None, "description": "Hostname or IP address"},
+            {"name": "port", "type": "integer", "required": False, "default": 5432, "description": "Port number"},
+            {"name": "database", "type": "string", "required": True, "default": None, "description": "Database name"},
+            {"name": "username", "type": "string", "required": True, "default": None, "description": "Database username"},
+            {"name": "password", "type": "string", "required": True, "default": None, "description": "Database password", "sensitive": True},
+            {"name": "ssl_mode", "type": "select", "required": False, "default": "require", "description": "SSL mode", "options": ["disable", "require", "verify-ca", "verify-full"]},
+        ],
+    },
+    {
+        "db_type": "mysql",
+        "display_name": "MySQL",
+        "logo": "mysql",
+        "status": "active",
+        "message": None,
+        "fields": [
+            {"name": "host", "type": "string", "required": True, "default": None, "description": "Hostname or IP address"},
+            {"name": "port", "type": "integer", "required": False, "default": 3306, "description": "Port number"},
+            {"name": "database", "type": "string", "required": True, "default": None, "description": "Database name"},
+            {"name": "username", "type": "string", "required": True, "default": None, "description": "Database username"},
+            {"name": "password", "type": "string", "required": True, "default": None, "description": "Database password", "sensitive": True},
+            {"name": "ssl", "type": "boolean", "required": False, "default": True, "description": "Enable SSL"},
+        ],
+    },
+    {
+        "db_type": "supabase",
+        "display_name": "Supabase",
+        "logo": "supabase",
+        "status": "active",
+        "message": None,
+        "fields": [
+            {"name": "host", "type": "string", "required": True, "default": None, "description": "Supabase database host"},
+            {"name": "port", "type": "integer", "required": False, "default": 5432, "description": "Port number (5432 direct, 6543 pooler)"},
+            {"name": "database", "type": "string", "required": False, "default": "postgres", "description": "Database name"},
+            {"name": "username", "type": "string", "required": True, "default": None, "description": "Database user"},
+            {"name": "password", "type": "string", "required": True, "default": None, "description": "Database password", "sensitive": True},
+            {"name": "ssl_mode", "type": "select", "required": False, "default": "require", "description": "SSL mode", "options": ["require", "verify-ca", "verify-full"]},
+        ],
+    },
+    {
+        "db_type": "sqlserver",
+        "display_name": "SQL Server",
+        "logo": "sqlserver",
+        "status": "inactive",
+        "message": "Coming soon",
+        "fields": [
+            {"name": "host", "type": "string", "required": True, "default": None, "description": "Hostname or IP address"},
+            {"name": "port", "type": "integer", "required": False, "default": 1433, "description": "Port number"},
+            {"name": "database", "type": "string", "required": True, "default": None, "description": "Database name"},
+            {"name": "username", "type": "string", "required": True, "default": None, "description": "Database username"},
+            {"name": "password", "type": "string", "required": True, "default": None, "description": "Database password", "sensitive": True},
+            {"name": "driver", "type": "string", "required": False, "default": None, "description": "ODBC driver name"},
+        ],
+    },
+    {
+        "db_type": "bigquery",
+        "display_name": "BigQuery",
+        "logo": "bigquery",
+        "status": "inactive",
+        "message": "Coming soon",
+        "fields": [
+            {"name": "project_id", "type": "string", "required": True, "default": None, "description": "GCP project ID"},
+            {"name": "dataset_id", "type": "string", "required": True, "default": None, "description": "BigQuery dataset name"},
+            {"name": "location", "type": "string", "required": False, "default": "US", "description": "Dataset location/region"},
+            {"name": "service_account_json", "type": "string", "required": True, "default": None, "description": "GCP Service Account key JSON", "sensitive": True},
+        ],
+    },
+    {
+        "db_type": "snowflake",
+        "display_name": "Snowflake",
+        "logo": "snowflake",
+        "status": "inactive",
+        "message": "Coming soon",
+        "fields": [
+            {"name": "account", "type": "string", "required": True, "default": None, "description": "Snowflake account identifier"},
+            {"name": "warehouse", "type": "string", "required": True, "default": None, "description": "Virtual warehouse name"},
+            {"name": "database", "type": "string", "required": True, "default": None, "description": "Database name"},
+            {"name": "schema", "type": "string", "required": False, "default": "PUBLIC", "description": "Schema name"},
+            {"name": "username", "type": "string", "required": True, "default": None, "description": "Snowflake username"},
+            {"name": "password", "type": "string", "required": True, "default": None, "description": "Snowflake password", "sensitive": True},
+            {"name": "role", "type": "string", "required": False, "default": None, "description": "Snowflake role"},
+        ],
+    },
+]
+
+
 # ---------------------------------------------------------------------------
 # Endpoints
 # ---------------------------------------------------------------------------
 
 
+@router.get(
+    "/database-clients/dbtypes",
+    summary="List supported database types",
+    response_description="All database types supported by DataEyond with their connection parameters.",
+)
+async def list_db_types():
+    """
+    Return every database type DataEyond can connect to, along with the
+    credential fields the frontend should render, a logo filename, and
+    an active/inactive status with an optional message.
+    """
+    return _DB_TYPES
+
+
 @router.post(
     "/database-clients",
     response_model=DatabaseClientResponse,
@@ -335,9 +446,14 @@ async def ingest_database_client(
     if client.user_id != user_id:
         raise HTTPException(status_code=403, detail="Access denied")
 
-    creds = decrypt_credentials_dict(client.credentials)
+    if client.status != "active":
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT,
+            detail="Cannot ingest from an inactive database connection.",
+        )
 
     try:
+        creds = decrypt_credentials_dict(client.credentials)
         with db_pipeline_service.engine_scope(
             db_type=client.db_type,
             credentials=creds,

src/config/settings.py

@@ -63,7 +63,7 @@ class Settings(BaseSettings):
 
     # DB credential encryption (Fernet key for user-registered database creds)
     dataeyond_db_credential_key: str = Field(
-        alias="dataeyond__db__credential__key", default=""
+        alias="dataeyond__db__credential__key"
     )
 
 

src/database_client/database_client_service.py

@@ -16,9 +16,46 @@ from src.utils.db_credential_encryption import (
 logger = get_logger("database_client_service")
 
 
+    # Fields that identify the same physical database per db_type.
+_CONNECTION_IDENTITY_KEYS: dict[str, tuple[str, ...]] = {
+    "postgres": ("host", "port", "database"),
+    "supabase": ("host", "port", "database"),
+    "mysql": ("host", "port", "database"),
+    "sqlserver": ("host", "port", "database"),
+    "bigquery": ("project_id", "dataset_id"),
+    "snowflake": ("account", "warehouse", "database"),
+}
+
+
 class DatabaseClientService:
     """Service for managing user-registered external database connections."""
 
+    async def _find_duplicate(
+        self,
+        db: AsyncSession,
+        user_id: str,
+        db_type: str,
+        credentials: dict,
+    ) -> Optional[DatabaseClient]:
+        """Return an existing client if it points to the same physical database."""
+        identity_keys = _CONNECTION_IDENTITY_KEYS.get(db_type, ())
+        if not identity_keys:
+            return None
+
+        result = await db.execute(
+            select(DatabaseClient).where(
+                DatabaseClient.user_id == user_id,
+                DatabaseClient.db_type == db_type,
+            )
+        )
+        for existing in result.scalars().all():
+            decrypted = decrypt_credentials_dict(existing.credentials)
+            if all(
+                decrypted.get(k) == credentials.get(k) for k in identity_keys
+            ):
+                return existing
+        return None
+
     async def create(
         self,
         db: AsyncSession,
@@ -29,8 +66,17 @@ class DatabaseClientService:
     ) -> DatabaseClient:
         """Register a new database client connection.
 
+        If a connection to the same physical database already exists for this
+        user, the existing record is returned instead of creating a duplicate.
         Credentials are encrypted before being stored.
         """
+        existing = await self._find_duplicate(db, user_id, db_type, credentials)
+        if existing:
+            logger.info(
+                f"Duplicate connection detected, returning existing client {existing.id}"
+            )
+            return existing
+
         client = DatabaseClient(
             id=str(uuid.uuid4()),
             user_id=user_id,

src/pipeline/db_pipeline/db_pipeline_service.py

@@ -128,7 +128,7 @@ class DbPipelineService:
             )
             return create_engine(url)
 
-        raise ValueError(f"Unsupported db_type: {db_type}")
+        raise NotImplementedError(f"Unsupported db_type: {db_type}")
 
     @contextmanager
     def engine_scope(

src/pipeline/db_pipeline/extractor.py

@@ -149,10 +149,10 @@ def profile_column(
             order_by="ORDER BY cnt DESC",
         )
         top = pd.read_sql(top_sql, engine)
-        profile["top_values"] = list(zip(top[col_name].tolist(), top["cnt"].tolist()))
+        profile["top_values"] = list(zip(top.iloc[:, 0].tolist(), top["cnt"].tolist()))
 
     sample = pd.read_sql(_head_query(engine, qc, qt, 5), engine)
-    profile["sample_values"] = sample[col_name].tolist()
+    profile["sample_values"] = sample.iloc[:, 0].tolist()
 
     return profile