1st try

.env

@@ -6,8 +6,8 @@ DOTENV_PUBLIC_KEY="02432e6a4895a1e967061d0588df908fba1730139bdf599d3021034d4035a
 
 # dotenvx encrypt -ek "SURREAL_*"
 # .env
-GOOGLE_API_KEY=encrypted:BEpFDkAjzT+MXhXpTo0MtlI+PCdMT2xc+qTQQGmHQifUWx4RxNG8ylRvCCB3jFe4fejMhBMBsDXrpt9OiurXDVvoStO4jl6QuVq1TfWHRrqbBYFCVRRlZnr7mx+MCwkhp8sWHWg=
-GROQ_API_KEY=encrypted:BGb7PzEU1NUTXUjCslpPqu9mCgwCO7G5cf2B4jp5AciPl2t6lwEhKvFLknRGzLVPu7RHoVN7Y+D78IhgJf3mDnuk9Oiz6b9KkDzXOpRjmQhO22GVnqPM8WtAgqBQx6BY5FGAaik=
+GOOGLE_API_KEY=AIzaSyAFlllmL6s0l-SKezU6sZRQ7ZzNQODgQro
+GROQ_API_KEY=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
 
 SURREAL_URL=ws://127.0.0.1:8000/rpc
 SURREAL_NAMESPACE=open_notebook

Dockerfile

@@ -0,0 +1,60 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Set PYTHONPATH to include /app
+ENV PYTHONPATH=/app
+
+# Set Hugging Face cache directories (writable in HF Spaces)
+ENV HF_HOME=/tmp
+ENV TRANSFORMERS_CACHE=/tmp
+ENV SENTENCE_TRANSFORMERS_HOME=/tmp
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    curl \
+    build-essential && \    
+    curl -sSf https://install.surrealdb.com | sh && \
+    curl -fsS https://dotenvx.sh | sh && \
+    rm -rf /var/lib/apt/lists/* 
+
+# Copy requirements.txt for dependency installation
+COPY requirements.txt ./
+
+# Install Python dependencies from requirements.txt
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+# Explicitly ensure surreal-commands is installed (belt-and-suspenders approach)
+# RUN pip install --no-cache-dir surreal-commands>=1.2.0
+# requirements.txt
+
+# Pre-download sentence-transformers model at build time
+# This will be cached in the Docker image
+RUN python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('all-MiniLM-L6-v2')"
+
+# Copy application code
+COPY api/ ./open_notebook/ ./commands/ ./migrations/ ./prompts/ run_api.py start.sh ./
+
+# Make start script executable
+RUN chmod +x start.sh
+
+# Set environment variables for SurrealDB connection
+ENV SURREAL_URL=ws://localhost:8000/rpc
+ENV SURREAL_ADDRESS=localhost
+ENV SURREAL_PORT=8000
+ENV SURREAL_USER=root
+ENV SURREAL_PASS=root
+ENV SURREAL_NAMESPACE=open_notebook
+ENV SURREAL_DATABASE=main
+
+# Set API configuration for Hugging Face Spaces
+ENV API_HOST=0.0.0.0
+ENV API_PORT=7860
+ENV API_RELOAD=false
+
+# Expose Hugging Face Spaces port
+EXPOSE 7860
+
+# Run the start script
+CMD ["dotenvx", "./start.sh"]

api/CLAUDE.md

@@ -0,0 +1,260 @@
+# API Module
+
+FastAPI-based REST backend exposing services for notebooks, sources, notes, chat, podcasts, and AI model management.
+
+## Purpose
+
+FastAPI application serving three architectural layers: routes (HTTP endpoints), services (business logic), and models (request/response schemas). Integrates LangGraph workflows (chat, ask, source_chat), SurrealDB persistence, and AI providers via Esperanto.
+
+## Architecture Overview
+
+**Three layers**:
+1. **Routes** (`routers/*`): HTTP endpoints mapping to services
+2. **Services** (`*_service.py`): Business logic orchestrating domain models, database, graphs, AI providers
+3. **Models** (`models.py`): Pydantic request/response schemas with validation
+
+**Startup flow**:
+- Load .env environment variables
+- Initialize CORS middleware + password auth middleware
+- Run database migrations via AsyncMigrationManager on lifespan startup
+- Run podcast profile data migration (legacy string to model registry conversion)
+- Register all routers
+
+**Key services**:
+- `chat_service.py`: Invokes chat graph with messages, context
+- `podcast_service.py`: Orchestrates outline + transcript generation
+- `sources_service.py`: Content ingestion, vectorization, metadata
+- `notes_service.py`: Note creation, linking to sources/insights
+- `transformations_service.py`: Applies transformations to content
+- `models_service.py`: Manages AI provider/model configuration
+- `episode_profiles_service.py`: Manages podcast speaker/episode profiles
+
+## Component Catalog
+
+### Main Application
+- **main.py**: FastAPI app initialization, CORS setup, auth middleware, lifespan event, router registration
+- **Lifespan handler**: Runs AsyncMigrationManager on startup (database schema migration)
+- **Auth middleware**: PasswordAuthMiddleware protects endpoints (password-based access control)
+
+### Services (Business Logic)
+- **chat_service.py**: Invokes chat.py graph; handles message history via SqliteSaver
+- **podcast_service.py**: Generates outline (outline.jinja), then transcript (transcript.jinja) for episodes
+- **sources_service.py**: Ingests files/URLs (content_core), extracts text, vectorizes, saves to SurrealDB
+- **transformations_service.py**: Applies transformations via transformation.py graph
+- **models_service.py**: Manages ModelManager config (AI provider overrides)
+- **episode_profiles_service.py**: CRUD for EpisodeProfile and SpeakerProfile models
+- **insights_service.py**: Generates and retrieves source insights
+- **notes_service.py**: Creates notes linked to sources/insights
+
+### Models (Schemas)
+- **models.py**: Pydantic schemas for request/response validation
+- Request bodies: ChatRequest, CreateNoteRequest, PodcastGenerationRequest, etc.
+- Response bodies: ChatResponse, NoteResponse, PodcastResponse, etc.
+- Custom validators for enum fields, file paths, model references
+
+### Routers
+- **routers/chat.py**: POST /chat
+- **routers/source_chat.py**: POST /source/{source_id}/chat
+- **routers/podcasts.py**: POST /podcasts, GET /podcasts/{id}, POST /podcasts/episodes/{id}/retry, etc.
+- **routers/notes.py**: POST /notes, GET /notes/{id}
+- **routers/sources.py**: POST /sources, GET /sources/{id}, DELETE /sources/{id}
+- **routers/models.py**: GET /models, POST /models/config
+- **routers/credentials.py**: CRUD + test + discover + migrate for credential management
+- **routers/transformations.py**: POST /transformations
+- **routers/insights.py**: GET /sources/{source_id}/insights
+- **routers/auth.py**: POST /auth/password (password-based auth)
+- **routers/languages.py**: GET /languages (available podcast languages via pycountry+babel)
+- **routers/commands.py**: GET /commands/{command_id} (job status tracking)
+
+## Common Patterns
+
+- **Service injection via FastAPI**: Routers import services directly; no DI framework
+- **Async/await throughout**: All DB queries, graph invocations, AI calls are async
+- **SurrealDB transactions**: Services use repo_query, repo_create, repo_upsert from database layer
+- **Config override pattern**: Models/config override via models_service passed to graph.ainvoke(config=...)
+- **Error handling**: Custom exception hierarchy (`open_notebook.exceptions`) with global FastAPI exception handlers mapping to HTTP status codes (see Error Handling section below). LangGraph nodes use `classify_error()` to convert raw LLM provider errors into typed exceptions with user-friendly messages.
+- **Logging**: loguru logger in main.py; services expected to log key operations
+- **Response normalization**: All responses follow standard schema (data + metadata structure)
+
+## Key Dependencies
+
+- `fastapi`: FastAPI app, routers, HTTPException
+- `pydantic`: Validation models with Field, field_validator
+- `open_notebook.graphs`: chat, ask, source_chat, source, transformation graphs
+- `open_notebook.database`: SurrealDB repository functions (repo_query, repo_create, repo_upsert)
+- `open_notebook.domain`: Notebook, Source, Note, SourceInsight models
+- `open_notebook.ai.provision`: provision_langchain_model() factory
+- `ai_prompter`: Prompter for template rendering
+- `content_core`: extract_content() for file/URL processing
+- `esperanto`: AI provider client library (LLM, embeddings, TTS)
+- `surreal_commands`: Job queue for async operations (podcast generation)
+- `loguru`: Structured logging
+
+## Important Quirks & Gotchas
+
+- **Migration auto-run**: Database schema migrations run on every API startup (via lifespan); no manual migration steps
+- **PasswordAuthMiddleware is basic**: Uses simple password check; production deployments should replace with OAuth/JWT
+- **No request rate limiting**: No built-in rate limiting; deployment must add via proxy/middleware
+- **Service state is stateless**: Services don't cache results; each request re-queries database/AI models
+- **Graph invocation is blocking**: chat/podcast workflows may take minutes; no timeout handling in services
+- **Command job fire-and-forget**: podcast_service.py submits jobs but doesn't wait (async job queue pattern)
+- **Model override scoping**: Model config override via RunnableConfig is per-request only (not persistent)
+- **CORS open by default**: main.py CORS settings allow all origins (restrict before production)
+- **No OpenAPI security scheme**: API docs available without auth (disable before production)
+- **Services don't validate user permission**: All endpoints trust authentication layer; no per-notebook permission checks
+
+## Error Handling
+
+### Global Exception Handlers (`main.py`)
+
+FastAPI exception handlers map custom exception types from `open_notebook.exceptions` to HTTP status codes. All error responses include CORS headers.
+
+| Exception Class | HTTP Status | Use Case |
+|----------------|-------------|----------|
+| `NotFoundError` | 404 | Resource not found |
+| `InvalidInputError` | 400 | Bad request data |
+| `AuthenticationError` | 401 | Invalid/missing API key |
+| `RateLimitError` | 429 | Provider rate limit exceeded |
+| `ConfigurationError` | 422 | Wrong model name, missing config |
+| `NetworkError` | 502 | Cannot reach AI provider |
+| `ExternalServiceError` | 502 | Provider returned error (500/503, context length) |
+| `OpenNotebookError` (base) | 500 | Any other application error |
+
+### Error Classification (`open_notebook.utils.error_classifier`)
+
+The `classify_error()` function maps raw exceptions from LLM providers/Esperanto/LangChain into the typed exceptions above with user-friendly messages. Used in all LangGraph graph nodes and SSE streaming handlers.
+
+**Flow**: Raw exception → keyword matching → `(ExceptionClass, user_message)` → raised → caught by global handler → HTTP response with descriptive message.
+
+### Frontend Integration
+
+The frontend `getApiErrorMessage()` helper (`lib/utils/error-handler.ts`) tries i18n mapping first, then falls back to displaying the backend's descriptive error message directly.
+
+---
+
+## How to Add New Endpoint
+
+1. Create router file in `routers/` (e.g., `routers/new_feature.py`)
+2. Import router into `main.py` and register: `app.include_router(new_feature.router, tags=["new_feature"])`
+3. Create service in `new_feature_service.py` with business logic
+4. Define request/response schemas in `models.py` (or create `new_feature_models.py`)
+5. Implement router functions calling service methods
+6. Test with `uv run uvicorn api.main:app --host 0.0.0.0 --port 5055`
+
+## Testing Patterns
+
+- **Interactive docs**: http://localhost:5055/docs (Swagger UI)
+- **Direct service tests**: Import service, call methods directly with test data
+- **Mock graphs**: Replace graph.ainvoke() with mock for testing service logic
+- **Database: Use test database** (separate SurrealDB instance or mock repo_query)
+
+---
+
+## Credential Management (API Configuration UI)
+
+The Credential Management system enables users to configure AI provider credentials through the UI instead of environment variables. Keys are stored securely in SurrealDB (encrypted via Fernet) with database-first fallback to environment variables.
+
+### Router: `routers/credentials.py`
+
+**Endpoints**:
+
+| Method | Endpoint | Description |
+|--------|----------|-------------|
+| GET | `/credentials` | List all credentials (optional `?provider=` filter) |
+| GET | `/credentials/by-provider/{provider}` | List credentials for a provider |
+| POST | `/credentials` | Create a new credential |
+| GET | `/credentials/{credential_id}` | Get a specific credential |
+| PUT | `/credentials/{credential_id}` | Update a credential |
+| DELETE | `/credentials/{credential_id}` | Delete a credential |
+| POST | `/credentials/{credential_id}/test` | Test connection using credential |
+| POST | `/credentials/{credential_id}/discover` | Discover available models |
+| POST | `/credentials/{credential_id}/register-models` | Register discovered models |
+| POST | `/credentials/migrate-from-provider-config` | Migrate from legacy ProviderConfig |
+
+**Supported Providers** (13 total):
+- Simple API key: `openai`, `anthropic`, `google`, `groq`, `mistral`, `deepseek`, `xai`, `openrouter`, `voyage`, `elevenlabs`
+- URL-based: `ollama`
+- Multi-field: `azure`, `vertex`, `openai_compatible`
+
+**Security Features**:
+- NEVER returns actual API key values (only metadata)
+- URL validation (SSRF protection) on all URL fields via `_validate_url()`
+- Allows private IPs and localhost for self-hosted services (Ollama, LM Studio)
+- Requires `OPEN_NOTEBOOK_ENCRYPTION_KEY` to be set for storing credentials
+
+### Domain Model: `Credential` (`open_notebook/domain/credential.py`)
+
+Individual credential records replacing the old `ProviderConfig` singleton. Each credential stores:
+- Provider name, display name, modalities
+- Encrypted API key (via Fernet)
+- Provider-specific config (base_url, endpoint, api_version, etc.)
+
+### Integration with Key Provider (`open_notebook/ai/key_provider.py`)
+
+The `key_provider` module provisions DB-stored credentials into environment variables for Esperanto compatibility:
+
+**Database-first Pattern**:
+1. API endpoint saves keys to `Credential` records (encrypted in SurrealDB)
+2. Before model provisioning, `provision_provider_keys(provider)` checks DB, then env vars
+3. Keys from DB are set as environment variables for Esperanto compatibility
+4. Existing env vars remain unchanged if no DB config exists
+
+**Key Functions**:
+- `get_api_key(provider)`: Get API key (DB first, env fallback)
+- `provision_provider_keys(provider)`: Set env vars from DB for a provider
+- `provision_all_keys()`: Load all provider keys from DB into env vars
+
+### Authentication
+
+No changes to authentication. The `credentials` router uses the same `PasswordAuthMiddleware` as all other endpoints. Keys are protected by the same password-based auth.
+
+**Auth Flow** (unchanged from `api/auth.py`):
+- `PasswordAuthMiddleware`: Global middleware checking `Authorization: Bearer {password}` header
+- Default password: `open-notebook-change-me` (set `OPEN_NOTEBOOK_PASSWORD` in production)
+- Docker secrets support via `OPEN_NOTEBOOK_PASSWORD_FILE`
+
+### Connection Testing (`open_notebook/ai/connection_tester.py`)
+
+The `/credentials/{credential_id}/test` endpoint uses minimal API calls to verify credentials:
+- Loads Credential via `Credential.get(config_id)`, uses `credential.to_esperanto_config()`
+- Uses cheapest/smallest models per provider (TEST_MODELS map)
+- Returns success status and descriptive message
+- Special handlers for ollama, openai_compatible, and azure providers
+
+### Migration Workflows
+
+Two migration endpoints help users transition to the credential system:
+
+**From environment variables** (`POST /credentials/migrate-from-env`):
+1. Checks each provider for env var presence
+2. Creates Credential records from env var values
+3. Returns summary: migrated, skipped, errors
+
+**From legacy ProviderConfig** (`POST /credentials/migrate-from-provider-config`):
+1. Reads old ProviderConfig records from database
+2. Converts each to individual Credential records
+3. Returns summary: migrated, skipped, errors
+
+### Example Usage
+
+```python
+# Check status
+GET /credentials/status
+# Response: {"configured": {"openai": true, "anthropic": false}, "source": {"openai": "database", "anthropic": "none"}, "encryption_configured": true}
+
+# Create credential
+POST /credentials
+{"name": "My OpenAI Key", "provider": "openai", "modalities": ["language", "embedding"], "api_key": "sk-proj-..."}
+
+# Test connection
+POST /credentials/{credential_id}/test
+# Response: {"provider": "openai", "success": true, "message": "Connection successful"}
+
+# Discover models
+POST /credentials/{credential_id}/discover
+# Response: {"provider": "openai", "models": [{"model_id": "gpt-4", "name": "gpt-4", ...}], "credential_id": "..."}
+
+# Migrate from env
+POST /credentials/migrate-from-env
+# Response: {"message": "Migration complete. Migrated 3 providers.", "migrated": ["openai", "anthropic", "groq"], "skipped": [], "errors": []}
+```

api/auth.py

@@ -0,0 +1,114 @@
+from typing import Optional
+
+from fastapi import Depends, HTTPException, Request
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+from loguru import logger
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.responses import JSONResponse
+
+from open_notebook.utils.encryption import get_secret_from_env
+
+
+class PasswordAuthMiddleware(BaseHTTPMiddleware):
+    """
+    Middleware to check password authentication for all API requests.
+    Always active with default password if OPEN_NOTEBOOK_PASSWORD is not set.
+    Supports Docker secrets via OPEN_NOTEBOOK_PASSWORD_FILE.
+    """
+
+    def __init__(self, app, excluded_paths: Optional[list] = None):
+        super().__init__(app)
+        self.password = get_secret_from_env("OPEN_NOTEBOOK_PASSWORD")
+        self.excluded_paths = excluded_paths or [
+            "/",
+            "/health",
+            "/docs",
+            "/openapi.json",
+            "/redoc",
+        ]
+
+    async def dispatch(self, request: Request, call_next):
+        # Skip authentication if no password is set
+        if not self.password:
+            return await call_next(request)
+
+        # Skip authentication for excluded paths
+        if request.url.path in self.excluded_paths:
+            return await call_next(request)
+
+        # Skip authentication for CORS preflight requests (OPTIONS)
+        if request.method == "OPTIONS":
+            return await call_next(request)
+
+        # Check authorization header
+        auth_header = request.headers.get("Authorization")
+
+        if not auth_header:
+            return JSONResponse(
+                status_code=401,
+                content={"detail": "Missing authorization header"},
+                headers={"WWW-Authenticate": "Bearer"},
+            )
+
+        # Expected format: "Bearer {password}"
+        try:
+            scheme, credentials = auth_header.split(" ", 1)
+            if scheme.lower() != "bearer":
+                raise ValueError("Invalid authentication scheme")
+        except ValueError:
+            return JSONResponse(
+                status_code=401,
+                content={"detail": "Invalid authorization header format"},
+                headers={"WWW-Authenticate": "Bearer"},
+            )
+
+        # Check password
+        if credentials != self.password:
+            return JSONResponse(
+                status_code=401,
+                content={"detail": "Invalid password"},
+                headers={"WWW-Authenticate": "Bearer"},
+            )
+
+        # Password is correct, proceed with the request
+        response = await call_next(request)
+        return response
+
+
+# Optional: HTTPBearer security scheme for OpenAPI documentation
+security = HTTPBearer(auto_error=False)
+
+
+def check_api_password(
+    credentials: Optional[HTTPAuthorizationCredentials] = Depends(security),
+) -> bool:
+    """
+    Utility function to check API password.
+    Can be used as a dependency in individual routes if needed.
+    Supports Docker secrets via OPEN_NOTEBOOK_PASSWORD_FILE.
+    Returns True without checking credentials if OPEN_NOTEBOOK_PASSWORD is not configured.
+    Raises 401 if credentials are missing or don't match the configured password.
+    """
+    password = get_secret_from_env("OPEN_NOTEBOOK_PASSWORD")
+
+    # No password configured - skip authentication
+    if not password:
+        return True
+
+    # No credentials provided
+    if not credentials:
+        raise HTTPException(
+            status_code=401,
+            detail="Missing authorization",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    # Check password
+    if credentials.credentials != password:
+        raise HTTPException(
+            status_code=401,
+            detail="Invalid password",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    return True

api/chat_service.py

@@ -0,0 +1,168 @@
+"""
+Chat service for API operations.
+Provides async interface for chat functionality.
+"""
+
+import os
+from typing import Any, Dict, List, Optional
+
+import httpx
+from loguru import logger
+
+
+class ChatService:
+    """Service for chat-related API operations"""
+
+    def __init__(self):
+        self.base_url = os.getenv("API_BASE_URL", "http://127.0.0.1:5055")
+        # Add authentication header if password is set
+        self.headers = {}
+        password = os.getenv("OPEN_NOTEBOOK_PASSWORD")
+        if password:
+            self.headers["Authorization"] = f"Bearer {password}"
+
+    async def get_sessions(self, notebook_id: str) -> List[Dict[str, Any]]:
+        """Get all chat sessions for a notebook"""
+        try:
+            async with httpx.AsyncClient() as client:
+                response = await client.get(
+                    f"{self.base_url}/api/chat/sessions",
+                    params={"notebook_id": notebook_id},
+                    headers=self.headers,
+                )
+                response.raise_for_status()
+                return response.json()
+        except Exception as e:
+            logger.error(f"Error fetching chat sessions: {str(e)}")
+            raise
+
+    async def create_session(
+        self,
+        notebook_id: str,
+        title: Optional[str] = None,
+        model_override: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Create a new chat session"""
+        try:
+            data: Dict[str, Any] = {"notebook_id": notebook_id}
+            if title is not None:
+                data["title"] = title
+            if model_override is not None:
+                data["model_override"] = model_override
+
+            async with httpx.AsyncClient() as client:
+                response = await client.post(
+                    f"{self.base_url}/api/chat/sessions",
+                    json=data,
+                    headers=self.headers,
+                )
+                response.raise_for_status()
+                return response.json()
+        except Exception as e:
+            logger.error(f"Error creating chat session: {str(e)}")
+            raise
+
+    async def get_session(self, session_id: str) -> Dict[str, Any]:
+        """Get a specific session with messages"""
+        try:
+            async with httpx.AsyncClient() as client:
+                response = await client.get(
+                    f"{self.base_url}/api/chat/sessions/{session_id}",
+                    headers=self.headers,
+                )
+                response.raise_for_status()
+                return response.json()
+        except Exception as e:
+            logger.error(f"Error fetching session: {str(e)}")
+            raise
+
+    async def update_session(
+        self,
+        session_id: str,
+        title: Optional[str] = None,
+        model_override: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Update session properties"""
+        try:
+            data: Dict[str, Any] = {}
+            if title is not None:
+                data["title"] = title
+            if model_override is not None:
+                data["model_override"] = model_override
+
+            if not data:
+                raise ValueError(
+                    "At least one field must be provided to update a session"
+                )
+
+            async with httpx.AsyncClient() as client:
+                response = await client.put(
+                    f"{self.base_url}/api/chat/sessions/{session_id}",
+                    json=data,
+                    headers=self.headers,
+                )
+                response.raise_for_status()
+                return response.json()
+        except Exception as e:
+            logger.error(f"Error updating session: {str(e)}")
+            raise
+
+    async def delete_session(self, session_id: str) -> Dict[str, Any]:
+        """Delete a chat session"""
+        try:
+            async with httpx.AsyncClient() as client:
+                response = await client.delete(
+                    f"{self.base_url}/api/chat/sessions/{session_id}",
+                    headers=self.headers,
+                )
+                response.raise_for_status()
+                return response.json()
+        except Exception as e:
+            logger.error(f"Error deleting session: {str(e)}")
+            raise
+
+    async def execute_chat(
+        self,
+        session_id: str,
+        message: str,
+        context: Dict[str, Any],
+        model_override: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Execute a chat request"""
+        try:
+            data = {"session_id": session_id, "message": message, "context": context}
+            if model_override is not None:
+                data["model_override"] = model_override
+
+            # Short connect timeout (10s), long read timeout (10 min) for Ollama/local LLMs
+            timeout = httpx.Timeout(connect=10.0, read=600.0, write=30.0, pool=10.0)
+            async with httpx.AsyncClient(timeout=timeout) as client:
+                response = await client.post(
+                    f"{self.base_url}/api/chat/execute", json=data, headers=self.headers
+                )
+                response.raise_for_status()
+                return response.json()
+        except Exception as e:
+            logger.error(f"Error executing chat: {str(e)}")
+            raise
+
+    async def build_context(
+        self, notebook_id: str, context_config: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        """Build context for a notebook"""
+        try:
+            data = {"notebook_id": notebook_id, "context_config": context_config}
+
+            async with httpx.AsyncClient() as client:
+                response = await client.post(
+                    f"{self.base_url}/api/chat/context", json=data, headers=self.headers
+                )
+                response.raise_for_status()
+                return response.json()
+        except Exception as e:
+            logger.error(f"Error building context: {str(e)}")
+            raise
+
+
+# Global instance
+chat_service = ChatService()

api/client.py

@@ -0,0 +1,529 @@
+"""
+API client for Open Notebook API.
+This module provides a client interface to interact with the Open Notebook API.
+"""
+
+import os
+from typing import Any, Dict, List, Optional, Union
+
+import httpx
+from loguru import logger
+
+
+class APIClient:
+    """Client for Open Notebook API."""
+
+    def __init__(self, base_url: Optional[str] = None):
+        self.base_url = base_url or os.getenv("API_BASE_URL", "http://127.0.0.1:5055")
+        # Timeout increased to 5 minutes (300s) to accommodate slow LLM operations
+        # (transformations, insights) on slower hardware (Ollama, LM Studio, remote APIs)
+        # Configurable via API_CLIENT_TIMEOUT environment variable (in seconds)
+        timeout_str = os.getenv("API_CLIENT_TIMEOUT", "300.0")
+        try:
+            timeout_value = float(timeout_str)
+            # Validate timeout is within reasonable bounds (30s - 3600s / 1 hour)
+            if timeout_value < 30:
+                logger.warning(
+                    f"API_CLIENT_TIMEOUT={timeout_value}s is too low, using minimum of 30s"
+                )
+                timeout_value = 30.0
+            elif timeout_value > 3600:
+                logger.warning(
+                    f"API_CLIENT_TIMEOUT={timeout_value}s is too high, using maximum of 3600s"
+                )
+                timeout_value = 3600.0
+            self.timeout = timeout_value
+        except ValueError:
+            logger.error(
+                f"Invalid API_CLIENT_TIMEOUT value '{timeout_str}', using default 300s"
+            )
+            self.timeout = 300.0
+
+        # Add authentication header if password is set
+        self.headers = {}
+        password = os.getenv("OPEN_NOTEBOOK_PASSWORD")
+        if password:
+            self.headers["Authorization"] = f"Bearer {password}"
+
+    def _make_request(
+        self, method: str, endpoint: str, timeout: Optional[float] = None, **kwargs
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Make HTTP request to the API."""
+        url = f"{self.base_url}{endpoint}"
+        request_timeout = timeout if timeout is not None else self.timeout
+
+        # Merge headers
+        headers = kwargs.get("headers", {})
+        headers.update(self.headers)
+        kwargs["headers"] = headers
+
+        try:
+            with httpx.Client(timeout=request_timeout) as client:
+                response = client.request(method, url, **kwargs)
+                response.raise_for_status()
+                return response.json()
+        except httpx.RequestError as e:
+            logger.error(f"Request error for {method} {url}: {str(e)}")
+            raise ConnectionError(f"Failed to connect to API: {str(e)}")
+        except httpx.HTTPStatusError as e:
+            logger.error(
+                f"HTTP error {e.response.status_code} for {method} {url}: {e.response.text}"
+            )
+            raise RuntimeError(
+                f"API request failed: {e.response.status_code} - {e.response.text}"
+            )
+        except Exception as e:
+            logger.error(f"Unexpected error for {method} {url}: {str(e)}")
+            raise
+
+    # Notebooks API methods
+    def get_notebooks(
+        self, archived: Optional[bool] = None, order_by: str = "updated desc"
+    ) -> List[Dict[Any, Any]]:
+        """Get all notebooks."""
+        params: Dict[str, Any] = {"order_by": order_by}
+        if archived is not None:
+            params["archived"] = str(archived).lower()
+
+        result = self._make_request("GET", "/api/notebooks", params=params)
+        return result if isinstance(result, list) else [result]
+
+    def create_notebook(
+        self, name: str, description: str = ""
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Create a new notebook."""
+        data = {"name": name, "description": description}
+        return self._make_request("POST", "/api/notebooks", json=data)
+
+    def get_notebook(
+        self, notebook_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get a specific notebook."""
+        return self._make_request("GET", f"/api/notebooks/{notebook_id}")
+
+    def update_notebook(
+        self, notebook_id: str, **updates
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Update a notebook."""
+        return self._make_request("PUT", f"/api/notebooks/{notebook_id}", json=updates)
+
+    def delete_notebook(
+        self, notebook_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Delete a notebook."""
+        return self._make_request("DELETE", f"/api/notebooks/{notebook_id}")
+
+    # Search API methods
+    def search(
+        self,
+        query: str,
+        search_type: str = "text",
+        limit: int = 100,
+        search_sources: bool = True,
+        search_notes: bool = True,
+        minimum_score: float = 0.2,
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Search the knowledge base."""
+        data = {
+            "query": query,
+            "type": search_type,
+            "limit": limit,
+            "search_sources": search_sources,
+            "search_notes": search_notes,
+            "minimum_score": minimum_score,
+        }
+        return self._make_request("POST", "/api/search", json=data)
+
+    def ask_simple(
+        self,
+        question: str,
+        strategy_model: str,
+        answer_model: str,
+        final_answer_model: str,
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Ask the knowledge base a question (simple, non-streaming)."""
+        data = {
+            "question": question,
+            "strategy_model": strategy_model,
+            "answer_model": answer_model,
+            "final_answer_model": final_answer_model,
+        }
+        # Use configured timeout for long-running ask operations
+        return self._make_request(
+            "POST", "/api/search/ask/simple", json=data, timeout=self.timeout
+        )
+
+    # Models API methods
+    def get_models(self, model_type: Optional[str] = None) -> List[Dict[Any, Any]]:
+        """Get all models with optional type filtering."""
+        params = {}
+        if model_type:
+            params["type"] = model_type
+        result = self._make_request("GET", "/api/models", params=params)
+        return result if isinstance(result, list) else [result]
+
+    def create_model(
+        self, name: str, provider: str, model_type: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Create a new model."""
+        data = {
+            "name": name,
+            "provider": provider,
+            "type": model_type,
+        }
+        return self._make_request("POST", "/api/models", json=data)
+
+    def delete_model(
+        self, model_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Delete a model."""
+        return self._make_request("DELETE", f"/api/models/{model_id}")
+
+    def get_default_models(self) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get default model assignments."""
+        return self._make_request("GET", "/api/models/defaults")
+
+    def update_default_models(
+        self, **defaults
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Update default model assignments."""
+        return self._make_request("PUT", "/api/models/defaults", json=defaults)
+
+    # Transformations API methods
+    def get_transformations(self) -> List[Dict[Any, Any]]:
+        """Get all transformations."""
+        result = self._make_request("GET", "/api/transformations")
+        return result if isinstance(result, list) else [result]
+
+    def create_transformation(
+        self,
+        name: str,
+        title: str,
+        description: str,
+        prompt: str,
+        apply_default: bool = False,
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Create a new transformation."""
+        data = {
+            "name": name,
+            "title": title,
+            "description": description,
+            "prompt": prompt,
+            "apply_default": apply_default,
+        }
+        return self._make_request("POST", "/api/transformations", json=data)
+
+    def get_transformation(
+        self, transformation_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get a specific transformation."""
+        return self._make_request("GET", f"/api/transformations/{transformation_id}")
+
+    def update_transformation(
+        self, transformation_id: str, **updates
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Update a transformation."""
+        return self._make_request(
+            "PUT", f"/api/transformations/{transformation_id}", json=updates
+        )
+
+    def delete_transformation(
+        self, transformation_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Delete a transformation."""
+        return self._make_request("DELETE", f"/api/transformations/{transformation_id}")
+
+    def execute_transformation(
+        self, transformation_id: str, input_text: str, model_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Execute a transformation on input text."""
+        data = {
+            "transformation_id": transformation_id,
+            "input_text": input_text,
+            "model_id": model_id,
+        }
+        # Use configured timeout for transformation operations
+        return self._make_request(
+            "POST", "/api/transformations/execute", json=data, timeout=self.timeout
+        )
+
+    # Notes API methods
+    def get_notes(self, notebook_id: Optional[str] = None) -> List[Dict[Any, Any]]:
+        """Get all notes with optional notebook filtering."""
+        params = {}
+        if notebook_id:
+            params["notebook_id"] = notebook_id
+        result = self._make_request("GET", "/api/notes", params=params)
+        return result if isinstance(result, list) else [result]
+
+    def create_note(
+        self,
+        content: str,
+        title: Optional[str] = None,
+        note_type: str = "human",
+        notebook_id: Optional[str] = None,
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Create a new note."""
+        data = {
+            "content": content,
+            "note_type": note_type,
+        }
+        if title:
+            data["title"] = title
+        if notebook_id:
+            data["notebook_id"] = notebook_id
+        return self._make_request("POST", "/api/notes", json=data)
+
+    def get_note(self, note_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get a specific note."""
+        return self._make_request("GET", f"/api/notes/{note_id}")
+
+    def update_note(
+        self, note_id: str, **updates
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Update a note."""
+        return self._make_request("PUT", f"/api/notes/{note_id}", json=updates)
+
+    def delete_note(self, note_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Delete a note."""
+        return self._make_request("DELETE", f"/api/notes/{note_id}")
+
+    # Embedding API methods
+    def embed_content(
+        self, item_id: str, item_type: str, async_processing: bool = False
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Embed content for vector search."""
+        data = {
+            "item_id": item_id,
+            "item_type": item_type,
+            "async_processing": async_processing,
+        }
+        # Use configured timeout for embedding operations
+        return self._make_request("POST", "/api/embed", json=data, timeout=self.timeout)
+
+    def rebuild_embeddings(
+        self,
+        mode: str = "existing",
+        include_sources: bool = True,
+        include_notes: bool = True,
+        include_insights: bool = True,
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Rebuild embeddings in bulk.
+
+        Note: This operation can take a long time for large databases.
+        Consider increasing API_CLIENT_TIMEOUT to 600-900s for bulk rebuilds.
+        """
+        data = {
+            "mode": mode,
+            "include_sources": include_sources,
+            "include_notes": include_notes,
+            "include_insights": include_insights,
+        }
+        # Use double the configured timeout for bulk rebuild operations (or configured value if already high)
+        rebuild_timeout = max(self.timeout, min(self.timeout * 2, 3600.0))
+        return self._make_request(
+            "POST", "/api/embeddings/rebuild", json=data, timeout=rebuild_timeout
+        )
+
+    def get_rebuild_status(
+        self, command_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get status of a rebuild operation."""
+        return self._make_request("GET", f"/api/embeddings/rebuild/{command_id}/status")
+
+    # Settings API methods
+    def get_settings(self) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get all application settings."""
+        return self._make_request("GET", "/api/settings")
+
+    def update_settings(
+        self, **settings
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Update application settings."""
+        return self._make_request("PUT", "/api/settings", json=settings)
+
+    # Context API methods
+    def get_notebook_context(
+        self, notebook_id: str, context_config: Optional[Dict] = None
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get context for a notebook."""
+        data: Dict[str, Any] = {"notebook_id": notebook_id}
+        if context_config:
+            data["context_config"] = context_config
+        result = self._make_request(
+            "POST", f"/api/notebooks/{notebook_id}/context", json=data
+        )
+        return result if isinstance(result, dict) else {}
+
+    # Sources API methods
+    def get_sources(self, notebook_id: Optional[str] = None) -> List[Dict[Any, Any]]:
+        """Get all sources with optional notebook filtering."""
+        params = {}
+        if notebook_id:
+            params["notebook_id"] = notebook_id
+        result = self._make_request("GET", "/api/sources", params=params)
+        return result if isinstance(result, list) else [result]
+
+    def create_source(
+        self,
+        notebook_id: Optional[str] = None,
+        notebooks: Optional[List[str]] = None,
+        source_type: str = "text",
+        url: Optional[str] = None,
+        file_path: Optional[str] = None,
+        content: Optional[str] = None,
+        title: Optional[str] = None,
+        transformations: Optional[List[str]] = None,
+        embed: bool = False,
+        delete_source: bool = False,
+        async_processing: bool = False,
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Create a new source."""
+        data = {
+            "type": source_type,
+            "embed": embed,
+            "delete_source": delete_source,
+            "async_processing": async_processing,
+        }
+
+        # Handle backward compatibility for notebook_id vs notebooks
+        if notebooks:
+            data["notebooks"] = notebooks
+        elif notebook_id:
+            data["notebook_id"] = notebook_id
+        else:
+            raise ValueError("Either notebook_id or notebooks must be provided")
+
+        if url:
+            data["url"] = url
+        if file_path:
+            data["file_path"] = file_path
+        if content:
+            data["content"] = content
+        if title:
+            data["title"] = title
+        if transformations:
+            data["transformations"] = transformations
+
+        # Use configured timeout for source creation (especially PDF processing with OCR)
+        return self._make_request(
+            "POST", "/api/sources/json", json=data, timeout=self.timeout
+        )
+
+    def get_source(self, source_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get a specific source."""
+        return self._make_request("GET", f"/api/sources/{source_id}")
+
+    def get_source_status(
+        self, source_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get processing status for a source."""
+        return self._make_request("GET", f"/api/sources/{source_id}/status")
+
+    def update_source(
+        self, source_id: str, **updates
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Update a source."""
+        return self._make_request("PUT", f"/api/sources/{source_id}", json=updates)
+
+    def delete_source(
+        self, source_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Delete a source."""
+        return self._make_request("DELETE", f"/api/sources/{source_id}")
+
+    # Insights API methods
+    def get_source_insights(self, source_id: str) -> List[Dict[Any, Any]]:
+        """Get all insights for a specific source."""
+        result = self._make_request("GET", f"/api/sources/{source_id}/insights")
+        return result if isinstance(result, list) else [result]
+
+    def get_insight(
+        self, insight_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get a specific insight."""
+        return self._make_request("GET", f"/api/insights/{insight_id}")
+
+    def delete_insight(
+        self, insight_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Delete a specific insight."""
+        return self._make_request("DELETE", f"/api/insights/{insight_id}")
+
+    def save_insight_as_note(
+        self, insight_id: str, notebook_id: Optional[str] = None
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Convert an insight to a note."""
+        data = {}
+        if notebook_id:
+            data["notebook_id"] = notebook_id
+        return self._make_request(
+            "POST", f"/api/insights/{insight_id}/save-as-note", json=data
+        )
+
+    def create_source_insight(
+        self, source_id: str, transformation_id: str, model_id: Optional[str] = None
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Create a new insight for a source by running a transformation."""
+        data = {"transformation_id": transformation_id}
+        if model_id:
+            data["model_id"] = model_id
+        return self._make_request(
+            "POST", f"/api/sources/{source_id}/insights", json=data
+        )
+
+    # Episode Profiles API methods
+    def get_episode_profiles(self) -> List[Dict[Any, Any]]:
+        """Get all episode profiles."""
+        result = self._make_request("GET", "/api/episode-profiles")
+        return result if isinstance(result, list) else [result]
+
+    def get_episode_profile(
+        self, profile_name: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get a specific episode profile by name."""
+        return self._make_request("GET", f"/api/episode-profiles/{profile_name}")
+
+    def create_episode_profile(
+        self,
+        name: str,
+        description: str = "",
+        speaker_config: str = "",
+        outline_provider: str = "",
+        outline_model: str = "",
+        transcript_provider: str = "",
+        transcript_model: str = "",
+        default_briefing: str = "",
+        num_segments: int = 5,
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Create a new episode profile."""
+        data = {
+            "name": name,
+            "description": description,
+            "speaker_config": speaker_config,
+            "outline_provider": outline_provider,
+            "outline_model": outline_model,
+            "transcript_provider": transcript_provider,
+            "transcript_model": transcript_model,
+            "default_briefing": default_briefing,
+            "num_segments": num_segments,
+        }
+        return self._make_request("POST", "/api/episode-profiles", json=data)
+
+    def update_episode_profile(
+        self, profile_id: str, **updates
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Update an episode profile."""
+        return self._make_request(
+            "PUT", f"/api/episode-profiles/{profile_id}", json=updates
+        )
+
+    def delete_episode_profile(
+        self, profile_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Delete an episode profile."""
+        return self._make_request("DELETE", f"/api/episode-profiles/{profile_id}")
+
+
+# Global client instance
+api_client = APIClient()

api/command_service.py

@@ -0,0 +1,92 @@
+from typing import Any, Dict, List, Optional
+
+from loguru import logger
+from surreal_commands import get_command_status, submit_command
+
+
+class CommandService:
+    """Generic service layer for command operations"""
+
+    @staticmethod
+    async def submit_command_job(
+        module_name: str,  # Actually app_name for surreal-commands
+        command_name: str,
+        command_args: Dict[str, Any],
+        context: Optional[Dict[str, Any]] = None,
+    ) -> str:
+        """Submit a generic command job for background processing"""
+        try:
+            # Ensure command modules are imported before submitting
+            # This is needed because submit_command validates against local registry
+            try:
+                import commands.podcast_commands  # noqa: F401
+            except ImportError as import_err:
+                logger.error(f"Failed to import command modules: {import_err}")
+                raise ValueError("Command modules not available")
+
+            # surreal-commands expects: submit_command(app_name, command_name, args)
+            cmd_id = submit_command(
+                module_name,  # This is actually the app name (e.g., "open_notebook")
+                command_name,  # Command name (e.g., "process_text")
+                command_args,  # Input data
+            )
+            # Convert RecordID to string if needed
+            if not cmd_id:
+                raise ValueError("Failed to get cmd_id from submit_command")
+            cmd_id_str = str(cmd_id)
+            logger.info(
+                f"Submitted command job: {cmd_id_str} for {module_name}.{command_name}"
+            )
+            return cmd_id_str
+
+        except Exception as e:
+            logger.error(f"Failed to submit command job: {e}")
+            raise
+
+    @staticmethod
+    async def get_command_status(job_id: str) -> Dict[str, Any]:
+        """Get status of any command job"""
+        try:
+            status = await get_command_status(job_id)
+            return {
+                "job_id": job_id,
+                "status": status.status if status else "unknown",
+                "result": status.result if status else None,
+                "error_message": getattr(status, "error_message", None)
+                if status
+                else None,
+                "created": str(status.created)
+                if status and hasattr(status, "created") and status.created
+                else None,
+                "updated": str(status.updated)
+                if status and hasattr(status, "updated") and status.updated
+                else None,
+                "progress": getattr(status, "progress", None) if status else None,
+            }
+        except Exception as e:
+            logger.error(f"Failed to get command status: {e}")
+            raise
+
+    @staticmethod
+    async def list_command_jobs(
+        module_filter: Optional[str] = None,
+        command_filter: Optional[str] = None,
+        status_filter: Optional[str] = None,
+        limit: int = 50,
+    ) -> List[Dict[str, Any]]:
+        """List command jobs with optional filtering"""
+        # This will be implemented with proper SurrealDB queries
+        # For now, return empty list as this is foundation phase
+        return []
+
+    @staticmethod
+    async def cancel_command_job(job_id: str) -> bool:
+        """Cancel a running command job"""
+        try:
+            # Implementation depends on surreal-commands cancellation support
+            # For now, just log the attempt
+            logger.info(f"Attempting to cancel job: {job_id}")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to cancel command job: {e}")
+            raise

api/context_service.py

@@ -0,0 +1,29 @@
+"""
+Context service layer using API.
+"""
+
+from typing import Any, Dict, List, Optional, Union
+
+from loguru import logger
+
+from api.client import api_client
+
+
+class ContextService:
+    """Service layer for context operations using API."""
+
+    def __init__(self):
+        logger.info("Using API for context operations")
+
+    def get_notebook_context(
+        self, notebook_id: str, context_config: Optional[Dict] = None
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get context for a notebook."""
+        result = api_client.get_notebook_context(
+            notebook_id=notebook_id, context_config=context_config
+        )
+        return result
+
+
+# Global service instance
+context_service = ContextService()

api/credentials_service.py

@@ -0,0 +1,890 @@
+"""
+Credentials Service
+
+Business logic for managing AI provider credentials.
+Extracted from the credentials router to follow the service layer pattern.
+
+All functions raise ValueError for business errors (router converts to HTTPException).
+"""
+
+import ipaddress
+import os
+import socket
+from typing import Dict, List, Optional
+from urllib.parse import urlparse
+
+import httpx
+from loguru import logger
+from pydantic import SecretStr
+
+from api.models import CredentialResponse
+from open_notebook.domain.credential import Credential
+from open_notebook.utils.encryption import get_secret_from_env
+
+# =============================================================================
+# Constants
+# =============================================================================
+
+# Provider environment variable configuration.
+# - "required": ALL listed env vars must be set for the provider to be considered configured.
+# - "required_any": at least ONE of the listed env vars must be set.
+# - "optional": additional env vars used during migration but not required.
+PROVIDER_ENV_CONFIG: Dict[str, dict] = {
+    "openai": {"required": ["OPENAI_API_KEY"]},
+    "anthropic": {"required": ["ANTHROPIC_API_KEY"]},
+    "google": {"required_any": ["GOOGLE_API_KEY", "GEMINI_API_KEY"]},
+    "groq": {"required": ["GROQ_API_KEY"]},
+    "mistral": {"required": ["MISTRAL_API_KEY"]},
+    "deepseek": {"required": ["DEEPSEEK_API_KEY"]},
+    "xai": {"required": ["XAI_API_KEY"]},
+    "openrouter": {"required": ["OPENROUTER_API_KEY"]},
+    "voyage": {"required": ["VOYAGE_API_KEY"]},
+    "elevenlabs": {"required": ["ELEVENLABS_API_KEY"]},
+    "ollama": {"required": ["OLLAMA_API_BASE"]},
+    "vertex": {
+        "required": ["VERTEX_PROJECT", "VERTEX_LOCATION"],
+        "optional": ["GOOGLE_APPLICATION_CREDENTIALS"],
+    },
+    "azure": {
+        "required": ["AZURE_OPENAI_API_KEY", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_API_VERSION"],
+        "optional": [
+            "AZURE_OPENAI_ENDPOINT_LLM",
+            "AZURE_OPENAI_ENDPOINT_EMBEDDING",
+            "AZURE_OPENAI_ENDPOINT_STT",
+            "AZURE_OPENAI_ENDPOINT_TTS",
+        ],
+    },
+    "openai_compatible": {
+        "required_any": ["OPENAI_COMPATIBLE_BASE_URL", "OPENAI_COMPATIBLE_API_KEY"],
+    },
+    "dashscope": {"required": ["DASHSCOPE_API_KEY"]},
+    "minimax": {"required": ["MINIMAX_API_KEY"]},
+}
+
+PROVIDER_MODALITIES: Dict[str, List[str]] = {
+    "openai": ["language", "embedding", "speech_to_text", "text_to_speech"],
+    "anthropic": ["language"],
+    "google": ["language", "embedding"],
+    "groq": ["language", "speech_to_text"],
+    "mistral": ["language", "embedding"],
+    "deepseek": ["language"],
+    "xai": ["language"],
+    "openrouter": ["language"],
+    "voyage": ["embedding"],
+    "elevenlabs": ["text_to_speech"],
+    "ollama": ["language", "embedding"],
+    "vertex": ["language", "embedding"],
+    "azure": ["language", "embedding", "speech_to_text", "text_to_speech"],
+    "openai_compatible": ["language", "embedding", "speech_to_text", "text_to_speech"],
+    "dashscope": ["language"],
+    "minimax": ["language"],
+}
+
+
+# =============================================================================
+# URL Validation (SSRF protection)
+# =============================================================================
+
+
+def validate_url(url: str, provider: str) -> None:
+    """
+    Validate URL format for API endpoints.
+
+    This is a self-hosted application, so we allow:
+    - Private IPs (10.x, 172.16-31.x, 192.168.x) for self-hosted services
+    - Localhost for local services (Ollama, LM Studio, etc.)
+
+    We only block:
+    - Invalid schemes (must be http or https)
+    - Malformed URLs
+    - Link-local addresses (169.254.x.x) - used for cloud metadata endpoints
+    - Hostnames that resolve to link-local addresses
+
+    Args:
+        url: The URL to validate
+        provider: The provider name (for logging/context)
+
+    Raises:
+        ValueError: If the URL is invalid
+    """
+    if not url or not url.strip():
+        return  # Empty URLs handled elsewhere
+
+    try:
+        parsed = urlparse(url.strip())
+
+        # Validate scheme - only http/https allowed
+        if parsed.scheme not in ("http", "https"):
+            raise ValueError(
+                f"Invalid URL scheme: '{parsed.scheme}'. Only http and https are allowed."
+            )
+
+        # Extract hostname
+        hostname = parsed.hostname
+        if not hostname:
+            raise ValueError("Invalid URL: hostname could not be determined.")
+
+        # Try to parse as IP address to check for dangerous addresses
+        try:
+            ip = ipaddress.ip_address(hostname)
+
+            # Block link-local addresses (169.254.x.x) - used for cloud metadata
+            # These are dangerous as they can expose cloud instance credentials
+            if ip.is_link_local:
+                raise ValueError(
+                    "Link-local addresses (169.254.x.x) are not allowed for security reasons. "
+                    "These addresses are used for cloud metadata endpoints."
+                )
+
+            # Block IPv4-mapped IPv6 addresses pointing to link-local
+            # e.g. ::ffff:169.254.169.254 bypasses IPv6 is_link_local check
+            if hasattr(ip, "ipv4_mapped") and ip.ipv4_mapped and ip.ipv4_mapped.is_link_local:
+                raise ValueError(
+                    "Link-local addresses (169.254.x.x) are not allowed for security reasons. "
+                    "These addresses are used for cloud metadata endpoints."
+                )
+
+        except ValueError as ve:
+            # Re-raise our own ValueErrors
+            if "Link-local" in str(ve) or "Invalid URL" in str(ve):
+                raise
+            # Not an IP address, it's a hostname - need to resolve and check
+            try:
+                # Resolve hostname to IP address
+                resolved_ips = socket.getaddrinfo(hostname, None)
+                for family, _, _, _, sockaddr in resolved_ips:
+                    ip_addr = sockaddr[0]
+                    try:
+                        parsed_ip = ipaddress.ip_address(ip_addr)
+                        if parsed_ip.is_link_local:
+                            raise ValueError(
+                                f"Hostname '{hostname}' resolves to a link-local address (169.254.x.x) which is not allowed for security reasons. "
+                                "These addresses are used for cloud metadata endpoints."
+                            )
+                        # Block IPv4-mapped IPv6 addresses pointing to link-local
+                        if (
+                            hasattr(parsed_ip, "ipv4_mapped")
+                            and parsed_ip.ipv4_mapped
+                            and parsed_ip.ipv4_mapped.is_link_local
+                        ):
+                            raise ValueError(
+                                f"Hostname '{hostname}' resolves to a link-local address (169.254.x.x) which is not allowed for security reasons. "
+                                "These addresses are used for cloud metadata endpoints."
+                            )
+                    except ValueError as inner_ve:
+                        if "link-local" in str(inner_ve).lower() or "Link-local" in str(inner_ve):
+                            raise
+                        # Skip non-IP addresses (e.g., IPv6 zones)
+                        continue
+            except socket.gaierror:
+                # Could not resolve hostname - allow it since the URL may be
+                # valid in the deployment environment (e.g., Azure endpoints,
+                # internal DNS names). We only block link-local addresses.
+                pass
+
+    except ValueError:
+        raise
+    except Exception:
+        raise ValueError("Invalid URL format. Check server logs for details.")
+
+
+# =============================================================================
+# Helpers
+# =============================================================================
+
+
+def require_encryption_key() -> None:
+    """Raise ValueError if encryption key is not configured."""
+    if not get_secret_from_env("OPEN_NOTEBOOK_ENCRYPTION_KEY"):
+        raise ValueError(
+            "Encryption key not configured. "
+            "Set OPEN_NOTEBOOK_ENCRYPTION_KEY to enable storing API keys."
+        )
+
+
+def credential_to_response(cred: Credential, model_count: int = 0) -> CredentialResponse:
+    """Convert a Credential domain object to API response."""
+    return CredentialResponse(
+        id=cred.id or "",
+        name=cred.name,
+        provider=cred.provider,
+        modalities=cred.modalities,
+        base_url=cred.base_url,
+        endpoint=cred.endpoint,
+        api_version=cred.api_version,
+        endpoint_llm=cred.endpoint_llm,
+        endpoint_embedding=cred.endpoint_embedding,
+        endpoint_stt=cred.endpoint_stt,
+        endpoint_tts=cred.endpoint_tts,
+        project=cred.project,
+        location=cred.location,
+        credentials_path=cred.credentials_path,
+        has_api_key=cred.api_key is not None,
+        created=str(cred.created) if cred.created else "",
+        updated=str(cred.updated) if cred.updated else "",
+        model_count=model_count,
+        decryption_error=cred.decryption_error,
+    )
+
+
+def check_env_configured(provider: str) -> bool:
+    """Check if a provider has sufficient env vars configured for migration."""
+    config = PROVIDER_ENV_CONFIG.get(provider)
+    if not config:
+        return False
+
+    if "required_any" in config:
+        return any(bool(os.environ.get(v, "").strip()) for v in config["required_any"])
+    elif "required" in config:
+        return all(bool(os.environ.get(v, "").strip()) for v in config["required"])
+    return False
+
+
+def get_default_modalities(provider: str) -> List[str]:
+    """Get default modalities for a provider."""
+    return PROVIDER_MODALITIES.get(provider.lower(), ["language"])
+
+
+def create_credential_from_env(provider: str) -> Credential:
+    """Create a Credential from environment variables for a given provider."""
+    modalities = get_default_modalities(provider)
+    name = "Default (Migrated from env)"
+
+    if provider == "ollama":
+        return Credential(
+            name=name,
+            provider=provider,
+            modalities=modalities,
+            base_url=os.environ.get("OLLAMA_API_BASE"),
+        )
+    elif provider == "vertex":
+        return Credential(
+            name=name,
+            provider=provider,
+            modalities=modalities,
+            project=os.environ.get("VERTEX_PROJECT"),
+            location=os.environ.get("VERTEX_LOCATION"),
+            credentials_path=os.environ.get("GOOGLE_APPLICATION_CREDENTIALS"),
+        )
+    elif provider == "azure":
+        return Credential(
+            name=name,
+            provider=provider,
+            modalities=modalities,
+            api_key=SecretStr(os.environ["AZURE_OPENAI_API_KEY"]),
+            endpoint=os.environ.get("AZURE_OPENAI_ENDPOINT"),
+            api_version=os.environ.get("AZURE_OPENAI_API_VERSION"),
+            endpoint_llm=os.environ.get("AZURE_OPENAI_ENDPOINT_LLM"),
+            endpoint_embedding=os.environ.get("AZURE_OPENAI_ENDPOINT_EMBEDDING"),
+            endpoint_stt=os.environ.get("AZURE_OPENAI_ENDPOINT_STT"),
+            endpoint_tts=os.environ.get("AZURE_OPENAI_ENDPOINT_TTS"),
+        )
+    elif provider == "openai_compatible":
+        api_key = os.environ.get("OPENAI_COMPATIBLE_API_KEY")
+        return Credential(
+            name=name,
+            provider=provider,
+            modalities=modalities,
+            api_key=SecretStr(api_key) if api_key else None,
+            base_url=os.environ.get("OPENAI_COMPATIBLE_BASE_URL"),
+        )
+    elif provider == "google":
+        # Support both GOOGLE_API_KEY and GEMINI_API_KEY (fallback)
+        api_key = os.environ.get("GOOGLE_API_KEY") or os.environ.get("GEMINI_API_KEY")
+        return Credential(
+            name=name,
+            provider=provider,
+            modalities=modalities,
+            api_key=SecretStr(api_key) if api_key else None,
+        )
+    else:
+        # Simple API key providers
+        config = PROVIDER_ENV_CONFIG.get(provider, {})
+        required = config.get("required", [])
+        env_var = required[0] if required else None
+        api_key = os.environ.get(env_var) if env_var else None
+        return Credential(
+            name=name,
+            provider=provider,
+            modalities=modalities,
+            api_key=SecretStr(api_key) if api_key else None,
+        )
+
+
+# =============================================================================
+# Service Functions
+# =============================================================================
+
+
+async def get_provider_status() -> dict:
+    """
+    Get configuration status: encryption key status, and per-provider
+    configured/source information.
+    """
+    encryption_configured = bool(get_secret_from_env("OPEN_NOTEBOOK_ENCRYPTION_KEY"))
+
+    configured: Dict[str, bool] = {}
+    source: Dict[str, str] = {}
+
+    for provider in PROVIDER_ENV_CONFIG:
+        env_configured = check_env_configured(provider)
+        try:
+            db_credentials = await Credential.get_by_provider(provider)
+            db_configured = len(db_credentials) > 0
+        except Exception:
+            db_configured = False
+
+        configured[provider] = db_configured or env_configured
+
+        if db_configured:
+            source[provider] = "database"
+        elif env_configured:
+            source[provider] = "environment"
+        else:
+            source[provider] = "none"
+
+    return {
+        "configured": configured,
+        "source": source,
+        "encryption_configured": encryption_configured,
+    }
+
+
+async def get_env_status() -> Dict[str, bool]:
+    """Check what's configured via environment variables."""
+    env_status: Dict[str, bool] = {}
+    for provider in PROVIDER_ENV_CONFIG:
+        env_status[provider] = check_env_configured(provider)
+    return env_status
+
+
+async def test_credential(credential_id: str) -> dict:
+    """
+    Test connection using a credential's configuration.
+
+    Returns dict with provider, success, message keys.
+    """
+    provider = "unknown"
+    try:
+        cred = await Credential.get(credential_id)
+        config = cred.to_esperanto_config()
+
+        from open_notebook.ai.connection_tester import (
+            _test_azure_connection,
+            _test_ollama_connection,
+            _test_openai_compatible_connection,
+        )
+
+        provider = cred.provider.lower()
+
+        # Handle special providers
+        if provider == "ollama":
+            base_url = config.get("base_url", "http://localhost:11434")
+            success, message = await _test_ollama_connection(base_url)
+            return {"provider": provider, "success": success, "message": message}
+
+        if provider == "openai_compatible":
+            base_url = config.get("base_url")
+            api_key = config.get("api_key")
+            if not base_url:
+                return {
+                    "provider": provider,
+                    "success": False,
+                    "message": "No base URL configured",
+                }
+            success, message = await _test_openai_compatible_connection(
+                base_url, api_key
+            )
+            return {"provider": provider, "success": success, "message": message}
+
+        if provider == "azure":
+            success, message = await _test_azure_connection(
+                endpoint=config.get("endpoint"),
+                api_key=config.get("api_key"),
+                api_version=config.get("api_version"),
+            )
+            return {"provider": provider, "success": success, "message": message}
+
+        # Standard provider: use Esperanto to create and test
+        from esperanto.factory import AIFactory
+
+        from open_notebook.ai.connection_tester import TEST_MODELS
+
+        if provider not in TEST_MODELS:
+            return {
+                "provider": provider,
+                "success": False,
+                "message": f"Unknown provider: {provider}",
+            }
+
+        test_model, test_type = TEST_MODELS[provider]
+        if not test_model:
+            return {
+                "provider": provider,
+                "success": False,
+                "message": f"No test model configured for {provider}",
+            }
+
+        if test_type == "language":
+            model = AIFactory.create_language(
+                model_name=test_model, provider=provider, config=config
+            )
+            lc_model = model.to_langchain()
+            await lc_model.ainvoke("Hi")
+            return {"provider": provider, "success": True, "message": "Connection successful"}
+
+        elif test_type == "embedding":
+            model = AIFactory.create_embedding(
+                model_name=test_model, provider=provider, config=config
+            )
+            await model.aembed(["test"])
+            return {"provider": provider, "success": True, "message": "Connection successful"}
+
+        elif test_type == "text_to_speech":
+            AIFactory.create_text_to_speech(model_name=test_model, provider=provider, config=config)
+            return {
+                "provider": provider,
+                "success": True,
+                "message": "Connection successful (key format valid)",
+            }
+
+        return {
+            "provider": provider,
+            "success": False,
+            "message": f"Unsupported test type: {test_type}",
+        }
+
+    except Exception as e:
+        error_msg = str(e)
+        if "401" in error_msg or "unauthorized" in error_msg.lower():
+            return {"provider": provider, "success": False, "message": "Invalid API key"}
+        elif "403" in error_msg or "forbidden" in error_msg.lower():
+            return {"provider": provider, "success": False, "message": "API key lacks required permissions"}
+        elif "rate" in error_msg.lower() and "limit" in error_msg.lower():
+            return {"provider": provider, "success": True, "message": "Rate limited - but connection works"}
+        elif "not found" in error_msg.lower() and "model" in error_msg.lower():
+            return {"provider": provider, "success": True, "message": "API key valid (test model not available)"}
+        else:
+            logger.debug(f"Test connection error for credential {credential_id}: {e}")
+            truncated = error_msg[:100] + "..." if len(error_msg) > 100 else error_msg
+            return {"provider": provider, "success": False, "message": f"Error: {truncated}"}
+
+
+async def discover_with_config(provider: str, config: dict) -> List[dict]:
+    """
+    Discover models using explicit config instead of env vars.
+
+    Returns model names only — no type classification.
+    The user chooses the model type when registering.
+    """
+    api_key = config.get("api_key")
+    base_url = config.get("base_url")
+
+    # Static model lists for providers without a listing API
+    STATIC_MODELS: Dict[str, List[str]] = {
+        "anthropic": [
+            "claude-opus-4-20250514",
+            "claude-sonnet-4-20250514",
+            "claude-3-5-sonnet-20241022",
+            "claude-3-5-haiku-20241022",
+            "claude-3-opus-20240229",
+            "claude-3-sonnet-20240229",
+            "claude-3-haiku-20240307",
+        ],
+        "voyage": [
+            "voyage-3", "voyage-3-lite", "voyage-code-3",
+            "voyage-finance-2", "voyage-law-2", "voyage-multilingual-2",
+        ],
+        "elevenlabs": [
+            "eleven_multilingual_v2", "eleven_turbo_v2_5",
+            "eleven_turbo_v2", "eleven_monolingual_v1",
+        ],
+    }
+
+    if provider in STATIC_MODELS:
+        if not api_key and provider != "ollama":
+            return []
+        return [
+            {"name": m, "provider": provider}
+            for m in STATIC_MODELS[provider]
+        ]
+
+    # API-based discovery URLs (OpenAI-style /models endpoints)
+    url_map = {
+        "openai": "https://api.openai.com/v1/models",
+        "groq": "https://api.groq.com/openai/v1/models",
+        "mistral": "https://api.mistral.ai/v1/models",
+        "deepseek": "https://api.deepseek.com/models",
+        "xai": "https://api.x.ai/v1/models",
+        "openrouter": "https://openrouter.ai/api/v1/models",
+        "dashscope": "https://dashscope.aliyuncs.com/compatible-mode/v1/models",
+        "minimax": "https://api.minimax.io/v1/models",
+    }
+
+    if provider == "ollama":
+        ollama_url = base_url or "http://localhost:11434"
+        try:
+            async with httpx.AsyncClient() as client:
+                response = await client.get(f"{ollama_url}/api/tags", timeout=10.0)
+                response.raise_for_status()
+                data = response.json()
+                return [
+                    {"name": m.get("name", ""), "provider": "ollama"}
+                    for m in data.get("models", [])
+                    if m.get("name")
+                ]
+        except Exception as e:
+            logger.warning(f"Failed to discover Ollama models: {e}")
+            return []
+
+    if provider == "openai_compatible":
+        if not base_url:
+            return []
+        try:
+            headers = {}
+            if api_key:
+                headers["Authorization"] = f"Bearer {api_key}"
+            async with httpx.AsyncClient() as client:
+                response = await client.get(
+                    f"{base_url.rstrip('/')}/models", headers=headers, timeout=30.0,
+                )
+                response.raise_for_status()
+                data = response.json()
+                return [
+                    {"name": m.get("id", ""), "provider": "openai_compatible"}
+                    for m in data.get("data", [])
+                    if m.get("id")
+                ]
+        except Exception as e:
+            logger.warning(f"Failed to discover openai_compatible models: {e}")
+            return []
+
+    if provider == "azure":
+        endpoint = config.get("endpoint")
+        api_version = config.get("api_version", "2024-10-21")
+        if not endpoint or not api_key:
+            return []
+        try:
+            url = f"{endpoint.rstrip('/')}/openai/models?api-version={api_version}"
+            headers = {"api-key": api_key}
+            async with httpx.AsyncClient() as client:
+                response = await client.get(url, headers=headers, timeout=30.0)
+                response.raise_for_status()
+                data = response.json()
+                return [
+                    {"name": m.get("id", ""), "provider": "azure"}
+                    for m in data.get("data", [])
+                    if m.get("id")
+                ]
+        except Exception as e:
+            logger.warning(f"Failed to discover Azure models: {e}")
+            return []
+
+    if provider == "vertex":
+        # Vertex AI requires service-account OAuth2 for model listing.
+        # Return a curated static list of well-known Vertex models instead.
+        VERTEX_MODELS = [
+            "gemini-2.0-flash",
+            "gemini-2.0-flash-lite",
+            "gemini-1.5-pro",
+            "gemini-1.5-flash",
+            "text-embedding-005",
+        ]
+        return [{"name": m, "provider": "vertex"} for m in VERTEX_MODELS]
+
+    if provider == "google":
+        try:
+            headers = {"X-Goog-Api-Key": api_key} if api_key else {}
+            async with httpx.AsyncClient() as client:
+                response = await client.get(
+                    "https://generativelanguage.googleapis.com/v1/models",
+                    headers=headers,
+                    timeout=30.0,
+                )
+                response.raise_for_status()
+                data = response.json()
+                return [
+                    {
+                        "name": model.get("name", "").replace("models/", ""),
+                        "provider": "google",
+                        "description": model.get("displayName"),
+                    }
+                    for model in data.get("models", [])
+                    if model.get("name")
+                ]
+        except Exception as e:
+            logger.warning(f"Failed to discover Google models: {e}")
+            return []
+
+    # Standard OpenAI-style API discovery
+    discovery_url = url_map.get(provider)
+    if not discovery_url or not api_key:
+        return []
+
+    try:
+        async with httpx.AsyncClient() as client:
+            response = await client.get(
+                discovery_url,
+                headers={"Authorization": f"Bearer {api_key}"},
+                timeout=30.0,
+            )
+            response.raise_for_status()
+            data = response.json()
+
+            return [
+                {
+                    "name": m.get("id", ""),
+                    "provider": provider,
+                    "description": m.get("name"),
+                }
+                for m in data.get("data", [])
+                if m.get("id")
+            ]
+    except Exception as e:
+        logger.warning(f"Failed to discover {provider} models: {e}")
+        return []
+
+
+async def register_models(credential_id: str, models_data: list) -> dict:
+    """
+    Register discovered models and link them to a credential.
+
+    Args:
+        credential_id: The credential ID to link models to
+        models_data: List of dicts with name, provider, model_type
+
+    Returns:
+        dict with created and existing counts
+    """
+    cred = await Credential.get(credential_id)
+
+    from open_notebook.ai.models import Model
+    from open_notebook.database.repository import repo_query
+
+    # Batch fetch existing models for this provider
+    existing_models = await repo_query(
+        "SELECT string::lowercase(name) as name, string::lowercase(type) as type FROM model "
+        "WHERE string::lowercase(provider) = $provider",
+        {"provider": cred.provider.lower()},
+    )
+    existing_keys = {(m["name"], m["type"]) for m in existing_models}
+
+    created = 0
+    existing = 0
+
+    for model_data in models_data:
+        key = (model_data.name.lower(), model_data.model_type.lower())
+        if key in existing_keys:
+            existing += 1
+            continue
+
+        new_model = Model(
+            name=model_data.name,
+            provider=model_data.provider or cred.provider,
+            type=model_data.model_type,
+            credential=cred.id,
+        )
+        await new_model.save()
+        created += 1
+
+    return {"created": created, "existing": existing}
+
+
+async def migrate_from_provider_config() -> dict:
+    """
+    Migrate existing ProviderConfig data to individual credential records.
+
+    Returns dict with message, migrated, skipped, errors.
+    """
+    logger.info("=== Starting ProviderConfig migration ===")
+
+    require_encryption_key()
+    logger.info("Encryption key verified")
+
+    from open_notebook.domain.provider_config import ProviderConfig
+
+    config = await ProviderConfig.get_instance()
+    logger.info(
+        f"Found ProviderConfig with {len(config.credentials)} provider(s): "
+        f"{', '.join(config.credentials.keys())}"
+    )
+
+    migrated = []
+    skipped = []
+    errors = []
+
+    for provider, credentials_list in config.credentials.items():
+        for old_cred in credentials_list:
+            try:
+                # Check if a credential already exists for this provider with same name
+                existing = await Credential.get_by_provider(provider)
+                names = [c.name for c in existing]
+                if old_cred.name in names:
+                    logger.info(
+                        f"[{provider}/{old_cred.name}] Already exists in DB, skipping"
+                    )
+                    skipped.append(f"{provider}/{old_cred.name}")
+                    continue
+
+                # Determine modalities from the provider type
+                modalities = get_default_modalities(provider)
+
+                logger.info(f"[{provider}/{old_cred.name}] Creating credential")
+                new_cred = Credential(
+                    name=old_cred.name,
+                    provider=provider,
+                    modalities=modalities,
+                    api_key=old_cred.api_key,
+                    base_url=old_cred.base_url,
+                    endpoint=old_cred.endpoint,
+                    api_version=old_cred.api_version,
+                    endpoint_llm=old_cred.endpoint_llm,
+                    endpoint_embedding=old_cred.endpoint_embedding,
+                    endpoint_stt=old_cred.endpoint_stt,
+                    endpoint_tts=old_cred.endpoint_tts,
+                    project=old_cred.project,
+                    location=old_cred.location,
+                    credentials_path=old_cred.credentials_path,
+                )
+                await new_cred.save()
+                logger.info(
+                    f"[{provider}/{old_cred.name}] Credential saved (id={new_cred.id})"
+                )
+
+                # Link existing models for this provider to the new credential
+                from open_notebook.ai.models import Model
+                from open_notebook.database.repository import repo_query
+
+                provider_models = await repo_query(
+                    "SELECT * FROM model WHERE string::lowercase(provider) = $provider AND credential IS NONE",
+                    {"provider": provider.lower()},
+                )
+                if provider_models:
+                    logger.info(
+                        f"[{provider}/{old_cred.name}] Linking {len(provider_models)} "
+                        f"unassigned model(s)"
+                    )
+                    for model_data in provider_models:
+                        model = Model(**model_data)
+                        model.credential = new_cred.id
+                        await model.save()
+
+                migrated.append(f"{provider}/{old_cred.name}")
+
+            except Exception as e:
+                logger.error(
+                    f"[{provider}/{old_cred.name}] Migration FAILED: "
+                    f"{type(e).__name__}: {e}",
+                    exc_info=True,
+                )
+                errors.append(f"{provider}/{old_cred.name}: {e}")
+
+    logger.info(
+        f"=== ProviderConfig migration complete === "
+        f"migrated={len(migrated)} skipped={len(skipped)} errors={len(errors)}"
+    )
+    if migrated:
+        logger.info(f"  Migrated: {', '.join(migrated)}")
+    if skipped:
+        logger.info(f"  Skipped: {', '.join(skipped)}")
+    if errors:
+        logger.error(f"  Errors: {'; '.join(errors)}")
+
+    return {
+        "message": f"Migration complete. Migrated {len(migrated)} credentials.",
+        "migrated": migrated,
+        "skipped": skipped,
+        "errors": errors,
+    }
+
+
+async def migrate_from_env() -> dict:
+    """
+    Migrate API keys from environment variables to credential records.
+
+    Returns dict with message, migrated, skipped, not_configured, errors.
+    """
+    logger.info("=== Starting environment variable migration ===")
+    logger.info(
+        f"Checking {len(PROVIDER_ENV_CONFIG)} providers: "
+        f"{', '.join(PROVIDER_ENV_CONFIG.keys())}"
+    )
+
+    require_encryption_key()
+    logger.info("Encryption key verified")
+
+    from open_notebook.ai.models import Model
+    from open_notebook.database.repository import repo_query
+
+    migrated = []
+    skipped = []
+    not_configured = []
+    errors = []
+
+    for provider in PROVIDER_ENV_CONFIG:
+        try:
+            if not check_env_configured(provider):
+                logger.debug(f"[{provider}] No env vars configured, skipping")
+                not_configured.append(provider)
+                continue
+
+            logger.info(f"[{provider}] Env vars detected, checking for existing credentials")
+
+            existing = await Credential.get_by_provider(provider)
+            if existing:
+                logger.info(
+                    f"[{provider}] Already has {len(existing)} credential(s) in DB, skipping"
+                )
+                skipped.append(provider)
+                continue
+
+            logger.info(f"[{provider}] Creating credential from env vars")
+            cred = create_credential_from_env(provider)
+            await cred.save()
+            logger.info(f"[{provider}] Credential saved successfully (id={cred.id})")
+
+            # Link unassigned models to this credential
+            provider_models = await repo_query(
+                "SELECT * FROM model WHERE string::lowercase(provider) = $provider AND credential IS NONE",
+                {"provider": provider.lower()},
+            )
+            if provider_models:
+                logger.info(
+                    f"[{provider}] Linking {len(provider_models)} unassigned model(s) "
+                    f"to credential {cred.id}"
+                )
+                for model_data in provider_models:
+                    model = Model(**model_data)
+                    model.credential = cred.id
+                    await model.save()
+            else:
+                logger.info(f"[{provider}] No unassigned models to link")
+
+            migrated.append(provider)
+
+        except Exception as e:
+            logger.error(
+                f"[{provider}] Migration FAILED: {type(e).__name__}: {e}",
+                exc_info=True,
+            )
+            errors.append(f"{provider}: {e}")
+
+    logger.info(
+        f"=== Environment variable migration complete === "
+        f"migrated={len(migrated)} skipped={len(skipped)} "
+        f"not_configured={len(not_configured)} errors={len(errors)}"
+    )
+    if migrated:
+        logger.info(f"  Migrated: {', '.join(migrated)}")
+    if skipped:
+        logger.info(f"  Skipped (already in DB): {', '.join(skipped)}")
+    if errors:
+        logger.error(f"  Errors: {'; '.join(errors)}")
+
+    return {
+        "message": f"Migration complete. Migrated {len(migrated)} providers.",
+        "migrated": migrated,
+        "skipped": skipped,
+        "not_configured": not_configured,
+        "errors": errors,
+    }

api/embedding_service.py

@@ -0,0 +1,27 @@
+"""
+Embedding service layer using API.
+"""
+
+from typing import Any, Dict, List, Union
+
+from loguru import logger
+
+from api.client import api_client
+
+
+class EmbeddingService:
+    """Service layer for embedding operations using API."""
+
+    def __init__(self):
+        logger.info("Using API for embedding operations")
+
+    def embed_content(
+        self, item_id: str, item_type: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Embed content for vector search."""
+        result = api_client.embed_content(item_id=item_id, item_type=item_type)
+        return result
+
+
+# Global service instance
+embedding_service = EmbeddingService()

api/episode_profiles_service.py

@@ -0,0 +1,112 @@
+"""
+Episode profiles service layer using API.
+"""
+
+from typing import List
+
+from loguru import logger
+
+from api.client import api_client
+from open_notebook.podcasts.models import EpisodeProfile
+
+
+class EpisodeProfilesService:
+    """Service layer for episode profiles operations using API."""
+
+    def __init__(self):
+        logger.info("Using API for episode profiles operations")
+
+    def get_all_episode_profiles(self) -> List[EpisodeProfile]:
+        """Get all episode profiles."""
+        profiles_data = api_client.get_episode_profiles()
+        # Convert API response to EpisodeProfile objects
+        profiles = []
+        for profile_data in profiles_data:
+            profile = EpisodeProfile(
+                name=profile_data["name"],
+                description=profile_data.get("description", ""),
+                speaker_config=profile_data["speaker_config"],
+                outline_provider=profile_data["outline_provider"],
+                outline_model=profile_data["outline_model"],
+                transcript_provider=profile_data["transcript_provider"],
+                transcript_model=profile_data["transcript_model"],
+                default_briefing=profile_data["default_briefing"],
+                num_segments=profile_data["num_segments"],
+            )
+            profile.id = profile_data["id"]
+            profiles.append(profile)
+        return profiles
+
+    def get_episode_profile(self, profile_name: str) -> EpisodeProfile:
+        """Get a specific episode profile by name."""
+        profile_response = api_client.get_episode_profile(profile_name)
+        profile_data = (
+            profile_response
+            if isinstance(profile_response, dict)
+            else profile_response[0]
+        )
+        profile = EpisodeProfile(
+            name=profile_data["name"],
+            description=profile_data.get("description", ""),
+            speaker_config=profile_data["speaker_config"],
+            outline_provider=profile_data["outline_provider"],
+            outline_model=profile_data["outline_model"],
+            transcript_provider=profile_data["transcript_provider"],
+            transcript_model=profile_data["transcript_model"],
+            default_briefing=profile_data["default_briefing"],
+            num_segments=profile_data["num_segments"],
+        )
+        profile.id = profile_data["id"]
+        return profile
+
+    def create_episode_profile(
+        self,
+        name: str,
+        description: str = "",
+        speaker_config: str = "",
+        outline_provider: str = "",
+        outline_model: str = "",
+        transcript_provider: str = "",
+        transcript_model: str = "",
+        default_briefing: str = "",
+        num_segments: int = 5,
+    ) -> EpisodeProfile:
+        """Create a new episode profile."""
+        profile_response = api_client.create_episode_profile(
+            name=name,
+            description=description,
+            speaker_config=speaker_config,
+            outline_provider=outline_provider,
+            outline_model=outline_model,
+            transcript_provider=transcript_provider,
+            transcript_model=transcript_model,
+            default_briefing=default_briefing,
+            num_segments=num_segments,
+        )
+        profile_data = (
+            profile_response
+            if isinstance(profile_response, dict)
+            else profile_response[0]
+        )
+        profile = EpisodeProfile(
+            name=profile_data["name"],
+            description=profile_data.get("description", ""),
+            speaker_config=profile_data["speaker_config"],
+            outline_provider=profile_data["outline_provider"],
+            outline_model=profile_data["outline_model"],
+            transcript_provider=profile_data["transcript_provider"],
+            transcript_model=profile_data["transcript_model"],
+            default_briefing=profile_data["default_briefing"],
+            num_segments=profile_data["num_segments"],
+        )
+        profile.id = profile_data["id"]
+        return profile
+
+    def delete_episode_profile(self, profile_id: str) -> bool:
+        """Delete an episode profile."""
+        api_client.delete_episode_profile(profile_id)
+        return True
+
+
+# Global service instance
+episode_profiles_service = EpisodeProfilesService()

api/insights_service.py

@@ -0,0 +1,100 @@
+"""
+Insights service layer using API.
+"""
+
+from typing import List, Optional
+
+from loguru import logger
+
+from api.client import api_client
+from open_notebook.domain.notebook import Note, SourceInsight
+
+
+class InsightsService:
+    """Service layer for insights operations using API."""
+
+    def __init__(self):
+        logger.info("Using API for insights operations")
+
+    def get_source_insights(self, source_id: str) -> List[SourceInsight]:
+        """Get all insights for a specific source."""
+        insights_data = api_client.get_source_insights(source_id)
+        # Convert API response to SourceInsight objects
+        insights = []
+        for insight_data in insights_data:
+            insight = SourceInsight(
+                insight_type=insight_data["insight_type"],
+                content=insight_data["content"],
+            )
+            insight.id = insight_data["id"]
+            insight.created = insight_data["created"]
+            insight.updated = insight_data["updated"]
+            insights.append(insight)
+        return insights
+
+    def get_insight(self, insight_id: str) -> SourceInsight:
+        """Get a specific insight."""
+        insight_response = api_client.get_insight(insight_id)
+        insight_data = (
+            insight_response
+            if isinstance(insight_response, dict)
+            else insight_response[0]
+        )
+        insight = SourceInsight(
+            insight_type=insight_data["insight_type"],
+            content=insight_data["content"],
+        )
+        insight.id = insight_data["id"]
+        insight.created = insight_data["created"]
+        insight.updated = insight_data["updated"]
+        # Note: source_id from API response is not stored; use await insight.get_source() if needed
+        return insight
+
+    def delete_insight(self, insight_id: str) -> bool:
+        """Delete a specific insight."""
+        api_client.delete_insight(insight_id)
+        return True
+
+    def save_insight_as_note(
+        self, insight_id: str, notebook_id: Optional[str] = None
+    ) -> Note:
+        """Convert an insight to a note."""
+        note_response = api_client.save_insight_as_note(insight_id, notebook_id)
+        note_data = (
+            note_response if isinstance(note_response, dict) else note_response[0]
+        )
+        note = Note(
+            title=note_data["title"],
+            content=note_data["content"],
+            note_type=note_data["note_type"],
+        )
+        note.id = note_data["id"]
+        note.created = note_data["created"]
+        note.updated = note_data["updated"]
+        return note
+
+    def create_source_insight(
+        self, source_id: str, transformation_id: str, model_id: Optional[str] = None
+    ) -> SourceInsight:
+        """Create a new insight for a source by running a transformation."""
+        insight_response = api_client.create_source_insight(
+            source_id, transformation_id, model_id
+        )
+        insight_data = (
+            insight_response
+            if isinstance(insight_response, dict)
+            else insight_response[0]
+        )
+        insight = SourceInsight(
+            insight_type=insight_data["insight_type"],
+            content=insight_data["content"],
+        )
+        insight.id = insight_data["id"]
+        insight.created = insight_data["created"]
+        insight.updated = insight_data["updated"]
+        # Note: source_id from API response is not stored; use await insight.get_source() if needed
+        return insight
+
+
+# Global service instance
+insights_service = InsightsService()

api/main.py

@@ -0,0 +1,322 @@
+# Load environment variables
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import os
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from loguru import logger
+from starlette.exceptions import HTTPException as StarletteHTTPException
+
+from api.auth import PasswordAuthMiddleware
+from api.routers import (
+    auth,
+    chat,
+    config,
+    context,
+    credentials,
+    embedding,
+    embedding_rebuild,
+    episode_profiles,
+    insights,
+    languages,
+    models,
+    notebooks,
+    notes,
+    podcasts,
+    search,
+    settings,
+    source_chat,
+    sources,
+    speaker_profiles,
+    transformations,
+)
+from api.routers import commands as commands_router
+from open_notebook.database.async_migrate import AsyncMigrationManager
+from open_notebook.exceptions import (
+    AuthenticationError,
+    ConfigurationError,
+    ExternalServiceError,
+    InvalidInputError,
+    NetworkError,
+    NotFoundError,
+    OpenNotebookError,
+    RateLimitError,
+)
+from open_notebook.utils.encryption import get_secret_from_env
+
+
+def _parse_cors_origins(raw: str) -> list[str]:
+    """Parse CORS_ORIGINS env value into a list of origins."""
+    value = raw.strip()
+    if value == "*":
+        return ["*"]
+    return [origin.strip() for origin in value.split(",") if origin.strip()]
+
+
+# Parsed once at module load; CORS_ORIGINS changes require a restart.
+_cors_origins_raw = os.getenv("CORS_ORIGINS")
+CORS_ALLOWED_ORIGINS = _parse_cors_origins(_cors_origins_raw or "*")
+CORS_IS_DEFAULT_WILDCARD = _cors_origins_raw is None
+
+
+def _cors_headers(request: Request) -> dict[str, str]:
+    """
+    Build CORS headers for error responses.
+
+    Mirrors Starlette CORSMiddleware behavior: reflects the request Origin
+    when the origin is allowed (or when wildcard is configured, since
+    browsers reject `Access-Control-Allow-Origin: *` combined with
+    credentials). Omits `Access-Control-Allow-Origin` for disallowed
+    origins so the browser blocks the error body from leaking cross-origin.
+    """
+    origin = request.headers.get("origin")
+    headers: dict[str, str] = {
+        "Access-Control-Allow-Credentials": "true",
+        "Access-Control-Allow-Methods": "*",
+        "Access-Control-Allow-Headers": "*",
+    }
+
+    if origin and ("*" in CORS_ALLOWED_ORIGINS or origin in CORS_ALLOWED_ORIGINS):
+        headers["Access-Control-Allow-Origin"] = origin
+        headers["Vary"] = "Origin"
+
+    return headers
+
+
+# Import commands to register them in the API process
+try:
+    logger.info("Commands imported in API process")
+except Exception as e:
+    logger.error(f"Failed to import commands in API process: {e}")
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """
+    Lifespan event handler for the FastAPI application.
+    Runs database migrations automatically on startup.
+    """
+    # Startup: Security checks
+    logger.info("Starting API initialization...")
+
+    # Security check: Encryption key
+    if not get_secret_from_env("OPEN_NOTEBOOK_ENCRYPTION_KEY"):
+        logger.warning(
+            "OPEN_NOTEBOOK_ENCRYPTION_KEY not set. "
+            "API key encryption will fail until this is configured. "
+            "Set OPEN_NOTEBOOK_ENCRYPTION_KEY to any secret string."
+        )
+
+    # Run database migrations
+
+    try:
+        migration_manager = AsyncMigrationManager()
+        current_version = await migration_manager.get_current_version()
+        logger.info(f"Current database version: {current_version}")
+
+        if await migration_manager.needs_migration():
+            logger.warning("Database migrations are pending. Running migrations...")
+            await migration_manager.run_migration_up()
+            new_version = await migration_manager.get_current_version()
+            logger.success(
+                f"Migrations completed successfully. Database is now at version {new_version}"
+            )
+        else:
+            logger.info(
+                "Database is already at the latest version. No migrations needed."
+            )
+    except Exception as e:
+        logger.error(f"CRITICAL: Database migration failed: {str(e)}")
+        logger.exception(e)
+        # Fail fast - don't start the API with an outdated database schema
+        raise RuntimeError(f"Failed to run database migrations: {str(e)}") from e
+
+    # Run podcast profile data migration (legacy strings -> Model registry)
+    try:
+        from open_notebook.podcasts.migration import migrate_podcast_profiles
+
+        await migrate_podcast_profiles()
+    except Exception as e:
+        logger.warning(f"Podcast profile migration encountered errors: {e}")
+        # Non-fatal: profiles can be migrated manually via UI
+
+    logger.success("API initialization completed successfully")
+
+    # Yield control to the application
+    yield
+
+    # Shutdown: cleanup if needed
+    logger.info("API shutdown complete")
+
+
+app = FastAPI(
+    title="Open Notebook API",
+    description="API for Open Notebook - Research Assistant",
+    lifespan=lifespan,
+)
+
+if CORS_IS_DEFAULT_WILDCARD:
+    logger.warning(
+        "CORS_ORIGINS is not set — API accepts cross-origin requests from any "
+        "origin (default: '*'). For production deployments, set CORS_ORIGINS to "
+        "your frontend origin(s), e.g. "
+        "CORS_ORIGINS=https://notebook.example.com"
+    )
+else:
+    logger.info(f"CORS allowed origins: {CORS_ALLOWED_ORIGINS}")
+
+# Add password authentication middleware first
+# Exclude /api/auth/status and /api/config from authentication
+app.add_middleware(
+    PasswordAuthMiddleware,
+    excluded_paths=[
+        "/",
+        "/health",
+        "/docs",
+        "/openapi.json",
+        "/redoc",
+        "/api/auth/status",
+        "/api/config",
+    ],
+)
+
+# Add CORS middleware last (so it processes first)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=CORS_ALLOWED_ORIGINS,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+# Custom exception handler to ensure CORS headers are included in error responses
+# This helps when errors occur before the CORS middleware can process them
+@app.exception_handler(StarletteHTTPException)
+async def custom_http_exception_handler(request: Request, exc: StarletteHTTPException):
+    """
+    Custom exception handler that ensures CORS headers are included in error responses.
+    This is particularly important for 413 (Payload Too Large) errors during file uploads.
+
+    Note: If a reverse proxy (nginx, traefik) returns 413 before the request reaches
+    FastAPI, this handler won't be called. In that case, configure your reverse proxy
+    to add CORS headers to error responses.
+    """
+    return JSONResponse(
+        status_code=exc.status_code,
+        content={"detail": exc.detail},
+        headers={**(exc.headers or {}), **_cors_headers(request)},
+    )
+
+
+@app.exception_handler(NotFoundError)
+async def not_found_error_handler(request: Request, exc: NotFoundError):
+    return JSONResponse(
+        status_code=404,
+        content={"detail": str(exc)},
+        headers=_cors_headers(request),
+    )
+
+
+@app.exception_handler(InvalidInputError)
+async def invalid_input_error_handler(request: Request, exc: InvalidInputError):
+    return JSONResponse(
+        status_code=400,
+        content={"detail": str(exc)},
+        headers=_cors_headers(request),
+    )
+
+
+@app.exception_handler(AuthenticationError)
+async def authentication_error_handler(request: Request, exc: AuthenticationError):
+    return JSONResponse(
+        status_code=401,
+        content={"detail": str(exc)},
+        headers=_cors_headers(request),
+    )
+
+
+@app.exception_handler(RateLimitError)
+async def rate_limit_error_handler(request: Request, exc: RateLimitError):
+    return JSONResponse(
+        status_code=429,
+        content={"detail": str(exc)},
+        headers=_cors_headers(request),
+    )
+
+
+@app.exception_handler(ConfigurationError)
+async def configuration_error_handler(request: Request, exc: ConfigurationError):
+    return JSONResponse(
+        status_code=422,
+        content={"detail": str(exc)},
+        headers=_cors_headers(request),
+    )
+
+
+@app.exception_handler(NetworkError)
+async def network_error_handler(request: Request, exc: NetworkError):
+    return JSONResponse(
+        status_code=502,
+        content={"detail": str(exc)},
+        headers=_cors_headers(request),
+    )
+
+
+@app.exception_handler(ExternalServiceError)
+async def external_service_error_handler(request: Request, exc: ExternalServiceError):
+    return JSONResponse(
+        status_code=502,
+        content={"detail": str(exc)},
+        headers=_cors_headers(request),
+    )
+
+
+@app.exception_handler(OpenNotebookError)
+async def open_notebook_error_handler(request: Request, exc: OpenNotebookError):
+    return JSONResponse(
+        status_code=500,
+        content={"detail": str(exc)},
+        headers=_cors_headers(request),
+    )
+
+
+# Include routers
+app.include_router(auth.router, prefix="/api", tags=["auth"])
+app.include_router(config.router, prefix="/api", tags=["config"])
+app.include_router(notebooks.router, prefix="/api", tags=["notebooks"])
+app.include_router(search.router, prefix="/api", tags=["search"])
+app.include_router(models.router, prefix="/api", tags=["models"])
+app.include_router(transformations.router, prefix="/api", tags=["transformations"])
+app.include_router(notes.router, prefix="/api", tags=["notes"])
+app.include_router(embedding.router, prefix="/api", tags=["embedding"])
+app.include_router(
+    embedding_rebuild.router, prefix="/api/embeddings", tags=["embeddings"]
+)
+app.include_router(settings.router, prefix="/api", tags=["settings"])
+app.include_router(context.router, prefix="/api", tags=["context"])
+app.include_router(sources.router, prefix="/api", tags=["sources"])
+app.include_router(insights.router, prefix="/api", tags=["insights"])
+app.include_router(commands_router.router, prefix="/api", tags=["commands"])
+app.include_router(podcasts.router, prefix="/api", tags=["podcasts"])
+app.include_router(episode_profiles.router, prefix="/api", tags=["episode-profiles"])
+app.include_router(speaker_profiles.router, prefix="/api", tags=["speaker-profiles"])
+app.include_router(chat.router, prefix="/api", tags=["chat"])
+app.include_router(source_chat.router, prefix="/api", tags=["source-chat"])
+app.include_router(credentials.router, prefix="/api", tags=["credentials"])
+app.include_router(languages.router, prefix="/api", tags=["languages"])
+
+
+@app.get("/")
+async def root():
+    return {"message": "Open Notebook API is running"}
+
+
+@app.get("/health")
+async def health():
+    return {"status": "healthy"}

api/models.py

@@ -0,0 +1,686 @@
+from typing import Any, Dict, List, Literal, Optional
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+
+# Notebook models
+class NotebookCreate(BaseModel):
+    name: str = Field(..., description="Name of the notebook")
+    description: str = Field(default="", description="Description of the notebook")
+
+
+class NotebookUpdate(BaseModel):
+    name: Optional[str] = Field(None, description="Name of the notebook")
+    description: Optional[str] = Field(None, description="Description of the notebook")
+    archived: Optional[bool] = Field(
+        None, description="Whether the notebook is archived"
+    )
+
+
+class NotebookResponse(BaseModel):
+    id: str
+    name: str
+    description: str
+    archived: bool
+    created: str
+    updated: str
+    source_count: int
+    note_count: int
+
+
+# Search models
+class SearchRequest(BaseModel):
+    query: str = Field(..., description="Search query")
+    type: Literal["text", "vector"] = Field("text", description="Search type")
+    limit: int = Field(100, description="Maximum number of results", le=1000)
+    search_sources: bool = Field(True, description="Include sources in search")
+    search_notes: bool = Field(True, description="Include notes in search")
+    minimum_score: float = Field(
+        0.2, description="Minimum score for vector search", ge=0, le=1
+    )
+
+
+class SearchResponse(BaseModel):
+    results: List[Dict[str, Any]] = Field(..., description="Search results")
+    total_count: int = Field(..., description="Total number of results")
+    search_type: str = Field(..., description="Type of search performed")
+
+
+class AskRequest(BaseModel):
+    question: str = Field(..., description="Question to ask the knowledge base")
+    strategy_model: str = Field(..., description="Model ID for query strategy")
+    answer_model: str = Field(..., description="Model ID for individual answers")
+    final_answer_model: str = Field(..., description="Model ID for final answer")
+
+
+class AskResponse(BaseModel):
+    answer: str = Field(..., description="Final answer from the knowledge base")
+    question: str = Field(..., description="Original question")
+
+
+# Models API models
+class ModelCreate(BaseModel):
+    name: str = Field(..., description="Model name (e.g., gpt-5-mini, claude, gemini)")
+    provider: str = Field(
+        ..., description="Provider name (e.g., openai, anthropic, gemini)"
+    )
+    type: str = Field(
+        ...,
+        description="Model type (language, embedding, text_to_speech, speech_to_text)",
+    )
+    credential: Optional[str] = Field(
+        None, description="Credential ID to link this model to"
+    )
+
+
+class ModelResponse(BaseModel):
+    id: str
+    name: str
+    provider: str
+    type: str
+    credential: Optional[str] = None
+    created: str
+    updated: str
+
+
+class DefaultModelsResponse(BaseModel):
+    default_chat_model: Optional[str] = None
+    default_transformation_model: Optional[str] = None
+    large_context_model: Optional[str] = None
+    default_text_to_speech_model: Optional[str] = None
+    default_speech_to_text_model: Optional[str] = None
+    default_embedding_model: Optional[str] = None
+    default_tools_model: Optional[str] = None
+
+
+class ProviderAvailabilityResponse(BaseModel):
+    available: List[str] = Field(..., description="List of available providers")
+    unavailable: List[str] = Field(..., description="List of unavailable providers")
+    supported_types: Dict[str, List[str]] = Field(
+        ..., description="Provider to supported model types mapping"
+    )
+
+
+# Transformations API models
+class TransformationCreate(BaseModel):
+    name: str = Field(..., description="Transformation name")
+    title: str = Field(..., description="Display title for the transformation")
+    description: str = Field(
+        ..., description="Description of what this transformation does"
+    )
+    prompt: str = Field(..., description="The transformation prompt")
+    apply_default: bool = Field(
+        False, description="Whether to apply this transformation by default"
+    )
+
+
+class TransformationUpdate(BaseModel):
+    name: Optional[str] = Field(None, description="Transformation name")
+    title: Optional[str] = Field(
+        None, description="Display title for the transformation"
+    )
+    description: Optional[str] = Field(
+        None, description="Description of what this transformation does"
+    )
+    prompt: Optional[str] = Field(None, description="The transformation prompt")
+    apply_default: Optional[bool] = Field(
+        None, description="Whether to apply this transformation by default"
+    )
+
+
+class TransformationResponse(BaseModel):
+    id: str
+    name: str
+    title: str
+    description: str
+    prompt: str
+    apply_default: bool
+    created: str
+    updated: str
+
+
+class TransformationExecuteRequest(BaseModel):
+    model_config = ConfigDict(protected_namespaces=())
+
+    transformation_id: str = Field(
+        ..., description="ID of the transformation to execute"
+    )
+    input_text: str = Field(..., description="Text to transform")
+    model_id: str = Field(..., description="Model ID to use for the transformation")
+
+
+class TransformationExecuteResponse(BaseModel):
+    model_config = ConfigDict(protected_namespaces=())
+
+    output: str = Field(..., description="Transformed text")
+    transformation_id: str = Field(..., description="ID of the transformation used")
+    model_id: str = Field(..., description="Model ID used")
+
+
+# Default Prompt API models
+class DefaultPromptResponse(BaseModel):
+    transformation_instructions: str = Field(
+        ..., description="Default transformation instructions"
+    )
+
+
+class DefaultPromptUpdate(BaseModel):
+    transformation_instructions: str = Field(
+        ..., description="Default transformation instructions"
+    )
+
+
+# Notes API models
+class NoteCreate(BaseModel):
+    title: Optional[str] = Field(None, description="Note title")
+    content: str = Field(..., description="Note content")
+    note_type: Optional[str] = Field("human", description="Type of note (human, ai)")
+    notebook_id: Optional[str] = Field(
+        None, description="Notebook ID to add the note to"
+    )
+
+
+class NoteUpdate(BaseModel):
+    title: Optional[str] = Field(None, description="Note title")
+    content: Optional[str] = Field(None, description="Note content")
+    note_type: Optional[str] = Field(None, description="Type of note (human, ai)")
+
+
+class NoteResponse(BaseModel):
+    id: str
+    title: Optional[str]
+    content: Optional[str]
+    note_type: Optional[str]
+    created: str
+    updated: str
+    command_id: Optional[str] = None
+
+
+# Embedding API models
+class EmbedRequest(BaseModel):
+    item_id: str = Field(..., description="ID of the item to embed")
+    item_type: str = Field(..., description="Type of item (source, note)")
+    async_processing: bool = Field(
+        False, description="Process asynchronously in background"
+    )
+
+
+class EmbedResponse(BaseModel):
+    success: bool = Field(..., description="Whether embedding was successful")
+    message: str = Field(..., description="Result message")
+    item_id: str = Field(..., description="ID of the item that was embedded")
+    item_type: str = Field(..., description="Type of item that was embedded")
+    command_id: Optional[str] = Field(
+        None, description="Command ID for async processing"
+    )
+
+
+# Rebuild request/response models
+class RebuildRequest(BaseModel):
+    mode: Literal["existing", "all"] = Field(
+        ...,
+        description="Rebuild mode: 'existing' only re-embeds items with embeddings, 'all' embeds everything",
+    )
+    include_sources: bool = Field(True, description="Include sources in rebuild")
+    include_notes: bool = Field(True, description="Include notes in rebuild")
+    include_insights: bool = Field(True, description="Include insights in rebuild")
+
+
+class RebuildResponse(BaseModel):
+    command_id: str = Field(..., description="Command ID to track progress")
+    total_items: int = Field(..., description="Estimated number of items to process")
+    message: str = Field(..., description="Status message")
+
+
+class RebuildProgress(BaseModel):
+    processed: int = Field(..., description="Number of items processed")
+    total: int = Field(..., description="Total items to process")
+    percentage: float = Field(..., description="Progress percentage")
+
+
+class RebuildStats(BaseModel):
+    sources: int = Field(0, description="Sources processed")
+    notes: int = Field(0, description="Notes processed")
+    insights: int = Field(0, description="Insights processed")
+    failed: int = Field(0, description="Failed items")
+
+
+class RebuildStatusResponse(BaseModel):
+    command_id: str = Field(..., description="Command ID")
+    status: str = Field(..., description="Status: queued, running, completed, failed")
+    progress: Optional[RebuildProgress] = None
+    stats: Optional[RebuildStats] = None
+    started_at: Optional[str] = None
+    completed_at: Optional[str] = None
+    error_message: Optional[str] = None
+
+
+# Settings API models
+class SettingsResponse(BaseModel):
+    default_content_processing_engine_doc: Optional[str] = None
+    default_content_processing_engine_url: Optional[str] = None
+    default_embedding_option: Optional[str] = None
+    auto_delete_files: Optional[str] = None
+    youtube_preferred_languages: Optional[List[str]] = None
+
+
+class SettingsUpdate(BaseModel):
+    default_content_processing_engine_doc: Optional[str] = None
+    default_content_processing_engine_url: Optional[str] = None
+    default_embedding_option: Optional[str] = None
+    auto_delete_files: Optional[str] = None
+    youtube_preferred_languages: Optional[List[str]] = None
+
+
+# Sources API models
+class AssetModel(BaseModel):
+    file_path: Optional[str] = None
+    url: Optional[str] = None
+
+
+class SourceCreate(BaseModel):
+    # Backward compatibility: support old single notebook_id
+    notebook_id: Optional[str] = Field(
+        None, description="Notebook ID to add the source to (deprecated, use notebooks)"
+    )
+    # New multi-notebook support
+    notebooks: Optional[List[str]] = Field(
+        None, description="List of notebook IDs to add the source to"
+    )
+    # Required fields
+    type: str = Field(..., description="Source type: link, upload, or text")
+    url: Optional[str] = Field(None, description="URL for link type")
+    file_path: Optional[str] = Field(None, description="File path for upload type")
+    content: Optional[str] = Field(None, description="Text content for text type")
+    title: Optional[str] = Field(None, description="Source title")
+    transformations: Optional[List[str]] = Field(
+        default_factory=list, description="Transformation IDs to apply"
+    )
+    embed: bool = Field(False, description="Whether to embed content for vector search")
+    delete_source: bool = Field(
+        False, description="Whether to delete uploaded file after processing"
+    )
+    # New async processing support
+    async_processing: bool = Field(
+        False, description="Whether to process source asynchronously"
+    )
+
+    @model_validator(mode="after")
+    def validate_notebook_fields(self):
+        # Ensure only one of notebook_id or notebooks is provided
+        if self.notebook_id is not None and self.notebooks is not None:
+            raise ValueError(
+                "Cannot specify both 'notebook_id' and 'notebooks'. Use 'notebooks' for multi-notebook support."
+            )
+
+        # Convert single notebook_id to notebooks array for internal processing
+        if self.notebook_id is not None:
+            self.notebooks = [self.notebook_id]
+            # Keep notebook_id for backward compatibility in response
+
+        # Set empty array if no notebooks specified (allow sources without notebooks)
+        if self.notebooks is None:
+            self.notebooks = []
+
+        return self
+
+
+class SourceUpdate(BaseModel):
+    title: Optional[str] = Field(None, description="Source title")
+    topics: Optional[List[str]] = Field(None, description="Source topics")
+
+
+class SourceResponse(BaseModel):
+    id: str
+    title: Optional[str]
+    topics: Optional[List[str]]
+    asset: Optional[AssetModel]
+    full_text: Optional[str]
+    embedded: bool
+    embedded_chunks: int
+    file_available: Optional[bool] = None
+    created: str
+    updated: str
+    # New fields for async processing
+    command_id: Optional[str] = None
+    status: Optional[str] = None
+    processing_info: Optional[Dict] = None
+    # Notebook associations
+    notebooks: Optional[List[str]] = None
+
+
+class SourceListResponse(BaseModel):
+    id: str
+    title: Optional[str]
+    topics: Optional[List[str]]
+    asset: Optional[AssetModel]
+    embedded: bool  # Boolean flag indicating if source has embeddings
+    embedded_chunks: int  # Number of embedded chunks
+    insights_count: int
+    created: str
+    updated: str
+    file_available: Optional[bool] = None
+    # Status fields for async processing
+    command_id: Optional[str] = None
+    status: Optional[str] = None
+    processing_info: Optional[Dict[str, Any]] = None
+
+
+# Context API models
+class ContextConfig(BaseModel):
+    sources: Dict[str, str] = Field(
+        default_factory=dict, description="Source inclusion config {source_id: level}"
+    )
+    notes: Dict[str, str] = Field(
+        default_factory=dict, description="Note inclusion config {note_id: level}"
+    )
+
+
+class ContextRequest(BaseModel):
+    notebook_id: str = Field(..., description="Notebook ID to get context for")
+    context_config: Optional[ContextConfig] = Field(
+        None, description="Context configuration"
+    )
+
+
+class ContextResponse(BaseModel):
+    notebook_id: str
+    sources: List[Dict[str, Any]] = Field(..., description="Source context data")
+    notes: List[Dict[str, Any]] = Field(..., description="Note context data")
+    total_tokens: Optional[int] = Field(None, description="Estimated token count")
+
+
+# Insights API models
+class SourceInsightResponse(BaseModel):
+    id: str
+    source_id: str
+    insight_type: str
+    content: str
+    created: str
+    updated: str
+
+
+class InsightCreationResponse(BaseModel):
+    """Response for async insight creation."""
+
+    status: Literal["pending"] = "pending"
+    message: str = "Insight generation started"
+    source_id: str
+    transformation_id: str
+    command_id: Optional[str] = None
+
+
+class SaveAsNoteRequest(BaseModel):
+    notebook_id: Optional[str] = Field(None, description="Notebook ID to add note to")
+
+
+class CreateSourceInsightRequest(BaseModel):
+    model_config = ConfigDict(protected_namespaces=())
+
+    transformation_id: str = Field(..., description="ID of transformation to apply")
+    model_id: Optional[str] = Field(
+        None, description="Model ID (uses default if not provided)"
+    )
+
+
+# Source status response
+class SourceStatusResponse(BaseModel):
+    status: Optional[str] = Field(None, description="Processing status")
+    message: str = Field(..., description="Descriptive message about the status")
+    processing_info: Optional[Dict[str, Any]] = Field(
+        None, description="Detailed processing information"
+    )
+    command_id: Optional[str] = Field(None, description="Command ID if available")
+
+
+# Error response
+class ErrorResponse(BaseModel):
+    error: str
+    message: str
+
+
+# API Key Configuration models
+class SetApiKeyRequest(BaseModel):
+    """Request to set an API key for a provider."""
+
+    api_key: Optional[str] = Field(None, description="API key for the provider")
+    base_url: Optional[str] = Field(
+        None, description="Base URL for URL-based providers (Ollama, OpenAI-compatible)"
+    )
+    endpoint: Optional[str] = Field(
+        None, description="Endpoint URL for Azure OpenAI"
+    )
+    api_version: Optional[str] = Field(
+        None, description="API version for Azure OpenAI"
+    )
+    endpoint_llm: Optional[str] = Field(
+        None, description="Service-specific endpoint for LLM (Azure)"
+    )
+    endpoint_embedding: Optional[str] = Field(
+        None, description="Service-specific endpoint for embedding (Azure)"
+    )
+    endpoint_stt: Optional[str] = Field(
+        None, description="Service-specific endpoint for STT (Azure)"
+    )
+    endpoint_tts: Optional[str] = Field(
+        None, description="Service-specific endpoint for TTS (Azure)"
+    )
+    service_type: Optional[Literal["llm", "embedding", "stt", "tts"]] = Field(
+        None,
+        description="Service type for OpenAI-compatible providers (llm, embedding, stt, tts)",
+    )
+    # Vertex AI specific fields
+    vertex_project: Optional[str] = Field(
+        None, description="Google Cloud Project ID for Vertex AI"
+    )
+    vertex_location: Optional[str] = Field(
+        None, description="Google Cloud Region for Vertex AI (e.g., us-central1)"
+    )
+    vertex_credentials_path: Optional[str] = Field(
+        None, description="Path to Google Cloud service account JSON file"
+    )
+
+    @field_validator(
+        "api_key",
+        "base_url",
+        "endpoint",
+        "api_version",
+        "endpoint_llm",
+        "endpoint_embedding",
+        "endpoint_stt",
+        "endpoint_tts",
+        "vertex_project",
+        "vertex_location",
+        "vertex_credentials_path",
+        mode="before",
+    )
+    @classmethod
+    def validate_not_empty_string(cls, v: Optional[str]) -> Optional[str]:
+        """Reject empty strings - convert to None or raise error."""
+        if v is not None:
+            stripped = v.strip()
+            if not stripped:
+                return None  # Treat empty/whitespace-only as None
+            return stripped
+        return v
+
+
+class ApiKeyStatusResponse(BaseModel):
+    """Response showing which providers are configured and their source."""
+
+    configured: Dict[str, bool] = Field(
+        ..., description="Map of provider name to whether it is configured"
+    )
+    source: Dict[str, Literal["database", "environment", "none"]] = Field(
+        ...,
+        description="Map of provider name to configuration source (database, environment, or none)",
+    )
+    encryption_configured: bool = Field(
+        ...,
+        description="Whether OPEN_NOTEBOOK_ENCRYPTION_KEY is set (required to store keys in database)",
+    )
+
+
+class TestConnectionResponse(BaseModel):
+    """Response from testing a provider connection."""
+
+    provider: str = Field(..., description="Provider name that was tested")
+    success: bool = Field(..., description="Whether connection test succeeded")
+    message: str = Field(..., description="Result message with details")
+
+
+class MigrateFromEnvRequest(BaseModel):
+    """Request to migrate API keys from environment variables to database."""
+
+    force: bool = Field(
+        False, description="Force overwrite existing database configurations"
+    )
+
+
+class MigrationResult(BaseModel):
+    """Response from migrating API keys from environment to database."""
+
+    message: str = Field(..., description="Summary message")
+    migrated: List[str] = Field(
+        default_factory=list, description="Providers successfully migrated"
+    )
+    skipped: List[str] = Field(
+        default_factory=list, description="Providers skipped (already in DB)"
+    )
+    errors: List[str] = Field(
+        default_factory=list, description="Migration errors by provider"
+    )
+
+
+# Notebook delete cascade models
+# Credential models
+class CreateCredentialRequest(BaseModel):
+    """Request to create a new credential."""
+
+    name: str = Field(..., description="Credential name")
+    provider: str = Field(..., description="Provider name (openai, anthropic, etc.)")
+    modalities: List[str] = Field(
+        default_factory=list,
+        description="Supported modalities (language, embedding, text_to_speech, speech_to_text)",
+    )
+    api_key: Optional[str] = Field(None, description="API key (stored encrypted)")
+    base_url: Optional[str] = Field(None, description="Base URL")
+    endpoint: Optional[str] = Field(None, description="Endpoint URL (Azure)")
+    api_version: Optional[str] = Field(None, description="API version (Azure)")
+    endpoint_llm: Optional[str] = Field(None, description="LLM endpoint")
+    endpoint_embedding: Optional[str] = Field(None, description="Embedding endpoint")
+    endpoint_stt: Optional[str] = Field(None, description="STT endpoint")
+    endpoint_tts: Optional[str] = Field(None, description="TTS endpoint")
+    project: Optional[str] = Field(None, description="Project ID (Vertex)")
+    location: Optional[str] = Field(None, description="Location (Vertex)")
+    credentials_path: Optional[str] = Field(
+        None, description="Credentials file path (Vertex)"
+    )
+
+
+class UpdateCredentialRequest(BaseModel):
+    """Request to update an existing credential."""
+
+    name: Optional[str] = Field(None, description="Credential name")
+    modalities: Optional[List[str]] = Field(None, description="Supported modalities")
+    api_key: Optional[str] = Field(None, description="API key (stored encrypted)")
+    base_url: Optional[str] = Field(None, description="Base URL")
+    endpoint: Optional[str] = Field(None, description="Endpoint URL")
+    api_version: Optional[str] = Field(None, description="API version")
+    endpoint_llm: Optional[str] = Field(None, description="LLM endpoint")
+    endpoint_embedding: Optional[str] = Field(None, description="Embedding endpoint")
+    endpoint_stt: Optional[str] = Field(None, description="STT endpoint")
+    endpoint_tts: Optional[str] = Field(None, description="TTS endpoint")
+    project: Optional[str] = Field(None, description="Project ID")
+    location: Optional[str] = Field(None, description="Location")
+    credentials_path: Optional[str] = Field(None, description="Credentials path")
+
+
+class CredentialResponse(BaseModel):
+    """Response for a credential (never includes api_key)."""
+
+    id: str
+    name: str
+    provider: str
+    modalities: List[str]
+    base_url: Optional[str] = None
+    endpoint: Optional[str] = None
+    api_version: Optional[str] = None
+    endpoint_llm: Optional[str] = None
+    endpoint_embedding: Optional[str] = None
+    endpoint_stt: Optional[str] = None
+    endpoint_tts: Optional[str] = None
+    project: Optional[str] = None
+    location: Optional[str] = None
+    credentials_path: Optional[str] = None
+    has_api_key: bool = False
+    created: str
+    updated: str
+    model_count: int = 0
+    decryption_error: Optional[str] = None
+
+
+class CredentialDeleteResponse(BaseModel):
+    """Response for credential deletion."""
+
+    message: str
+    deleted_models: int = 0
+
+
+class DiscoveredModelResponse(BaseModel):
+    """A model discovered from a provider."""
+
+    name: str
+    provider: str
+    model_type: Optional[str] = None
+    description: Optional[str] = None
+
+
+class DiscoverModelsResponse(BaseModel):
+    """Response from model discovery."""
+
+    credential_id: str
+    provider: str
+    discovered: List[DiscoveredModelResponse]
+
+
+class RegisterModelData(BaseModel):
+    """A model to register with user-specified type."""
+
+    name: str
+    provider: str
+    model_type: str  # Required: user specifies the type
+
+
+class RegisterModelsRequest(BaseModel):
+    """Request to register discovered models."""
+
+    models: List[RegisterModelData]
+
+
+class RegisterModelsResponse(BaseModel):
+    """Response from model registration."""
+
+    created: int
+    existing: int
+
+
+class NotebookDeletePreview(BaseModel):
+    notebook_id: str = Field(..., description="ID of the notebook")
+    notebook_name: str = Field(..., description="Name of the notebook")
+    note_count: int = Field(..., description="Number of notes that will be deleted")
+    exclusive_source_count: int = Field(
+        ..., description="Number of sources only in this notebook"
+    )
+    shared_source_count: int = Field(
+        ..., description="Number of sources shared with other notebooks"
+    )
+
+
+class NotebookDeleteResponse(BaseModel):
+    message: str = Field(..., description="Success message")
+    deleted_notes: int = Field(..., description="Number of notes deleted")
+    deleted_sources: int = Field(..., description="Number of exclusive sources deleted")
+    unlinked_sources: int = Field(
+        ..., description="Number of sources unlinked from notebook"
+    )

api/models_service.py

@@ -0,0 +1,112 @@
+"""
+Models service layer using API.
+"""
+
+from typing import List, Optional
+
+from loguru import logger
+
+from api.client import api_client
+from open_notebook.ai.models import DefaultModels, Model
+
+
+class ModelsService:
+    """Service layer for models operations using API."""
+
+    def __init__(self):
+        logger.info("Using API for models operations")
+
+    def get_all_models(self, model_type: Optional[str] = None) -> List[Model]:
+        """Get all models with optional type filtering."""
+        models_data = api_client.get_models(model_type=model_type)
+        # Convert API response to Model objects
+        models = []
+        for model_data in models_data:
+            model = Model(
+                name=model_data["name"],
+                provider=model_data["provider"],
+                type=model_data["type"],
+            )
+            model.id = model_data["id"]
+            model.created = model_data["created"]
+            model.updated = model_data["updated"]
+            models.append(model)
+        return models
+
+    def create_model(self, name: str, provider: str, model_type: str) -> Model:
+        """Create a new model."""
+        response = api_client.create_model(name, provider, model_type)
+        model_data = response if isinstance(response, dict) else response[0]
+        model = Model(
+            name=model_data["name"],
+            provider=model_data["provider"],
+            type=model_data["type"],
+        )
+        model.id = model_data["id"]
+        model.created = model_data["created"]
+        model.updated = model_data["updated"]
+        return model
+
+    def delete_model(self, model_id: str) -> bool:
+        """Delete a model."""
+        api_client.delete_model(model_id)
+        return True
+
+    def get_default_models(self) -> DefaultModels:
+        """Get default model assignments."""
+        response = api_client.get_default_models()
+        defaults_data = response if isinstance(response, dict) else response[0]
+        defaults = DefaultModels()
+
+        # Set the values from API response
+        defaults.default_chat_model = defaults_data.get("default_chat_model")
+        defaults.default_transformation_model = defaults_data.get(
+            "default_transformation_model"
+        )
+        defaults.large_context_model = defaults_data.get("large_context_model")
+        defaults.default_text_to_speech_model = defaults_data.get(
+            "default_text_to_speech_model"
+        )
+        defaults.default_speech_to_text_model = defaults_data.get(
+            "default_speech_to_text_model"
+        )
+        defaults.default_embedding_model = defaults_data.get("default_embedding_model")
+        defaults.default_tools_model = defaults_data.get("default_tools_model")
+
+        return defaults
+
+    def update_default_models(self, defaults: DefaultModels) -> DefaultModels:
+        """Update default model assignments."""
+        updates = {
+            "default_chat_model": defaults.default_chat_model,
+            "default_transformation_model": defaults.default_transformation_model,
+            "large_context_model": defaults.large_context_model,
+            "default_text_to_speech_model": defaults.default_text_to_speech_model,
+            "default_speech_to_text_model": defaults.default_speech_to_text_model,
+            "default_embedding_model": defaults.default_embedding_model,
+            "default_tools_model": defaults.default_tools_model,
+        }
+
+        response = api_client.update_default_models(**updates)
+        defaults_data = response if isinstance(response, dict) else response[0]
+
+        # Update the defaults object with the response
+        defaults.default_chat_model = defaults_data.get("default_chat_model")
+        defaults.default_transformation_model = defaults_data.get(
+            "default_transformation_model"
+        )
+        defaults.large_context_model = defaults_data.get("large_context_model")
+        defaults.default_text_to_speech_model = defaults_data.get(
+            "default_text_to_speech_model"
+        )
+        defaults.default_speech_to_text_model = defaults_data.get(
+            "default_speech_to_text_model"
+        )
+        defaults.default_embedding_model = defaults_data.get("default_embedding_model")
+        defaults.default_tools_model = defaults_data.get("default_tools_model")
+
+        return defaults
+
+
+# Global service instance
+models_service = ModelsService()

api/notebook_service.py

@@ -0,0 +1,87 @@
+"""
+Notebook service layer using API.
+"""
+
+from typing import List, Optional
+
+from loguru import logger
+
+from api.client import api_client
+from open_notebook.domain.notebook import Notebook
+
+
+class NotebookService:
+    """Service layer for notebook operations using API."""
+
+    def __init__(self):
+        logger.info("Using API for notebook operations")
+
+    def get_all_notebooks(self, order_by: str = "updated desc") -> List[Notebook]:
+        """Get all notebooks."""
+        notebooks_data = api_client.get_notebooks(order_by=order_by)
+        # Convert API response to Notebook objects
+        notebooks = []
+        for nb_data in notebooks_data:
+            nb = Notebook(
+                name=nb_data["name"],
+                description=nb_data["description"],
+                archived=nb_data["archived"],
+            )
+            nb.id = nb_data["id"]
+            nb.created = nb_data["created"]
+            nb.updated = nb_data["updated"]
+            notebooks.append(nb)
+        return notebooks
+
+    def get_notebook(self, notebook_id: str) -> Optional[Notebook]:
+        """Get a specific notebook."""
+        response = api_client.get_notebook(notebook_id)
+        nb_data = response if isinstance(response, dict) else response[0]
+        nb = Notebook(
+            name=nb_data["name"],
+            description=nb_data["description"],
+            archived=nb_data["archived"],
+        )
+        nb.id = nb_data["id"]
+        nb.created = nb_data["created"]
+        nb.updated = nb_data["updated"]
+        return nb
+
+    def create_notebook(self, name: str, description: str = "") -> Notebook:
+        """Create a new notebook."""
+        response = api_client.create_notebook(name, description)
+        nb_data = response if isinstance(response, dict) else response[0]
+        nb = Notebook(
+            name=nb_data["name"],
+            description=nb_data["description"],
+            archived=nb_data["archived"],
+        )
+        nb.id = nb_data["id"]
+        nb.created = nb_data["created"]
+        nb.updated = nb_data["updated"]
+        return nb
+
+    def update_notebook(self, notebook: Notebook) -> Notebook:
+        """Update a notebook."""
+        updates = {
+            "name": notebook.name,
+            "description": notebook.description,
+            "archived": notebook.archived,
+        }
+        response = api_client.update_notebook(notebook.id or "", **updates)
+        nb_data = response if isinstance(response, dict) else response[0]
+        # Update the notebook object with the response
+        notebook.name = nb_data["name"]
+        notebook.description = nb_data["description"]
+        notebook.archived = nb_data["archived"]
+        notebook.updated = nb_data["updated"]
+        return notebook
+
+    def delete_notebook(self, notebook: Notebook) -> bool:
+        """Delete a notebook."""
+        api_client.delete_notebook(notebook.id or "")
+        return True
+
+
+# Global service instance
+notebook_service = NotebookService()

api/notes_service.py

@@ -0,0 +1,103 @@
+"""
+Notes service layer using API.
+"""
+
+from typing import List, Optional
+
+from loguru import logger
+
+from api.client import api_client
+from open_notebook.domain.notebook import Note
+
+
+class NotesService:
+    """Service layer for notes operations using API."""
+
+    def __init__(self):
+        logger.info("Using API for notes operations")
+
+    def get_all_notes(self, notebook_id: Optional[str] = None) -> List[Note]:
+        """Get all notes with optional notebook filtering."""
+        notes_data = api_client.get_notes(notebook_id=notebook_id)
+        # Convert API response to Note objects
+        notes = []
+        for note_data in notes_data:
+            note = Note(
+                title=note_data["title"],
+                content=note_data["content"],
+                note_type=note_data["note_type"],
+            )
+            note.id = note_data["id"]
+            note.created = note_data["created"]
+            note.updated = note_data["updated"]
+            notes.append(note)
+        return notes
+
+    def get_note(self, note_id: str) -> Note:
+        """Get a specific note."""
+        note_response = api_client.get_note(note_id)
+        note_data = (
+            note_response if isinstance(note_response, dict) else note_response[0]
+        )
+        note = Note(
+            title=note_data["title"],
+            content=note_data["content"],
+            note_type=note_data["note_type"],
+        )
+        note.id = note_data["id"]
+        note.created = note_data["created"]
+        note.updated = note_data["updated"]
+        return note
+
+    def create_note(
+        self,
+        content: str,
+        title: Optional[str] = None,
+        note_type: str = "human",
+        notebook_id: Optional[str] = None,
+    ) -> Note:
+        """Create a new note."""
+        note_response = api_client.create_note(
+            content=content, title=title, note_type=note_type, notebook_id=notebook_id
+        )
+        note_data = (
+            note_response if isinstance(note_response, dict) else note_response[0]
+        )
+        note = Note(
+            title=note_data["title"],
+            content=note_data["content"],
+            note_type=note_data["note_type"],
+        )
+        note.id = note_data["id"]
+        note.created = note_data["created"]
+        note.updated = note_data["updated"]
+        return note
+
+    def update_note(self, note: Note) -> Note:
+        """Update a note."""
+        updates = {
+            "title": note.title,
+            "content": note.content,
+            "note_type": note.note_type,
+        }
+        note_response = api_client.update_note(note.id or "", **updates)
+        note_data = (
+            note_response if isinstance(note_response, dict) else note_response[0]
+        )
+
+        # Update the note object with the response
+        note.title = note_data["title"]
+        note.content = note_data["content"]
+        note.note_type = note_data["note_type"]
+        note.updated = note_data["updated"]
+
+        return note
+
+    def delete_note(self, note_id: str) -> bool:
+        """Delete a note."""
+        api_client.delete_note(note_id)
+        return True
+
+
+# Global service instance
+notes_service = NotesService()

api/podcast_api_service.py

@@ -0,0 +1,125 @@
+"""
+Podcast service layer using API client.
+This replaces direct httpx calls in the Streamlit pages.
+"""
+
+from typing import Any, Dict, List
+
+from loguru import logger
+
+from api.client import api_client
+
+
+class PodcastAPIService:
+    """Service layer for podcast operations using API client."""
+
+    def __init__(self):
+        logger.info("Using API client for podcast operations")
+
+    # Episode methods
+    def get_episodes(self) -> List[Dict[Any, Any]]:
+        """Get all podcast episodes."""
+        result = api_client._make_request("GET", "/api/podcasts/episodes")
+        return result if isinstance(result, list) else [result]
+
+    def delete_episode(self, episode_id: str) -> bool:
+        """Delete a podcast episode."""
+        try:
+            api_client._make_request("DELETE", f"/api/podcasts/episodes/{episode_id}")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to delete episode: {e}")
+            return False
+
+    # Episode Profile methods
+    def get_episode_profiles(self) -> List[Dict]:
+        """Get all episode profiles."""
+        return api_client.get_episode_profiles()
+
+    def create_episode_profile(self, profile_data: Dict) -> bool:
+        """Create a new episode profile."""
+        try:
+            api_client.create_episode_profile(**profile_data)
+            return True
+        except Exception as e:
+            logger.error(f"Failed to create episode profile: {e}")
+            return False
+
+    def update_episode_profile(self, profile_id: str, profile_data: Dict) -> bool:
+        """Update an episode profile."""
+        try:
+            api_client.update_episode_profile(profile_id, **profile_data)
+            return True
+        except Exception as e:
+            logger.error(f"Failed to update episode profile: {e}")
+            return False
+
+    def delete_episode_profile(self, profile_id: str) -> bool:
+        """Delete an episode profile."""
+        try:
+            api_client.delete_episode_profile(profile_id)
+            return True
+        except Exception as e:
+            logger.error(f"Failed to delete episode profile: {e}")
+            return False
+
+    def duplicate_episode_profile(self, profile_id: str) -> bool:
+        """Duplicate an episode profile."""
+        try:
+            api_client._make_request(
+                "POST", f"/api/episode-profiles/{profile_id}/duplicate"
+            )
+            return True
+        except Exception as e:
+            logger.error(f"Failed to duplicate episode profile: {e}")
+            return False
+
+    # Speaker Profile methods
+    def get_speaker_profiles(self) -> List[Dict[Any, Any]]:
+        """Get all speaker profiles."""
+        result = api_client._make_request("GET", "/api/speaker-profiles")
+        return result if isinstance(result, list) else [result]
+
+    def create_speaker_profile(self, profile_data: Dict) -> bool:
+        """Create a new speaker profile."""
+        try:
+            api_client._make_request("POST", "/api/speaker-profiles", json=profile_data)
+            return True
+        except Exception as e:
+            logger.error(f"Failed to create speaker profile: {e}")
+            return False
+
+    def update_speaker_profile(self, profile_id: str, profile_data: Dict) -> bool:
+        """Update a speaker profile."""
+        try:
+            api_client._make_request(
+                "PUT", f"/api/speaker-profiles/{profile_id}", json=profile_data
+            )
+            return True
+        except Exception as e:
+            logger.error(f"Failed to update speaker profile: {e}")
+            return False
+
+    def delete_speaker_profile(self, profile_id: str) -> bool:
+        """Delete a speaker profile."""
+        try:
+            api_client._make_request("DELETE", f"/api/speaker-profiles/{profile_id}")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to delete speaker profile: {e}")
+            return False
+
+    def duplicate_speaker_profile(self, profile_id: str) -> bool:
+        """Duplicate a speaker profile."""
+        try:
+            api_client._make_request(
+                "POST", f"/api/speaker-profiles/{profile_id}/duplicate"
+            )
+            return True
+        except Exception as e:
+            logger.error(f"Failed to duplicate speaker profile: {e}")
+            return False
+
+
+# Global service instance
+podcast_api_service = PodcastAPIService()

api/podcast_service.py

@@ -0,0 +1,206 @@
+from typing import Any, Dict, Optional
+
+from fastapi import HTTPException
+from loguru import logger
+from pydantic import BaseModel
+from surreal_commands import get_command_status, submit_command
+
+from open_notebook.domain.notebook import Notebook
+from open_notebook.podcasts.models import EpisodeProfile, PodcastEpisode, SpeakerProfile
+
+
+class PodcastGenerationRequest(BaseModel):
+    """Request model for podcast generation"""
+
+    episode_profile: str
+    speaker_profile: str
+    episode_name: str
+    content: Optional[str] = None
+    notebook_id: Optional[str] = None
+    briefing_suffix: Optional[str] = None
+
+
+class PodcastGenerationResponse(BaseModel):
+    """Response model for podcast generation"""
+
+    job_id: str
+    status: str
+    message: str
+    episode_profile: str
+    episode_name: str
+
+
+class PodcastService:
+    """Service layer for podcast operations"""
+
+    @staticmethod
+    async def submit_generation_job(
+        episode_profile_name: str,
+        speaker_profile_name: str,
+        episode_name: str,
+        notebook_id: Optional[str] = None,
+        content: Optional[str] = None,
+        briefing_suffix: Optional[str] = None,
+    ) -> str:
+        """Submit a podcast generation job for background processing"""
+        try:
+            # Validate episode profile exists
+            episode_profile = await EpisodeProfile.get_by_name(episode_profile_name)
+            if not episode_profile:
+                raise ValueError(f"Episode profile '{episode_profile_name}' not found")
+
+            # Validate speaker profile exists
+            speaker_profile = await SpeakerProfile.get_by_name(speaker_profile_name)
+            if not speaker_profile:
+                raise ValueError(f"Speaker profile '{speaker_profile_name}' not found")
+
+            # Get content from notebook if not provided directly
+            if not content and notebook_id:
+                try:
+                    notebook = await Notebook.get(notebook_id)
+                    # Get notebook context (this may need to be adjusted based on actual Notebook implementation)
+                    content = (
+                        await notebook.get_context()
+                        if hasattr(notebook, "get_context")
+                        else str(notebook)
+                    )
+                except Exception as e:
+                    logger.warning(
+                        f"Failed to get notebook content, using notebook_id as content: {e}"
+                    )
+                    content = f"Notebook ID: {notebook_id}"
+
+            if not content:
+                raise ValueError(
+                    "Content is required - provide either content or notebook_id"
+                )
+
+            # Prepare command arguments
+            command_args = {
+                "episode_profile": episode_profile_name,
+                "speaker_profile": speaker_profile_name,
+                "episode_name": episode_name,
+                "content": str(content),
+                "briefing_suffix": briefing_suffix,
+            }
+
+            # Ensure command modules are imported before submitting
+            # This is needed because submit_command validates against local registry
+            try:
+                import commands.podcast_commands  # noqa: F401
+            except ImportError as import_err:
+                logger.error(f"Failed to import podcast commands: {import_err}")
+                raise ValueError("Podcast commands not available")
+
+            # Submit command to surreal-commands
+            job_id = submit_command("open_notebook", "generate_podcast", command_args)
+
+            # Convert RecordID to string if needed
+            if not job_id:
+                raise ValueError("Failed to get job_id from submit_command")
+            job_id_str = str(job_id)
+            logger.info(
+                f"Submitted podcast generation job: {job_id_str} for episode '{episode_name}'"
+            )
+            return job_id_str
+
+        except Exception as e:
+            logger.error(f"Failed to submit podcast generation job: {e}")
+            raise HTTPException(
+                status_code=500,
+                detail=f"Failed to submit podcast generation job: {str(e)}",
+            )
+
+    @staticmethod
+    async def get_job_status(job_id: str) -> Dict[str, Any]:
+        """Get status of a podcast generation job"""
+        try:
+            status = await get_command_status(job_id)
+            return {
+                "job_id": job_id,
+                "status": status.status if status else "unknown",
+                "result": status.result if status else None,
+                "error_message": getattr(status, "error_message", None)
+                if status
+                else None,
+                "created": str(status.created)
+                if status and hasattr(status, "created") and status.created
+                else None,
+                "updated": str(status.updated)
+                if status and hasattr(status, "updated") and status.updated
+                else None,
+                "progress": getattr(status, "progress", None) if status else None,
+            }
+        except Exception as e:
+            logger.error(f"Failed to get podcast job status: {e}")
+            raise HTTPException(
+                status_code=500, detail=f"Failed to get job status: {str(e)}"
+            )
+
+    @staticmethod
+    async def list_episodes() -> list:
+        """List all podcast episodes"""
+        try:
+            episodes = await PodcastEpisode.get_all(order_by="created desc")
+            return episodes
+        except Exception as e:
+            logger.error(f"Failed to list podcast episodes: {e}")
+            raise HTTPException(
+                status_code=500, detail=f"Failed to list episodes: {str(e)}"
+            )
+
+    @staticmethod
+    async def get_episode(episode_id: str) -> PodcastEpisode:
+        """Get a specific podcast episode"""
+        try:
+            episode = await PodcastEpisode.get(episode_id)
+            return episode
+        except Exception as e:
+            logger.error(f"Failed to get podcast episode {episode_id}: {e}")
+            raise HTTPException(status_code=404, detail=f"Episode not found: {str(e)}")
+
+
+class DefaultProfiles:
+    """Utility class for creating default profiles (if needed beyond migration data)"""
+
+    @staticmethod
+    async def create_default_episode_profiles():
+        """Create default episode profiles if they don't exist"""
+        try:
+            # Check if profiles already exist
+            existing = await EpisodeProfile.get_all()
+            if existing:
+                logger.info(f"Episode profiles already exist: {len(existing)} found")
+                return existing
+
+            # This would create profiles, but since we have migration data,
+            # this is mainly for future extensibility
+            logger.info(
+                "Default episode profiles should be created via database migration"
+            )
+            return []
+
+        except Exception as e:
+            logger.error(f"Failed to create default episode profiles: {e}")
+            raise
+
+    @staticmethod
+    async def create_default_speaker_profiles():
+        """Create default speaker profiles if they don't exist"""
+        try:
+            # Check if profiles already exist
+            existing = await SpeakerProfile.get_all()
+            if existing:
+                logger.info(f"Speaker profiles already exist: {len(existing)} found")
+                return existing
+
+            # This would create profiles, but since we have migration data,
+            # this is mainly for future extensibility
+            logger.info(
+                "Default speaker profiles should be created via database migration"
+            )
+            return []
+
+        except Exception as e:
+            logger.error(f"Failed to create default speaker profiles: {e}")
+            raise

api/routers/auth.py

@@ -0,0 +1,27 @@
+"""
+Authentication router for Open Notebook API.
+Provides endpoints to check authentication status.
+"""
+
+from fastapi import APIRouter
+
+from open_notebook.utils.encryption import get_secret_from_env
+
+router = APIRouter(prefix="/auth", tags=["auth"])
+
+
+@router.get("/status")
+async def get_auth_status():
+    """
+    Check if authentication is enabled.
+    Returns whether a password is required to access the API.
+    Supports Docker secrets via OPEN_NOTEBOOK_PASSWORD_FILE.
+    """
+    auth_enabled = bool(get_secret_from_env("OPEN_NOTEBOOK_PASSWORD"))
+
+    return {
+        "auth_enabled": auth_enabled,
+        "message": "Authentication is required"
+        if auth_enabled
+        else "Authentication is disabled",
+    }

api/routers/chat.py

@@ -0,0 +1,526 @@
+import asyncio
+import traceback
+from typing import Any, Dict, List, Optional
+
+from fastapi import APIRouter, HTTPException, Query
+from langchain_core.runnables import RunnableConfig
+from loguru import logger
+from pydantic import BaseModel, Field
+
+from open_notebook.database.repository import ensure_record_id, repo_query
+from open_notebook.domain.notebook import ChatSession, Note, Notebook, Source
+from open_notebook.exceptions import (
+    NotFoundError,
+)
+from open_notebook.graphs.chat import graph as chat_graph
+from open_notebook.utils.graph_utils import get_session_message_count
+
+router = APIRouter()
+
+
+# Request/Response models
+class CreateSessionRequest(BaseModel):
+    notebook_id: str = Field(..., description="Notebook ID to create session for")
+    title: Optional[str] = Field(None, description="Optional session title")
+    model_override: Optional[str] = Field(
+        None, description="Optional model override for this session"
+    )
+
+
+class UpdateSessionRequest(BaseModel):
+    title: Optional[str] = Field(None, description="New session title")
+    model_override: Optional[str] = Field(
+        None, description="Model override for this session"
+    )
+
+
+class ChatMessage(BaseModel):
+    id: str = Field(..., description="Message ID")
+    type: str = Field(..., description="Message type (human|ai)")
+    content: str = Field(..., description="Message content")
+    timestamp: Optional[str] = Field(None, description="Message timestamp")
+
+
+class ChatSessionResponse(BaseModel):
+    id: str = Field(..., description="Session ID")
+    title: str = Field(..., description="Session title")
+    notebook_id: Optional[str] = Field(None, description="Notebook ID")
+    created: str = Field(..., description="Creation timestamp")
+    updated: str = Field(..., description="Last update timestamp")
+    message_count: Optional[int] = Field(
+        None, description="Number of messages in session"
+    )
+    model_override: Optional[str] = Field(
+        None, description="Model override for this session"
+    )
+
+
+class ChatSessionWithMessagesResponse(ChatSessionResponse):
+    messages: List[ChatMessage] = Field(
+        default_factory=list, description="Session messages"
+    )
+
+
+class ExecuteChatRequest(BaseModel):
+    session_id: str = Field(..., description="Chat session ID")
+    message: str = Field(..., description="User message content")
+    context: Dict[str, Any] = Field(
+        ..., description="Chat context with sources and notes"
+    )
+    model_override: Optional[str] = Field(
+        None, description="Optional model override for this message"
+    )
+
+
+class ExecuteChatResponse(BaseModel):
+    session_id: str = Field(..., description="Session ID")
+    messages: List[ChatMessage] = Field(..., description="Updated message list")
+
+
+class BuildContextRequest(BaseModel):
+    notebook_id: str = Field(..., description="Notebook ID")
+    context_config: Dict[str, Any] = Field(..., description="Context configuration")
+
+
+class BuildContextResponse(BaseModel):
+    context: Dict[str, Any] = Field(..., description="Built context data")
+    token_count: int = Field(..., description="Estimated token count")
+    char_count: int = Field(..., description="Character count")
+
+
+class SuccessResponse(BaseModel):
+    success: bool = Field(True, description="Operation success status")
+    message: str = Field(..., description="Success message")
+
+
+@router.get("/chat/sessions", response_model=List[ChatSessionResponse])
+async def get_sessions(notebook_id: str = Query(..., description="Notebook ID")):
+    """Get all chat sessions for a notebook."""
+    try:
+        # Get notebook to verify it exists
+        notebook = await Notebook.get(notebook_id)
+        if not notebook:
+            raise HTTPException(status_code=404, detail="Notebook not found")
+
+        # Get sessions for this notebook
+        sessions_list = await notebook.get_chat_sessions()
+
+        results = []
+        for session in sessions_list:
+            session_id = str(session.id)
+
+            # Get message count from LangGraph state
+            msg_count = await get_session_message_count(chat_graph, session_id)
+
+            results.append(
+                ChatSessionResponse(
+                    id=session.id or "",
+                    title=session.title or "Untitled Session",
+                    notebook_id=notebook_id,
+                    created=str(session.created),
+                    updated=str(session.updated),
+                    message_count=msg_count,
+                    model_override=getattr(session, "model_override", None),
+                )
+            )
+
+        return results
+    except NotFoundError:
+        raise HTTPException(status_code=404, detail="Notebook not found")
+    except Exception as e:
+        logger.error(f"Error fetching chat sessions: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error fetching chat sessions: {str(e)}"
+        )
+
+
+@router.post("/chat/sessions", response_model=ChatSessionResponse)
+async def create_session(request: CreateSessionRequest):
+    """Create a new chat session."""
+    try:
+        # Verify notebook exists
+        notebook = await Notebook.get(request.notebook_id)
+        if not notebook:
+            raise HTTPException(status_code=404, detail="Notebook not found")
+
+        # Create new session
+        session = ChatSession(
+            title=request.title
+            or f"Chat Session {asyncio.get_event_loop().time():.0f}",
+            model_override=request.model_override,
+        )
+        await session.save()
+
+        # Relate session to notebook
+        await session.relate_to_notebook(request.notebook_id)
+
+        return ChatSessionResponse(
+            id=session.id or "",
+            title=session.title or "",
+            notebook_id=request.notebook_id,
+            created=str(session.created),
+            updated=str(session.updated),
+            message_count=0,
+            model_override=session.model_override,
+        )
+    except NotFoundError:
+        raise HTTPException(status_code=404, detail="Notebook not found")
+    except Exception as e:
+        logger.error(f"Error creating chat session: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error creating chat session: {str(e)}"
+        )
+
+
+@router.get(
+    "/chat/sessions/{session_id}", response_model=ChatSessionWithMessagesResponse
+)
+async def get_session(session_id: str):
+    """Get a specific session with its messages."""
+    try:
+        # Get session
+        # Ensure session_id has proper table prefix
+        full_session_id = (
+            session_id
+            if session_id.startswith("chat_session:")
+            else f"chat_session:{session_id}"
+        )
+        session = await ChatSession.get(full_session_id)
+        if not session:
+            raise HTTPException(status_code=404, detail="Session not found")
+
+        # Get session state from LangGraph to retrieve messages
+        # Use sync get_state() in a thread since SqliteSaver doesn't support async
+        thread_state = await asyncio.to_thread(
+            chat_graph.get_state,
+            config=RunnableConfig(configurable={"thread_id": full_session_id}),
+        )
+
+        # Extract messages from state
+        messages: list[ChatMessage] = []
+        if thread_state and thread_state.values and "messages" in thread_state.values:
+            for msg in thread_state.values["messages"]:
+                messages.append(
+                    ChatMessage(
+                        id=getattr(msg, "id", f"msg_{len(messages)}"),
+                        type=msg.type if hasattr(msg, "type") else "unknown",
+                        content=msg.content if hasattr(msg, "content") else str(msg),
+                        timestamp=None,  # LangChain messages don't have timestamps by default
+                    )
+                )
+
+        # Find notebook_id (we need to query the relationship)
+        # Ensure session_id has proper table prefix
+        full_session_id = (
+            session_id
+            if session_id.startswith("chat_session:")
+            else f"chat_session:{session_id}"
+        )
+
+        notebook_query = await repo_query(
+            "SELECT out FROM refers_to WHERE in = $session_id",
+            {"session_id": ensure_record_id(full_session_id)},
+        )
+
+        notebook_id = notebook_query[0]["out"] if notebook_query else None
+
+        if not notebook_id:
+            # This might be an old session created before API migration
+            logger.warning(
+                f"No notebook relationship found for session {session_id} - may be an orphaned session"
+            )
+
+        return ChatSessionWithMessagesResponse(
+            id=session.id or "",
+            title=session.title or "Untitled Session",
+            notebook_id=notebook_id,
+            created=str(session.created),
+            updated=str(session.updated),
+            message_count=len(messages),
+            messages=messages,
+            model_override=getattr(session, "model_override", None),
+        )
+    except NotFoundError:
+        raise HTTPException(status_code=404, detail="Session not found")
+    except Exception as e:
+        logger.error(f"Error fetching session: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error fetching session: {str(e)}")
+
+
+@router.put("/chat/sessions/{session_id}", response_model=ChatSessionResponse)
+async def update_session(session_id: str, request: UpdateSessionRequest):
+    """Update session title."""
+    try:
+        # Ensure session_id has proper table prefix
+        full_session_id = (
+            session_id
+            if session_id.startswith("chat_session:")
+            else f"chat_session:{session_id}"
+        )
+        session = await ChatSession.get(full_session_id)
+        if not session:
+            raise HTTPException(status_code=404, detail="Session not found")
+
+        update_data = request.model_dump(exclude_unset=True)
+
+        if "title" in update_data:
+            session.title = update_data["title"]
+
+        if "model_override" in update_data:
+            session.model_override = update_data["model_override"]
+
+        await session.save()
+
+        # Find notebook_id
+        # Ensure session_id has proper table prefix
+        full_session_id = (
+            session_id
+            if session_id.startswith("chat_session:")
+            else f"chat_session:{session_id}"
+        )
+        notebook_query = await repo_query(
+            "SELECT out FROM refers_to WHERE in = $session_id",
+            {"session_id": ensure_record_id(full_session_id)},
+        )
+        notebook_id = notebook_query[0]["out"] if notebook_query else None
+
+        # Get message count from LangGraph state
+        msg_count = await get_session_message_count(chat_graph, full_session_id)
+
+        return ChatSessionResponse(
+            id=session.id or "",
+            title=session.title or "",
+            notebook_id=notebook_id,
+            created=str(session.created),
+            updated=str(session.updated),
+            message_count=msg_count,
+            model_override=session.model_override,
+        )
+    except NotFoundError:
+        raise HTTPException(status_code=404, detail="Session not found")
+    except Exception as e:
+        logger.error(f"Error updating session: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error updating session: {str(e)}")
+
+
+@router.delete("/chat/sessions/{session_id}", response_model=SuccessResponse)
+async def delete_session(session_id: str):
+    """Delete a chat session."""
+    try:
+        # Ensure session_id has proper table prefix
+        full_session_id = (
+            session_id
+            if session_id.startswith("chat_session:")
+            else f"chat_session:{session_id}"
+        )
+        session = await ChatSession.get(full_session_id)
+        if not session:
+            raise HTTPException(status_code=404, detail="Session not found")
+
+        await session.delete()
+
+        return SuccessResponse(success=True, message="Session deleted successfully")
+    except NotFoundError:
+        raise HTTPException(status_code=404, detail="Session not found")
+    except Exception as e:
+        logger.error(f"Error deleting session: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error deleting session: {str(e)}")
+
+
+@router.post("/chat/execute", response_model=ExecuteChatResponse)
+async def execute_chat(request: ExecuteChatRequest):
+    """Execute a chat request and get AI response."""
+    try:
+        # Verify session exists
+        # Ensure session_id has proper table prefix
+        full_session_id = (
+            request.session_id
+            if request.session_id.startswith("chat_session:")
+            else f"chat_session:{request.session_id}"
+        )
+        session = await ChatSession.get(full_session_id)
+        if not session:
+            raise HTTPException(status_code=404, detail="Session not found")
+
+        # Fetch notebook linked to this session
+        notebook_query = await repo_query(
+            "SELECT out FROM refers_to WHERE in = $session_id",
+            {"session_id": ensure_record_id(full_session_id)},
+        )
+        notebook = None
+        if notebook_query:
+            notebook = await Notebook.get(notebook_query[0]["out"])
+
+        # Determine model override (per-request override takes precedence over session-level)
+        model_override = (
+            request.model_override
+            if request.model_override is not None
+            else getattr(session, "model_override", None)
+        )
+
+        # Get current state
+        # Use sync get_state() in a thread since SqliteSaver doesn't support async
+        current_state = await asyncio.to_thread(
+            chat_graph.get_state,
+            config=RunnableConfig(configurable={"thread_id": full_session_id}),
+        )
+
+        # Prepare state for execution
+        state_values = current_state.values if current_state else {}
+        state_values["messages"] = state_values.get("messages", [])
+        state_values["context"] = request.context
+        state_values["notebook"] = notebook
+        state_values["model_override"] = model_override
+
+        # Add user message to state
+        from langchain_core.messages import HumanMessage
+
+        user_message = HumanMessage(content=request.message)
+        state_values["messages"].append(user_message)
+
+        # Execute chat graph
+        result = chat_graph.invoke(
+            input=state_values,  # type: ignore[arg-type]
+            config=RunnableConfig(
+                configurable={
+                    "thread_id": full_session_id,
+                    "model_id": model_override,
+                }
+            ),
+        )
+
+        # Update session timestamp
+        await session.save()
+
+        # Convert messages to response format
+        messages: list[ChatMessage] = []
+        for msg in result.get("messages", []):
+            messages.append(
+                ChatMessage(
+                    id=getattr(msg, "id", f"msg_{len(messages)}"),
+                    type=msg.type if hasattr(msg, "type") else "unknown",
+                    content=msg.content if hasattr(msg, "content") else str(msg),
+                    timestamp=None,
+                )
+            )
+
+        return ExecuteChatResponse(session_id=request.session_id, messages=messages)
+    except NotFoundError:
+        raise HTTPException(status_code=404, detail="Session not found")
+    except Exception as e:
+        # Log detailed error with context for debugging
+        logger.error(
+            f"Error executing chat: {str(e)}\n"
+            f"  Session ID: {request.session_id}\n"
+            f"  Model override: {request.model_override}\n"
+            f"  Traceback:\n{traceback.format_exc()}"
+        )
+        raise HTTPException(status_code=500, detail=f"Error executing chat: {str(e)}")
+
+
+@router.post("/chat/context", response_model=BuildContextResponse)
+async def build_context(request: BuildContextRequest):
+    """Build context for a notebook based on context configuration."""
+    try:
+        # Verify notebook exists
+        notebook = await Notebook.get(request.notebook_id)
+        if not notebook:
+            raise HTTPException(status_code=404, detail="Notebook not found")
+
+        context_data: dict[str, list[dict[str, str]]] = {"sources": [], "notes": []}
+        total_content = ""
+
+        # Process context configuration if provided
+        if request.context_config:
+            # Process sources
+            for source_id, status in request.context_config.get("sources", {}).items():
+                if "not in" in status:
+                    continue
+
+                try:
+                    # Add table prefix if not present
+                    full_source_id = (
+                        source_id
+                        if source_id.startswith("source:")
+                        else f"source:{source_id}"
+                    )
+
+                    try:
+                        source = await Source.get(full_source_id)
+                    except Exception:
+                        continue
+
+                    if "insights" in status:
+                        source_context = await source.get_context(context_size="short")
+                        context_data["sources"].append(source_context)
+                        total_content += str(source_context)
+                    elif "full content" in status:
+                        source_context = await source.get_context(context_size="long")
+                        context_data["sources"].append(source_context)
+                        total_content += str(source_context)
+                except Exception as e:
+                    logger.warning(f"Error processing source {source_id}: {str(e)}")
+                    continue
+
+            # Process notes
+            for note_id, status in request.context_config.get("notes", {}).items():
+                if "not in" in status:
+                    continue
+
+                try:
+                    # Add table prefix if not present
+                    full_note_id = (
+                        note_id if note_id.startswith("note:") else f"note:{note_id}"
+                    )
+                    note = await Note.get(full_note_id)
+                    if not note:
+                        continue
+
+                    if "full content" in status:
+                        note_context = note.get_context(context_size="long")
+                        context_data["notes"].append(note_context)
+                        total_content += str(note_context)
+                except Exception as e:
+                    logger.warning(f"Error processing note {note_id}: {str(e)}")
+                    continue
+        else:
+            # Default behavior - include all sources and notes with short context
+            sources = await notebook.get_sources()
+            for source in sources:
+                try:
+                    source_context = await source.get_context(context_size="short")
+                    context_data["sources"].append(source_context)
+                    total_content += str(source_context)
+                except Exception as e:
+                    logger.warning(f"Error processing source {source.id}: {str(e)}")
+                    continue
+
+            notes = await notebook.get_notes()
+            for note in notes:
+                try:
+                    note_context = note.get_context(context_size="short")
+                    context_data["notes"].append(note_context)
+                    total_content += str(note_context)
+                except Exception as e:
+                    logger.warning(f"Error processing note {note.id}: {str(e)}")
+                    continue
+
+        # Calculate character and token counts
+        char_count = len(total_content)
+        # Use token count utility if available
+        try:
+            from open_notebook.utils import token_count
+
+            estimated_tokens = token_count(total_content) if total_content else 0
+        except ImportError:
+            # Fallback to simple estimation
+            estimated_tokens = char_count // 4
+
+        return BuildContextResponse(
+            context=context_data, token_count=estimated_tokens, char_count=char_count
+        )
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error building context: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error building context: {str(e)}")

api/routers/commands.py

@@ -0,0 +1,166 @@
+from typing import Any, Dict, List, Optional
+
+from fastapi import APIRouter, HTTPException, Query
+from loguru import logger
+from pydantic import BaseModel, Field
+from surreal_commands import registry
+
+from api.command_service import CommandService
+
+router = APIRouter()
+
+
+class CommandExecutionRequest(BaseModel):
+    command: str = Field(
+        ..., description="Command function name (e.g., 'process_text')"
+    )
+    app: str = Field(..., description="Application name (e.g., 'open_notebook')")
+    input: Dict[str, Any] = Field(..., description="Arguments to pass to the command")
+
+
+class CommandJobResponse(BaseModel):
+    job_id: str
+    status: str
+    message: str
+
+
+class CommandJobStatusResponse(BaseModel):
+    job_id: str
+    status: str
+    result: Optional[Dict[str, Any]] = None
+    error_message: Optional[str] = None
+    created: Optional[str] = None
+    updated: Optional[str] = None
+    progress: Optional[Dict[str, Any]] = None
+
+
+@router.post("/commands/jobs", response_model=CommandJobResponse)
+async def execute_command(request: CommandExecutionRequest):
+    """
+    Submit a command for background processing.
+    Returns immediately with job ID for status tracking.
+
+    Example request:
+    {
+        "command": "process_text",
+        "app": "open_notebook",
+        "input": {
+            "text": "Hello world",
+            "operation": "uppercase"
+        }
+    }
+    """
+    try:
+        # Submit command using app name (not module name)
+        job_id = await CommandService.submit_command_job(
+            module_name=request.app,  # This should be "open_notebook"
+            command_name=request.command,
+            command_args=request.input,
+        )
+
+        return CommandJobResponse(
+            job_id=job_id,
+            status="submitted",
+            message=f"Command '{request.command}' submitted successfully",
+        )
+
+    except Exception as e:
+        logger.error(f"Error submitting command: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail="Failed to submit command"
+        )
+
+
+@router.get("/commands/jobs/{job_id}", response_model=CommandJobStatusResponse)
+async def get_command_job_status(job_id: str):
+    """Get the status of a specific command job"""
+    try:
+        status_data = await CommandService.get_command_status(job_id)
+        return CommandJobStatusResponse(**status_data)
+
+    except Exception as e:
+        logger.error(f"Error fetching job status: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail="Failed to fetch job status"
+        )
+
+
+@router.get("/commands/jobs", response_model=List[Dict[str, Any]])
+async def list_command_jobs(
+    command_filter: Optional[str] = Query(None, description="Filter by command name"),
+    status_filter: Optional[str] = Query(None, description="Filter by status"),
+    limit: int = Query(50, description="Maximum number of jobs to return"),
+):
+    """List command jobs with optional filtering"""
+    try:
+        jobs = await CommandService.list_command_jobs(
+            command_filter=command_filter, status_filter=status_filter, limit=limit
+        )
+        return jobs
+
+    except Exception as e:
+        logger.error(f"Error listing command jobs: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail="Failed to list command jobs"
+        )
+
+
+@router.delete("/commands/jobs/{job_id}")
+async def cancel_command_job(job_id: str):
+    """Cancel a running command job"""
+    try:
+        success = await CommandService.cancel_command_job(job_id)
+        return {"job_id": job_id, "cancelled": success}
+
+    except Exception as e:
+        logger.error(f"Error cancelling command job: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail="Failed to cancel command job"
+        )
+
+
+@router.get("/commands/registry/debug")
+async def debug_registry():
+    """Debug endpoint to see what commands are registered"""
+    try:
+        # Get all registered commands
+        all_items = registry.get_all_commands()
+
+        # Create JSON-serializable data
+        command_items = []
+        for item in all_items:
+            try:
+                command_items.append(
+                    {
+                        "app_id": item.app_id,
+                        "name": item.name,
+                        "full_id": f"{item.app_id}.{item.name}",
+                    }
+                )
+            except Exception as item_error:
+                logger.error(f"Error processing item: {item_error}")
+
+        # Get the basic command structure
+        try:
+            commands_dict: dict[str, list[str]] = {}
+            for item in all_items:
+                if item.app_id not in commands_dict:
+                    commands_dict[item.app_id] = []
+                commands_dict[item.app_id].append(item.name)
+        except Exception:
+            commands_dict = {}
+
+        return {
+            "total_commands": len(all_items),
+            "commands_by_app": commands_dict,
+            "command_items": command_items,
+        }
+
+    except Exception as e:
+        logger.error(f"Error debugging registry: {str(e)}")
+        return {
+            "error": str(e),
+            "total_commands": 0,
+            "commands_by_app": {},
+            "command_items": [],
+        }

api/routers/config.py

@@ -0,0 +1,160 @@
+import asyncio
+import os
+import time
+import tomllib
+from pathlib import Path
+from typing import Optional
+
+from fastapi import APIRouter, Request
+from loguru import logger
+
+from open_notebook.database.repository import repo_query
+from open_notebook.utils.version_utils import (
+    compare_versions,
+    get_version_from_github_async,
+)
+
+router = APIRouter()
+
+# In-memory cache for version check results
+_version_cache: dict = {
+    "latest_version": None,
+    "has_update": False,
+    "timestamp": 0,
+    "check_failed": False,
+}
+
+# Cache TTL in seconds (24 hours)
+VERSION_CACHE_TTL = 24 * 60 * 60
+
+
+def get_version() -> str:
+    """Read version from pyproject.toml"""
+    try:
+        pyproject_path = Path(__file__).parent.parent.parent / "pyproject.toml"
+        with open(pyproject_path, "rb") as f:
+            pyproject = tomllib.load(f)
+            return pyproject.get("project", {}).get("version", "unknown")
+    except Exception as e:
+        logger.warning(f"Could not read version from pyproject.toml: {e}")
+        return "unknown"
+
+
+async def get_latest_version_cached(current_version: str) -> tuple[Optional[str], bool]:
+    """
+    Check for the latest version from GitHub with caching.
+
+    Returns:
+        tuple: (latest_version, has_update)
+        - latest_version: str or None if check failed
+        - has_update: bool indicating if update is available
+    """
+    global _version_cache
+
+    # Check if cache is still valid (within TTL)
+    cache_age = time.time() - _version_cache["timestamp"]
+    if _version_cache["timestamp"] > 0 and cache_age < VERSION_CACHE_TTL:
+        logger.debug(f"Using cached version check result (age: {cache_age:.0f}s)")
+        return _version_cache["latest_version"], _version_cache["has_update"]
+
+    # Cache expired or not yet set
+    if _version_cache["timestamp"] > 0:
+        logger.info(f"Version cache expired (age: {cache_age:.0f}s), refreshing...")
+
+    # Perform version check with strict error handling
+    try:
+        logger.info("Checking for latest version from GitHub...")
+
+        # Fetch latest version from GitHub with 10-second timeout
+        latest_version = await get_version_from_github_async(
+            "https://github.com/lfnovo/open-notebook", "main"
+        )
+
+        logger.info(
+            f"Latest version from GitHub: {latest_version}, Current version: {current_version}"
+        )
+
+        # Compare versions
+        has_update = compare_versions(current_version, latest_version) < 0
+
+        # Cache the result
+        _version_cache["latest_version"] = latest_version
+        _version_cache["has_update"] = has_update
+        _version_cache["timestamp"] = time.time()
+        _version_cache["check_failed"] = False
+
+        logger.info(f"Version check complete. Update available: {has_update}")
+
+        return latest_version, has_update
+
+    except Exception as e:
+        logger.warning(f"Version check failed: {e}")
+
+        # Cache the failure to avoid repeated attempts
+        _version_cache["latest_version"] = None
+        _version_cache["has_update"] = False
+        _version_cache["timestamp"] = time.time()
+        _version_cache["check_failed"] = True
+
+        return None, False
+
+
+async def check_database_health() -> dict:
+    """
+    Check if database is reachable using a lightweight query.
+
+    Returns:
+        dict with 'status' ("online" | "offline") and optional 'error'
+    """
+    try:
+        # 2-second timeout for database health check
+        result = await asyncio.wait_for(repo_query("RETURN 1"), timeout=2.0)
+        if result:
+            return {"status": "online"}
+        return {"status": "offline", "error": "Empty result"}
+    except asyncio.TimeoutError:
+        logger.warning("Database health check timed out after 2 seconds")
+        return {"status": "offline", "error": "Health check timeout"}
+    except Exception as e:
+        logger.warning(f"Database health check failed: {e}")
+        return {"status": "offline", "error": str(e)}
+
+
+@router.get("/config")
+async def get_config(request: Request):
+    """
+    Get frontend configuration.
+
+    Returns version information and health status.
+    Note: The frontend determines the API URL via its own runtime-config endpoint,
+    so this endpoint no longer returns apiUrl.
+
+    Also checks for version updates from GitHub (with caching and error handling).
+    """
+    # Get current version
+    current_version = get_version()
+
+    # Check for updates (with caching and error handling)
+    # This MUST NOT break the endpoint - wrapped in try-except as extra safety
+    latest_version = None
+    has_update = False
+
+    try:
+        latest_version, has_update = await get_latest_version_cached(current_version)
+    except Exception as e:
+        # Extra safety: ensure version check never breaks the config endpoint
+        logger.error(f"Unexpected error during version check: {e}")
+
+    # Check database health
+    db_health = await check_database_health()
+    db_status = db_health["status"]
+
+    if db_status == "offline":
+        logger.warning(f"Database offline: {db_health.get('error', 'Unknown error')}")
+
+    return {
+        "version": current_version,
+        "latestVersion": latest_version,
+        "hasUpdate": has_update,
+        "dbStatus": db_status,
+    }

api/routers/context.py

@@ -0,0 +1,115 @@
+from fastapi import APIRouter, HTTPException
+from loguru import logger
+
+from api.models import ContextRequest, ContextResponse
+from open_notebook.domain.notebook import Note, Notebook, Source
+from open_notebook.exceptions import InvalidInputError
+from open_notebook.utils import token_count
+
+router = APIRouter()
+
+
+@router.post("/notebooks/{notebook_id}/context", response_model=ContextResponse)
+async def get_notebook_context(notebook_id: str, context_request: ContextRequest):
+    """Get context for a notebook based on configuration."""
+    try:
+        # Verify notebook exists
+        notebook = await Notebook.get(notebook_id)
+        if not notebook:
+            raise HTTPException(status_code=404, detail="Notebook not found")
+
+        context_data: dict[str, list[dict[str, str]]] = {"note": [], "source": []}
+        total_content = ""
+
+        # Process context configuration if provided
+        if context_request.context_config:
+            # Process sources
+            for source_id, status in context_request.context_config.sources.items():
+                if "not in" in status:
+                    continue
+
+                try:
+                    # Add table prefix if not present
+                    full_source_id = (
+                        source_id
+                        if source_id.startswith("source:")
+                        else f"source:{source_id}"
+                    )
+
+                    try:
+                        source = await Source.get(full_source_id)
+                    except Exception:
+                        continue
+
+                    if "insights" in status:
+                        source_context = await source.get_context(context_size="short")
+                        context_data["source"].append(source_context)
+                        total_content += str(source_context)
+                    elif "full content" in status:
+                        source_context = await source.get_context(context_size="long")
+                        context_data["source"].append(source_context)
+                        total_content += str(source_context)
+                except Exception as e:
+                    logger.warning(f"Error processing source {source_id}: {str(e)}")
+                    continue
+
+            # Process notes
+            for note_id, status in context_request.context_config.notes.items():
+                if "not in" in status:
+                    continue
+
+                try:
+                    # Add table prefix if not present
+                    full_note_id = (
+                        note_id if note_id.startswith("note:") else f"note:{note_id}"
+                    )
+                    note = await Note.get(full_note_id)
+                    if not note:
+                        continue
+
+                    if "full content" in status:
+                        note_context = note.get_context(context_size="long")
+                        context_data["note"].append(note_context)
+                        total_content += str(note_context)
+                except Exception as e:
+                    logger.warning(f"Error processing note {note_id}: {str(e)}")
+                    continue
+        else:
+            # Default behavior - include all sources and notes with short context
+            sources = await notebook.get_sources()
+            for source in sources:
+                try:
+                    source_context = await source.get_context(context_size="short")
+                    context_data["source"].append(source_context)
+                    total_content += str(source_context)
+                except Exception as e:
+                    logger.warning(f"Error processing source {source.id}: {str(e)}")
+                    continue
+
+            notes = await notebook.get_notes()
+            for note in notes:
+                try:
+                    note_context = note.get_context(context_size="short")
+                    context_data["note"].append(note_context)
+                    total_content += str(note_context)
+                except Exception as e:
+                    logger.warning(f"Error processing note {note.id}: {str(e)}")
+                    continue
+
+        # Calculate estimated token count
+        estimated_tokens = token_count(total_content) if total_content else 0
+
+        return ContextResponse(
+            notebook_id=notebook_id,
+            sources=context_data["source"],
+            notes=context_data["note"],
+            total_tokens=estimated_tokens,
+        )
+
+    except HTTPException:
+        raise
+    except InvalidInputError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        logger.error(f"Error getting context for notebook {notebook_id}: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error getting context: {str(e)}")

api/routers/credentials.py

@@ -0,0 +1,426 @@
+"""
+Credentials Router
+
+Thin HTTP layer for managing individual AI provider credentials.
+Business logic lives in api.credentials_service.
+
+Endpoints:
+- GET /credentials - List all credentials
+- GET /credentials/by-provider/{provider} - List credentials for a provider
+- POST /credentials - Create a new credential
+- GET /credentials/{credential_id} - Get a specific credential
+- PUT /credentials/{credential_id} - Update a credential
+- DELETE /credentials/{credential_id} - Delete a credential
+- POST /credentials/{credential_id}/test - Test connection
+- POST /credentials/{credential_id}/discover - Discover models
+- POST /credentials/{credential_id}/register-models - Register models
+
+NEVER returns actual API key values - only metadata.
+"""
+
+from typing import List, Optional
+
+from fastapi import APIRouter, HTTPException, Query
+from loguru import logger
+from pydantic import SecretStr
+
+from api.credentials_service import (
+    credential_to_response,
+    discover_with_config,
+    get_provider_status,
+    register_models,
+    require_encryption_key,
+    validate_url,
+)
+from api.credentials_service import (
+    get_env_status as svc_get_env_status,
+)
+from api.credentials_service import (
+    migrate_from_env as svc_migrate_from_env,
+)
+from api.credentials_service import (
+    migrate_from_provider_config as svc_migrate_from_provider_config,
+)
+from api.credentials_service import (
+    test_credential as svc_test_credential,
+)
+from api.models import (
+    CreateCredentialRequest,
+    CredentialDeleteResponse,
+    CredentialResponse,
+    DiscoveredModelResponse,
+    DiscoverModelsResponse,
+    RegisterModelsRequest,
+    RegisterModelsResponse,
+    UpdateCredentialRequest,
+)
+from open_notebook.database.repository import ensure_record_id, repo_delete, repo_query
+from open_notebook.domain.credential import Credential
+
+router = APIRouter(prefix="/credentials", tags=["credentials"])
+
+
+def _handle_value_error(e: ValueError, status_code: int = 400) -> HTTPException:
+    """Convert a ValueError from the service layer to an HTTPException."""
+    return HTTPException(status_code=status_code, detail=str(e))
+
+
+# =============================================================================
+# Status endpoints
+# =============================================================================
+
+
+@router.get("/status")
+async def get_status():
+    """
+    Get configuration status: encryption key status, and per-provider
+    configured/source information.
+    """
+    try:
+        return await get_provider_status()
+    except Exception as e:
+        logger.error(f"Error fetching status: {e}")
+        raise HTTPException(status_code=500, detail="Failed to fetch credential status")
+
+
+@router.get("/env-status")
+async def get_env_status():
+    """Check what's configured via environment variables."""
+    try:
+        return await svc_get_env_status()
+    except Exception as e:
+        logger.error(f"Error checking env status: {e}")
+        raise HTTPException(status_code=500, detail="Failed to check environment status")
+
+
+# =============================================================================
+# CRUD endpoints
+# =============================================================================
+
+
+@router.get("", response_model=List[CredentialResponse])
+async def list_credentials(
+    provider: Optional[str] = Query(None, description="Filter by provider"),
+):
+    """List all credentials, optionally filtered by provider."""
+    try:
+        if provider:
+            credentials = await Credential.get_by_provider(provider)
+        else:
+            credentials = await Credential.get_all(order_by="provider, created")
+
+        result = []
+        for cred in credentials:
+            models = await cred.get_linked_models()
+            result.append(credential_to_response(cred, len(models)))
+
+        return result
+
+    except Exception as e:
+        logger.error(f"Error listing credentials: {e}")
+        raise HTTPException(status_code=500, detail="Failed to list credentials")
+
+
+@router.get("/by-provider/{provider}", response_model=List[CredentialResponse])
+async def list_credentials_by_provider(provider: str):
+    """List all credentials for a specific provider."""
+    try:
+        credentials = await Credential.get_by_provider(provider.lower())
+        result = []
+        for cred in credentials:
+            models = await cred.get_linked_models()
+            result.append(credential_to_response(cred, len(models)))
+        return result
+    except Exception as e:
+        logger.error(f"Error listing credentials for {provider}: {e}")
+        raise HTTPException(status_code=500, detail="Failed to list credentials for provider")
+
+
+@router.post("", response_model=CredentialResponse, status_code=201)
+async def create_credential(request: CreateCredentialRequest):
+    """Create a new credential."""
+    try:
+        require_encryption_key()
+    except ValueError as e:
+        raise _handle_value_error(e)
+
+    # Validate all URL fields
+    for url_field in [
+        request.base_url, request.endpoint, request.endpoint_llm,
+        request.endpoint_embedding, request.endpoint_stt, request.endpoint_tts,
+    ]:
+        if url_field:
+            try:
+                validate_url(url_field, request.provider)
+            except ValueError as e:
+                raise _handle_value_error(e)
+
+    try:
+        cred = Credential(
+            name=request.name,
+            provider=request.provider.lower(),
+            modalities=request.modalities,
+            api_key=SecretStr(request.api_key) if request.api_key else None,
+            base_url=request.base_url,
+            endpoint=request.endpoint,
+            api_version=request.api_version,
+            endpoint_llm=request.endpoint_llm,
+            endpoint_embedding=request.endpoint_embedding,
+            endpoint_stt=request.endpoint_stt,
+            endpoint_tts=request.endpoint_tts,
+            project=request.project,
+            location=request.location,
+            credentials_path=request.credentials_path,
+        )
+        await cred.save()
+        return credential_to_response(cred, 0)
+
+    except Exception as e:
+        logger.error(f"Error creating credential: {e}")
+        raise HTTPException(status_code=500, detail="Failed to create credential")
+
+
+@router.get("/{credential_id}", response_model=CredentialResponse)
+async def get_credential(credential_id: str):
+    """Get a specific credential by ID. Never returns api_key."""
+    try:
+        cred = await Credential.get(credential_id)
+        models = await cred.get_linked_models()
+        return credential_to_response(cred, len(models))
+    except Exception as e:
+        logger.error(f"Error fetching credential {credential_id}: {e}")
+        raise HTTPException(status_code=404, detail="Credential not found")
+
+
+@router.put("/{credential_id}", response_model=CredentialResponse)
+async def update_credential(credential_id: str, request: UpdateCredentialRequest):
+    """Update an existing credential."""
+    try:
+        require_encryption_key()
+    except ValueError as e:
+        raise _handle_value_error(e)
+
+    # Validate all URL fields being updated
+    for url_field in [
+        request.base_url, request.endpoint, request.endpoint_llm,
+        request.endpoint_embedding, request.endpoint_stt, request.endpoint_tts,
+    ]:
+        if url_field:
+            try:
+                validate_url(url_field, "update")
+            except ValueError as e:
+                raise _handle_value_error(e)
+
+    try:
+        cred = await Credential.get(credential_id)
+
+        if request.name is not None:
+            cred.name = request.name
+        if request.modalities is not None:
+            cred.modalities = request.modalities
+        if request.api_key is not None:
+            cred.api_key = SecretStr(request.api_key)
+        if request.base_url is not None:
+            cred.base_url = request.base_url or None
+        if request.endpoint is not None:
+            cred.endpoint = request.endpoint or None
+        if request.api_version is not None:
+            cred.api_version = request.api_version or None
+        if request.endpoint_llm is not None:
+            cred.endpoint_llm = request.endpoint_llm or None
+        if request.endpoint_embedding is not None:
+            cred.endpoint_embedding = request.endpoint_embedding or None
+        if request.endpoint_stt is not None:
+            cred.endpoint_stt = request.endpoint_stt or None
+        if request.endpoint_tts is not None:
+            cred.endpoint_tts = request.endpoint_tts or None
+        if request.project is not None:
+            cred.project = request.project or None
+        if request.location is not None:
+            cred.location = request.location or None
+        if request.credentials_path is not None:
+            cred.credentials_path = request.credentials_path or None
+
+        await cred.save()
+        models = await cred.get_linked_models()
+        return credential_to_response(cred, len(models))
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error updating credential {credential_id}: {e}")
+        raise HTTPException(status_code=500, detail="Failed to update credential")
+
+
+@router.delete("/{credential_id}", response_model=CredentialDeleteResponse)
+async def delete_credential(
+    credential_id: str,
+    migrate_to: Optional[str] = Query(
+        None, description="Migrate linked models to this credential ID"
+    ),
+):
+    """
+    Delete a credential.
+
+    If the credential has linked models:
+    - Pass migrate_to=<credential_id> to reassign them to another credential
+    - Otherwise, linked models are cascade-deleted automatically
+    """
+    try:
+        try:
+            cred = await Credential.get(credential_id)
+        except ValueError as decrypt_err:
+            # Credential exists but can't be decrypted (wrong encryption key).
+            # Fall back to direct DB operations for deletion.
+            logger.warning(
+                f"Cannot decrypt credential {credential_id}, "
+                f"falling back to direct delete: {decrypt_err}"
+            )
+
+            # Query linked models
+            linked = await repo_query(
+                "SELECT * FROM model WHERE credential = $cred_id",
+                {"cred_id": ensure_record_id(credential_id)},
+            )
+            deleted_models = 0
+
+            if linked and migrate_to:
+                # Migrate models to another credential
+                target_cred = await Credential.get(migrate_to)
+                for model_row in linked:
+                    model_id = str(model_row.get("id", ""))
+                    if model_id:
+                        await repo_query(
+                            "UPDATE $model_id SET credential = $target_id",
+                            {
+                                "model_id": ensure_record_id(model_id),
+                                "target_id": ensure_record_id(target_cred.id),
+                            },
+                        )
+            elif linked:
+                # Cascade-delete linked models
+                for model_row in linked:
+                    model_id = str(model_row.get("id", ""))
+                    if model_id:
+                        await repo_delete(model_id)
+                        deleted_models += 1
+
+            # Delete the credential itself
+            await repo_delete(credential_id)
+
+            return CredentialDeleteResponse(
+                message="Credential deleted successfully",
+                deleted_models=deleted_models,
+            )
+
+        linked_models = await cred.get_linked_models()
+
+        deleted_models = 0
+
+        if linked_models and migrate_to:
+            # Migrate models to another credential
+            target_cred = await Credential.get(migrate_to)
+            for model in linked_models:
+                model.credential = target_cred.id
+                await model.save()
+
+        elif linked_models:
+            # Cascade-delete linked models (default behavior when no migrate_to)
+            for model in linked_models:
+                await model.delete()
+                deleted_models += 1
+
+        # Delete the credential
+        await cred.delete()
+
+        return CredentialDeleteResponse(
+            message="Credential deleted successfully",
+            deleted_models=deleted_models,
+        )
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error deleting credential {credential_id}: {e}")
+        raise HTTPException(status_code=500, detail="Failed to delete credential")
+
+
+# =============================================================================
+# Test / Discover / Register endpoints
+# =============================================================================
+
+
+@router.post("/{credential_id}/test")
+async def test_credential(credential_id: str):
+    """Test connection using this credential's configuration."""
+    return await svc_test_credential(credential_id)
+
+
+@router.post("/{credential_id}/discover", response_model=DiscoverModelsResponse)
+async def discover_models_for_credential(credential_id: str):
+    """Discover available models using this credential's API key."""
+    try:
+        cred = await Credential.get(credential_id)
+        config = cred.to_esperanto_config()
+        provider = cred.provider.lower()
+
+        discovered = await discover_with_config(provider, config)
+
+        return DiscoverModelsResponse(
+            credential_id=cred.id or "",
+            provider=provider,
+            discovered=[
+                DiscoveredModelResponse(
+                    name=d["name"],
+                    provider=d["provider"],
+                    description=d.get("description"),
+                )
+                for d in discovered
+            ],
+        )
+
+    except Exception as e:
+        logger.error(f"Error discovering models for credential {credential_id}: {e}")
+        raise HTTPException(status_code=500, detail="Failed to discover models")
+
+
+@router.post("/{credential_id}/register-models", response_model=RegisterModelsResponse)
+async def register_models_for_credential(
+    credential_id: str, request: RegisterModelsRequest
+):
+    """Register discovered models and link them to this credential."""
+    try:
+        result = await register_models(credential_id, request.models)
+        return RegisterModelsResponse(**result)
+    except Exception as e:
+        logger.error(f"Error registering models for credential {credential_id}: {e}")
+        raise HTTPException(status_code=500, detail="Failed to register models")
+
+
+# =============================================================================
+# Migration endpoints
+# =============================================================================
+
+
+@router.post("/migrate-from-provider-config")
+async def migrate_from_provider_config():
+    """Migrate existing ProviderConfig data to individual credential records."""
+    try:
+        return await svc_migrate_from_provider_config()
+    except ValueError as e:
+        raise _handle_value_error(e)
+    except Exception as e:
+        logger.error(f"ProviderConfig migration FAILED: {type(e).__name__}: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail="Migration from provider config failed")
+
+
+@router.post("/migrate-from-env")
+async def migrate_from_env():
+    """Migrate API keys from environment variables to credential records."""
+    try:
+        return await svc_migrate_from_env()
+    except ValueError as e:
+        raise _handle_value_error(e)
+    except Exception as e:
+        logger.error(f"Env migration FAILED: {type(e).__name__}: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail="Migration from environment variables failed")

api/routers/embedding.py

@@ -0,0 +1,113 @@
+from fastapi import APIRouter, HTTPException
+from loguru import logger
+
+from api.command_service import CommandService
+from api.models import EmbedRequest, EmbedResponse
+from open_notebook.ai.models import model_manager
+from open_notebook.domain.notebook import Note, Source
+
+router = APIRouter()
+
+
+@router.post("/embed", response_model=EmbedResponse)
+async def embed_content(embed_request: EmbedRequest):
+    """Embed content for vector search."""
+    try:
+        # Check if embedding model is available
+        if not await model_manager.get_embedding_model():
+            raise HTTPException(
+                status_code=400,
+                detail="No embedding model configured. Please configure one in the Models section.",
+            )
+
+        item_id = embed_request.item_id
+        item_type = embed_request.item_type.lower()
+
+        # Validate item type
+        if item_type not in ["source", "note"]:
+            raise HTTPException(
+                status_code=400, detail="Item type must be either 'source' or 'note'"
+            )
+
+        # Branch based on processing mode
+        if embed_request.async_processing:
+            # ASYNC PATH: Submit command for background processing
+            logger.info(f"Using async processing for {item_type} {item_id}")
+
+            try:
+                # Import commands to ensure they're registered
+                import commands.embedding_commands  # noqa: F401
+
+                # Submit type-specific command
+                if item_type == "source":
+                    command_name = "embed_source"
+                    command_input = {"source_id": item_id}
+                else:  # note
+                    command_name = "embed_note"
+                    command_input = {"note_id": item_id}
+
+                command_id = await CommandService.submit_command_job(
+                    "open_notebook",
+                    command_name,
+                    command_input,
+                )
+
+                logger.info(f"Submitted async {command_name} command: {command_id}")
+
+                return EmbedResponse(
+                    success=True,
+                    message="Embedding queued for background processing",
+                    item_id=item_id,
+                    item_type=item_type,
+                    command_id=command_id,
+                )
+
+            except Exception as e:
+                logger.error(f"Failed to submit async embedding command: {e}")
+                raise HTTPException(
+                    status_code=500, detail=f"Failed to queue embedding: {str(e)}"
+                )
+
+        else:
+            # DOMAIN MODEL PATH: Submit job via domain model convenience methods
+            # These methods internally call submit_command() - still fire-and-forget
+            logger.info(f"Using domain model path for {item_type} {item_id}")
+
+            command_id = None
+
+            # Get the item and submit embedding job
+            if item_type == "source":
+                source_item = await Source.get(item_id)
+                if not source_item:
+                    raise HTTPException(status_code=404, detail="Source not found")
+
+                # Submit embed_source job (returns command_id for tracking)
+                command_id = await source_item.vectorize()
+                message = "Source embedding job submitted"
+
+            elif item_type == "note":
+                note_item = await Note.get(item_id)
+                if not note_item:
+                    raise HTTPException(status_code=404, detail="Note not found")
+
+                # Note.save() internally submits embed_note command and returns command_id
+                command_id = await note_item.save()
+                message = "Note embedding job submitted"
+
+            return EmbedResponse(
+                success=True,
+                message=message,
+                item_id=item_id,
+                item_type=item_type,
+                command_id=command_id,
+            )
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(
+            f"Error embedding {embed_request.item_type} {embed_request.item_id}: {str(e)}"
+        )
+        raise HTTPException(
+            status_code=500, detail=f"Error embedding content: {str(e)}"
+        )

api/routers/embedding_rebuild.py

@@ -0,0 +1,192 @@
+from fastapi import APIRouter, HTTPException
+from loguru import logger
+from surreal_commands import get_command_status
+
+from api.command_service import CommandService
+from api.models import (
+    RebuildProgress,
+    RebuildRequest,
+    RebuildResponse,
+    RebuildStats,
+    RebuildStatusResponse,
+)
+from open_notebook.database.repository import repo_query
+
+router = APIRouter()
+
+
+@router.post("/rebuild", response_model=RebuildResponse)
+async def start_rebuild(request: RebuildRequest):
+    """
+    Start a background job to rebuild embeddings.
+
+    - **mode**: "existing" (re-embed items with embeddings) or "all" (embed everything)
+    - **include_sources**: Include sources in rebuild (default: true)
+    - **include_notes**: Include notes in rebuild (default: true)
+    - **include_insights**: Include insights in rebuild (default: true)
+
+    Returns command ID to track progress and estimated item count.
+    """
+    try:
+        logger.info(f"Starting rebuild request: mode={request.mode}")
+
+        # Import commands to ensure they're registered
+        import commands.embedding_commands  # noqa: F401
+
+        # Estimate total items (quick count query)
+        # This is a rough estimate before the command runs
+        total_estimate = 0
+
+        if request.include_sources:
+            if request.mode == "existing":
+                # Count sources with embeddings
+                result = await repo_query(
+                    """
+                    SELECT VALUE count(array::distinct(
+                        SELECT VALUE source.id
+                        FROM source_embedding
+                        WHERE embedding != none AND array::len(embedding) > 0
+                    )) as count FROM {}
+                    """
+                )
+            else:
+                # Count all sources with content
+                result = await repo_query(
+                    "SELECT VALUE count() as count FROM source WHERE full_text != none GROUP ALL"
+                )
+
+            if result and isinstance(result[0], dict):
+                total_estimate += result[0].get("count", 0)
+            elif result:
+                total_estimate += result[0] if isinstance(result[0], int) else 0
+
+        if request.include_notes:
+            if request.mode == "existing":
+                result = await repo_query(
+                    "SELECT VALUE count() as count FROM note WHERE embedding != none AND array::len(embedding) > 0 GROUP ALL"
+                )
+            else:
+                result = await repo_query(
+                    "SELECT VALUE count() as count FROM note WHERE content != none GROUP ALL"
+                )
+
+            if result and isinstance(result[0], dict):
+                total_estimate += result[0].get("count", 0)
+            elif result:
+                total_estimate += result[0] if isinstance(result[0], int) else 0
+
+        if request.include_insights:
+            if request.mode == "existing":
+                result = await repo_query(
+                    "SELECT VALUE count() as count FROM source_insight WHERE embedding != none AND array::len(embedding) > 0 GROUP ALL"
+                )
+            else:
+                result = await repo_query(
+                    "SELECT VALUE count() as count FROM source_insight GROUP ALL"
+                )
+
+            if result and isinstance(result[0], dict):
+                total_estimate += result[0].get("count", 0)
+            elif result:
+                total_estimate += result[0] if isinstance(result[0], int) else 0
+
+        logger.info(f"Estimated {total_estimate} items to process")
+
+        # Submit command
+        command_id = await CommandService.submit_command_job(
+            "open_notebook",
+            "rebuild_embeddings",
+            {
+                "mode": request.mode,
+                "include_sources": request.include_sources,
+                "include_notes": request.include_notes,
+                "include_insights": request.include_insights,
+            },
+        )
+
+        logger.info(f"Submitted rebuild command: {command_id}")
+
+        return RebuildResponse(
+            command_id=command_id,
+            total_items=total_estimate,
+            message=f"Rebuild operation started. Estimated {total_estimate} items to process.",
+        )
+
+    except Exception as e:
+        logger.error(f"Failed to start rebuild: {e}")
+        logger.exception(e)
+        raise HTTPException(
+            status_code=500, detail=f"Failed to start rebuild operation: {str(e)}"
+        )
+
+
+@router.get("/rebuild/{command_id}/status", response_model=RebuildStatusResponse)
+async def get_rebuild_status(command_id: str):
+    """
+    Get the status of a rebuild operation.
+
+    Returns:
+    - **status**: queued, running, completed, failed
+    - **progress**: processed count, total count, percentage
+    - **stats**: breakdown by type (sources, notes, insights, failed)
+    - **timestamps**: started_at, completed_at
+    """
+    try:
+        # Get command status from surreal_commands
+        status = await get_command_status(command_id)
+
+        if not status:
+            raise HTTPException(status_code=404, detail="Rebuild command not found")
+
+        # Build response based on status
+        response = RebuildStatusResponse(
+            command_id=command_id,
+            status=status.status,
+        )
+
+        # Extract metadata from command result
+        if status.result and isinstance(status.result, dict):
+            result = status.result
+
+            # Build progress info
+            if "total_items" in result and "jobs_submitted" in result:
+                total = result["total_items"]
+                submitted = result["jobs_submitted"]
+                response.progress = RebuildProgress(
+                    processed=submitted,
+                    total=total,
+                    percentage=round((submitted / total * 100) if total > 0 else 0, 2),
+                )
+
+            # Build stats
+            response.stats = RebuildStats(
+                sources=result.get("sources_submitted", 0),
+                notes=result.get("notes_submitted", 0),
+                insights=result.get("insights_submitted", 0),
+                failed=result.get("failed_submissions", 0),
+            )
+
+        # Add timestamps
+        if hasattr(status, "created") and status.created:
+            response.started_at = str(status.created)
+        if hasattr(status, "updated") and status.updated:
+            response.completed_at = str(status.updated)
+
+        # Add error message if failed
+        if (
+            status.status == "failed"
+            and status.result
+            and isinstance(status.result, dict)
+        ):
+            response.error_message = status.result.get("error_message", "Unknown error")
+
+        return response
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Failed to get rebuild status: {e}")
+        logger.exception(e)
+        raise HTTPException(
+            status_code=500, detail=f"Failed to get rebuild status: {str(e)}"
+        )

api/routers/episode_profiles.py

@@ -0,0 +1,226 @@
+from typing import List, Optional
+
+from fastapi import APIRouter, HTTPException
+from loguru import logger
+from pydantic import BaseModel, Field
+
+from open_notebook.podcasts.models import EpisodeProfile
+
+router = APIRouter()
+
+
+class EpisodeProfileResponse(BaseModel):
+    id: str
+    name: str
+    description: str
+    speaker_config: str
+    outline_llm: Optional[str] = None
+    transcript_llm: Optional[str] = None
+    language: Optional[str] = None
+    default_briefing: str
+    num_segments: int
+    # Legacy fields (for display/migration awareness)
+    outline_provider: Optional[str] = None
+    outline_model: Optional[str] = None
+    transcript_provider: Optional[str] = None
+    transcript_model: Optional[str] = None
+
+
+def _profile_to_response(profile: EpisodeProfile) -> EpisodeProfileResponse:
+    return EpisodeProfileResponse(
+        id=str(profile.id),
+        name=profile.name,
+        description=profile.description or "",
+        speaker_config=profile.speaker_config,
+        outline_llm=profile.outline_llm,
+        transcript_llm=profile.transcript_llm,
+        language=profile.language,
+        default_briefing=profile.default_briefing,
+        num_segments=profile.num_segments,
+        outline_provider=profile.outline_provider,
+        outline_model=profile.outline_model,
+        transcript_provider=profile.transcript_provider,
+        transcript_model=profile.transcript_model,
+    )
+
+
+@router.get("/episode-profiles", response_model=List[EpisodeProfileResponse])
+async def list_episode_profiles():
+    """List all available episode profiles"""
+    try:
+        profiles = await EpisodeProfile.get_all(order_by="name asc")
+        return [_profile_to_response(p) for p in profiles]
+    except Exception as e:
+        logger.error(f"Failed to fetch episode profiles: {e}")
+        raise HTTPException(
+            status_code=500, detail="Failed to fetch episode profiles"
+        )
+
+
+@router.get("/episode-profiles/{profile_name}", response_model=EpisodeProfileResponse)
+async def get_episode_profile(profile_name: str):
+    """Get a specific episode profile by name"""
+    try:
+        profile = await EpisodeProfile.get_by_name(profile_name)
+
+        if not profile:
+            raise HTTPException(
+                status_code=404, detail=f"Episode profile '{profile_name}' not found"
+            )
+
+        return _profile_to_response(profile)
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Failed to fetch episode profile '{profile_name}': {e}")
+        raise HTTPException(
+            status_code=500, detail="Failed to fetch episode profile"
+        )
+
+
+class EpisodeProfileCreate(BaseModel):
+    name: str = Field(..., description="Unique profile name")
+    description: str = Field("", description="Profile description")
+    speaker_config: str = Field(..., description="Reference to speaker profile name")
+    outline_llm: Optional[str] = Field(None, description="Model record ID for outline")
+    transcript_llm: Optional[str] = Field(
+        None, description="Model record ID for transcript"
+    )
+    language: Optional[str] = Field(None, description="Podcast language code")
+    default_briefing: str = Field(..., description="Default briefing template")
+    num_segments: int = Field(default=5, description="Number of podcast segments")
+    # Legacy fields (accepted but not required)
+    outline_provider: Optional[str] = None
+    outline_model: Optional[str] = None
+    transcript_provider: Optional[str] = None
+    transcript_model: Optional[str] = None
+
+
+@router.post("/episode-profiles", response_model=EpisodeProfileResponse)
+async def create_episode_profile(profile_data: EpisodeProfileCreate):
+    """Create a new episode profile"""
+    try:
+        profile = EpisodeProfile(
+            name=profile_data.name,
+            description=profile_data.description,
+            speaker_config=profile_data.speaker_config,
+            outline_llm=profile_data.outline_llm,
+            transcript_llm=profile_data.transcript_llm,
+            language=profile_data.language,
+            default_briefing=profile_data.default_briefing,
+            num_segments=profile_data.num_segments,
+            outline_provider=profile_data.outline_provider,
+            outline_model=profile_data.outline_model,
+            transcript_provider=profile_data.transcript_provider,
+            transcript_model=profile_data.transcript_model,
+        )
+
+        await profile.save()
+        return _profile_to_response(profile)
+
+    except Exception as e:
+        logger.error(f"Failed to create episode profile: {e}")
+        raise HTTPException(
+            status_code=500, detail="Failed to create episode profile"
+        )
+
+
+@router.put("/episode-profiles/{profile_id}", response_model=EpisodeProfileResponse)
+async def update_episode_profile(profile_id: str, profile_data: EpisodeProfileCreate):
+    """Update an existing episode profile"""
+    try:
+        profile = await EpisodeProfile.get(profile_id)
+
+        if not profile:
+            raise HTTPException(
+                status_code=404, detail=f"Episode profile '{profile_id}' not found"
+            )
+
+        profile.name = profile_data.name
+        profile.description = profile_data.description
+        profile.speaker_config = profile_data.speaker_config
+        profile.outline_llm = profile_data.outline_llm
+        profile.transcript_llm = profile_data.transcript_llm
+        profile.language = profile_data.language
+        profile.default_briefing = profile_data.default_briefing
+        profile.num_segments = profile_data.num_segments
+        profile.outline_provider = profile_data.outline_provider
+        profile.outline_model = profile_data.outline_model
+        profile.transcript_provider = profile_data.transcript_provider
+        profile.transcript_model = profile_data.transcript_model
+
+        await profile.save()
+        return _profile_to_response(profile)
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Failed to update episode profile: {e}")
+        raise HTTPException(
+            status_code=500, detail="Failed to update episode profile"
+        )
+
+
+@router.delete("/episode-profiles/{profile_id}")
+async def delete_episode_profile(profile_id: str):
+    """Delete an episode profile"""
+    try:
+        profile = await EpisodeProfile.get(profile_id)
+
+        if not profile:
+            raise HTTPException(
+                status_code=404, detail=f"Episode profile '{profile_id}' not found"
+            )
+
+        await profile.delete()
+
+        return {"message": "Episode profile deleted successfully"}
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Failed to delete episode profile: {e}")
+        raise HTTPException(
+            status_code=500, detail="Failed to delete episode profile"
+        )
+
+
+@router.post(
+    "/episode-profiles/{profile_id}/duplicate", response_model=EpisodeProfileResponse
+)
+async def duplicate_episode_profile(profile_id: str):
+    """Duplicate an episode profile"""
+    try:
+        original = await EpisodeProfile.get(profile_id)
+
+        if not original:
+            raise HTTPException(
+                status_code=404, detail=f"Episode profile '{profile_id}' not found"
+            )
+
+        duplicate = EpisodeProfile(
+            name=f"{original.name} - Copy",
+            description=original.description,
+            speaker_config=original.speaker_config,
+            outline_llm=original.outline_llm,
+            transcript_llm=original.transcript_llm,
+            language=original.language,
+            default_briefing=original.default_briefing,
+            num_segments=original.num_segments,
+            outline_provider=original.outline_provider,
+            outline_model=original.outline_model,
+            transcript_provider=original.transcript_provider,
+            transcript_model=original.transcript_model,
+        )
+
+        await duplicate.save()
+        return _profile_to_response(duplicate)
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Failed to duplicate episode profile: {e}")
+        raise HTTPException(
+            status_code=500, detail="Failed to duplicate episode profile"
+        )

api/routers/insights.py

@@ -0,0 +1,82 @@
+from fastapi import APIRouter, HTTPException
+from loguru import logger
+
+from api.models import NoteResponse, SaveAsNoteRequest, SourceInsightResponse
+from open_notebook.domain.notebook import SourceInsight
+from open_notebook.exceptions import InvalidInputError
+
+router = APIRouter()
+
+
+@router.get("/insights/{insight_id}", response_model=SourceInsightResponse)
+async def get_insight(insight_id: str):
+    """Get a specific insight by ID."""
+    try:
+        insight = await SourceInsight.get(insight_id)
+        if not insight:
+            raise HTTPException(status_code=404, detail="Insight not found")
+
+        # Get source ID from the insight relationship
+        source = await insight.get_source()
+
+        return SourceInsightResponse(
+            id=insight.id or "",
+            source_id=source.id or "",
+            insight_type=insight.insight_type,
+            content=insight.content,
+            created=str(insight.created),
+            updated=str(insight.updated),
+        )
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error fetching insight {insight_id}: {str(e)}")
+        raise HTTPException(status_code=500, detail="Error fetching insight")
+
+
+@router.delete("/insights/{insight_id}")
+async def delete_insight(insight_id: str):
+    """Delete a specific insight."""
+    try:
+        insight = await SourceInsight.get(insight_id)
+        if not insight:
+            raise HTTPException(status_code=404, detail="Insight not found")
+
+        await insight.delete()
+
+        return {"message": "Insight deleted successfully"}
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error deleting insight {insight_id}: {str(e)}")
+        raise HTTPException(status_code=500, detail="Error deleting insight")
+
+
+@router.post("/insights/{insight_id}/save-as-note", response_model=NoteResponse)
+async def save_insight_as_note(insight_id: str, request: SaveAsNoteRequest):
+    """Convert an insight to a note."""
+    try:
+        insight = await SourceInsight.get(insight_id)
+        if not insight:
+            raise HTTPException(status_code=404, detail="Insight not found")
+
+        # Use the existing save_as_note method from the domain model
+        note = await insight.save_as_note(request.notebook_id)
+
+        return NoteResponse(
+            id=note.id or "",
+            title=note.title,
+            content=note.content,
+            note_type=note.note_type,
+            created=str(note.created),
+            updated=str(note.updated),
+        )
+    except HTTPException:
+        raise
+    except InvalidInputError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        logger.error(f"Error saving insight {insight_id} as note: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail="Error saving insight as note"
+        )

api/routers/languages.py

@@ -0,0 +1,83 @@
+from typing import List
+
+import pycountry
+from babel import Locale
+from babel.core import get_global
+from fastapi import APIRouter
+from pydantic import BaseModel
+
+router = APIRouter()
+
+# Additional regional variants for languages where the distinction matters
+# (TTS accent, vocabulary, spelling differences)
+_EXTRA_VARIANTS = [
+    "pt_PT",
+    "en_GB",
+    "en_AU",
+    "en_IN",
+    "es_MX",
+    "es_AR",
+    "es_CO",
+    "fr_CA",
+    "fr_CH",
+    "zh_TW",
+    "zh_HK",
+    "de_AT",
+    "de_CH",
+    "ar_SA",
+    "nl_BE",
+]
+
+
+class LanguageResponse(BaseModel):
+    code: str
+    name: str
+
+
+@router.get("/languages", response_model=List[LanguageResponse])
+async def list_languages():
+    """List available languages as BCP 47 locale codes (e.g. pt-BR, en-US)."""
+    likely_subtags = get_global("likely_subtags")
+    languages = []
+    seen = set()
+
+    # 1. For each language, resolve its default locale via CLDR likely subtags
+    for lang in pycountry.languages:
+        if not hasattr(lang, "alpha_2"):
+            continue
+
+        code = lang.alpha_2
+        likely = likely_subtags.get(code)
+
+        if likely:
+            try:
+                loc = Locale.parse(likely)
+                if loc.territory:
+                    bcp47 = f"{loc.language}-{loc.territory}"
+                    display = loc.get_display_name("en")
+                    if bcp47 not in seen:
+                        seen.add(bcp47)
+                        languages.append(LanguageResponse(code=bcp47, name=display))
+                    continue
+            except Exception:
+                pass
+
+        # Fallback: bare language code
+        if code not in seen:
+            seen.add(code)
+            languages.append(LanguageResponse(code=code, name=lang.name))
+
+    # 2. Add important regional variants
+    for locale_str in _EXTRA_VARIANTS:
+        try:
+            loc = Locale.parse(locale_str)
+            bcp47 = f"{loc.language}-{loc.territory}"
+            if bcp47 not in seen:
+                seen.add(bcp47)
+                display = loc.get_display_name("en")
+                languages.append(LanguageResponse(code=bcp47, name=display))
+        except Exception:
+            pass
+
+    languages.sort(key=lambda x: x.name)
+    return languages

api/routers/models.py

@@ -0,0 +1,776 @@
+import os
+import traceback
+from typing import Dict, List, Optional
+
+from esperanto import AIFactory
+from fastapi import APIRouter, HTTPException, Query
+from loguru import logger
+from pydantic import BaseModel
+
+from api.models import (
+    DefaultModelsResponse,
+    ModelCreate,
+    ModelResponse,
+    ProviderAvailabilityResponse,
+)
+from open_notebook.domain.credential import Credential
+from open_notebook.ai.connection_tester import test_individual_model
+from open_notebook.ai.key_provider import provision_provider_keys
+from open_notebook.ai.model_discovery import (
+    discover_provider_models,
+    get_provider_model_count,
+    sync_all_providers,
+    sync_provider_models,
+)
+from open_notebook.ai.models import DefaultModels, Model
+from open_notebook.exceptions import InvalidInputError
+
+router = APIRouter()
+
+
+# =============================================================================
+# Model Discovery Response Models
+# =============================================================================
+
+
+class DiscoveredModelResponse(BaseModel):
+    """Response model for a discovered model."""
+
+    name: str
+    provider: str
+    model_type: str
+    description: Optional[str] = None
+
+
+class ProviderSyncResponse(BaseModel):
+    """Response model for provider sync operation."""
+
+    provider: str
+    discovered: int
+    new: int
+    existing: int
+
+
+class AllProvidersSyncResponse(BaseModel):
+    """Response model for syncing all providers."""
+
+    results: Dict[str, ProviderSyncResponse]
+    total_discovered: int
+    total_new: int
+
+
+class ProviderModelCountResponse(BaseModel):
+    """Response model for provider model counts."""
+
+    provider: str
+    counts: Dict[str, int]
+    total: int
+
+
+class AutoAssignResult(BaseModel):
+    """Response model for auto-assign operation."""
+
+    assigned: Dict[str, str]  # slot_name -> model_id
+    skipped: List[str]  # slots already assigned
+    missing: List[str]  # slots with no available models
+
+
+class ModelTestResponse(BaseModel):
+    """Response model for individual model test."""
+
+    success: bool
+    message: str
+    details: Optional[str] = None
+
+
+# Provider priority for auto-assignment (higher priority first)
+PROVIDER_PRIORITY = [
+    "openai",
+    "anthropic",
+    "google",
+    "mistral",
+    "groq",
+    "deepseek",
+    "xai",
+    "openrouter",
+    "ollama",
+    "azure",
+    "openai_compatible",
+    "dashscope",
+    "minimax",
+]
+
+# Model preference patterns (preferred models within each provider)
+MODEL_PREFERENCES = {
+    "openai": ["gpt-4o", "gpt-4", "gpt-3.5-turbo"],
+    "anthropic": ["claude-3-5-sonnet", "claude-3-opus", "claude-3-sonnet"],
+    "google": ["gemini-2.0", "gemini-1.5-pro", "gemini-pro"],
+    "mistral": ["mistral-large", "mixtral"],
+    "groq": ["llama-3.3", "llama-3.1", "mixtral"],
+    "dashscope": ["qwen-max", "qwen-plus", "qwen-turbo"],
+    "minimax": ["MiniMax-M2.5", "MiniMax-M2.5-highspeed"],
+}
+
+
+async def _check_provider_has_credential(provider: str) -> bool:
+    """Check if a provider has any credentials configured in the database."""
+    try:
+        credentials = await Credential.get_by_provider(provider)
+        return len(credentials) > 0
+    except Exception:
+        pass
+    return False
+
+
+def _check_azure_support(mode: str) -> bool:
+    """
+    Check if Azure OpenAI provider is available for a specific mode.
+
+    Args:
+        mode: One of 'LLM', 'EMBEDDING', 'STT', 'TTS'
+
+    Returns:
+        bool: True if either generic or mode-specific env vars are set
+    """
+    # Check generic configuration (applies to all modes)
+    generic = (
+        os.environ.get("AZURE_OPENAI_API_KEY") is not None
+        and os.environ.get("AZURE_OPENAI_ENDPOINT") is not None
+        and os.environ.get("AZURE_OPENAI_API_VERSION") is not None
+    )
+
+    # Check mode-specific configuration (takes precedence)
+    specific = (
+        os.environ.get(f"AZURE_OPENAI_API_KEY_{mode}") is not None
+        and os.environ.get(f"AZURE_OPENAI_ENDPOINT_{mode}") is not None
+        and os.environ.get(f"AZURE_OPENAI_API_VERSION_{mode}") is not None
+    )
+
+    return generic or specific
+
+
+def _check_openai_compatible_support(mode: str) -> bool:
+    """
+    Check if OpenAI-compatible provider is available for a specific mode.
+
+    Args:
+        mode: One of 'LLM', 'EMBEDDING', 'STT', 'TTS'
+
+    Returns:
+        bool: True if either generic or mode-specific env var is set
+    """
+    generic = os.environ.get("OPENAI_COMPATIBLE_BASE_URL") is not None
+    specific = os.environ.get(f"OPENAI_COMPATIBLE_BASE_URL_{mode}") is not None
+    generic_key = os.environ.get("OPENAI_COMPATIBLE_API_KEY") is not None
+    specific_key = os.environ.get(f"OPENAI_COMPATIBLE_API_KEY_{mode}") is not None
+    return generic or specific or generic_key or specific_key
+
+
+@router.get("/models", response_model=List[ModelResponse])
+async def get_models(
+    type: Optional[str] = Query(None, description="Filter by model type"),
+):
+    """Get all configured models with optional type filtering."""
+    try:
+        if type:
+            models = await Model.get_models_by_type(type)
+        else:
+            models = await Model.get_all()
+
+        return [
+            ModelResponse(
+                id=model.id,
+                name=model.name,
+                provider=model.provider,
+                type=model.type,
+                credential=model.credential,
+                created=str(model.created),
+                updated=str(model.updated),
+            )
+            for model in models
+        ]
+    except Exception as e:
+        logger.error(f"Error fetching models: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error fetching models: {str(e)}")
+
+
+@router.post("/models", response_model=ModelResponse)
+async def create_model(model_data: ModelCreate):
+    """Create a new model configuration."""
+    try:
+        # Validate model type
+        valid_types = ["language", "embedding", "text_to_speech", "speech_to_text"]
+        if model_data.type not in valid_types:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Invalid model type. Must be one of: {valid_types}",
+            )
+
+        # Check for duplicate model name under the same provider and type (case-insensitive)
+        from open_notebook.database.repository import repo_query
+
+        existing = await repo_query(
+            "SELECT * FROM model WHERE string::lowercase(provider) = $provider AND string::lowercase(name) = $name AND string::lowercase(type) = $type LIMIT 1",
+            {
+                "provider": model_data.provider.lower(),
+                "name": model_data.name.lower(),
+                "type": model_data.type.lower(),
+            },
+        )
+        if existing:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Model '{model_data.name}' already exists for provider '{model_data.provider}' with type '{model_data.type}'",
+            )
+
+        new_model = Model(
+            name=model_data.name,
+            provider=model_data.provider,
+            type=model_data.type,
+            credential=model_data.credential,
+        )
+        await new_model.save()
+
+        return ModelResponse(
+            id=new_model.id or "",
+            name=new_model.name,
+            provider=new_model.provider,
+            type=new_model.type,
+            credential=new_model.credential,
+            created=str(new_model.created),
+            updated=str(new_model.updated),
+        )
+    except HTTPException:
+        raise
+    except InvalidInputError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        logger.error(f"Error creating model: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error creating model: {str(e)}")
+
+
+@router.delete("/models/{model_id}")
+async def delete_model(model_id: str):
+    """Delete a model configuration."""
+    try:
+        model = await Model.get(model_id)
+        if not model:
+            raise HTTPException(status_code=404, detail="Model not found")
+
+        await model.delete()
+
+        return {"message": "Model deleted successfully"}
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error deleting model {model_id}: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error deleting model: {str(e)}")
+
+
+@router.post("/models/{model_id}/test", response_model=ModelTestResponse)
+async def test_model(model_id: str):
+    """Test if a specific model is correctly configured and functional."""
+    try:
+        model = await Model.get(model_id)
+        if not model:
+            raise HTTPException(status_code=404, detail="Model not found")
+    except HTTPException:
+        raise
+    except Exception:
+        raise HTTPException(status_code=404, detail="Model not found")
+
+    try:
+        success, message = await test_individual_model(model)
+        return ModelTestResponse(success=success, message=message)
+    except Exception as e:
+        logger.error(f"Error testing model {model_id}: {traceback.format_exc()}")
+        return ModelTestResponse(
+            success=False,
+            message=str(e)[:200],
+        )
+
+
+@router.get("/models/defaults", response_model=DefaultModelsResponse)
+async def get_default_models():
+    """Get default model assignments."""
+    try:
+        defaults = await DefaultModels.get_instance()
+
+        return DefaultModelsResponse(
+            default_chat_model=defaults.default_chat_model,  # type: ignore[attr-defined]
+            default_transformation_model=defaults.default_transformation_model,  # type: ignore[attr-defined]
+            large_context_model=defaults.large_context_model,  # type: ignore[attr-defined]
+            default_text_to_speech_model=defaults.default_text_to_speech_model,  # type: ignore[attr-defined]
+            default_speech_to_text_model=defaults.default_speech_to_text_model,  # type: ignore[attr-defined]
+            default_embedding_model=defaults.default_embedding_model,  # type: ignore[attr-defined]
+            default_tools_model=defaults.default_tools_model,  # type: ignore[attr-defined]
+        )
+    except Exception as e:
+        logger.error(f"Error fetching default models: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error fetching default models: {str(e)}"
+        )
+
+
+@router.put("/models/defaults", response_model=DefaultModelsResponse)
+async def update_default_models(defaults_data: DefaultModelsResponse):
+    """Update default model assignments."""
+    try:
+        defaults = await DefaultModels.get_instance()
+
+        # Update only provided fields
+        if defaults_data.default_chat_model is not None:
+            defaults.default_chat_model = defaults_data.default_chat_model  # type: ignore[attr-defined]
+        if defaults_data.default_transformation_model is not None:
+            defaults.default_transformation_model = (
+                defaults_data.default_transformation_model
+            )  # type: ignore[attr-defined]
+        if defaults_data.large_context_model is not None:
+            defaults.large_context_model = defaults_data.large_context_model  # type: ignore[attr-defined]
+        if defaults_data.default_text_to_speech_model is not None:
+            defaults.default_text_to_speech_model = (
+                defaults_data.default_text_to_speech_model
+            )  # type: ignore[attr-defined]
+        if defaults_data.default_speech_to_text_model is not None:
+            defaults.default_speech_to_text_model = (
+                defaults_data.default_speech_to_text_model
+            )  # type: ignore[attr-defined]
+        if defaults_data.default_embedding_model is not None:
+            defaults.default_embedding_model = defaults_data.default_embedding_model  # type: ignore[attr-defined]
+        if defaults_data.default_tools_model is not None:
+            defaults.default_tools_model = defaults_data.default_tools_model  # type: ignore[attr-defined]
+
+        await defaults.update()
+
+        # No cache refresh needed - next access will fetch fresh data from DB
+
+        return DefaultModelsResponse(
+            default_chat_model=defaults.default_chat_model,  # type: ignore[attr-defined]
+            default_transformation_model=defaults.default_transformation_model,  # type: ignore[attr-defined]
+            large_context_model=defaults.large_context_model,  # type: ignore[attr-defined]
+            default_text_to_speech_model=defaults.default_text_to_speech_model,  # type: ignore[attr-defined]
+            default_speech_to_text_model=defaults.default_speech_to_text_model,  # type: ignore[attr-defined]
+            default_embedding_model=defaults.default_embedding_model,  # type: ignore[attr-defined]
+            default_tools_model=defaults.default_tools_model,  # type: ignore[attr-defined]
+        )
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error updating default models: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error updating default models: {str(e)}"
+        )
+
+
+@router.get("/models/providers", response_model=ProviderAvailabilityResponse)
+async def get_provider_availability():
+    """Get provider availability based on database config and environment variables."""
+    try:
+        # Check which providers have credentials in the database or env vars
+        # For each provider, check DB credentials first, then env vars as fallback
+
+        # Simple env var mapping for backward compatibility
+        env_var_map = {
+            "openai": "OPENAI_API_KEY",
+            "anthropic": "ANTHROPIC_API_KEY",
+            "google": "GOOGLE_API_KEY",
+            "groq": "GROQ_API_KEY",
+            "mistral": "MISTRAL_API_KEY",
+            "deepseek": "DEEPSEEK_API_KEY",
+            "xai": "XAI_API_KEY",
+            "openrouter": "OPENROUTER_API_KEY",
+            "voyage": "VOYAGE_API_KEY",
+            "elevenlabs": "ELEVENLABS_API_KEY",
+            "ollama": "OLLAMA_API_BASE",
+            "dashscope": "DASHSCOPE_API_KEY",
+            "minimax": "MINIMAX_API_KEY",
+        }
+
+        provider_status = {}
+
+        # Check simple providers: credential in DB or env var
+        for provider, env_var in env_var_map.items():
+            has_cred = await _check_provider_has_credential(provider)
+            has_env = os.environ.get(env_var) is not None
+            provider_status[provider] = has_cred or has_env
+
+        # Google also supports GEMINI_API_KEY
+        if not provider_status.get("google"):
+            provider_status["google"] = os.environ.get("GEMINI_API_KEY") is not None
+
+        # Vertex: DB credential or env vars
+        provider_status["vertex"] = (
+            await _check_provider_has_credential("vertex")
+            or os.environ.get("VERTEX_PROJECT") is not None
+        )
+
+        # Azure: DB credential or env vars
+        provider_status["azure"] = (
+            await _check_provider_has_credential("azure")
+            or _check_azure_support("LLM")
+            or _check_azure_support("EMBEDDING")
+            or _check_azure_support("STT")
+            or _check_azure_support("TTS")
+        )
+
+        # OpenAI-compatible: DB credential or env vars
+        provider_status["openai-compatible"] = (
+            await _check_provider_has_credential("openai_compatible")
+            or _check_openai_compatible_support("LLM")
+            or _check_openai_compatible_support("EMBEDDING")
+            or _check_openai_compatible_support("STT")
+            or _check_openai_compatible_support("TTS")
+        )
+
+        available_providers = [k for k, v in provider_status.items() if v]
+        unavailable_providers = [k for k, v in provider_status.items() if not v]
+
+        # Get supported model types from Esperanto
+        esperanto_available = AIFactory.get_available_providers()
+
+        # Build supported types mapping only for available providers
+        supported_types: dict[str, list[str]] = {}
+        for provider in available_providers:
+            supported_types[provider] = []
+
+            # Map Esperanto model types to our environment variable modes
+            mode_mapping = {
+                "language": "LLM",
+                "embedding": "EMBEDDING",
+                "speech_to_text": "STT",
+                "text_to_speech": "TTS",
+            }
+
+            # Special handling for openai-compatible to check mode-specific availability
+            if provider == "openai-compatible":
+                has_db_cred = await _check_provider_has_credential("openai_compatible")
+                for model_type, mode in mode_mapping.items():
+                    if (
+                        model_type in esperanto_available
+                        and provider in esperanto_available[model_type]
+                    ):
+                        if has_db_cred or _check_openai_compatible_support(mode):
+                            supported_types[provider].append(model_type)
+            # Special handling for azure to check mode-specific availability
+            elif provider == "azure":
+                has_db_cred = await _check_provider_has_credential("azure")
+                for model_type, mode in mode_mapping.items():
+                    if (
+                        model_type in esperanto_available
+                        and provider in esperanto_available[model_type]
+                    ):
+                        if has_db_cred or _check_azure_support(mode):
+                            supported_types[provider].append(model_type)
+            else:
+                # Standard provider detection
+                for model_type, providers in esperanto_available.items():
+                    if provider in providers:
+                        supported_types[provider].append(model_type)
+
+        return ProviderAvailabilityResponse(
+            available=available_providers,
+            unavailable=unavailable_providers,
+            supported_types=supported_types,
+        )
+    except Exception as e:
+        logger.error(f"Error checking provider availability: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error checking provider availability: {str(e)}"
+        )
+
+
+# =============================================================================
+# Model Discovery Endpoints
+# =============================================================================
+
+
+@router.get(
+    "/models/discover/{provider}", response_model=List[DiscoveredModelResponse]
+)
+async def discover_models(provider: str):
+    """
+    Discover available models from a provider without registering them.
+
+    This endpoint queries the provider's API to list available models
+    but does not save them to the database. Use the sync endpoint
+    to both discover and register models.
+    """
+    try:
+        # Provision DB-stored credentials into env vars before discovery
+        await provision_provider_keys(provider)
+        discovered = await discover_provider_models(provider)
+        return [
+            DiscoveredModelResponse(
+                name=m.name,
+                provider=m.provider,
+                model_type=m.model_type,
+                description=m.description,
+            )
+            for m in discovered
+        ]
+    except Exception as e:
+        logger.error(f"Error discovering models for {provider}: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail="Error discovering models. Check server logs for details."
+        )
+
+
+@router.post("/models/sync/{provider}", response_model=ProviderSyncResponse)
+async def sync_models(provider: str):
+    """
+    Sync models for a specific provider.
+
+    Discovers available models from the provider's API and registers
+    any new models in the database. Existing models are skipped.
+
+    Returns counts of discovered, new, and existing models.
+    """
+    try:
+        # Provision DB-stored credentials into env vars before discovery
+        await provision_provider_keys(provider)
+        discovered, new, existing = await sync_provider_models(
+            provider, auto_register=True
+        )
+        return ProviderSyncResponse(
+            provider=provider,
+            discovered=discovered,
+            new=new,
+            existing=existing,
+        )
+    except Exception as e:
+        logger.error(f"Error syncing models for {provider}: {str(e)}")
+        raise HTTPException(status_code=500, detail="Error syncing models. Check server logs for details.")
+
+
+@router.post("/models/sync", response_model=AllProvidersSyncResponse)
+async def sync_all_models():
+    """
+    Sync models for all configured providers.
+
+    Discovers and registers models from all providers that have
+    valid API keys configured. This is useful for initial setup
+    or periodic refresh of available models.
+    """
+    try:
+        results = await sync_all_providers()
+
+        response_results = {}
+        total_discovered = 0
+        total_new = 0
+
+        for provider, (discovered, new, existing) in results.items():
+            response_results[provider] = ProviderSyncResponse(
+                provider=provider,
+                discovered=discovered,
+                new=new,
+                existing=existing,
+            )
+            total_discovered += discovered
+            total_new += new
+
+        return AllProvidersSyncResponse(
+            results=response_results,
+            total_discovered=total_discovered,
+            total_new=total_new,
+        )
+    except Exception as e:
+        logger.error(f"Error syncing all models: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error syncing all models: {str(e)}"
+        )
+
+
+@router.get("/models/count/{provider}", response_model=ProviderModelCountResponse)
+async def get_model_count(provider: str):
+    """
+    Get count of registered models for a provider, grouped by type.
+
+    Returns counts for each model type (language, embedding,
+    speech_to_text, text_to_speech) as well as total count.
+    """
+    try:
+        counts = await get_provider_model_count(provider)
+        total = sum(counts.values())
+        return ProviderModelCountResponse(
+            provider=provider,
+            counts=counts,
+            total=total,
+        )
+    except Exception as e:
+        logger.error(f"Error getting model count for {provider}: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error getting model count: {str(e)}"
+        )
+
+
+@router.get("/models/by-provider/{provider}", response_model=List[ModelResponse])
+async def get_models_by_provider(provider: str):
+    """
+    Get all registered models for a specific provider.
+
+    Returns models from the database that belong to the specified provider.
+    """
+    try:
+        from open_notebook.database.repository import repo_query
+
+        models = await repo_query(
+            "SELECT * FROM model WHERE provider = $provider ORDER BY type, name",
+            {"provider": provider},
+        )
+
+        return [
+            ModelResponse(
+                id=model.get("id", ""),
+                name=model.get("name", ""),
+                provider=model.get("provider", ""),
+                type=model.get("type", ""),
+                credential=model.get("credential"),
+                created=str(model.get("created", "")),
+                updated=str(model.get("updated", "")),
+            )
+            for model in models
+        ]
+    except Exception as e:
+        logger.error(f"Error fetching models for {provider}: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error fetching models: {str(e)}"
+        )
+
+
+def _get_preferred_model(
+    models: List[Dict], provider_priority: List[str], model_preferences: Dict
+) -> Optional[Dict]:
+    """
+    Select the best model from a list based on provider priority and model preferences.
+
+    Args:
+        models: List of model dictionaries with 'provider', 'name', 'id' keys
+        provider_priority: List of providers in preference order
+        model_preferences: Dict mapping provider to list of preferred model name patterns
+
+    Returns:
+        The best model dict, or None if no models available
+    """
+    if not models:
+        return None
+
+    # Group models by provider
+    by_provider: Dict[str, List[Dict]] = {}
+    for model in models:
+        provider = model.get("provider", "")
+        if provider not in by_provider:
+            by_provider[provider] = []
+        by_provider[provider].append(model)
+
+    # Find first provider with models (in priority order)
+    for provider in provider_priority:
+        if provider in by_provider:
+            provider_models = by_provider[provider]
+
+            # Check for preferred models within this provider
+            if provider in model_preferences:
+                for preference in model_preferences[provider]:
+                    for model in provider_models:
+                        if preference.lower() in model.get("name", "").lower():
+                            return model
+
+            # Fall back to first model from this provider
+            return provider_models[0]
+
+    # Fall back to first model from any provider
+    return models[0] if models else None
+
+
+@router.post("/models/auto-assign", response_model=AutoAssignResult)
+async def auto_assign_defaults():
+    """
+    Auto-assign default models based on available models.
+
+    This endpoint intelligently assigns the first available model of each
+    required type to the corresponding default slot. It uses provider
+    priority (preferring premium providers like OpenAI, Anthropic) and
+    model preferences within each provider.
+
+    Returns:
+        - assigned: Dict of slot names to assigned model IDs
+        - skipped: List of slots that already have models assigned
+        - missing: List of slots with no available models
+    """
+    try:
+        from open_notebook.database.repository import repo_query
+
+        # Get current defaults
+        defaults = await DefaultModels.get_instance()
+
+        # Get all models grouped by type
+        all_models = await repo_query(
+            "SELECT * FROM model ORDER BY provider, name",
+            {},
+        )
+
+        # Group models by type
+        models_by_type: Dict[str, List[Dict]] = {
+            "language": [],
+            "embedding": [],
+            "text_to_speech": [],
+            "speech_to_text": [],
+        }
+
+        for model in all_models:
+            model_type = model.get("type", "")
+            if model_type in models_by_type:
+                models_by_type[model_type].append(model)
+
+        # Define slot configuration: (slot_name, model_type, current_value)
+        slot_configs = [
+            ("default_chat_model", "language", defaults.default_chat_model),  # type: ignore[attr-defined]
+            ("default_transformation_model", "language", defaults.default_transformation_model),  # type: ignore[attr-defined]
+            ("default_tools_model", "language", defaults.default_tools_model),  # type: ignore[attr-defined]
+            ("large_context_model", "language", defaults.large_context_model),  # type: ignore[attr-defined]
+            ("default_embedding_model", "embedding", defaults.default_embedding_model),  # type: ignore[attr-defined]
+            ("default_text_to_speech_model", "text_to_speech", defaults.default_text_to_speech_model),  # type: ignore[attr-defined]
+            ("default_speech_to_text_model", "speech_to_text", defaults.default_speech_to_text_model),  # type: ignore[attr-defined]
+        ]
+
+        assigned: Dict[str, str] = {}
+        skipped: List[str] = []
+        missing: List[str] = []
+
+        for slot_name, model_type, current_value in slot_configs:
+            if current_value:
+                # Slot already has a value
+                skipped.append(slot_name)
+                continue
+
+            available_models = models_by_type.get(model_type, [])
+            if not available_models:
+                # No models of this type available
+                missing.append(slot_name)
+                continue
+
+            # Select best model for this slot
+            best_model = _get_preferred_model(
+                available_models, PROVIDER_PRIORITY, MODEL_PREFERENCES
+            )
+
+            if best_model:
+                model_id = best_model.get("id", "")
+                assigned[slot_name] = model_id
+                # Update the defaults object
+                setattr(defaults, slot_name, model_id)
+
+        # Save updated defaults if any assignments were made
+        if assigned:
+            await defaults.update()
+
+        return AutoAssignResult(
+            assigned=assigned,
+            skipped=skipped,
+            missing=missing,
+        )
+
+    except Exception as e:
+        logger.error(f"Error auto-assigning defaults: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error auto-assigning defaults: {str(e)}"
+        )

api/routers/notebooks.py

@@ -0,0 +1,354 @@
+from typing import List, Optional
+
+from fastapi import APIRouter, HTTPException, Query
+from loguru import logger
+
+from api.models import (
+    NotebookCreate,
+    NotebookDeletePreview,
+    NotebookDeleteResponse,
+    NotebookResponse,
+    NotebookUpdate,
+)
+from open_notebook.database.repository import ensure_record_id, repo_query
+from open_notebook.domain.notebook import Notebook, Source
+from open_notebook.exceptions import InvalidInputError
+
+router = APIRouter()
+
+
+@router.get("/notebooks", response_model=List[NotebookResponse])
+async def get_notebooks(
+    archived: Optional[bool] = Query(None, description="Filter by archived status"),
+    order_by: str = Query("updated desc", description="Order by field and direction"),
+):
+    """Get all notebooks with optional filtering and ordering."""
+    try:
+        # Validate order_by against allowlist to prevent SurrealQL injection
+        allowed_fields = {"name", "created", "updated"}
+        allowed_directions = {"asc", "desc"}
+
+        parts = order_by.strip().lower().split()
+        if len(parts) == 1:
+            if parts[0] not in allowed_fields:
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"Invalid order_by field: '{order_by}'. Allowed fields: {', '.join(sorted(allowed_fields))}",
+                )
+            validated_order_by = parts[0]
+        elif len(parts) == 2:
+            if parts[0] not in allowed_fields or parts[1] not in allowed_directions:
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"Invalid order_by: '{order_by}'. Allowed fields: {', '.join(sorted(allowed_fields))}. Allowed directions: asc, desc",
+                )
+            validated_order_by = f"{parts[0]} {parts[1]}"
+        else:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Invalid order_by format: '{order_by}'. Expected 'field' or 'field direction'",
+            )
+
+        # Build the query with counts
+        query = f"""
+            SELECT *,
+            count(<-reference.in) as source_count,
+            count(<-artifact.in) as note_count
+            FROM notebook
+            ORDER BY {validated_order_by}
+        """
+
+        result = await repo_query(query)
+
+        # Filter by archived status if specified
+        if archived is not None:
+            result = [nb for nb in result if nb.get("archived") == archived]
+
+        return [
+            NotebookResponse(
+                id=str(nb.get("id", "")),
+                name=nb.get("name", ""),
+                description=nb.get("description", ""),
+                archived=nb.get("archived", False),
+                created=str(nb.get("created", "")),
+                updated=str(nb.get("updated", "")),
+                source_count=nb.get("source_count", 0),
+                note_count=nb.get("note_count", 0),
+            )
+            for nb in result
+        ]
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error fetching notebooks: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error fetching notebooks: {str(e)}"
+        )
+
+
+@router.post("/notebooks", response_model=NotebookResponse)
+async def create_notebook(notebook: NotebookCreate):
+    """Create a new notebook."""
+    try:
+        new_notebook = Notebook(
+            name=notebook.name,
+            description=notebook.description,
+        )
+        await new_notebook.save()
+
+        return NotebookResponse(
+            id=new_notebook.id or "",
+            name=new_notebook.name,
+            description=new_notebook.description,
+            archived=new_notebook.archived or False,
+            created=str(new_notebook.created),
+            updated=str(new_notebook.updated),
+            source_count=0,  # New notebook has no sources
+            note_count=0,  # New notebook has no notes
+        )
+    except InvalidInputError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        logger.error(f"Error creating notebook: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error creating notebook: {str(e)}"
+        )
+
+
+@router.get(
+    "/notebooks/{notebook_id}/delete-preview", response_model=NotebookDeletePreview
+)
+async def get_notebook_delete_preview(notebook_id: str):
+    """Get a preview of what will be deleted when this notebook is deleted."""
+    try:
+        notebook = await Notebook.get(notebook_id)
+        if not notebook:
+            raise HTTPException(status_code=404, detail="Notebook not found")
+
+        preview = await notebook.get_delete_preview()
+
+        return NotebookDeletePreview(
+            notebook_id=str(notebook.id),
+            notebook_name=notebook.name,
+            note_count=preview["note_count"],
+            exclusive_source_count=preview["exclusive_source_count"],
+            shared_source_count=preview["shared_source_count"],
+        )
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error getting delete preview for notebook {notebook_id}: {e}")
+        raise HTTPException(
+            status_code=500,
+            detail=f"Error fetching notebook deletion preview: {str(e)}",
+        )
+
+
+@router.get("/notebooks/{notebook_id}", response_model=NotebookResponse)
+async def get_notebook(notebook_id: str):
+    """Get a specific notebook by ID."""
+    try:
+        # Query with counts for single notebook
+        query = """
+            SELECT *,
+            count(<-reference.in) as source_count,
+            count(<-artifact.in) as note_count
+            FROM $notebook_id
+        """
+        result = await repo_query(query, {"notebook_id": ensure_record_id(notebook_id)})
+
+        if not result:
+            raise HTTPException(status_code=404, detail="Notebook not found")
+
+        nb = result[0]
+        return NotebookResponse(
+            id=str(nb.get("id", "")),
+            name=nb.get("name", ""),
+            description=nb.get("description", ""),
+            archived=nb.get("archived", False),
+            created=str(nb.get("created", "")),
+            updated=str(nb.get("updated", "")),
+            source_count=nb.get("source_count", 0),
+            note_count=nb.get("note_count", 0),
+        )
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error fetching notebook {notebook_id}: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error fetching notebook: {str(e)}"
+        )
+
+
+@router.put("/notebooks/{notebook_id}", response_model=NotebookResponse)
+async def update_notebook(notebook_id: str, notebook_update: NotebookUpdate):
+    """Update a notebook."""
+    try:
+        notebook = await Notebook.get(notebook_id)
+        if not notebook:
+            raise HTTPException(status_code=404, detail="Notebook not found")
+
+        # Update only provided fields
+        if notebook_update.name is not None:
+            notebook.name = notebook_update.name
+        if notebook_update.description is not None:
+            notebook.description = notebook_update.description
+        if notebook_update.archived is not None:
+            notebook.archived = notebook_update.archived
+
+        await notebook.save()
+
+        # Query with counts after update
+        query = """
+            SELECT *,
+            count(<-reference.in) as source_count,
+            count(<-artifact.in) as note_count
+            FROM $notebook_id
+        """
+        result = await repo_query(query, {"notebook_id": ensure_record_id(notebook_id)})
+
+        if result:
+            nb = result[0]
+            return NotebookResponse(
+                id=str(nb.get("id", "")),
+                name=nb.get("name", ""),
+                description=nb.get("description", ""),
+                archived=nb.get("archived", False),
+                created=str(nb.get("created", "")),
+                updated=str(nb.get("updated", "")),
+                source_count=nb.get("source_count", 0),
+                note_count=nb.get("note_count", 0),
+            )
+
+        # Fallback if query fails
+        return NotebookResponse(
+            id=notebook.id or "",
+            name=notebook.name,
+            description=notebook.description,
+            archived=notebook.archived or False,
+            created=str(notebook.created),
+            updated=str(notebook.updated),
+            source_count=0,
+            note_count=0,
+        )
+    except HTTPException:
+        raise
+    except InvalidInputError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        logger.error(f"Error updating notebook {notebook_id}: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error updating notebook: {str(e)}"
+        )
+
+
+@router.post("/notebooks/{notebook_id}/sources/{source_id}")
+async def add_source_to_notebook(notebook_id: str, source_id: str):
+    """Add an existing source to a notebook (create the reference)."""
+    try:
+        # Check if notebook exists
+        notebook = await Notebook.get(notebook_id)
+        if not notebook:
+            raise HTTPException(status_code=404, detail="Notebook not found")
+
+        # Check if source exists
+        source = await Source.get(source_id)
+        if not source:
+            raise HTTPException(status_code=404, detail="Source not found")
+
+        # Check if reference already exists (idempotency)
+        existing_ref = await repo_query(
+            "SELECT * FROM reference WHERE out = $source_id AND in = $notebook_id",
+            {
+                "notebook_id": ensure_record_id(notebook_id),
+                "source_id": ensure_record_id(source_id),
+            },
+        )
+
+        # If reference doesn't exist, create it
+        if not existing_ref:
+            await repo_query(
+                "RELATE $source_id->reference->$notebook_id",
+                {
+                    "notebook_id": ensure_record_id(notebook_id),
+                    "source_id": ensure_record_id(source_id),
+                },
+            )
+
+        return {"message": "Source linked to notebook successfully"}
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(
+            f"Error linking source {source_id} to notebook {notebook_id}: {str(e)}"
+        )
+        raise HTTPException(
+            status_code=500, detail=f"Error linking source to notebook: {str(e)}"
+        )
+
+
+@router.delete("/notebooks/{notebook_id}/sources/{source_id}")
+async def remove_source_from_notebook(notebook_id: str, source_id: str):
+    """Remove a source from a notebook (delete the reference)."""
+    try:
+        # Check if notebook exists
+        notebook = await Notebook.get(notebook_id)
+        if not notebook:
+            raise HTTPException(status_code=404, detail="Notebook not found")
+
+        # Delete the reference record linking source to notebook
+        await repo_query(
+            "DELETE FROM reference WHERE out = $notebook_id AND in = $source_id",
+            {
+                "notebook_id": ensure_record_id(notebook_id),
+                "source_id": ensure_record_id(source_id),
+            },
+        )
+
+        return {"message": "Source removed from notebook successfully"}
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(
+            f"Error removing source {source_id} from notebook {notebook_id}: {str(e)}"
+        )
+        raise HTTPException(
+            status_code=500, detail=f"Error removing source from notebook: {str(e)}"
+        )
+
+
+@router.delete("/notebooks/{notebook_id}", response_model=NotebookDeleteResponse)
+async def delete_notebook(
+    notebook_id: str,
+    delete_exclusive_sources: bool = Query(
+        False,
+        description="Whether to delete sources that belong only to this notebook",
+    ),
+):
+    """
+    Delete a notebook with cascade deletion.
+
+    Always deletes all notes associated with the notebook.
+    If delete_exclusive_sources is True, also deletes sources that belong only
+    to this notebook (not linked to any other notebooks).
+    """
+    try:
+        notebook = await Notebook.get(notebook_id)
+        if not notebook:
+            raise HTTPException(status_code=404, detail="Notebook not found")
+
+        result = await notebook.delete(delete_exclusive_sources=delete_exclusive_sources)
+
+        return NotebookDeleteResponse(
+            message="Notebook deleted successfully",
+            deleted_notes=result["deleted_notes"],
+            deleted_sources=result["deleted_sources"],
+            unlinked_sources=result["unlinked_sources"],
+        )
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error deleting notebook {notebook_id}: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error deleting notebook: {str(e)}"
+        )

api/routers/notes.py

@@ -0,0 +1,189 @@
+from typing import List, Literal, Optional
+
+from fastapi import APIRouter, HTTPException, Query
+from loguru import logger
+
+from api.models import NoteCreate, NoteResponse, NoteUpdate
+from open_notebook.domain.notebook import Note
+from open_notebook.exceptions import InvalidInputError
+
+router = APIRouter()
+
+
+@router.get("/notes", response_model=List[NoteResponse])
+async def get_notes(
+    notebook_id: Optional[str] = Query(None, description="Filter by notebook ID"),
+):
+    """Get all notes with optional notebook filtering."""
+    try:
+        if notebook_id:
+            # Get notes for a specific notebook
+            from open_notebook.domain.notebook import Notebook
+
+            notebook = await Notebook.get(notebook_id)
+            if not notebook:
+                raise HTTPException(status_code=404, detail="Notebook not found")
+            notes = await notebook.get_notes()
+        else:
+            # Get all notes
+            notes = await Note.get_all(order_by="updated desc")
+
+        return [
+            NoteResponse(
+                id=note.id or "",
+                title=note.title,
+                content=note.content,
+                note_type=note.note_type,
+                created=str(note.created),
+                updated=str(note.updated),
+            )
+            for note in notes
+        ]
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error fetching notes: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error fetching notes: {str(e)}")
+
+
+@router.post("/notes", response_model=NoteResponse)
+async def create_note(note_data: NoteCreate):
+    """Create a new note."""
+    try:
+        # Auto-generate title if not provided and it's an AI note
+        title = note_data.title
+        if not title and note_data.note_type == "ai" and note_data.content:
+            from open_notebook.graphs.prompt import graph as prompt_graph
+
+            prompt = "Based on the Note below, please provide a Title for this content, with max 15 words"
+            result = await prompt_graph.ainvoke(
+                {  # type: ignore[arg-type]
+                    "input_text": note_data.content,
+                    "prompt": prompt,
+                }
+            )
+            title = result.get("output", "Untitled Note")
+
+        # Validate note_type
+        note_type: Optional[Literal["human", "ai"]] = None
+        if note_data.note_type in ("human", "ai"):
+            note_type = note_data.note_type  # type: ignore[assignment]
+        elif note_data.note_type is not None:
+            raise HTTPException(
+                status_code=400, detail="note_type must be 'human' or 'ai'"
+            )
+
+        new_note = Note(
+            title=title,
+            content=note_data.content,
+            note_type=note_type,
+        )
+        command_id = await new_note.save()
+
+        # Add to notebook if specified
+        if note_data.notebook_id:
+            from open_notebook.domain.notebook import Notebook
+
+            notebook = await Notebook.get(note_data.notebook_id)
+            if not notebook:
+                raise HTTPException(status_code=404, detail="Notebook not found")
+            await new_note.add_to_notebook(note_data.notebook_id)
+
+        return NoteResponse(
+            id=new_note.id or "",
+            title=new_note.title,
+            content=new_note.content,
+            note_type=new_note.note_type,
+            created=str(new_note.created),
+            updated=str(new_note.updated),
+            command_id=str(command_id) if command_id else None,
+        )
+    except HTTPException:
+        raise
+    except InvalidInputError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        logger.error(f"Error creating note: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error creating note: {str(e)}")
+
+
+@router.get("/notes/{note_id}", response_model=NoteResponse)
+async def get_note(note_id: str):
+    """Get a specific note by ID."""
+    try:
+        note = await Note.get(note_id)
+        if not note:
+            raise HTTPException(status_code=404, detail="Note not found")
+
+        return NoteResponse(
+            id=note.id or "",
+            title=note.title,
+            content=note.content,
+            note_type=note.note_type,
+            created=str(note.created),
+            updated=str(note.updated),
+        )
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error fetching note {note_id}: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error fetching note: {str(e)}")
+
+
+@router.put("/notes/{note_id}", response_model=NoteResponse)
+async def update_note(note_id: str, note_update: NoteUpdate):
+    """Update a note."""
+    try:
+        note = await Note.get(note_id)
+        if not note:
+            raise HTTPException(status_code=404, detail="Note not found")
+
+        # Update only provided fields
+        if note_update.title is not None:
+            note.title = note_update.title
+        if note_update.content is not None:
+            note.content = note_update.content
+        if note_update.note_type is not None:
+            if note_update.note_type in ("human", "ai"):
+                note.note_type = note_update.note_type  # type: ignore[assignment]
+            else:
+                raise HTTPException(
+                    status_code=400, detail="note_type must be 'human' or 'ai'"
+                )
+
+        command_id = await note.save()
+
+        return NoteResponse(
+            id=note.id or "",
+            title=note.title,
+            content=note.content,
+            note_type=note.note_type,
+            created=str(note.created),
+            updated=str(note.updated),
+            command_id=str(command_id) if command_id else None,
+        )
+    except HTTPException:
+        raise
+    except InvalidInputError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        logger.error(f"Error updating note {note_id}: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error updating note: {str(e)}")
+
+
+@router.delete("/notes/{note_id}")
+async def delete_note(note_id: str):
+    """Delete a note."""
+    try:
+        note = await Note.get(note_id)
+        if not note:
+            raise HTTPException(status_code=404, detail="Note not found")
+
+        await note.delete()
+
+        return {"message": "Note deleted successfully"}
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error deleting note {note_id}: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error deleting note: {str(e)}")

api/routers/podcasts.py

@@ -0,0 +1,299 @@
+from pathlib import Path
+from typing import List, Optional
+from urllib.parse import unquote, urlparse
+
+from fastapi import APIRouter, HTTPException
+from fastapi.responses import FileResponse
+from loguru import logger
+from pydantic import BaseModel
+
+from api.podcast_service import (
+    PodcastGenerationRequest,
+    PodcastGenerationResponse,
+    PodcastService,
+)
+
+router = APIRouter()
+
+
+class PodcastEpisodeResponse(BaseModel):
+    id: str
+    name: str
+    episode_profile: dict
+    speaker_profile: dict
+    briefing: str
+    audio_file: Optional[str] = None
+    audio_url: Optional[str] = None
+    transcript: Optional[dict] = None
+    outline: Optional[dict] = None
+    created: Optional[str] = None
+    job_status: Optional[str] = None
+    error_message: Optional[str] = None
+
+
+def _resolve_audio_path(audio_file: str) -> Path:
+    if audio_file.startswith("file://"):
+        parsed = urlparse(audio_file)
+        return Path(unquote(parsed.path))
+    return Path(audio_file)
+
+
+@router.post("/podcasts/generate", response_model=PodcastGenerationResponse)
+async def generate_podcast(request: PodcastGenerationRequest):
+    """
+    Generate a podcast episode using Episode Profiles.
+    Returns immediately with job ID for status tracking.
+    """
+    try:
+        job_id = await PodcastService.submit_generation_job(
+            episode_profile_name=request.episode_profile,
+            speaker_profile_name=request.speaker_profile,
+            episode_name=request.episode_name,
+            notebook_id=request.notebook_id,
+            content=request.content,
+            briefing_suffix=request.briefing_suffix,
+        )
+
+        return PodcastGenerationResponse(
+            job_id=job_id,
+            status="submitted",
+            message=f"Podcast generation started for episode '{request.episode_name}'",
+            episode_profile=request.episode_profile,
+            episode_name=request.episode_name,
+        )
+
+    except Exception as e:
+        logger.error(f"Error generating podcast: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail="Failed to generate podcast"
+        )
+
+
+@router.get("/podcasts/jobs/{job_id}")
+async def get_podcast_job_status(job_id: str):
+    """Get the status of a podcast generation job"""
+    try:
+        status_data = await PodcastService.get_job_status(job_id)
+        return status_data
+
+    except Exception as e:
+        logger.error(f"Error fetching podcast job status: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail="Failed to fetch job status"
+        )
+
+
+@router.get("/podcasts/episodes", response_model=List[PodcastEpisodeResponse])
+async def list_podcast_episodes():
+    """List all podcast episodes"""
+    try:
+        episodes = await PodcastService.list_episodes()
+
+        response_episodes = []
+        for episode in episodes:
+            # Skip incomplete episodes without command or audio
+            if not episode.command and not episode.audio_file:
+                continue
+
+            # Get job status and error message if available
+            job_status = None
+            error_message = None
+            if episode.command:
+                try:
+                    detail = await episode.get_job_detail()
+                    job_status = detail["status"]
+                    error_message = detail["error_message"]
+                except Exception:
+                    job_status = "unknown"
+            else:
+                # No command but has audio file = completed import
+                job_status = "completed"
+
+            audio_url = None
+            if episode.audio_file:
+                audio_path = _resolve_audio_path(episode.audio_file)
+                if audio_path.exists():
+                    audio_url = f"/api/podcasts/episodes/{episode.id}/audio"
+
+            response_episodes.append(
+                PodcastEpisodeResponse(
+                    id=str(episode.id),
+                    name=episode.name,
+                    episode_profile=episode.episode_profile,
+                    speaker_profile=episode.speaker_profile,
+                    briefing=episode.briefing,
+                    audio_file=episode.audio_file,
+                    audio_url=audio_url,
+                    transcript=episode.transcript,
+                    outline=episode.outline,
+                    created=str(episode.created) if episode.created else None,
+                    job_status=job_status,
+                    error_message=error_message,
+                )
+            )
+
+        return response_episodes
+
+    except Exception as e:
+        logger.error(f"Error listing podcast episodes: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail="Failed to list podcast episodes"
+        )
+
+
+@router.get("/podcasts/episodes/{episode_id}", response_model=PodcastEpisodeResponse)
+async def get_podcast_episode(episode_id: str):
+    """Get a specific podcast episode"""
+    try:
+        episode = await PodcastService.get_episode(episode_id)
+
+        # Get job status and error message if available
+        job_status = None
+        error_message = None
+        if episode.command:
+            try:
+                detail = await episode.get_job_detail()
+                job_status = detail["status"]
+                error_message = detail["error_message"]
+            except Exception:
+                job_status = "unknown"
+        else:
+            # No command but has audio file = completed import
+            job_status = "completed" if episode.audio_file else "unknown"
+
+        audio_url = None
+        if episode.audio_file:
+            audio_path = _resolve_audio_path(episode.audio_file)
+            if audio_path.exists():
+                audio_url = f"/api/podcasts/episodes/{episode.id}/audio"
+
+        return PodcastEpisodeResponse(
+            id=str(episode.id),
+            name=episode.name,
+            episode_profile=episode.episode_profile,
+            speaker_profile=episode.speaker_profile,
+            briefing=episode.briefing,
+            audio_file=episode.audio_file,
+            audio_url=audio_url,
+            transcript=episode.transcript,
+            outline=episode.outline,
+            created=str(episode.created) if episode.created else None,
+            job_status=job_status,
+            error_message=error_message,
+        )
+
+    except Exception as e:
+        logger.error(f"Error fetching podcast episode: {str(e)}")
+        raise HTTPException(status_code=404, detail="Episode not found")
+
+
+@router.get("/podcasts/episodes/{episode_id}/audio")
+async def stream_podcast_episode_audio(episode_id: str):
+    """Stream the audio file associated with a podcast episode"""
+    try:
+        episode = await PodcastService.get_episode(episode_id)
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error fetching podcast episode for audio: {str(e)}")
+        raise HTTPException(status_code=404, detail="Episode not found")
+
+    if not episode.audio_file:
+        raise HTTPException(status_code=404, detail="Episode has no audio file")
+
+    audio_path = _resolve_audio_path(episode.audio_file)
+    if not audio_path.exists():
+        raise HTTPException(status_code=404, detail="Audio file not found on disk")
+
+    return FileResponse(
+        audio_path,
+        media_type="audio/mpeg",
+        filename=audio_path.name,
+    )
+
+
+@router.post("/podcasts/episodes/{episode_id}/retry")
+async def retry_podcast_episode(episode_id: str):
+    """Retry a failed podcast episode by deleting it and submitting a new job"""
+    try:
+        episode = await PodcastService.get_episode(episode_id)
+
+        # Validate episode is in a failed state
+        detail = await episode.get_job_detail()
+        if detail["status"] not in ("failed", "error"):
+            raise HTTPException(
+                status_code=400,
+                detail=f"Episode is not in a failed state (current: {detail['status']})",
+            )
+
+        # Extract params for re-submission
+        ep_profile_name = episode.episode_profile.get("name")
+        sp_profile_name = episode.speaker_profile.get("name")
+        episode_name = episode.name
+        content = episode.content
+
+        if not ep_profile_name or not sp_profile_name:
+            raise HTTPException(
+                status_code=400,
+                detail="Cannot retry: episode or speaker profile name missing from stored data",
+            )
+
+        # Delete audio file if any
+        if episode.audio_file:
+            audio_path = _resolve_audio_path(episode.audio_file)
+            if audio_path.exists():
+                try:
+                    audio_path.unlink()
+                except Exception as e:
+                    logger.warning(f"Failed to delete audio file {audio_path}: {e}")
+
+        # Delete the failed episode
+        await episode.delete()
+
+        # Submit a new job
+        job_id = await PodcastService.submit_generation_job(
+            episode_profile_name=ep_profile_name,
+            speaker_profile_name=sp_profile_name,
+            episode_name=episode_name,
+            content=content,
+        )
+
+        return {"job_id": job_id, "message": "Retry submitted successfully"}
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error retrying podcast episode: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail="Failed to retry episode"
+        )
+
+
+@router.delete("/podcasts/episodes/{episode_id}")
+async def delete_podcast_episode(episode_id: str):
+    """Delete a podcast episode and its associated audio file"""
+    try:
+        # Get the episode first to check if it exists and get the audio file path
+        episode = await PodcastService.get_episode(episode_id)
+
+        # Delete the physical audio file if it exists
+        if episode.audio_file:
+            audio_path = _resolve_audio_path(episode.audio_file)
+            if audio_path.exists():
+                try:
+                    audio_path.unlink()
+                    logger.info(f"Deleted audio file: {audio_path}")
+                except Exception as e:
+                    logger.warning(f"Failed to delete audio file {audio_path}: {e}")
+
+        # Delete the episode from the database
+        await episode.delete()
+
+        logger.info(f"Deleted podcast episode: {episode_id}")
+        return {"message": "Episode deleted successfully", "episode_id": episode_id}
+
+    except Exception as e:
+        logger.error(f"Error deleting podcast episode: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail="Failed to delete episode"
+        )

api/routers/search.py

@@ -0,0 +1,217 @@
+import json
+from typing import AsyncGenerator
+
+from fastapi import APIRouter, HTTPException
+from fastapi.responses import StreamingResponse
+from loguru import logger
+
+from api.models import AskRequest, AskResponse, SearchRequest, SearchResponse
+from open_notebook.ai.models import Model, model_manager
+from open_notebook.domain.notebook import text_search, vector_search
+from open_notebook.exceptions import DatabaseOperationError, InvalidInputError
+from open_notebook.graphs.ask import graph as ask_graph
+
+router = APIRouter()
+
+
+@router.post("/search", response_model=SearchResponse)
+async def search_knowledge_base(search_request: SearchRequest):
+    """Search the knowledge base using text or vector search."""
+    try:
+        if search_request.type == "vector":
+            # Check if embedding model is available for vector search
+            if not await model_manager.get_embedding_model():
+                raise HTTPException(
+                    status_code=400,
+                    detail="Vector search requires an embedding model. Please configure one in the Models section.",
+                )
+
+            results = await vector_search(
+                keyword=search_request.query,
+                results=search_request.limit,
+                source=search_request.search_sources,
+                note=search_request.search_notes,
+                minimum_score=search_request.minimum_score,
+            )
+        else:
+            # Text search
+            results = await text_search(
+                keyword=search_request.query,
+                results=search_request.limit,
+                source=search_request.search_sources,
+                note=search_request.search_notes,
+            )
+
+        return SearchResponse(
+            results=results or [],
+            total_count=len(results) if results else 0,
+            search_type=search_request.type,
+        )
+
+    except InvalidInputError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except DatabaseOperationError as e:
+        logger.error(f"Database error during search: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Search failed: {str(e)}")
+    except Exception as e:
+        logger.error(f"Unexpected error during search: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Search failed: {str(e)}")
+
+
+async def stream_ask_response(
+    question: str, strategy_model: Model, answer_model: Model, final_answer_model: Model
+) -> AsyncGenerator[str, None]:
+    """Stream the ask response as Server-Sent Events."""
+    try:
+        final_answer = None
+
+        async for chunk in ask_graph.astream(
+            input=dict(question=question),  # type: ignore[arg-type]
+            config=dict(
+                configurable=dict(
+                    strategy_model=strategy_model.id,
+                    answer_model=answer_model.id,
+                    final_answer_model=final_answer_model.id,
+                )
+            ),
+            stream_mode="updates",
+        ):
+            if "agent" in chunk:
+                strategy_data = {
+                    "type": "strategy",
+                    "reasoning": chunk["agent"]["strategy"].reasoning,
+                    "searches": [
+                        {"term": search.term, "instructions": search.instructions}
+                        for search in chunk["agent"]["strategy"].searches
+                    ],
+                }
+                yield f"data: {json.dumps(strategy_data)}\n\n"
+
+            elif "provide_answer" in chunk:
+                for answer in chunk["provide_answer"]["answers"]:
+                    answer_data = {"type": "answer", "content": answer}
+                    yield f"data: {json.dumps(answer_data)}\n\n"
+
+            elif "write_final_answer" in chunk:
+                final_answer = chunk["write_final_answer"]["final_answer"]
+                final_data = {"type": "final_answer", "content": final_answer}
+                yield f"data: {json.dumps(final_data)}\n\n"
+
+        # Send completion signal
+        completion_data = {"type": "complete", "final_answer": final_answer}
+        yield f"data: {json.dumps(completion_data)}\n\n"
+
+    except Exception as e:
+        from open_notebook.utils.error_classifier import classify_error
+
+        _, user_message = classify_error(e)
+        logger.error(f"Error in ask streaming: {str(e)}")
+        error_data = {"type": "error", "message": user_message}
+        yield f"data: {json.dumps(error_data)}\n\n"
+
+
+@router.post("/search/ask")
+async def ask_knowledge_base(ask_request: AskRequest):
+    """Ask the knowledge base a question using AI models."""
+    try:
+        # Validate models exist
+        strategy_model = await Model.get(ask_request.strategy_model)
+        answer_model = await Model.get(ask_request.answer_model)
+        final_answer_model = await Model.get(ask_request.final_answer_model)
+
+        if not strategy_model:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Strategy model {ask_request.strategy_model} not found",
+            )
+        if not answer_model:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Answer model {ask_request.answer_model} not found",
+            )
+        if not final_answer_model:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Final answer model {ask_request.final_answer_model} not found",
+            )
+
+        # Check if embedding model is available
+        if not await model_manager.get_embedding_model():
+            raise HTTPException(
+                status_code=400,
+                detail="Ask feature requires an embedding model. Please configure one in the Models section.",
+            )
+
+        # For streaming response
+        return StreamingResponse(
+            stream_ask_response(
+                ask_request.question, strategy_model, answer_model, final_answer_model
+            ),
+            media_type="text/plain",
+        )
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error in ask endpoint: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Ask operation failed: {str(e)}")
+
+
+@router.post("/search/ask/simple", response_model=AskResponse)
+async def ask_knowledge_base_simple(ask_request: AskRequest):
+    """Ask the knowledge base a question and return a simple response (non-streaming)."""
+    try:
+        # Validate models exist
+        strategy_model = await Model.get(ask_request.strategy_model)
+        answer_model = await Model.get(ask_request.answer_model)
+        final_answer_model = await Model.get(ask_request.final_answer_model)
+
+        if not strategy_model:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Strategy model {ask_request.strategy_model} not found",
+            )
+        if not answer_model:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Answer model {ask_request.answer_model} not found",
+            )
+        if not final_answer_model:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Final answer model {ask_request.final_answer_model} not found",
+            )
+
+        # Check if embedding model is available
+        if not await model_manager.get_embedding_model():
+            raise HTTPException(
+                status_code=400,
+                detail="Ask feature requires an embedding model. Please configure one in the Models section.",
+            )
+
+        # Run the ask graph and get final result
+        final_answer = None
+        async for chunk in ask_graph.astream(
+            input=dict(question=ask_request.question),  # type: ignore[arg-type]
+            config=dict(
+                configurable=dict(
+                    strategy_model=strategy_model.id,
+                    answer_model=answer_model.id,
+                    final_answer_model=final_answer_model.id,
+                )
+            ),
+            stream_mode="updates",
+        ):
+            if "write_final_answer" in chunk:
+                final_answer = chunk["write_final_answer"]["final_answer"]
+
+        if not final_answer:
+            raise HTTPException(status_code=500, detail="No answer generated")
+
+        return AskResponse(answer=final_answer, question=ask_request.question)
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error in ask simple endpoint: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Ask operation failed: {str(e)}")

api/routers/settings.py

@@ -0,0 +1,88 @@
+from fastapi import APIRouter, HTTPException
+from loguru import logger
+
+from api.models import SettingsResponse, SettingsUpdate
+from open_notebook.domain.content_settings import ContentSettings
+from open_notebook.exceptions import InvalidInputError
+
+router = APIRouter()
+
+
+@router.get("/settings", response_model=SettingsResponse)
+async def get_settings():
+    """Get all application settings."""
+    try:
+        settings: ContentSettings = await ContentSettings.get_instance()  # type: ignore[assignment]
+
+        return SettingsResponse(
+            default_content_processing_engine_doc=settings.default_content_processing_engine_doc,
+            default_content_processing_engine_url=settings.default_content_processing_engine_url,
+            default_embedding_option=settings.default_embedding_option,
+            auto_delete_files=settings.auto_delete_files,
+            youtube_preferred_languages=settings.youtube_preferred_languages,
+        )
+    except Exception as e:
+        logger.error(f"Error fetching settings: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail="Error fetching settings"
+        )
+
+
+@router.put("/settings", response_model=SettingsResponse)
+async def update_settings(settings_update: SettingsUpdate):
+    """Update application settings."""
+    try:
+        settings: ContentSettings = await ContentSettings.get_instance()  # type: ignore[assignment]
+
+        # Update only provided fields
+        if settings_update.default_content_processing_engine_doc is not None:
+            # Cast to proper literal type
+            from typing import Literal, cast
+
+            settings.default_content_processing_engine_doc = cast(
+                Literal["auto", "docling", "simple"],
+                settings_update.default_content_processing_engine_doc,
+            )
+        if settings_update.default_content_processing_engine_url is not None:
+            from typing import Literal, cast
+
+            settings.default_content_processing_engine_url = cast(
+                Literal["auto", "firecrawl", "jina", "simple"],
+                settings_update.default_content_processing_engine_url,
+            )
+        if settings_update.default_embedding_option is not None:
+            from typing import Literal, cast
+
+            settings.default_embedding_option = cast(
+                Literal["ask", "always", "never"],
+                settings_update.default_embedding_option,
+            )
+        if settings_update.auto_delete_files is not None:
+            from typing import Literal, cast
+
+            settings.auto_delete_files = cast(
+                Literal["yes", "no"], settings_update.auto_delete_files
+            )
+        if settings_update.youtube_preferred_languages is not None:
+            settings.youtube_preferred_languages = (
+                settings_update.youtube_preferred_languages
+            )
+
+        await settings.update()
+
+        return SettingsResponse(
+            default_content_processing_engine_doc=settings.default_content_processing_engine_doc,
+            default_content_processing_engine_url=settings.default_content_processing_engine_url,
+            default_embedding_option=settings.default_embedding_option,
+            auto_delete_files=settings.auto_delete_files,
+            youtube_preferred_languages=settings.youtube_preferred_languages,
+        )
+    except HTTPException:
+        raise
+    except InvalidInputError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        logger.error(f"Error updating settings: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail="Error updating settings"
+        )

api/routers/source_chat.py

@@ -0,0 +1,554 @@
+import asyncio
+import json
+from typing import AsyncGenerator, List, Optional
+
+from fastapi import APIRouter, HTTPException, Path
+from fastapi.responses import StreamingResponse
+from langchain_core.messages import HumanMessage
+from langchain_core.runnables import RunnableConfig
+from loguru import logger
+from pydantic import BaseModel, Field
+
+from open_notebook.database.repository import ensure_record_id, repo_query
+from open_notebook.domain.notebook import ChatSession, Source
+from open_notebook.exceptions import (
+    NotFoundError,
+)
+from open_notebook.graphs.source_chat import source_chat_graph as source_chat_graph
+from open_notebook.utils.graph_utils import get_session_message_count
+
+router = APIRouter()
+
+
+# Request/Response models
+class CreateSourceChatSessionRequest(BaseModel):
+    source_id: str = Field(..., description="Source ID to create chat session for")
+    title: Optional[str] = Field(None, description="Optional session title")
+    model_override: Optional[str] = Field(
+        None, description="Optional model override for this session"
+    )
+
+class UpdateSourceChatSessionRequest(BaseModel):
+    title: Optional[str] = Field(None, description="New session title")
+    model_override: Optional[str] = Field(
+        None, description="Model override for this session"
+    )
+
+class ChatMessage(BaseModel):
+    id: str = Field(..., description="Message ID")
+    type: str = Field(..., description="Message type (human|ai)")
+    content: str = Field(..., description="Message content")
+    timestamp: Optional[str] = Field(None, description="Message timestamp")
+
+
+class ContextIndicator(BaseModel):
+    sources: List[str] = Field(
+        default_factory=list, description="Source IDs used in context"
+    )
+    insights: List[str] = Field(
+        default_factory=list, description="Insight IDs used in context"
+    )
+    notes: List[str] = Field(
+        default_factory=list, description="Note IDs used in context"
+    )
+
+class SourceChatSessionResponse(BaseModel):
+    id: str = Field(..., description="Session ID")
+    title: str = Field(..., description="Session title")
+    source_id: str = Field(..., description="Source ID")
+    model_override: Optional[str] = Field(
+        None, description="Model override for this session"
+    )
+    created: str = Field(..., description="Creation timestamp")
+    updated: str = Field(..., description="Last update timestamp")
+    message_count: Optional[int] = Field(
+        None, description="Number of messages in session"
+    )
+
+class SourceChatSessionWithMessagesResponse(SourceChatSessionResponse):
+    messages: List[ChatMessage] = Field(
+        default_factory=list, description="Session messages"
+    )
+    context_indicators: Optional[ContextIndicator] = Field(
+        None, description="Context indicators from last response"
+    )
+
+class SendMessageRequest(BaseModel):
+    message: str = Field(..., description="User message content")
+    model_override: Optional[str] = Field(
+        None, description="Optional model override for this message"
+    )
+
+class SuccessResponse(BaseModel):
+    success: bool = Field(True, description="Operation success status")
+    message: str = Field(..., description="Success message")
+
+
+@router.post(
+    "/sources/{source_id}/chat/sessions", response_model=SourceChatSessionResponse
+)
+async def create_source_chat_session(
+    request: CreateSourceChatSessionRequest,
+    source_id: str = Path(..., description="Source ID"),
+):
+    """Create a new chat session for a source."""
+    try:
+        # Verify source exists
+        full_source_id = (
+            source_id if source_id.startswith("source:") else f"source:{source_id}"
+        )
+        source = await Source.get(full_source_id)
+        if not source:
+            raise HTTPException(status_code=404, detail="Source not found")
+
+        # Create new session with model_override support
+        session = ChatSession(
+            title=request.title or f"Source Chat {asyncio.get_event_loop().time():.0f}",
+            model_override=request.model_override,
+        )
+        await session.save()
+
+        # Relate session to source using "refers_to" relation
+        await session.relate("refers_to", full_source_id)
+
+        return SourceChatSessionResponse(
+            id=session.id or "",
+            title=session.title or "Untitled Session",
+            source_id=source_id,
+            model_override=session.model_override,
+            created=str(session.created),
+            updated=str(session.updated),
+            message_count=0,
+        )
+    except NotFoundError:
+        raise HTTPException(status_code=404, detail="Source not found")
+    except Exception as e:
+        logger.error(f"Error creating source chat session: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error creating source chat session: {str(e)}"
+        )
+
+
+@router.get(
+    "/sources/{source_id}/chat/sessions", response_model=List[SourceChatSessionResponse]
+)
+async def get_source_chat_sessions(source_id: str = Path(..., description="Source ID")):
+    """Get all chat sessions for a source."""
+    try:
+        # Verify source exists
+        full_source_id = (
+            source_id if source_id.startswith("source:") else f"source:{source_id}"
+        )
+        source = await Source.get(full_source_id)
+        if not source:
+            raise HTTPException(status_code=404, detail="Source not found")
+
+        # Get sessions that refer to this source - first get relations, then sessions
+        relations = await repo_query(
+            "SELECT in FROM refers_to WHERE out = $source_id",
+            {"source_id": ensure_record_id(full_source_id)},
+        )
+
+        sessions = []
+        for relation in relations:
+            session_id_raw = relation.get("in")
+            if session_id_raw:
+                session_id = str(session_id_raw)
+
+                session_result = await repo_query(
+                    "SELECT * FROM $id", {"id": ensure_record_id(session_id)}
+                )
+                if session_result and len(session_result) > 0:
+                    session_data = session_result[0]
+
+                    # Get message count from LangGraph state
+                    msg_count = await get_session_message_count(
+                        source_chat_graph, session_id
+                    )
+
+                    sessions.append(
+                        SourceChatSessionResponse(
+                            id=session_data.get("id") or "",
+                            title=session_data.get("title") or "Untitled Session",
+                            source_id=source_id,
+                            model_override=session_data.get("model_override"),
+                            created=str(session_data.get("created")),
+                            updated=str(session_data.get("updated")),
+                            message_count=msg_count,
+                        )
+                    )
+
+        # Sort sessions by created date (newest first)
+        sessions.sort(key=lambda x: x.created, reverse=True)
+        return sessions
+    except NotFoundError:
+        raise HTTPException(status_code=404, detail="Source not found")
+    except Exception as e:
+        logger.error(f"Error fetching source chat sessions: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error fetching source chat sessions: {str(e)}"
+        )
+
+
+@router.get(
+    "/sources/{source_id}/chat/sessions/{session_id}",
+    response_model=SourceChatSessionWithMessagesResponse,
+)
+async def get_source_chat_session(
+    source_id: str = Path(..., description="Source ID"),
+    session_id: str = Path(..., description="Session ID"),
+):
+    """Get a specific source chat session with its messages."""
+    try:
+        # Verify source exists
+        full_source_id = (
+            source_id if source_id.startswith("source:") else f"source:{source_id}"
+        )
+        source = await Source.get(full_source_id)
+        if not source:
+            raise HTTPException(status_code=404, detail="Source not found")
+
+        # Get session
+        full_session_id = (
+            session_id
+            if session_id.startswith("chat_session:")
+            else f"chat_session:{session_id}"
+        )
+        session = await ChatSession.get(full_session_id)
+        if not session:
+            raise HTTPException(status_code=404, detail="Session not found")
+
+        # Verify session is related to this source
+        relation_query = await repo_query(
+            "SELECT * FROM refers_to WHERE in = $session_id AND out = $source_id",
+            {
+                "session_id": ensure_record_id(full_session_id),
+                "source_id": ensure_record_id(full_source_id),
+            },
+        )
+
+        if not relation_query:
+            raise HTTPException(
+                status_code=404, detail="Session not found for this source"
+            )
+
+        # Get session state from LangGraph to retrieve messages
+        # Use sync get_state() in a thread since SqliteSaver doesn't support async
+        thread_state = await asyncio.to_thread(
+            source_chat_graph.get_state,
+            config=RunnableConfig(configurable={"thread_id": full_session_id}),
+        )
+
+        # Extract messages from state
+        messages: list[ChatMessage] = []
+        context_indicators = None
+
+        if thread_state and thread_state.values:
+            # Extract messages
+            if "messages" in thread_state.values:
+                for msg in thread_state.values["messages"]:
+                    messages.append(
+                        ChatMessage(
+                            id=getattr(msg, "id", f"msg_{len(messages)}"),
+                            type=msg.type if hasattr(msg, "type") else "unknown",
+                            content=msg.content
+                            if hasattr(msg, "content")
+                            else str(msg),
+                            timestamp=None,  # LangChain messages don't have timestamps by default
+                        )
+                    )
+
+            # Extract context indicators from the last state
+            if "context_indicators" in thread_state.values:
+                context_data = thread_state.values["context_indicators"]
+                context_indicators = ContextIndicator(
+                    sources=context_data.get("sources", []),
+                    insights=context_data.get("insights", []),
+                    notes=context_data.get("notes", []),
+                )
+
+        return SourceChatSessionWithMessagesResponse(
+            id=session.id or "",
+            title=session.title or "Untitled Session",
+            source_id=source_id,
+            model_override=getattr(session, "model_override", None),
+            created=str(session.created),
+            updated=str(session.updated),
+            message_count=len(messages),
+            messages=messages,
+            context_indicators=context_indicators,
+        )
+    except NotFoundError:
+        raise HTTPException(status_code=404, detail="Source or session not found")
+    except Exception as e:
+        logger.error(f"Error fetching source chat session: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error fetching source chat session: {str(e)}"
+        )
+
+
+@router.put(
+    "/sources/{source_id}/chat/sessions/{session_id}",
+    response_model=SourceChatSessionResponse,
+)
+async def update_source_chat_session(
+    request: UpdateSourceChatSessionRequest,
+    source_id: str = Path(..., description="Source ID"),
+    session_id: str = Path(..., description="Session ID"),
+):
+    """Update source chat session title and/or model override."""
+    try:
+        # Verify source exists
+        full_source_id = (
+            source_id if source_id.startswith("source:") else f"source:{source_id}"
+        )
+        source = await Source.get(full_source_id)
+        if not source:
+            raise HTTPException(status_code=404, detail="Source not found")
+
+        # Get session
+        full_session_id = (
+            session_id
+            if session_id.startswith("chat_session:")
+            else f"chat_session:{session_id}"
+        )
+        session = await ChatSession.get(full_session_id)
+        if not session:
+            raise HTTPException(status_code=404, detail="Session not found")
+
+        # Verify session is related to this source
+        relation_query = await repo_query(
+            "SELECT * FROM refers_to WHERE in = $session_id AND out = $source_id",
+            {
+                "session_id": ensure_record_id(full_session_id),
+                "source_id": ensure_record_id(full_source_id),
+            },
+        )
+
+        if not relation_query:
+            raise HTTPException(
+                status_code=404, detail="Session not found for this source"
+            )
+
+        # Update session fields
+        if request.title is not None:
+            session.title = request.title
+        if request.model_override is not None:
+            session.model_override = request.model_override
+
+        await session.save()
+
+        # Get message count from LangGraph state
+        msg_count = await get_session_message_count(source_chat_graph, full_session_id)
+
+        return SourceChatSessionResponse(
+            id=session.id or "",
+            title=session.title or "Untitled Session",
+            source_id=source_id,
+            model_override=getattr(session, "model_override", None),
+            created=str(session.created),
+            updated=str(session.updated),
+            message_count=msg_count,
+        )
+    except NotFoundError:
+        raise HTTPException(status_code=404, detail="Source or session not found")
+    except Exception as e:
+        logger.error(f"Error updating source chat session: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error updating source chat session: {str(e)}"
+        )
+
+
+@router.delete(
+    "/sources/{source_id}/chat/sessions/{session_id}", response_model=SuccessResponse
+)
+async def delete_source_chat_session(
+    source_id: str = Path(..., description="Source ID"),
+    session_id: str = Path(..., description="Session ID"),
+):
+    """Delete a source chat session."""
+    try:
+        # Verify source exists
+        full_source_id = (
+            source_id if source_id.startswith("source:") else f"source:{source_id}"
+        )
+        source = await Source.get(full_source_id)
+        if not source:
+            raise HTTPException(status_code=404, detail="Source not found")
+
+        # Get session
+        full_session_id = (
+            session_id
+            if session_id.startswith("chat_session:")
+            else f"chat_session:{session_id}"
+        )
+        session = await ChatSession.get(full_session_id)
+        if not session:
+            raise HTTPException(status_code=404, detail="Session not found")
+
+        # Verify session is related to this source
+        relation_query = await repo_query(
+            "SELECT * FROM refers_to WHERE in = $session_id AND out = $source_id",
+            {
+                "session_id": ensure_record_id(full_session_id),
+                "source_id": ensure_record_id(full_source_id),
+            },
+        )
+
+        if not relation_query:
+            raise HTTPException(
+                status_code=404, detail="Session not found for this source"
+            )
+
+        await session.delete()
+
+        return SuccessResponse(
+            success=True, message="Source chat session deleted successfully"
+        )
+    except NotFoundError:
+        raise HTTPException(status_code=404, detail="Source or session not found")
+    except Exception as e:
+        logger.error(f"Error deleting source chat session: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error deleting source chat session: {str(e)}"
+        )
+
+
+async def stream_source_chat_response(
+    session_id: str, source_id: str, message: str, model_override: Optional[str] = None
+) -> AsyncGenerator[str, None]:
+    """Stream the source chat response as Server-Sent Events."""
+    try:
+        # Get current state
+        # Use sync get_state() in a thread since SqliteSaver doesn't support async
+        current_state = await asyncio.to_thread(
+            source_chat_graph.get_state,
+            config=RunnableConfig(configurable={"thread_id": session_id}),
+        )
+
+        # Prepare state for execution
+        state_values = current_state.values if current_state else {}
+        state_values["messages"] = state_values.get("messages", [])
+        state_values["source_id"] = source_id
+        state_values["model_override"] = model_override
+
+        # Add user message to state
+        user_message = HumanMessage(content=message)
+        state_values["messages"].append(user_message)
+
+        # Send user message event
+        user_event = {"type": "user_message", "content": message, "timestamp": None}
+        yield f"data: {json.dumps(user_event)}\n\n"
+
+        # Execute source chat graph synchronously (like notebook chat does)
+        result = source_chat_graph.invoke(
+            input=state_values,  # type: ignore[arg-type]
+            config=RunnableConfig(
+                configurable={"thread_id": session_id, "model_id": model_override}
+            ),
+        )
+
+        # Stream the complete AI response
+        if "messages" in result:
+            for msg in result["messages"]:
+                if hasattr(msg, "type") and msg.type == "ai":
+                    ai_event = {
+                        "type": "ai_message",
+                        "content": msg.content if hasattr(msg, "content") else str(msg),
+                        "timestamp": None,
+                    }
+                    yield f"data: {json.dumps(ai_event)}\n\n"
+
+        # Stream context indicators
+        if "context_indicators" in result:
+            context_event = {
+                "type": "context_indicators",
+                "data": result["context_indicators"],
+            }
+            yield f"data: {json.dumps(context_event)}\n\n"
+
+        # Send completion signal
+        completion_event = {"type": "complete"}
+        yield f"data: {json.dumps(completion_event)}\n\n"
+
+    except Exception as e:
+        from open_notebook.utils.error_classifier import classify_error
+
+        _, user_message = classify_error(e)
+        logger.error(f"Error in source chat streaming: {str(e)}")
+        error_event = {"type": "error", "message": user_message}
+        yield f"data: {json.dumps(error_event)}\n\n"
+
+
+@router.post("/sources/{source_id}/chat/sessions/{session_id}/messages")
+async def send_message_to_source_chat(
+    request: SendMessageRequest,
+    source_id: str = Path(..., description="Source ID"),
+    session_id: str = Path(..., description="Session ID"),
+):
+    """Send a message to source chat session with SSE streaming response."""
+    try:
+        # Verify source exists
+        full_source_id = (
+            source_id if source_id.startswith("source:") else f"source:{source_id}"
+        )
+        source = await Source.get(full_source_id)
+        if not source:
+            raise HTTPException(status_code=404, detail="Source not found")
+
+        # Verify session exists and is related to source
+        full_session_id = (
+            session_id
+            if session_id.startswith("chat_session:")
+            else f"chat_session:{session_id}"
+        )
+        session = await ChatSession.get(full_session_id)
+        if not session:
+            raise HTTPException(status_code=404, detail="Session not found")
+
+        # Verify session is related to this source
+        relation_query = await repo_query(
+            "SELECT * FROM refers_to WHERE in = $session_id AND out = $source_id",
+            {
+                "session_id": ensure_record_id(full_session_id),
+                "source_id": ensure_record_id(full_source_id),
+            },
+        )
+
+        if not relation_query:
+            raise HTTPException(
+                status_code=404, detail="Session not found for this source"
+            )
+
+        if not request.message:
+            raise HTTPException(status_code=400, detail="Message content is required")
+
+        # Determine model override (request override takes precedence over session override)
+        model_override = request.model_override or getattr(
+            session, "model_override", None
+        )
+
+        # Update session timestamp
+        await session.save()
+
+        # Return streaming response
+        return StreamingResponse(
+            stream_source_chat_response(
+                session_id=full_session_id,
+                source_id=full_source_id,
+                message=request.message,
+                model_override=model_override,
+            ),
+            media_type="text/plain",
+            headers={
+                "Cache-Control": "no-cache",
+                "Connection": "keep-alive",
+                "Content-Type": "text/plain; charset=utf-8",
+            },
+        )
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error sending message to source chat: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error sending message: {str(e)}")

api/routers/sources.py

@@ -0,0 +1,1045 @@
+import asyncio
+import os
+from pathlib import Path
+from typing import Any, List, Optional
+
+from fastapi import (
+    APIRouter,
+    Depends,
+    File,
+    Form,
+    HTTPException,
+    Query,
+    UploadFile,
+)
+from fastapi.responses import FileResponse, Response
+from loguru import logger
+from surreal_commands import execute_command_sync, submit_command
+
+from api.command_service import CommandService
+from api.models import (
+    AssetModel,
+    CreateSourceInsightRequest,
+    InsightCreationResponse,
+    SourceCreate,
+    SourceInsightResponse,
+    SourceListResponse,
+    SourceResponse,
+    SourceStatusResponse,
+    SourceUpdate,
+)
+from commands.source_commands import SourceProcessingInput
+from open_notebook.config import UPLOADS_FOLDER
+from open_notebook.database.repository import ensure_record_id, repo_query
+from open_notebook.domain.notebook import Asset, Notebook, Source
+from open_notebook.domain.transformation import Transformation
+from open_notebook.exceptions import InvalidInputError
+
+router = APIRouter()
+
+
+def generate_unique_filename(original_filename: str, upload_folder: str) -> str:
+    """Generate unique filename like Streamlit app (append counter if file exists)."""
+    file_path = Path(upload_folder)
+    file_path.mkdir(parents=True, exist_ok=True)
+
+    # Strip directory components to prevent path traversal
+    safe_filename = os.path.basename(original_filename)
+    if not safe_filename:
+        raise ValueError("Invalid filename")
+
+    # Split filename and extension
+    stem = Path(safe_filename).stem
+    suffix = Path(safe_filename).suffix
+
+    # Check if file exists and generate unique name
+    counter = 0
+    while True:
+        if counter == 0:
+            new_filename = safe_filename
+        else:
+            new_filename = f"{stem} ({counter}){suffix}"
+
+        full_path = file_path / new_filename
+        # Verify resolved path stays within upload folder
+        resolved = full_path.resolve()
+        if not str(resolved).startswith(str(file_path.resolve()) + os.sep):
+            raise ValueError("Invalid filename: path traversal detected")
+        if not resolved.exists():
+            return str(resolved)
+        counter += 1
+
+
+async def save_uploaded_file(upload_file: UploadFile) -> str:
+    """Save uploaded file to uploads folder and return file path."""
+    if not upload_file.filename:
+        raise ValueError("No filename provided")
+
+    # Generate unique filename
+    file_path = generate_unique_filename(upload_file.filename, UPLOADS_FOLDER)
+
+    try:
+        # Save file
+        with open(file_path, "wb") as f:
+            content = await upload_file.read()
+            f.write(content)
+
+        logger.info(f"Saved uploaded file to: {file_path}")
+        return file_path
+    except Exception as e:
+        logger.error(f"Failed to save uploaded file: {e}")
+        # Clean up partial file if it exists
+        if os.path.exists(file_path):
+            os.unlink(file_path)
+        raise
+
+
+def parse_source_form_data(
+    type: str = Form(...),
+    notebook_id: Optional[str] = Form(None),
+    notebooks: Optional[str] = Form(None),  # JSON string of notebook IDs
+    url: Optional[str] = Form(None),
+    content: Optional[str] = Form(None),
+    title: Optional[str] = Form(None),
+    transformations: Optional[str] = Form(None),  # JSON string of transformation IDs
+    embed: str = Form("false"),  # Accept as string, convert to bool
+    delete_source: str = Form("false"),  # Accept as string, convert to bool
+    async_processing: str = Form("false"),  # Accept as string, convert to bool
+    file: Optional[UploadFile] = File(None),
+) -> tuple[SourceCreate, Optional[UploadFile]]:
+    """Parse form data into SourceCreate model and return upload file separately."""
+    import json
+
+    # Convert string booleans to actual booleans
+    def str_to_bool(value: str) -> bool:
+        return value.lower() in ("true", "1", "yes", "on")
+
+    embed_bool = str_to_bool(embed)
+    delete_source_bool = str_to_bool(delete_source)
+    async_processing_bool = str_to_bool(async_processing)
+
+    # Parse JSON strings
+    notebooks_list = None
+    if notebooks:
+        try:
+            notebooks_list = json.loads(notebooks)
+        except json.JSONDecodeError:
+            logger.error(f"Invalid JSON in notebooks field: {notebooks}")
+            raise ValueError("Invalid JSON in notebooks field")
+
+    transformations_list = []
+    if transformations:
+        try:
+            transformations_list = json.loads(transformations)
+        except json.JSONDecodeError:
+            logger.error(f"Invalid JSON in transformations field: {transformations}")
+            raise ValueError("Invalid JSON in transformations field")
+
+    # Create SourceCreate instance
+    try:
+        source_data = SourceCreate(
+            type=type,
+            notebook_id=notebook_id,
+            notebooks=notebooks_list,
+            url=url,
+            content=content,
+            title=title,
+            file_path=None,  # Will be set later if file is uploaded
+            transformations=transformations_list,
+            embed=embed_bool,
+            delete_source=delete_source_bool,
+            async_processing=async_processing_bool,
+        )
+        pass  # SourceCreate instance created successfully
+    except Exception as e:
+        logger.error(f"Failed to create SourceCreate instance: {e}")
+        raise
+
+    return source_data, file
+
+
+@router.get("/sources", response_model=List[SourceListResponse])
+async def get_sources(
+    notebook_id: Optional[str] = Query(None, description="Filter by notebook ID"),
+    limit: int = Query(
+        50, ge=1, le=100, description="Number of sources to return (1-100)"
+    ),
+    offset: int = Query(0, ge=0, description="Number of sources to skip"),
+    sort_by: str = Query(
+        "updated", description="Field to sort by (created or updated)"
+    ),
+    sort_order: str = Query("desc", description="Sort order (asc or desc)"),
+):
+    """Get sources with pagination and sorting support."""
+    try:
+        # Validate sort parameters
+        if sort_by not in ["created", "updated"]:
+            raise HTTPException(
+                status_code=400, detail="sort_by must be 'created' or 'updated'"
+            )
+        if sort_order.lower() not in ["asc", "desc"]:
+            raise HTTPException(
+                status_code=400, detail="sort_order must be 'asc' or 'desc'"
+            )
+
+        # Build ORDER BY clause
+        order_clause = f"ORDER BY {sort_by} {sort_order.upper()}"
+
+        # Build the query
+        if notebook_id:
+            # Verify notebook exists first
+            notebook = await Notebook.get(notebook_id)
+            if not notebook:
+                raise HTTPException(status_code=404, detail="Notebook not found")
+
+            # Query sources for specific notebook - include command field with FETCH
+            query = f"""
+                SELECT id, asset, created, title, updated, topics, command,
+                (SELECT VALUE count() FROM source_insight WHERE source = $parent.id GROUP ALL)[0].count OR 0 AS insights_count,
+                (SELECT VALUE id FROM source_embedding WHERE source = $parent.id LIMIT 1) != [] AS embedded
+                FROM (select value in from reference where out=$notebook_id)
+                {order_clause}
+                LIMIT $limit START $offset
+                FETCH command
+            """
+            result = await repo_query(
+                query,
+                {
+                    "notebook_id": ensure_record_id(notebook_id),
+                    "limit": limit,
+                    "offset": offset,
+                },
+            )
+        else:
+            # Query all sources - include command field with FETCH
+            query = f"""
+                SELECT id, asset, created, title, updated, topics, command,
+                (SELECT VALUE count() FROM source_insight WHERE source = $parent.id GROUP ALL)[0].count OR 0 AS insights_count,
+                (SELECT VALUE id FROM source_embedding WHERE source = $parent.id LIMIT 1) != [] AS embedded
+                FROM source
+                {order_clause}
+                LIMIT $limit START $offset
+                FETCH command
+            """
+            result = await repo_query(query, {"limit": limit, "offset": offset})
+
+        # Convert result to response model
+        # Command data is already fetched via FETCH command clause
+        response_list = []
+        for row in result:
+            command = row.get("command")
+            command_id = None
+            status = None
+            processing_info = None
+
+            # Extract status from fetched command object (already resolved by FETCH)
+            if command and isinstance(command, dict):
+                command_id = str(command.get("id")) if command.get("id") else None
+                status = command.get("status")
+                # Extract execution metadata from nested result structure
+                result_data = command.get("result")
+                execution_metadata = (
+                    result_data.get("execution_metadata", {})
+                    if isinstance(result_data, dict)
+                    else {}
+                )
+                processing_info = {
+                    "started_at": execution_metadata.get("started_at"),
+                    "completed_at": execution_metadata.get("completed_at"),
+                    "error": command.get("error_message"),
+                }
+            elif command:
+                # Command exists but FETCH failed to resolve it (broken reference)
+                command_id = str(command)
+                status = "unknown"
+
+            response_list.append(
+                SourceListResponse(
+                    id=row["id"],
+                    title=row.get("title"),
+                    topics=row.get("topics") or [],
+                    asset=AssetModel(
+                        file_path=row["asset"].get("file_path")
+                        if row.get("asset")
+                        else None,
+                        url=row["asset"].get("url") if row.get("asset") else None,
+                    )
+                    if row.get("asset")
+                    else None,
+                    embedded=row.get("embedded", False),
+                    embedded_chunks=0,  # Not needed in list view
+                    insights_count=row.get("insights_count", 0),
+                    created=str(row["created"]),
+                    updated=str(row["updated"]),
+                    # Status fields from fetched command
+                    command_id=command_id,
+                    status=status,
+                    processing_info=processing_info,
+                )
+            )
+
+        return response_list
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error fetching sources: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error fetching sources: {str(e)}")
+
+
+@router.post("/sources", response_model=SourceResponse)
+async def create_source(
+    form_data: tuple[SourceCreate, Optional[UploadFile]] = Depends(
+        parse_source_form_data
+    ),
+):
+    """Create a new source with support for both JSON and multipart form data."""
+    source_data, upload_file = form_data
+
+    # Initialize file_path before try block so exception handlers can reference it
+    file_path = None
+
+    try:
+        # Verify all specified notebooks exist (backward compatibility support)
+        for notebook_id in source_data.notebooks or []:
+            notebook = await Notebook.get(notebook_id)
+            if not notebook:
+                raise HTTPException(
+                    status_code=404, detail=f"Notebook {notebook_id} not found"
+                )
+
+        # Handle file upload if provided
+        if upload_file and source_data.type == "upload":
+            try:
+                file_path = await save_uploaded_file(upload_file)
+            except Exception as e:
+                logger.error(f"File upload failed: {e}")
+                raise HTTPException(
+                    status_code=400, detail=f"File upload failed: {str(e)}"
+                )
+
+        # Prepare content_state for processing
+        content_state: dict[str, Any] = {}
+
+        if source_data.type == "link":
+            if not source_data.url:
+                raise HTTPException(
+                    status_code=400, detail="URL is required for link type"
+                )
+            content_state["url"] = source_data.url
+        elif source_data.type == "upload":
+            # Use uploaded file path or provided file_path (backward compatibility)
+            final_file_path = file_path or source_data.file_path
+            if not final_file_path:
+                raise HTTPException(
+                    status_code=400,
+                    detail="File upload or file_path is required for upload type",
+                )
+            # Validate file_path is within the uploads directory to prevent LFI
+            uploads_resolved = Path(UPLOADS_FOLDER).resolve()
+            file_resolved = Path(final_file_path).resolve()
+            if not str(file_resolved).startswith(str(uploads_resolved) + os.sep):
+                raise HTTPException(
+                    status_code=400,
+                    detail="Invalid file path: must be within the uploads directory",
+                )
+            content_state["file_path"] = final_file_path
+            content_state["delete_source"] = source_data.delete_source
+        elif source_data.type == "text":
+            if not source_data.content:
+                raise HTTPException(
+                    status_code=400, detail="Content is required for text type"
+                )
+            content_state["content"] = source_data.content
+        else:
+            raise HTTPException(
+                status_code=400,
+                detail="Invalid source type. Must be link, upload, or text",
+            )
+
+        # Validate transformations exist
+        transformation_ids = source_data.transformations or []
+        for trans_id in transformation_ids:
+            transformation = await Transformation.get(trans_id)
+            if not transformation:
+                raise HTTPException(
+                    status_code=404, detail=f"Transformation {trans_id} not found"
+                )
+
+        # Branch based on processing mode
+        if source_data.async_processing:
+            # ASYNC PATH: Create source record first, then queue command
+            logger.info("Using async processing path")
+
+            # Create source record with asset - let SurrealDB generate the ID
+            # Persist asset before save so it's available for retry if processing fails
+            if source_data.type == "link":
+                source_asset = Asset(url=source_data.url)
+            elif source_data.type == "upload":
+                source_asset = Asset(file_path=file_path or source_data.file_path)
+            else:
+                source_asset = None
+
+            source = Source(
+                title=source_data.title or "Processing...",
+                topics=[],
+                asset=source_asset,
+            )
+            await source.save()
+
+            # Add source to notebooks immediately so it appears in the UI
+            # The source_graph will skip adding duplicates
+            for notebook_id in source_data.notebooks or []:
+                await source.add_to_notebook(notebook_id)
+
+            try:
+                # Import command modules to ensure they're registered
+                import commands.source_commands  # noqa: F401
+
+                # Submit command for background processing
+                command_input = SourceProcessingInput(
+                    source_id=str(source.id),
+                    content_state=content_state,
+                    notebook_ids=source_data.notebooks,
+                    transformations=transformation_ids,
+                    embed=source_data.embed,
+                )
+
+                command_id = await CommandService.submit_command_job(
+                    "open_notebook",  # app name
+                    "process_source",  # command name
+                    command_input.model_dump(),
+                )
+
+                logger.info(f"Submitted async processing command: {command_id}")
+
+                # Update source with command reference immediately
+                # command_id already includes 'command:' prefix
+                source.command = ensure_record_id(command_id)
+                await source.save()
+
+                # Return source with command info
+                return SourceResponse(
+                    id=source.id or "",
+                    title=source.title,
+                    topics=source.topics or [],
+                    asset=None,  # Will be populated after processing
+                    full_text=None,  # Will be populated after processing
+                    embedded=False,  # Will be updated after processing
+                    embedded_chunks=0,
+                    created=str(source.created),
+                    updated=str(source.updated),
+                    command_id=command_id,
+                    status="new",
+                    processing_info={"async": True, "queued": True},
+                )
+
+            except Exception as e:
+                logger.error(f"Failed to submit async processing command: {e}")
+                # Clean up source record on command submission failure
+                try:
+                    await source.delete()
+                except Exception:
+                    pass
+                # Clean up uploaded file if we created it
+                if file_path and upload_file:
+                    try:
+                        os.unlink(file_path)
+                    except Exception:
+                        pass
+                raise HTTPException(
+                    status_code=500, detail=f"Failed to queue processing: {str(e)}"
+                )
+
+        else:
+            # SYNC PATH: Execute synchronously using execute_command_sync
+            logger.info("Using sync processing path")
+
+            try:
+                # Import command modules to ensure they're registered
+                import commands.source_commands  # noqa: F401
+
+                # Create source record - let SurrealDB generate the ID
+                source = Source(
+                    title=source_data.title or "Processing...",
+                    topics=[],
+                )
+                await source.save()
+
+                # Add source to notebooks immediately so it appears in the UI
+                # The source_graph will skip adding duplicates
+                for notebook_id in source_data.notebooks or []:
+                    await source.add_to_notebook(notebook_id)
+
+                # Execute command synchronously
+                command_input = SourceProcessingInput(
+                    source_id=str(source.id),
+                    content_state=content_state,
+                    notebook_ids=source_data.notebooks,
+                    transformations=transformation_ids,
+                    embed=source_data.embed,
+                )
+
+                # Run in thread pool to avoid blocking the event loop
+                # execute_command_sync uses asyncio.run() internally which can't
+                # be called from an already-running event loop (FastAPI)
+                result = await asyncio.to_thread(
+                    execute_command_sync,
+                    "open_notebook",  # app name
+                    "process_source",  # command name
+                    command_input.model_dump(),
+                    timeout=300,  # 5 minute timeout for sync processing
+                )
+
+                if not result.is_success():
+                    logger.error(f"Sync processing failed: {result.error_message}")
+                    # Clean up source record
+                    try:
+                        await source.delete()
+                    except Exception:
+                        pass
+                    # Clean up uploaded file if we created it
+                    if file_path and upload_file:
+                        try:
+                            os.unlink(file_path)
+                        except Exception:
+                            pass
+                    raise HTTPException(
+                        status_code=500,
+                        detail=f"Processing failed: {result.error_message}",
+                    )
+
+                # Get the processed source
+                if not source.id:
+                    raise HTTPException(status_code=500, detail="Source ID is missing")
+                processed_source = await Source.get(source.id)
+                if not processed_source:
+                    raise HTTPException(
+                        status_code=500, detail="Processed source not found"
+                    )
+
+                embedded_chunks = await processed_source.get_embedded_chunks()
+                return SourceResponse(
+                    id=processed_source.id or "",
+                    title=processed_source.title,
+                    topics=processed_source.topics or [],
+                    asset=AssetModel(
+                        file_path=processed_source.asset.file_path
+                        if processed_source.asset
+                        else None,
+                        url=processed_source.asset.url
+                        if processed_source.asset
+                        else None,
+                    )
+                    if processed_source.asset
+                    else None,
+                    full_text=processed_source.full_text,
+                    embedded=embedded_chunks > 0,
+                    embedded_chunks=embedded_chunks,
+                    created=str(processed_source.created),
+                    updated=str(processed_source.updated),
+                    # No command_id or status for sync processing (legacy behavior)
+                )
+
+            except Exception as e:
+                logger.error(f"Sync processing failed: {e}")
+                # Clean up uploaded file if we created it
+                if file_path and upload_file:
+                    try:
+                        os.unlink(file_path)
+                    except Exception:
+                        pass
+                raise
+
+    except HTTPException:
+        # Clean up uploaded file on HTTP exceptions if we created it
+        if file_path and upload_file:
+            try:
+                os.unlink(file_path)
+            except Exception:
+                pass
+        raise
+    except InvalidInputError as e:
+        # Clean up uploaded file on validation errors if we created it
+        if file_path and upload_file:
+            try:
+                os.unlink(file_path)
+            except Exception:
+                pass
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        logger.error(f"Error creating source: {str(e)}")
+        # Clean up uploaded file on unexpected errors if we created it
+        if file_path and upload_file:
+            try:
+                os.unlink(file_path)
+            except Exception:
+                pass
+        raise HTTPException(status_code=500, detail=f"Error creating source: {str(e)}")
+
+
+@router.post("/sources/json", response_model=SourceResponse)
+async def create_source_json(source_data: SourceCreate):
+    """Create a new source using JSON payload (legacy endpoint for backward compatibility)."""
+    # Convert to form data format and call main endpoint
+    form_data = (source_data, None)
+    return await create_source(form_data)
+
+
+async def _resolve_source_file(source_id: str) -> tuple[str, str]:
+    source = await Source.get(source_id)
+    if not source:
+        raise HTTPException(status_code=404, detail="Source not found")
+
+    file_path = source.asset.file_path if source.asset else None
+    if not file_path:
+        raise HTTPException(status_code=404, detail="Source has no file to download")
+
+    safe_root = os.path.realpath(UPLOADS_FOLDER)
+    resolved_path = os.path.realpath(file_path)
+
+    if not resolved_path.startswith(safe_root):
+        logger.warning(
+            f"Blocked download outside uploads directory for source {source_id}: {resolved_path}"
+        )
+        raise HTTPException(status_code=403, detail="Access to file denied")
+
+    if not os.path.exists(resolved_path):
+        raise HTTPException(status_code=404, detail="File not found on server")
+
+    filename = os.path.basename(resolved_path)
+    return resolved_path, filename
+
+
+def _is_source_file_available(source: Source) -> Optional[bool]:
+    if not source or not source.asset or not source.asset.file_path:
+        return None
+
+    file_path = source.asset.file_path
+    safe_root = os.path.realpath(UPLOADS_FOLDER)
+    resolved_path = os.path.realpath(file_path)
+
+    if not resolved_path.startswith(safe_root):
+        return False
+
+    return os.path.exists(resolved_path)
+
+
+@router.get("/sources/{source_id}", response_model=SourceResponse)
+async def get_source(source_id: str):
+    """Get a specific source by ID."""
+    try:
+        source = await Source.get(source_id)
+        if not source:
+            raise HTTPException(status_code=404, detail="Source not found")
+
+        # Get status information if command exists
+        status = None
+        processing_info = None
+        if source.command:
+            try:
+                status = await source.get_status()
+                processing_info = await source.get_processing_progress()
+            except Exception as e:
+                logger.warning(f"Failed to get status for source {source_id}: {e}")
+                status = "unknown"
+
+        embedded_chunks = await source.get_embedded_chunks()
+
+        # Get associated notebooks
+        notebooks_query = await repo_query(
+            "SELECT VALUE out FROM reference WHERE in = $source_id",
+            {"source_id": ensure_record_id(source.id or source_id)},
+        )
+        notebook_ids = (
+            [str(nb_id) for nb_id in notebooks_query] if notebooks_query else []
+        )
+
+        return SourceResponse(
+            id=source.id or "",
+            title=source.title,
+            topics=source.topics or [],
+            asset=AssetModel(
+                file_path=source.asset.file_path if source.asset else None,
+                url=source.asset.url if source.asset else None,
+            )
+            if source.asset
+            else None,
+            full_text=source.full_text,
+            embedded=embedded_chunks > 0,
+            embedded_chunks=embedded_chunks,
+            file_available=_is_source_file_available(source),
+            created=str(source.created),
+            updated=str(source.updated),
+            # Status fields
+            command_id=str(source.command) if source.command else None,
+            status=status,
+            processing_info=processing_info,
+            # Notebook associations
+            notebooks=notebook_ids,
+        )
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error fetching source {source_id}: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error fetching source: {str(e)}")
+
+
+@router.head("/sources/{source_id}/download")
+async def check_source_file(source_id: str):
+    """Check if a source has a downloadable file."""
+    try:
+        await _resolve_source_file(source_id)
+        return Response(status_code=200)
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error checking file for source {source_id}: {str(e)}")
+        raise HTTPException(status_code=500, detail="Failed to verify file")
+
+
+@router.get("/sources/{source_id}/download")
+async def download_source_file(source_id: str):
+    """Download the original file associated with an uploaded source."""
+    try:
+        resolved_path, filename = await _resolve_source_file(source_id)
+        return FileResponse(
+            path=resolved_path,
+            filename=filename,
+            media_type="application/octet-stream",
+        )
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error downloading file for source {source_id}: {str(e)}")
+        raise HTTPException(status_code=500, detail="Failed to download source file")
+
+
+@router.get("/sources/{source_id}/status", response_model=SourceStatusResponse)
+async def get_source_status(source_id: str):
+    """Get processing status for a source."""
+    try:
+        # First, verify source exists
+        source = await Source.get(source_id)
+        if not source:
+            raise HTTPException(status_code=404, detail="Source not found")
+
+        # Check if this is a legacy source (no command)
+        if not source.command:
+            return SourceStatusResponse(
+                status=None,
+                message="Legacy source (completed before async processing)",
+                processing_info=None,
+                command_id=None,
+            )
+
+        # Get command status and processing info
+        try:
+            status = await source.get_status()
+            processing_info = await source.get_processing_progress()
+
+            # Generate descriptive message based on status
+            if status == "completed":
+                message = "Source processing completed successfully"
+            elif status == "failed":
+                message = "Source processing failed"
+            elif status == "running":
+                message = "Source processing in progress"
+            elif status == "queued":
+                message = "Source processing queued"
+            elif status == "unknown":
+                message = "Source processing status unknown"
+            else:
+                message = f"Source processing status: {status}"
+
+            return SourceStatusResponse(
+                status=status,
+                message=message,
+                processing_info=processing_info,
+                command_id=str(source.command) if source.command else None,
+            )
+
+        except Exception as e:
+            logger.warning(f"Failed to get status for source {source_id}: {e}")
+            return SourceStatusResponse(
+                status="unknown",
+                message="Failed to retrieve processing status",
+                processing_info=None,
+                command_id=str(source.command) if source.command else None,
+            )
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error fetching status for source {source_id}: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error fetching source status: {str(e)}"
+        )
+
+
+@router.put("/sources/{source_id}", response_model=SourceResponse)
+async def update_source(source_id: str, source_update: SourceUpdate):
+    """Update a source."""
+    try:
+        source = await Source.get(source_id)
+        if not source:
+            raise HTTPException(status_code=404, detail="Source not found")
+
+        # Update only provided fields
+        if source_update.title is not None:
+            source.title = source_update.title
+        if source_update.topics is not None:
+            source.topics = source_update.topics
+
+        await source.save()
+
+        embedded_chunks = await source.get_embedded_chunks()
+        return SourceResponse(
+            id=source.id or "",
+            title=source.title,
+            topics=source.topics or [],
+            asset=AssetModel(
+                file_path=source.asset.file_path if source.asset else None,
+                url=source.asset.url if source.asset else None,
+            )
+            if source.asset
+            else None,
+            full_text=source.full_text,
+            embedded=embedded_chunks > 0,
+            embedded_chunks=embedded_chunks,
+            created=str(source.created),
+            updated=str(source.updated),
+        )
+    except HTTPException:
+        raise
+    except InvalidInputError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        logger.error(f"Error updating source {source_id}: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error updating source: {str(e)}")
+
+
+@router.post("/sources/{source_id}/retry", response_model=SourceResponse)
+async def retry_source_processing(source_id: str):
+    """Retry processing for a failed or stuck source."""
+    try:
+        # First, verify source exists
+        source = await Source.get(source_id)
+        if not source:
+            raise HTTPException(status_code=404, detail="Source not found")
+
+        # Check if source already has a running command
+        if source.command:
+            try:
+                status = await source.get_status()
+                if status in ["running", "queued"]:
+                    raise HTTPException(
+                        status_code=400,
+                        detail="Source is already processing. Cannot retry while processing is active.",
+                    )
+            except Exception as e:
+                logger.warning(
+                    f"Failed to check current status for source {source_id}: {e}"
+                )
+                # Continue with retry if we can't check status
+
+        # Get notebooks that this source belongs to
+        query = "SELECT notebook FROM reference WHERE source = $source_id"
+        references = await repo_query(query, {"source_id": source_id})
+        notebook_ids = [str(ref["notebook"]) for ref in references]
+
+        if not notebook_ids:
+            raise HTTPException(
+                status_code=400, detail="Source is not associated with any notebooks"
+            )
+
+        # Prepare content_state based on source asset
+        content_state = {}
+        if source.asset:
+            if source.asset.file_path:
+                content_state = {
+                    "file_path": source.asset.file_path,
+                    "delete_source": False,  # Don't delete on retry
+                }
+            elif source.asset.url:
+                content_state = {"url": source.asset.url}
+            else:
+                raise HTTPException(
+                    status_code=400, detail="Source asset has no file_path or url"
+                )
+        else:
+            # Check if it's a text source by trying to get full_text
+            if source.full_text:
+                content_state = {"content": source.full_text}
+            else:
+                raise HTTPException(
+                    status_code=400, detail="Cannot determine source content for retry"
+                )
+
+        try:
+            # Import command modules to ensure they're registered
+            import commands.source_commands  # noqa: F401
+
+            # Submit new command for background processing
+            command_input = SourceProcessingInput(
+                source_id=str(source.id),
+                content_state=content_state,
+                notebook_ids=notebook_ids,
+                transformations=[],  # Use default transformations on retry
+                embed=True,  # Always embed on retry
+            )
+
+            command_id = await CommandService.submit_command_job(
+                "open_notebook",  # app name
+                "process_source",  # command name
+                command_input.model_dump(),
+            )
+
+            logger.info(
+                f"Submitted retry processing command: {command_id} for source {source_id}"
+            )
+
+            # Update source with new command ID
+            source.command = ensure_record_id(f"command:{command_id}")
+            await source.save()
+
+            # Get current embedded chunks count
+            embedded_chunks = await source.get_embedded_chunks()
+
+            # Return updated source response
+            return SourceResponse(
+                id=source.id or "",
+                title=source.title,
+                topics=source.topics or [],
+                asset=AssetModel(
+                    file_path=source.asset.file_path if source.asset else None,
+                    url=source.asset.url if source.asset else None,
+                )
+                if source.asset
+                else None,
+                full_text=source.full_text,
+                embedded=embedded_chunks > 0,
+                embedded_chunks=embedded_chunks,
+                created=str(source.created),
+                updated=str(source.updated),
+                command_id=command_id,
+                status="queued",
+                processing_info={"retry": True, "queued": True},
+            )
+
+        except Exception as e:
+            logger.error(
+                f"Failed to submit retry processing command for source {source_id}: {e}"
+            )
+            raise HTTPException(
+                status_code=500, detail=f"Failed to queue retry processing: {str(e)}"
+            )
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error retrying source processing for {source_id}: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error retrying source processing: {str(e)}"
+        )
+
+
+@router.delete("/sources/{source_id}")
+async def delete_source(source_id: str):
+    """Delete a source."""
+    try:
+        source = await Source.get(source_id)
+        if not source:
+            raise HTTPException(status_code=404, detail="Source not found")
+
+        await source.delete()
+
+        return {"message": "Source deleted successfully"}
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error deleting source {source_id}: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error deleting source: {str(e)}")
+
+
+@router.get("/sources/{source_id}/insights", response_model=List[SourceInsightResponse])
+async def get_source_insights(source_id: str):
+    """Get all insights for a specific source."""
+    try:
+        source = await Source.get(source_id)
+        if not source:
+            raise HTTPException(status_code=404, detail="Source not found")
+
+        insights = await source.get_insights()
+        return [
+            SourceInsightResponse(
+                id=insight.id or "",
+                source_id=source_id,
+                insight_type=insight.insight_type,
+                content=insight.content,
+                created=str(insight.created),
+                updated=str(insight.updated),
+            )
+            for insight in insights
+        ]
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error fetching insights for source {source_id}: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error fetching insights: {str(e)}"
+        )
+
+
+@router.post(
+    "/sources/{source_id}/insights",
+    response_model=InsightCreationResponse,
+    status_code=202,
+)
+async def create_source_insight(source_id: str, request: CreateSourceInsightRequest):
+    """
+    Start insight generation for a source by running a transformation.
+
+    This endpoint returns immediately with a 202 Accepted status.
+    The transformation runs asynchronously in the background via the job queue.
+    Poll GET /sources/{source_id}/insights to see when the insight is ready.
+    """
+    try:
+        # Validate source exists
+        source = await Source.get(source_id)
+        if not source:
+            raise HTTPException(status_code=404, detail="Source not found")
+
+        # Validate transformation exists
+        transformation = await Transformation.get(request.transformation_id)
+        if not transformation:
+            raise HTTPException(status_code=404, detail="Transformation not found")
+
+        # Submit transformation as background job (fire-and-forget)
+        command_id = submit_command(
+            "open_notebook",
+            "run_transformation",
+            {
+                "source_id": source_id,
+                "transformation_id": request.transformation_id,
+            },
+        )
+        logger.info(
+            f"Submitted run_transformation command {command_id} for source {source_id}"
+        )
+
+        # Return immediately with command_id for status tracking
+        return InsightCreationResponse(
+            status="pending",
+            message="Insight generation started",
+            source_id=source_id,
+            transformation_id=request.transformation_id,
+            command_id=str(command_id),
+        )
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error starting insight generation for source {source_id}: {e}")
+        raise HTTPException(
+            status_code=500, detail=f"Error starting insight generation: {str(e)}"
+        )

api/routers/speaker_profiles.py

@@ -0,0 +1,190 @@
+from typing import Any, Dict, List, Optional
+
+from fastapi import APIRouter, HTTPException
+from loguru import logger
+from pydantic import BaseModel, Field
+
+from open_notebook.podcasts.models import SpeakerProfile
+
+router = APIRouter()
+
+
+class SpeakerProfileResponse(BaseModel):
+    id: str
+    name: str
+    description: str
+    voice_model: Optional[str] = None
+    speakers: List[Dict[str, Any]]
+    # Legacy fields (for display/migration awareness)
+    tts_provider: Optional[str] = None
+    tts_model: Optional[str] = None
+
+
+def _profile_to_response(profile: SpeakerProfile) -> SpeakerProfileResponse:
+    return SpeakerProfileResponse(
+        id=str(profile.id),
+        name=profile.name,
+        description=profile.description or "",
+        voice_model=profile.voice_model,
+        speakers=profile.speakers,
+        tts_provider=profile.tts_provider,
+        tts_model=profile.tts_model,
+    )
+
+
+@router.get("/speaker-profiles", response_model=List[SpeakerProfileResponse])
+async def list_speaker_profiles():
+    """List all available speaker profiles"""
+    try:
+        profiles = await SpeakerProfile.get_all(order_by="name asc")
+        return [_profile_to_response(p) for p in profiles]
+    except Exception as e:
+        logger.error(f"Failed to fetch speaker profiles: {e}")
+        raise HTTPException(
+            status_code=500, detail="Failed to fetch speaker profiles"
+        )
+
+
+@router.get("/speaker-profiles/{profile_name}", response_model=SpeakerProfileResponse)
+async def get_speaker_profile(profile_name: str):
+    """Get a specific speaker profile by name"""
+    try:
+        profile = await SpeakerProfile.get_by_name(profile_name)
+
+        if not profile:
+            raise HTTPException(
+                status_code=404, detail=f"Speaker profile '{profile_name}' not found"
+            )
+
+        return _profile_to_response(profile)
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Failed to fetch speaker profile '{profile_name}': {e}")
+        raise HTTPException(
+            status_code=500, detail="Failed to fetch speaker profile"
+        )
+
+
+class SpeakerProfileCreate(BaseModel):
+    name: str = Field(..., description="Unique profile name")
+    description: str = Field("", description="Profile description")
+    voice_model: Optional[str] = Field(None, description="Model record ID for TTS")
+    speakers: List[Dict[str, Any]] = Field(
+        ..., description="Array of speaker configurations"
+    )
+    # Legacy fields (accepted but not required)
+    tts_provider: Optional[str] = None
+    tts_model: Optional[str] = None
+
+
+@router.post("/speaker-profiles", response_model=SpeakerProfileResponse)
+async def create_speaker_profile(profile_data: SpeakerProfileCreate):
+    """Create a new speaker profile"""
+    try:
+        profile = SpeakerProfile(
+            name=profile_data.name,
+            description=profile_data.description,
+            voice_model=profile_data.voice_model,
+            speakers=profile_data.speakers,
+            tts_provider=profile_data.tts_provider,
+            tts_model=profile_data.tts_model,
+        )
+
+        await profile.save()
+        return _profile_to_response(profile)
+
+    except Exception as e:
+        logger.error(f"Failed to create speaker profile: {e}")
+        raise HTTPException(
+            status_code=500, detail="Failed to create speaker profile"
+        )
+
+
+@router.put("/speaker-profiles/{profile_id}", response_model=SpeakerProfileResponse)
+async def update_speaker_profile(profile_id: str, profile_data: SpeakerProfileCreate):
+    """Update an existing speaker profile"""
+    try:
+        profile = await SpeakerProfile.get(profile_id)
+
+        if not profile:
+            raise HTTPException(
+                status_code=404, detail=f"Speaker profile '{profile_id}' not found"
+            )
+
+        profile.name = profile_data.name
+        profile.description = profile_data.description
+        profile.voice_model = profile_data.voice_model
+        profile.speakers = profile_data.speakers
+        profile.tts_provider = profile_data.tts_provider
+        profile.tts_model = profile_data.tts_model
+
+        await profile.save()
+        return _profile_to_response(profile)
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Failed to update speaker profile: {e}")
+        raise HTTPException(
+            status_code=500, detail="Failed to update speaker profile"
+        )
+
+
+@router.delete("/speaker-profiles/{profile_id}")
+async def delete_speaker_profile(profile_id: str):
+    """Delete a speaker profile"""
+    try:
+        profile = await SpeakerProfile.get(profile_id)
+
+        if not profile:
+            raise HTTPException(
+                status_code=404, detail=f"Speaker profile '{profile_id}' not found"
+            )
+
+        await profile.delete()
+
+        return {"message": "Speaker profile deleted successfully"}
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Failed to delete speaker profile: {e}")
+        raise HTTPException(
+            status_code=500, detail="Failed to delete speaker profile"
+        )
+
+
+@router.post(
+    "/speaker-profiles/{profile_id}/duplicate", response_model=SpeakerProfileResponse
+)
+async def duplicate_speaker_profile(profile_id: str):
+    """Duplicate a speaker profile"""
+    try:
+        original = await SpeakerProfile.get(profile_id)
+
+        if not original:
+            raise HTTPException(
+                status_code=404, detail=f"Speaker profile '{profile_id}' not found"
+            )
+
+        duplicate = SpeakerProfile(
+            name=f"{original.name} - Copy",
+            description=original.description,
+            voice_model=original.voice_model,
+            speakers=original.speakers,
+            tts_provider=original.tts_provider,
+            tts_model=original.tts_model,
+        )
+
+        await duplicate.save()
+        return _profile_to_response(duplicate)
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Failed to duplicate speaker profile: {e}")
+        raise HTTPException(
+            status_code=500, detail="Failed to duplicate speaker profile"
+        )

api/routers/transformations.py

@@ -0,0 +1,252 @@
+from typing import List
+
+from fastapi import APIRouter, HTTPException
+from loguru import logger
+
+from api.models import (
+    DefaultPromptResponse,
+    DefaultPromptUpdate,
+    TransformationCreate,
+    TransformationExecuteRequest,
+    TransformationExecuteResponse,
+    TransformationResponse,
+    TransformationUpdate,
+)
+from open_notebook.ai.models import Model
+from open_notebook.domain.transformation import DefaultPrompts, Transformation
+from open_notebook.exceptions import InvalidInputError, OpenNotebookError
+from open_notebook.graphs.transformation import graph as transformation_graph
+
+router = APIRouter()
+
+
+@router.get("/transformations", response_model=List[TransformationResponse])
+async def get_transformations():
+    """Get all transformations."""
+    try:
+        transformations = await Transformation.get_all(order_by="name asc")
+
+        return [
+            TransformationResponse(
+                id=transformation.id or "",
+                name=transformation.name,
+                title=transformation.title,
+                description=transformation.description,
+                prompt=transformation.prompt,
+                apply_default=transformation.apply_default,
+                created=str(transformation.created),
+                updated=str(transformation.updated),
+            )
+            for transformation in transformations
+        ]
+    except Exception as e:
+        logger.error(f"Error fetching transformations: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error fetching transformations: {str(e)}"
+        )
+
+
+@router.post("/transformations", response_model=TransformationResponse)
+async def create_transformation(transformation_data: TransformationCreate):
+    """Create a new transformation."""
+    try:
+        new_transformation = Transformation(
+            name=transformation_data.name,
+            title=transformation_data.title,
+            description=transformation_data.description,
+            prompt=transformation_data.prompt,
+            apply_default=transformation_data.apply_default,
+        )
+        await new_transformation.save()
+
+        return TransformationResponse(
+            id=new_transformation.id or "",
+            name=new_transformation.name,
+            title=new_transformation.title,
+            description=new_transformation.description,
+            prompt=new_transformation.prompt,
+            apply_default=new_transformation.apply_default,
+            created=str(new_transformation.created),
+            updated=str(new_transformation.updated),
+        )
+    except InvalidInputError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        logger.error(f"Error creating transformation: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error creating transformation: {str(e)}"
+        )
+
+
+@router.post("/transformations/execute", response_model=TransformationExecuteResponse)
+async def execute_transformation(execute_request: TransformationExecuteRequest):
+    """Execute a transformation on input text."""
+    try:
+        # Validate transformation exists
+        transformation = await Transformation.get(execute_request.transformation_id)
+        if not transformation:
+            raise HTTPException(status_code=404, detail="Transformation not found")
+
+        # Validate model exists
+        model = await Model.get(execute_request.model_id)
+        if not model:
+            raise HTTPException(status_code=404, detail="Model not found")
+
+        # Execute the transformation
+        result = await transformation_graph.ainvoke(
+            dict(  # type: ignore[arg-type]
+                input_text=execute_request.input_text,
+                transformation=transformation,
+            ),
+            config=dict(configurable={"model_id": execute_request.model_id}),
+        )
+
+        return TransformationExecuteResponse(
+            output=result["output"],
+            transformation_id=execute_request.transformation_id,
+            model_id=execute_request.model_id,
+        )
+
+    except HTTPException:
+        raise
+    except OpenNotebookError:
+        raise  # Let global exception handlers return proper status codes
+    except Exception as e:
+        logger.error(f"Error executing transformation: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error executing transformation: {str(e)}"
+        )
+
+
+@router.get("/transformations/default-prompt", response_model=DefaultPromptResponse)
+async def get_default_prompt():
+    """Get the default transformation prompt."""
+    try:
+        default_prompts: DefaultPrompts = await DefaultPrompts.get_instance()  # type: ignore[assignment]
+
+        return DefaultPromptResponse(
+            transformation_instructions=default_prompts.transformation_instructions
+            or ""
+        )
+    except Exception as e:
+        logger.error(f"Error fetching default prompt: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error fetching default prompt: {str(e)}"
+        )
+
+
+@router.put("/transformations/default-prompt", response_model=DefaultPromptResponse)
+async def update_default_prompt(prompt_update: DefaultPromptUpdate):
+    """Update the default transformation prompt."""
+    try:
+        default_prompts: DefaultPrompts = await DefaultPrompts.get_instance()  # type: ignore[assignment]
+
+        default_prompts.transformation_instructions = (
+            prompt_update.transformation_instructions
+        )
+        await default_prompts.update()
+
+        return DefaultPromptResponse(
+            transformation_instructions=default_prompts.transformation_instructions
+        )
+    except Exception as e:
+        logger.error(f"Error updating default prompt: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error updating default prompt: {str(e)}"
+        )
+
+
+@router.get(
+    "/transformations/{transformation_id}", response_model=TransformationResponse
+)
+async def get_transformation(transformation_id: str):
+    """Get a specific transformation by ID."""
+    try:
+        transformation = await Transformation.get(transformation_id)
+        if not transformation:
+            raise HTTPException(status_code=404, detail="Transformation not found")
+
+        return TransformationResponse(
+            id=transformation.id or "",
+            name=transformation.name,
+            title=transformation.title,
+            description=transformation.description,
+            prompt=transformation.prompt,
+            apply_default=transformation.apply_default,
+            created=str(transformation.created),
+            updated=str(transformation.updated),
+        )
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error fetching transformation {transformation_id}: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error fetching transformation: {str(e)}"
+        )
+
+
+@router.put(
+    "/transformations/{transformation_id}", response_model=TransformationResponse
+)
+async def update_transformation(
+    transformation_id: str, transformation_update: TransformationUpdate
+):
+    """Update a transformation."""
+    try:
+        transformation = await Transformation.get(transformation_id)
+        if not transformation:
+            raise HTTPException(status_code=404, detail="Transformation not found")
+
+        # Update only provided fields
+        if transformation_update.name is not None:
+            transformation.name = transformation_update.name
+        if transformation_update.title is not None:
+            transformation.title = transformation_update.title
+        if transformation_update.description is not None:
+            transformation.description = transformation_update.description
+        if transformation_update.prompt is not None:
+            transformation.prompt = transformation_update.prompt
+        if transformation_update.apply_default is not None:
+            transformation.apply_default = transformation_update.apply_default
+
+        await transformation.save()
+
+        return TransformationResponse(
+            id=transformation.id or "",
+            name=transformation.name,
+            title=transformation.title,
+            description=transformation.description,
+            prompt=transformation.prompt,
+            apply_default=transformation.apply_default,
+            created=str(transformation.created),
+            updated=str(transformation.updated),
+        )
+    except HTTPException:
+        raise
+    except InvalidInputError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        logger.error(f"Error updating transformation {transformation_id}: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error updating transformation: {str(e)}"
+        )
+
+
+@router.delete("/transformations/{transformation_id}")
+async def delete_transformation(transformation_id: str):
+    """Delete a transformation."""
+    try:
+        transformation = await Transformation.get(transformation_id)
+        if not transformation:
+            raise HTTPException(status_code=404, detail="Transformation not found")
+
+        await transformation.delete()
+
+        return {"message": "Transformation deleted successfully"}
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error deleting transformation {transformation_id}: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error deleting transformation: {str(e)}"
+        )

api/search_service.py

@@ -0,0 +1,58 @@
+"""
+Search service layer using API.
+"""
+
+from typing import Any, Dict, List, Union
+
+from loguru import logger
+
+from api.client import api_client
+
+
+class SearchService:
+    """Service layer for search operations using API."""
+
+    def __init__(self):
+        logger.info("Using API for search operations")
+
+    def search(
+        self,
+        query: str,
+        search_type: str = "text",
+        limit: int = 100,
+        search_sources: bool = True,
+        search_notes: bool = True,
+        minimum_score: float = 0.2,
+    ) -> List[Dict[str, Any]]:
+        """Search the knowledge base."""
+        response = api_client.search(
+            query=query,
+            search_type=search_type,
+            limit=limit,
+            search_sources=search_sources,
+            search_notes=search_notes,
+            minimum_score=minimum_score,
+        )
+        if isinstance(response, dict):
+            return response.get("results", [])
+        return []
+
+    def ask_knowledge_base(
+        self,
+        question: str,
+        strategy_model: str,
+        answer_model: str,
+        final_answer_model: str,
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Ask the knowledge base a question."""
+        response = api_client.ask_simple(
+            question=question,
+            strategy_model=strategy_model,
+            answer_model=answer_model,
+            final_answer_model=final_answer_model,
+        )
+        return response
+
+
+# Global service instance
+search_service = SearchService()

api/settings_service.py

@@ -0,0 +1,79 @@
+"""
+Settings service layer using API.
+"""
+
+from loguru import logger
+
+from api.client import api_client
+from open_notebook.domain.content_settings import ContentSettings
+
+
+class SettingsService:
+    """Service layer for settings operations using API."""
+
+    def __init__(self):
+        logger.info("Using API for settings operations")
+
+    def get_settings(self) -> ContentSettings:
+        """Get application settings."""
+        settings_response = api_client.get_settings()
+        settings_data = (
+            settings_response
+            if isinstance(settings_response, dict)
+            else settings_response[0]
+        )
+
+        # Create ContentSettings object from API response
+        settings = ContentSettings(
+            default_content_processing_engine_doc=settings_data.get(
+                "default_content_processing_engine_doc"
+            ),
+            default_content_processing_engine_url=settings_data.get(
+                "default_content_processing_engine_url"
+            ),
+            default_embedding_option=settings_data.get("default_embedding_option"),
+            auto_delete_files=settings_data.get("auto_delete_files"),
+            youtube_preferred_languages=settings_data.get(
+                "youtube_preferred_languages"
+            ),
+        )
+
+        return settings
+
+    def update_settings(self, settings: ContentSettings) -> ContentSettings:
+        """Update application settings."""
+        updates = {
+            "default_content_processing_engine_doc": settings.default_content_processing_engine_doc,
+            "default_content_processing_engine_url": settings.default_content_processing_engine_url,
+            "default_embedding_option": settings.default_embedding_option,
+            "auto_delete_files": settings.auto_delete_files,
+            "youtube_preferred_languages": settings.youtube_preferred_languages,
+        }
+
+        settings_response = api_client.update_settings(**updates)
+        settings_data = (
+            settings_response
+            if isinstance(settings_response, dict)
+            else settings_response[0]
+        )
+
+        # Update the settings object with the response
+        settings.default_content_processing_engine_doc = settings_data.get(
+            "default_content_processing_engine_doc"
+        )
+        settings.default_content_processing_engine_url = settings_data.get(
+            "default_content_processing_engine_url"
+        )
+        settings.default_embedding_option = settings_data.get(
+            "default_embedding_option"
+        )
+        settings.auto_delete_files = settings_data.get("auto_delete_files")
+        settings.youtube_preferred_languages = settings_data.get(
+            "youtube_preferred_languages"
+        )
+
+        return settings
+
+
+# Global service instance
+settings_service = SettingsService()

api/sources_service.py

@@ -0,0 +1,324 @@
+"""
+Sources service layer using API.
+"""
+
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Union
+
+from loguru import logger
+
+from api.client import api_client
+from open_notebook.domain.notebook import Asset, Source
+
+
+@dataclass
+class SourceProcessingResult:
+    """Result of source creation with optional async processing info."""
+
+    source: Source
+    is_async: bool = False
+    command_id: Optional[str] = None
+    status: Optional[str] = None
+    processing_info: Optional[Dict] = None
+
+
+@dataclass
+class SourceWithMetadata:
+    """Source object with additional metadata from API."""
+
+    source: Source
+    embedded_chunks: int
+
+    # Expose common source properties for easy access
+    @property
+    def id(self):
+        return self.source.id
+
+    @property
+    def title(self):
+        return self.source.title
+
+    @title.setter
+    def title(self, value):
+        self.source.title = value
+
+    @property
+    def topics(self):
+        return self.source.topics
+
+    @property
+    def asset(self):
+        return self.source.asset
+
+    @property
+    def full_text(self):
+        return self.source.full_text
+
+    @property
+    def created(self):
+        return self.source.created
+
+    @property
+    def updated(self):
+        return self.source.updated
+
+
+class SourcesService:
+    """Service layer for sources operations using API."""
+
+    def __init__(self):
+        logger.info("Using API for sources operations")
+
+    def get_all_sources(
+        self, notebook_id: Optional[str] = None
+    ) -> List[SourceWithMetadata]:
+        """Get all sources with optional notebook filtering."""
+        sources_data = api_client.get_sources(notebook_id=notebook_id)
+        # Convert API response to SourceWithMetadata objects
+        sources = []
+        for source_data in sources_data:
+            source = Source(
+                title=source_data["title"],
+                topics=source_data["topics"],
+                asset=Asset(
+                    file_path=source_data["asset"]["file_path"]
+                    if source_data["asset"]
+                    else None,
+                    url=source_data["asset"]["url"] if source_data["asset"] else None,
+                )
+                if source_data["asset"]
+                else None,
+            )
+            source.id = source_data["id"]
+            source.created = source_data["created"]
+            source.updated = source_data["updated"]
+
+            # Wrap in SourceWithMetadata
+            source_with_metadata = SourceWithMetadata(
+                source=source, embedded_chunks=source_data.get("embedded_chunks", 0)
+            )
+            sources.append(source_with_metadata)
+        return sources
+
+    def get_source(self, source_id: str) -> SourceWithMetadata:
+        """Get a specific source."""
+        response = api_client.get_source(source_id)
+        source_data = response if isinstance(response, dict) else response[0]
+        source = Source(
+            title=source_data["title"],
+            topics=source_data["topics"],
+            full_text=source_data["full_text"],
+            asset=Asset(
+                file_path=source_data["asset"]["file_path"]
+                if source_data["asset"]
+                else None,
+                url=source_data["asset"]["url"] if source_data["asset"] else None,
+            )
+            if source_data["asset"]
+            else None,
+        )
+        source.id = source_data["id"]
+        source.created = source_data["created"]
+        source.updated = source_data["updated"]
+
+        return SourceWithMetadata(
+            source=source, embedded_chunks=source_data.get("embedded_chunks", 0)
+        )
+
+    def create_source(
+        self,
+        notebook_id: Optional[str] = None,
+        source_type: str = "text",
+        url: Optional[str] = None,
+        file_path: Optional[str] = None,
+        content: Optional[str] = None,
+        title: Optional[str] = None,
+        transformations: Optional[List[str]] = None,
+        embed: bool = False,
+        delete_source: bool = False,
+        notebooks: Optional[List[str]] = None,
+        async_processing: bool = False,
+    ) -> Union[Source, SourceProcessingResult]:
+        """
+        Create a new source with support for async processing.
+
+        Args:
+            notebook_id: Single notebook ID (deprecated, use notebooks parameter)
+            source_type: Type of source (link, upload, text)
+            url: URL for link sources
+            file_path: File path for upload sources
+            content: Text content for text sources
+            title: Optional source title
+            transformations: List of transformation IDs to apply
+            embed: Whether to embed content for vector search
+            delete_source: Whether to delete uploaded file after processing
+            notebooks: List of notebook IDs to add source to (preferred over notebook_id)
+            async_processing: Whether to process source asynchronously
+
+        Returns:
+            Source object for sync processing (backward compatibility)
+            SourceProcessingResult for async processing (contains additional metadata)
+        """
+        source_data = api_client.create_source(
+            notebook_id=notebook_id,
+            notebooks=notebooks,
+            source_type=source_type,
+            url=url,
+            file_path=file_path,
+            content=content,
+            title=title,
+            transformations=transformations,
+            embed=embed,
+            delete_source=delete_source,
+            async_processing=async_processing,
+        )
+
+        # Create Source object from response
+        response_data = source_data if isinstance(source_data, dict) else source_data[0]
+        source = Source(
+            title=response_data["title"],
+            topics=response_data.get("topics") or [],
+            full_text=response_data.get("full_text"),
+            asset=Asset(
+                file_path=response_data["asset"]["file_path"]
+                if response_data.get("asset")
+                else None,
+                url=response_data["asset"]["url"]
+                if response_data.get("asset")
+                else None,
+            )
+            if response_data.get("asset")
+            else None,
+        )
+        source.id = response_data["id"]
+        source.created = response_data["created"]
+        source.updated = response_data["updated"]
+
+        # Check if this is an async processing response
+        if (
+            response_data.get("command_id")
+            or response_data.get("status")
+            or response_data.get("processing_info")
+        ):
+            # Ensure source_data is a dict for accessing attributes
+            source_data_dict = (
+                source_data if isinstance(source_data, dict) else source_data[0]
+            )
+            # Return enhanced result for async processing
+            return SourceProcessingResult(
+                source=source,
+                is_async=True,
+                command_id=source_data_dict.get("command_id"),
+                status=source_data_dict.get("status"),
+                processing_info=source_data_dict.get("processing_info"),
+            )
+        else:
+            # Return simple Source for backward compatibility
+            return source
+
+    def get_source_status(self, source_id: str) -> Dict:
+        """Get processing status for a source."""
+        response = api_client.get_source_status(source_id)
+        return response if isinstance(response, dict) else response[0]
+
+    def create_source_async(
+        self,
+        notebook_id: Optional[str] = None,
+        source_type: str = "text",
+        url: Optional[str] = None,
+        file_path: Optional[str] = None,
+        content: Optional[str] = None,
+        title: Optional[str] = None,
+        transformations: Optional[List[str]] = None,
+        embed: bool = False,
+        delete_source: bool = False,
+        notebooks: Optional[List[str]] = None,
+    ) -> SourceProcessingResult:
+        """
+        Create a new source with async processing enabled.
+
+        This is a convenience method that always uses async processing.
+        Returns a SourceProcessingResult with processing status information.
+        """
+        result = self.create_source(
+            notebook_id=notebook_id,
+            notebooks=notebooks,
+            source_type=source_type,
+            url=url,
+            file_path=file_path,
+            content=content,
+            title=title,
+            transformations=transformations,
+            embed=embed,
+            delete_source=delete_source,
+            async_processing=True,
+        )
+
+        # Since we forced async_processing=True, this should always be a SourceProcessingResult
+        if isinstance(result, SourceProcessingResult):
+            return result
+        else:
+            # Fallback: wrap Source in SourceProcessingResult
+            return SourceProcessingResult(
+                source=result,
+                is_async=False,  # This shouldn't happen, but handle it gracefully
+            )
+
+    def is_source_processing_complete(self, source_id: str) -> bool:
+        """
+        Check if a source's async processing is complete.
+
+        Returns True if processing is complete (success or failure),
+        False if still processing or queued.
+        """
+        try:
+            status_data = self.get_source_status(source_id)
+            status = status_data.get("status")
+            return status in [
+                "completed",
+                "failed",
+                None,
+            ]  # None indicates legacy/sync source
+        except Exception as e:
+            logger.error(f"Error checking source processing status: {e}")
+            return True  # Assume complete on error
+
+    def update_source(self, source: Source) -> Source:
+        """Update a source."""
+        if not source.id:
+            raise ValueError("Source ID is required for update")
+
+        updates = {
+            "title": source.title,
+            "topics": source.topics,
+        }
+        source_data = api_client.update_source(source.id, **updates)
+
+        # Ensure source_data is a dict
+        source_data_dict = (
+            source_data if isinstance(source_data, dict) else source_data[0]
+        )
+
+        # Update the source object with the response
+        source.title = source_data_dict["title"]
+        source.topics = source_data_dict["topics"]
+        source.updated = source_data_dict["updated"]
+
+        return source
+
+    def delete_source(self, source_id: str) -> bool:
+        """Delete a source."""
+        api_client.delete_source(source_id)
+        return True
+
+
+# Global service instance
+sources_service = SourcesService()
+
+# Export important classes for easy importing
+__all__ = [
+    "SourcesService",
+    "SourceWithMetadata",
+    "SourceProcessingResult",
+    "sources_service",
+]

api/transformations_service.py

@@ -0,0 +1,141 @@
+"""
+Transformations service layer using API.
+"""
+
+from datetime import datetime
+from typing import Any, Dict, List, Union
+
+from loguru import logger
+
+from api.client import api_client
+from open_notebook.domain.transformation import Transformation
+
+
+class TransformationsService:
+    """Service layer for transformations operations using API."""
+
+    def __init__(self):
+        logger.info("Using API for transformations operations")
+
+    def get_all_transformations(self) -> List[Transformation]:
+        """Get all transformations."""
+        transformations_data = api_client.get_transformations()
+        # Convert API response to Transformation objects
+        transformations = []
+        for trans_data in transformations_data:
+            transformation = Transformation(
+                name=trans_data["name"],
+                title=trans_data["title"],
+                description=trans_data["description"],
+                prompt=trans_data["prompt"],
+                apply_default=trans_data["apply_default"],
+            )
+            transformation.id = trans_data["id"]
+            transformation.created = datetime.fromisoformat(
+                trans_data["created"].replace("Z", "+00:00")
+            )
+            transformation.updated = datetime.fromisoformat(
+                trans_data["updated"].replace("Z", "+00:00")
+            )
+            transformations.append(transformation)
+        return transformations
+
+    def get_transformation(self, transformation_id: str) -> Transformation:
+        """Get a specific transformation."""
+        response = api_client.get_transformation(transformation_id)
+        trans_data = response if isinstance(response, dict) else response[0]
+        transformation = Transformation(
+            name=trans_data["name"],
+            title=trans_data["title"],
+            description=trans_data["description"],
+            prompt=trans_data["prompt"],
+            apply_default=trans_data["apply_default"],
+        )
+        transformation.id = trans_data["id"]
+        transformation.created = datetime.fromisoformat(
+            trans_data["created"].replace("Z", "+00:00")
+        )
+        transformation.updated = datetime.fromisoformat(
+            trans_data["updated"].replace("Z", "+00:00")
+        )
+        return transformation
+
+    def create_transformation(
+        self,
+        name: str,
+        title: str,
+        description: str,
+        prompt: str,
+        apply_default: bool = False,
+    ) -> Transformation:
+        """Create a new transformation."""
+        response = api_client.create_transformation(
+            name=name,
+            title=title,
+            description=description,
+            prompt=prompt,
+            apply_default=apply_default,
+        )
+        trans_data = response if isinstance(response, dict) else response[0]
+        transformation = Transformation(
+            name=trans_data["name"],
+            title=trans_data["title"],
+            description=trans_data["description"],
+            prompt=trans_data["prompt"],
+            apply_default=trans_data["apply_default"],
+        )
+        transformation.id = trans_data["id"]
+        transformation.created = datetime.fromisoformat(
+            trans_data["created"].replace("Z", "+00:00")
+        )
+        transformation.updated = datetime.fromisoformat(
+            trans_data["updated"].replace("Z", "+00:00")
+        )
+        return transformation
+
+    def update_transformation(self, transformation: Transformation) -> Transformation:
+        """Update a transformation."""
+        if not transformation.id:
+            raise ValueError("Transformation ID is required for update")
+
+        updates = {
+            "name": transformation.name,
+            "title": transformation.title,
+            "description": transformation.description,
+            "prompt": transformation.prompt,
+            "apply_default": transformation.apply_default,
+        }
+        response = api_client.update_transformation(transformation.id, **updates)
+        trans_data = response if isinstance(response, dict) else response[0]
+
+        # Update the transformation object with the response
+        transformation.name = trans_data["name"]
+        transformation.title = trans_data["title"]
+        transformation.description = trans_data["description"]
+        transformation.prompt = trans_data["prompt"]
+        transformation.apply_default = trans_data["apply_default"]
+        transformation.updated = datetime.fromisoformat(
+            trans_data["updated"].replace("Z", "+00:00")
+        )
+
+        return transformation
+
+    def delete_transformation(self, transformation_id: str) -> bool:
+        """Delete a transformation."""
+        api_client.delete_transformation(transformation_id)
+        return True
+
+    def execute_transformation(
+        self, transformation_id: str, input_text: str, model_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Execute a transformation on input text."""
+        result = api_client.execute_transformation(
+            transformation_id=transformation_id,
+            input_text=input_text,
+            model_id=model_id,
+        )
+        return result
+
+
+# Global service instance
+transformations_service = TransformationsService()

commands/CLAUDE.md

@@ -0,0 +1,68 @@
+# Commands Module
+
+**Purpose**: Defines async command handlers for long-running operations via `surreal-commands` job queue system.
+
+## Key Components
+
+### Embedding Commands
+
+- **`embed_note_command`**: Embeds a single note using unified embedding pipeline with content-type aware processing. Uses MARKDOWN content type detection. Retry: 5 attempts, exponential jitter 1-60s.
+- **`embed_insight_command`**: Embeds a single source insight. Uses MARKDOWN content type. Retry: 5 attempts, exponential jitter 1-60s.
+- **`embed_source_command`**: Embeds a source by chunking full_text with content-type aware splitters (HTML, Markdown, plain), then batch embedding all chunks (batches of 50 with per-batch retry). Retry: 5 attempts, exponential jitter 1-60s.
+- **`create_insight_command`**: Creates a source insight with automatic retry on transaction conflicts. Creates the DB record, then submits `embed_insight` command (fire-and-forget). Retry: 5 attempts, exponential jitter 1-60s. Used by `Source.add_insight()`.
+- **`rebuild_embeddings_command`**: Submits individual embed_* commands for all sources/notes/insights. Returns immediately; actual embedding happens async. No retry (coordinator only).
+
+### Other Commands
+
+- **`process_source_command`**: Ingests content through `source_graph`, creates embeddings (optional), and generates insights. Retries on transaction conflicts (exp. jitter, max 15×, 1-120s).
+- **`run_transformation_command`**: Runs a transformation on an existing source to generate an insight. Executes the transformation graph (LLM call) then creates insight via `create_insight_command`. Used by `POST /sources/{id}/insights` API endpoint. Retry: 5 attempts, exponential jitter 1-60s.
+- **`generate_podcast_command`**: Creates podcasts via podcast-creator library. Resolves model registry references and credentials for all profiles before invoking podcast-creator. Validates that outline_llm, transcript_llm, and voice_model are configured.
+- **`process_text_command`** (example): Test fixture for text operations (uppercase, lowercase, reverse, word_count).
+- **`analyze_data_command`** (example): Test fixture for numeric aggregations.
+
+## Important Patterns
+
+- **Pydantic I/O**: All commands use `CommandInput`/`CommandOutput` subclasses for type safety and serialization.
+- **Error handling**: Permanent errors (ValueError) return failure output; all other exceptions auto-retry via surreal-commands.
+- **Retry configuration**: Uses `stop_on: [ValueError]` (blocklist approach) - retries all exceptions EXCEPT ValueError. This is more resilient than allowlist as new exception types auto-retry.
+- **Fire-and-forget embedding**: Domain models submit embed_* commands via `submit_command()` without waiting. Commands process asynchronously.
+- **Content-type aware chunking**: `embed_source_command` uses `chunk_text()` with automatic content type detection (HTML, Markdown, plain text) for optimal text splitting. Default: 1500 char chunks with 225 char overlap.
+- **Batch embedding**: `embed_source_command` uses `generate_embeddings()` which automatically batches texts (default 50) with per-batch retry to avoid exceeding provider payload limits.
+- **Mean pooling for large content**: `embed_note_command` and `embed_insight_command` use `generate_embedding()` which handles content larger than chunk size via mean pooling.
+- **Model dumping**: Recursive `full_model_dump()` utility converts Pydantic models → dicts for DB/API responses.
+- **Logging**: Uses `loguru.logger` throughout; logs execution start/end and key metrics (processing time, counts).
+- **Time tracking**: All commands measure `start_time` → `processing_time` for monitoring.
+
+## Dependencies
+
+**External**: `surreal_commands` (command decorator, job queue, submit_command), `loguru`, `pydantic`, `podcast_creator`
+**Internal**: `open_notebook.domain.notebook` (Source, Note, SourceInsight), `open_notebook.utils.chunking` (chunk_text, detect_content_type), `open_notebook.utils.embedding` (generate_embedding, generate_embeddings), `open_notebook.database.repository` (repo_query, repo_insert)
+
+## Quirks & Edge Cases
+
+- **source_commands**: `ensure_record_id()` wraps command IDs for DB storage; transaction conflicts trigger exponential backoff retry. ValueError exceptions are permanent (not retried).
+- **embedding_commands**: Content type detection uses file extension as primary source, heuristics as fallback. Chunks >1800 chars trigger secondary splitting. Empty/whitespace-only content returns ValueError (not retried).
+- **rebuild_embeddings_command**: Returns "jobs_submitted" not "processed_items" - embedding is async. Individual commands handle failures with their own retries.
+- **podcast_commands**: Profiles loaded from SurrealDB by name; model configs (credentials) resolved for ALL profiles before podcast-creator validation. Validates outline_llm/transcript_llm/voice_model are set. Episode records created mid-execution.
+- **Example commands**: Accept optional `delay_seconds` for testing async behavior; not for production.
+
+## Code Example
+
+```python
+@command("process_source", app="open_notebook", retry={
+    "max_attempts": 5,
+    "wait_strategy": "exponential_jitter",
+    "stop_on": [ValueError],  # Don't retry validation errors
+})
+async def process_source_command(input_data: SourceProcessingInput) -> SourceProcessingOutput:
+    start_time = time.time()
+    try:
+        transformations = [await Transformation.get(id) for id in input_data.transformations]
+        source = await Source.get(input_data.source_id)
+        result = await source_graph.ainvoke({...})
+        return SourceProcessingOutput(success=True, ...)
+    except ValueError as e:
+        return SourceProcessingOutput(success=False, error_message=str(e))  # No retry
+    except Exception as e:
+        raise  # Retry all other exceptions
+```

commands/__init__.py

@@ -0,0 +1,24 @@
+"""Surreal-commands integration for Open Notebook"""
+
+from .embedding_commands import (
+    embed_insight_command,
+    embed_note_command,
+    embed_source_command,
+    rebuild_embeddings_command,
+)
+from .example_commands import analyze_data_command, process_text_command
+from .podcast_commands import generate_podcast_command
+from .source_commands import process_source_command
+
+__all__ = [
+    # Embedding commands
+    "embed_note_command",
+    "embed_insight_command",
+    "embed_source_command",
+    "rebuild_embeddings_command",
+    # Other commands
+    "generate_podcast_command",
+    "process_source_command",
+    "process_text_command",
+    "analyze_data_command",
+]

commands/embedding_commands.py

@@ -0,0 +1,787 @@
+import time
+from typing import Dict, List, Literal, Optional
+
+from loguru import logger
+from pydantic import BaseModel
+from surreal_commands import CommandInput, CommandOutput, command, submit_command
+
+from open_notebook.ai.models import model_manager
+from open_notebook.database.repository import ensure_record_id, repo_insert, repo_query
+from open_notebook.exceptions import ConfigurationError
+from open_notebook.domain.notebook import Note, Source, SourceInsight
+from open_notebook.utils.chunking import ContentType, chunk_text, detect_content_type
+from open_notebook.utils.embedding import generate_embedding, generate_embeddings
+
+
+def full_model_dump(model):
+    if isinstance(model, BaseModel):
+        return model.model_dump()
+    elif isinstance(model, dict):
+        return {k: full_model_dump(v) for k, v in model.items()}
+    elif isinstance(model, list):
+        return [full_model_dump(item) for item in model]
+    else:
+        return model
+
+
+def get_command_id(input_data: CommandInput) -> str:
+    """Extract command_id from input_data's execution context, or return 'unknown'."""
+    if input_data.execution_context:
+        return str(input_data.execution_context.command_id)
+    return "unknown"
+
+
+class RebuildEmbeddingsInput(CommandInput):
+    mode: Literal["existing", "all"]
+    include_sources: bool = True
+    include_notes: bool = True
+    include_insights: bool = True
+
+
+class RebuildEmbeddingsOutput(CommandOutput):
+    success: bool
+    total_items: int
+    jobs_submitted: int  # Count of embedding commands submitted
+    failed_submissions: int  # Count of items that failed to submit
+    sources_submitted: int = 0
+    notes_submitted: int = 0
+    insights_submitted: int = 0
+    processing_time: float
+    error_message: Optional[str] = None
+
+
+# =============================================================================
+# NEW EMBEDDING COMMANDS (Phase 3)
+# =============================================================================
+
+
+class CreateInsightInput(CommandInput):
+    """Input for creating a source insight with automatic retry on conflicts."""
+
+    source_id: str
+    insight_type: str
+    content: str
+
+
+class CreateInsightOutput(CommandOutput):
+    """Output from insight creation command."""
+
+    success: bool
+    insight_id: Optional[str] = None
+    processing_time: float
+    error_message: Optional[str] = None
+
+
+class EmbedNoteInput(CommandInput):
+    """Input for embedding a single note."""
+
+    note_id: str
+
+
+class EmbedNoteOutput(CommandOutput):
+    """Output from note embedding command."""
+
+    success: bool
+    note_id: str
+    processing_time: float
+    error_message: Optional[str] = None
+
+
+class EmbedInsightInput(CommandInput):
+    """Input for embedding a single source insight."""
+
+    insight_id: str
+
+
+class EmbedInsightOutput(CommandOutput):
+    """Output from insight embedding command."""
+
+    success: bool
+    insight_id: str
+    processing_time: float
+    error_message: Optional[str] = None
+
+
+class EmbedSourceInput(CommandInput):
+    """Input for embedding a source (creates multiple chunk embeddings)."""
+
+    source_id: str
+
+
+class EmbedSourceOutput(CommandOutput):
+    """Output from source embedding command."""
+
+    success: bool
+    source_id: str
+    chunks_created: int
+    processing_time: float
+    error_message: Optional[str] = None
+
+
+@command(
+    "embed_note",
+    app="open_notebook",
+    retry={
+        "max_attempts": 5,
+        "wait_strategy": "exponential_jitter",
+        "wait_min": 1,
+        "wait_max": 60,
+        "stop_on": [ValueError, ConfigurationError],  # Don't retry validation/config errors
+        "retry_log_level": "debug",
+    },
+)
+async def embed_note_command(input_data: EmbedNoteInput) -> EmbedNoteOutput:
+    """
+    Generate and store embedding for a single note.
+
+    Uses the unified embedding pipeline with automatic chunking and mean pooling
+    for notes that exceed the chunk size limit.
+
+    Flow:
+    1. Load Note by ID
+    2. Generate embedding via generate_embedding() (auto-chunks + mean pools if needed)
+    3. UPSERT note embedding in database
+
+    Retry Strategy:
+    - Retries up to 5 times for transient failures (network, timeout, etc.)
+    - Uses exponential-jitter backoff (1-60s)
+    - Does NOT retry permanent failures (ValueError for validation errors)
+    """
+    start_time = time.time()
+
+    try:
+        logger.info(f"Starting embedding for note: {input_data.note_id}")
+
+        # 1. Load note
+        note = await Note.get(input_data.note_id)
+        if not note:
+            raise ValueError(f"Note '{input_data.note_id}' not found")
+
+        if not note.content or not note.content.strip():
+            raise ValueError(f"Note '{input_data.note_id}' has no content to embed")
+
+        # 2. Generate embedding (auto-chunks + mean pools if needed)
+        # Notes are typically markdown content
+        cmd_id = get_command_id(input_data)
+        embedding = await generate_embedding(
+            note.content, content_type=ContentType.MARKDOWN, command_id=cmd_id
+        )
+
+        # 3. UPSERT embedding into note record
+        await repo_query(
+            "UPDATE $note_id SET embedding = $embedding",
+            {
+                "note_id": ensure_record_id(input_data.note_id),
+                "embedding": embedding,
+            },
+        )
+
+        processing_time = time.time() - start_time
+        logger.info(
+            f"Successfully embedded note {input_data.note_id} in {processing_time:.2f}s"
+        )
+
+        return EmbedNoteOutput(
+            success=True,
+            note_id=input_data.note_id,
+            processing_time=processing_time,
+        )
+
+    except ValueError as e:
+        # Permanent failure - don't retry
+        processing_time = time.time() - start_time
+        cmd_id = get_command_id(input_data)
+        logger.error(
+            f"Failed to embed note {input_data.note_id} (command: {cmd_id}): {e}"
+        )
+        return EmbedNoteOutput(
+            success=False,
+            note_id=input_data.note_id,
+            processing_time=processing_time,
+            error_message=str(e),
+        )
+    except Exception as e:
+        # Transient failure - will be retried (surreal-commands logs final failure)
+        cmd_id = get_command_id(input_data)
+        logger.debug(
+            f"Transient error embedding note {input_data.note_id} "
+            f"(command: {cmd_id}): {e}"
+        )
+        raise
+
+
+@command(
+    "embed_insight",
+    app="open_notebook",
+    retry={
+        "max_attempts": 5,
+        "wait_strategy": "exponential_jitter",
+        "wait_min": 1,
+        "wait_max": 60,
+        "stop_on": [ValueError, ConfigurationError],  # Don't retry validation/config errors
+        "retry_log_level": "debug",
+    },
+)
+async def embed_insight_command(input_data: EmbedInsightInput) -> EmbedInsightOutput:
+    """
+    Generate and store embedding for a single source insight.
+
+    Uses the unified embedding pipeline with automatic chunking and mean pooling
+    for insights that exceed the chunk size limit.
+
+    Flow:
+    1. Load SourceInsight by ID
+    2. Generate embedding via generate_embedding() (auto-chunks + mean pools if needed)
+    3. UPSERT insight embedding in database
+
+    Retry Strategy:
+    - Retries up to 5 times for transient failures (network, timeout, etc.)
+    - Uses exponential-jitter backoff (1-60s)
+    - Does NOT retry permanent failures (ValueError for validation errors)
+    """
+    start_time = time.time()
+
+    try:
+        logger.info(f"Starting embedding for insight: {input_data.insight_id}")
+
+        # 1. Load insight
+        insight = await SourceInsight.get(input_data.insight_id)
+        if not insight:
+            raise ValueError(f"Insight '{input_data.insight_id}' not found")
+
+        if not insight.content or not insight.content.strip():
+            raise ValueError(
+                f"Insight '{input_data.insight_id}' has no content to embed"
+            )
+
+        # 2. Generate embedding (auto-chunks + mean pools if needed)
+        # Insights are typically markdown content (generated by LLM)
+        cmd_id = get_command_id(input_data)
+        embedding = await generate_embedding(
+            insight.content, content_type=ContentType.MARKDOWN, command_id=cmd_id
+        )
+
+        # 3. UPSERT embedding into insight record
+        await repo_query(
+            "UPDATE $insight_id SET embedding = $embedding",
+            {
+                "insight_id": ensure_record_id(input_data.insight_id),
+                "embedding": embedding,
+            },
+        )
+
+        processing_time = time.time() - start_time
+        logger.info(
+            f"Successfully embedded insight {input_data.insight_id} in {processing_time:.2f}s"
+        )
+
+        return EmbedInsightOutput(
+            success=True,
+            insight_id=input_data.insight_id,
+            processing_time=processing_time,
+        )
+
+    except ValueError as e:
+        # Permanent failure - don't retry
+        processing_time = time.time() - start_time
+        cmd_id = get_command_id(input_data)
+        logger.error(
+            f"Failed to embed insight {input_data.insight_id} (command: {cmd_id}): {e}"
+        )
+        return EmbedInsightOutput(
+            success=False,
+            insight_id=input_data.insight_id,
+            processing_time=processing_time,
+            error_message=str(e),
+        )
+    except Exception as e:
+        # Transient failure - will be retried (surreal-commands logs final failure)
+        cmd_id = get_command_id(input_data)
+        logger.debug(
+            f"Transient error embedding insight {input_data.insight_id} "
+            f"(command: {cmd_id}): {e}"
+        )
+        raise
+
+
+@command(
+    "embed_source",
+    app="open_notebook",
+    retry={
+        "max_attempts": 5,
+        "wait_strategy": "exponential_jitter",
+        "wait_min": 1,
+        "wait_max": 60,
+        "stop_on": [ValueError, ConfigurationError],  # Don't retry validation/config errors
+        "retry_log_level": "debug",
+    },
+)
+async def embed_source_command(input_data: EmbedSourceInput) -> EmbedSourceOutput:
+    """
+    Generate and store embeddings for a source document.
+
+    Creates multiple chunk embeddings stored in the source_embedding table.
+    Uses content-type aware chunking based on file extension or content heuristics.
+
+    Flow:
+    1. Load Source by ID
+    2. DELETE existing source_embedding records for this source
+    3. Detect content type from file path or content
+    4. Chunk text using appropriate splitter
+    5. Generate embeddings for all chunks in batches
+    6. Bulk INSERT source_embedding records
+
+    Retry Strategy:
+    - Retries up to 5 times for transient failures (network, timeout, etc.)
+    - Uses exponential-jitter backoff (1-60s)
+    - Does NOT retry permanent failures (ValueError for validation errors)
+    """
+    start_time = time.time()
+
+    try:
+        logger.info(f"Starting embedding for source: {input_data.source_id}")
+
+        # 1. Load source
+        source = await Source.get(input_data.source_id)
+        if not source:
+            raise ValueError(f"Source '{input_data.source_id}' not found")
+
+        if not source.full_text or not source.full_text.strip():
+            raise ValueError(f"Source '{input_data.source_id}' has no text to embed")
+
+        # 2. DELETE existing embeddings (idempotency)
+        logger.debug(f"Deleting existing embeddings for source {input_data.source_id}")
+        await repo_query(
+            "DELETE source_embedding WHERE source = $source_id",
+            {"source_id": ensure_record_id(input_data.source_id)},
+        )
+
+        # 3. Detect content type from file path if available
+        file_path = source.asset.file_path if source.asset else None
+        content_type = detect_content_type(source.full_text, file_path)
+        logger.debug(f"Detected content type: {content_type.value}")
+
+        # 4. Chunk text using appropriate splitter
+        chunks = chunk_text(source.full_text, content_type=content_type)
+        total_chunks = len(chunks)
+
+        # Log chunk statistics for debugging
+        chunk_sizes = [len(c) for c in chunks]
+        logger.info(
+            f"Created {total_chunks} chunks for source {input_data.source_id} "
+            f"(sizes: min={min(chunk_sizes) if chunk_sizes else 0}, "
+            f"max={max(chunk_sizes) if chunk_sizes else 0}, "
+            f"avg={sum(chunk_sizes)//len(chunk_sizes) if chunk_sizes else 0} chars)"
+        )
+
+        if total_chunks == 0:
+            raise ValueError("No chunks created after splitting text")
+
+        # 5. Generate embeddings for all chunks in batches
+        cmd_id = get_command_id(input_data)
+        logger.debug(f"Generating embeddings for {total_chunks} chunks")
+        embeddings = await generate_embeddings(chunks, command_id=cmd_id)
+
+        # Verify we got embeddings for all chunks
+        if len(embeddings) != len(chunks):
+            raise ValueError(
+                f"Embedding count mismatch: got {len(embeddings)} embeddings "
+                f"for {len(chunks)} chunks"
+            )
+
+        # 6. Bulk INSERT source_embedding records
+        records = [
+            {
+                "source": ensure_record_id(input_data.source_id),
+                "order": idx,
+                "content": chunk,
+                "embedding": embedding,
+            }
+            for idx, (chunk, embedding) in enumerate(zip(chunks, embeddings))
+        ]
+
+        logger.debug(f"Inserting {len(records)} source_embedding records")
+        await repo_insert("source_embedding", records)
+
+        processing_time = time.time() - start_time
+        logger.info(
+            f"Successfully embedded source {input_data.source_id}: "
+            f"{total_chunks} chunks in {processing_time:.2f}s"
+        )
+
+        return EmbedSourceOutput(
+            success=True,
+            source_id=input_data.source_id,
+            chunks_created=total_chunks,
+            processing_time=processing_time,
+        )
+
+    except ValueError as e:
+        # Permanent failure - don't retry
+        processing_time = time.time() - start_time
+        cmd_id = get_command_id(input_data)
+        logger.error(
+            f"Failed to embed source {input_data.source_id} (command: {cmd_id}): {e}"
+        )
+        return EmbedSourceOutput(
+            success=False,
+            source_id=input_data.source_id,
+            chunks_created=0,
+            processing_time=processing_time,
+            error_message=str(e),
+        )
+    except Exception as e:
+        # Transient failure - will be retried (surreal-commands logs final failure)
+        cmd_id = get_command_id(input_data)
+        logger.debug(
+            f"Transient error embedding source {input_data.source_id} "
+            f"(command: {cmd_id}): {e}"
+        )
+        raise
+
+
+@command(
+    "create_insight",
+    app="open_notebook",
+    retry={
+        "max_attempts": 5,
+        "wait_strategy": "exponential_jitter",
+        "wait_min": 1,
+        "wait_max": 60,
+        "stop_on": [ValueError, ConfigurationError],  # Don't retry validation/config errors
+        "retry_log_level": "debug",
+    },
+)
+async def create_insight_command(
+    input_data: CreateInsightInput,
+) -> CreateInsightOutput:
+    """
+    Create a source insight with automatic retry on transaction conflicts.
+
+    This command wraps the CREATE source_insight operation with retry logic
+    to handle SurrealDB transaction conflicts that occur during batch imports
+    when multiple parallel transformations try to create insights concurrently.
+
+    Flow:
+    1. CREATE source_insight record in database
+    2. Submit embed_insight command (fire-and-forget) for async embedding
+    3. Return the insight_id
+
+    Retry Strategy:
+    - Retries up to 5 times for transient failures (network, timeout, etc.)
+    - Uses exponential-jitter backoff (1-60s)
+    - Does NOT retry permanent failures (ValueError for validation errors)
+    """
+    start_time = time.time()
+
+    try:
+        logger.info(
+            f"Creating insight for source {input_data.source_id}: "
+            f"type={input_data.insight_type}"
+        )
+
+        # 1. Create insight record in database
+        result = await repo_query(
+            """
+            CREATE source_insight CONTENT {
+                "source": $source_id,
+                "insight_type": $insight_type,
+                "content": $content
+            };
+            """,
+            {
+                "source_id": ensure_record_id(input_data.source_id),
+                "insight_type": input_data.insight_type,
+                "content": input_data.content,
+            },
+        )
+
+        if not result or len(result) == 0:
+            raise ValueError("Failed to create insight - no result returned")
+
+        insight_id = str(result[0].get("id", ""))
+        if not insight_id:
+            raise ValueError("Failed to create insight - no ID in result")
+
+        # 2. Submit embedding command (fire-and-forget)
+        submit_command(
+            "open_notebook",
+            "embed_insight",
+            {"insight_id": insight_id},
+        )
+        logger.debug(f"Submitted embed_insight command for {insight_id}")
+
+        processing_time = time.time() - start_time
+        logger.info(
+            f"Successfully created insight {insight_id} for source "
+            f"{input_data.source_id} in {processing_time:.2f}s"
+        )
+
+        return CreateInsightOutput(
+            success=True,
+            insight_id=insight_id,
+            processing_time=processing_time,
+        )
+
+    except ValueError as e:
+        # Permanent failure - don't retry
+        processing_time = time.time() - start_time
+        cmd_id = get_command_id(input_data)
+        logger.error(
+            f"Failed to create insight for source {input_data.source_id} "
+            f"(command: {cmd_id}): {e}"
+        )
+        return CreateInsightOutput(
+            success=False,
+            processing_time=processing_time,
+            error_message=str(e),
+        )
+    except Exception as e:
+        # Transient failure - will be retried (surreal-commands logs final failure)
+        cmd_id = get_command_id(input_data)
+        logger.debug(
+            f"Transient error creating insight for source {input_data.source_id} "
+            f"(command: {cmd_id}): {e}"
+        )
+        raise
+
+
+async def collect_items_for_rebuild(
+    mode: str,
+    include_sources: bool,
+    include_notes: bool,
+    include_insights: bool,
+) -> Dict[str, List[str]]:
+    """
+    Collect items to rebuild based on mode and include flags.
+
+    Returns:
+        Dict with keys: 'sources', 'notes', 'insights' containing lists of item IDs
+    """
+    items: Dict[str, List[str]] = {"sources": [], "notes": [], "insights": []}
+
+    if include_sources:
+        if mode == "existing":
+            # Query sources with embeddings (via source_embedding table)
+            result = await repo_query(
+                """
+                RETURN array::distinct(
+                    SELECT VALUE source.id
+                    FROM source_embedding
+                    WHERE embedding != none AND array::len(embedding) > 0
+                )
+                """
+            )
+            # RETURN returns the array directly as the result (not nested)
+            if result:
+                items["sources"] = [str(item) for item in result]
+            else:
+                items["sources"] = []
+        else:  # mode == "all"
+            # Query all sources with non-empty content
+            result = await repo_query(
+                "SELECT id FROM source WHERE full_text != none AND string::trim(full_text) != ''"
+            )
+            items["sources"] = [str(item["id"]) for item in result] if result else []
+
+        logger.info(f"Collected {len(items['sources'])} sources for rebuild")
+
+    if include_notes:
+        if mode == "existing":
+            # Query notes with embeddings
+            result = await repo_query(
+                "SELECT id FROM note WHERE embedding != none AND array::len(embedding) > 0"
+            )
+        else:  # mode == "all"
+            # Query all notes with non-empty content
+            result = await repo_query(
+                "SELECT id FROM note WHERE content != none AND string::trim(content) != ''"
+            )
+
+        items["notes"] = [str(item["id"]) for item in result] if result else []
+        logger.info(f"Collected {len(items['notes'])} notes for rebuild")
+
+    if include_insights:
+        if mode == "existing":
+            # Query insights with embeddings
+            result = await repo_query(
+                "SELECT id FROM source_insight WHERE embedding != none AND array::len(embedding) > 0"
+            )
+        else:  # mode == "all"
+            # Query all insights with non-empty content
+            result = await repo_query(
+                "SELECT id FROM source_insight WHERE content != none AND string::trim(content) != ''"
+            )
+
+        items["insights"] = [str(item["id"]) for item in result] if result else []
+        logger.info(f"Collected {len(items['insights'])} insights for rebuild")
+
+    return items
+
+
+@command("rebuild_embeddings", app="open_notebook", retry=None)
+async def rebuild_embeddings_command(
+    input_data: RebuildEmbeddingsInput,
+) -> RebuildEmbeddingsOutput:
+    """
+    Rebuild embeddings for sources, notes, and/or insights.
+
+    This command submits individual embedding jobs for each item:
+    - embed_source for sources
+    - embed_note for notes
+    - embed_insight for insights
+
+    The command returns after submitting all jobs. Actual embedding
+    happens asynchronously via the individual commands (which have
+    their own retry strategies).
+
+    Retry Strategy:
+    - Retries disabled (retry=None) for this coordinator command
+    - Individual embed_* commands handle their own retries
+    """
+    start_time = time.time()
+
+    try:
+        logger.info("=" * 60)
+        logger.info(f"Starting embedding rebuild with mode={input_data.mode}")
+        logger.info(
+            f"Include: sources={input_data.include_sources}, notes={input_data.include_notes}, insights={input_data.include_insights}"
+        )
+        logger.info("=" * 60)
+
+        # Check embedding model availability (fail fast)
+        EMBEDDING_MODEL = await model_manager.get_embedding_model()
+        if not EMBEDDING_MODEL:
+            raise ValueError(
+                "No embedding model configured. Please configure one in the Models section."
+            )
+
+        logger.info(f"Embedding model configured: {EMBEDDING_MODEL}")
+
+        # Collect items to process (returns IDs only)
+        items = await collect_items_for_rebuild(
+            input_data.mode,
+            input_data.include_sources,
+            input_data.include_notes,
+            input_data.include_insights,
+        )
+
+        total_items = (
+            len(items["sources"]) + len(items["notes"]) + len(items["insights"])
+        )
+        logger.info(f"Total items to rebuild: {total_items}")
+
+        if total_items == 0:
+            logger.warning("No items found to rebuild")
+            return RebuildEmbeddingsOutput(
+                success=True,
+                total_items=0,
+                jobs_submitted=0,
+                failed_submissions=0,
+                processing_time=time.time() - start_time,
+            )
+
+        # Initialize counters
+        sources_submitted = 0
+        notes_submitted = 0
+        insights_submitted = 0
+        failed_submissions = 0
+
+        # Submit embed_source commands for sources
+        logger.info(f"\nSubmitting {len(items['sources'])} source embedding jobs...")
+        for idx, source_id in enumerate(items["sources"], 1):
+            try:
+                submit_command(
+                    "open_notebook",
+                    "embed_source",
+                    {"source_id": source_id},
+                )
+                sources_submitted += 1
+
+                if idx % 50 == 0 or idx == len(items["sources"]):
+                    logger.info(
+                        f"  Progress: {idx}/{len(items['sources'])} source jobs submitted"
+                    )
+
+            except Exception as e:
+                logger.error(f"Failed to submit embed_source for {source_id}: {e}")
+                failed_submissions += 1
+
+        # Submit embed_note commands for notes
+        logger.info(f"\nSubmitting {len(items['notes'])} note embedding jobs...")
+        for idx, note_id in enumerate(items["notes"], 1):
+            try:
+                submit_command(
+                    "open_notebook",
+                    "embed_note",
+                    {"note_id": note_id},
+                )
+                notes_submitted += 1
+
+                if idx % 50 == 0 or idx == len(items["notes"]):
+                    logger.info(
+                        f"  Progress: {idx}/{len(items['notes'])} note jobs submitted"
+                    )
+
+            except Exception as e:
+                logger.error(f"Failed to submit embed_note for {note_id}: {e}")
+                failed_submissions += 1
+
+        # Submit embed_insight commands for insights
+        logger.info(f"\nSubmitting {len(items['insights'])} insight embedding jobs...")
+        for idx, insight_id in enumerate(items["insights"], 1):
+            try:
+                submit_command(
+                    "open_notebook",
+                    "embed_insight",
+                    {"insight_id": insight_id},
+                )
+                insights_submitted += 1
+
+                if idx % 50 == 0 or idx == len(items["insights"]):
+                    logger.info(
+                        f"  Progress: {idx}/{len(items['insights'])} insight jobs submitted"
+                    )
+
+            except Exception as e:
+                logger.error(f"Failed to submit embed_insight for {insight_id}: {e}")
+                failed_submissions += 1
+
+        processing_time = time.time() - start_time
+        jobs_submitted = sources_submitted + notes_submitted + insights_submitted
+
+        logger.info("=" * 60)
+        logger.info("REBUILD JOBS SUBMITTED")
+        logger.info(f"  Total jobs submitted: {jobs_submitted}/{total_items}")
+        logger.info(f"  Sources: {sources_submitted}")
+        logger.info(f"  Notes: {notes_submitted}")
+        logger.info(f"  Insights: {insights_submitted}")
+        logger.info(f"  Failed submissions: {failed_submissions}")
+        logger.info(f"  Submission time: {processing_time:.2f}s")
+        logger.info("  Note: Actual embedding happens asynchronously")
+        logger.info("=" * 60)
+
+        return RebuildEmbeddingsOutput(
+            success=True,
+            total_items=total_items,
+            jobs_submitted=jobs_submitted,
+            failed_submissions=failed_submissions,
+            sources_submitted=sources_submitted,
+            notes_submitted=notes_submitted,
+            insights_submitted=insights_submitted,
+            processing_time=processing_time,
+        )
+
+    except Exception as e:
+        processing_time = time.time() - start_time
+        logger.error(f"Rebuild embeddings failed: {e}")
+        logger.exception(e)
+
+        return RebuildEmbeddingsOutput(
+            success=False,
+            total_items=0,
+            jobs_submitted=0,
+            failed_submissions=0,
+            processing_time=processing_time,
+            error_message=str(e),
+        )

commands/example_commands.py

@@ -0,0 +1,142 @@
+import asyncio
+import time
+from typing import List, Optional
+
+from loguru import logger
+from pydantic import BaseModel
+from surreal_commands import command
+
+
+class TextProcessingInput(BaseModel):
+    text: str
+    operation: str = "uppercase"  # uppercase, lowercase, word_count, reverse
+    delay_seconds: Optional[int] = None  # For testing async behavior
+
+
+class TextProcessingOutput(BaseModel):
+    success: bool
+    original_text: str
+    processed_text: Optional[str] = None
+    word_count: Optional[int] = None
+    processing_time: float
+    error_message: Optional[str] = None
+
+
+class DataAnalysisInput(BaseModel):
+    numbers: List[float]
+    analysis_type: str = "basic"  # basic, detailed
+    delay_seconds: Optional[int] = None
+
+
+class DataAnalysisOutput(BaseModel):
+    success: bool
+    analysis_type: str
+    count: int
+    sum: Optional[float] = None
+    average: Optional[float] = None
+    min_value: Optional[float] = None
+    max_value: Optional[float] = None
+    processing_time: float
+    error_message: Optional[str] = None
+
+
+@command("process_text", app="open_notebook")
+async def process_text_command(input_data: TextProcessingInput) -> TextProcessingOutput:
+    """
+    Example command for text processing. Tests basic command functionality
+    and demonstrates different processing types.
+    """
+    start_time = time.time()
+
+    try:
+        logger.info(f"Processing text with operation: {input_data.operation}")
+
+        # Simulate processing delay if specified
+        if input_data.delay_seconds:
+            await asyncio.sleep(input_data.delay_seconds)
+
+        processed_text = None
+        word_count = None
+
+        if input_data.operation == "uppercase":
+            processed_text = input_data.text.upper()
+        elif input_data.operation == "lowercase":
+            processed_text = input_data.text.lower()
+        elif input_data.operation == "reverse":
+            processed_text = input_data.text[::-1]
+        elif input_data.operation == "word_count":
+            word_count = len(input_data.text.split())
+            processed_text = f"Word count: {word_count}"
+        else:
+            raise ValueError(f"Unknown operation: {input_data.operation}")
+
+        processing_time = time.time() - start_time
+
+        return TextProcessingOutput(
+            success=True,
+            original_text=input_data.text,
+            processed_text=processed_text,
+            word_count=word_count,
+            processing_time=processing_time,
+        )
+
+    except Exception as e:
+        processing_time = time.time() - start_time
+        logger.error(f"Text processing failed: {e}")
+        return TextProcessingOutput(
+            success=False,
+            original_text=input_data.text,
+            processing_time=processing_time,
+            error_message=str(e),
+        )
+
+
+@command("analyze_data", app="open_notebook")
+async def analyze_data_command(input_data: DataAnalysisInput) -> DataAnalysisOutput:
+    """
+    Example command for data analysis. Tests command with complex input/output
+    and demonstrates error handling.
+    """
+    start_time = time.time()
+
+    try:
+        logger.info(
+            f"Analyzing {len(input_data.numbers)} numbers with {input_data.analysis_type} analysis"
+        )
+
+        # Simulate processing delay if specified
+        if input_data.delay_seconds:
+            await asyncio.sleep(input_data.delay_seconds)
+
+        if not input_data.numbers:
+            raise ValueError("No numbers provided for analysis")
+
+        count = len(input_data.numbers)
+        sum_value = sum(input_data.numbers)
+        average = sum_value / count
+        min_value = min(input_data.numbers)
+        max_value = max(input_data.numbers)
+
+        processing_time = time.time() - start_time
+
+        return DataAnalysisOutput(
+            success=True,
+            analysis_type=input_data.analysis_type,
+            count=count,
+            sum=sum_value,
+            average=average,
+            min_value=min_value,
+            max_value=max_value,
+            processing_time=processing_time,
+        )
+
+    except Exception as e:
+        processing_time = time.time() - start_time
+        logger.error(f"Data analysis failed: {e}")
+        return DataAnalysisOutput(
+            success=False,
+            analysis_type=input_data.analysis_type,
+            count=0,
+            processing_time=processing_time,
+            error_message=str(e),
+        )