Deploy Open Notebook to HuggingFace Spaces

.dockerignore

@@ -0,0 +1,52 @@
+# Git
+.git
+.gitignore
+
+# Python
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+.venv
+venv
+ENV
+env
+.pytest_cache
+.mypy_cache
+.ruff_cache
+
+# Frontend
+frontend/node_modules
+frontend/.next
+frontend/dist
+frontend/out
+frontend/.env*
+frontend/*.log
+
+# Project data
+.antigravity
+.gemini
+tmp
+data
+mydata
+notebook_data
+surreal_data
+surreal-data
+surreal_single_data
+*.db
+*.log
+docker.env
+.env
+docker-compose*
+
+# Documentation & CI (not needed in image)
+docs
+.github
+
+# IDE and OS files
+.vscode
+.idea
+*.swp
+*.swo
+*~
+.DS_Store

.env.example

@@ -0,0 +1,59 @@
+# Open Notebook Configuration
+# Copy this file to .env and customize as needed
+
+# =============================================================================
+# REQUIRED
+# =============================================================================
+
+# Encryption key for storing API credentials securely in the database
+# Change this to any secret string (minimum 16 characters recommended)
+OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-to-a-secret-string
+
+# =============================================================================
+# DATABASE (Default values work with docker-compose.yml)
+# =============================================================================
+
+SURREAL_URL=ws://surrealdb:8000/rpc
+SURREAL_USER=root
+SURREAL_PASSWORD=root
+SURREAL_NAMESPACE=open_notebook
+SURREAL_DATABASE=open_notebook
+
+# =============================================================================
+# OPTIONAL: AI Provider API Keys
+# =============================================================================
+# You can configure these via the UI (Settings → API Keys) or set them here
+# UI configuration is recommended for better security and flexibility
+
+# OpenAI
+# OPENAI_API_KEY=sk-...
+
+# Anthropic
+# ANTHROPIC_API_KEY=sk-ant-...
+
+# Google AI
+# GOOGLE_API_KEY=...
+
+# Groq
+# GROQ_API_KEY=gsk_...
+
+# =============================================================================
+# OPTIONAL: Advanced Configuration
+# =============================================================================
+
+# External API URL (for webhooks, callbacks, etc.)
+# API_URL=http://localhost:5055
+
+# Ollama endpoint (if running locally)
+# OLLAMA_BASE_URL=http://ollama:11434
+
+# Content processing
+# CHUNK_SIZE=1500
+# CHUNK_OVERLAP=150
+
+# Security
+# BASIC_AUTH_USERNAME=admin
+# BASIC_AUTH_PASSWORD=secret
+
+# For more configuration options, see:
+# https://github.com/lfnovo/open-notebook/blob/main/docs/5-CONFIGURATION/environment-reference.md

.gitattributes

@@ -0,0 +1,11 @@
+# Ensure shell scripts always use LF so they run in Linux containers (e.g. Docker)
+*.sh text eol=lf
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.svg filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+*.ico filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,104 @@
+name: 🐛 Bug Report
+description: Report a bug or unexpected behavior (app is running but misbehaving)
+title: "[Bug]: "
+labels: ["bug", "needs-triage"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for reporting a bug! Please fill out the information below to help us understand and fix the issue.
+
+        **Note**: If you're having installation or setup issues, please use the "Installation Issue" template instead.
+
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What did you do when it broke?
+      description: Describe the steps you took that led to the bug
+      placeholder: |
+        1. I went to the Notebooks page
+        2. I clicked on "Create New Notebook"
+        3. I filled in the form and clicked "Save"
+        4. Then the error occurred...
+    validations:
+      required: true
+
+  - type: textarea
+    id: how-broke
+    attributes:
+      label: How did it break?
+      description: What happened that was unexpected? What did you expect to happen instead?
+      placeholder: |
+        Expected: The notebook should be created and I should see it in the list
+        Actual: I got an error message saying "Failed to create notebook"
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs-screenshots
+    attributes:
+      label: Logs or Screenshots
+      description: |
+        Please provide any error messages, logs, or screenshots that might help us understand the issue.
+
+        **How to get logs:**
+        - Docker: `docker compose logs -f open_notebook`
+        - Check browser console (F12 → Console tab)
+      placeholder: |
+        Paste logs here or drag and drop screenshots.
+
+        Error messages, stack traces, or browser console errors are very helpful!
+    validations:
+      required: false
+
+  - type: dropdown
+    id: version
+    attributes:
+      label: Open Notebook Version
+      description: Which version are you using?
+      options:
+        - v1-latest (Docker)
+        - v1-latest-single (Docker, deprecated)
+        - Latest from main branch
+        - Other (please specify in additional context)
+    validations:
+      required: true
+
+  - type: textarea
+    id: environment
+    attributes:
+      label: Environment
+      description: What environment are you running in?
+      placeholder: |
+        - OS: Ubuntu 22.04 / Windows 11 / macOS 14
+        - Browser: Chrome 120
+    validations:
+      required: false
+
+  - type: textarea
+    id: additional-context
+    attributes:
+      label: Additional Context
+      description: Any other information that might be helpful
+      placeholder: "This started happening after I upgraded to v1.5.0..."
+    validations:
+      required: false
+
+  - type: checkboxes
+    id: willing-to-contribute
+    attributes:
+      label: Contribution
+      description: Would you like to work on fixing this bug?
+      options:
+        - label: I am a developer and would like to work on fixing this issue (pending maintainer approval)
+          required: false
+
+  - type: markdown
+    attributes:
+      value: |
+        ---
+        **Next Steps:**
+        1. A maintainer will review your bug report
+        2. If you checked the box above and want to fix it, please propose your solution approach
+        3. Wait for assignment before starting development
+        4. See our [Contributing Guide](https://github.com/lfnovo/open-notebook/blob/main/CONTRIBUTING.md) for more details

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,11 @@
+blank_issues_enabled: false
+contact_links:
+  - name: 💬 Discord Community
+    url: https://discord.gg/37XJPXfz2w
+    about: Get help from the community and share ideas
+  - name: 🤖 Installation Assistant (ChatGPT)
+    url: https://chatgpt.com/g/g-68776e2765b48191bd1bae3f30212631-open-notebook-installation-assistant
+    about: CustomGPT that knows all our docs. Really useful. Try it.
+  - name: 📚 Documentation
+    url: https://github.com/lfnovo/open-notebook/tree/main/docs
+    about: Browse our comprehensive documentation

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,65 @@
+name: ✨ Feature Suggestion
+description: Suggest a new feature or improvement for Open Notebook
+title: "[Feature]: "
+labels: ["enhancement", "needs-triage"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to suggest a feature! Your ideas help make Open Notebook better for everyone.
+
+  - type: textarea
+    id: feature-description
+    attributes:
+      label: Feature Description
+      description: What feature would you like to see added or improved?
+      placeholder: "I would like to be able to..."
+    validations:
+      required: true
+
+  - type: textarea
+    id: why-helpful
+    attributes:
+      label: Why would this be helpful?
+      description: Explain how this feature would benefit you and other users
+      placeholder: "This would help because..."
+    validations:
+      required: true
+
+  - type: textarea
+    id: proposed-solution
+    attributes:
+      label: Proposed Solution (Optional)
+      description: If you have ideas on how to implement this feature, please share them
+      placeholder: "This could be implemented by..."
+    validations:
+      required: false
+
+  - type: textarea
+    id: additional-context
+    attributes:
+      label: Additional Context
+      description: Any other context, screenshots, or examples that might be helpful
+      placeholder: "For example, other tools do this by..."
+    validations:
+      required: false
+
+  - type: checkboxes
+    id: willing-to-contribute
+    attributes:
+      label: Contribution
+      description: Would you like to work on implementing this feature?
+      options:
+        - label: I am a developer and would like to work on implementing this feature (pending maintainer approval)
+          required: false
+
+  - type: markdown
+    attributes:
+      value: |
+        ---
+        **Next Steps:**
+        1. A maintainer will review your feature request
+        2. If approved and you checked the box above, the issue will be assigned to you
+        3. Please wait for assignment before starting development
+        4. See our [Contributing Guide](https://github.com/lfnovo/open-notebook/blob/main/CONTRIBUTING.md) for more details
+

.github/ISSUE_TEMPLATE/installation_issue.yml

@@ -0,0 +1,148 @@
+name: 🔧 Installation Issue
+description: Report problems with installation, setup, or connectivity
+title: "[Install]: "
+labels: ["installation", "needs-triage"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        ## ⚠️ Before You Continue
+
+        **Please try these resources first:**
+
+        1. 🤖 **[Installation Assistant ChatGPT](https://chatgpt.com/g/g-68776e2765b48191bd1bae3f30212631-open-notebook-installation-assistant)** - Our AI assistant can help you troubleshoot most installation issues instantly!
+
+        2. 📚 **[Installation Guide](https://github.com/lfnovo/open-notebook/blob/main/docs/getting-started/installation.md)** - Comprehensive setup instructions
+
+        3. 🐋 **[Docker Deployment Guide](https://github.com/lfnovo/open-notebook/blob/main/docs/deployment/docker.md)** - Detailed Docker setup
+
+        4. 🦙 **Ollama Issues?** Read our [Ollama Guide](https://github.com/lfnovo/open-notebook/blob/main/docs/features/ollama.md) first
+
+        5. 💬 **[Discord Community](https://discord.gg/37XJPXfz2w)** - Get real-time help from the community
+
+        ---
+
+        If you've tried the above and still need help, please fill out the form below with as much detail as possible.
+
+  - type: dropdown
+    id: installation-method
+    attributes:
+      label: Installation Method
+      description: How are you trying to install Open Notebook?
+      options:
+        - Docker (docker-compose - recommended)
+        - Docker (single container - v1-latest-single, deprecated)
+        - Local development (make start-all)
+        - Other (please specify below)
+    validations:
+      required: true
+
+  - type: textarea
+    id: issue-description
+    attributes:
+      label: What is the issue?
+      description: Describe the installation or setup problem you're experiencing
+      placeholder: |
+        Example: "I can't connect to the database" or "The container won't start" or "Getting 404 errors when accessing the UI"
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Logs
+      description: |
+        Please provide relevant logs. **This is very important for diagnosing issues!**
+
+        **How to get logs:**
+        - Docker single container: `docker logs open-notebook`
+        - Docker Compose: `docker compose logs -f`
+        - Specific service: `docker compose logs -f open_notebook`
+      placeholder: |
+        Paste your logs here. Include the full error message and stack trace if available.
+      render: shell
+    validations:
+      required: false
+
+  - type: textarea
+    id: docker-compose
+    attributes:
+      label: Docker Compose Configuration
+      description: |
+        If using Docker Compose, please paste your `docker-compose.yml` file here.
+
+        **⚠️ IMPORTANT: Redact any sensitive information (API keys, passwords, etc.)**
+      placeholder: |
+        services:
+          open_notebook:
+            image: lfnovo/open_notebook:v1-latest-single
+            ports:
+              - "8502:8502"
+              - "5055:5055"
+            environment:
+              - OPENAI_API_KEY=sk-***REDACTED***
+            ...
+      render: yaml
+    validations:
+      required: false
+
+  - type: textarea
+    id: env-file
+    attributes:
+      label: Environment File
+      description: |
+        If using an `.env` or `docker.env` file, please paste it here.
+
+        **⚠️ IMPORTANT: REDACT ALL API KEYS AND PASSWORDS!**
+      placeholder: |
+        SURREAL_URL=ws://surrealdb:8000/rpc
+        SURREAL_USER=root
+        SURREAL_PASSWORD=***REDACTED***
+        OPENAI_API_KEY=sk-***REDACTED***
+        ANTHROPIC_API_KEY=sk-ant-***REDACTED***
+      render: shell
+    validations:
+      required: false
+
+  - type: textarea
+    id: system-info
+    attributes:
+      label: System Information
+      description: Tell us about your setup
+      placeholder: |
+        - Operating System: Ubuntu 22.04 / Windows 11 / macOS 14
+        - Docker version: `docker --version`
+        - Docker Compose version: `docker compose version`
+        - Architecture: amd64 / arm64 (Apple Silicon)
+        - Available disk space: `df -h`
+        - Available memory: `free -h` (Linux) or Activity Monitor (Mac)
+    validations:
+      required: false
+
+  - type: textarea
+    id: additional-context
+    attributes:
+      label: Additional Context
+      description: Any other information that might be helpful
+      placeholder: |
+        - Are you behind a corporate proxy or firewall?
+        - Are you using a VPN?
+        - Have you made any custom modifications?
+        - Did this work before and suddenly break?
+    validations:
+      required: false
+
+  - type: checkboxes
+    id: checklist
+    attributes:
+      label: Pre-submission Checklist
+      description: Please confirm you've tried these steps
+      options:
+        - label: I tried the [Installation Assistant ChatGPT](https://chatgpt.com/g/g-68776e2765b48191bd1bae3f30212631-open-notebook-installation-assistant)
+          required: false
+        - label: I read the relevant documentation ([Installation Guide](https://github.com/lfnovo/open-notebook/blob/main/docs/getting-started/installation.md) or [Ollama Guide](https://github.com/lfnovo/open-notebook/blob/main/docs/features/ollama.md))
+          required: false
+        - label: I searched existing issues to see if this was already reported
+          required: true
+        - label: I redacted all sensitive information (API keys, passwords, etc.)
+          required: true

.github/pull_request_template.md

@@ -0,0 +1,107 @@
+## Description
+
+<!-- Provide a clear and concise description of what this PR does -->
+
+## Related Issue
+
+<!-- This PR should be linked to an approved issue. If not, please create an issue first. -->
+
+Fixes #<!-- issue number -->
+
+## Type of Change
+
+<!-- Mark the relevant option with an "x" -->
+
+- [ ] Bug fix (non-breaking change that fixes an issue)
+- [ ] New feature (non-breaking change that adds functionality)
+- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
+- [ ] Documentation update
+- [ ] Code refactoring (no functional changes)
+- [ ] Performance improvement
+- [ ] Test coverage improvement
+
+## How Has This Been Tested?
+
+<!-- Describe the tests you ran and/or how you verified your changes work -->
+
+- [ ] Tested locally with Docker
+- [ ] Tested locally with development setup
+- [ ] Added new unit tests
+- [ ] Existing tests pass (`uv run pytest`)
+- [ ] Manual testing performed (describe below)
+
+**Test Details:**
+<!-- Describe your testing approach -->
+
+## Design Alignment
+
+<!-- This section helps ensure your PR aligns with our project vision -->
+
+**Which design principles does this PR support?** (See [DESIGN_PRINCIPLES.md](../DESIGN_PRINCIPLES.md))
+
+- [ ] Privacy First
+- [ ] Simplicity Over Features
+- [ ] API-First Architecture
+- [ ] Multi-Provider Flexibility
+- [ ] Extensibility Through Standards
+- [ ] Async-First for Performance
+
+**Explanation:**
+<!-- Brief explanation of how your changes align with these principles -->
+
+## Checklist
+
+<!-- Mark completed items with an "x" -->
+
+### Code Quality
+- [ ] My code follows PEP 8 style guidelines (Python)
+- [ ] My code follows TypeScript best practices (Frontend)
+- [ ] I have added type hints to my code (Python)
+- [ ] I have added JSDoc comments where appropriate (TypeScript)
+- [ ] I have performed a self-review of my code
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] My changes generate no new warnings or errors
+
+### Testing
+- [ ] I have added tests that prove my fix is effective or that my feature works
+- [ ] New and existing unit tests pass locally with my changes
+- [ ] I ran linting: `make ruff` or `ruff check . --fix`
+- [ ] I ran type checking: `make lint` or `uv run python -m mypy .`
+
+### Documentation
+- [ ] I have updated the relevant documentation in `/docs` (if applicable)
+- [ ] I have added/updated docstrings for new/modified functions
+- [ ] I have updated the API documentation (if API changes were made)
+- [ ] I have added comments to complex logic
+
+### Database Changes
+- [ ] I have created migration scripts for any database schema changes (in `/migrations`)
+- [ ] Migration includes both up and down scripts
+- [ ] Migration has been tested locally
+
+### Breaking Changes
+- [ ] This PR includes breaking changes
+- [ ] I have documented the migration path for users
+- [ ] I have updated MIGRATION.md (if applicable)
+
+## Screenshots (if applicable)
+
+<!-- Add screenshots for UI changes -->
+
+## Additional Context
+
+<!-- Add any other context about the PR here -->
+
+## Pre-Submission Verification
+
+Before submitting, please verify:
+
+- [ ] I have read [CONTRIBUTING.md](../CONTRIBUTING.md)
+- [ ] I have read [DESIGN_PRINCIPLES.md](../DESIGN_PRINCIPLES.md)
+- [ ] This PR addresses an approved issue that was assigned to me
+- [ ] I have not included unrelated changes in this PR
+- [ ] My PR title follows conventional commits format (e.g., "feat: add user authentication")
+
+---
+
+**Thank you for contributing to Open Notebook!** 🎉

.github/workflows/build-and-release.yml

@@ -0,0 +1,298 @@
+name: Build and Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      push_latest:
+        description: 'Also push v1-latest tags'
+        required: true
+        default: false
+        type: boolean
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+  packages: write
+
+env:
+  GHCR_IMAGE: ghcr.io/lfnovo/open-notebook
+  DOCKERHUB_IMAGE: lfnovo/open_notebook
+
+jobs:
+  extract-version:
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+      has_dockerhub_secrets: ${{ steps.check.outputs.has_dockerhub_secrets }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Extract version from pyproject.toml
+        id: version
+        run: |
+          VERSION=$(grep -m1 '^version = ' pyproject.toml | cut -d'"' -f2)
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo "Extracted version: $VERSION"
+
+      - name: Check for Docker Hub credentials
+        id: check
+        env:
+          SECRET_DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          SECRET_DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        run: |
+          if [[ -n ""$SECRET_DOCKER_USERNAME"" && -n ""$SECRET_DOCKER_PASSWORD"" ]]; then
+            echo "has_dockerhub_secrets=true" >> $GITHUB_OUTPUT
+            echo "Docker Hub credentials available"
+          else
+            echo "has_dockerhub_secrets=false" >> $GITHUB_OUTPUT
+            echo "Docker Hub credentials not available - will only push to GHCR"
+          fi
+
+  build-regular:
+    needs: extract-version
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Free up disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo docker image prune --all --force
+          df -h
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Login to Docker Hub
+        if: needs.extract-version.outputs.has_dockerhub_secrets == 'true'
+        uses: docker/login-action@v4
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Cache Docker layers
+        uses: actions/cache@v5
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-regular-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-regular-
+
+      - name: Prepare Docker tags for regular build
+        id: tags-regular
+        env:
+          ENV_GHCR_IMAGE: ${{ env.GHCR_IMAGE }}
+          GITHUB_EVENT_INPUTS_PUSH_LATEST: ${{ github.event.inputs.push_latest }}
+          GITHUB_EVENT_NAME: ${{ github.event_name }}
+          GITHUB_EVENT_RELEASE_PRERELEASE: ${{ github.event.release.prerelease }}
+          ENV_DOCKERHUB_IMAGE: ${{ env.DOCKERHUB_IMAGE }}
+        run: |
+          TAGS=""$ENV_GHCR_IMAGE":${{ needs.extract-version.outputs.version }}"
+
+          # Determine if we should push latest tags
+          PUSH_LATEST=""$GITHUB_EVENT_INPUTS_PUSH_LATEST""
+          if [[ -z "$PUSH_LATEST" ]]; then
+            PUSH_LATEST="false"
+          fi
+
+          # Add GHCR latest tag if requested or for non-prerelease releases
+          if [[ "$PUSH_LATEST" == "true" ]] || [[ ""$GITHUB_EVENT_NAME"" == "release" && ""$GITHUB_EVENT_RELEASE_PRERELEASE"" != "true" ]]; then
+            TAGS="${TAGS},"$ENV_GHCR_IMAGE":v1-latest"
+          fi
+
+          # Add Docker Hub tags if credentials available
+          if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then
+            TAGS="${TAGS},"$ENV_DOCKERHUB_IMAGE":${{ needs.extract-version.outputs.version }}"
+
+            if [[ "$PUSH_LATEST" == "true" ]] || [[ ""$GITHUB_EVENT_NAME"" == "release" && ""$GITHUB_EVENT_RELEASE_PRERELEASE"" != "true" ]]; then
+              TAGS="${TAGS},"$ENV_DOCKERHUB_IMAGE":v1-latest"
+            fi
+          fi
+
+          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
+          echo "Generated tags: ${TAGS}"
+
+      - name: Build and push regular image
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.tags-regular.outputs.tags }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
+
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+  build-single:
+    needs: extract-version
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Free up disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo docker image prune --all --force
+          df -h
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Login to Docker Hub
+        if: needs.extract-version.outputs.has_dockerhub_secrets == 'true'
+        uses: docker/login-action@v4
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Cache Docker layers
+        uses: actions/cache@v5
+        with:
+          path: /tmp/.buildx-cache-single
+          key: ${{ runner.os }}-buildx-single-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-single-
+
+      - name: Prepare Docker tags for single build
+        id: tags-single
+        env:
+          ENV_GHCR_IMAGE: ${{ env.GHCR_IMAGE }}
+          GITHUB_EVENT_INPUTS_PUSH_LATEST: ${{ github.event.inputs.push_latest }}
+          GITHUB_EVENT_NAME: ${{ github.event_name }}
+          GITHUB_EVENT_RELEASE_PRERELEASE: ${{ github.event.release.prerelease }}
+          ENV_DOCKERHUB_IMAGE: ${{ env.DOCKERHUB_IMAGE }}
+        run: |
+          TAGS=""$ENV_GHCR_IMAGE":${{ needs.extract-version.outputs.version }}-single"
+
+          # Determine if we should push latest tags
+          PUSH_LATEST=""$GITHUB_EVENT_INPUTS_PUSH_LATEST""
+          if [[ -z "$PUSH_LATEST" ]]; then
+            PUSH_LATEST="false"
+          fi
+
+          # Add GHCR latest tag if requested or for non-prerelease releases
+          if [[ "$PUSH_LATEST" == "true" ]] || [[ ""$GITHUB_EVENT_NAME"" == "release" && ""$GITHUB_EVENT_RELEASE_PRERELEASE"" != "true" ]]; then
+            TAGS="${TAGS},"$ENV_GHCR_IMAGE":v1-latest-single"
+          fi
+
+          # Add Docker Hub tags if credentials available
+          if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then
+            TAGS="${TAGS},"$ENV_DOCKERHUB_IMAGE":${{ needs.extract-version.outputs.version }}-single"
+
+            if [[ "$PUSH_LATEST" == "true" ]] || [[ ""$GITHUB_EVENT_NAME"" == "release" && ""$GITHUB_EVENT_RELEASE_PRERELEASE"" != "true" ]]; then
+              TAGS="${TAGS},"$ENV_DOCKERHUB_IMAGE":v1-latest-single"
+            fi
+          fi
+
+          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
+          echo "Generated tags: ${TAGS}"
+
+      - name: Build and push single-container image
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: ./Dockerfile.single
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.tags-single.outputs.tags }}
+          cache-from: type=local,src=/tmp/.buildx-cache-single
+          cache-to: type=local,dest=/tmp/.buildx-cache-single-new,mode=max
+
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache-single
+          mv /tmp/.buildx-cache-single-new /tmp/.buildx-cache-single
+
+  summary:
+    needs: [extract-version, build-regular, build-single]
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Build Summary
+        env:
+          GITHUB_EVENT_INPUTS_PUSH_LATEST_____FALSE_: ${{ github.event.inputs.push_latest || 'false' }}
+          ENV_GHCR_IMAGE: ${{ env.GHCR_IMAGE }}
+          ENV_DOCKERHUB_IMAGE: ${{ env.DOCKERHUB_IMAGE }}
+          GITHUB_EVENT_INPUTS_PUSH_LATEST: ${{ github.event.inputs.push_latest }}
+        run: |
+          echo "## Build Summary" >> $GITHUB_STEP_SUMMARY
+          echo "**Version:** ${{ needs.extract-version.outputs.version }}" >> $GITHUB_STEP_SUMMARY
+          echo "**Push v1-Latest:** "$GITHUB_EVENT_INPUTS_PUSH_LATEST_____FALSE_"" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### Registries:" >> $GITHUB_STEP_SUMMARY
+          echo "✅ **GHCR:** \`"$ENV_GHCR_IMAGE"\`" >> $GITHUB_STEP_SUMMARY
+          if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then
+            echo "✅ **Docker Hub:** \`"$ENV_DOCKERHUB_IMAGE"\`" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "⏭️ **Docker Hub:** Skipped (credentials not configured)" >> $GITHUB_STEP_SUMMARY
+          fi
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### Images Built:" >> $GITHUB_STEP_SUMMARY
+
+          if [[ "${{ needs.build-regular.result }}" == "success" ]]; then
+            echo "✅ **Regular (GHCR):** \`"$ENV_GHCR_IMAGE":${{ needs.extract-version.outputs.version }}\`" >> $GITHUB_STEP_SUMMARY
+            if [[ ""$GITHUB_EVENT_INPUTS_PUSH_LATEST"" == "true" ]]; then
+              echo "✅ **Regular v1-Latest (GHCR):** \`"$ENV_GHCR_IMAGE":v1-latest\`" >> $GITHUB_STEP_SUMMARY
+            fi
+            if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then
+              echo "✅ **Regular (Docker Hub):** \`"$ENV_DOCKERHUB_IMAGE":${{ needs.extract-version.outputs.version }}\`" >> $GITHUB_STEP_SUMMARY
+              if [[ ""$GITHUB_EVENT_INPUTS_PUSH_LATEST"" == "true" ]]; then
+                echo "✅ **Regular v1-Latest (Docker Hub):** \`"$ENV_DOCKERHUB_IMAGE":v1-latest\`" >> $GITHUB_STEP_SUMMARY
+              fi
+            fi
+          elif [[ "${{ needs.build-regular.result }}" == "skipped" ]]; then
+            echo "⏭️ **Regular:** Skipped" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "❌ **Regular:** Failed" >> $GITHUB_STEP_SUMMARY
+          fi
+
+          if [[ "${{ needs.build-single.result }}" == "success" ]]; then
+            echo "✅ **Single (GHCR):** \`"$ENV_GHCR_IMAGE":${{ needs.extract-version.outputs.version }}-single\`" >> $GITHUB_STEP_SUMMARY
+            if [[ ""$GITHUB_EVENT_INPUTS_PUSH_LATEST"" == "true" ]]; then
+              echo "✅ **Single v1-Latest (GHCR):** \`"$ENV_GHCR_IMAGE":v1-latest-single\`" >> $GITHUB_STEP_SUMMARY
+            fi
+            if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then
+              echo "✅ **Single (Docker Hub):** \`"$ENV_DOCKERHUB_IMAGE":${{ needs.extract-version.outputs.version }}-single\`" >> $GITHUB_STEP_SUMMARY
+              if [[ ""$GITHUB_EVENT_INPUTS_PUSH_LATEST"" == "true" ]]; then
+                echo "✅ **Single v1-Latest (Docker Hub):** \`"$ENV_DOCKERHUB_IMAGE":v1-latest-single\`" >> $GITHUB_STEP_SUMMARY
+              fi
+            fi
+          elif [[ "${{ needs.build-single.result }}" == "skipped" ]]; then
+            echo "⏭️ **Single:** Skipped" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "❌ **Single:** Failed" >> $GITHUB_STEP_SUMMARY
+          fi
+
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### Platforms:" >> $GITHUB_STEP_SUMMARY
+          echo "- linux/amd64" >> $GITHUB_STEP_SUMMARY
+          echo "- linux/arm64" >> $GITHUB_STEP_SUMMARY

.github/workflows/build-dev.yml

@@ -0,0 +1,273 @@
+name: Development Build
+
+on:
+  pull_request:
+    branches: [ main ]
+  push:
+    branches: [ main ]
+    paths-ignore:
+      - '**.md'
+      - 'docs/**'
+      - 'notebooks/**'
+      - '.github/workflows/claude*.yml'
+  workflow_dispatch:
+    inputs:
+      platform:
+        description: 'Platform to build'
+        required: true
+        default: 'linux/amd64'
+        type: choice
+        options:
+          - linux/amd64
+          - linux/arm64
+          - linux/amd64,linux/arm64
+
+permissions:
+  contents: read
+  packages: write
+
+env:
+  GHCR_IMAGE: ghcr.io/lfnovo/open-notebook
+  DOCKERHUB_IMAGE: lfnovo/open_notebook
+
+jobs:
+  extract-version:
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+      has_dockerhub_secrets: ${{ steps.check.outputs.has_dockerhub_secrets }}
+      is_push_to_main: ${{ steps.check.outputs.is_push_to_main }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Extract version from pyproject.toml
+        id: version
+        run: |
+          VERSION=$(grep -m1 '^version = ' pyproject.toml | cut -d'"' -f2)
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo "Extracted version: $VERSION"
+
+      - name: Check environment
+        id: check
+        env:
+          SECRET_DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          SECRET_DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        run: |
+          # Check for Docker Hub credentials
+          if [[ -n "$SECRET_DOCKER_USERNAME" && -n "$SECRET_DOCKER_PASSWORD" ]]; then
+            echo "has_dockerhub_secrets=true" >> $GITHUB_OUTPUT
+            echo "Docker Hub credentials available"
+          else
+            echo "has_dockerhub_secrets=false" >> $GITHUB_OUTPUT
+            echo "Docker Hub credentials not available"
+          fi
+
+          # Check if this is a push to main (not a PR)
+          if [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/main" ]]; then
+            echo "is_push_to_main=true" >> $GITHUB_OUTPUT
+            echo "This is a push to main - will publish v1-dev tags"
+          else
+            echo "is_push_to_main=false" >> $GITHUB_OUTPUT
+            echo "This is a PR or manual run - test build only"
+          fi
+
+  build-regular:
+    needs: extract-version
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Free up disk space
+        if: needs.extract-version.outputs.is_push_to_main == 'true'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo docker image prune --all --force
+          df -h
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Login to GitHub Container Registry
+        if: needs.extract-version.outputs.is_push_to_main == 'true'
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Login to Docker Hub
+        if: needs.extract-version.outputs.is_push_to_main == 'true' && needs.extract-version.outputs.has_dockerhub_secrets == 'true'
+        uses: docker/login-action@v4
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Cache Docker layers
+        uses: actions/cache@v5
+        with:
+          path: /tmp/.buildx-cache-dev
+          key: ${{ runner.os }}-buildx-dev-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-dev-
+
+      - name: Prepare Docker tags
+        id: tags
+        run: |
+          if [[ "${{ needs.extract-version.outputs.is_push_to_main }}" == "true" ]]; then
+            # Push to main: build and push v1-dev tags
+            TAGS="${{ env.GHCR_IMAGE }}:v1-dev"
+            if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then
+              TAGS="${TAGS},${{ env.DOCKERHUB_IMAGE }}:v1-dev"
+            fi
+            echo "tags=${TAGS}" >> $GITHUB_OUTPUT
+            echo "push=true" >> $GITHUB_OUTPUT
+            echo "platforms=linux/amd64,linux/arm64" >> $GITHUB_OUTPUT
+          else
+            # PR or manual: test build only
+            echo "tags=${{ env.DOCKERHUB_IMAGE }}:${{ needs.extract-version.outputs.version }}-dev" >> $GITHUB_OUTPUT
+            echo "push=false" >> $GITHUB_OUTPUT
+            echo "platforms=${{ github.event.inputs.platform || 'linux/amd64' }}" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Build and push regular image
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: ./Dockerfile
+          platforms: ${{ steps.tags.outputs.platforms }}
+          push: ${{ steps.tags.outputs.push }}
+          tags: ${{ steps.tags.outputs.tags }}
+          cache-from: type=local,src=/tmp/.buildx-cache-dev
+          cache-to: type=local,dest=/tmp/.buildx-cache-dev-new,mode=max
+
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache-dev
+          mv /tmp/.buildx-cache-dev-new /tmp/.buildx-cache-dev
+
+  build-single:
+    needs: extract-version
+    # Only build single image on push to main
+    if: needs.extract-version.outputs.is_push_to_main == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Free up disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo docker image prune --all --force
+          df -h
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Login to Docker Hub
+        if: needs.extract-version.outputs.has_dockerhub_secrets == 'true'
+        uses: docker/login-action@v4
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Cache Docker layers
+        uses: actions/cache@v5
+        with:
+          path: /tmp/.buildx-cache-dev-single
+          key: ${{ runner.os }}-buildx-dev-single-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-dev-single-
+
+      - name: Prepare Docker tags
+        id: tags
+        run: |
+          TAGS="${{ env.GHCR_IMAGE }}:v1-dev-single"
+          if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then
+            TAGS="${TAGS},${{ env.DOCKERHUB_IMAGE }}:v1-dev-single"
+          fi
+          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
+
+      - name: Build and push single-container image
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: ./Dockerfile.single
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.tags.outputs.tags }}
+          cache-from: type=local,src=/tmp/.buildx-cache-dev-single
+          cache-to: type=local,dest=/tmp/.buildx-cache-dev-single-new,mode=max
+
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache-dev-single
+          mv /tmp/.buildx-cache-dev-single-new /tmp/.buildx-cache-dev-single
+
+  summary:
+    needs: [extract-version, build-regular, build-single]
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Development Build Summary
+        run: |
+          echo "## Development Build Summary" >> $GITHUB_STEP_SUMMARY
+          echo "**Version:** ${{ needs.extract-version.outputs.version }}" >> $GITHUB_STEP_SUMMARY
+          echo "**Event:** ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY
+          echo "**Push to Main:** ${{ needs.extract-version.outputs.is_push_to_main }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+          if [[ "${{ needs.extract-version.outputs.is_push_to_main }}" == "true" ]]; then
+            echo "### Published Tags:" >> $GITHUB_STEP_SUMMARY
+
+            if [[ "${{ needs.build-regular.result }}" == "success" ]]; then
+              echo "✅ **Regular:** \`${{ env.GHCR_IMAGE }}:v1-dev\`" >> $GITHUB_STEP_SUMMARY
+              if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then
+                echo "✅ **Regular (Docker Hub):** \`${{ env.DOCKERHUB_IMAGE }}:v1-dev\`" >> $GITHUB_STEP_SUMMARY
+              fi
+            else
+              echo "❌ **Regular:** Build failed" >> $GITHUB_STEP_SUMMARY
+            fi
+
+            if [[ "${{ needs.build-single.result }}" == "success" ]]; then
+              echo "✅ **Single:** \`${{ env.GHCR_IMAGE }}:v1-dev-single\`" >> $GITHUB_STEP_SUMMARY
+              if [[ "${{ needs.extract-version.outputs.has_dockerhub_secrets }}" == "true" ]]; then
+                echo "✅ **Single (Docker Hub):** \`${{ env.DOCKERHUB_IMAGE }}:v1-dev-single\`" >> $GITHUB_STEP_SUMMARY
+              fi
+            elif [[ "${{ needs.build-single.result }}" == "skipped" ]]; then
+              echo "⏭️ **Single:** Skipped" >> $GITHUB_STEP_SUMMARY
+            else
+              echo "❌ **Single:** Build failed" >> $GITHUB_STEP_SUMMARY
+            fi
+
+            echo "" >> $GITHUB_STEP_SUMMARY
+            echo "### Platforms:" >> $GITHUB_STEP_SUMMARY
+            echo "- linux/amd64" >> $GITHUB_STEP_SUMMARY
+            echo "- linux/arm64" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "### Test Build Results:" >> $GITHUB_STEP_SUMMARY
+            if [[ "${{ needs.build-regular.result }}" == "success" ]]; then
+              echo "✅ **Dockerfile:** Build successful" >> $GITHUB_STEP_SUMMARY
+            else
+              echo "❌ **Dockerfile:** Build failed" >> $GITHUB_STEP_SUMMARY
+            fi
+            echo "" >> $GITHUB_STEP_SUMMARY
+            echo "### Notes:" >> $GITHUB_STEP_SUMMARY
+            echo "- This is a test build (no images pushed to registry)" >> $GITHUB_STEP_SUMMARY
+            echo "- Merge to main to publish \`v1-dev\` tags" >> $GITHUB_STEP_SUMMARY
+            echo "- For stable releases, use the 'Build and Release' workflow" >> $GITHUB_STEP_SUMMARY
+          fi

.github/workflows/test.yml

@@ -0,0 +1,59 @@
+name: Tests
+
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'docs/**'
+      - '.github/workflows/claude*.yml'
+
+permissions:
+  contents: read
+
+jobs:
+  backend:
+    name: Backend Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v8.1.0
+        with:
+          enable-cache: true
+
+      - name: Set up Python
+        run: uv python install
+
+      - name: Install dependencies
+        run: uv sync
+
+      - name: Run tests
+        run: uv run pytest tests/ -v
+
+  frontend:
+    name: Frontend Tests
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: frontend
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: 22
+          cache: npm
+          cache-dependency-path: frontend/package-lock.json
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run tests
+        run: npm test

.gitignore

@@ -0,0 +1,144 @@
+.env
+prompts/patterns/user/
+/notebooks/
+data/
+.uploads/
+sqlite-db/
+surreal-data/
+docker.env
+notebook_data/
+# Python-specific
+*.py[cod]
+__pycache__/
+*.so
+todo.md
+temp/
+google-credentials.json
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+/lib/
+/lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# PyCharm
+.idea/
+
+# VS Code
+.vscode/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# macOS
+.DS_Store
+
+# Windows
+Thumbs.db
+ehthumbs.db
+desktop.ini
+
+# Linux
+*~
+
+# Log files
+*.log
+
+# Database files
+*.db
+*.sqlite3
+
+.quarentena
+
+claude-logs/
+.claude/sessions
+**/claude-logs
+
+
+docs/custom_gpt
+doc_exports/
+
+specs/
+.claude
+.sisyphus
+
+.playwright-mcp/
+
+
+
+*.local.yml
+**/*.local.md
+.harness/
+
+.mcp.json

.python-version

@@ -0,0 +1 @@
+3.12

.worktreeinclude

@@ -0,0 +1,5 @@
+.env
+.env.local
+.env.*
+**/.claude/settings.local.json
+CLAUDE.local.md

CHANGELOG.md

@@ -0,0 +1,367 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [1.9.0] - 2026-06-02
+
+### Added
+- **New audio providers**, surfacing the capabilities added in Esperanto 2.21–2.22:
+  - **Mistral Voxtral** speech-to-text (`voxtral-*-latest`) and text-to-speech (`voxtral-mini-tts`), reusing the existing Mistral credential (#827)
+  - **Deepgram** text-to-speech (Aura voice catalog) as a new provider (`DEEPGRAM_API_KEY`) (#827)
+  - **xAI** text-to-speech (#827)
+  - **Google** speech-to-text & text-to-speech, **Vertex** text-to-speech, and **ElevenLabs** speech-to-text (Scribe), completing the audio provider matrix (#828)
+- Optional per-credential **`num_ctx`** (context window) override for Ollama models, configurable in Settings → API Keys and translated across all 13 locales (#825)
+- `OPEN_NOTEBOOK_EMBEDDING_BATCH_SIZE` environment variable to override the embedding batch size; default remains `50`. Helps with CPU-only local embedding and stricter OpenAI-compatible endpoints (#735)
+- `CORS_ORIGINS` environment variable to configure the API's allowed origins (comma-separated). Default remains `*` for backward compatibility; the API now logs a startup warning prompting users to set it for production deployments. Exception responses honor the configured origins when explicitly set (#585, #597, #730)
+- `OPEN_NOTEBOOK_MIN_CHUNK_SIZE` environment variable (default: 5 tokens) to filter out degenerate tiny chunks before embedding. Set to `0` to disable.
+
+### Changed
+- Bumped **Esperanto 2.20.0 → 2.22.0**. Beyond the new audio providers above, this inherits several upstream fixes and behavior changes (see below).
+
+### Inherited from Esperanto 2.21–2.22
+- **Fixed:** OpenRouter LLM and embedding requests now send a proper JSON body (previously sent a malformed form-encoded payload).
+- **Fixed:** OpenAI-compatible endpoints (e.g. llama.cpp) that return null embeddings now raise a clear, descriptive error instead of an opaque `TypeError`.
+- **Fixed:** Streaming tool calls now return proper `ToolCall` objects across Anthropic, Google, Vertex, and Ollama.
+- **Fixed:** `base_url` trailing slashes are normalized across providers, preventing double-slash URLs (and 301 redirects) for Ollama and other self-hosted endpoints.
+- **Fixed:** Ollama "thinking" models (e.g. Qwen) now merge their reasoning content correctly.
+- **Fixed:** Model discovery honors a custom `base_url` (LiteLLM/vLLM/OpenAI-compatible proxies).
+- **Behavior change:** the Ollama default context window (`num_ctx`) is now **8192** (was 128000) to avoid out-of-memory errors on consumer GPUs. Raise it per-credential via the new `num_ctx` field if your hardware allows.
+- **Behavior change:** the Google embedding default model is now `gemini-embedding-001` (the previous default, `text-embedding-004`, was removed from Google's API). If you used Google embeddings with the old default, re-create the model and re-embed your content (embedding dimensions changed).
+- **Fixed:** Google TTS default model updated to a currently-working preview model.
+
+### Fixed
+- URL source embedding no longer crashes with `TypeError: float() argument must be a string or a real number, not 'NoneType'` when header-based splitters emit single-character fragments from complex HTML pages (e.g. Wikipedia, Project Gutenberg). Such chunks are now filtered before being sent to the embedding provider (#764)
+- Language toggle now uses `t('common.german')` instead of a hardcoded "Deutsch" label, matching the pattern used by every other language entry (follow-up to #794)
+- Speech-to-text model connection tests now transcribe a short bundled speech clip instead of silence, so a passing test returns real text instead of a blank transcription (#838)
+
+## [1.8.5] - 2026-04-14
+
+### Changed
+- Embedding chunking is now token-based instead of character-based, improving chunk sizing consistency for CJK and mixed-language content (#542, #749)
+- `OPEN_NOTEBOOK_CHUNK_SIZE` and `OPEN_NOTEBOOK_CHUNK_OVERLAP` semantics changed from characters to tokens; default reduced from 1200 characters to 400 tokens to stay safely below the 512-token ceiling of BERT-family embedders (e.g. mxbai-embed-large) after accounting for tokenizer mismatch and splitter overshoot. Existing stored embeddings are unaffected; only new ingestions use the new chunking.
+
+### Fixed
+- Credentials endpoint no longer crashes (500) when encryption key doesn't match stored credentials (#740)
+- Broken credentials are now shown with a decryption warning and can still be deleted
+- DELETE endpoint for broken credentials supports model migration (`migrate_to` parameter)
+
+## [1.8.4] - 2026-04-09
+
+### Security
+- Fix Remote Code Execution (RCE) via Jinja2 Server-Side Template Injection in transformations (CVSS 9.2 Critical)
+- Fix arbitrary file write via path traversal in file upload (CVSS 7.0 High)
+- Fix arbitrary file read via Local File Inclusion in source creation (CVSS 8.2 High)
+
+### Dependencies
+- Bump ai-prompter to >=0.4.0 (uses Jinja2 SandboxedEnvironment to prevent SSTI)
+
+## [1.8.3] - 2026-04-07
+
+### Security
+- Fix SurrealDB injection via unsanitized `order_by` query parameter in `GET /api/notebooks` (CVSS 8.7 High)
+- Add allowlist validation for sorting parameters in notebooks endpoint
+- Replace f-string query interpolation with parameterized `$variable` binding in source chat and migration queries
+- Add defensive validation in `get_all()` base method to prevent injection via `order_by` parameter
+
+## [1.8.2] - 2026-04-06
+
+### Added
+- DashScope (Qwen) and MiniMax provider support via Esperanto v2.20.0 (#725)
+- Source list auto-refresh after adding a new source via URL, file upload, or text (#721)
+
+### Fixed
+- Source asset persistence — failed sources now persist their asset (URL/file path), making them identifiable and retryable (#722)
+- Source title preservation — user-set custom titles are no longer overwritten after background processing (#722)
+- Credential cascade delete — deleting a credential now removes linked models instead of returning a 409 error (#722)
+- Podcast directory names — uses UUID for episode directories, fixing filesystem errors with special characters (#666)
+- Tiktoken offline handling — API no longer crashes in air-gapped environments (#622)
+- SurrealDB healthcheck — removed incompatible healthcheck from Docker Compose (#656)
+- Esperanto embedding fixes — base_url/api_key config issues across multiple embedding providers (#664, #665)
+
+### Docs
+- Deprecated single-container Docker image in favor of Docker Compose (#723)
+
+### Dependencies
+- Bump esperanto to >=2.20.0
+
+## [1.8.1] - 2026-03-10
+
+### Added
+- i18n support for Bengali (bn-IN) (#643)
+- Podcast language support via podcast-creator 0.12.0 (#645)
+- Upgrade default Azure API version for model testing and fetching (#638)
+
+### Fixed
+- Tiktoken network errors in offline/air-gapped Docker deployments — pre-downloads encoding at build time (#264, #622)
+- SurrealDB getting stuck (#656)
+
+### Dependencies
+- Bump esperanto to 2.19.5 (#657)
+- Bump langgraph from 1.0.6 to 1.0.10rc1 (#658)
+- Bump authlib from 1.6.6 to 1.6.7 (#649)
+- Bump lxml-html-clean from 0.4.3 to 0.4.4 (#646)
+- Bump rollup from 4.55.1 to 4.59.0 (#635)
+- Bump minimatch in frontend (#634)
+- Bump tar from 7.5.9 to 7.5.11 (#650, #659)
+
+## [1.7.4] - 2026-02-18
+
+### Fixed
+- Embedding large documents (3MB+) fails with 413 Payload Too Large (#594)
+- `generate_embeddings()` now batches texts in groups of 50 with per-batch retry, preventing provider payload limits from being exceeded
+- 413 errors now classified with user-friendly message in error classifier
+- Misleading "Created 0 embedded chunks" log in `process_source_command` — embedding is fire-and-forget, so the count was always 0; now logs "embedding submitted" instead
+
+## [1.7.3] - 2026-02-17
+
+### Added
+- Retry button for failed podcast episodes in the UI (#211, #218)
+- Error details displayed on failed podcast episodes (#185, #355)
+- `POST /podcasts/episodes/{id}/retry` API endpoint for re-submitting failed episodes
+- `error_message` field in podcast episode API responses
+
+### Fixed
+- Podcast generation failures now correctly marked as "failed" instead of "completed" (#300, #335)
+- Disabled automatic retries for podcast generation to prevent duplicate episode records (#302)
+
+### Dependencies
+- Bump podcast-creator to >= 0.11.2
+- Bump esperanto to >= 2.19.4
+
+## [1.7.2] - 2026-02-16
+
+### Added
+- Error classification utility that maps LLM provider errors to user-friendly messages (#506)
+- Global exception handlers in FastAPI for all custom exception types with proper HTTP status codes
+- `getApiErrorMessage()` frontend helper that falls back to backend messages when no i18n mapping exists
+
+### Fixed
+- LLM errors (invalid API key, wrong model, rate limits) now show descriptive messages instead of "An unexpected error occurred" (#590)
+- SSE streaming error events in source chat and ask hooks were swallowed by inner JSON parse catch blocks
+- Transformation execution errors were caught and re-wrapped as generic 500s instead of using proper status codes
+- Fail fast when source content extraction returns empty instead of retrying (#589)
+- Chat input and message overflow with long unbroken strings (#588)
+- Word-wrap overflow in source cards, note editor, inline edit, note titles, and dialog content (#588)
+- Translation proxy shadowing `name` keys (#588)
+- OpenAI-compatible provider name handling via Esperanto update (#583)
+
+### Changed
+- `ValueError` replaced with `ConfigurationError` in model provisioning for proper error classification
+- `ConfigurationError` added to command retry `stop_on` lists to avoid retrying permanent config failures
+
+### Dependencies
+- Bump esperanto to 2.19.3 (#583)
+- Bump podcast-creator to 0.9.1
+
+## [1.7.1] - 2026-02-14
+
+### Added
+- French (fr-FR) language support (#581)
+- CI test workflow and improved i18n validation (#580)
+- Expose embed `command_id` in note API responses (#545)
+
+### Fixed
+- ElevenLabs TTS credential passthrough via Esperanto update (#578)
+- Handle empty/whitespace source content without retry loop (#576)
+- Increase transformation `max_tokens` and update Esperanto dep (#568)
+- Turn the embedding field into optional (#557)
+
+### Docs
+- Fix docker container names in local setup guides (#577)
+
+### Dependencies
+- Bump langchain-core from 1.2.7 to 1.2.11 (#564)
+- Bump cryptography from 46.0.3 to 46.0.5 (#563)
+
+## [1.7.0] - 2026-02-10
+
+### Added
+- **Credential-Based Provider Management** (#477)
+  - New Settings → API Keys page for managing AI provider credentials via the UI
+  - Support for 14 providers: OpenAI, Anthropic, Google, Groq, Mistral, DeepSeek, xAI, OpenRouter, Voyage AI, ElevenLabs, Ollama, Azure OpenAI, OpenAI-Compatible, and Vertex AI
+  - Secure storage of API keys in SurrealDB with field-level encryption (Fernet AES-128-CBC + HMAC-SHA256)
+  - One-click connection testing, model discovery, and model registration per credential
+  - Migration tool to import existing environment variable keys into the credential system
+  - Azure OpenAI support with service-specific endpoints (LLM, Embedding, STT, TTS)
+  - OpenAI-Compatible support with per-service URL configurations
+  - Vertex AI support with project, location, and credentials path
+  - Environment variable API keys deprecated in favor of Settings UI
+
+- **Security Enhancements**
+  - Docker secrets support via `_FILE` suffix pattern (e.g., `OPEN_NOTEBOOK_PASSWORD_FILE`)
+  - Default encryption key derived from "0p3n-N0t3b0ok" for easy setup (change in production!)
+  - Default password "open-notebook-change-me" for out-of-box experience (change in production!)
+  - URL validation for SSRF protection - blocks private IPs and localhost (except for Ollama which runs locally)
+  - Security warnings logged when using default credentials
+
+- HTML clipboard detection for text sources (#426)
+  - When pasting content, automatically detects HTML format (e.g., from Word, web pages)
+  - Shows info message when HTML is detected, informing user it will be converted to Markdown
+  - Preserves formatting that would be lost with plain text paste
+  - Bump content-core to 0.11.0 for HTML to Markdown conversion support
+
+- **Improved Getting Started Experience**
+  - Simplified docker-compose.yml in repository root (single official file)
+  - Added examples/ folder with ready-made configurations:
+    - `docker-compose-ollama.yml` - Local AI with Ollama
+    - `docker-compose-speaches.yml` - Local TTS/STT with Speaches
+    - `docker-compose-full-local.yml` - 100% local setup (Ollama + Speaches)
+  - Inline quick start in README (no need to navigate to docs)
+  - Cross-references between docker-compose examples and documentation
+  - .env.example template with all configuration options
+
+### Fixed
+- Azure form race condition: all configuration now saved in single atomic request
+- Migration API "error error" display: added proper MigrationResult model with message field
+- Connection tester for Ollama providers: improved error handling and URL validation
+- SqliteSaver async compatibility issues in chat system (#509, #525, #538)
+- Re-embedding failures with empty content (#513, #515)
+- Deletion cascade for notes and sources (#77)
+- YouTube content availability issues (#494)
+- Large document embedding errors (#489)
+
+### Security
+- API keys are encrypted at rest using Fernet symmetric encryption
+- Keys are never returned to the frontend, only configuration status
+- SSRF protection prevents internal network access via URL validation
+
+### Docs
+- Complete documentation update for credential-based system across 25 files
+- All quick-start, installation, and configuration guides now use Settings UI workflow
+- Environment variable API key instructions moved to deprecated/legacy sections
+- Fixed broken links in installation docs
+- Added comprehensive examples/ folder with documented docker-compose configurations
+- Updated local-tts.md and local-stt.md with links to ready-made examples
+
+### Internationalization
+- Added Russian (ru-RU) language support (#524)
+- Added Italian (it-IT) language support (#508)
+
+## [1.6.2] - 2026-01-24
+
+### Fixed
+- Connection error with llama.cpp and OpenAI-compatible providers (#465)
+  - Bump Esperanto to 2.17.2 which fixes LangChain connection errors caused by garbage collection
+
+## [1.6.1] - 2026-01-22
+
+### Fixed
+- "Failed to send message" error with unhelpful logs when chat model is not configured (#358)
+  - Added detailed error logging with model selection context and full traceback
+  - Improved error messages to guide users to Settings → Models
+  - Added warnings when default models are not configured
+
+### Docs
+- Ollama troubleshooting: Added "Model Name Configuration" section emphasizing exact model names from `ollama list`
+- Added troubleshooting entry for "Failed to send message" error with step-by-step solutions
+- Updated AI Chat Issues documentation with model configuration guidance
+
+
+## [1.6.0] - 2026-01-21
+
+### Added
+- Content-type aware text chunking with automatic HTML, Markdown, and plain text detection (#350, #142)
+- Unified embedding generation with mean pooling for large content that exceeds model context limits
+- Dedicated embedding commands: `embed_note`, `embed_insight`, `embed_source`
+- New utility modules: `chunking.py` and `embedding.py` in `open_notebook/utils/`
+- Japanese (ja-JP) language support (#450)
+
+### Changed
+- Embedding is now fire-and-forget: domain models submit embedding commands asynchronously after save
+- `rebuild_embeddings_command` now delegates to individual embed_* commands instead of inline processing
+- Chunk size reduced to 1500 characters for better compatibility with Ollama embedding models
+- Bump Esperanto to 2.16 for increased Ollama context window support
+
+### Removed
+- Legacy embedding commands: `embed_single_item_command`, `embed_chunk_command`, `vectorize_source_command`
+- `needs_embedding()` and `get_embedding_content()` methods from domain models
+- `split_text()` function from text_utils (replaced by `chunk_text()` in chunking module)
+
+### Fixed
+- Embedding failures when content exceeds model context limits (#350, #142)
+- Empty note titles when saving from chat (clean thinking tags from prompt graph output)
+- Orphaned embedding/insight records when deleting sources (cascade delete)
+- Search results crash with null parent_id (defensive frontend check)
+- Database migration 10 cleans up existing orphaned records
+
+## [1.5.2] - 2026-01-15
+
+### Performance
+- Improved source listing speed by 20-30x (#436, closes #351)
+  - Added database indexes on `source` field for `source_insight` and `source_embedding` tables
+  - Use SurrealDB `FETCH` clause for command status instead of N async calls
+
+## [1.5.1] - 2026-01-15
+
+### Fixed
+- Podcast dialog infinite loop error caused by excessive translation Proxy accesses in loops
+- Podcast dialog UI freezing when typing episode name or additional instructions
+- Removed incorrect translation keys for user-defined episode profiles (user content should not be translated)
+
+## [1.5.0] - 2026-01-15
+
+### Added
+- Internationalization (i18n) support with Chinese (Simplified and Traditional) translations (#371, closes #344, #349, #360)
+- Frontend test infrastructure with Vitest (#371)
+- Language toggle component for switching UI language (#371)
+- Date localization using date-fns locales (#371)
+- Error message translation system (#371)
+
+### Fixed
+- Accessibility improvements: added missing `id`, `name`, and `autoComplete` attributes to form inputs (#371)
+- Added `DialogDescription` to dialogs for Radix UI accessibility compliance (#371)
+- Fixed "Collapsible is changing from uncontrolled to controlled" warning in SettingsForm (#371)
+- Fixed lint command for Next.js 16 compatibility (`eslint` instead of `next lint`)
+
+### Changed
+- Dockerfile optimizations: better layer caching, `--no-install-recommends` for smaller images (#371)
+- Dockerfile.single refactored into 3 separate build stages for better caching (#371)
+
+## [1.4.0] - 2026-01-14
+
+### Added
+- CTA button to empty state notebook list for better onboarding (#408)
+- Offline deployment support for Docker containers (#414)
+
+### Fixed
+- Large file uploads (>10MB) by upgrading to Next.js 16 (#423)
+- Orphaned uploaded files when sources are removed (#421)
+- Broken documentation links to ai-providers.md (#419)
+- ZIP support indication removed from UI (#418)
+- Duplicate Claude Code workflow runs on PRs (#417)
+- Claude Code review workflow now runs on PRs from forks (#416)
+
+### Changed
+- Upgraded Next.js from 15.4.10 to 16.1.1 (#423)
+- Upgraded React from 19.1.0 to 19.2.3 (#423)
+- Renamed `middleware.ts` to `proxy.ts` for Next.js 16 compatibility (#423)
+
+### Dependencies
+- next: 15.4.10 → 16.1.1
+- react: 19.1.0 → 19.2.3
+- react-dom: 19.1.0 → 19.2.3
+
+## [1.2.4] - 2025-12-14
+
+### Added
+- Infinite scroll for notebook sources - no more 50 source limit (#325)
+- Markdown table rendering in chat responses, search results, and insights (#325)
+
+### Fixed
+- Timeout errors with Ollama and local LLMs - increased to 10 minutes (#325)
+- "Unable to Connect to API Server" on Docker startup - frontend now waits for API health check (#325, #315)
+- SSL issues with langchain (#274)
+- Query key consistency for source mutations to properly refresh infinite scroll (#325)
+- Docker compose start-all flow (#323)
+
+### Changed
+- Timeout configuration now uses granular httpx.Timeout (short connect, long read) (#325)
+
+### Dependencies
+- Updated next.js to 15.4.10
+- Updated httpx to >=0.27.0 for SSL fix

CLAUDE.md

@@ -0,0 +1,218 @@
+# Open Notebook - Root CLAUDE.md
+
+This file provides architectural guidance for contributors working on Open Notebook at the project level.
+
+## Project Overview
+
+**Open Notebook** is an open-source, privacy-focused alternative to Google's Notebook LM. It's an AI-powered research assistant enabling users to upload multi-modal content (PDFs, audio, video, web pages), generate intelligent notes, search semantically, chat with AI models, and produce professional podcasts—all with complete control over data and choice of AI providers.
+
+**Key Values**: Privacy-first, multi-provider AI support, fully self-hosted option, open-source transparency.
+
+---
+
+## Three-Tier Architecture
+
+```
+┌─────────────────────────────────────────────────────────┐
+│              Frontend (React/Next.js)                    │
+│              frontend/ @ port 3000                       │
+├─────────────────────────────────────────────────────────┤
+│ - Notebooks, sources, notes, chat, podcasts, search UI  │
+│ - Zustand state management, TanStack Query (React Query)│
+│ - Shadcn/ui component library with Tailwind CSS         │
+└────────────────────────┬────────────────────────────────┘
+                         │ HTTP REST
+┌────────────────────────▼────────────────────────────────┐
+│              API (FastAPI)                              │
+│              api/ @ port 5055                           │
+├─────────────────────────────────────────────────────────┤
+│ - REST endpoints for notebooks, sources, notes, chat    │
+│ - LangGraph workflow orchestration                      │
+│ - Job queue for async operations (podcasts)             │
+│ - Multi-provider AI provisioning via Esperanto          │
+└────────────────────────┬────────────────────────────────┘
+                         │ SurrealQL
+┌────────────────────────▼────────────────────────────────┐
+│         Database (SurrealDB)                            │
+│         Graph database @ port 8000                      │
+├─────────────────────────────────────────────────────────┤
+│ - Records: Notebook, Source, Note, ChatSession, Credential│
+│ - Relationships: source-to-notebook, note-to-source     │
+│ - Vector embeddings for semantic search                 │
+└─────────────────────────────────────────────────────────┘
+```
+
+---
+
+## Useful sources
+
+User documentation is at @docs/
+
+## Tech Stack
+
+### Frontend (`frontend/`)
+- **Framework**: Next.js 16 (React 19)
+- **Language**: TypeScript
+- **State Management**: Zustand
+- **Data Fetching**: TanStack Query (React Query)
+- **Styling**: Tailwind CSS + Shadcn/ui
+- **Build Tool**: Webpack (via Next.js)
+- **i18n compatible**: All front-end changes must also consider the translation keys
+
+### API Backend (`api/` + `open_notebook/`)
+- **Framework**: FastAPI 0.104+
+- **Language**: Python 3.11+
+- **Workflows**: LangGraph state machines
+- **Database**: SurrealDB async driver
+- **AI Providers**: Esperanto library (8+ providers: OpenAI, Anthropic, Google, Groq, Ollama, Mistral, DeepSeek, xAI)
+- **Job Queue**: Surreal-Commands for async jobs (podcasts)
+- **Logging**: Loguru
+- **Validation**: Pydantic v2
+- **Testing**: Pytest
+
+### Database
+- **SurrealDB**: Graph database with built-in embedding storage and vector search
+- **Schema Migrations**: Automatic on API startup via AsyncMigrationManager
+
+### Additional Services
+- **Content Processing**: content-core library (file/URL extraction)
+- **Prompts**: AI-Prompter with Jinja2 templating
+- **Podcast Generation**: podcast-creator library
+- **Embeddings**: Multi-provider via Esperanto
+
+---
+
+## Architecture Highlights
+
+### 1. Async-First Design
+- All database queries, graph invocations, and API calls are async (await)
+- SurrealDB async driver with connection pooling
+- FastAPI handles concurrent requests efficiently
+
+### 2. LangGraph Workflows
+- **source.py**: Content ingestion (extract → embed → save)
+- **chat.py**: Conversational agent with message history
+- **ask.py**: Search + synthesis (retrieve relevant sources → LLM)
+- **transformation.py**: Custom transformations on sources
+- All use `provision_langchain_model()` for smart model selection
+
+### 3. Multi-Provider AI
+- **Esperanto library**: Unified interface to 8+ AI providers
+- **Credential system**: Individual encrypted credential records per provider; models link to credentials for direct config
+- **ModelManager**: Factory pattern with fallback logic; uses credential config when available, env vars as fallback
+- **Smart selection**: Detects large contexts, prefers long-context models
+- **Override support**: Per-request model configuration
+
+### 4. Database Schema
+- **Automatic migrations**: AsyncMigrationManager runs on API startup
+- **SurrealDB graph model**: Records with relationships and embeddings
+- **Vector search**: Built-in semantic search across all content
+- **Transactions**: Repo functions handle ACID operations
+
+### 5. Authentication
+- **Current**: Simple password middleware (insecure, dev-only)
+- **Production**: Replace with OAuth/JWT (see CONFIGURATION.md)
+
+---
+
+## Important Quirks & Gotchas
+
+### API Startup
+- **Migrations run automatically** on startup; check logs for errors
+- **Must start API before UI**: UI depends on API for all data
+- **SurrealDB must be running**: API fails without database connection
+
+### Frontend-Backend Communication
+- **Base API URL**: Configured in `.env.local` (default: http://localhost:5055)
+- **CORS enabled**: Configured in `api/main.py` (allow all origins in dev)
+- **Rate limiting**: Not built-in; add at proxy layer for production
+
+### LangGraph Workflows
+- **Blocking operations**: Chat/podcast workflows may take minutes; no timeout
+- **State persistence**: Uses SQLite checkpoint storage in `/data/sqlite-db/`
+- **Model fallback**: If primary model fails, falls back to cheaper/smaller model
+
+### Podcast Generation
+- **Async job queue**: `podcast_service.py` submits jobs but doesn't wait
+- **Track status**: Use `/commands/{command_id}` endpoint to poll status
+- **TTS failures**: Fall back to silent audio if speech synthesis fails
+
+### Content Processing
+- **File extraction**: Uses content-core library; supports 50+ file types
+- **URL handling**: Extracts text + metadata from web pages
+- **Large files**: Content processing is sync; may block API briefly
+
+---
+
+## Component References
+
+See dedicated CLAUDE.md files for detailed guidance:
+
+- **[frontend/CLAUDE.md](frontend/CLAUDE.md)**: React/Next.js architecture, state management, API integration
+- **[api/CLAUDE.md](api/CLAUDE.md)**: FastAPI structure, service pattern, endpoint development
+- **[open_notebook/CLAUDE.md](open_notebook/CLAUDE.md)**: Backend core, domain models, LangGraph workflows, AI provisioning
+- **[open_notebook/domain/CLAUDE.md](open_notebook/domain/CLAUDE.md)**: Data models, repository pattern, search functions
+- **[open_notebook/ai/CLAUDE.md](open_notebook/ai/CLAUDE.md)**: ModelManager, AI provider integration, Esperanto usage
+- **[open_notebook/graphs/CLAUDE.md](open_notebook/graphs/CLAUDE.md)**: LangGraph workflow design, state machines
+- **[open_notebook/database/CLAUDE.md](open_notebook/database/CLAUDE.md)**: SurrealDB operations, migrations, async patterns
+
+---
+
+## Documentation Map
+
+- **[README.md](README.md)**: Project overview, features, quick start
+- **[docs/index.md](docs/index.md)**: Complete user & deployment documentation
+- **[CONFIGURATION.md](CONFIGURATION.md)**: Environment variables, model configuration
+- **[CONTRIBUTING.md](CONTRIBUTING.md)**: Contribution guidelines
+- **[MAINTAINER_GUIDE.md](MAINTAINER_GUIDE.md)**: Release & maintenance procedures
+
+---
+
+## Testing Strategy
+
+- **Unit tests**: `tests/test_domain.py`, `test_models_api.py`
+- **Graph tests**: `tests/test_graphs.py` (workflow integration)
+- **Utils tests**: `tests/test_utils.py`, `tests/test_chunking.py`, `tests/test_embedding.py`
+- **Run all**: `uv run pytest tests/`
+- **Coverage**: Check with `pytest --cov`
+
+---
+
+## Common Tasks
+
+### Add a New API Endpoint
+1. Create router in `api/routers/feature.py`
+2. Create service in `api/feature_service.py`
+3. Define schemas in `api/models.py`
+4. Register router in `api/main.py`
+5. Test via http://localhost:5055/docs
+
+### Add a New LangGraph Workflow
+1. Create `open_notebook/graphs/workflow_name.py`
+2. Define StateDict and node functions
+3. Build graph with `.add_node()` / `.add_edge()`
+4. Invoke in service: `graph.ainvoke({"input": ...}, config={"..."})`
+5. Test with sample data in `tests/`
+
+### Add Database Migration
+1. Create `migrations/XXX_description.surql`
+2. Write SurrealQL schema changes
+3. Create `migrations/XXX_description_down.surql` (optional rollback)
+4. API auto-detects on startup; migration runs if newer than recorded version
+
+### Deploy to Production
+1. Review [CONFIGURATION.md](CONFIGURATION.md) for security settings
+2. Use `make docker-release` for multi-platform image
+3. Push to Docker Hub / GitHub Container Registry
+4. Deploy `docker compose --profile multi up`
+5. Verify migrations via API logs
+
+---
+
+## Support & Community
+
+- **Documentation**: https://open-notebook.ai
+- **Discord**: https://discord.gg/37XJPXfz2w
+- **Issues**: https://github.com/lfnovo/open-notebook/issues
+- **License**: MIT (see LICENSE)
+

CONFIGURATION.md

@@ -0,0 +1,36 @@
+# Configuration Guide
+
+**📍 This file has moved!**
+
+All configuration documentation has been consolidated into the new documentation structure.
+
+👉 **[Read the Configuration Guide](docs/5-CONFIGURATION/index.md)**
+
+---
+
+## Quick Links
+
+- **AI Provider Setup** → [AI Providers](docs/5-CONFIGURATION/ai-providers.md)
+- **Environment Variables Reference** → [Environment Reference](docs/5-CONFIGURATION/environment-reference.md)
+- **Database Configuration** → [Database Setup](docs/5-CONFIGURATION/database.md)
+- **Server Configuration** → [Server Settings](docs/5-CONFIGURATION/server.md)
+- **Security Setup** → [Security Configuration](docs/5-CONFIGURATION/security.md)
+- **Reverse Proxy** → [Reverse Proxy Setup](docs/5-CONFIGURATION/reverse-proxy.md)
+- **Advanced Tuning** → [Advanced Configuration](docs/5-CONFIGURATION/advanced.md)
+
+---
+
+## What You'll Find
+
+The new configuration documentation includes:
+
+- **Complete environment variable reference** with examples
+- **Provider-specific setup guides** for OpenAI, Anthropic, Google, Groq, Ollama, and more
+- **Production deployment configurations** with security best practices
+- **Reverse proxy examples** for Nginx, Caddy, Traefik
+- **Database tuning** for performance optimization
+- **Troubleshooting guides** for common configuration issues
+
+---
+
+For all configuration details, see **[docs/5-CONFIGURATION/](docs/5-CONFIGURATION/index.md)**.

CONTRIBUTING.md

@@ -0,0 +1,29 @@
+# Contributing to Open Notebook
+
+**📍 This file has moved!**
+
+All contribution guidelines have been consolidated into the new development documentation structure.
+
+👉 **[Read the Contributing Guide](docs/7-DEVELOPMENT/contributing.md)**
+
+---
+
+## Quick Links
+
+- **Want to contribute code?** → [Contributing Guide](docs/7-DEVELOPMENT/contributing.md)
+- **Want to understand the architecture?** → [Architecture Overview](docs/7-DEVELOPMENT/architecture.md)
+- **Want to understand our design philosophy?** → [Design Principles](docs/7-DEVELOPMENT/design-principles.md)
+- **Are you a maintainer?** → [Maintainer Guide](docs/7-DEVELOPMENT/maintainer-guide.md)
+- **New developer?** → [Quick Start](docs/7-DEVELOPMENT/quick-start.md)
+
+---
+
+## The Issue-First Workflow
+
+**TL;DR**: Create an issue first, get it assigned, THEN code.
+
+This prevents wasted effort and ensures your work aligns with the project. [See details →](docs/7-DEVELOPMENT/contributing.md)
+
+---
+
+For all contribution details, see **[docs/7-DEVELOPMENT/contributing.md](docs/7-DEVELOPMENT/contributing.md)**.

Dockerfile

@@ -0,0 +1,149 @@
+# ==============================================================================
+# Open Notebook - HuggingFace Spaces Dockerfile
+# Based on Dockerfile.single, adapted for HuggingFace Spaces constraints:
+#   - Single container with supervisord
+#   - Nginx reverse proxy on port 7860 (HF mandatory port)
+#   - Runs as UID 1000 (HF mandatory user)
+#   - Persistent storage at /data/
+# ==============================================================================
+
+# Stage 1: Frontend Builder
+FROM node:20-slim AS frontend-builder
+WORKDIR /app/frontend
+
+# Copy dependency files first to leverage cache
+COPY frontend/package.json frontend/package-lock.json ./
+ARG NPM_REGISTRY=https://registry.npmjs.org/
+RUN npm config set registry ${NPM_REGISTRY} \
+ && npm config set fetch-retries 5 \
+ && npm config set fetch-retry-mintimeout 20000 \
+ && npm config set fetch-retry-maxtimeout 120000
+# Retry npm ci to survive transient registry ECONNRESETs
+RUN i=0; until npm ci; do \
+      i=$((i+1)); \
+      if [ "$i" -ge 5 ]; then echo "npm ci failed after $i attempts"; exit 1; fi; \
+      echo "npm ci failed (attempt $i); retrying in 15s"; sleep 15; \
+    done
+
+# Copy the rest of the frontend source
+COPY frontend/ ./
+# Build the frontend
+RUN npm run build
+
+# Stage 2: SurrealDB binary (pinned to v2 to match docker-compose.yml)
+FROM surrealdb/surrealdb:v2 AS surreal-binary
+
+# Stage 3: Backend Builder
+FROM python:3.12-slim-bookworm AS backend-builder
+# Install build dependencies
+RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends build-essential && rm -rf /var/lib/apt/lists/*
+# Install uv
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+WORKDIR /app
+
+# Set build optimization environment variables
+ENV UV_HTTP_TIMEOUT=120
+
+# Copy dependency files first
+COPY pyproject.toml uv.lock ./
+COPY open_notebook/__init__.py ./open_notebook/__init__.py
+# Install dependencies
+RUN uv sync --frozen --no-dev
+
+# Pre-download tiktoken encoding so the app works offline (issue #264).
+ENV TIKTOKEN_CACHE_DIR=/app/tiktoken-cache
+RUN mkdir -p /app/tiktoken-cache && \
+    .venv/bin/python -c "import tiktoken; tiktoken.get_encoding('o200k_base')"
+
+# ==============================================================================
+# Stage 4: Runtime (HuggingFace Spaces optimized)
+# ==============================================================================
+FROM python:3.12-slim-bookworm AS runtime
+
+# Install runtime dependencies (including nginx for reverse proxy)
+# All apt installs MUST happen before switching to non-root user
+RUN apt-get update && apt-get upgrade -y && apt-get install -y \
+    ffmpeg \
+    supervisor \
+    nginx \
+    curl \
+    && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
+    && apt-get install -y nodejs \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install SurrealDB binary (from pinned v2 image)
+COPY --from=surreal-binary /surreal /usr/local/bin/surreal
+
+# Install uv
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+# ==============================================================================
+# HuggingFace Spaces: Create user with UID 1000 (mandatory)
+# ==============================================================================
+RUN useradd -m -u 1000 user
+
+WORKDIR /app
+
+# Copy backend virtualenv and source code
+COPY --from=backend-builder /app/.venv /app/.venv
+COPY . /app/
+
+# Copy pre-downloaded tiktoken encoding from builder
+COPY --from=backend-builder /app/tiktoken-cache /app/tiktoken-cache
+
+# Copy built frontend from standalone output
+COPY --from=frontend-builder /app/frontend/.next/standalone /app/frontend/
+COPY --from=frontend-builder /app/frontend/.next/static /app/frontend/.next/static
+COPY --from=frontend-builder /app/frontend/public /app/frontend/public
+
+# Bind Next.js to all interfaces
+ENV HOSTNAME=0.0.0.0
+# Point the app at the pre-baked tiktoken encoding
+ENV TIKTOKEN_CACHE_DIR=/app/tiktoken-cache
+
+# ==============================================================================
+# Setup directories, Nginx, and permissions for UID 1000
+# ==============================================================================
+
+# Copy HuggingFace-specific configuration files
+COPY nginx.hf.conf /etc/nginx/sites-available/default
+COPY supervisord.hf.conf /etc/supervisor/conf.d/supervisord.conf
+COPY start.sh /app/start.sh
+RUN chmod +x /app/start.sh
+
+# Ensure wait-for-api script is executable
+RUN chmod +x /app/scripts/wait-for-api.sh
+
+# Setup directories with proper permissions for UID 1000
+RUN mkdir -p /app/data /data /var/log/supervisor /var/log/nginx \
+    /var/lib/nginx /var/lib/nginx/body /var/lib/nginx/proxy \
+    /var/lib/nginx/fastcgi /var/lib/nginx/uwsgi /var/lib/nginx/scgi \
+    /run /tmp/nginx
+
+# Set ownership for non-root user (UID 1000)
+RUN chown -R 1000:1000 /app /data /var/log/supervisor /var/log/nginx \
+    /var/lib/nginx /run /tmp/nginx /etc/nginx && \
+    chown -R 1000:1000 /var/run 2>/dev/null || true
+
+# Configure nginx sites
+RUN rm -f /etc/nginx/sites-enabled/default && \
+    ln -sf /etc/nginx/sites-available/default /etc/nginx/sites-enabled/default
+
+# ==============================================================================
+# Environment defaults for HuggingFace
+# ==============================================================================
+ENV SURREAL_URL=ws://localhost:8000/rpc
+ENV SURREAL_USER=root
+ENV SURREAL_PASSWORD=root
+ENV SURREAL_NAMESPACE=open_notebook
+ENV SURREAL_DATABASE=open_notebook
+ENV OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-in-hf-secrets
+
+# Switch to non-root user (HF Spaces requirement)
+USER user
+
+# Expose HuggingFace's mandatory port
+EXPOSE 7860
+
+# Start via our initialization script
+CMD ["/app/start.sh"]

Dockerfile.hf

@@ -0,0 +1,149 @@
+# ==============================================================================
+# Open Notebook - HuggingFace Spaces Dockerfile
+# Based on Dockerfile.single, adapted for HuggingFace Spaces constraints:
+#   - Single container with supervisord
+#   - Nginx reverse proxy on port 7860 (HF mandatory port)
+#   - Runs as UID 1000 (HF mandatory user)
+#   - Persistent storage at /data/
+# ==============================================================================
+
+# Stage 1: Frontend Builder
+FROM node:20-slim AS frontend-builder
+WORKDIR /app/frontend
+
+# Copy dependency files first to leverage cache
+COPY frontend/package.json frontend/package-lock.json ./
+ARG NPM_REGISTRY=https://registry.npmjs.org/
+RUN npm config set registry ${NPM_REGISTRY} \
+ && npm config set fetch-retries 5 \
+ && npm config set fetch-retry-mintimeout 20000 \
+ && npm config set fetch-retry-maxtimeout 120000
+# Retry npm ci to survive transient registry ECONNRESETs
+RUN i=0; until npm ci; do \
+      i=$((i+1)); \
+      if [ "$i" -ge 5 ]; then echo "npm ci failed after $i attempts"; exit 1; fi; \
+      echo "npm ci failed (attempt $i); retrying in 15s"; sleep 15; \
+    done
+
+# Copy the rest of the frontend source
+COPY frontend/ ./
+# Build the frontend
+RUN npm run build
+
+# Stage 2: SurrealDB binary (pinned to v2 to match docker-compose.yml)
+FROM surrealdb/surrealdb:v2 AS surreal-binary
+
+# Stage 3: Backend Builder
+FROM python:3.12-slim-bookworm AS backend-builder
+# Install build dependencies
+RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends build-essential && rm -rf /var/lib/apt/lists/*
+# Install uv
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+WORKDIR /app
+
+# Set build optimization environment variables
+ENV UV_HTTP_TIMEOUT=120
+
+# Copy dependency files first
+COPY pyproject.toml uv.lock ./
+COPY open_notebook/__init__.py ./open_notebook/__init__.py
+# Install dependencies
+RUN uv sync --frozen --no-dev
+
+# Pre-download tiktoken encoding so the app works offline (issue #264).
+ENV TIKTOKEN_CACHE_DIR=/app/tiktoken-cache
+RUN mkdir -p /app/tiktoken-cache && \
+    .venv/bin/python -c "import tiktoken; tiktoken.get_encoding('o200k_base')"
+
+# ==============================================================================
+# Stage 4: Runtime (HuggingFace Spaces optimized)
+# ==============================================================================
+FROM python:3.12-slim-bookworm AS runtime
+
+# Install runtime dependencies (including nginx for reverse proxy)
+# All apt installs MUST happen before switching to non-root user
+RUN apt-get update && apt-get upgrade -y && apt-get install -y \
+    ffmpeg \
+    supervisor \
+    nginx \
+    curl \
+    && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
+    && apt-get install -y nodejs \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install SurrealDB binary (from pinned v2 image)
+COPY --from=surreal-binary /surreal /usr/local/bin/surreal
+
+# Install uv
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+# ==============================================================================
+# HuggingFace Spaces: Create user with UID 1000 (mandatory)
+# ==============================================================================
+RUN useradd -m -u 1000 user
+
+WORKDIR /app
+
+# Copy backend virtualenv and source code
+COPY --from=backend-builder /app/.venv /app/.venv
+COPY . /app/
+
+# Copy pre-downloaded tiktoken encoding from builder
+COPY --from=backend-builder /app/tiktoken-cache /app/tiktoken-cache
+
+# Copy built frontend from standalone output
+COPY --from=frontend-builder /app/frontend/.next/standalone /app/frontend/
+COPY --from=frontend-builder /app/frontend/.next/static /app/frontend/.next/static
+COPY --from=frontend-builder /app/frontend/public /app/frontend/public
+
+# Bind Next.js to all interfaces
+ENV HOSTNAME=0.0.0.0
+# Point the app at the pre-baked tiktoken encoding
+ENV TIKTOKEN_CACHE_DIR=/app/tiktoken-cache
+
+# ==============================================================================
+# Setup directories, Nginx, and permissions for UID 1000
+# ==============================================================================
+
+# Copy HuggingFace-specific configuration files
+COPY nginx.hf.conf /etc/nginx/sites-available/default
+COPY supervisord.hf.conf /etc/supervisor/conf.d/supervisord.conf
+COPY start.sh /app/start.sh
+RUN chmod +x /app/start.sh
+
+# Ensure wait-for-api script is executable
+RUN chmod +x /app/scripts/wait-for-api.sh
+
+# Setup directories with proper permissions for UID 1000
+RUN mkdir -p /app/data /data /var/log/supervisor /var/log/nginx \
+    /var/lib/nginx /var/lib/nginx/body /var/lib/nginx/proxy \
+    /var/lib/nginx/fastcgi /var/lib/nginx/uwsgi /var/lib/nginx/scgi \
+    /run /tmp/nginx
+
+# Set ownership for non-root user (UID 1000)
+RUN chown -R 1000:1000 /app /data /var/log/supervisor /var/log/nginx \
+    /var/lib/nginx /run /tmp/nginx /etc/nginx && \
+    chown -R 1000:1000 /var/run 2>/dev/null || true
+
+# Configure nginx sites
+RUN rm -f /etc/nginx/sites-enabled/default && \
+    ln -sf /etc/nginx/sites-available/default /etc/nginx/sites-enabled/default
+
+# ==============================================================================
+# Environment defaults for HuggingFace
+# ==============================================================================
+ENV SURREAL_URL=ws://localhost:8000/rpc
+ENV SURREAL_USER=root
+ENV SURREAL_PASSWORD=root
+ENV SURREAL_NAMESPACE=open_notebook
+ENV SURREAL_DATABASE=open_notebook
+ENV OPEN_NOTEBOOK_ENCRYPTION_KEY=change-me-in-hf-secrets
+
+# Switch to non-root user (HF Spaces requirement)
+USER user
+
+# Expose HuggingFace's mandatory port
+EXPOSE 7860
+
+# Start via our initialization script
+CMD ["/app/start.sh"]

Dockerfile.single

@@ -0,0 +1,106 @@
+# Stage 1: Frontend Builder
+FROM node:20-slim AS frontend-builder
+WORKDIR /app/frontend
+
+# Copy dependency files first to leverage cache
+COPY frontend/package.json frontend/package-lock.json ./
+ARG NPM_REGISTRY=https://registry.npmjs.org/
+RUN npm config set registry ${NPM_REGISTRY} \
+ && npm config set fetch-retries 5 \
+ && npm config set fetch-retry-mintimeout 20000 \
+ && npm config set fetch-retry-maxtimeout 120000
+# Retry npm ci to survive transient registry ECONNRESETs, which are common on
+# the QEMU-emulated arm64 leg of the multi-arch build.
+RUN i=0; until npm ci; do \
+      i=$((i+1)); \
+      if [ "$i" -ge 5 ]; then echo "npm ci failed after $i attempts"; exit 1; fi; \
+      echo "npm ci failed (attempt $i); retrying in 15s"; sleep 15; \
+    done
+
+# Copy the rest of the frontend source
+COPY frontend/ ./
+# Build the frontend
+RUN npm run build
+
+# Stage 2: SurrealDB binary (pinned to v2 to match docker-compose.yml)
+FROM surrealdb/surrealdb:v2 AS surreal-binary
+
+# Stage 4: Backend Builder
+FROM python:3.12-slim-bookworm AS backend-builder
+# Install build dependencies
+RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends build-essential && rm -rf /var/lib/apt/lists/*
+# Install uv
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+WORKDIR /app
+
+# Set build optimization environment variables
+ENV UV_HTTP_TIMEOUT=120
+
+# Copy dependency files first
+COPY pyproject.toml uv.lock ./
+COPY open_notebook/__init__.py ./open_notebook/__init__.py
+# Install dependencies
+RUN uv sync --frozen --no-dev
+
+# Pre-download tiktoken encoding so the app works offline (issue #264).
+# /app/tiktoken-cache is intentionally outside /app/data/ so that volume mounts
+# of /app/data (for user data persistence) do not hide the pre-baked encoding.
+# config.py reads TIKTOKEN_CACHE_DIR from the environment to pick up this path.
+ENV TIKTOKEN_CACHE_DIR=/app/tiktoken-cache
+RUN mkdir -p /app/tiktoken-cache && \
+    .venv/bin/python -c "import tiktoken; tiktoken.get_encoding('o200k_base')"
+
+# Stage 5: Runtime
+FROM python:3.12-slim-bookworm AS runtime
+
+# Install runtime dependencies
+RUN apt-get update && apt-get upgrade -y && apt-get install -y \
+    ffmpeg \
+    supervisor \
+    curl \
+    && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
+    && apt-get install -y nodejs \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install SurrealDB (copied from pinned v2 image to match docker-compose.yml)
+COPY --from=surreal-binary /surreal /usr/local/bin/surreal
+
+# Install uv (optional but helpful for some scripts)
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+WORKDIR /app
+
+# Copy backend virtualenv and source code
+COPY --from=backend-builder /app/.venv /app/.venv
+COPY . /app/
+
+# Copy pre-downloaded tiktoken encoding from builder (outside /data/ — volume-mount safe)
+COPY --from=backend-builder /app/tiktoken-cache /app/tiktoken-cache
+
+# Copy built frontend from standalone output
+COPY --from=frontend-builder /app/frontend/.next/standalone /app/frontend/
+COPY --from=frontend-builder /app/frontend/.next/static /app/frontend/.next/static
+COPY --from=frontend-builder /app/frontend/public /app/frontend/public
+
+# Bind Next.js to all interfaces (required for Docker networking and reverse proxies)
+ENV HOSTNAME=0.0.0.0
+# Point the app at the pre-baked tiktoken encoding (see open_notebook/config.py)
+ENV TIKTOKEN_CACHE_DIR=/app/tiktoken-cache
+
+# Setup directories and permissions
+RUN mkdir -p /app/data /mydata
+
+# Ensure wait-for-api script is executable
+RUN chmod +x /app/scripts/wait-for-api.sh
+
+# Copy supervisord configuration
+COPY supervisord.single.conf /etc/supervisor/conf.d/supervisord.conf
+
+# Create log directories
+RUN mkdir -p /var/log/supervisor
+
+# Expose ports
+EXPOSE 8502 5055
+
+# Set startup command
+CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]

LICENSE

@@ -0,0 +1,17 @@
+MIT License
+Copyright (c) 2024 Luis Novo
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

MAINTAINER_GUIDE.md

@@ -0,0 +1,19 @@
+# Maintainer Guide
+
+**📍 This file has moved!**
+
+All maintainer guidelines have been consolidated into the new development documentation structure.
+
+👉 **[Read the Maintainer Guide](docs/7-DEVELOPMENT/maintainer-guide.md)**
+
+---
+
+## Quick Links
+
+- **Maintainer Guide** → [docs/7-DEVELOPMENT/maintainer-guide.md](docs/7-DEVELOPMENT/maintainer-guide.md)
+- **Contributing Guide** → [docs/7-DEVELOPMENT/contributing.md](docs/7-DEVELOPMENT/contributing.md)
+- **Design Principles** → [docs/7-DEVELOPMENT/design-principles.md](docs/7-DEVELOPMENT/design-principles.md)
+
+---
+
+For all maintainer details, see **[docs/7-DEVELOPMENT/maintainer-guide.md](docs/7-DEVELOPMENT/maintainer-guide.md)**.

Makefile

@@ -0,0 +1,210 @@
+.PHONY: run frontend check ruff database lint api start-all stop-all status clean-cache worker worker-start worker-stop worker-restart
+.PHONY: docker-buildx-prepare docker-buildx-clean docker-buildx-reset
+.PHONY: docker-push docker-push-latest docker-release docker-build-local tag export-docs
+
+# Get version from pyproject.toml
+VERSION := $(shell grep -m1 version pyproject.toml | cut -d'"' -f2)
+
+# Image names for both registries
+DOCKERHUB_IMAGE := lfnovo/open_notebook
+GHCR_IMAGE := ghcr.io/lfnovo/open-notebook
+
+# Build platforms
+PLATFORMS := linux/amd64,linux/arm64
+
+database:
+	docker compose up -d surrealdb
+
+run:
+	@echo "⚠️  Warning: Starting frontend only. For full functionality, use 'make start-all'"
+	cd frontend && npm run dev
+
+frontend:
+	cd frontend && npm run dev
+
+lint:
+	uv run python -m mypy .
+
+ruff:
+	ruff check . --fix
+
+# === Docker Build Setup ===
+docker-buildx-prepare:
+	@docker buildx inspect multi-platform-builder >/dev/null 2>&1 || \
+		docker buildx create --use --name multi-platform-builder --driver docker-container
+	@docker buildx use multi-platform-builder
+
+docker-buildx-clean:
+	@echo "🧹 Cleaning up buildx builders..."
+	@docker buildx rm multi-platform-builder 2>/dev/null || true
+	@docker ps -a | grep buildx_buildkit | awk '{print $$1}' | xargs -r docker rm -f 2>/dev/null || true
+	@echo "✅ Buildx cleanup complete!"
+
+docker-buildx-reset: docker-buildx-clean docker-buildx-prepare
+	@echo "✅ Buildx reset complete!"
+
+# === Docker Build Targets ===
+
+# Build production image for local platform only (no push)
+docker-build-local:
+	@echo "🔨 Building production image locally ($(shell uname -m))..."
+	docker build \
+		-t $(DOCKERHUB_IMAGE):$(VERSION) \
+		-t $(DOCKERHUB_IMAGE):local \
+		.
+	@echo "✅ Built $(DOCKERHUB_IMAGE):$(VERSION) and $(DOCKERHUB_IMAGE):local"
+	@echo "Run with: docker run -p 5055:5055 -p 3000:3000 $(DOCKERHUB_IMAGE):local"
+
+# Build and push version tags ONLY (no latest) for both regular and single images
+docker-push: docker-buildx-prepare
+	@echo "📤 Building and pushing version $(VERSION) to both registries..."
+	@echo "🔨 Building regular image..."
+	docker buildx build --pull \
+		--platform $(PLATFORMS) \
+		--progress=plain \
+		-t $(DOCKERHUB_IMAGE):$(VERSION) \
+		-t $(GHCR_IMAGE):$(VERSION) \
+		--push \
+		.
+	@echo "🔨 Building single-container image..."
+	docker buildx build --pull \
+		--platform $(PLATFORMS) \
+		--progress=plain \
+		-f Dockerfile.single \
+		-t $(DOCKERHUB_IMAGE):$(VERSION)-single \
+		-t $(GHCR_IMAGE):$(VERSION)-single \
+		--push \
+		.
+	@echo "✅ Pushed version $(VERSION) to both registries (latest NOT updated)"
+	@echo "  📦 Docker Hub:"
+	@echo "    - $(DOCKERHUB_IMAGE):$(VERSION)"
+	@echo "    - $(DOCKERHUB_IMAGE):$(VERSION)-single"
+	@echo "  📦 GHCR:"
+	@echo "    - $(GHCR_IMAGE):$(VERSION)"
+	@echo "    - $(GHCR_IMAGE):$(VERSION)-single"
+
+# Update v1-latest tags to current version (both regular and single images)
+docker-push-latest: docker-buildx-prepare
+	@echo "📤 Updating v1-latest tags to version $(VERSION)..."
+	@echo "🔨 Building regular image with latest tag..."
+	docker buildx build --pull \
+		--platform $(PLATFORMS) \
+		--progress=plain \
+		-t $(DOCKERHUB_IMAGE):$(VERSION) \
+		-t $(DOCKERHUB_IMAGE):v1-latest \
+		-t $(GHCR_IMAGE):$(VERSION) \
+		-t $(GHCR_IMAGE):v1-latest \
+		--push \
+		.
+	@echo "🔨 Building single-container image with latest tag..."
+	docker buildx build --pull \
+		--platform $(PLATFORMS) \
+		--progress=plain \
+		-f Dockerfile.single \
+		-t $(DOCKERHUB_IMAGE):$(VERSION)-single \
+		-t $(DOCKERHUB_IMAGE):v1-latest-single \
+		-t $(GHCR_IMAGE):$(VERSION)-single \
+		-t $(GHCR_IMAGE):v1-latest-single \
+		--push \
+		.
+	@echo "✅ Updated v1-latest to version $(VERSION)"
+	@echo "  📦 Docker Hub:"
+	@echo "    - $(DOCKERHUB_IMAGE):$(VERSION) → v1-latest"
+	@echo "    - $(DOCKERHUB_IMAGE):$(VERSION)-single → v1-latest-single"
+	@echo "  📦 GHCR:"
+	@echo "    - $(GHCR_IMAGE):$(VERSION) → v1-latest"
+	@echo "    - $(GHCR_IMAGE):$(VERSION)-single → v1-latest-single"
+
+# Full release: push version AND update latest tags
+docker-release: docker-push-latest
+	@echo "✅ Full release complete for version $(VERSION)"
+
+tag:
+	@version=$$(grep '^version = ' pyproject.toml | sed 's/version = "\(.*\)"/\1/'); \
+	echo "Creating tag v$$version"; \
+	git tag "v$$version"; \
+	git push origin "v$$version"
+
+
+dev:
+	docker compose -f docker-compose.dev.yml up --build 
+
+full:
+	docker compose -f docker-compose.full.yml up --build 
+
+
+api:
+	uv run --env-file .env run_api.py
+
+.PHONY: worker worker-start worker-stop worker-restart
+
+worker: worker-start
+
+worker-start:
+	@echo "Starting surreal-commands worker..."
+	uv run --env-file .env surreal-commands-worker --import-modules commands
+
+worker-stop:
+	@echo "Stopping surreal-commands worker..."
+	pkill -f "surreal-commands-worker" || true
+
+worker-restart: worker-stop
+	@sleep 2
+	@$(MAKE) worker-start
+
+# === Service Management ===
+start-all:
+	@echo "🚀 Starting Open Notebook (Database + API + Worker + Frontend)..."
+	@echo "📊 Starting SurrealDB..."
+	@docker compose -f docker-compose.dev.yml up -d surrealdb
+	@sleep 3
+	@echo "🔧 Starting API backend..."
+	@uv run run_api.py &
+	@sleep 3
+	@echo "⚙️ Starting background worker..."
+	@uv run --env-file .env surreal-commands-worker --import-modules commands &
+	@sleep 2
+	@echo "🌐 Starting Next.js frontend..."
+	@echo "✅ All services started!"
+	@echo "📱 Frontend: http://localhost:3000"
+	@echo "🔗 API: http://localhost:5055"
+	@echo "📚 API Docs: http://localhost:5055/docs"
+	cd frontend && npm run dev
+
+stop-all:
+	@echo "🛑 Stopping all Open Notebook services..."
+	@pkill -f "next dev" || true
+	@pkill -f "surreal-commands-worker" || true
+	@pkill -f "run_api.py" || true
+	@pkill -f "uvicorn api.main:app" || true
+	@docker compose down
+	@echo "✅ All services stopped!"
+
+status:
+	@echo "📊 Open Notebook Service Status:"
+	@echo "Database (SurrealDB):"
+	@docker compose ps surrealdb 2>/dev/null || echo "  ❌ Not running"
+	@echo "API Backend:"
+	@pgrep -f "run_api.py\|uvicorn api.main:app" >/dev/null && echo "  ✅ Running" || echo "  ❌ Not running"
+	@echo "Background Worker:"
+	@pgrep -f "surreal-commands-worker" >/dev/null && echo "  ✅ Running" || echo "  ❌ Not running"
+	@echo "Next.js Frontend:"
+	@pgrep -f "next dev" >/dev/null && echo "  ✅ Running" || echo "  ❌ Not running"
+
+# === Documentation Export ===
+export-docs:
+	@echo "📚 Exporting documentation..."
+	@uv run python scripts/export_docs.py
+	@echo "✅ Documentation export complete!"
+
+# === Cleanup ===
+clean-cache:
+	@echo "🧹 Cleaning cache directories..."
+	@find . -name "__pycache__" -type d -exec rm -rf {} + 2>/dev/null || true
+	@find . -name ".mypy_cache" -type d -exec rm -rf {} + 2>/dev/null || true
+	@find . -name ".ruff_cache" -type d -exec rm -rf {} + 2>/dev/null || true
+	@find . -name ".pytest_cache" -type d -exec rm -rf {} + 2>/dev/null || true
+	@find . -name "*.pyc" -type f -delete 2>/dev/null || true
+	@find . -name "*.pyo" -type f -delete 2>/dev/null || true
+	@find . -name "*.pyd" -type f -delete 2>/dev/null || true
+	@echo "✅ Cache directories cleaned!"

README.dev.md

@@ -0,0 +1,449 @@
+# Developer Guide
+
+This guide is for developers working on Open Notebook. For end-user documentation, see [README.md](README.md) and [docs/](docs/).
+
+## Quick Start for Development
+
+```bash
+# 1. Clone and setup
+git clone https://github.com/lfnovo/open-notebook.git
+cd open-notebook
+
+# 2. Copy environment files
+cp .env.example .env
+cp .env.example docker.env
+
+# 3. Install dependencies
+uv sync
+
+# 4. Start all services (recommended for development)
+make start-all
+```
+
+## Development Workflows
+
+### When to Use What?
+
+| Workflow | Use Case | Speed | Production Parity |
+|----------|----------|-------|-------------------|
+| **Local Services** (`make start-all`) | Day-to-day development, fastest iteration | ⚡⚡⚡ Fast | Medium |
+| **Docker Compose** (`make dev`) | Testing containerized setup | ⚡⚡ Medium | High |
+| **Local Docker Build** (`make docker-build-local`) | Testing Dockerfile changes | ⚡ Slow | Very High |
+| **Multi-platform Build** (`make docker-push`) | Publishing releases | 🐌 Very Slow | Exact |
+
+---
+
+## 1. Local Development (Recommended)
+
+**Best for:** Daily development, hot reload, debugging
+
+### Setup
+
+```bash
+# Start database
+make database
+
+# Start all services (DB + API + Worker + Frontend)
+make start-all
+```
+
+### What This Does
+
+1. Starts SurrealDB in Docker (port 8000)
+2. Starts FastAPI backend (port 5055)
+3. Starts background worker (surreal-commands)
+4. Starts Next.js frontend (port 3000)
+
+### Individual Services
+
+```bash
+# Just the database
+make database
+
+# Just the API
+make api
+
+# Just the frontend
+make frontend
+
+# Just the worker
+make worker
+```
+
+### Checking Status
+
+```bash
+# See what's running
+make status
+
+# Stop everything
+make stop-all
+```
+
+### Advantages
+- ✅ Fastest iteration (hot reload)
+- ✅ Easy debugging (direct process access)
+- ✅ Low resource usage
+- ✅ Direct log access
+
+### Disadvantages
+- ❌ Doesn't test Docker build
+- ❌ Environment may differ from production
+- ❌ Requires local Python/Node setup
+
+---
+
+## 2. Docker Compose Development
+
+**Best for:** Testing containerized setup, CI/CD verification
+
+```bash
+# Start with dev profile
+make dev
+
+# Or full stack
+make full
+```
+
+### Configuration Files
+
+- `docker-compose.dev.yml` - Development setup
+- `docker-compose.full.yml` - Full stack setup
+- `docker-compose.yml` - Base configuration
+
+### Advantages
+- ✅ Closer to production environment
+- ✅ Isolated dependencies
+- ✅ Easy to share exact environment
+
+### Disadvantages
+- ❌ Slower rebuilds
+- ❌ More complex debugging
+- ❌ Higher resource usage
+
+---
+
+## 3. Testing Production Docker Images
+
+**Best for:** Verifying Dockerfile changes before publishing
+
+### Build Locally
+
+```bash
+# Build production image for your platform only
+make docker-build-local
+```
+
+This creates two tags:
+- `lfnovo/open_notebook:<version>` (from pyproject.toml)
+- `lfnovo/open_notebook:local`
+
+### Run Locally
+
+```bash
+docker run -p 5055:5055 -p 3000:3000 lfnovo/open_notebook:local
+```
+
+### When to Use
+- ✅ Before pushing to registry
+- ✅ Testing Dockerfile changes
+- ✅ Debugging production-specific issues
+- ✅ Verifying build process
+
+---
+
+## 4. Publishing Docker Images
+
+### Workflow
+
+```bash
+# 1. Test locally first
+make docker-build-local
+
+# 2. If successful, push version tag (no latest update)
+make docker-push
+
+# 3. Test the pushed version in staging/production
+
+# 4. When ready, promote to latest
+make docker-push-latest
+```
+
+### Available Commands
+
+| Command | What It Does | Updates Latest? |
+|---------|--------------|-----------------|
+| `make docker-build-local` | Build for current platform only | No registry push |
+| `make docker-push` | Push version tags to registries | ❌ No |
+| `make docker-push-latest` | Push version + update v1-latest | ✅ Yes |
+| `make docker-release` | Full release (same as docker-push-latest) | ✅ Yes |
+
+### Publishing Details
+
+- **Platforms:** `linux/amd64`, `linux/arm64`
+- **Registries:** Docker Hub + GitHub Container Registry
+- **Image Variants:** Regular + Single-container (`-single`)
+- **Version Source:** `pyproject.toml`
+
+### Creating Git Tags
+
+```bash
+# Create and push git tag matching pyproject.toml version
+make tag
+```
+
+---
+
+## Code Quality
+
+```bash
+# Run linter with auto-fix
+make ruff
+
+# Run type checking
+make lint
+
+# Run tests
+uv run pytest tests/
+
+# Clean cache directories
+make clean-cache
+```
+
+---
+
+## Common Development Tasks
+
+### Adding a New Feature
+
+1. Create feature branch
+2. Develop using `make start-all`
+3. Write tests
+4. Run `make ruff` and `make lint`
+5. Test with `make docker-build-local`
+6. Create PR
+
+### Fixing a Bug
+
+1. Reproduce locally with `make start-all`
+2. Add test case demonstrating bug
+3. Fix the bug
+4. Verify test passes
+5. Check with `make docker-build-local`
+
+### Updating Dependencies
+
+```bash
+# Add Python dependency
+uv add package-name
+
+# Update dependencies
+uv sync
+
+# Frontend dependencies
+cd frontend && npm install package-name
+```
+
+### Adding a New Language (i18n)
+
+Open Notebook supports internationalization. To add a new language:
+
+1. **Create locale file**: Copy an existing locale as template
+   ```bash
+   cp frontend/src/lib/locales/en-US/index.ts frontend/src/lib/locales/pt-BR/index.ts
+   ```
+
+2. **Translate all strings** in the new file. The structure includes:
+   - `common`: Shared UI elements (buttons, labels)
+   - `notebooks`, `sources`, `notes`: Feature-specific strings
+   - `chat`, `search`, `podcasts`: Module-specific strings
+   - `apiErrors`: Error message translations
+
+3. **Register the locale** in `frontend/src/lib/locales/index.ts`:
+   ```typescript
+   import { ptBR } from './pt-BR'
+
+   export const locales = {
+     'en-US': enUS,
+     'zh-CN': zhCN,
+     'zh-TW': zhTW,
+     'pt-BR': ptBR,  // Add your locale
+   }
+   ```
+
+4. **Add date-fns locale** in `frontend/src/lib/utils/date-locale.ts`:
+   ```typescript
+   import { zhCN, enUS, zhTW, ptBR } from 'date-fns/locale'
+
+   const LOCALE_MAP: Record<string, Locale> = {
+     'zh-CN': zhCN,
+     'zh-TW': zhTW,
+     'en-US': enUS,
+     'pt-BR': ptBR,  // Add your locale
+   }
+   ```
+
+5. **Test**: Switch languages using the language toggle in the UI header.
+
+### Database Migrations
+
+Database migrations run **automatically** when the API starts.
+
+1. Create migration file: `migrations/XXX_description.surql`
+2. Write SurrealQL schema changes
+3. (Optional) Create rollback: `migrations/XXX_description_down.surql`
+4. Restart API - migration runs on startup
+
+---
+
+## Troubleshooting
+
+### Services Won't Start
+
+```bash
+# Check status
+make status
+
+# Check database
+docker compose ps surrealdb
+
+# View logs
+docker compose logs surrealdb
+
+# Restart everything
+make stop-all
+make start-all
+```
+
+### Port Already in Use
+
+```bash
+# Find process using port
+lsof -i :5055
+lsof -i :3000
+lsof -i :8000
+
+# Kill stuck processes
+make stop-all
+```
+
+### Database Connection Issues
+
+```bash
+# Verify SurrealDB is running
+docker compose ps surrealdb
+
+# Check connection settings in .env
+cat .env | grep SURREAL
+```
+
+### Docker Build Fails
+
+```bash
+# Clean Docker cache
+docker builder prune
+
+# Reset buildx
+make docker-buildx-reset
+
+# Try local build first
+make docker-build-local
+```
+
+---
+
+## Project Structure
+
+```
+open-notebook/
+├── api/                    # FastAPI backend
+├── frontend/               # Next.js React frontend
+├── open_notebook/          # Python core library
+│   ├── domain/            # Domain models
+│   ├── graphs/            # LangGraph workflows
+│   ├── ai/                # AI provider integration
+│   └── database/          # SurrealDB operations
+├── migrations/             # Database migrations
+├── tests/                  # Test suite
+├── docs/                   # User documentation
+└── Makefile               # Development commands
+```
+
+See component-specific CLAUDE.md files for detailed architecture:
+- [frontend/CLAUDE.md](frontend/CLAUDE.md)
+- [api/CLAUDE.md](api/CLAUDE.md)
+- [open_notebook/CLAUDE.md](open_notebook/CLAUDE.md)
+
+---
+
+## Environment Variables
+
+### Required for Local Development
+
+```bash
+# .env file
+SURREAL_URL=ws://localhost:8000
+SURREAL_USER=root
+SURREAL_PASS=root
+SURREAL_DB=open_notebook
+SURREAL_NS=production
+
+# AI Provider (at least one required)
+OPENAI_API_KEY=sk-...
+# OR
+ANTHROPIC_API_KEY=sk-ant-...
+# OR configure other providers (see docs/5-CONFIGURATION/)
+```
+
+See [docs/5-CONFIGURATION/](docs/5-CONFIGURATION/) for complete configuration guide.
+
+---
+
+## Performance Tips
+
+### Speed Up Local Development
+
+1. **Use `make start-all`** instead of Docker for daily work
+2. **Keep SurrealDB running** between sessions (`make database`)
+3. **Use `make docker-build-local`** only when testing Dockerfile changes
+4. **Skip multi-platform builds** until ready to publish
+
+### Reduce Resource Usage
+
+```bash
+# Stop unused services
+make stop-all
+
+# Clean up Docker
+docker system prune -a
+
+# Clean Python cache
+make clean-cache
+```
+
+---
+
+## TODO: Sections to Add
+
+- [ ] Frontend development guide (hot reload, component structure)
+- [ ] API development guide (adding endpoints, services)
+- [ ] LangGraph workflow development
+- [ ] Testing strategy and coverage
+- [ ] Debugging tips (VSCode/PyCharm setup)
+- [ ] CI/CD pipeline overview
+- [ ] Release process checklist
+- [ ] Common error messages and solutions
+
+---
+
+## Resources
+
+- **Documentation:** https://open-notebook.ai
+- **Discord:** https://discord.gg/37XJPXfz2w
+- **Issues:** https://github.com/lfnovo/open-notebook/issues
+- **Contributing:** [CONTRIBUTING.md](CONTRIBUTING.md)
+- **Maintainer Guide:** [MAINTAINER_GUIDE.md](MAINTAINER_GUIDE.md)
+
+---
+
+**Last Updated:** January 2025

README.hf.md

@@ -0,0 +1,75 @@
+---
+title: Open Notebook
+emoji: 📓
+colorFrom: blue
+colorTo: purple
+sdk: docker
+app_port: 7860
+startup_duration_timeout: 1h
+pinned: false
+license: mit
+suggested_hardware: cpu-upgrade
+---
+
+# 📓 Open Notebook
+
+An open source, privacy-focused alternative to Google's Notebook LM!
+
+## Features
+
+- 🔒 **Privacy First** - Self-hosted, your data stays under your control
+- 🤖 **18+ AI Providers** - OpenAI, Anthropic, Google, Ollama, and more
+- 📚 **Multi-modal Content** - PDFs, videos, audio, web pages
+- 🎙️ **Podcast Generation** - Advanced multi-speaker podcasts
+- 🔍 **Intelligent Search** - Full-text and vector search
+- 💬 **Context-Aware Chat** - AI conversations powered by your research
+- 🌐 **REST API** - Full programmatic access at `/docs`
+- 🔌 **MCP Integration** - Connect with Claude Desktop, VS Code, etc.
+
+## Quick Start
+
+1. **Configure AI Provider**: Go to Settings → API Keys → Add your provider
+2. **Create a Notebook**: Start organizing your research
+3. **Add Sources**: Upload PDFs, paste URLs, or add text content
+4. **Chat & Generate**: Ask questions, generate insights, create podcasts
+
+## API Access
+
+- **Swagger Docs**: [/docs](/docs)
+- **REST API**: All endpoints at `/api/v1/...`
+
+## MCP Integration
+
+Connect your local AI assistants to this Space:
+
+```json
+{
+  "mcpServers": {
+    "open-notebook": {
+      "command": "uvx",
+      "args": ["open-notebook-mcp"],
+      "env": {
+        "OPEN_NOTEBOOK_URL": "https://YOUR-USERNAME-open-notebook.hf.space"
+      }
+    }
+  }
+}
+```
+
+## Configuration
+
+Set these in your Space **Settings → Secrets**:
+
+| Secret | Required | Description |
+|--------|----------|-------------|
+| `OPEN_NOTEBOOK_ENCRYPTION_KEY` | ✅ | Encrypts API keys in database |
+| `BASIC_AUTH_USERNAME` | Optional | Password-protect the UI |
+| `BASIC_AUTH_PASSWORD` | Optional | Password-protect the UI |
+
+## Links
+
+- [GitHub Repository](https://github.com/lfnovo/open-notebook)
+- [Documentation](https://www.open-notebook.ai)
+- [Discord Community](https://discord.gg/37XJPXfz2w)
+
+> **⚠️ Persistent Storage**: Enable Persistent Storage in Space Settings to keep your data between restarts.

README.md

@@ -0,0 +1,75 @@
+---
+title: Open Notebook
+emoji: 📓
+colorFrom: blue
+colorTo: purple
+sdk: docker
+app_port: 7860
+startup_duration_timeout: 1h
+pinned: false
+license: mit
+suggested_hardware: cpu-upgrade
+---
+
+# 📓 Open Notebook
+
+An open source, privacy-focused alternative to Google's Notebook LM!
+
+## Features
+
+- 🔒 **Privacy First** - Self-hosted, your data stays under your control
+- 🤖 **18+ AI Providers** - OpenAI, Anthropic, Google, Ollama, and more
+- 📚 **Multi-modal Content** - PDFs, videos, audio, web pages
+- 🎙️ **Podcast Generation** - Advanced multi-speaker podcasts
+- 🔍 **Intelligent Search** - Full-text and vector search
+- 💬 **Context-Aware Chat** - AI conversations powered by your research
+- 🌐 **REST API** - Full programmatic access at `/docs`
+- 🔌 **MCP Integration** - Connect with Claude Desktop, VS Code, etc.
+
+## Quick Start
+
+1. **Configure AI Provider**: Go to Settings → API Keys → Add your provider
+2. **Create a Notebook**: Start organizing your research
+3. **Add Sources**: Upload PDFs, paste URLs, or add text content
+4. **Chat & Generate**: Ask questions, generate insights, create podcasts
+
+## API Access
+
+- **Swagger Docs**: [/docs](/docs)
+- **REST API**: All endpoints at `/api/v1/...`
+
+## MCP Integration
+
+Connect your local AI assistants to this Space:
+
+```json
+{
+  "mcpServers": {
+    "open-notebook": {
+      "command": "uvx",
+      "args": ["open-notebook-mcp"],
+      "env": {
+        "OPEN_NOTEBOOK_URL": "https://YOUR-USERNAME-open-notebook.hf.space"
+      }
+    }
+  }
+}
+```
+
+## Configuration
+
+Set these in your Space **Settings → Secrets**:
+
+| Secret | Required | Description |
+|--------|----------|-------------|
+| `OPEN_NOTEBOOK_ENCRYPTION_KEY` | ✅ | Encrypts API keys in database |
+| `BASIC_AUTH_USERNAME` | Optional | Password-protect the UI |
+| `BASIC_AUTH_PASSWORD` | Optional | Password-protect the UI |
+
+## Links
+
+- [GitHub Repository](https://github.com/lfnovo/open-notebook)
+- [Documentation](https://www.open-notebook.ai)
+- [Discord Community](https://discord.gg/37XJPXfz2w)
+
+> **⚠️ Persistent Storage**: Enable Persistent Storage in Space Settings to keep your data between restarts.

api/CLAUDE.md

@@ -0,0 +1,260 @@
+# API Module
+
+FastAPI-based REST backend exposing services for notebooks, sources, notes, chat, podcasts, and AI model management.
+
+## Purpose
+
+FastAPI application serving three architectural layers: routes (HTTP endpoints), services (business logic), and models (request/response schemas). Integrates LangGraph workflows (chat, ask, source_chat), SurrealDB persistence, and AI providers via Esperanto.
+
+## Architecture Overview
+
+**Three layers**:
+1. **Routes** (`routers/*`): HTTP endpoints mapping to services
+2. **Services** (`*_service.py`): Business logic orchestrating domain models, database, graphs, AI providers
+3. **Models** (`models.py`): Pydantic request/response schemas with validation
+
+**Startup flow**:
+- Load .env environment variables
+- Initialize CORS middleware + password auth middleware
+- Run database migrations via AsyncMigrationManager on lifespan startup
+- Run podcast profile data migration (legacy string to model registry conversion)
+- Register all routers
+
+**Key services**:
+- `chat_service.py`: Invokes chat graph with messages, context
+- `podcast_service.py`: Orchestrates outline + transcript generation
+- `sources_service.py`: Content ingestion, vectorization, metadata
+- `notes_service.py`: Note creation, linking to sources/insights
+- `transformations_service.py`: Applies transformations to content
+- `models_service.py`: Manages AI provider/model configuration
+- `episode_profiles_service.py`: Manages podcast speaker/episode profiles
+
+## Component Catalog
+
+### Main Application
+- **main.py**: FastAPI app initialization, CORS setup, auth middleware, lifespan event, router registration
+- **Lifespan handler**: Runs AsyncMigrationManager on startup (database schema migration)
+- **Auth middleware**: PasswordAuthMiddleware protects endpoints (password-based access control)
+
+### Services (Business Logic)
+- **chat_service.py**: Invokes chat.py graph; handles message history via SqliteSaver
+- **podcast_service.py**: Generates outline (outline.jinja), then transcript (transcript.jinja) for episodes
+- **sources_service.py**: Ingests files/URLs (content_core), extracts text, vectorizes, saves to SurrealDB
+- **transformations_service.py**: Applies transformations via transformation.py graph
+- **models_service.py**: Manages ModelManager config (AI provider overrides)
+- **episode_profiles_service.py**: CRUD for EpisodeProfile and SpeakerProfile models
+- **insights_service.py**: Generates and retrieves source insights
+- **notes_service.py**: Creates notes linked to sources/insights
+
+### Models (Schemas)
+- **models.py**: Pydantic schemas for request/response validation
+- Request bodies: ChatRequest, CreateNoteRequest, PodcastGenerationRequest, etc.
+- Response bodies: ChatResponse, NoteResponse, PodcastResponse, etc.
+- Custom validators for enum fields, file paths, model references
+
+### Routers
+- **routers/chat.py**: POST /chat
+- **routers/source_chat.py**: POST /source/{source_id}/chat
+- **routers/podcasts.py**: POST /podcasts, GET /podcasts/{id}, POST /podcasts/episodes/{id}/retry, etc.
+- **routers/notes.py**: POST /notes, GET /notes/{id}
+- **routers/sources.py**: POST /sources, GET /sources/{id}, DELETE /sources/{id}
+- **routers/models.py**: GET /models, POST /models/config
+- **routers/credentials.py**: CRUD + test + discover + migrate for credential management
+- **routers/transformations.py**: POST /transformations
+- **routers/insights.py**: GET /sources/{source_id}/insights
+- **routers/auth.py**: POST /auth/password (password-based auth)
+- **routers/languages.py**: GET /languages (available podcast languages via pycountry+babel)
+- **routers/commands.py**: GET /commands/{command_id} (job status tracking)
+
+## Common Patterns
+
+- **Service injection via FastAPI**: Routers import services directly; no DI framework
+- **Async/await throughout**: All DB queries, graph invocations, AI calls are async
+- **SurrealDB transactions**: Services use repo_query, repo_create, repo_upsert from database layer
+- **Config override pattern**: Models/config override via models_service passed to graph.ainvoke(config=...)
+- **Error handling**: Custom exception hierarchy (`open_notebook.exceptions`) with global FastAPI exception handlers mapping to HTTP status codes (see Error Handling section below). LangGraph nodes use `classify_error()` to convert raw LLM provider errors into typed exceptions with user-friendly messages.
+- **Logging**: loguru logger in main.py; services expected to log key operations
+- **Response normalization**: All responses follow standard schema (data + metadata structure)
+
+## Key Dependencies
+
+- `fastapi`: FastAPI app, routers, HTTPException
+- `pydantic`: Validation models with Field, field_validator
+- `open_notebook.graphs`: chat, ask, source_chat, source, transformation graphs
+- `open_notebook.database`: SurrealDB repository functions (repo_query, repo_create, repo_upsert)
+- `open_notebook.domain`: Notebook, Source, Note, SourceInsight models
+- `open_notebook.ai.provision`: provision_langchain_model() factory
+- `ai_prompter`: Prompter for template rendering
+- `content_core`: extract_content() for file/URL processing
+- `esperanto`: AI provider client library (LLM, embeddings, TTS)
+- `surreal_commands`: Job queue for async operations (podcast generation)
+- `loguru`: Structured logging
+
+## Important Quirks & Gotchas
+
+- **Migration auto-run**: Database schema migrations run on every API startup (via lifespan); no manual migration steps
+- **PasswordAuthMiddleware is basic**: Uses simple password check; production deployments should replace with OAuth/JWT
+- **No request rate limiting**: No built-in rate limiting; deployment must add via proxy/middleware
+- **Service state is stateless**: Services don't cache results; each request re-queries database/AI models
+- **Graph invocation is blocking**: chat/podcast workflows may take minutes; no timeout handling in services
+- **Command job fire-and-forget**: podcast_service.py submits jobs but doesn't wait (async job queue pattern)
+- **Model override scoping**: Model config override via RunnableConfig is per-request only (not persistent)
+- **CORS open by default**: main.py CORS settings allow all origins (restrict before production)
+- **No OpenAPI security scheme**: API docs available without auth (disable before production)
+- **Services don't validate user permission**: All endpoints trust authentication layer; no per-notebook permission checks
+
+## Error Handling
+
+### Global Exception Handlers (`main.py`)
+
+FastAPI exception handlers map custom exception types from `open_notebook.exceptions` to HTTP status codes. All error responses include CORS headers.
+
+| Exception Class | HTTP Status | Use Case |
+|----------------|-------------|----------|
+| `NotFoundError` | 404 | Resource not found |
+| `InvalidInputError` | 400 | Bad request data |
+| `AuthenticationError` | 401 | Invalid/missing API key |
+| `RateLimitError` | 429 | Provider rate limit exceeded |
+| `ConfigurationError` | 422 | Wrong model name, missing config |
+| `NetworkError` | 502 | Cannot reach AI provider |
+| `ExternalServiceError` | 502 | Provider returned error (500/503, context length) |
+| `OpenNotebookError` (base) | 500 | Any other application error |
+
+### Error Classification (`open_notebook.utils.error_classifier`)
+
+The `classify_error()` function maps raw exceptions from LLM providers/Esperanto/LangChain into the typed exceptions above with user-friendly messages. Used in all LangGraph graph nodes and SSE streaming handlers.
+
+**Flow**: Raw exception → keyword matching → `(ExceptionClass, user_message)` → raised → caught by global handler → HTTP response with descriptive message.
+
+### Frontend Integration
+
+The frontend `getApiErrorMessage()` helper (`lib/utils/error-handler.ts`) tries i18n mapping first, then falls back to displaying the backend's descriptive error message directly.
+
+---
+
+## How to Add New Endpoint
+
+1. Create router file in `routers/` (e.g., `routers/new_feature.py`)
+2. Import router into `main.py` and register: `app.include_router(new_feature.router, tags=["new_feature"])`
+3. Create service in `new_feature_service.py` with business logic
+4. Define request/response schemas in `models.py` (or create `new_feature_models.py`)
+5. Implement router functions calling service methods
+6. Test with `uv run uvicorn api.main:app --host 0.0.0.0 --port 5055`
+
+## Testing Patterns
+
+- **Interactive docs**: http://localhost:5055/docs (Swagger UI)
+- **Direct service tests**: Import service, call methods directly with test data
+- **Mock graphs**: Replace graph.ainvoke() with mock for testing service logic
+- **Database: Use test database** (separate SurrealDB instance or mock repo_query)
+
+---
+
+## Credential Management (API Configuration UI)
+
+The Credential Management system enables users to configure AI provider credentials through the UI instead of environment variables. Keys are stored securely in SurrealDB (encrypted via Fernet) with database-first fallback to environment variables.
+
+### Router: `routers/credentials.py`
+
+**Endpoints**:
+
+| Method | Endpoint | Description |
+|--------|----------|-------------|
+| GET | `/credentials` | List all credentials (optional `?provider=` filter) |
+| GET | `/credentials/by-provider/{provider}` | List credentials for a provider |
+| POST | `/credentials` | Create a new credential |
+| GET | `/credentials/{credential_id}` | Get a specific credential |
+| PUT | `/credentials/{credential_id}` | Update a credential |
+| DELETE | `/credentials/{credential_id}` | Delete a credential |
+| POST | `/credentials/{credential_id}/test` | Test connection using credential |
+| POST | `/credentials/{credential_id}/discover` | Discover available models |
+| POST | `/credentials/{credential_id}/register-models` | Register discovered models |
+| POST | `/credentials/migrate-from-provider-config` | Migrate from legacy ProviderConfig |
+
+**Supported Providers** (13 total):
+- Simple API key: `openai`, `anthropic`, `google`, `groq`, `mistral`, `deepseek`, `xai`, `openrouter`, `voyage`, `elevenlabs`
+- URL-based: `ollama`
+- Multi-field: `azure`, `vertex`, `openai_compatible`
+
+**Security Features**:
+- NEVER returns actual API key values (only metadata)
+- URL validation (SSRF protection) on all URL fields via `_validate_url()`
+- Allows private IPs and localhost for self-hosted services (Ollama, LM Studio)
+- Requires `OPEN_NOTEBOOK_ENCRYPTION_KEY` to be set for storing credentials
+
+### Domain Model: `Credential` (`open_notebook/domain/credential.py`)
+
+Individual credential records replacing the old `ProviderConfig` singleton. Each credential stores:
+- Provider name, display name, modalities
+- Encrypted API key (via Fernet)
+- Provider-specific config (base_url, endpoint, api_version, etc.)
+
+### Integration with Key Provider (`open_notebook/ai/key_provider.py`)
+
+The `key_provider` module provisions DB-stored credentials into environment variables for Esperanto compatibility:
+
+**Database-first Pattern**:
+1. API endpoint saves keys to `Credential` records (encrypted in SurrealDB)
+2. Before model provisioning, `provision_provider_keys(provider)` checks DB, then env vars
+3. Keys from DB are set as environment variables for Esperanto compatibility
+4. Existing env vars remain unchanged if no DB config exists
+
+**Key Functions**:
+- `get_api_key(provider)`: Get API key (DB first, env fallback)
+- `provision_provider_keys(provider)`: Set env vars from DB for a provider
+- `provision_all_keys()`: Load all provider keys from DB into env vars
+
+### Authentication
+
+No changes to authentication. The `credentials` router uses the same `PasswordAuthMiddleware` as all other endpoints. Keys are protected by the same password-based auth.
+
+**Auth Flow** (unchanged from `api/auth.py`):
+- `PasswordAuthMiddleware`: Global middleware checking `Authorization: Bearer {password}` header
+- Default password: `open-notebook-change-me` (set `OPEN_NOTEBOOK_PASSWORD` in production)
+- Docker secrets support via `OPEN_NOTEBOOK_PASSWORD_FILE`
+
+### Connection Testing (`open_notebook/ai/connection_tester.py`)
+
+The `/credentials/{credential_id}/test` endpoint uses minimal API calls to verify credentials:
+- Loads Credential via `Credential.get(config_id)`, uses `credential.to_esperanto_config()`
+- Uses cheapest/smallest models per provider (TEST_MODELS map)
+- Returns success status and descriptive message
+- Special handlers for ollama, openai_compatible, and azure providers
+
+### Migration Workflows
+
+Two migration endpoints help users transition to the credential system:
+
+**From environment variables** (`POST /credentials/migrate-from-env`):
+1. Checks each provider for env var presence
+2. Creates Credential records from env var values
+3. Returns summary: migrated, skipped, errors
+
+**From legacy ProviderConfig** (`POST /credentials/migrate-from-provider-config`):
+1. Reads old ProviderConfig records from database
+2. Converts each to individual Credential records
+3. Returns summary: migrated, skipped, errors
+
+### Example Usage
+
+```python
+# Check status
+GET /credentials/status
+# Response: {"configured": {"openai": true, "anthropic": false}, "source": {"openai": "database", "anthropic": "none"}, "encryption_configured": true}
+
+# Create credential
+POST /credentials
+{"name": "My OpenAI Key", "provider": "openai", "modalities": ["language", "embedding"], "api_key": "sk-proj-..."}
+
+# Test connection
+POST /credentials/{credential_id}/test
+# Response: {"provider": "openai", "success": true, "message": "Connection successful"}
+
+# Discover models
+POST /credentials/{credential_id}/discover
+# Response: {"provider": "openai", "models": [{"model_id": "gpt-4", "name": "gpt-4", ...}], "credential_id": "..."}
+
+# Migrate from env
+POST /credentials/migrate-from-env
+# Response: {"message": "Migration complete. Migrated 3 providers.", "migrated": ["openai", "anthropic", "groq"], "skipped": [], "errors": []}
+```

api/auth.py

@@ -0,0 +1,114 @@
+from typing import Optional
+
+from fastapi import Depends, HTTPException, Request
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+from loguru import logger
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.responses import JSONResponse
+
+from open_notebook.utils.encryption import get_secret_from_env
+
+
+class PasswordAuthMiddleware(BaseHTTPMiddleware):
+    """
+    Middleware to check password authentication for all API requests.
+    Always active with default password if OPEN_NOTEBOOK_PASSWORD is not set.
+    Supports Docker secrets via OPEN_NOTEBOOK_PASSWORD_FILE.
+    """
+
+    def __init__(self, app, excluded_paths: Optional[list] = None):
+        super().__init__(app)
+        self.password = get_secret_from_env("OPEN_NOTEBOOK_PASSWORD")
+        self.excluded_paths = excluded_paths or [
+            "/",
+            "/health",
+            "/docs",
+            "/openapi.json",
+            "/redoc",
+        ]
+
+    async def dispatch(self, request: Request, call_next):
+        # Skip authentication if no password is set
+        if not self.password:
+            return await call_next(request)
+
+        # Skip authentication for excluded paths
+        if request.url.path in self.excluded_paths:
+            return await call_next(request)
+
+        # Skip authentication for CORS preflight requests (OPTIONS)
+        if request.method == "OPTIONS":
+            return await call_next(request)
+
+        # Check authorization header
+        auth_header = request.headers.get("Authorization")
+
+        if not auth_header:
+            return JSONResponse(
+                status_code=401,
+                content={"detail": "Missing authorization header"},
+                headers={"WWW-Authenticate": "Bearer"},
+            )
+
+        # Expected format: "Bearer {password}"
+        try:
+            scheme, credentials = auth_header.split(" ", 1)
+            if scheme.lower() != "bearer":
+                raise ValueError("Invalid authentication scheme")
+        except ValueError:
+            return JSONResponse(
+                status_code=401,
+                content={"detail": "Invalid authorization header format"},
+                headers={"WWW-Authenticate": "Bearer"},
+            )
+
+        # Check password
+        if credentials != self.password:
+            return JSONResponse(
+                status_code=401,
+                content={"detail": "Invalid password"},
+                headers={"WWW-Authenticate": "Bearer"},
+            )
+
+        # Password is correct, proceed with the request
+        response = await call_next(request)
+        return response
+
+
+# Optional: HTTPBearer security scheme for OpenAPI documentation
+security = HTTPBearer(auto_error=False)
+
+
+def check_api_password(
+    credentials: Optional[HTTPAuthorizationCredentials] = Depends(security),
+) -> bool:
+    """
+    Utility function to check API password.
+    Can be used as a dependency in individual routes if needed.
+    Supports Docker secrets via OPEN_NOTEBOOK_PASSWORD_FILE.
+    Returns True without checking credentials if OPEN_NOTEBOOK_PASSWORD is not configured.
+    Raises 401 if credentials are missing or don't match the configured password.
+    """
+    password = get_secret_from_env("OPEN_NOTEBOOK_PASSWORD")
+
+    # No password configured - skip authentication
+    if not password:
+        return True
+
+    # No credentials provided
+    if not credentials:
+        raise HTTPException(
+            status_code=401,
+            detail="Missing authorization",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    # Check password
+    if credentials.credentials != password:
+        raise HTTPException(
+            status_code=401,
+            detail="Invalid password",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    return True

api/chat_service.py

@@ -0,0 +1,168 @@
+"""
+Chat service for API operations.
+Provides async interface for chat functionality.
+"""
+
+import os
+from typing import Any, Dict, List, Optional
+
+import httpx
+from loguru import logger
+
+
+class ChatService:
+    """Service for chat-related API operations"""
+
+    def __init__(self):
+        self.base_url = os.getenv("API_BASE_URL", "http://127.0.0.1:5055")
+        # Add authentication header if password is set
+        self.headers = {}
+        password = os.getenv("OPEN_NOTEBOOK_PASSWORD")
+        if password:
+            self.headers["Authorization"] = f"Bearer {password}"
+
+    async def get_sessions(self, notebook_id: str) -> List[Dict[str, Any]]:
+        """Get all chat sessions for a notebook"""
+        try:
+            async with httpx.AsyncClient() as client:
+                response = await client.get(
+                    f"{self.base_url}/api/chat/sessions",
+                    params={"notebook_id": notebook_id},
+                    headers=self.headers,
+                )
+                response.raise_for_status()
+                return response.json()
+        except Exception as e:
+            logger.error(f"Error fetching chat sessions: {str(e)}")
+            raise
+
+    async def create_session(
+        self,
+        notebook_id: str,
+        title: Optional[str] = None,
+        model_override: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Create a new chat session"""
+        try:
+            data: Dict[str, Any] = {"notebook_id": notebook_id}
+            if title is not None:
+                data["title"] = title
+            if model_override is not None:
+                data["model_override"] = model_override
+
+            async with httpx.AsyncClient() as client:
+                response = await client.post(
+                    f"{self.base_url}/api/chat/sessions",
+                    json=data,
+                    headers=self.headers,
+                )
+                response.raise_for_status()
+                return response.json()
+        except Exception as e:
+            logger.error(f"Error creating chat session: {str(e)}")
+            raise
+
+    async def get_session(self, session_id: str) -> Dict[str, Any]:
+        """Get a specific session with messages"""
+        try:
+            async with httpx.AsyncClient() as client:
+                response = await client.get(
+                    f"{self.base_url}/api/chat/sessions/{session_id}",
+                    headers=self.headers,
+                )
+                response.raise_for_status()
+                return response.json()
+        except Exception as e:
+            logger.error(f"Error fetching session: {str(e)}")
+            raise
+
+    async def update_session(
+        self,
+        session_id: str,
+        title: Optional[str] = None,
+        model_override: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Update session properties"""
+        try:
+            data: Dict[str, Any] = {}
+            if title is not None:
+                data["title"] = title
+            if model_override is not None:
+                data["model_override"] = model_override
+
+            if not data:
+                raise ValueError(
+                    "At least one field must be provided to update a session"
+                )
+
+            async with httpx.AsyncClient() as client:
+                response = await client.put(
+                    f"{self.base_url}/api/chat/sessions/{session_id}",
+                    json=data,
+                    headers=self.headers,
+                )
+                response.raise_for_status()
+                return response.json()
+        except Exception as e:
+            logger.error(f"Error updating session: {str(e)}")
+            raise
+
+    async def delete_session(self, session_id: str) -> Dict[str, Any]:
+        """Delete a chat session"""
+        try:
+            async with httpx.AsyncClient() as client:
+                response = await client.delete(
+                    f"{self.base_url}/api/chat/sessions/{session_id}",
+                    headers=self.headers,
+                )
+                response.raise_for_status()
+                return response.json()
+        except Exception as e:
+            logger.error(f"Error deleting session: {str(e)}")
+            raise
+
+    async def execute_chat(
+        self,
+        session_id: str,
+        message: str,
+        context: Dict[str, Any],
+        model_override: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Execute a chat request"""
+        try:
+            data = {"session_id": session_id, "message": message, "context": context}
+            if model_override is not None:
+                data["model_override"] = model_override
+
+            # Short connect timeout (10s), long read timeout (10 min) for Ollama/local LLMs
+            timeout = httpx.Timeout(connect=10.0, read=600.0, write=30.0, pool=10.0)
+            async with httpx.AsyncClient(timeout=timeout) as client:
+                response = await client.post(
+                    f"{self.base_url}/api/chat/execute", json=data, headers=self.headers
+                )
+                response.raise_for_status()
+                return response.json()
+        except Exception as e:
+            logger.error(f"Error executing chat: {str(e)}")
+            raise
+
+    async def build_context(
+        self, notebook_id: str, context_config: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        """Build context for a notebook"""
+        try:
+            data = {"notebook_id": notebook_id, "context_config": context_config}
+
+            async with httpx.AsyncClient() as client:
+                response = await client.post(
+                    f"{self.base_url}/api/chat/context", json=data, headers=self.headers
+                )
+                response.raise_for_status()
+                return response.json()
+        except Exception as e:
+            logger.error(f"Error building context: {str(e)}")
+            raise
+
+
+# Global instance
+chat_service = ChatService()

api/client.py

@@ -0,0 +1,529 @@
+"""
+API client for Open Notebook API.
+This module provides a client interface to interact with the Open Notebook API.
+"""
+
+import os
+from typing import Any, Dict, List, Optional, Union
+
+import httpx
+from loguru import logger
+
+
+class APIClient:
+    """Client for Open Notebook API."""
+
+    def __init__(self, base_url: Optional[str] = None):
+        self.base_url = base_url or os.getenv("API_BASE_URL", "http://127.0.0.1:5055")
+        # Timeout increased to 5 minutes (300s) to accommodate slow LLM operations
+        # (transformations, insights) on slower hardware (Ollama, LM Studio, remote APIs)
+        # Configurable via API_CLIENT_TIMEOUT environment variable (in seconds)
+        timeout_str = os.getenv("API_CLIENT_TIMEOUT", "300.0")
+        try:
+            timeout_value = float(timeout_str)
+            # Validate timeout is within reasonable bounds (30s - 3600s / 1 hour)
+            if timeout_value < 30:
+                logger.warning(
+                    f"API_CLIENT_TIMEOUT={timeout_value}s is too low, using minimum of 30s"
+                )
+                timeout_value = 30.0
+            elif timeout_value > 3600:
+                logger.warning(
+                    f"API_CLIENT_TIMEOUT={timeout_value}s is too high, using maximum of 3600s"
+                )
+                timeout_value = 3600.0
+            self.timeout = timeout_value
+        except ValueError:
+            logger.error(
+                f"Invalid API_CLIENT_TIMEOUT value '{timeout_str}', using default 300s"
+            )
+            self.timeout = 300.0
+
+        # Add authentication header if password is set
+        self.headers = {}
+        password = os.getenv("OPEN_NOTEBOOK_PASSWORD")
+        if password:
+            self.headers["Authorization"] = f"Bearer {password}"
+
+    def _make_request(
+        self, method: str, endpoint: str, timeout: Optional[float] = None, **kwargs
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Make HTTP request to the API."""
+        url = f"{self.base_url}{endpoint}"
+        request_timeout = timeout if timeout is not None else self.timeout
+
+        # Merge headers
+        headers = kwargs.get("headers", {})
+        headers.update(self.headers)
+        kwargs["headers"] = headers
+
+        try:
+            with httpx.Client(timeout=request_timeout) as client:
+                response = client.request(method, url, **kwargs)
+                response.raise_for_status()
+                return response.json()
+        except httpx.RequestError as e:
+            logger.error(f"Request error for {method} {url}: {str(e)}")
+            raise ConnectionError(f"Failed to connect to API: {str(e)}")
+        except httpx.HTTPStatusError as e:
+            logger.error(
+                f"HTTP error {e.response.status_code} for {method} {url}: {e.response.text}"
+            )
+            raise RuntimeError(
+                f"API request failed: {e.response.status_code} - {e.response.text}"
+            )
+        except Exception as e:
+            logger.error(f"Unexpected error for {method} {url}: {str(e)}")
+            raise
+
+    # Notebooks API methods
+    def get_notebooks(
+        self, archived: Optional[bool] = None, order_by: str = "updated desc"
+    ) -> List[Dict[Any, Any]]:
+        """Get all notebooks."""
+        params: Dict[str, Any] = {"order_by": order_by}
+        if archived is not None:
+            params["archived"] = str(archived).lower()
+
+        result = self._make_request("GET", "/api/notebooks", params=params)
+        return result if isinstance(result, list) else [result]
+
+    def create_notebook(
+        self, name: str, description: str = ""
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Create a new notebook."""
+        data = {"name": name, "description": description}
+        return self._make_request("POST", "/api/notebooks", json=data)
+
+    def get_notebook(
+        self, notebook_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get a specific notebook."""
+        return self._make_request("GET", f"/api/notebooks/{notebook_id}")
+
+    def update_notebook(
+        self, notebook_id: str, **updates
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Update a notebook."""
+        return self._make_request("PUT", f"/api/notebooks/{notebook_id}", json=updates)
+
+    def delete_notebook(
+        self, notebook_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Delete a notebook."""
+        return self._make_request("DELETE", f"/api/notebooks/{notebook_id}")
+
+    # Search API methods
+    def search(
+        self,
+        query: str,
+        search_type: str = "text",
+        limit: int = 100,
+        search_sources: bool = True,
+        search_notes: bool = True,
+        minimum_score: float = 0.2,
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Search the knowledge base."""
+        data = {
+            "query": query,
+            "type": search_type,
+            "limit": limit,
+            "search_sources": search_sources,
+            "search_notes": search_notes,
+            "minimum_score": minimum_score,
+        }
+        return self._make_request("POST", "/api/search", json=data)
+
+    def ask_simple(
+        self,
+        question: str,
+        strategy_model: str,
+        answer_model: str,
+        final_answer_model: str,
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Ask the knowledge base a question (simple, non-streaming)."""
+        data = {
+            "question": question,
+            "strategy_model": strategy_model,
+            "answer_model": answer_model,
+            "final_answer_model": final_answer_model,
+        }
+        # Use configured timeout for long-running ask operations
+        return self._make_request(
+            "POST", "/api/search/ask/simple", json=data, timeout=self.timeout
+        )
+
+    # Models API methods
+    def get_models(self, model_type: Optional[str] = None) -> List[Dict[Any, Any]]:
+        """Get all models with optional type filtering."""
+        params = {}
+        if model_type:
+            params["type"] = model_type
+        result = self._make_request("GET", "/api/models", params=params)
+        return result if isinstance(result, list) else [result]
+
+    def create_model(
+        self, name: str, provider: str, model_type: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Create a new model."""
+        data = {
+            "name": name,
+            "provider": provider,
+            "type": model_type,
+        }
+        return self._make_request("POST", "/api/models", json=data)
+
+    def delete_model(
+        self, model_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Delete a model."""
+        return self._make_request("DELETE", f"/api/models/{model_id}")
+
+    def get_default_models(self) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get default model assignments."""
+        return self._make_request("GET", "/api/models/defaults")
+
+    def update_default_models(
+        self, **defaults
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Update default model assignments."""
+        return self._make_request("PUT", "/api/models/defaults", json=defaults)
+
+    # Transformations API methods
+    def get_transformations(self) -> List[Dict[Any, Any]]:
+        """Get all transformations."""
+        result = self._make_request("GET", "/api/transformations")
+        return result if isinstance(result, list) else [result]
+
+    def create_transformation(
+        self,
+        name: str,
+        title: str,
+        description: str,
+        prompt: str,
+        apply_default: bool = False,
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Create a new transformation."""
+        data = {
+            "name": name,
+            "title": title,
+            "description": description,
+            "prompt": prompt,
+            "apply_default": apply_default,
+        }
+        return self._make_request("POST", "/api/transformations", json=data)
+
+    def get_transformation(
+        self, transformation_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get a specific transformation."""
+        return self._make_request("GET", f"/api/transformations/{transformation_id}")
+
+    def update_transformation(
+        self, transformation_id: str, **updates
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Update a transformation."""
+        return self._make_request(
+            "PUT", f"/api/transformations/{transformation_id}", json=updates
+        )
+
+    def delete_transformation(
+        self, transformation_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Delete a transformation."""
+        return self._make_request("DELETE", f"/api/transformations/{transformation_id}")
+
+    def execute_transformation(
+        self, transformation_id: str, input_text: str, model_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Execute a transformation on input text."""
+        data = {
+            "transformation_id": transformation_id,
+            "input_text": input_text,
+            "model_id": model_id,
+        }
+        # Use configured timeout for transformation operations
+        return self._make_request(
+            "POST", "/api/transformations/execute", json=data, timeout=self.timeout
+        )
+
+    # Notes API methods
+    def get_notes(self, notebook_id: Optional[str] = None) -> List[Dict[Any, Any]]:
+        """Get all notes with optional notebook filtering."""
+        params = {}
+        if notebook_id:
+            params["notebook_id"] = notebook_id
+        result = self._make_request("GET", "/api/notes", params=params)
+        return result if isinstance(result, list) else [result]
+
+    def create_note(
+        self,
+        content: str,
+        title: Optional[str] = None,
+        note_type: str = "human",
+        notebook_id: Optional[str] = None,
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Create a new note."""
+        data = {
+            "content": content,
+            "note_type": note_type,
+        }
+        if title:
+            data["title"] = title
+        if notebook_id:
+            data["notebook_id"] = notebook_id
+        return self._make_request("POST", "/api/notes", json=data)
+
+    def get_note(self, note_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get a specific note."""
+        return self._make_request("GET", f"/api/notes/{note_id}")
+
+    def update_note(
+        self, note_id: str, **updates
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Update a note."""
+        return self._make_request("PUT", f"/api/notes/{note_id}", json=updates)
+
+    def delete_note(self, note_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Delete a note."""
+        return self._make_request("DELETE", f"/api/notes/{note_id}")
+
+    # Embedding API methods
+    def embed_content(
+        self, item_id: str, item_type: str, async_processing: bool = False
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Embed content for vector search."""
+        data = {
+            "item_id": item_id,
+            "item_type": item_type,
+            "async_processing": async_processing,
+        }
+        # Use configured timeout for embedding operations
+        return self._make_request("POST", "/api/embed", json=data, timeout=self.timeout)
+
+    def rebuild_embeddings(
+        self,
+        mode: str = "existing",
+        include_sources: bool = True,
+        include_notes: bool = True,
+        include_insights: bool = True,
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Rebuild embeddings in bulk.
+
+        Note: This operation can take a long time for large databases.
+        Consider increasing API_CLIENT_TIMEOUT to 600-900s for bulk rebuilds.
+        """
+        data = {
+            "mode": mode,
+            "include_sources": include_sources,
+            "include_notes": include_notes,
+            "include_insights": include_insights,
+        }
+        # Use double the configured timeout for bulk rebuild operations (or configured value if already high)
+        rebuild_timeout = max(self.timeout, min(self.timeout * 2, 3600.0))
+        return self._make_request(
+            "POST", "/api/embeddings/rebuild", json=data, timeout=rebuild_timeout
+        )
+
+    def get_rebuild_status(
+        self, command_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get status of a rebuild operation."""
+        return self._make_request("GET", f"/api/embeddings/rebuild/{command_id}/status")
+
+    # Settings API methods
+    def get_settings(self) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get all application settings."""
+        return self._make_request("GET", "/api/settings")
+
+    def update_settings(
+        self, **settings
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Update application settings."""
+        return self._make_request("PUT", "/api/settings", json=settings)
+
+    # Context API methods
+    def get_notebook_context(
+        self, notebook_id: str, context_config: Optional[Dict] = None
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get context for a notebook."""
+        data: Dict[str, Any] = {"notebook_id": notebook_id}
+        if context_config:
+            data["context_config"] = context_config
+        result = self._make_request(
+            "POST", f"/api/notebooks/{notebook_id}/context", json=data
+        )
+        return result if isinstance(result, dict) else {}
+
+    # Sources API methods
+    def get_sources(self, notebook_id: Optional[str] = None) -> List[Dict[Any, Any]]:
+        """Get all sources with optional notebook filtering."""
+        params = {}
+        if notebook_id:
+            params["notebook_id"] = notebook_id
+        result = self._make_request("GET", "/api/sources", params=params)
+        return result if isinstance(result, list) else [result]
+
+    def create_source(
+        self,
+        notebook_id: Optional[str] = None,
+        notebooks: Optional[List[str]] = None,
+        source_type: str = "text",
+        url: Optional[str] = None,
+        file_path: Optional[str] = None,
+        content: Optional[str] = None,
+        title: Optional[str] = None,
+        transformations: Optional[List[str]] = None,
+        embed: bool = False,
+        delete_source: bool = False,
+        async_processing: bool = False,
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Create a new source."""
+        data = {
+            "type": source_type,
+            "embed": embed,
+            "delete_source": delete_source,
+            "async_processing": async_processing,
+        }
+
+        # Handle backward compatibility for notebook_id vs notebooks
+        if notebooks:
+            data["notebooks"] = notebooks
+        elif notebook_id:
+            data["notebook_id"] = notebook_id
+        else:
+            raise ValueError("Either notebook_id or notebooks must be provided")
+
+        if url:
+            data["url"] = url
+        if file_path:
+            data["file_path"] = file_path
+        if content:
+            data["content"] = content
+        if title:
+            data["title"] = title
+        if transformations:
+            data["transformations"] = transformations
+
+        # Use configured timeout for source creation (especially PDF processing with OCR)
+        return self._make_request(
+            "POST", "/api/sources/json", json=data, timeout=self.timeout
+        )
+
+    def get_source(self, source_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get a specific source."""
+        return self._make_request("GET", f"/api/sources/{source_id}")
+
+    def get_source_status(
+        self, source_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get processing status for a source."""
+        return self._make_request("GET", f"/api/sources/{source_id}/status")
+
+    def update_source(
+        self, source_id: str, **updates
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Update a source."""
+        return self._make_request("PUT", f"/api/sources/{source_id}", json=updates)
+
+    def delete_source(
+        self, source_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Delete a source."""
+        return self._make_request("DELETE", f"/api/sources/{source_id}")
+
+    # Insights API methods
+    def get_source_insights(self, source_id: str) -> List[Dict[Any, Any]]:
+        """Get all insights for a specific source."""
+        result = self._make_request("GET", f"/api/sources/{source_id}/insights")
+        return result if isinstance(result, list) else [result]
+
+    def get_insight(
+        self, insight_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get a specific insight."""
+        return self._make_request("GET", f"/api/insights/{insight_id}")
+
+    def delete_insight(
+        self, insight_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Delete a specific insight."""
+        return self._make_request("DELETE", f"/api/insights/{insight_id}")
+
+    def save_insight_as_note(
+        self, insight_id: str, notebook_id: Optional[str] = None
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Convert an insight to a note."""
+        data = {}
+        if notebook_id:
+            data["notebook_id"] = notebook_id
+        return self._make_request(
+            "POST", f"/api/insights/{insight_id}/save-as-note", json=data
+        )
+
+    def create_source_insight(
+        self, source_id: str, transformation_id: str, model_id: Optional[str] = None
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Create a new insight for a source by running a transformation."""
+        data = {"transformation_id": transformation_id}
+        if model_id:
+            data["model_id"] = model_id
+        return self._make_request(
+            "POST", f"/api/sources/{source_id}/insights", json=data
+        )
+
+    # Episode Profiles API methods
+    def get_episode_profiles(self) -> List[Dict[Any, Any]]:
+        """Get all episode profiles."""
+        result = self._make_request("GET", "/api/episode-profiles")
+        return result if isinstance(result, list) else [result]
+
+    def get_episode_profile(
+        self, profile_name: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get a specific episode profile by name."""
+        return self._make_request("GET", f"/api/episode-profiles/{profile_name}")
+
+    def create_episode_profile(
+        self,
+        name: str,
+        description: str = "",
+        speaker_config: str = "",
+        outline_provider: str = "",
+        outline_model: str = "",
+        transcript_provider: str = "",
+        transcript_model: str = "",
+        default_briefing: str = "",
+        num_segments: int = 5,
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Create a new episode profile."""
+        data = {
+            "name": name,
+            "description": description,
+            "speaker_config": speaker_config,
+            "outline_provider": outline_provider,
+            "outline_model": outline_model,
+            "transcript_provider": transcript_provider,
+            "transcript_model": transcript_model,
+            "default_briefing": default_briefing,
+            "num_segments": num_segments,
+        }
+        return self._make_request("POST", "/api/episode-profiles", json=data)
+
+    def update_episode_profile(
+        self, profile_id: str, **updates
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Update an episode profile."""
+        return self._make_request(
+            "PUT", f"/api/episode-profiles/{profile_id}", json=updates
+        )
+
+    def delete_episode_profile(
+        self, profile_id: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Delete an episode profile."""
+        return self._make_request("DELETE", f"/api/episode-profiles/{profile_id}")
+
+
+# Global client instance
+api_client = APIClient()

api/command_service.py

@@ -0,0 +1,92 @@
+from typing import Any, Dict, List, Optional
+
+from loguru import logger
+from surreal_commands import get_command_status, submit_command
+
+
+class CommandService:
+    """Generic service layer for command operations"""
+
+    @staticmethod
+    async def submit_command_job(
+        module_name: str,  # Actually app_name for surreal-commands
+        command_name: str,
+        command_args: Dict[str, Any],
+        context: Optional[Dict[str, Any]] = None,
+    ) -> str:
+        """Submit a generic command job for background processing"""
+        try:
+            # Ensure command modules are imported before submitting
+            # This is needed because submit_command validates against local registry
+            try:
+                import commands.podcast_commands  # noqa: F401
+            except ImportError as import_err:
+                logger.error(f"Failed to import command modules: {import_err}")
+                raise ValueError("Command modules not available")
+
+            # surreal-commands expects: submit_command(app_name, command_name, args)
+            cmd_id = submit_command(
+                module_name,  # This is actually the app name (e.g., "open_notebook")
+                command_name,  # Command name (e.g., "process_text")
+                command_args,  # Input data
+            )
+            # Convert RecordID to string if needed
+            if not cmd_id:
+                raise ValueError("Failed to get cmd_id from submit_command")
+            cmd_id_str = str(cmd_id)
+            logger.info(
+                f"Submitted command job: {cmd_id_str} for {module_name}.{command_name}"
+            )
+            return cmd_id_str
+
+        except Exception as e:
+            logger.error(f"Failed to submit command job: {e}")
+            raise
+
+    @staticmethod
+    async def get_command_status(job_id: str) -> Dict[str, Any]:
+        """Get status of any command job"""
+        try:
+            status = await get_command_status(job_id)
+            return {
+                "job_id": job_id,
+                "status": status.status if status else "unknown",
+                "result": status.result if status else None,
+                "error_message": getattr(status, "error_message", None)
+                if status
+                else None,
+                "created": str(status.created)
+                if status and hasattr(status, "created") and status.created
+                else None,
+                "updated": str(status.updated)
+                if status and hasattr(status, "updated") and status.updated
+                else None,
+                "progress": getattr(status, "progress", None) if status else None,
+            }
+        except Exception as e:
+            logger.error(f"Failed to get command status: {e}")
+            raise
+
+    @staticmethod
+    async def list_command_jobs(
+        module_filter: Optional[str] = None,
+        command_filter: Optional[str] = None,
+        status_filter: Optional[str] = None,
+        limit: int = 50,
+    ) -> List[Dict[str, Any]]:
+        """List command jobs with optional filtering"""
+        # This will be implemented with proper SurrealDB queries
+        # For now, return empty list as this is foundation phase
+        return []
+
+    @staticmethod
+    async def cancel_command_job(job_id: str) -> bool:
+        """Cancel a running command job"""
+        try:
+            # Implementation depends on surreal-commands cancellation support
+            # For now, just log the attempt
+            logger.info(f"Attempting to cancel job: {job_id}")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to cancel command job: {e}")
+            raise

api/context_service.py

@@ -0,0 +1,29 @@
+"""
+Context service layer using API.
+"""
+
+from typing import Any, Dict, List, Optional, Union
+
+from loguru import logger
+
+from api.client import api_client
+
+
+class ContextService:
+    """Service layer for context operations using API."""
+
+    def __init__(self):
+        logger.info("Using API for context operations")
+
+    def get_notebook_context(
+        self, notebook_id: str, context_config: Optional[Dict] = None
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Get context for a notebook."""
+        result = api_client.get_notebook_context(
+            notebook_id=notebook_id, context_config=context_config
+        )
+        return result
+
+
+# Global service instance
+context_service = ContextService()

api/credentials_service.py

@@ -0,0 +1,915 @@
+"""
+Credentials Service
+
+Business logic for managing AI provider credentials.
+Extracted from the credentials router to follow the service layer pattern.
+
+All functions raise ValueError for business errors (router converts to HTTPException).
+"""
+
+import ipaddress
+import os
+import socket
+from typing import Dict, List, Optional
+from urllib.parse import urlparse
+
+import httpx
+from loguru import logger
+from pydantic import SecretStr
+
+from api.models import CredentialResponse
+from open_notebook.ai.model_discovery import classify_model_type
+from open_notebook.domain.credential import Credential
+from open_notebook.utils.encryption import get_secret_from_env
+
+# =============================================================================
+# Constants
+# =============================================================================
+
+# Provider environment variable configuration.
+# - "required": ALL listed env vars must be set for the provider to be considered configured.
+# - "required_any": at least ONE of the listed env vars must be set.
+# - "optional": additional env vars used during migration but not required.
+PROVIDER_ENV_CONFIG: Dict[str, dict] = {
+    "openai": {"required": ["OPENAI_API_KEY"]},
+    "anthropic": {"required": ["ANTHROPIC_API_KEY"]},
+    "google": {"required_any": ["GOOGLE_API_KEY", "GEMINI_API_KEY"]},
+    "groq": {"required": ["GROQ_API_KEY"]},
+    "mistral": {"required": ["MISTRAL_API_KEY"]},
+    "deepseek": {"required": ["DEEPSEEK_API_KEY"]},
+    "xai": {"required": ["XAI_API_KEY"]},
+    "openrouter": {"required": ["OPENROUTER_API_KEY"]},
+    "voyage": {"required": ["VOYAGE_API_KEY"]},
+    "elevenlabs": {"required": ["ELEVENLABS_API_KEY"]},
+    "deepgram": {"required": ["DEEPGRAM_API_KEY"]},
+    "ollama": {"required": ["OLLAMA_API_BASE"]},
+    "vertex": {
+        "required": ["VERTEX_PROJECT", "VERTEX_LOCATION"],
+        "optional": ["GOOGLE_APPLICATION_CREDENTIALS"],
+    },
+    "azure": {
+        "required": ["AZURE_OPENAI_API_KEY", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_API_VERSION"],
+        "optional": [
+            "AZURE_OPENAI_ENDPOINT_LLM",
+            "AZURE_OPENAI_ENDPOINT_EMBEDDING",
+            "AZURE_OPENAI_ENDPOINT_STT",
+            "AZURE_OPENAI_ENDPOINT_TTS",
+        ],
+    },
+    "openai_compatible": {
+        "required_any": ["OPENAI_COMPATIBLE_BASE_URL", "OPENAI_COMPATIBLE_API_KEY"],
+    },
+    "dashscope": {"required": ["DASHSCOPE_API_KEY"]},
+    "minimax": {"required": ["MINIMAX_API_KEY"]},
+}
+
+PROVIDER_MODALITIES: Dict[str, List[str]] = {
+    "openai": ["language", "embedding", "speech_to_text", "text_to_speech"],
+    "anthropic": ["language"],
+    "google": ["language", "embedding", "speech_to_text", "text_to_speech"],
+    "groq": ["language", "speech_to_text"],
+    "mistral": ["language", "embedding", "speech_to_text", "text_to_speech"],
+    "deepseek": ["language"],
+    "xai": ["language", "text_to_speech"],
+    "openrouter": ["language", "embedding"],
+    "voyage": ["embedding"],
+    "elevenlabs": ["text_to_speech", "speech_to_text"],
+    "deepgram": ["text_to_speech"],
+    "ollama": ["language", "embedding"],
+    "vertex": ["language", "embedding", "text_to_speech"],
+    "azure": ["language", "embedding", "speech_to_text", "text_to_speech"],
+    "openai_compatible": ["language", "embedding", "speech_to_text", "text_to_speech"],
+    "dashscope": ["language"],
+    "minimax": ["language"],
+}
+
+
+# =============================================================================
+# URL Validation (SSRF protection)
+# =============================================================================
+
+
+def validate_url(url: str, provider: str) -> None:
+    """
+    Validate URL format for API endpoints.
+
+    This is a self-hosted application, so we allow:
+    - Private IPs (10.x, 172.16-31.x, 192.168.x) for self-hosted services
+    - Localhost for local services (Ollama, LM Studio, etc.)
+
+    We only block:
+    - Invalid schemes (must be http or https)
+    - Malformed URLs
+    - Link-local addresses (169.254.x.x) - used for cloud metadata endpoints
+    - Hostnames that resolve to link-local addresses
+
+    Args:
+        url: The URL to validate
+        provider: The provider name (for logging/context)
+
+    Raises:
+        ValueError: If the URL is invalid
+    """
+    if not url or not url.strip():
+        return  # Empty URLs handled elsewhere
+
+    try:
+        parsed = urlparse(url.strip())
+
+        # Validate scheme - only http/https allowed
+        if parsed.scheme not in ("http", "https"):
+            raise ValueError(
+                f"Invalid URL scheme: '{parsed.scheme}'. Only http and https are allowed."
+            )
+
+        # Extract hostname
+        hostname = parsed.hostname
+        if not hostname:
+            raise ValueError("Invalid URL: hostname could not be determined.")
+
+        # Try to parse as IP address to check for dangerous addresses
+        try:
+            ip = ipaddress.ip_address(hostname)
+
+            # Block link-local addresses (169.254.x.x) - used for cloud metadata
+            # These are dangerous as they can expose cloud instance credentials
+            if ip.is_link_local:
+                raise ValueError(
+                    "Link-local addresses (169.254.x.x) are not allowed for security reasons. "
+                    "These addresses are used for cloud metadata endpoints."
+                )
+
+            # Block IPv4-mapped IPv6 addresses pointing to link-local
+            # e.g. ::ffff:169.254.169.254 bypasses IPv6 is_link_local check
+            if hasattr(ip, "ipv4_mapped") and ip.ipv4_mapped and ip.ipv4_mapped.is_link_local:
+                raise ValueError(
+                    "Link-local addresses (169.254.x.x) are not allowed for security reasons. "
+                    "These addresses are used for cloud metadata endpoints."
+                )
+
+        except ValueError as ve:
+            # Re-raise our own ValueErrors
+            if "Link-local" in str(ve) or "Invalid URL" in str(ve):
+                raise
+            # Not an IP address, it's a hostname - need to resolve and check
+            try:
+                # Resolve hostname to IP address
+                resolved_ips = socket.getaddrinfo(hostname, None)
+                for family, _, _, _, sockaddr in resolved_ips:
+                    ip_addr = sockaddr[0]
+                    try:
+                        parsed_ip = ipaddress.ip_address(ip_addr)
+                        if parsed_ip.is_link_local:
+                            raise ValueError(
+                                f"Hostname '{hostname}' resolves to a link-local address (169.254.x.x) which is not allowed for security reasons. "
+                                "These addresses are used for cloud metadata endpoints."
+                            )
+                        # Block IPv4-mapped IPv6 addresses pointing to link-local
+                        if (
+                            hasattr(parsed_ip, "ipv4_mapped")
+                            and parsed_ip.ipv4_mapped
+                            and parsed_ip.ipv4_mapped.is_link_local
+                        ):
+                            raise ValueError(
+                                f"Hostname '{hostname}' resolves to a link-local address (169.254.x.x) which is not allowed for security reasons. "
+                                "These addresses are used for cloud metadata endpoints."
+                            )
+                    except ValueError as inner_ve:
+                        if "link-local" in str(inner_ve).lower() or "Link-local" in str(inner_ve):
+                            raise
+                        # Skip non-IP addresses (e.g., IPv6 zones)
+                        continue
+            except socket.gaierror:
+                # Could not resolve hostname - allow it since the URL may be
+                # valid in the deployment environment (e.g., Azure endpoints,
+                # internal DNS names). We only block link-local addresses.
+                pass
+
+    except ValueError:
+        raise
+    except Exception:
+        raise ValueError("Invalid URL format. Check server logs for details.")
+
+
+# =============================================================================
+# Helpers
+# =============================================================================
+
+
+def require_encryption_key() -> None:
+    """Raise ValueError if encryption key is not configured."""
+    if not get_secret_from_env("OPEN_NOTEBOOK_ENCRYPTION_KEY"):
+        raise ValueError(
+            "Encryption key not configured. "
+            "Set OPEN_NOTEBOOK_ENCRYPTION_KEY to enable storing API keys."
+        )
+
+
+def credential_to_response(cred: Credential, model_count: int = 0) -> CredentialResponse:
+    """Convert a Credential domain object to API response."""
+    return CredentialResponse(
+        id=cred.id or "",
+        name=cred.name,
+        provider=cred.provider,
+        modalities=cred.modalities,
+        base_url=cred.base_url,
+        endpoint=cred.endpoint,
+        api_version=cred.api_version,
+        endpoint_llm=cred.endpoint_llm,
+        endpoint_embedding=cred.endpoint_embedding,
+        endpoint_stt=cred.endpoint_stt,
+        endpoint_tts=cred.endpoint_tts,
+        project=cred.project,
+        location=cred.location,
+        credentials_path=cred.credentials_path,
+        num_ctx=cred.num_ctx,
+        has_api_key=cred.api_key is not None,
+        created=str(cred.created) if cred.created else "",
+        updated=str(cred.updated) if cred.updated else "",
+        model_count=model_count,
+        decryption_error=cred.decryption_error,
+    )
+
+
+def check_env_configured(provider: str) -> bool:
+    """Check if a provider has sufficient env vars configured for migration."""
+    config = PROVIDER_ENV_CONFIG.get(provider)
+    if not config:
+        return False
+
+    if "required_any" in config:
+        return any(bool(os.environ.get(v, "").strip()) for v in config["required_any"])
+    elif "required" in config:
+        return all(bool(os.environ.get(v, "").strip()) for v in config["required"])
+    return False
+
+
+def get_default_modalities(provider: str) -> List[str]:
+    """Get default modalities for a provider."""
+    return PROVIDER_MODALITIES.get(provider.lower(), ["language"])
+
+
+def create_credential_from_env(provider: str) -> Credential:
+    """Create a Credential from environment variables for a given provider."""
+    modalities = get_default_modalities(provider)
+    name = "Default (Migrated from env)"
+
+    if provider == "ollama":
+        return Credential(
+            name=name,
+            provider=provider,
+            modalities=modalities,
+            base_url=os.environ.get("OLLAMA_API_BASE"),
+        )
+    elif provider == "vertex":
+        return Credential(
+            name=name,
+            provider=provider,
+            modalities=modalities,
+            project=os.environ.get("VERTEX_PROJECT"),
+            location=os.environ.get("VERTEX_LOCATION"),
+            credentials_path=os.environ.get("GOOGLE_APPLICATION_CREDENTIALS"),
+        )
+    elif provider == "azure":
+        return Credential(
+            name=name,
+            provider=provider,
+            modalities=modalities,
+            api_key=SecretStr(os.environ["AZURE_OPENAI_API_KEY"]),
+            endpoint=os.environ.get("AZURE_OPENAI_ENDPOINT"),
+            api_version=os.environ.get("AZURE_OPENAI_API_VERSION"),
+            endpoint_llm=os.environ.get("AZURE_OPENAI_ENDPOINT_LLM"),
+            endpoint_embedding=os.environ.get("AZURE_OPENAI_ENDPOINT_EMBEDDING"),
+            endpoint_stt=os.environ.get("AZURE_OPENAI_ENDPOINT_STT"),
+            endpoint_tts=os.environ.get("AZURE_OPENAI_ENDPOINT_TTS"),
+        )
+    elif provider == "openai_compatible":
+        api_key = os.environ.get("OPENAI_COMPATIBLE_API_KEY")
+        return Credential(
+            name=name,
+            provider=provider,
+            modalities=modalities,
+            api_key=SecretStr(api_key) if api_key else None,
+            base_url=os.environ.get("OPENAI_COMPATIBLE_BASE_URL"),
+        )
+    elif provider == "google":
+        # Support both GOOGLE_API_KEY and GEMINI_API_KEY (fallback)
+        api_key = os.environ.get("GOOGLE_API_KEY") or os.environ.get("GEMINI_API_KEY")
+        return Credential(
+            name=name,
+            provider=provider,
+            modalities=modalities,
+            api_key=SecretStr(api_key) if api_key else None,
+        )
+    else:
+        # Simple API key providers
+        config = PROVIDER_ENV_CONFIG.get(provider, {})
+        required = config.get("required", [])
+        env_var = required[0] if required else None
+        api_key = os.environ.get(env_var) if env_var else None
+        return Credential(
+            name=name,
+            provider=provider,
+            modalities=modalities,
+            api_key=SecretStr(api_key) if api_key else None,
+        )
+
+
+# =============================================================================
+# Service Functions
+# =============================================================================
+
+
+async def get_provider_status() -> dict:
+    """
+    Get configuration status: encryption key status, and per-provider
+    configured/source information.
+    """
+    encryption_configured = bool(get_secret_from_env("OPEN_NOTEBOOK_ENCRYPTION_KEY"))
+
+    configured: Dict[str, bool] = {}
+    source: Dict[str, str] = {}
+
+    for provider in PROVIDER_ENV_CONFIG:
+        env_configured = check_env_configured(provider)
+        try:
+            db_credentials = await Credential.get_by_provider(provider)
+            db_configured = len(db_credentials) > 0
+        except Exception:
+            db_configured = False
+
+        configured[provider] = db_configured or env_configured
+
+        if db_configured:
+            source[provider] = "database"
+        elif env_configured:
+            source[provider] = "environment"
+        else:
+            source[provider] = "none"
+
+    return {
+        "configured": configured,
+        "source": source,
+        "encryption_configured": encryption_configured,
+    }
+
+
+async def get_env_status() -> Dict[str, bool]:
+    """Check what's configured via environment variables."""
+    env_status: Dict[str, bool] = {}
+    for provider in PROVIDER_ENV_CONFIG:
+        env_status[provider] = check_env_configured(provider)
+    return env_status
+
+
+async def test_credential(credential_id: str) -> dict:
+    """
+    Test connection using a credential's configuration.
+
+    Returns dict with provider, success, message keys.
+    """
+    provider = "unknown"
+    try:
+        cred = await Credential.get(credential_id)
+        config = cred.to_esperanto_config()
+
+        from open_notebook.ai.connection_tester import (
+            _test_azure_connection,
+            _test_ollama_connection,
+            _test_openai_compatible_connection,
+        )
+
+        provider = cred.provider.lower()
+
+        # Handle special providers
+        if provider == "ollama":
+            base_url = config.get("base_url", "http://localhost:11434")
+            success, message = await _test_ollama_connection(base_url)
+            return {"provider": provider, "success": success, "message": message}
+
+        if provider == "openai_compatible":
+            base_url = config.get("base_url")
+            api_key = config.get("api_key")
+            if not base_url:
+                return {
+                    "provider": provider,
+                    "success": False,
+                    "message": "No base URL configured",
+                }
+            success, message = await _test_openai_compatible_connection(
+                base_url, api_key
+            )
+            return {"provider": provider, "success": success, "message": message}
+
+        if provider == "azure":
+            success, message = await _test_azure_connection(
+                endpoint=config.get("endpoint"),
+                api_key=config.get("api_key"),
+                api_version=config.get("api_version"),
+            )
+            return {"provider": provider, "success": success, "message": message}
+
+        # Standard provider: use Esperanto to create and test
+        from esperanto.factory import AIFactory
+
+        from open_notebook.ai.connection_tester import TEST_MODELS
+
+        if provider not in TEST_MODELS:
+            return {
+                "provider": provider,
+                "success": False,
+                "message": f"Unknown provider: {provider}",
+            }
+
+        test_model, test_type = TEST_MODELS[provider]
+        if not test_model:
+            return {
+                "provider": provider,
+                "success": False,
+                "message": f"No test model configured for {provider}",
+            }
+
+        if test_type == "language":
+            model = AIFactory.create_language(
+                model_name=test_model, provider=provider, config=config
+            )
+            lc_model = model.to_langchain()
+            await lc_model.ainvoke("Hi")
+            return {"provider": provider, "success": True, "message": "Connection successful"}
+
+        elif test_type == "embedding":
+            model = AIFactory.create_embedding(
+                model_name=test_model, provider=provider, config=config
+            )
+            await model.aembed(["test"])
+            return {"provider": provider, "success": True, "message": "Connection successful"}
+
+        elif test_type == "text_to_speech":
+            AIFactory.create_text_to_speech(model_name=test_model, provider=provider, config=config)
+            return {
+                "provider": provider,
+                "success": True,
+                "message": "Connection successful (key format valid)",
+            }
+
+        return {
+            "provider": provider,
+            "success": False,
+            "message": f"Unsupported test type: {test_type}",
+        }
+
+    except Exception as e:
+        error_msg = str(e)
+        if "401" in error_msg or "unauthorized" in error_msg.lower():
+            return {"provider": provider, "success": False, "message": "Invalid API key"}
+        elif "403" in error_msg or "forbidden" in error_msg.lower():
+            return {"provider": provider, "success": False, "message": "API key lacks required permissions"}
+        elif "rate" in error_msg.lower() and "limit" in error_msg.lower():
+            return {"provider": provider, "success": True, "message": "Rate limited - but connection works"}
+        elif "not found" in error_msg.lower() and "model" in error_msg.lower():
+            return {"provider": provider, "success": True, "message": "API key valid (test model not available)"}
+        else:
+            logger.debug(f"Test connection error for credential {credential_id}: {e}")
+            truncated = error_msg[:100] + "..." if len(error_msg) > 100 else error_msg
+            return {"provider": provider, "success": False, "message": f"Error: {truncated}"}
+
+
+async def discover_with_config(provider: str, config: dict) -> List[dict]:
+    """
+    Discover models using explicit config instead of env vars.
+
+    Returns model names only — no type classification.
+    The user chooses the model type when registering.
+    """
+    api_key = config.get("api_key")
+    base_url = config.get("base_url")
+
+    def models_endpoint(url: str) -> str:
+        trimmed = url.rstrip("/")
+        if trimmed.endswith("/models"):
+            return trimmed
+        return f"{trimmed}/models"
+
+    # Static model lists for providers without a listing API
+    STATIC_MODELS: Dict[str, List[str]] = {
+        "anthropic": [
+            "claude-opus-4-20250514",
+            "claude-sonnet-4-20250514",
+            "claude-3-5-sonnet-20241022",
+            "claude-3-5-haiku-20241022",
+            "claude-3-opus-20240229",
+            "claude-3-sonnet-20240229",
+            "claude-3-haiku-20240307",
+        ],
+        "voyage": [
+            "voyage-3", "voyage-3-lite", "voyage-code-3",
+            "voyage-finance-2", "voyage-law-2", "voyage-multilingual-2",
+        ],
+        "elevenlabs": [
+            "eleven_multilingual_v2", "eleven_turbo_v2_5",
+            "eleven_turbo_v2", "eleven_monolingual_v1",
+            "scribe_v1",  # speech-to-text
+        ],
+        "deepgram": [
+            "aura-2-thalia-en", "aura-2-andromeda-en", "aura-2-helena-en",
+            "aura-2-apollo-en", "aura-2-arcas-en", "aura-2-asteria-en",
+            "aura-2-athena-en", "aura-2-hera-en", "aura-2-hermes-en",
+            "aura-2-atlas-en",
+        ],
+    }
+
+    if provider in STATIC_MODELS:
+        if not api_key and provider != "ollama":
+            return []
+        return [
+            {"name": m, "provider": provider}
+            for m in STATIC_MODELS[provider]
+        ]
+
+    # API-based discovery URLs (OpenAI-style /models endpoints)
+    url_map = {
+        "openai": "https://api.openai.com/v1/models",
+        "groq": "https://api.groq.com/openai/v1/models",
+        "mistral": "https://api.mistral.ai/v1/models",
+        "deepseek": "https://api.deepseek.com/models",
+        "xai": "https://api.x.ai/v1/models",
+        "openrouter": "https://openrouter.ai/api/v1/models",
+        "dashscope": "https://dashscope.aliyuncs.com/compatible-mode/v1/models",
+        "minimax": "https://api.minimax.io/v1/models",
+    }
+
+    if provider == "ollama":
+        ollama_url = base_url or "http://localhost:11434"
+        try:
+            async with httpx.AsyncClient() as client:
+                response = await client.get(f"{ollama_url}/api/tags", timeout=10.0)
+                response.raise_for_status()
+                data = response.json()
+                return [
+                    {
+                        "name": m.get("name", ""),
+                        "provider": "ollama",
+                        "model_type": classify_model_type(m.get("name", ""), "ollama"),
+                    }
+                    for m in data.get("models", [])
+                    if m.get("name")
+                ]
+        except Exception as e:
+            logger.warning(f"Failed to discover Ollama models: {e}")
+            return []
+
+    if provider == "openai_compatible":
+        if not base_url:
+            return []
+        try:
+            headers = {}
+            if api_key:
+                headers["Authorization"] = f"Bearer {api_key}"
+            async with httpx.AsyncClient() as client:
+                response = await client.get(
+                    models_endpoint(base_url),
+                    headers=headers,
+                    timeout=30.0,
+                )
+                response.raise_for_status()
+                data = response.json()
+                return [
+                    {"name": m.get("id", ""), "provider": "openai_compatible"}
+                    for m in data.get("data", [])
+                    if m.get("id")
+                ]
+        except Exception as e:
+            logger.warning(f"Failed to discover openai_compatible models: {e}")
+            return []
+
+    if provider == "azure":
+        endpoint = config.get("endpoint")
+        api_version = config.get("api_version", "2024-10-21")
+        if not endpoint or not api_key:
+            return []
+        try:
+            url = f"{endpoint.rstrip('/')}/openai/models?api-version={api_version}"
+            headers = {"api-key": api_key}
+            async with httpx.AsyncClient() as client:
+                response = await client.get(url, headers=headers, timeout=30.0)
+                response.raise_for_status()
+                data = response.json()
+                return [
+                    {"name": m.get("id", ""), "provider": "azure"}
+                    for m in data.get("data", [])
+                    if m.get("id")
+                ]
+        except Exception as e:
+            logger.warning(f"Failed to discover Azure models: {e}")
+            return []
+
+    if provider == "vertex":
+        # Vertex AI requires service-account OAuth2 for model listing.
+        # Return a curated static list of well-known Vertex models instead.
+        VERTEX_MODELS = [
+            "gemini-2.0-flash",
+            "gemini-2.0-flash-lite",
+            "gemini-1.5-pro",
+            "gemini-1.5-flash",
+            "text-embedding-005",
+        ]
+        return [{"name": m, "provider": "vertex"} for m in VERTEX_MODELS]
+
+    if provider == "google":
+        try:
+            headers = {"X-Goog-Api-Key": api_key} if api_key else {}
+            async with httpx.AsyncClient() as client:
+                response = await client.get(
+                    "https://generativelanguage.googleapis.com/v1/models",
+                    headers=headers,
+                    timeout=30.0,
+                )
+                response.raise_for_status()
+                data = response.json()
+                return [
+                    {
+                        "name": model.get("name", "").replace("models/", ""),
+                        "provider": "google",
+                        "description": model.get("displayName"),
+                    }
+                    for model in data.get("models", [])
+                    if model.get("name")
+                ]
+        except Exception as e:
+            logger.warning(f"Failed to discover Google models: {e}")
+            return []
+
+    # Standard OpenAI-style API discovery
+    discovery_url = url_map.get(provider)
+    if provider == "openai" and base_url:
+        discovery_url = models_endpoint(base_url)
+    if not discovery_url or not api_key:
+        return []
+
+    try:
+        async with httpx.AsyncClient() as client:
+            response = await client.get(
+                discovery_url,
+                headers={"Authorization": f"Bearer {api_key}"},
+                timeout=30.0,
+            )
+            response.raise_for_status()
+            data = response.json()
+
+            return [
+                {
+                    "name": m.get("id", ""),
+                    "provider": provider,
+                    "description": m.get("name"),
+                }
+                for m in data.get("data", [])
+                if m.get("id")
+            ]
+    except Exception as e:
+        logger.warning(f"Failed to discover {provider} models: {e}")
+        return []
+
+
+async def register_models(credential_id: str, models_data: list) -> dict:
+    """
+    Register discovered models and link them to a credential.
+
+    Args:
+        credential_id: The credential ID to link models to
+        models_data: List of dicts with name, provider, model_type
+
+    Returns:
+        dict with created and existing counts
+    """
+    cred = await Credential.get(credential_id)
+
+    from open_notebook.ai.models import Model
+    from open_notebook.database.repository import repo_query
+
+    # Batch fetch existing models for this provider
+    existing_models = await repo_query(
+        "SELECT string::lowercase(name) as name, string::lowercase(type) as type FROM model "
+        "WHERE string::lowercase(provider) = $provider",
+        {"provider": cred.provider.lower()},
+    )
+    existing_keys = {(m["name"], m["type"]) for m in existing_models}
+
+    created = 0
+    existing = 0
+
+    for model_data in models_data:
+        key = (model_data.name.lower(), model_data.model_type.lower())
+        if key in existing_keys:
+            existing += 1
+            continue
+
+        new_model = Model(
+            name=model_data.name,
+            provider=model_data.provider or cred.provider,
+            type=model_data.model_type,
+            credential=cred.id,
+        )
+        await new_model.save()
+        created += 1
+
+    return {"created": created, "existing": existing}
+
+
+async def migrate_from_provider_config() -> dict:
+    """
+    Migrate existing ProviderConfig data to individual credential records.
+
+    Returns dict with message, migrated, skipped, errors.
+    """
+    logger.info("=== Starting ProviderConfig migration ===")
+
+    require_encryption_key()
+    logger.info("Encryption key verified")
+
+    from open_notebook.domain.provider_config import ProviderConfig
+
+    config = await ProviderConfig.get_instance()
+    logger.info(
+        f"Found ProviderConfig with {len(config.credentials)} provider(s): "
+        f"{', '.join(config.credentials.keys())}"
+    )
+
+    migrated = []
+    skipped = []
+    errors = []
+
+    for provider, credentials_list in config.credentials.items():
+        for old_cred in credentials_list:
+            try:
+                # Check if a credential already exists for this provider with same name
+                existing = await Credential.get_by_provider(provider)
+                names = [c.name for c in existing]
+                if old_cred.name in names:
+                    logger.info(
+                        f"[{provider}/{old_cred.name}] Already exists in DB, skipping"
+                    )
+                    skipped.append(f"{provider}/{old_cred.name}")
+                    continue
+
+                # Determine modalities from the provider type
+                modalities = get_default_modalities(provider)
+
+                logger.info(f"[{provider}/{old_cred.name}] Creating credential")
+                new_cred = Credential(
+                    name=old_cred.name,
+                    provider=provider,
+                    modalities=modalities,
+                    api_key=old_cred.api_key,
+                    base_url=old_cred.base_url,
+                    endpoint=old_cred.endpoint,
+                    api_version=old_cred.api_version,
+                    endpoint_llm=old_cred.endpoint_llm,
+                    endpoint_embedding=old_cred.endpoint_embedding,
+                    endpoint_stt=old_cred.endpoint_stt,
+                    endpoint_tts=old_cred.endpoint_tts,
+                    project=old_cred.project,
+                    location=old_cred.location,
+                    credentials_path=old_cred.credentials_path,
+                )
+                await new_cred.save()
+                logger.info(
+                    f"[{provider}/{old_cred.name}] Credential saved (id={new_cred.id})"
+                )
+
+                # Link existing models for this provider to the new credential
+                from open_notebook.ai.models import Model
+                from open_notebook.database.repository import repo_query
+
+                provider_models = await repo_query(
+                    "SELECT * FROM model WHERE string::lowercase(provider) = $provider AND credential IS NONE",
+                    {"provider": provider.lower()},
+                )
+                if provider_models:
+                    logger.info(
+                        f"[{provider}/{old_cred.name}] Linking {len(provider_models)} "
+                        f"unassigned model(s)"
+                    )
+                    for model_data in provider_models:
+                        model = Model(**model_data)
+                        model.credential = new_cred.id
+                        await model.save()
+
+                migrated.append(f"{provider}/{old_cred.name}")
+
+            except Exception as e:
+                logger.error(
+                    f"[{provider}/{old_cred.name}] Migration FAILED: "
+                    f"{type(e).__name__}: {e}",
+                    exc_info=True,
+                )
+                errors.append(f"{provider}/{old_cred.name}: {e}")
+
+    logger.info(
+        f"=== ProviderConfig migration complete === "
+        f"migrated={len(migrated)} skipped={len(skipped)} errors={len(errors)}"
+    )
+    if migrated:
+        logger.info(f"  Migrated: {', '.join(migrated)}")
+    if skipped:
+        logger.info(f"  Skipped: {', '.join(skipped)}")
+    if errors:
+        logger.error(f"  Errors: {'; '.join(errors)}")
+
+    return {
+        "message": f"Migration complete. Migrated {len(migrated)} credentials.",
+        "migrated": migrated,
+        "skipped": skipped,
+        "errors": errors,
+    }
+
+
+async def migrate_from_env() -> dict:
+    """
+    Migrate API keys from environment variables to credential records.
+
+    Returns dict with message, migrated, skipped, not_configured, errors.
+    """
+    logger.info("=== Starting environment variable migration ===")
+    logger.info(
+        f"Checking {len(PROVIDER_ENV_CONFIG)} providers: "
+        f"{', '.join(PROVIDER_ENV_CONFIG.keys())}"
+    )
+
+    require_encryption_key()
+    logger.info("Encryption key verified")
+
+    from open_notebook.ai.models import Model
+    from open_notebook.database.repository import repo_query
+
+    migrated = []
+    skipped = []
+    not_configured = []
+    errors = []
+
+    for provider in PROVIDER_ENV_CONFIG:
+        try:
+            if not check_env_configured(provider):
+                logger.debug(f"[{provider}] No env vars configured, skipping")
+                not_configured.append(provider)
+                continue
+
+            logger.info(f"[{provider}] Env vars detected, checking for existing credentials")
+
+            existing = await Credential.get_by_provider(provider)
+            if existing:
+                logger.info(
+                    f"[{provider}] Already has {len(existing)} credential(s) in DB, skipping"
+                )
+                skipped.append(provider)
+                continue
+
+            logger.info(f"[{provider}] Creating credential from env vars")
+            cred = create_credential_from_env(provider)
+            await cred.save()
+            logger.info(f"[{provider}] Credential saved successfully (id={cred.id})")
+
+            # Link unassigned models to this credential
+            provider_models = await repo_query(
+                "SELECT * FROM model WHERE string::lowercase(provider) = $provider AND credential IS NONE",
+                {"provider": provider.lower()},
+            )
+            if provider_models:
+                logger.info(
+                    f"[{provider}] Linking {len(provider_models)} unassigned model(s) "
+                    f"to credential {cred.id}"
+                )
+                for model_data in provider_models:
+                    model = Model(**model_data)
+                    model.credential = cred.id
+                    await model.save()
+            else:
+                logger.info(f"[{provider}] No unassigned models to link")
+
+            migrated.append(provider)
+
+        except Exception as e:
+            logger.error(
+                f"[{provider}] Migration FAILED: {type(e).__name__}: {e}",
+                exc_info=True,
+            )
+            errors.append(f"{provider}: {e}")
+
+    logger.info(
+        f"=== Environment variable migration complete === "
+        f"migrated={len(migrated)} skipped={len(skipped)} "
+        f"not_configured={len(not_configured)} errors={len(errors)}"
+    )
+    if migrated:
+        logger.info(f"  Migrated: {', '.join(migrated)}")
+    if skipped:
+        logger.info(f"  Skipped (already in DB): {', '.join(skipped)}")
+    if errors:
+        logger.error(f"  Errors: {'; '.join(errors)}")
+
+    return {
+        "message": f"Migration complete. Migrated {len(migrated)} providers.",
+        "migrated": migrated,
+        "skipped": skipped,
+        "not_configured": not_configured,
+        "errors": errors,
+    }

api/embedding_service.py

@@ -0,0 +1,27 @@
+"""
+Embedding service layer using API.
+"""
+
+from typing import Any, Dict, List, Union
+
+from loguru import logger
+
+from api.client import api_client
+
+
+class EmbeddingService:
+    """Service layer for embedding operations using API."""
+
+    def __init__(self):
+        logger.info("Using API for embedding operations")
+
+    def embed_content(
+        self, item_id: str, item_type: str
+    ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]:
+        """Embed content for vector search."""
+        result = api_client.embed_content(item_id=item_id, item_type=item_type)
+        return result
+
+
+# Global service instance
+embedding_service = EmbeddingService()

api/episode_profiles_service.py

@@ -0,0 +1,112 @@
+"""
+Episode profiles service layer using API.
+"""
+
+from typing import List
+
+from loguru import logger
+
+from api.client import api_client
+from open_notebook.podcasts.models import EpisodeProfile
+
+
+class EpisodeProfilesService:
+    """Service layer for episode profiles operations using API."""
+
+    def __init__(self):
+        logger.info("Using API for episode profiles operations")
+
+    def get_all_episode_profiles(self) -> List[EpisodeProfile]:
+        """Get all episode profiles."""
+        profiles_data = api_client.get_episode_profiles()
+        # Convert API response to EpisodeProfile objects
+        profiles = []
+        for profile_data in profiles_data:
+            profile = EpisodeProfile(
+                name=profile_data["name"],
+                description=profile_data.get("description", ""),
+                speaker_config=profile_data["speaker_config"],
+                outline_provider=profile_data["outline_provider"],
+                outline_model=profile_data["outline_model"],
+                transcript_provider=profile_data["transcript_provider"],
+                transcript_model=profile_data["transcript_model"],
+                default_briefing=profile_data["default_briefing"],
+                num_segments=profile_data["num_segments"],
+            )
+            profile.id = profile_data["id"]
+            profiles.append(profile)
+        return profiles
+
+    def get_episode_profile(self, profile_name: str) -> EpisodeProfile:
+        """Get a specific episode profile by name."""
+        profile_response = api_client.get_episode_profile(profile_name)
+        profile_data = (
+            profile_response
+            if isinstance(profile_response, dict)
+            else profile_response[0]
+        )
+        profile = EpisodeProfile(
+            name=profile_data["name"],
+            description=profile_data.get("description", ""),
+            speaker_config=profile_data["speaker_config"],
+            outline_provider=profile_data["outline_provider"],
+            outline_model=profile_data["outline_model"],
+            transcript_provider=profile_data["transcript_provider"],
+            transcript_model=profile_data["transcript_model"],
+            default_briefing=profile_data["default_briefing"],
+            num_segments=profile_data["num_segments"],
+        )
+        profile.id = profile_data["id"]
+        return profile
+
+    def create_episode_profile(
+        self,
+        name: str,
+        description: str = "",
+        speaker_config: str = "",
+        outline_provider: str = "",
+        outline_model: str = "",
+        transcript_provider: str = "",
+        transcript_model: str = "",
+        default_briefing: str = "",
+        num_segments: int = 5,
+    ) -> EpisodeProfile:
+        """Create a new episode profile."""
+        profile_response = api_client.create_episode_profile(
+            name=name,
+            description=description,
+            speaker_config=speaker_config,
+            outline_provider=outline_provider,
+            outline_model=outline_model,
+            transcript_provider=transcript_provider,
+            transcript_model=transcript_model,
+            default_briefing=default_briefing,
+            num_segments=num_segments,
+        )
+        profile_data = (
+            profile_response
+            if isinstance(profile_response, dict)
+            else profile_response[0]
+        )
+        profile = EpisodeProfile(
+            name=profile_data["name"],
+            description=profile_data.get("description", ""),
+            speaker_config=profile_data["speaker_config"],
+            outline_provider=profile_data["outline_provider"],
+            outline_model=profile_data["outline_model"],
+            transcript_provider=profile_data["transcript_provider"],
+            transcript_model=profile_data["transcript_model"],
+            default_briefing=profile_data["default_briefing"],
+            num_segments=profile_data["num_segments"],
+        )
+        profile.id = profile_data["id"]
+        return profile
+
+    def delete_episode_profile(self, profile_id: str) -> bool:
+        """Delete an episode profile."""
+        api_client.delete_episode_profile(profile_id)
+        return True
+
+
+# Global service instance
+episode_profiles_service = EpisodeProfilesService()

api/insights_service.py

@@ -0,0 +1,100 @@
+"""
+Insights service layer using API.
+"""
+
+from typing import List, Optional
+
+from loguru import logger
+
+from api.client import api_client
+from open_notebook.domain.notebook import Note, SourceInsight
+
+
+class InsightsService:
+    """Service layer for insights operations using API."""
+
+    def __init__(self):
+        logger.info("Using API for insights operations")
+
+    def get_source_insights(self, source_id: str) -> List[SourceInsight]:
+        """Get all insights for a specific source."""
+        insights_data = api_client.get_source_insights(source_id)
+        # Convert API response to SourceInsight objects
+        insights = []
+        for insight_data in insights_data:
+            insight = SourceInsight(
+                insight_type=insight_data["insight_type"],
+                content=insight_data["content"],
+            )
+            insight.id = insight_data["id"]
+            insight.created = insight_data["created"]
+            insight.updated = insight_data["updated"]
+            insights.append(insight)
+        return insights
+
+    def get_insight(self, insight_id: str) -> SourceInsight:
+        """Get a specific insight."""
+        insight_response = api_client.get_insight(insight_id)
+        insight_data = (
+            insight_response
+            if isinstance(insight_response, dict)
+            else insight_response[0]
+        )
+        insight = SourceInsight(
+            insight_type=insight_data["insight_type"],
+            content=insight_data["content"],
+        )
+        insight.id = insight_data["id"]
+        insight.created = insight_data["created"]
+        insight.updated = insight_data["updated"]
+        # Note: source_id from API response is not stored; use await insight.get_source() if needed
+        return insight
+
+    def delete_insight(self, insight_id: str) -> bool:
+        """Delete a specific insight."""
+        api_client.delete_insight(insight_id)
+        return True
+
+    def save_insight_as_note(
+        self, insight_id: str, notebook_id: Optional[str] = None
+    ) -> Note:
+        """Convert an insight to a note."""
+        note_response = api_client.save_insight_as_note(insight_id, notebook_id)
+        note_data = (
+            note_response if isinstance(note_response, dict) else note_response[0]
+        )
+        note = Note(
+            title=note_data["title"],
+            content=note_data["content"],
+            note_type=note_data["note_type"],
+        )
+        note.id = note_data["id"]
+        note.created = note_data["created"]
+        note.updated = note_data["updated"]
+        return note
+
+    def create_source_insight(
+        self, source_id: str, transformation_id: str, model_id: Optional[str] = None
+    ) -> SourceInsight:
+        """Create a new insight for a source by running a transformation."""
+        insight_response = api_client.create_source_insight(
+            source_id, transformation_id, model_id
+        )
+        insight_data = (
+            insight_response
+            if isinstance(insight_response, dict)
+            else insight_response[0]
+        )
+        insight = SourceInsight(
+            insight_type=insight_data["insight_type"],
+            content=insight_data["content"],
+        )
+        insight.id = insight_data["id"]
+        insight.created = insight_data["created"]
+        insight.updated = insight_data["updated"]
+        # Note: source_id from API response is not stored; use await insight.get_source() if needed
+        return insight
+
+
+# Global service instance
+insights_service = InsightsService()

api/main.py

@@ -0,0 +1,322 @@
+# Load environment variables
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import os
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from loguru import logger
+from starlette.exceptions import HTTPException as StarletteHTTPException
+
+from api.auth import PasswordAuthMiddleware
+from api.routers import (
+    auth,
+    chat,
+    config,
+    context,
+    credentials,
+    embedding,
+    embedding_rebuild,
+    episode_profiles,
+    insights,
+    languages,
+    models,
+    notebooks,
+    notes,
+    podcasts,
+    search,
+    settings,
+    source_chat,
+    sources,
+    speaker_profiles,
+    transformations,
+)
+from api.routers import commands as commands_router
+from open_notebook.database.async_migrate import AsyncMigrationManager
+from open_notebook.exceptions import (
+    AuthenticationError,
+    ConfigurationError,
+    ExternalServiceError,
+    InvalidInputError,
+    NetworkError,
+    NotFoundError,
+    OpenNotebookError,
+    RateLimitError,
+)
+from open_notebook.utils.encryption import get_secret_from_env
+
+
+def _parse_cors_origins(raw: str) -> list[str]:
+    """Parse CORS_ORIGINS env value into a list of origins."""
+    value = raw.strip()
+    if value == "*":
+        return ["*"]
+    return [origin.strip() for origin in value.split(",") if origin.strip()]
+
+
+# Parsed once at module load; CORS_ORIGINS changes require a restart.
+_cors_origins_raw = os.getenv("CORS_ORIGINS")
+CORS_ALLOWED_ORIGINS = _parse_cors_origins(_cors_origins_raw or "*")
+CORS_IS_DEFAULT_WILDCARD = _cors_origins_raw is None
+
+
+def _cors_headers(request: Request) -> dict[str, str]:
+    """
+    Build CORS headers for error responses.
+
+    Mirrors Starlette CORSMiddleware behavior: reflects the request Origin
+    when the origin is allowed (or when wildcard is configured, since
+    browsers reject `Access-Control-Allow-Origin: *` combined with
+    credentials). Omits `Access-Control-Allow-Origin` for disallowed
+    origins so the browser blocks the error body from leaking cross-origin.
+    """
+    origin = request.headers.get("origin")
+    headers: dict[str, str] = {
+        "Access-Control-Allow-Credentials": "true",
+        "Access-Control-Allow-Methods": "*",
+        "Access-Control-Allow-Headers": "*",
+    }
+
+    if origin and ("*" in CORS_ALLOWED_ORIGINS or origin in CORS_ALLOWED_ORIGINS):
+        headers["Access-Control-Allow-Origin"] = origin
+        headers["Vary"] = "Origin"
+
+    return headers
+
+
+# Import commands to register them in the API process
+try:
+    logger.info("Commands imported in API process")
+except Exception as e:
+    logger.error(f"Failed to import commands in API process: {e}")
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """
+    Lifespan event handler for the FastAPI application.
+    Runs database migrations automatically on startup.
+    """
+    # Startup: Security checks
+    logger.info("Starting API initialization...")
+
+    # Security check: Encryption key
+    if not get_secret_from_env("OPEN_NOTEBOOK_ENCRYPTION_KEY"):
+        logger.warning(
+            "OPEN_NOTEBOOK_ENCRYPTION_KEY not set. "
+            "API key encryption will fail until this is configured. "
+            "Set OPEN_NOTEBOOK_ENCRYPTION_KEY to any secret string."
+        )
+
+    # Run database migrations
+
+    try:
+        migration_manager = AsyncMigrationManager()
+        current_version = await migration_manager.get_current_version()
+        logger.info(f"Current database version: {current_version}")
+
+        if await migration_manager.needs_migration():
+            logger.warning("Database migrations are pending. Running migrations...")
+            await migration_manager.run_migration_up()
+            new_version = await migration_manager.get_current_version()
+            logger.success(
+                f"Migrations completed successfully. Database is now at version {new_version}"
+            )
+        else:
+            logger.info(
+                "Database is already at the latest version. No migrations needed."
+            )
+    except Exception as e:
+        logger.error(f"CRITICAL: Database migration failed: {str(e)}")
+        logger.exception(e)
+        # Fail fast - don't start the API with an outdated database schema
+        raise RuntimeError(f"Failed to run database migrations: {str(e)}") from e
+
+    # Run podcast profile data migration (legacy strings -> Model registry)
+    try:
+        from open_notebook.podcasts.migration import migrate_podcast_profiles
+
+        await migrate_podcast_profiles()
+    except Exception as e:
+        logger.warning(f"Podcast profile migration encountered errors: {e}")
+        # Non-fatal: profiles can be migrated manually via UI
+
+    logger.success("API initialization completed successfully")
+
+    # Yield control to the application
+    yield
+
+    # Shutdown: cleanup if needed
+    logger.info("API shutdown complete")
+
+
+app = FastAPI(
+    title="Open Notebook API",
+    description="API for Open Notebook - Research Assistant",
+    lifespan=lifespan,
+)
+
+if CORS_IS_DEFAULT_WILDCARD:
+    logger.warning(
+        "CORS_ORIGINS is not set — API accepts cross-origin requests from any "
+        "origin (default: '*'). For production deployments, set CORS_ORIGINS to "
+        "your frontend origin(s), e.g. "
+        "CORS_ORIGINS=https://notebook.example.com"
+    )
+else:
+    logger.info(f"CORS allowed origins: {CORS_ALLOWED_ORIGINS}")
+
+# Add password authentication middleware first
+# Exclude /api/auth/status and /api/config from authentication
+app.add_middleware(
+    PasswordAuthMiddleware,
+    excluded_paths=[
+        "/",
+        "/health",
+        "/docs",
+        "/openapi.json",
+        "/redoc",
+        "/api/auth/status",
+        "/api/config",
+    ],
+)
+
+# Add CORS middleware last (so it processes first)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=CORS_ALLOWED_ORIGINS,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+# Custom exception handler to ensure CORS headers are included in error responses
+# This helps when errors occur before the CORS middleware can process them
+@app.exception_handler(StarletteHTTPException)
+async def custom_http_exception_handler(request: Request, exc: StarletteHTTPException):
+    """
+    Custom exception handler that ensures CORS headers are included in error responses.
+    This is particularly important for 413 (Payload Too Large) errors during file uploads.
+
+    Note: If a reverse proxy (nginx, traefik) returns 413 before the request reaches
+    FastAPI, this handler won't be called. In that case, configure your reverse proxy
+    to add CORS headers to error responses.
+    """
+    return JSONResponse(
+        status_code=exc.status_code,
+        content={"detail": exc.detail},
+        headers={**(exc.headers or {}), **_cors_headers(request)},
+    )
+
+
+@app.exception_handler(NotFoundError)
+async def not_found_error_handler(request: Request, exc: NotFoundError):
+    return JSONResponse(
+        status_code=404,
+        content={"detail": str(exc)},
+        headers=_cors_headers(request),
+    )
+
+
+@app.exception_handler(InvalidInputError)
+async def invalid_input_error_handler(request: Request, exc: InvalidInputError):
+    return JSONResponse(
+        status_code=400,
+        content={"detail": str(exc)},
+        headers=_cors_headers(request),
+    )
+
+
+@app.exception_handler(AuthenticationError)
+async def authentication_error_handler(request: Request, exc: AuthenticationError):
+    return JSONResponse(
+        status_code=401,
+        content={"detail": str(exc)},
+        headers=_cors_headers(request),
+    )
+
+
+@app.exception_handler(RateLimitError)
+async def rate_limit_error_handler(request: Request, exc: RateLimitError):
+    return JSONResponse(
+        status_code=429,
+        content={"detail": str(exc)},
+        headers=_cors_headers(request),
+    )
+
+
+@app.exception_handler(ConfigurationError)
+async def configuration_error_handler(request: Request, exc: ConfigurationError):
+    return JSONResponse(
+        status_code=422,
+        content={"detail": str(exc)},
+        headers=_cors_headers(request),
+    )
+
+
+@app.exception_handler(NetworkError)
+async def network_error_handler(request: Request, exc: NetworkError):
+    return JSONResponse(
+        status_code=502,
+        content={"detail": str(exc)},
+        headers=_cors_headers(request),
+    )
+
+
+@app.exception_handler(ExternalServiceError)
+async def external_service_error_handler(request: Request, exc: ExternalServiceError):
+    return JSONResponse(
+        status_code=502,
+        content={"detail": str(exc)},
+        headers=_cors_headers(request),
+    )
+
+
+@app.exception_handler(OpenNotebookError)
+async def open_notebook_error_handler(request: Request, exc: OpenNotebookError):
+    return JSONResponse(
+        status_code=500,
+        content={"detail": str(exc)},
+        headers=_cors_headers(request),
+    )
+
+
+# Include routers
+app.include_router(auth.router, prefix="/api", tags=["auth"])
+app.include_router(config.router, prefix="/api", tags=["config"])
+app.include_router(notebooks.router, prefix="/api", tags=["notebooks"])
+app.include_router(search.router, prefix="/api", tags=["search"])
+app.include_router(models.router, prefix="/api", tags=["models"])
+app.include_router(transformations.router, prefix="/api", tags=["transformations"])
+app.include_router(notes.router, prefix="/api", tags=["notes"])
+app.include_router(embedding.router, prefix="/api", tags=["embedding"])
+app.include_router(
+    embedding_rebuild.router, prefix="/api/embeddings", tags=["embeddings"]
+)
+app.include_router(settings.router, prefix="/api", tags=["settings"])
+app.include_router(context.router, prefix="/api", tags=["context"])
+app.include_router(sources.router, prefix="/api", tags=["sources"])
+app.include_router(insights.router, prefix="/api", tags=["insights"])
+app.include_router(commands_router.router, prefix="/api", tags=["commands"])
+app.include_router(podcasts.router, prefix="/api", tags=["podcasts"])
+app.include_router(episode_profiles.router, prefix="/api", tags=["episode-profiles"])
+app.include_router(speaker_profiles.router, prefix="/api", tags=["speaker-profiles"])
+app.include_router(chat.router, prefix="/api", tags=["chat"])
+app.include_router(source_chat.router, prefix="/api", tags=["source-chat"])
+app.include_router(credentials.router, prefix="/api", tags=["credentials"])
+app.include_router(languages.router, prefix="/api", tags=["languages"])
+
+
+@app.get("/")
+async def root():
+    return {"message": "Open Notebook API is running"}
+
+
+@app.get("/health")
+async def health():
+    return {"status": "healthy"}

api/models.py

@@ -0,0 +1,693 @@
+from typing import Any, Dict, List, Literal, Optional
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+
+# Notebook models
+class NotebookCreate(BaseModel):
+    name: str = Field(..., description="Name of the notebook")
+    description: str = Field(default="", description="Description of the notebook")
+
+
+class NotebookUpdate(BaseModel):
+    name: Optional[str] = Field(None, description="Name of the notebook")
+    description: Optional[str] = Field(None, description="Description of the notebook")
+    archived: Optional[bool] = Field(
+        None, description="Whether the notebook is archived"
+    )
+
+
+class NotebookResponse(BaseModel):
+    id: str
+    name: str
+    description: str
+    archived: bool
+    created: str
+    updated: str
+    source_count: int
+    note_count: int
+
+
+# Search models
+class SearchRequest(BaseModel):
+    query: str = Field(..., description="Search query")
+    type: Literal["text", "vector"] = Field("text", description="Search type")
+    limit: int = Field(100, description="Maximum number of results", le=1000)
+    search_sources: bool = Field(True, description="Include sources in search")
+    search_notes: bool = Field(True, description="Include notes in search")
+    minimum_score: float = Field(
+        0.2, description="Minimum score for vector search", ge=0, le=1
+    )
+
+
+class SearchResponse(BaseModel):
+    results: List[Dict[str, Any]] = Field(..., description="Search results")
+    total_count: int = Field(..., description="Total number of results")
+    search_type: str = Field(..., description="Type of search performed")
+
+
+class AskRequest(BaseModel):
+    question: str = Field(..., description="Question to ask the knowledge base")
+    strategy_model: str = Field(..., description="Model ID for query strategy")
+    answer_model: str = Field(..., description="Model ID for individual answers")
+    final_answer_model: str = Field(..., description="Model ID for final answer")
+
+
+class AskResponse(BaseModel):
+    answer: str = Field(..., description="Final answer from the knowledge base")
+    question: str = Field(..., description="Original question")
+
+
+# Models API models
+class ModelCreate(BaseModel):
+    name: str = Field(..., description="Model name (e.g., gpt-5-mini, claude, gemini)")
+    provider: str = Field(
+        ..., description="Provider name (e.g., openai, anthropic, gemini)"
+    )
+    type: str = Field(
+        ...,
+        description="Model type (language, embedding, text_to_speech, speech_to_text)",
+    )
+    credential: Optional[str] = Field(
+        None, description="Credential ID to link this model to"
+    )
+
+
+class ModelResponse(BaseModel):
+    id: str
+    name: str
+    provider: str
+    type: str
+    credential: Optional[str] = None
+    created: str
+    updated: str
+
+
+class DefaultModelsResponse(BaseModel):
+    default_chat_model: Optional[str] = None
+    default_transformation_model: Optional[str] = None
+    large_context_model: Optional[str] = None
+    default_text_to_speech_model: Optional[str] = None
+    default_speech_to_text_model: Optional[str] = None
+    default_embedding_model: Optional[str] = None
+    default_tools_model: Optional[str] = None
+
+
+class ProviderAvailabilityResponse(BaseModel):
+    available: List[str] = Field(..., description="List of available providers")
+    unavailable: List[str] = Field(..., description="List of unavailable providers")
+    supported_types: Dict[str, List[str]] = Field(
+        ..., description="Provider to supported model types mapping"
+    )
+
+
+# Transformations API models
+class TransformationCreate(BaseModel):
+    name: str = Field(..., description="Transformation name")
+    title: str = Field(..., description="Display title for the transformation")
+    description: str = Field(
+        ..., description="Description of what this transformation does"
+    )
+    prompt: str = Field(..., description="The transformation prompt")
+    apply_default: bool = Field(
+        False, description="Whether to apply this transformation by default"
+    )
+
+
+class TransformationUpdate(BaseModel):
+    name: Optional[str] = Field(None, description="Transformation name")
+    title: Optional[str] = Field(
+        None, description="Display title for the transformation"
+    )
+    description: Optional[str] = Field(
+        None, description="Description of what this transformation does"
+    )
+    prompt: Optional[str] = Field(None, description="The transformation prompt")
+    apply_default: Optional[bool] = Field(
+        None, description="Whether to apply this transformation by default"
+    )
+
+
+class TransformationResponse(BaseModel):
+    id: str
+    name: str
+    title: str
+    description: str
+    prompt: str
+    apply_default: bool
+    created: str
+    updated: str
+
+
+class TransformationExecuteRequest(BaseModel):
+    model_config = ConfigDict(protected_namespaces=())
+
+    transformation_id: str = Field(
+        ..., description="ID of the transformation to execute"
+    )
+    input_text: str = Field(..., description="Text to transform")
+    model_id: str = Field(..., description="Model ID to use for the transformation")
+
+
+class TransformationExecuteResponse(BaseModel):
+    model_config = ConfigDict(protected_namespaces=())
+
+    output: str = Field(..., description="Transformed text")
+    transformation_id: str = Field(..., description="ID of the transformation used")
+    model_id: str = Field(..., description="Model ID used")
+
+
+# Default Prompt API models
+class DefaultPromptResponse(BaseModel):
+    transformation_instructions: str = Field(
+        ..., description="Default transformation instructions"
+    )
+
+
+class DefaultPromptUpdate(BaseModel):
+    transformation_instructions: str = Field(
+        ..., description="Default transformation instructions"
+    )
+
+
+# Notes API models
+class NoteCreate(BaseModel):
+    title: Optional[str] = Field(None, description="Note title")
+    content: str = Field(..., description="Note content")
+    note_type: Optional[str] = Field("human", description="Type of note (human, ai)")
+    notebook_id: Optional[str] = Field(
+        None, description="Notebook ID to add the note to"
+    )
+
+
+class NoteUpdate(BaseModel):
+    title: Optional[str] = Field(None, description="Note title")
+    content: Optional[str] = Field(None, description="Note content")
+    note_type: Optional[str] = Field(None, description="Type of note (human, ai)")
+
+
+class NoteResponse(BaseModel):
+    id: str
+    title: Optional[str]
+    content: Optional[str]
+    note_type: Optional[str]
+    created: str
+    updated: str
+    command_id: Optional[str] = None
+
+
+# Embedding API models
+class EmbedRequest(BaseModel):
+    item_id: str = Field(..., description="ID of the item to embed")
+    item_type: str = Field(..., description="Type of item (source, note)")
+    async_processing: bool = Field(
+        False, description="Process asynchronously in background"
+    )
+
+
+class EmbedResponse(BaseModel):
+    success: bool = Field(..., description="Whether embedding was successful")
+    message: str = Field(..., description="Result message")
+    item_id: str = Field(..., description="ID of the item that was embedded")
+    item_type: str = Field(..., description="Type of item that was embedded")
+    command_id: Optional[str] = Field(
+        None, description="Command ID for async processing"
+    )
+
+
+# Rebuild request/response models
+class RebuildRequest(BaseModel):
+    mode: Literal["existing", "all"] = Field(
+        ...,
+        description="Rebuild mode: 'existing' only re-embeds items with embeddings, 'all' embeds everything",
+    )
+    include_sources: bool = Field(True, description="Include sources in rebuild")
+    include_notes: bool = Field(True, description="Include notes in rebuild")
+    include_insights: bool = Field(True, description="Include insights in rebuild")
+
+
+class RebuildResponse(BaseModel):
+    command_id: str = Field(..., description="Command ID to track progress")
+    total_items: int = Field(..., description="Estimated number of items to process")
+    message: str = Field(..., description="Status message")
+
+
+class RebuildProgress(BaseModel):
+    processed: int = Field(..., description="Number of items processed")
+    total: int = Field(..., description="Total items to process")
+    percentage: float = Field(..., description="Progress percentage")
+
+
+class RebuildStats(BaseModel):
+    sources: int = Field(0, description="Sources processed")
+    notes: int = Field(0, description="Notes processed")
+    insights: int = Field(0, description="Insights processed")
+    failed: int = Field(0, description="Failed items")
+
+
+class RebuildStatusResponse(BaseModel):
+    command_id: str = Field(..., description="Command ID")
+    status: str = Field(..., description="Status: queued, running, completed, failed")
+    progress: Optional[RebuildProgress] = None
+    stats: Optional[RebuildStats] = None
+    started_at: Optional[str] = None
+    completed_at: Optional[str] = None
+    error_message: Optional[str] = None
+
+
+# Settings API models
+class SettingsResponse(BaseModel):
+    default_content_processing_engine_doc: Optional[str] = None
+    default_content_processing_engine_url: Optional[str] = None
+    default_embedding_option: Optional[str] = None
+    auto_delete_files: Optional[str] = None
+    youtube_preferred_languages: Optional[List[str]] = None
+
+
+class SettingsUpdate(BaseModel):
+    default_content_processing_engine_doc: Optional[str] = None
+    default_content_processing_engine_url: Optional[str] = None
+    default_embedding_option: Optional[str] = None
+    auto_delete_files: Optional[str] = None
+    youtube_preferred_languages: Optional[List[str]] = None
+
+
+# Sources API models
+class AssetModel(BaseModel):
+    file_path: Optional[str] = None
+    url: Optional[str] = None
+
+
+class SourceCreate(BaseModel):
+    # Backward compatibility: support old single notebook_id
+    notebook_id: Optional[str] = Field(
+        None, description="Notebook ID to add the source to (deprecated, use notebooks)"
+    )
+    # New multi-notebook support
+    notebooks: Optional[List[str]] = Field(
+        None, description="List of notebook IDs to add the source to"
+    )
+    # Required fields
+    type: str = Field(..., description="Source type: link, upload, or text")
+    url: Optional[str] = Field(None, description="URL for link type")
+    file_path: Optional[str] = Field(None, description="File path for upload type")
+    content: Optional[str] = Field(None, description="Text content for text type")
+    title: Optional[str] = Field(None, description="Source title")
+    transformations: Optional[List[str]] = Field(
+        default_factory=list, description="Transformation IDs to apply"
+    )
+    embed: bool = Field(False, description="Whether to embed content for vector search")
+    delete_source: bool = Field(
+        False, description="Whether to delete uploaded file after processing"
+    )
+    # New async processing support
+    async_processing: bool = Field(
+        False, description="Whether to process source asynchronously"
+    )
+
+    @model_validator(mode="after")
+    def validate_notebook_fields(self):
+        # Ensure only one of notebook_id or notebooks is provided
+        if self.notebook_id is not None and self.notebooks is not None:
+            raise ValueError(
+                "Cannot specify both 'notebook_id' and 'notebooks'. Use 'notebooks' for multi-notebook support."
+            )
+
+        # Convert single notebook_id to notebooks array for internal processing
+        if self.notebook_id is not None:
+            self.notebooks = [self.notebook_id]
+            # Keep notebook_id for backward compatibility in response
+
+        # Set empty array if no notebooks specified (allow sources without notebooks)
+        if self.notebooks is None:
+            self.notebooks = []
+
+        return self
+
+
+class SourceUpdate(BaseModel):
+    title: Optional[str] = Field(None, description="Source title")
+    topics: Optional[List[str]] = Field(None, description="Source topics")
+
+
+class SourceResponse(BaseModel):
+    id: str
+    title: Optional[str]
+    topics: Optional[List[str]]
+    asset: Optional[AssetModel]
+    full_text: Optional[str]
+    embedded: bool
+    embedded_chunks: int
+    file_available: Optional[bool] = None
+    created: str
+    updated: str
+    # New fields for async processing
+    command_id: Optional[str] = None
+    status: Optional[str] = None
+    processing_info: Optional[Dict] = None
+    # Notebook associations
+    notebooks: Optional[List[str]] = None
+
+
+class SourceListResponse(BaseModel):
+    id: str
+    title: Optional[str]
+    topics: Optional[List[str]]
+    asset: Optional[AssetModel]
+    embedded: bool  # Boolean flag indicating if source has embeddings
+    embedded_chunks: int  # Number of embedded chunks
+    insights_count: int
+    created: str
+    updated: str
+    file_available: Optional[bool] = None
+    # Status fields for async processing
+    command_id: Optional[str] = None
+    status: Optional[str] = None
+    processing_info: Optional[Dict[str, Any]] = None
+
+
+# Context API models
+class ContextConfig(BaseModel):
+    sources: Dict[str, str] = Field(
+        default_factory=dict, description="Source inclusion config {source_id: level}"
+    )
+    notes: Dict[str, str] = Field(
+        default_factory=dict, description="Note inclusion config {note_id: level}"
+    )
+
+
+class ContextRequest(BaseModel):
+    notebook_id: str = Field(..., description="Notebook ID to get context for")
+    context_config: Optional[ContextConfig] = Field(
+        None, description="Context configuration"
+    )
+
+
+class ContextResponse(BaseModel):
+    notebook_id: str
+    sources: List[Dict[str, Any]] = Field(..., description="Source context data")
+    notes: List[Dict[str, Any]] = Field(..., description="Note context data")
+    total_tokens: Optional[int] = Field(None, description="Estimated token count")
+
+
+# Insights API models
+class SourceInsightResponse(BaseModel):
+    id: str
+    source_id: str
+    insight_type: str
+    content: str
+    created: str
+    updated: str
+
+
+class InsightCreationResponse(BaseModel):
+    """Response for async insight creation."""
+
+    status: Literal["pending"] = "pending"
+    message: str = "Insight generation started"
+    source_id: str
+    transformation_id: str
+    command_id: Optional[str] = None
+
+
+class SaveAsNoteRequest(BaseModel):
+    notebook_id: Optional[str] = Field(None, description="Notebook ID to add note to")
+
+
+class CreateSourceInsightRequest(BaseModel):
+    model_config = ConfigDict(protected_namespaces=())
+
+    transformation_id: str = Field(..., description="ID of transformation to apply")
+    model_id: Optional[str] = Field(
+        None, description="Model ID (uses default if not provided)"
+    )
+
+
+# Source status response
+class SourceStatusResponse(BaseModel):
+    status: Optional[str] = Field(None, description="Processing status")
+    message: str = Field(..., description="Descriptive message about the status")
+    processing_info: Optional[Dict[str, Any]] = Field(
+        None, description="Detailed processing information"
+    )
+    command_id: Optional[str] = Field(None, description="Command ID if available")
+
+
+# Error response
+class ErrorResponse(BaseModel):
+    error: str
+    message: str
+
+
+# API Key Configuration models
+class SetApiKeyRequest(BaseModel):
+    """Request to set an API key for a provider."""
+
+    api_key: Optional[str] = Field(None, description="API key for the provider")
+    base_url: Optional[str] = Field(
+        None, description="Base URL for URL-based providers (Ollama, OpenAI-compatible)"
+    )
+    endpoint: Optional[str] = Field(
+        None, description="Endpoint URL for Azure OpenAI"
+    )
+    api_version: Optional[str] = Field(
+        None, description="API version for Azure OpenAI"
+    )
+    endpoint_llm: Optional[str] = Field(
+        None, description="Service-specific endpoint for LLM (Azure)"
+    )
+    endpoint_embedding: Optional[str] = Field(
+        None, description="Service-specific endpoint for embedding (Azure)"
+    )
+    endpoint_stt: Optional[str] = Field(
+        None, description="Service-specific endpoint for STT (Azure)"
+    )
+    endpoint_tts: Optional[str] = Field(
+        None, description="Service-specific endpoint for TTS (Azure)"
+    )
+    service_type: Optional[Literal["llm", "embedding", "stt", "tts"]] = Field(
+        None,
+        description="Service type for OpenAI-compatible providers (llm, embedding, stt, tts)",
+    )
+    # Vertex AI specific fields
+    vertex_project: Optional[str] = Field(
+        None, description="Google Cloud Project ID for Vertex AI"
+    )
+    vertex_location: Optional[str] = Field(
+        None, description="Google Cloud Region for Vertex AI (e.g., us-central1)"
+    )
+    vertex_credentials_path: Optional[str] = Field(
+        None, description="Path to Google Cloud service account JSON file"
+    )
+
+    @field_validator(
+        "api_key",
+        "base_url",
+        "endpoint",
+        "api_version",
+        "endpoint_llm",
+        "endpoint_embedding",
+        "endpoint_stt",
+        "endpoint_tts",
+        "vertex_project",
+        "vertex_location",
+        "vertex_credentials_path",
+        mode="before",
+    )
+    @classmethod
+    def validate_not_empty_string(cls, v: Optional[str]) -> Optional[str]:
+        """Reject empty strings - convert to None or raise error."""
+        if v is not None:
+            stripped = v.strip()
+            if not stripped:
+                return None  # Treat empty/whitespace-only as None
+            return stripped
+        return v
+
+
+class ApiKeyStatusResponse(BaseModel):
+    """Response showing which providers are configured and their source."""
+
+    configured: Dict[str, bool] = Field(
+        ..., description="Map of provider name to whether it is configured"
+    )
+    source: Dict[str, Literal["database", "environment", "none"]] = Field(
+        ...,
+        description="Map of provider name to configuration source (database, environment, or none)",
+    )
+    encryption_configured: bool = Field(
+        ...,
+        description="Whether OPEN_NOTEBOOK_ENCRYPTION_KEY is set (required to store keys in database)",
+    )
+
+
+class TestConnectionResponse(BaseModel):
+    """Response from testing a provider connection."""
+
+    provider: str = Field(..., description="Provider name that was tested")
+    success: bool = Field(..., description="Whether connection test succeeded")
+    message: str = Field(..., description="Result message with details")
+
+
+class MigrateFromEnvRequest(BaseModel):
+    """Request to migrate API keys from environment variables to database."""
+
+    force: bool = Field(
+        False, description="Force overwrite existing database configurations"
+    )
+
+
+class MigrationResult(BaseModel):
+    """Response from migrating API keys from environment to database."""
+
+    message: str = Field(..., description="Summary message")
+    migrated: List[str] = Field(
+        default_factory=list, description="Providers successfully migrated"
+    )
+    skipped: List[str] = Field(
+        default_factory=list, description="Providers skipped (already in DB)"
+    )
+    errors: List[str] = Field(
+        default_factory=list, description="Migration errors by provider"
+    )
+
+
+# Notebook delete cascade models
+# Credential models
+class CreateCredentialRequest(BaseModel):
+    """Request to create a new credential."""
+
+    name: str = Field(..., description="Credential name")
+    provider: str = Field(..., description="Provider name (openai, anthropic, etc.)")
+    modalities: List[str] = Field(
+        default_factory=list,
+        description="Supported modalities (language, embedding, text_to_speech, speech_to_text)",
+    )
+    api_key: Optional[str] = Field(None, description="API key (stored encrypted)")
+    base_url: Optional[str] = Field(None, description="Base URL")
+    endpoint: Optional[str] = Field(None, description="Endpoint URL (Azure)")
+    api_version: Optional[str] = Field(None, description="API version (Azure)")
+    endpoint_llm: Optional[str] = Field(None, description="LLM endpoint")
+    endpoint_embedding: Optional[str] = Field(None, description="Embedding endpoint")
+    endpoint_stt: Optional[str] = Field(None, description="STT endpoint")
+    endpoint_tts: Optional[str] = Field(None, description="TTS endpoint")
+    project: Optional[str] = Field(None, description="Project ID (Vertex)")
+    location: Optional[str] = Field(None, description="Location (Vertex)")
+    credentials_path: Optional[str] = Field(
+        None, description="Credentials file path (Vertex)"
+    )
+    num_ctx: Optional[int] = Field(
+        None, description="Context window size (Ollama only; defaults to 8192)"
+    )
+
+
+class UpdateCredentialRequest(BaseModel):
+    """Request to update an existing credential."""
+
+    name: Optional[str] = Field(None, description="Credential name")
+    modalities: Optional[List[str]] = Field(None, description="Supported modalities")
+    api_key: Optional[str] = Field(None, description="API key (stored encrypted)")
+    base_url: Optional[str] = Field(None, description="Base URL")
+    endpoint: Optional[str] = Field(None, description="Endpoint URL")
+    api_version: Optional[str] = Field(None, description="API version")
+    endpoint_llm: Optional[str] = Field(None, description="LLM endpoint")
+    endpoint_embedding: Optional[str] = Field(None, description="Embedding endpoint")
+    endpoint_stt: Optional[str] = Field(None, description="STT endpoint")
+    endpoint_tts: Optional[str] = Field(None, description="TTS endpoint")
+    project: Optional[str] = Field(None, description="Project ID")
+    location: Optional[str] = Field(None, description="Location")
+    credentials_path: Optional[str] = Field(None, description="Credentials path")
+    num_ctx: Optional[int] = Field(
+        None, description="Context window size (Ollama only; defaults to 8192)"
+    )
+
+
+class CredentialResponse(BaseModel):
+    """Response for a credential (never includes api_key)."""
+
+    id: str
+    name: str
+    provider: str
+    modalities: List[str]
+    base_url: Optional[str] = None
+    endpoint: Optional[str] = None
+    api_version: Optional[str] = None
+    endpoint_llm: Optional[str] = None
+    endpoint_embedding: Optional[str] = None
+    endpoint_stt: Optional[str] = None
+    endpoint_tts: Optional[str] = None
+    project: Optional[str] = None
+    location: Optional[str] = None
+    credentials_path: Optional[str] = None
+    num_ctx: Optional[int] = None
+    has_api_key: bool = False
+    created: str
+    updated: str
+    model_count: int = 0
+    decryption_error: Optional[str] = None
+
+
+class CredentialDeleteResponse(BaseModel):
+    """Response for credential deletion."""
+
+    message: str
+    deleted_models: int = 0
+
+
+class DiscoveredModelResponse(BaseModel):
+    """A model discovered from a provider."""
+
+    name: str
+    provider: str
+    model_type: Optional[str] = None
+    description: Optional[str] = None
+
+
+class DiscoverModelsResponse(BaseModel):
+    """Response from model discovery."""
+
+    credential_id: str
+    provider: str
+    discovered: List[DiscoveredModelResponse]
+
+
+class RegisterModelData(BaseModel):
+    """A model to register with user-specified type."""
+
+    name: str
+    provider: str
+    model_type: str  # Required: user specifies the type
+
+
+class RegisterModelsRequest(BaseModel):
+    """Request to register discovered models."""
+
+    models: List[RegisterModelData]
+
+
+class RegisterModelsResponse(BaseModel):
+    """Response from model registration."""
+
+    created: int
+    existing: int
+
+
+class NotebookDeletePreview(BaseModel):
+    notebook_id: str = Field(..., description="ID of the notebook")
+    notebook_name: str = Field(..., description="Name of the notebook")
+    note_count: int = Field(..., description="Number of notes that will be deleted")
+    exclusive_source_count: int = Field(
+        ..., description="Number of sources only in this notebook"
+    )
+    shared_source_count: int = Field(
+        ..., description="Number of sources shared with other notebooks"
+    )
+
+
+class NotebookDeleteResponse(BaseModel):
+    message: str = Field(..., description="Success message")
+    deleted_notes: int = Field(..., description="Number of notes deleted")
+    deleted_sources: int = Field(..., description="Number of exclusive sources deleted")
+    unlinked_sources: int = Field(
+        ..., description="Number of sources unlinked from notebook"
+    )

api/models_service.py

@@ -0,0 +1,112 @@
+"""
+Models service layer using API.
+"""
+
+from typing import List, Optional
+
+from loguru import logger
+
+from api.client import api_client
+from open_notebook.ai.models import DefaultModels, Model
+
+
+class ModelsService:
+    """Service layer for models operations using API."""
+
+    def __init__(self):
+        logger.info("Using API for models operations")
+
+    def get_all_models(self, model_type: Optional[str] = None) -> List[Model]:
+        """Get all models with optional type filtering."""
+        models_data = api_client.get_models(model_type=model_type)
+        # Convert API response to Model objects
+        models = []
+        for model_data in models_data:
+            model = Model(
+                name=model_data["name"],
+                provider=model_data["provider"],
+                type=model_data["type"],
+            )
+            model.id = model_data["id"]
+            model.created = model_data["created"]
+            model.updated = model_data["updated"]
+            models.append(model)
+        return models
+
+    def create_model(self, name: str, provider: str, model_type: str) -> Model:
+        """Create a new model."""
+        response = api_client.create_model(name, provider, model_type)
+        model_data = response if isinstance(response, dict) else response[0]
+        model = Model(
+            name=model_data["name"],
+            provider=model_data["provider"],
+            type=model_data["type"],
+        )
+        model.id = model_data["id"]
+        model.created = model_data["created"]
+        model.updated = model_data["updated"]
+        return model
+
+    def delete_model(self, model_id: str) -> bool:
+        """Delete a model."""
+        api_client.delete_model(model_id)
+        return True
+
+    def get_default_models(self) -> DefaultModels:
+        """Get default model assignments."""
+        response = api_client.get_default_models()
+        defaults_data = response if isinstance(response, dict) else response[0]
+        defaults = DefaultModels()
+
+        # Set the values from API response
+        defaults.default_chat_model = defaults_data.get("default_chat_model")
+        defaults.default_transformation_model = defaults_data.get(
+            "default_transformation_model"
+        )
+        defaults.large_context_model = defaults_data.get("large_context_model")
+        defaults.default_text_to_speech_model = defaults_data.get(
+            "default_text_to_speech_model"
+        )
+        defaults.default_speech_to_text_model = defaults_data.get(
+            "default_speech_to_text_model"
+        )
+        defaults.default_embedding_model = defaults_data.get("default_embedding_model")
+        defaults.default_tools_model = defaults_data.get("default_tools_model")
+
+        return defaults
+
+    def update_default_models(self, defaults: DefaultModels) -> DefaultModels:
+        """Update default model assignments."""
+        updates = {
+            "default_chat_model": defaults.default_chat_model,
+            "default_transformation_model": defaults.default_transformation_model,
+            "large_context_model": defaults.large_context_model,
+            "default_text_to_speech_model": defaults.default_text_to_speech_model,
+            "default_speech_to_text_model": defaults.default_speech_to_text_model,
+            "default_embedding_model": defaults.default_embedding_model,
+            "default_tools_model": defaults.default_tools_model,
+        }
+
+        response = api_client.update_default_models(**updates)
+        defaults_data = response if isinstance(response, dict) else response[0]
+
+        # Update the defaults object with the response
+        defaults.default_chat_model = defaults_data.get("default_chat_model")
+        defaults.default_transformation_model = defaults_data.get(
+            "default_transformation_model"
+        )
+        defaults.large_context_model = defaults_data.get("large_context_model")
+        defaults.default_text_to_speech_model = defaults_data.get(
+            "default_text_to_speech_model"
+        )
+        defaults.default_speech_to_text_model = defaults_data.get(
+            "default_speech_to_text_model"
+        )
+        defaults.default_embedding_model = defaults_data.get("default_embedding_model")
+        defaults.default_tools_model = defaults_data.get("default_tools_model")
+
+        return defaults
+
+
+# Global service instance
+models_service = ModelsService()

api/notebook_service.py

@@ -0,0 +1,87 @@
+"""
+Notebook service layer using API.
+"""
+
+from typing import List, Optional
+
+from loguru import logger
+
+from api.client import api_client
+from open_notebook.domain.notebook import Notebook
+
+
+class NotebookService:
+    """Service layer for notebook operations using API."""
+
+    def __init__(self):
+        logger.info("Using API for notebook operations")
+
+    def get_all_notebooks(self, order_by: str = "updated desc") -> List[Notebook]:
+        """Get all notebooks."""
+        notebooks_data = api_client.get_notebooks(order_by=order_by)
+        # Convert API response to Notebook objects
+        notebooks = []
+        for nb_data in notebooks_data:
+            nb = Notebook(
+                name=nb_data["name"],
+                description=nb_data["description"],
+                archived=nb_data["archived"],
+            )
+            nb.id = nb_data["id"]
+            nb.created = nb_data["created"]
+            nb.updated = nb_data["updated"]
+            notebooks.append(nb)
+        return notebooks
+
+    def get_notebook(self, notebook_id: str) -> Optional[Notebook]:
+        """Get a specific notebook."""
+        response = api_client.get_notebook(notebook_id)
+        nb_data = response if isinstance(response, dict) else response[0]
+        nb = Notebook(
+            name=nb_data["name"],
+            description=nb_data["description"],
+            archived=nb_data["archived"],
+        )
+        nb.id = nb_data["id"]
+        nb.created = nb_data["created"]
+        nb.updated = nb_data["updated"]
+        return nb
+
+    def create_notebook(self, name: str, description: str = "") -> Notebook:
+        """Create a new notebook."""
+        response = api_client.create_notebook(name, description)
+        nb_data = response if isinstance(response, dict) else response[0]
+        nb = Notebook(
+            name=nb_data["name"],
+            description=nb_data["description"],
+            archived=nb_data["archived"],
+        )
+        nb.id = nb_data["id"]
+        nb.created = nb_data["created"]
+        nb.updated = nb_data["updated"]
+        return nb
+
+    def update_notebook(self, notebook: Notebook) -> Notebook:
+        """Update a notebook."""
+        updates = {
+            "name": notebook.name,
+            "description": notebook.description,
+            "archived": notebook.archived,
+        }
+        response = api_client.update_notebook(notebook.id or "", **updates)
+        nb_data = response if isinstance(response, dict) else response[0]
+        # Update the notebook object with the response
+        notebook.name = nb_data["name"]
+        notebook.description = nb_data["description"]
+        notebook.archived = nb_data["archived"]
+        notebook.updated = nb_data["updated"]
+        return notebook
+
+    def delete_notebook(self, notebook: Notebook) -> bool:
+        """Delete a notebook."""
+        api_client.delete_notebook(notebook.id or "")
+        return True
+
+
+# Global service instance
+notebook_service = NotebookService()

api/notes_service.py

@@ -0,0 +1,103 @@
+"""
+Notes service layer using API.
+"""
+
+from typing import List, Optional
+
+from loguru import logger
+
+from api.client import api_client
+from open_notebook.domain.notebook import Note
+
+
+class NotesService:
+    """Service layer for notes operations using API."""
+
+    def __init__(self):
+        logger.info("Using API for notes operations")
+
+    def get_all_notes(self, notebook_id: Optional[str] = None) -> List[Note]:
+        """Get all notes with optional notebook filtering."""
+        notes_data = api_client.get_notes(notebook_id=notebook_id)
+        # Convert API response to Note objects
+        notes = []
+        for note_data in notes_data:
+            note = Note(
+                title=note_data["title"],
+                content=note_data["content"],
+                note_type=note_data["note_type"],
+            )
+            note.id = note_data["id"]
+            note.created = note_data["created"]
+            note.updated = note_data["updated"]
+            notes.append(note)
+        return notes
+
+    def get_note(self, note_id: str) -> Note:
+        """Get a specific note."""
+        note_response = api_client.get_note(note_id)
+        note_data = (
+            note_response if isinstance(note_response, dict) else note_response[0]
+        )
+        note = Note(
+            title=note_data["title"],
+            content=note_data["content"],
+            note_type=note_data["note_type"],
+        )
+        note.id = note_data["id"]
+        note.created = note_data["created"]
+        note.updated = note_data["updated"]
+        return note
+
+    def create_note(
+        self,
+        content: str,
+        title: Optional[str] = None,
+        note_type: str = "human",
+        notebook_id: Optional[str] = None,
+    ) -> Note:
+        """Create a new note."""
+        note_response = api_client.create_note(
+            content=content, title=title, note_type=note_type, notebook_id=notebook_id
+        )
+        note_data = (
+            note_response if isinstance(note_response, dict) else note_response[0]
+        )
+        note = Note(
+            title=note_data["title"],
+            content=note_data["content"],
+            note_type=note_data["note_type"],
+        )
+        note.id = note_data["id"]
+        note.created = note_data["created"]
+        note.updated = note_data["updated"]
+        return note
+
+    def update_note(self, note: Note) -> Note:
+        """Update a note."""
+        updates = {
+            "title": note.title,
+            "content": note.content,
+            "note_type": note.note_type,
+        }
+        note_response = api_client.update_note(note.id or "", **updates)
+        note_data = (
+            note_response if isinstance(note_response, dict) else note_response[0]
+        )
+
+        # Update the note object with the response
+        note.title = note_data["title"]
+        note.content = note_data["content"]
+        note.note_type = note_data["note_type"]
+        note.updated = note_data["updated"]
+
+        return note
+
+    def delete_note(self, note_id: str) -> bool:
+        """Delete a note."""
+        api_client.delete_note(note_id)
+        return True
+
+
+# Global service instance
+notes_service = NotesService()

api/podcast_api_service.py

@@ -0,0 +1,125 @@
+"""
+Podcast service layer using API client.
+This replaces direct httpx calls in the Streamlit pages.
+"""
+
+from typing import Any, Dict, List
+
+from loguru import logger
+
+from api.client import api_client
+
+
+class PodcastAPIService:
+    """Service layer for podcast operations using API client."""
+
+    def __init__(self):
+        logger.info("Using API client for podcast operations")
+
+    # Episode methods
+    def get_episodes(self) -> List[Dict[Any, Any]]:
+        """Get all podcast episodes."""
+        result = api_client._make_request("GET", "/api/podcasts/episodes")
+        return result if isinstance(result, list) else [result]
+
+    def delete_episode(self, episode_id: str) -> bool:
+        """Delete a podcast episode."""
+        try:
+            api_client._make_request("DELETE", f"/api/podcasts/episodes/{episode_id}")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to delete episode: {e}")
+            return False
+
+    # Episode Profile methods
+    def get_episode_profiles(self) -> List[Dict]:
+        """Get all episode profiles."""
+        return api_client.get_episode_profiles()
+
+    def create_episode_profile(self, profile_data: Dict) -> bool:
+        """Create a new episode profile."""
+        try:
+            api_client.create_episode_profile(**profile_data)
+            return True
+        except Exception as e:
+            logger.error(f"Failed to create episode profile: {e}")
+            return False
+
+    def update_episode_profile(self, profile_id: str, profile_data: Dict) -> bool:
+        """Update an episode profile."""
+        try:
+            api_client.update_episode_profile(profile_id, **profile_data)
+            return True
+        except Exception as e:
+            logger.error(f"Failed to update episode profile: {e}")
+            return False
+
+    def delete_episode_profile(self, profile_id: str) -> bool:
+        """Delete an episode profile."""
+        try:
+            api_client.delete_episode_profile(profile_id)
+            return True
+        except Exception as e:
+            logger.error(f"Failed to delete episode profile: {e}")
+            return False
+
+    def duplicate_episode_profile(self, profile_id: str) -> bool:
+        """Duplicate an episode profile."""
+        try:
+            api_client._make_request(
+                "POST", f"/api/episode-profiles/{profile_id}/duplicate"
+            )
+            return True
+        except Exception as e:
+            logger.error(f"Failed to duplicate episode profile: {e}")
+            return False
+
+    # Speaker Profile methods
+    def get_speaker_profiles(self) -> List[Dict[Any, Any]]:
+        """Get all speaker profiles."""
+        result = api_client._make_request("GET", "/api/speaker-profiles")
+        return result if isinstance(result, list) else [result]
+
+    def create_speaker_profile(self, profile_data: Dict) -> bool:
+        """Create a new speaker profile."""
+        try:
+            api_client._make_request("POST", "/api/speaker-profiles", json=profile_data)
+            return True
+        except Exception as e:
+            logger.error(f"Failed to create speaker profile: {e}")
+            return False
+
+    def update_speaker_profile(self, profile_id: str, profile_data: Dict) -> bool:
+        """Update a speaker profile."""
+        try:
+            api_client._make_request(
+                "PUT", f"/api/speaker-profiles/{profile_id}", json=profile_data
+            )
+            return True
+        except Exception as e:
+            logger.error(f"Failed to update speaker profile: {e}")
+            return False
+
+    def delete_speaker_profile(self, profile_id: str) -> bool:
+        """Delete a speaker profile."""
+        try:
+            api_client._make_request("DELETE", f"/api/speaker-profiles/{profile_id}")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to delete speaker profile: {e}")
+            return False
+
+    def duplicate_speaker_profile(self, profile_id: str) -> bool:
+        """Duplicate a speaker profile."""
+        try:
+            api_client._make_request(
+                "POST", f"/api/speaker-profiles/{profile_id}/duplicate"
+            )
+            return True
+        except Exception as e:
+            logger.error(f"Failed to duplicate speaker profile: {e}")
+            return False
+
+
+# Global service instance
+podcast_api_service = PodcastAPIService()

api/podcast_service.py

@@ -0,0 +1,206 @@
+from typing import Any, Dict, Optional
+
+from fastapi import HTTPException
+from loguru import logger
+from pydantic import BaseModel
+from surreal_commands import get_command_status, submit_command
+
+from open_notebook.domain.notebook import Notebook
+from open_notebook.podcasts.models import EpisodeProfile, PodcastEpisode, SpeakerProfile
+
+
+class PodcastGenerationRequest(BaseModel):
+    """Request model for podcast generation"""
+
+    episode_profile: str
+    speaker_profile: str
+    episode_name: str
+    content: Optional[str] = None
+    notebook_id: Optional[str] = None
+    briefing_suffix: Optional[str] = None
+
+
+class PodcastGenerationResponse(BaseModel):
+    """Response model for podcast generation"""
+
+    job_id: str
+    status: str
+    message: str
+    episode_profile: str
+    episode_name: str
+
+
+class PodcastService:
+    """Service layer for podcast operations"""
+
+    @staticmethod
+    async def submit_generation_job(
+        episode_profile_name: str,
+        speaker_profile_name: str,
+        episode_name: str,
+        notebook_id: Optional[str] = None,
+        content: Optional[str] = None,
+        briefing_suffix: Optional[str] = None,
+    ) -> str:
+        """Submit a podcast generation job for background processing"""
+        try:
+            # Validate episode profile exists
+            episode_profile = await EpisodeProfile.get_by_name(episode_profile_name)
+            if not episode_profile:
+                raise ValueError(f"Episode profile '{episode_profile_name}' not found")
+
+            # Validate speaker profile exists
+            speaker_profile = await SpeakerProfile.get_by_name(speaker_profile_name)
+            if not speaker_profile:
+                raise ValueError(f"Speaker profile '{speaker_profile_name}' not found")
+
+            # Get content from notebook if not provided directly
+            if not content and notebook_id:
+                try:
+                    notebook = await Notebook.get(notebook_id)
+                    # Get notebook context (this may need to be adjusted based on actual Notebook implementation)
+                    content = (
+                        await notebook.get_context()
+                        if hasattr(notebook, "get_context")
+                        else str(notebook)
+                    )
+                except Exception as e:
+                    logger.warning(
+                        f"Failed to get notebook content, using notebook_id as content: {e}"
+                    )
+                    content = f"Notebook ID: {notebook_id}"
+
+            if not content:
+                raise ValueError(
+                    "Content is required - provide either content or notebook_id"
+                )
+
+            # Prepare command arguments
+            command_args = {
+                "episode_profile": episode_profile_name,
+                "speaker_profile": speaker_profile_name,
+                "episode_name": episode_name,
+                "content": str(content),
+                "briefing_suffix": briefing_suffix,
+            }
+
+            # Ensure command modules are imported before submitting
+            # This is needed because submit_command validates against local registry
+            try:
+                import commands.podcast_commands  # noqa: F401
+            except ImportError as import_err:
+                logger.error(f"Failed to import podcast commands: {import_err}")
+                raise ValueError("Podcast commands not available")
+
+            # Submit command to surreal-commands
+            job_id = submit_command("open_notebook", "generate_podcast", command_args)
+
+            # Convert RecordID to string if needed
+            if not job_id:
+                raise ValueError("Failed to get job_id from submit_command")
+            job_id_str = str(job_id)
+            logger.info(
+                f"Submitted podcast generation job: {job_id_str} for episode '{episode_name}'"
+            )
+            return job_id_str
+
+        except Exception as e:
+            logger.error(f"Failed to submit podcast generation job: {e}")
+            raise HTTPException(
+                status_code=500,
+                detail=f"Failed to submit podcast generation job: {str(e)}",
+            )
+
+    @staticmethod
+    async def get_job_status(job_id: str) -> Dict[str, Any]:
+        """Get status of a podcast generation job"""
+        try:
+            status = await get_command_status(job_id)
+            return {
+                "job_id": job_id,
+                "status": status.status if status else "unknown",
+                "result": status.result if status else None,
+                "error_message": getattr(status, "error_message", None)
+                if status
+                else None,
+                "created": str(status.created)
+                if status and hasattr(status, "created") and status.created
+                else None,
+                "updated": str(status.updated)
+                if status and hasattr(status, "updated") and status.updated
+                else None,
+                "progress": getattr(status, "progress", None) if status else None,
+            }
+        except Exception as e:
+            logger.error(f"Failed to get podcast job status: {e}")
+            raise HTTPException(
+                status_code=500, detail=f"Failed to get job status: {str(e)}"
+            )
+
+    @staticmethod
+    async def list_episodes() -> list:
+        """List all podcast episodes"""
+        try:
+            episodes = await PodcastEpisode.get_all(order_by="created desc")
+            return episodes
+        except Exception as e:
+            logger.error(f"Failed to list podcast episodes: {e}")
+            raise HTTPException(
+                status_code=500, detail=f"Failed to list episodes: {str(e)}"
+            )
+
+    @staticmethod
+    async def get_episode(episode_id: str) -> PodcastEpisode:
+        """Get a specific podcast episode"""
+        try:
+            episode = await PodcastEpisode.get(episode_id)
+            return episode
+        except Exception as e:
+            logger.error(f"Failed to get podcast episode {episode_id}: {e}")
+            raise HTTPException(status_code=404, detail=f"Episode not found: {str(e)}")
+
+
+class DefaultProfiles:
+    """Utility class for creating default profiles (if needed beyond migration data)"""
+
+    @staticmethod
+    async def create_default_episode_profiles():
+        """Create default episode profiles if they don't exist"""
+        try:
+            # Check if profiles already exist
+            existing = await EpisodeProfile.get_all()
+            if existing:
+                logger.info(f"Episode profiles already exist: {len(existing)} found")
+                return existing
+
+            # This would create profiles, but since we have migration data,
+            # this is mainly for future extensibility
+            logger.info(
+                "Default episode profiles should be created via database migration"
+            )
+            return []
+
+        except Exception as e:
+            logger.error(f"Failed to create default episode profiles: {e}")
+            raise
+
+    @staticmethod
+    async def create_default_speaker_profiles():
+        """Create default speaker profiles if they don't exist"""
+        try:
+            # Check if profiles already exist
+            existing = await SpeakerProfile.get_all()
+            if existing:
+                logger.info(f"Speaker profiles already exist: {len(existing)} found")
+                return existing
+
+            # This would create profiles, but since we have migration data,
+            # this is mainly for future extensibility
+            logger.info(
+                "Default speaker profiles should be created via database migration"
+            )
+            return []
+
+        except Exception as e:
+            logger.error(f"Failed to create default speaker profiles: {e}")
+            raise

api/routers/auth.py

@@ -0,0 +1,27 @@
+"""
+Authentication router for Open Notebook API.
+Provides endpoints to check authentication status.
+"""
+
+from fastapi import APIRouter
+
+from open_notebook.utils.encryption import get_secret_from_env
+
+router = APIRouter(prefix="/auth", tags=["auth"])
+
+
+@router.get("/status")
+async def get_auth_status():
+    """
+    Check if authentication is enabled.
+    Returns whether a password is required to access the API.
+    Supports Docker secrets via OPEN_NOTEBOOK_PASSWORD_FILE.
+    """
+    auth_enabled = bool(get_secret_from_env("OPEN_NOTEBOOK_PASSWORD"))
+
+    return {
+        "auth_enabled": auth_enabled,
+        "message": "Authentication is required"
+        if auth_enabled
+        else "Authentication is disabled",
+    }

api/routers/chat.py

@@ -0,0 +1,526 @@
+import asyncio
+import traceback
+from typing import Any, Dict, List, Optional
+
+from fastapi import APIRouter, HTTPException, Query
+from langchain_core.runnables import RunnableConfig
+from loguru import logger
+from pydantic import BaseModel, Field
+
+from open_notebook.database.repository import ensure_record_id, repo_query
+from open_notebook.domain.notebook import ChatSession, Note, Notebook, Source
+from open_notebook.exceptions import (
+    NotFoundError,
+)
+from open_notebook.graphs.chat import graph as chat_graph
+from open_notebook.utils.graph_utils import get_session_message_count
+
+router = APIRouter()
+
+
+# Request/Response models
+class CreateSessionRequest(BaseModel):
+    notebook_id: str = Field(..., description="Notebook ID to create session for")
+    title: Optional[str] = Field(None, description="Optional session title")
+    model_override: Optional[str] = Field(
+        None, description="Optional model override for this session"
+    )
+
+
+class UpdateSessionRequest(BaseModel):
+    title: Optional[str] = Field(None, description="New session title")
+    model_override: Optional[str] = Field(
+        None, description="Model override for this session"
+    )
+
+
+class ChatMessage(BaseModel):
+    id: str = Field(..., description="Message ID")
+    type: str = Field(..., description="Message type (human|ai)")
+    content: str = Field(..., description="Message content")
+    timestamp: Optional[str] = Field(None, description="Message timestamp")
+
+
+class ChatSessionResponse(BaseModel):
+    id: str = Field(..., description="Session ID")
+    title: str = Field(..., description="Session title")
+    notebook_id: Optional[str] = Field(None, description="Notebook ID")
+    created: str = Field(..., description="Creation timestamp")
+    updated: str = Field(..., description="Last update timestamp")
+    message_count: Optional[int] = Field(
+        None, description="Number of messages in session"
+    )
+    model_override: Optional[str] = Field(
+        None, description="Model override for this session"
+    )
+
+
+class ChatSessionWithMessagesResponse(ChatSessionResponse):
+    messages: List[ChatMessage] = Field(
+        default_factory=list, description="Session messages"
+    )
+
+
+class ExecuteChatRequest(BaseModel):
+    session_id: str = Field(..., description="Chat session ID")
+    message: str = Field(..., description="User message content")
+    context: Dict[str, Any] = Field(
+        ..., description="Chat context with sources and notes"
+    )
+    model_override: Optional[str] = Field(
+        None, description="Optional model override for this message"
+    )
+
+
+class ExecuteChatResponse(BaseModel):
+    session_id: str = Field(..., description="Session ID")
+    messages: List[ChatMessage] = Field(..., description="Updated message list")
+
+
+class BuildContextRequest(BaseModel):
+    notebook_id: str = Field(..., description="Notebook ID")
+    context_config: Dict[str, Any] = Field(..., description="Context configuration")
+
+
+class BuildContextResponse(BaseModel):
+    context: Dict[str, Any] = Field(..., description="Built context data")
+    token_count: int = Field(..., description="Estimated token count")
+    char_count: int = Field(..., description="Character count")
+
+
+class SuccessResponse(BaseModel):
+    success: bool = Field(True, description="Operation success status")
+    message: str = Field(..., description="Success message")
+
+
+@router.get("/chat/sessions", response_model=List[ChatSessionResponse])
+async def get_sessions(notebook_id: str = Query(..., description="Notebook ID")):
+    """Get all chat sessions for a notebook."""
+    try:
+        # Get notebook to verify it exists
+        notebook = await Notebook.get(notebook_id)
+        if not notebook:
+            raise HTTPException(status_code=404, detail="Notebook not found")
+
+        # Get sessions for this notebook
+        sessions_list = await notebook.get_chat_sessions()
+
+        results = []
+        for session in sessions_list:
+            session_id = str(session.id)
+
+            # Get message count from LangGraph state
+            msg_count = await get_session_message_count(chat_graph, session_id)
+
+            results.append(
+                ChatSessionResponse(
+                    id=session.id or "",
+                    title=session.title or "Untitled Session",
+                    notebook_id=notebook_id,
+                    created=str(session.created),
+                    updated=str(session.updated),
+                    message_count=msg_count,
+                    model_override=getattr(session, "model_override", None),
+                )
+            )
+
+        return results
+    except NotFoundError:
+        raise HTTPException(status_code=404, detail="Notebook not found")
+    except Exception as e:
+        logger.error(f"Error fetching chat sessions: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error fetching chat sessions: {str(e)}"
+        )
+
+
+@router.post("/chat/sessions", response_model=ChatSessionResponse)
+async def create_session(request: CreateSessionRequest):
+    """Create a new chat session."""
+    try:
+        # Verify notebook exists
+        notebook = await Notebook.get(request.notebook_id)
+        if not notebook:
+            raise HTTPException(status_code=404, detail="Notebook not found")
+
+        # Create new session
+        session = ChatSession(
+            title=request.title
+            or f"Chat Session {asyncio.get_event_loop().time():.0f}",
+            model_override=request.model_override,
+        )
+        await session.save()
+
+        # Relate session to notebook
+        await session.relate_to_notebook(request.notebook_id)
+
+        return ChatSessionResponse(
+            id=session.id or "",
+            title=session.title or "",
+            notebook_id=request.notebook_id,
+            created=str(session.created),
+            updated=str(session.updated),
+            message_count=0,
+            model_override=session.model_override,
+        )
+    except NotFoundError:
+        raise HTTPException(status_code=404, detail="Notebook not found")
+    except Exception as e:
+        logger.error(f"Error creating chat session: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error creating chat session: {str(e)}"
+        )
+
+
+@router.get(
+    "/chat/sessions/{session_id}", response_model=ChatSessionWithMessagesResponse
+)
+async def get_session(session_id: str):
+    """Get a specific session with its messages."""
+    try:
+        # Get session
+        # Ensure session_id has proper table prefix
+        full_session_id = (
+            session_id
+            if session_id.startswith("chat_session:")
+            else f"chat_session:{session_id}"
+        )
+        session = await ChatSession.get(full_session_id)
+        if not session:
+            raise HTTPException(status_code=404, detail="Session not found")
+
+        # Get session state from LangGraph to retrieve messages
+        # Use sync get_state() in a thread since SqliteSaver doesn't support async
+        thread_state = await asyncio.to_thread(
+            chat_graph.get_state,
+            config=RunnableConfig(configurable={"thread_id": full_session_id}),
+        )
+
+        # Extract messages from state
+        messages: list[ChatMessage] = []
+        if thread_state and thread_state.values and "messages" in thread_state.values:
+            for msg in thread_state.values["messages"]:
+                messages.append(
+                    ChatMessage(
+                        id=getattr(msg, "id", f"msg_{len(messages)}"),
+                        type=msg.type if hasattr(msg, "type") else "unknown",
+                        content=msg.content if hasattr(msg, "content") else str(msg),
+                        timestamp=None,  # LangChain messages don't have timestamps by default
+                    )
+                )
+
+        # Find notebook_id (we need to query the relationship)
+        # Ensure session_id has proper table prefix
+        full_session_id = (
+            session_id
+            if session_id.startswith("chat_session:")
+            else f"chat_session:{session_id}"
+        )
+
+        notebook_query = await repo_query(
+            "SELECT out FROM refers_to WHERE in = $session_id",
+            {"session_id": ensure_record_id(full_session_id)},
+        )
+
+        notebook_id = notebook_query[0]["out"] if notebook_query else None
+
+        if not notebook_id:
+            # This might be an old session created before API migration
+            logger.warning(
+                f"No notebook relationship found for session {session_id} - may be an orphaned session"
+            )
+
+        return ChatSessionWithMessagesResponse(
+            id=session.id or "",
+            title=session.title or "Untitled Session",
+            notebook_id=notebook_id,
+            created=str(session.created),
+            updated=str(session.updated),
+            message_count=len(messages),
+            messages=messages,
+            model_override=getattr(session, "model_override", None),
+        )
+    except NotFoundError:
+        raise HTTPException(status_code=404, detail="Session not found")
+    except Exception as e:
+        logger.error(f"Error fetching session: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error fetching session: {str(e)}")
+
+
+@router.put("/chat/sessions/{session_id}", response_model=ChatSessionResponse)
+async def update_session(session_id: str, request: UpdateSessionRequest):
+    """Update session title."""
+    try:
+        # Ensure session_id has proper table prefix
+        full_session_id = (
+            session_id
+            if session_id.startswith("chat_session:")
+            else f"chat_session:{session_id}"
+        )
+        session = await ChatSession.get(full_session_id)
+        if not session:
+            raise HTTPException(status_code=404, detail="Session not found")
+
+        update_data = request.model_dump(exclude_unset=True)
+
+        if "title" in update_data:
+            session.title = update_data["title"]
+
+        if "model_override" in update_data:
+            session.model_override = update_data["model_override"]
+
+        await session.save()
+
+        # Find notebook_id
+        # Ensure session_id has proper table prefix
+        full_session_id = (
+            session_id
+            if session_id.startswith("chat_session:")
+            else f"chat_session:{session_id}"
+        )
+        notebook_query = await repo_query(
+            "SELECT out FROM refers_to WHERE in = $session_id",
+            {"session_id": ensure_record_id(full_session_id)},
+        )
+        notebook_id = notebook_query[0]["out"] if notebook_query else None
+
+        # Get message count from LangGraph state
+        msg_count = await get_session_message_count(chat_graph, full_session_id)
+
+        return ChatSessionResponse(
+            id=session.id or "",
+            title=session.title or "",
+            notebook_id=notebook_id,
+            created=str(session.created),
+            updated=str(session.updated),
+            message_count=msg_count,
+            model_override=session.model_override,
+        )
+    except NotFoundError:
+        raise HTTPException(status_code=404, detail="Session not found")
+    except Exception as e:
+        logger.error(f"Error updating session: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error updating session: {str(e)}")
+
+
+@router.delete("/chat/sessions/{session_id}", response_model=SuccessResponse)
+async def delete_session(session_id: str):
+    """Delete a chat session."""
+    try:
+        # Ensure session_id has proper table prefix
+        full_session_id = (
+            session_id
+            if session_id.startswith("chat_session:")
+            else f"chat_session:{session_id}"
+        )
+        session = await ChatSession.get(full_session_id)
+        if not session:
+            raise HTTPException(status_code=404, detail="Session not found")
+
+        await session.delete()
+
+        return SuccessResponse(success=True, message="Session deleted successfully")
+    except NotFoundError:
+        raise HTTPException(status_code=404, detail="Session not found")
+    except Exception as e:
+        logger.error(f"Error deleting session: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error deleting session: {str(e)}")
+
+
+@router.post("/chat/execute", response_model=ExecuteChatResponse)
+async def execute_chat(request: ExecuteChatRequest):
+    """Execute a chat request and get AI response."""
+    try:
+        # Verify session exists
+        # Ensure session_id has proper table prefix
+        full_session_id = (
+            request.session_id
+            if request.session_id.startswith("chat_session:")
+            else f"chat_session:{request.session_id}"
+        )
+        session = await ChatSession.get(full_session_id)
+        if not session:
+            raise HTTPException(status_code=404, detail="Session not found")
+
+        # Fetch notebook linked to this session
+        notebook_query = await repo_query(
+            "SELECT out FROM refers_to WHERE in = $session_id",
+            {"session_id": ensure_record_id(full_session_id)},
+        )
+        notebook = None
+        if notebook_query:
+            notebook = await Notebook.get(notebook_query[0]["out"])
+
+        # Determine model override (per-request override takes precedence over session-level)
+        model_override = (
+            request.model_override
+            if request.model_override is not None
+            else getattr(session, "model_override", None)
+        )
+
+        # Get current state
+        # Use sync get_state() in a thread since SqliteSaver doesn't support async
+        current_state = await asyncio.to_thread(
+            chat_graph.get_state,
+            config=RunnableConfig(configurable={"thread_id": full_session_id}),
+        )
+
+        # Prepare state for execution
+        state_values = current_state.values if current_state else {}
+        state_values["messages"] = state_values.get("messages", [])
+        state_values["context"] = request.context
+        state_values["notebook"] = notebook
+        state_values["model_override"] = model_override
+
+        # Add user message to state
+        from langchain_core.messages import HumanMessage
+
+        user_message = HumanMessage(content=request.message)
+        state_values["messages"].append(user_message)
+
+        # Execute chat graph
+        result = chat_graph.invoke(
+            input=state_values,  # type: ignore[arg-type]
+            config=RunnableConfig(
+                configurable={
+                    "thread_id": full_session_id,
+                    "model_id": model_override,
+                }
+            ),
+        )
+
+        # Update session timestamp
+        await session.save()
+
+        # Convert messages to response format
+        messages: list[ChatMessage] = []
+        for msg in result.get("messages", []):
+            messages.append(
+                ChatMessage(
+                    id=getattr(msg, "id", f"msg_{len(messages)}"),
+                    type=msg.type if hasattr(msg, "type") else "unknown",
+                    content=msg.content if hasattr(msg, "content") else str(msg),
+                    timestamp=None,
+                )
+            )
+
+        return ExecuteChatResponse(session_id=request.session_id, messages=messages)
+    except NotFoundError:
+        raise HTTPException(status_code=404, detail="Session not found")
+    except Exception as e:
+        # Log detailed error with context for debugging
+        logger.error(
+            f"Error executing chat: {str(e)}\n"
+            f"  Session ID: {request.session_id}\n"
+            f"  Model override: {request.model_override}\n"
+            f"  Traceback:\n{traceback.format_exc()}"
+        )
+        raise HTTPException(status_code=500, detail=f"Error executing chat: {str(e)}")
+
+
+@router.post("/chat/context", response_model=BuildContextResponse)
+async def build_context(request: BuildContextRequest):
+    """Build context for a notebook based on context configuration."""
+    try:
+        # Verify notebook exists
+        notebook = await Notebook.get(request.notebook_id)
+        if not notebook:
+            raise HTTPException(status_code=404, detail="Notebook not found")
+
+        context_data: dict[str, list[dict[str, str]]] = {"sources": [], "notes": []}
+        total_content = ""
+
+        # Process context configuration if provided
+        if request.context_config:
+            # Process sources
+            for source_id, status in request.context_config.get("sources", {}).items():
+                if "not in" in status:
+                    continue
+
+                try:
+                    # Add table prefix if not present
+                    full_source_id = (
+                        source_id
+                        if source_id.startswith("source:")
+                        else f"source:{source_id}"
+                    )
+
+                    try:
+                        source = await Source.get(full_source_id)
+                    except Exception:
+                        continue
+
+                    if "insights" in status:
+                        source_context = await source.get_context(context_size="short")
+                        context_data["sources"].append(source_context)
+                        total_content += str(source_context)
+                    elif "full content" in status:
+                        source_context = await source.get_context(context_size="long")
+                        context_data["sources"].append(source_context)
+                        total_content += str(source_context)
+                except Exception as e:
+                    logger.warning(f"Error processing source {source_id}: {str(e)}")
+                    continue
+
+            # Process notes
+            for note_id, status in request.context_config.get("notes", {}).items():
+                if "not in" in status:
+                    continue
+
+                try:
+                    # Add table prefix if not present
+                    full_note_id = (
+                        note_id if note_id.startswith("note:") else f"note:{note_id}"
+                    )
+                    note = await Note.get(full_note_id)
+                    if not note:
+                        continue
+
+                    if "full content" in status:
+                        note_context = note.get_context(context_size="long")
+                        context_data["notes"].append(note_context)
+                        total_content += str(note_context)
+                except Exception as e:
+                    logger.warning(f"Error processing note {note_id}: {str(e)}")
+                    continue
+        else:
+            # Default behavior - include all sources and notes with short context
+            sources = await notebook.get_sources()
+            for source in sources:
+                try:
+                    source_context = await source.get_context(context_size="short")
+                    context_data["sources"].append(source_context)
+                    total_content += str(source_context)
+                except Exception as e:
+                    logger.warning(f"Error processing source {source.id}: {str(e)}")
+                    continue
+
+            notes = await notebook.get_notes()
+            for note in notes:
+                try:
+                    note_context = note.get_context(context_size="short")
+                    context_data["notes"].append(note_context)
+                    total_content += str(note_context)
+                except Exception as e:
+                    logger.warning(f"Error processing note {note.id}: {str(e)}")
+                    continue
+
+        # Calculate character and token counts
+        char_count = len(total_content)
+        # Use token count utility if available
+        try:
+            from open_notebook.utils import token_count
+
+            estimated_tokens = token_count(total_content) if total_content else 0
+        except ImportError:
+            # Fallback to simple estimation
+            estimated_tokens = char_count // 4
+
+        return BuildContextResponse(
+            context=context_data, token_count=estimated_tokens, char_count=char_count
+        )
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error building context: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error building context: {str(e)}")

api/routers/commands.py

@@ -0,0 +1,166 @@
+from typing import Any, Dict, List, Optional
+
+from fastapi import APIRouter, HTTPException, Query
+from loguru import logger
+from pydantic import BaseModel, Field
+from surreal_commands import registry
+
+from api.command_service import CommandService
+
+router = APIRouter()
+
+
+class CommandExecutionRequest(BaseModel):
+    command: str = Field(
+        ..., description="Command function name (e.g., 'process_text')"
+    )
+    app: str = Field(..., description="Application name (e.g., 'open_notebook')")
+    input: Dict[str, Any] = Field(..., description="Arguments to pass to the command")
+
+
+class CommandJobResponse(BaseModel):
+    job_id: str
+    status: str
+    message: str
+
+
+class CommandJobStatusResponse(BaseModel):
+    job_id: str
+    status: str
+    result: Optional[Dict[str, Any]] = None
+    error_message: Optional[str] = None
+    created: Optional[str] = None
+    updated: Optional[str] = None
+    progress: Optional[Dict[str, Any]] = None
+
+
+@router.post("/commands/jobs", response_model=CommandJobResponse)
+async def execute_command(request: CommandExecutionRequest):
+    """
+    Submit a command for background processing.
+    Returns immediately with job ID for status tracking.
+
+    Example request:
+    {
+        "command": "process_text",
+        "app": "open_notebook",
+        "input": {
+            "text": "Hello world",
+            "operation": "uppercase"
+        }
+    }
+    """
+    try:
+        # Submit command using app name (not module name)
+        job_id = await CommandService.submit_command_job(
+            module_name=request.app,  # This should be "open_notebook"
+            command_name=request.command,
+            command_args=request.input,
+        )
+
+        return CommandJobResponse(
+            job_id=job_id,
+            status="submitted",
+            message=f"Command '{request.command}' submitted successfully",
+        )
+
+    except Exception as e:
+        logger.error(f"Error submitting command: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail="Failed to submit command"
+        )
+
+
+@router.get("/commands/jobs/{job_id}", response_model=CommandJobStatusResponse)
+async def get_command_job_status(job_id: str):
+    """Get the status of a specific command job"""
+    try:
+        status_data = await CommandService.get_command_status(job_id)
+        return CommandJobStatusResponse(**status_data)
+
+    except Exception as e:
+        logger.error(f"Error fetching job status: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail="Failed to fetch job status"
+        )
+
+
+@router.get("/commands/jobs", response_model=List[Dict[str, Any]])
+async def list_command_jobs(
+    command_filter: Optional[str] = Query(None, description="Filter by command name"),
+    status_filter: Optional[str] = Query(None, description="Filter by status"),
+    limit: int = Query(50, description="Maximum number of jobs to return"),
+):
+    """List command jobs with optional filtering"""
+    try:
+        jobs = await CommandService.list_command_jobs(
+            command_filter=command_filter, status_filter=status_filter, limit=limit
+        )
+        return jobs
+
+    except Exception as e:
+        logger.error(f"Error listing command jobs: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail="Failed to list command jobs"
+        )
+
+
+@router.delete("/commands/jobs/{job_id}")
+async def cancel_command_job(job_id: str):
+    """Cancel a running command job"""
+    try:
+        success = await CommandService.cancel_command_job(job_id)
+        return {"job_id": job_id, "cancelled": success}
+
+    except Exception as e:
+        logger.error(f"Error cancelling command job: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail="Failed to cancel command job"
+        )
+
+
+@router.get("/commands/registry/debug")
+async def debug_registry():
+    """Debug endpoint to see what commands are registered"""
+    try:
+        # Get all registered commands
+        all_items = registry.get_all_commands()
+
+        # Create JSON-serializable data
+        command_items = []
+        for item in all_items:
+            try:
+                command_items.append(
+                    {
+                        "app_id": item.app_id,
+                        "name": item.name,
+                        "full_id": f"{item.app_id}.{item.name}",
+                    }
+                )
+            except Exception as item_error:
+                logger.error(f"Error processing item: {item_error}")
+
+        # Get the basic command structure
+        try:
+            commands_dict: dict[str, list[str]] = {}
+            for item in all_items:
+                if item.app_id not in commands_dict:
+                    commands_dict[item.app_id] = []
+                commands_dict[item.app_id].append(item.name)
+        except Exception:
+            commands_dict = {}
+
+        return {
+            "total_commands": len(all_items),
+            "commands_by_app": commands_dict,
+            "command_items": command_items,
+        }
+
+    except Exception as e:
+        logger.error(f"Error debugging registry: {str(e)}")
+        return {
+            "error": str(e),
+            "total_commands": 0,
+            "commands_by_app": {},
+            "command_items": [],
+        }

api/routers/config.py

@@ -0,0 +1,160 @@
+import asyncio
+import os
+import time
+import tomllib
+from pathlib import Path
+from typing import Optional
+
+from fastapi import APIRouter, Request
+from loguru import logger
+
+from open_notebook.database.repository import repo_query
+from open_notebook.utils.version_utils import (
+    compare_versions,
+    get_version_from_github_async,
+)
+
+router = APIRouter()
+
+# In-memory cache for version check results
+_version_cache: dict = {
+    "latest_version": None,
+    "has_update": False,
+    "timestamp": 0,
+    "check_failed": False,
+}
+
+# Cache TTL in seconds (24 hours)
+VERSION_CACHE_TTL = 24 * 60 * 60
+
+
+def get_version() -> str:
+    """Read version from pyproject.toml"""
+    try:
+        pyproject_path = Path(__file__).parent.parent.parent / "pyproject.toml"
+        with open(pyproject_path, "rb") as f:
+            pyproject = tomllib.load(f)
+            return pyproject.get("project", {}).get("version", "unknown")
+    except Exception as e:
+        logger.warning(f"Could not read version from pyproject.toml: {e}")
+        return "unknown"
+
+
+async def get_latest_version_cached(current_version: str) -> tuple[Optional[str], bool]:
+    """
+    Check for the latest version from GitHub with caching.
+
+    Returns:
+        tuple: (latest_version, has_update)
+        - latest_version: str or None if check failed
+        - has_update: bool indicating if update is available
+    """
+    global _version_cache
+
+    # Check if cache is still valid (within TTL)
+    cache_age = time.time() - _version_cache["timestamp"]
+    if _version_cache["timestamp"] > 0 and cache_age < VERSION_CACHE_TTL:
+        logger.debug(f"Using cached version check result (age: {cache_age:.0f}s)")
+        return _version_cache["latest_version"], _version_cache["has_update"]
+
+    # Cache expired or not yet set
+    if _version_cache["timestamp"] > 0:
+        logger.info(f"Version cache expired (age: {cache_age:.0f}s), refreshing...")
+
+    # Perform version check with strict error handling
+    try:
+        logger.info("Checking for latest version from GitHub...")
+
+        # Fetch latest version from GitHub with 10-second timeout
+        latest_version = await get_version_from_github_async(
+            "https://github.com/lfnovo/open-notebook", "main"
+        )
+
+        logger.info(
+            f"Latest version from GitHub: {latest_version}, Current version: {current_version}"
+        )
+
+        # Compare versions
+        has_update = compare_versions(current_version, latest_version) < 0
+
+        # Cache the result
+        _version_cache["latest_version"] = latest_version
+        _version_cache["has_update"] = has_update
+        _version_cache["timestamp"] = time.time()
+        _version_cache["check_failed"] = False
+
+        logger.info(f"Version check complete. Update available: {has_update}")
+
+        return latest_version, has_update
+
+    except Exception as e:
+        logger.warning(f"Version check failed: {e}")
+
+        # Cache the failure to avoid repeated attempts
+        _version_cache["latest_version"] = None
+        _version_cache["has_update"] = False
+        _version_cache["timestamp"] = time.time()
+        _version_cache["check_failed"] = True
+
+        return None, False
+
+
+async def check_database_health() -> dict:
+    """
+    Check if database is reachable using a lightweight query.
+
+    Returns:
+        dict with 'status' ("online" | "offline") and optional 'error'
+    """
+    try:
+        # 2-second timeout for database health check
+        result = await asyncio.wait_for(repo_query("RETURN 1"), timeout=2.0)
+        if result:
+            return {"status": "online"}
+        return {"status": "offline", "error": "Empty result"}
+    except asyncio.TimeoutError:
+        logger.warning("Database health check timed out after 2 seconds")
+        return {"status": "offline", "error": "Health check timeout"}
+    except Exception as e:
+        logger.warning(f"Database health check failed: {e}")
+        return {"status": "offline", "error": str(e)}
+
+
+@router.get("/config")
+async def get_config(request: Request):
+    """
+    Get frontend configuration.
+
+    Returns version information and health status.
+    Note: The frontend determines the API URL via its own runtime-config endpoint,
+    so this endpoint no longer returns apiUrl.
+
+    Also checks for version updates from GitHub (with caching and error handling).
+    """
+    # Get current version
+    current_version = get_version()
+
+    # Check for updates (with caching and error handling)
+    # This MUST NOT break the endpoint - wrapped in try-except as extra safety
+    latest_version = None
+    has_update = False
+
+    try:
+        latest_version, has_update = await get_latest_version_cached(current_version)
+    except Exception as e:
+        # Extra safety: ensure version check never breaks the config endpoint
+        logger.error(f"Unexpected error during version check: {e}")
+
+    # Check database health
+    db_health = await check_database_health()
+    db_status = db_health["status"]
+
+    if db_status == "offline":
+        logger.warning(f"Database offline: {db_health.get('error', 'Unknown error')}")
+
+    return {
+        "version": current_version,
+        "latestVersion": latest_version,
+        "hasUpdate": has_update,
+        "dbStatus": db_status,
+    }