Deploy MiningNiti API with production RAG pipeline

.gitignore

@@ -0,0 +1,7 @@
+__pycache__/
+*.py[cod]
+venv/
+.env
+*.log
+.pytest_cache/
+test.db

Dockerfile

@@ -0,0 +1,24 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y \
+    libpq-dev gcc libmagic1 curl \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cpu
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+RUN python -c "from sentence_transformers import CrossEncoder; CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')"
+
+ENV TOKENIZERS_PARALLELISM=false
+ENV TRANSFORMERS_CACHE=/app/.cache/huggingface
+ENV SENTENCE_TRANSFORMERS_HOME=/app/.cache/sentence-transformers
+
+COPY . .
+
+EXPOSE 8000
+
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]

README.md

@@ -1,11 +1,19 @@
 ---
-title: Miningniti Api
-emoji: 🐨
-colorFrom: yellow
-colorTo: pink
+title: MiningNiti API
+emoji: ⛏️
+colorFrom: purple
+colorTo: blue
 sdk: docker
-pinned: false
-license: apache-2.0
+app_port: 8000
+pinned: true
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# MiningNiti API
+
+AI-powered document intelligence engine for the coal mining industry.
+
+## Features
+- Multi-agent AI pipeline (6 agents, 4 providers)
+- Production RAG with hybrid search + cross-encoder reranking
+- Compliance auto-auditing
+- Real-time streaming chat

alembic.ini

@@ -0,0 +1,112 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = alembic
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+file_template = %%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python>=3.9 or backports.zoneinfo library.
+# Any required deps can installed via pip[tz]
+# timezone = UTC
+
+# max length of characters to apply to the "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# set to 'true' to search source files recursively
+# in each "version_locations" directory
+# New in Alembic version 1.10
+# recursive_version_locations = false
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+# DATABASE_URL is loaded from environment variable in env.py
+# Do NOT hardcode the URL here — use .env file
+sqlalchemy.url = driver://user:pass@localhost/dbname
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# lint with attempts to fix using "ruff" - use the exec runner, against a script
+# hooks = ruff
+# ruff.type = exec
+# ruff.executable = %(here)s/.venv/bin/ruff
+# ruff.options = --fix REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARNING
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARNING
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

alembic/README

@@ -0,0 +1 @@
+Generic single-database configuration.

alembic/env.py

@@ -0,0 +1,84 @@
+"""
+Alembic Environment Configuration
+Loads DATABASE_URL from .env and connects all SQLAlchemy models.
+"""
+
+import os
+import sys
+from logging.config import fileConfig
+from pathlib import Path
+
+from sqlalchemy import engine_from_config, pool, text
+from alembic import context
+
+# ── Ensure 'backend/' is on sys.path so `app.*` imports work ──────────────────
+BACKEND_DIR = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(BACKEND_DIR))
+
+# ── Load settings (reads .env automatically via pydantic-settings) ─────────────
+from app.config import settings  # noqa: E402
+
+# ── Import ALL models so Alembic can detect their tables ──────────────────────
+from app.models.base import Base  # noqa: E402
+from app.models import user, document, chat, audit, prompt  # noqa: E402, F401
+
+# ── Alembic Config object ─────────────────────────────────────────────────────
+config = context.config
+
+# Override sqlalchemy.url with the value from our settings/.env
+config.set_main_option("sqlalchemy.url", settings.DATABASE_URL)
+
+# Interpret the config file for Python logging (if present)
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# Target metadata for autogenerate support
+target_metadata = Base.metadata
+
+
+def run_migrations_offline() -> None:
+    """
+    Run migrations in 'offline' mode.
+    Generates SQL scripts without a live DB connection.
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+        compare_type=True,
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """
+    Run migrations in 'online' mode.
+    Connects to the real database and applies migrations.
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            compare_type=True,
+            # Include schemas for pgvector extension objects
+            include_schemas=False,
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

alembic/script.py.mako

@@ -0,0 +1,26 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

alembic/versions/001_enable_pgvector.py

@@ -0,0 +1,153 @@
+"""
+Migration: Enable pgvector and migrate embeddings from JSONB to vector(768)
+
+Revision ID: 001
+Revises: (initial)
+Create Date: 2026-06-07
+
+Changes:
+  1. Enable the pgvector extension
+  2. Add page tracking columns to document_embeddings
+     (section_title, page_numbers)
+  3. Migrate embedding column from JSONB to vector(768) native type
+  4. Create HNSW index for fast ANN search (~5ms vs seconds)
+  5. Fix datetime.utcnow() deprecation in base models (add timezone info)
+  6. Add total_pages column to documents
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic
+revision = "001"
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ── Step 1: Enable pgvector extension ──────────────────────────────────────
+    op.execute("CREATE EXTENSION IF NOT EXISTS vector")
+    op.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm")  # For full-text search
+
+    # ── Step 2: Add page tracking columns to document_embeddings ───────────────
+    op.add_column(
+        "document_embeddings",
+        sa.Column("section_title", sa.String(500), nullable=True),
+    )
+    op.add_column(
+        "document_embeddings",
+        sa.Column(
+            "page_numbers",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=True,
+            comment="List of page numbers this chunk spans, e.g. [12, 13]",
+        ),
+    )
+
+    # ── Step 3: Add total_pages column to documents ────────────────────────────
+    # (page_count already exists — we just ensure it's correctly named)
+    # Add total_pages as an alias; keep page_count for backward compatibility
+    op.add_column(
+        "documents",
+        sa.Column("total_pages", sa.Integer(), nullable=True),
+    )
+
+    # ── Step 4: Migrate embedding column from JSONB → vector(768) ─────────────
+    # First add the new column
+    op.execute(
+        "ALTER TABLE document_embeddings ADD COLUMN embedding_vec vector(768)"
+    )
+
+    # Convert existing JSONB embeddings to vector type
+    # This handles both list-of-floats and null values safely
+    op.execute(
+        """
+        UPDATE document_embeddings
+        SET embedding_vec = (
+            SELECT array_agg(elem::float8)::vector(768)
+            FROM jsonb_array_elements_text(embedding) AS elem
+        )
+        WHERE embedding IS NOT NULL
+          AND jsonb_typeof(embedding) = 'array'
+          AND jsonb_array_length(embedding) = 768
+        """
+    )
+
+    # Drop the old JSONB column and rename new one
+    op.execute("ALTER TABLE document_embeddings DROP COLUMN embedding")
+    op.execute(
+        "ALTER TABLE document_embeddings RENAME COLUMN embedding_vec TO embedding"
+    )
+
+    # Make embedding NOT NULL (existing rows already converted)
+    op.execute(
+        "ALTER TABLE document_embeddings ALTER COLUMN embedding SET NOT NULL"
+    )
+
+    # ── Step 5: Create HNSW index for approximate nearest neighbor search ──────
+    # HNSW gives sub-5ms search up to ~1M vectors
+    # m=16: max connections per node (higher = better recall, more memory)
+    # ef_construction=200: build-time search depth (higher = better quality index)
+    op.execute(
+        """
+        CREATE INDEX idx_embeddings_hnsw
+        ON document_embeddings
+        USING hnsw (embedding vector_cosine_ops)
+        WITH (m = 16, ef_construction = 200)
+        """
+    )
+
+    # ── Step 6: Add composite index on (document_id, chunk_index) ──────────────
+    op.create_index(
+        "idx_embeddings_doc_chunk",
+        "document_embeddings",
+        ["document_id", "chunk_index"],
+        unique=True,
+    )
+
+    # ── Step 7: Add trigram index on documents for full-text search ────────────
+    op.execute(
+        """
+        CREATE INDEX idx_documents_title_trgm
+        ON documents
+        USING gin (title gin_trgm_ops)
+        """
+    )
+    op.execute(
+        """
+        CREATE INDEX idx_documents_filename_trgm
+        ON documents
+        USING gin (file_name gin_trgm_ops)
+        """
+    )
+
+
+def downgrade() -> None:
+    # Remove indexes
+    op.execute("DROP INDEX IF EXISTS idx_embeddings_hnsw")
+    op.execute("DROP INDEX IF EXISTS idx_documents_title_trgm")
+    op.execute("DROP INDEX IF EXISTS idx_documents_filename_trgm")
+    op.drop_index("idx_embeddings_doc_chunk", table_name="document_embeddings")
+
+    # Restore JSONB column
+    op.execute(
+        "ALTER TABLE document_embeddings ADD COLUMN embedding_jsonb jsonb"
+    )
+    op.execute(
+        """
+        UPDATE document_embeddings
+        SET embedding_jsonb = to_jsonb(embedding::float8[])
+        WHERE embedding IS NOT NULL
+        """
+    )
+    op.execute("ALTER TABLE document_embeddings DROP COLUMN embedding")
+    op.execute(
+        "ALTER TABLE document_embeddings RENAME COLUMN embedding_jsonb TO embedding"
+    )
+
+    # Remove added columns
+    op.drop_column("document_embeddings", "section_title")
+    op.drop_column("document_embeddings", "page_numbers")
+    op.drop_column("documents", "total_pages")

alembic/versions/002_hybrid_search_index.py

@@ -0,0 +1,43 @@
+"""Add GIN trigram index on chunk_text for hybrid search
+
+Revision ID: 002
+Revises: 001_enable_pgvector
+Create Date: 2025-01-01
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+revision = "002"
+down_revision = "001"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # Ensure pg_trgm extension exists (for trigram similarity)
+    op.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm")
+
+    # GIN trigram index on chunk_text for fast BM25-style keyword search
+    # Enables: WHERE chunk_text % :query (trigram similarity match)
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS idx_embeddings_chunk_text_trgm
+        ON document_embeddings
+        USING gin (chunk_text gin_trgm_ops)
+        """
+    )
+
+    # GIN trigram index on document content for full-document keyword search
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS idx_documents_content_trgm
+        ON documents
+        USING gin (content gin_trgm_ops)
+        """
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP INDEX IF EXISTS idx_embeddings_chunk_text_trgm")
+    op.execute("DROP INDEX IF EXISTS idx_documents_content_trgm")

app/README.md

@@ -0,0 +1,30 @@
+# MiningNiti Enterprise Backend
+
+Production-ready AI Document Intelligence Engine for the Mining Industry.
+
+## Structure
+
+```
+app/
+├── api/          # REST API endpoints
+├── agents/       # AI agents (LangGraph)
+├── core/         # Security, config, exceptions
+├── db/           # Database session & migrations
+├── models/       # SQLAlchemy ORM models
+├── schemas/      # Pydantic request/response schemas
+├── services/     # Business logic layer
+└── workers/      # Celery background tasks
+```
+
+## Quick Start
+
+```bash
+# Install dependencies
+pip install -r requirements.txt
+
+# Run development server
+uvicorn app.main:app --reload --port 8000
+
+# Run Celery worker
+celery -A app.workers.celery_app worker -l info
+```

app/__init__.py

@@ -0,0 +1,7 @@
+"""
+MiningNiti Enterprise Backend
+AI-Powered Document Intelligence for the Mining Industry
+"""
+
+__version__ = "2.0.0"
+__author__ = "MiningNiti Team"

app/agents/__init__.py

@@ -0,0 +1,20 @@
+"""
+AI Agents Module
+Specialized agents for mining document intelligence
+"""
+
+from app.agents.base import BaseAgent
+from app.agents.classifier import ClassifierAgent
+from app.agents.entity_extractor import EntityExtractorAgent
+from app.agents.orchestrator import AgentOrchestrator
+from app.agents.safety_analyzer import SafetyAnalyzerAgent
+from app.agents.summarizer import SummarizerAgent
+
+__all__ = [
+    "BaseAgent",
+    "ClassifierAgent",
+    "SafetyAnalyzerAgent",
+    "EntityExtractorAgent",
+    "SummarizerAgent",
+    "AgentOrchestrator",
+]

app/agents/base.py

@@ -0,0 +1,308 @@
+"""
+Base Agent
+Abstract base class for all mining intelligence agents.
+
+Improvements over v1:
+  - JSON mode via response_mime_type (no more regex JSON extraction)
+  - Retry with exponential backoff (3 retries), respecting Gemini retry_delay
+  - Processes full document via pages, not truncated to 3000 chars
+  - Confidence score required in all agent outputs
+  - Proper QuotaExceededError raised (no more silent empty-dict returns)
+"""
+
+import asyncio
+import logging
+import re
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional
+
+import google.generativeai as genai
+from google.generativeai.types import GenerationConfig
+
+from app.config import settings
+
+logger = logging.getLogger(__name__)
+
+# Configure Gemini once at module level
+genai.configure(api_key=settings.GEMINI_API_KEY)
+
+# Generation config that forces JSON output — no more regex parsing
+_JSON_GENERATION_CONFIG = GenerationConfig(
+    response_mime_type="application/json",
+    temperature=0.1,  # Low temperature for consistent structured output
+    top_p=0.95,
+)
+
+_MAX_RETRIES = 3
+_RETRY_BASE_DELAY = 2.0  # seconds — minimum delay between retries
+_MAX_RETRY_DELAY = 120.0  # seconds — cap for retry_delay parsed from API response
+
+
+class QuotaExceededError(RuntimeError):
+    """Raised when the Gemini API quota / rate-limit is exhausted."""
+
+
+def _parse_retry_delay(err_str: str) -> Optional[float]:
+    """
+    Extract the suggested retry_delay (in seconds) from a Gemini 429 error
+    message.  The error body contains a line like:
+        retry_delay { seconds: 31 }
+    Returns None if no delay can be parsed.
+    """
+    match = re.search(r"retry_delay\s*\{\s*seconds:\s*(\d+)", err_str)
+    if match:
+        return min(float(match.group(1)), _MAX_RETRY_DELAY)
+    # Fallback: look for "Please retry in X.Xs"
+    match2 = re.search(r"retry in (\d+\.?\d*)s", err_str)
+    if match2:
+        return min(float(match2.group(1)), _MAX_RETRY_DELAY)
+    return None
+
+
+class BaseAgent(ABC):
+    """
+    Abstract base class for mining document intelligence agents.
+
+    Each agent is responsible for a specific analysis task:
+    - Classification
+    - Safety Analysis
+    - Entity Extraction
+    - Summarization
+    """
+
+    def __init__(
+        self,
+        model_name: str = None,
+        provider: str = "gemini",
+        fallback_model: str = None,
+        fallback_provider: str = None,
+    ):
+        self.provider = provider
+        self.model_name = model_name or settings.GEMINI_MODEL
+        self.name = self.__class__.__name__
+
+        # Fallback config (e.g. Cerebras when Groq is rate-limited)
+        self.fallback_model = fallback_model
+        self.fallback_provider = fallback_provider
+        self._fallback_client = None
+        self._using_fallback = False
+
+        if self.fallback_provider and self.fallback_model:
+            self._init_fallback_client()
+
+        self._init_client()
+
+    def _init_client(self):
+        """Initialize the primary provider client."""
+        if self.provider == "gemini":
+            self.model = genai.GenerativeModel(
+                model_name=self.model_name,
+                generation_config=_JSON_GENERATION_CONFIG,
+            )
+        elif self.provider == "groq":
+            from app.services.llm_provider import get_groq_client
+
+            self.client = get_groq_client()
+        elif self.provider == "mistral":
+            from app.services.llm_provider import get_mistral_client
+
+            self.client = get_mistral_client()
+        elif self.provider == "cerebras":
+            from app.services.llm_provider import get_cerebras_client
+
+            self.client = get_cerebras_client()
+
+    def _init_fallback_client(self):
+        """Initialize the fallback provider client."""
+        if self.fallback_provider == "cerebras":
+            from app.services.llm_provider import get_cerebras_client
+
+            self._fallback_client = get_cerebras_client()
+        elif self.fallback_provider == "groq":
+            from app.services.llm_provider import get_groq_client
+
+            self._fallback_client = get_groq_client()
+        elif self.fallback_provider == "mistral":
+            from app.services.llm_provider import get_mistral_client
+
+            self._fallback_client = get_mistral_client()
+        logger.info(
+            f"{self.name}: Fallback configured — {self.fallback_provider}/{self.fallback_model}"
+        )
+
+    @abstractmethod
+    async def analyze(
+        self,
+        text: str,
+        context: Optional[Dict] = None,
+    ) -> Dict[str, Any]:
+        """
+        Analyze document text and return structured results.
+
+        Args:
+            text: Document text content (full text or representative sample)
+            context: Additional context (e.g., document category from classifier)
+
+        Returns:
+            Dictionary with agent-specific analysis results.
+            All results MUST include a 'confidence' key (0.0–1.0).
+        """
+
+    @property
+    @abstractmethod
+    def system_prompt(self) -> str:
+        """System prompt defining the agent's role and capabilities."""
+
+    # ── Generation with retry + fallback ───────────────────────────────────────
+
+    async def _call_openai_compat(self, client, model: str, prompt: str) -> str:
+        """Call an OpenAI-compatible provider and return text response."""
+        response = await client.chat.completions.create(
+            model=model,
+            messages=[
+                {"role": "system", "content": self.system_prompt},
+                {"role": "user", "content": prompt},
+            ],
+            response_format={"type": "json_object"},
+            temperature=0.1,
+        )
+        return response.choices[0].message.content or ""
+
+    async def _generate_json(self, prompt: str) -> Dict[str, Any]:
+        """
+        Generate structured JSON output with retry and automatic provider fallback.
+
+        Primary provider is tried first. On 429/rate-limit errors, if a fallback
+        is configured (e.g. Cerebras when Groq is rate-limited), the request
+        is retried on the fallback provider before raising QuotaExceededError.
+        """
+        import json
+
+        last_error: Optional[Exception] = None
+        for attempt in range(1, _MAX_RETRIES + 1):
+            try:
+                full_prompt = f"{self.system_prompt}\n\n{prompt}"
+
+                if self.provider == "gemini":
+                    response = await asyncio.to_thread(
+                        self.model.generate_content,
+                        full_prompt,
+                    )
+                    text_response = getattr(response, "text", "")
+                elif self.provider in ["groq", "mistral", "cerebras"]:
+                    text_response = await self._call_openai_compat(
+                        self.client, self.model_name, prompt
+                    )
+
+                try:
+                    return json.loads(text_response)
+                except (json.JSONDecodeError, AttributeError) as parse_err:
+                    logger.warning(
+                        f"{self.name} attempt {attempt}: JSON parse failed — {parse_err}. "
+                        f"Raw response: {text_response[:200]}"
+                    )
+                    last_error = parse_err
+                    delay = _RETRY_BASE_DELAY * (2 ** (attempt - 1))
+                    await asyncio.sleep(delay)
+                    continue
+
+            except Exception as e:
+                last_error = e
+                err_str = str(e)
+
+                is_quota = (
+                    "429" in err_str
+                    or "rate_limit" in err_str.lower()
+                    or "quota" in err_str.lower()
+                    or "RESOURCE_EXHAUSTED" in err_str
+                )
+
+                if is_quota:
+                    # Try fallback provider if available and not already using it
+                    if (
+                        self._fallback_client
+                        and self.fallback_model
+                        and not self._using_fallback
+                    ):
+                        logger.warning(
+                            f"{self.name}: Primary provider rate-limited. "
+                            f"Falling back to {self.fallback_provider}/{self.fallback_model}"
+                        )
+                        try:
+                            text_response = await self._call_openai_compat(
+                                self._fallback_client, self.fallback_model, prompt
+                            )
+                            result = json.loads(text_response)
+                            self._using_fallback = True
+                            return result
+                        except Exception as fb_err:
+                            logger.error(f"{self.name}: Fallback also failed: {fb_err}")
+                            # Fall through to raise QuotaExceededError
+
+                    # Parse the suggested wait time from the error body
+                    suggested_delay = _parse_retry_delay(err_str)
+
+                    if attempt < _MAX_RETRIES and suggested_delay is not None:
+                        logger.warning(
+                            f"{self.name}: Quota/rate-limit hit (attempt {attempt}/{_MAX_RETRIES}). "
+                            f"Waiting {suggested_delay}s..."
+                        )
+                        await asyncio.sleep(suggested_delay)
+                        continue
+
+                    logger.error(f"{self.name}: All providers exhausted — {e}")
+                    raise QuotaExceededError(
+                        f"Rate limit exceeded for {self.name}. "
+                        "Please try again later."
+                    ) from e
+
+                # Transient non-quota error — exponential backoff
+                delay = _RETRY_BASE_DELAY * (2 ** (attempt - 1))
+                logger.warning(
+                    f"{self.name} attempt {attempt}/{_MAX_RETRIES} failed: {e}. "
+                    f"Retrying in {delay}s..."
+                )
+                await asyncio.sleep(delay)
+
+        logger.error(f"{self.name} failed after {_MAX_RETRIES} attempts: {last_error}")
+        return {}
+
+    # ── Text helpers ───────────────────────────────────────────────────────────
+
+    def _prepare_text(self, text: str, max_chars: int = 15000) -> str:
+        """
+        Prepare text for agent analysis.
+
+        Instead of hard-truncating to 3000 chars (old behavior), we use up to
+        15000 chars (≈10 pages) to capture much more document content.
+        Long documents get the first 12000 chars + last 3000 chars to include
+        both the opening context and the conclusion/summary sections.
+        """
+        if len(text) <= max_chars:
+            return text
+
+        head = text[:12000]
+        tail = text[-3000:]
+        return head + "\n\n[... middle of document omitted for analysis ...]\n\n" + tail
+
+    # ── Kept for backward compatibility ───────────────────────────────────────
+
+    def _parse_json(self, text: str) -> Dict[str, Any]:
+        """Legacy JSON parser — kept for any subclass that still needs it."""
+        import json
+        import re
+
+        text = text.strip()
+        if text.startswith("```"):
+            lines = text.split("\n")
+            text = "\n".join(line for line in lines if not line.startswith("```"))
+        try:
+            return json.loads(text)
+        except json.JSONDecodeError:
+            match = re.search(r"\{.*\}", text, re.DOTALL)
+            if match:
+                try:
+                    return json.loads(match.group())
+                except json.JSONDecodeError:
+                    pass
+        return {}

app/agents/classifier.py

@@ -0,0 +1,107 @@
+"""
+Classifier Agent
+Document classification for mining industry categories
+"""
+
+import asyncio
+import logging
+from typing import Any, Dict, Optional
+
+from app.agents.base import BaseAgent
+from app.models.document import DocumentCategory
+
+logger = logging.getLogger(__name__)
+
+
+class ClassifierAgent(BaseAgent):
+    """
+    Document Classification Agent.
+
+    Categorizes mining documents into predefined categories:
+    - Safety protocols
+    - Equipment manuals
+    - Regulatory documents
+    - Incident reports
+    - Geological reports
+    - Environmental reports
+    - Training materials
+    - Permits
+    - Maintenance logs
+    """
+
+    def __init__(self):
+        super().__init__(model_name="llama-3.3-70b-versatile", provider="groq")
+
+    @property
+    def system_prompt(self) -> str:
+        return """You are a document classification agent specialized in the mining industry.
+
+Your task is to analyze documents and classify them into the appropriate category based on their content, structure, and purpose.
+
+Categories:
+1. safety_protocol - Safety procedures, guidelines, emergency protocols
+2. equipment_manual - Equipment operation guides, maintenance manuals
+3. regulatory - MSHA, OSHA, EPA regulations, compliance documents
+4. incident_report - Accident reports, incident investigations, near-miss reports
+5. geological - Drill logs, assay reports, geological surveys, core samples
+6. environmental - Environmental impact assessments, monitoring reports
+7. training - Training materials, certifications, competency assessments
+8. permit - Mining permits, licenses, applications
+9. maintenance - Maintenance schedules, repair logs, equipment inspections
+10. other - Documents that don't fit other categories
+
+Consider:
+- Document structure and formatting
+- Key terminology and language used
+- Purpose and intended audience
+- Regulatory references
+"""
+
+    async def analyze(
+        self, text: str, context: Optional[Dict] = None
+    ) -> Dict[str, Any]:
+        """
+        Classify document into mining category.
+
+        Returns:
+            {
+                "category": str (DocumentCategory value),
+                "subcategory": str,
+                "confidence": float (0-1),
+                "reasoning": str
+            }
+        """
+        prompt = f"""Analyze this mining document and classify it.
+
+Document content ({len(text)} chars total, showing up to 15000):
+{self._prepare_text(text)}
+
+Respond with a JSON object:
+{{
+  "category": "<one of: safety_protocol|equipment_manual|regulatory|incident_report|geological|environmental|training|permit|maintenance|other>",
+  "subcategory": "<more specific type if applicable, else null>",
+  "confidence": <0.0-1.0>,
+  "reasoning": "<brief explanation of classification decision>"
+}}
+"""
+        result = await self._generate_json(prompt)
+
+        category_str = (result.get("category") or "other").lower().strip()
+        category_map = {
+            "safety_protocol": DocumentCategory.SAFETY_PROTOCOL,
+            "equipment_manual": DocumentCategory.EQUIPMENT_MANUAL,
+            "regulatory": DocumentCategory.REGULATORY,
+            "incident_report": DocumentCategory.INCIDENT_REPORT,
+            "geological": DocumentCategory.GEOLOGICAL,
+            "environmental": DocumentCategory.ENVIRONMENTAL,
+            "training": DocumentCategory.TRAINING,
+            "permit": DocumentCategory.PERMIT,
+            "maintenance": DocumentCategory.MAINTENANCE,
+        }
+
+        return {
+            "category": category_map.get(category_str, DocumentCategory.OTHER).value,
+            "subcategory": result.get("subcategory"),
+            "confidence": float(result.get("confidence") or 0.5),
+            "reasoning": result.get("reasoning", ""),
+        }

app/agents/compliance_auditor.py

@@ -0,0 +1,129 @@
+"""
+Compliance Auditor Agent
+Cross-references a regulation clause against operational document evidence
+to determine compliance status (compliant / gap / missing).
+
+Uses Gemini for the nuanced cross-referencing task that requires reasoning
+across multiple evidence chunks.
+"""
+
+import logging
+from typing import Any, Dict, List, Optional
+
+from app.agents.base import BaseAgent
+
+logger = logging.getLogger(__name__)
+
+
+class ComplianceAuditorAgent(BaseAgent):
+    """
+    Regulatory Compliance Auditor Agent.
+
+    Takes a single regulation clause and a set of evidence chunks from
+    operational documents, then assesses whether the operational documents
+    adequately address the clause requirements.
+    """
+
+    def __init__(self):
+        super().__init__(model_name="llama-3.3-70b-versatile", provider="groq")
+
+    @property
+    def system_prompt(self) -> str:
+        return """You are a regulatory compliance auditor specializing in the mining industry.
+
+Your expertise includes:
+- MSHA (Mine Safety and Health Administration) regulations (30 CFR)
+- OSHA safety standards (29 CFR 1910, 1926)
+- DGMS (Directorate General of Mines Safety) regulations
+- EPA environmental regulations for mining operations
+- State-level mining regulations and permits
+
+Your task: Given a REGULATION CLAUSE and EVIDENCE CHUNKS from operational documents,
+assess whether the operational documents adequately address the clause requirements.
+
+Assessment statuses:
+- "compliant": The operational documents clearly address the regulation clause requirements
+- "gap": The operational documents partially address the clause but have gaps or deficiencies
+- "missing": The operational documents do not address this clause at all, or the evidence is insufficient
+
+Be strict but fair. If the evidence is thin but directionally correct, mark as "gap" not "missing".
+Only mark "compliant" if the evidence clearly and adequately addresses the clause.
+
+Always cite specific evidence in your assessment. If no evidence is provided, mark as "missing".
+"""
+
+    async def analyze(
+        self,
+        text: str,
+        context: Optional[Dict] = None,
+    ) -> Dict[str, Any]:
+        """
+        Assess compliance for a single regulation clause.
+
+        Args:
+            text: The regulation clause text
+            context: Must contain:
+                - evidence_chunks: List of dicts with chunk_text, document_title,
+                  page_numbers, relevance_score
+                - clause_section: Section title from the regulation document
+
+        Returns:
+            {
+                "status": "compliant" | "gap" | "missing",
+                "assessment": str,
+                "confidence": float (0.0-1.0),
+                "recommendations": list[str],
+            }
+        """
+        evidence_chunks: List[Dict] = (
+            context.get("evidence_chunks", []) if context else []
+        )
+        clause_section = context.get("clause_section", "") if context else ""
+
+        # Format evidence for the prompt
+        if evidence_chunks:
+            evidence_text = "\n\n".join(
+                f"[Evidence {i+1}] From '{chunk.get('document_title', 'Unknown')}'"
+                f" (Pages {chunk.get('page_numbers', ['?'])}, "
+                f"Relevance: {chunk.get('relevance_score', 0):.0%}):\n"
+                f"{chunk.get('chunk_text', '')}"
+                for i, chunk in enumerate(evidence_chunks)
+            )
+        else:
+            evidence_text = "No relevant evidence found in operational documents."
+
+        section_hint = (
+            f"\nRegulation Section: {clause_section}" if clause_section else ""
+        )
+
+        prompt = (
+            "Assess compliance for the following regulation clause against "
+            "the provided operational document evidence.\n\n"
+            f"REGULATION CLAUSE{text}:\n{text}\n\n"
+            f"{section_hint}\n\n"
+            f"EVIDENCE FROM OPERATIONAL DOCUMENTS:\n{evidence_text}\n\n"
+            "Respond with a JSON object:\n"
+            "{\n"
+            '  "status": "<compliant|gap|missing>",\n'
+            '  "assessment": "<2-4 sentence explanation of compliance status, '
+            'citing specific evidence where available>",\n'
+            '  "confidence": <0.0-1.0>,\n'
+            '  "recommendations": ["<specific actionable recommendation to close gaps>"]\n'
+            "}\n"
+        )
+
+        result = await self._generate_json(prompt)
+
+        status = (result.get("status") or "missing").lower()
+        if status not in ("compliant", "gap", "missing"):
+            status = "missing"
+
+        return {
+            "status": status,
+            "assessment": result.get(
+                "assessment",
+                "Assessment could not be generated.",
+            ),
+            "confidence": float(result.get("confidence") or 0.5),
+            "recommendations": result.get("recommendations", []),
+        }

app/agents/entity_extractor.py

@@ -0,0 +1,130 @@
+"""
+Entity Extractor Agent
+Mining-specific Named Entity Recognition
+"""
+
+import logging
+from typing import Any, Dict, List, Optional
+
+from app.agents.base import BaseAgent
+
+logger = logging.getLogger(__name__)
+
+
+class EntityExtractorAgent(BaseAgent):
+    """
+    Named Entity Recognition Agent for Mining Documents.
+
+    Extracts mining-specific entities:
+    - Equipment names and models
+    - Chemical compounds and gases
+    - Mine locations and sections
+    - Personnel and roles
+    - Dates and schedules
+    - Regulatory references
+    """
+
+    def __init__(self):
+        # Cerebras: 1M tokens/day free, 2600+ TPS, 60K TPM — better for high-volume extraction than Groq
+        super().__init__(model_name="gpt-oss-120b", provider="cerebras")
+
+    @property
+    def system_prompt(self) -> str:
+        return """You are a named entity extraction agent specialized in mining documents.
+
+Extract the following entity types:
+
+1. EQUIPMENT
+   - Mining machinery (excavators, haul trucks, drills)
+   - Brand names and models (Caterpillar D11, Komatsu PC8000)
+   - Equipment IDs and serial numbers
+   - Tools and instruments
+
+2. CHEMICALS
+   - Gases (methane, CO, H2S, oxygen)
+   - Minerals and ores
+   - Explosives and blasting agents
+   - Dust types (coal dust, silica)
+   - Hazardous substances
+
+3. LOCATIONS
+   - Mine names
+   - Sections and portals
+   - Underground levels
+   - Surface areas
+   - Geographic coordinates
+
+4. PERSONNEL
+   - Names (anonymize if needed)
+   - Roles (Safety Officer, Foreman, Engineer)
+   - Departments and teams
+   - Certifications
+
+5. DATES
+   - Specific dates
+   - Deadlines
+   - Scheduled events
+   - Time periods
+
+6. REGULATIONS
+   - MSHA regulations (30 CFR citations)
+   - OSHA standards
+   - EPA requirements
+   - State regulations
+   - Company policies
+
+Be precise and avoid duplicates. Extract exactly as written in the document.
+"""
+
+    async def analyze(
+        self, text: str, context: Optional[Dict] = None
+    ) -> Dict[str, Any]:
+        """
+        Extract named entities from document.
+
+        Returns dict with keys: equipment, chemicals, locations, personnel,
+        dates, regulations (all List[str]), entity_count (int)
+        """
+        prompt = f"""Extract all mining-specific named entities from this document.
+
+Document content ({len(text)} chars total, showing up to 15000):
+{self._prepare_text(text)}
+
+Respond with a JSON object:
+{{
+  "equipment": ["<equipment name or model>"],
+  "chemicals": ["<chemical compound, gas, or mineral>"],
+  "locations": ["<mine name, section, or area>"],
+  "personnel": ["<role or name>"],
+  "dates": ["<date or time period>"],
+  "regulations": ["<e.g. 30 CFR 75.400, OSHA 1910.134>"]
+}}
+
+Notes:
+- List each unique entity only once
+- Use exact text from document
+- For personnel, prefer roles over names for privacy
+- Include regulation citations in standard format
+"""
+        result = await self._generate_json(prompt)
+
+        entities = {
+            "equipment": self._deduplicate(result.get("equipment", [])),
+            "chemicals": self._deduplicate(result.get("chemicals", [])),
+            "locations": self._deduplicate(result.get("locations", [])),
+            "personnel": self._deduplicate(result.get("personnel", [])),
+            "dates": self._deduplicate(result.get("dates", [])),
+            "regulations": self._deduplicate(result.get("regulations", [])),
+        }
+        entities["entity_count"] = sum(len(v) for v in entities.values())
+        return entities
+
+    def _deduplicate(self, items: List[str]) -> List[str]:
+        """Remove duplicates while preserving order"""
+        seen = set()
+        result = []
+        for item in items:
+            if item and item.lower() not in seen:
+                seen.add(item.lower())
+                result.append(item)
+        return result

app/agents/orchestrator.py

@@ -0,0 +1,252 @@
+"""
+Agent Orchestrator
+Coordinates multi-agent document analysis pipeline.
+
+Improvements over v1:
+  - Accepts pages parameter from extractor for future per-page analysis
+  - Runs safety/entity/summary with small delay between calls to avoid simultaneous quota hits
+  - Adds per-agent timing metrics
+  - All agents use JSON mode (no regex) + exponential backoff retry
+  - Surfaces QuotaExceededError instead of silently returning empty data
+"""
+
+import asyncio
+import logging
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from app.agents.base import QuotaExceededError
+from app.agents.classifier import ClassifierAgent
+from app.agents.entity_extractor import EntityExtractorAgent
+from app.agents.safety_analyzer import SafetyAnalyzerAgent
+from app.agents.summarizer import SummarizerAgent
+
+logger = logging.getLogger(__name__)
+
+
+import functools
+
+
+def exponential_backoff_wrapper(max_retries: int = 3, base_delay: float = 2.0):
+    """Exponential Backoff Utility Wrapper for AI Agent execution pipelines."""
+
+    def decorator(func):
+        @functools.wraps(func)
+        async def wrapper(*args, **kwargs):
+            last_error = None
+            for attempt in range(1, max_retries + 1):
+                try:
+                    return await func(*args, **kwargs)
+                except Exception as e:
+                    err_str = str(e).lower()
+                    is_rate_limit = any(
+                        term in err_str
+                        for term in [
+                            "429",
+                            "quota",
+                            "timeout",
+                            "rate_limit",
+                            "resource_exhausted",
+                            "too many requests",
+                        ]
+                    )
+                    if is_rate_limit:
+                        last_error = e
+                        if attempt < max_retries:
+                            delay = base_delay * (2 ** (attempt - 1))
+                            logger.warning(
+                                f"Audit Trail: {func.__name__} attempt {attempt} failed (429/Timeout). Retrying in {delay}s..."
+                            )
+                            await asyncio.sleep(delay)
+                        else:
+                            logger.error(
+                                f"Audit Trail: {func.__name__} exhausted {max_retries} retries."
+                            )
+                            raise QuotaExceededError(
+                                f"Rate limit/timeout exceeded in {func.__name__}"
+                            ) from e
+                    else:
+                        raise
+            if last_error is not None:
+                raise last_error
+            raise RuntimeError(f"Task {func.__name__} failed with no attempts made")
+
+        return wrapper
+
+    return decorator
+
+
+class AgentOrchestrator:
+    """
+    Multi-Agent Orchestrator for Document Intelligence.
+
+    Execution order:
+    1. ClassifierAgent   — runs first (result feeds category context to others)
+    2. SafetyAnalyzerAgent  ┐
+    3. EntityExtractorAgent ├── run in parallel after classification (Multi-Provider)
+    4. SummarizerAgent      ┘
+    """
+
+    def __init__(self):
+        self.classifier = ClassifierAgent()
+        self.safety_analyzer = SafetyAnalyzerAgent()
+        self.entity_extractor = EntityExtractorAgent()
+        self.summarizer = SummarizerAgent()
+
+    async def analyze_document(
+        self,
+        text: str,
+        pages: Optional[List] = None,
+    ) -> Dict[str, Any]:
+        """
+        Run full multi-agent analysis pipeline on document.
+        """
+        start_time = datetime.now(timezone.utc)
+        logger.info("Starting multi-agent document analysis")
+
+        agent_timings: Dict[str, int] = {}
+
+        try:
+            # ── Step 1: Classification (feeds category context to other agents) ──
+            t0 = datetime.now(timezone.utc)
+            logger.info("Running ClassifierAgent...")
+
+            @exponential_backoff_wrapper()
+            async def _run_classifier():
+                return await self.classifier.analyze(text)
+
+            classification = await _run_classifier()
+            agent_timings["classifier_ms"] = int(
+                (datetime.now(timezone.utc) - t0).total_seconds() * 1000
+            )
+
+            category = classification.get("category", "other")
+            context = {"category": category}
+
+            # ── Step 2: Parallel agents using category context ─────────────────
+            logger.info(
+                "Running parallel agents across multiple providers (Safety, Entity, Summary)..."
+            )
+            t1 = datetime.now(timezone.utc)
+
+            @exponential_backoff_wrapper()
+            async def _run_safety():
+                # Only run safety analysis on relevant document categories
+                non_safety_categories = [
+                    "regulatory",
+                    "geological",
+                    "environmental",
+                    "permit",
+                    "other",
+                ]
+                if category in non_safety_categories:
+                    logger.info(
+                        f"Routing: Document is {category}. Bypassing Safety Analyzer."
+                    )
+                    return {
+                        "status": "not_applicable",
+                        "score": None,
+                        "hazards": [],
+                        "recommendations": [
+                            f"Safety analysis bypassed for {category} document"
+                        ],
+                    }
+                return await self.safety_analyzer.analyze(text, context)
+
+            @exponential_backoff_wrapper()
+            async def _run_entities():
+                return await self.entity_extractor.analyze(text, context)
+
+            @exponential_backoff_wrapper()
+            async def _run_summary():
+                return await self.summarizer.analyze(text, context)
+
+            safety, entities, summary = await asyncio.gather(
+                _run_safety(),
+                _run_entities(),
+                _run_summary(),
+                return_exceptions=True,
+            )
+
+            agent_timings["parallel_agents_ms"] = int(
+                (datetime.now(timezone.utc) - t1).total_seconds() * 1000
+            )
+
+            # Handle per-agent exceptions gracefully
+            def _quota_error_result(agent_name: str, exc: Exception) -> dict:
+                is_quota = isinstance(exc, QuotaExceededError)
+                logger.error(f"{agent_name} failed: {exc}")
+                return {
+                    "error": str(exc),
+                    "quota_exceeded": is_quota,
+                    "status": "quota_exceeded" if is_quota else "error",
+                }
+
+            if isinstance(safety, Exception):
+                safety = {
+                    **_quota_error_result("SafetyAnalyzerAgent", safety),
+                    "score": None,
+                    "hazards": [],
+                    "recommendations": [],
+                }
+
+            if isinstance(entities, Exception):
+                entities = {
+                    **_quota_error_result("EntityExtractorAgent", entities),
+                    "equipment": [],
+                    "chemicals": [],
+                    "locations": [],
+                    "personnel": [],
+                    "dates": [],
+                    "regulations": [],
+                }
+
+            if isinstance(summary, Exception):
+                summary = {
+                    **_quota_error_result("SummarizerAgent", summary),
+                    "summary": "Analysis failed — Gemini API quota exceeded. Please try again later.",
+                    "key_points": [],
+                }
+
+            total_ms = int(
+                (datetime.now(timezone.utc) - start_time).total_seconds() * 1000
+            )
+            logger.info(f"Multi-agent analysis completed in {total_ms}ms")
+
+            return {
+                "classification": classification,
+                "safety": safety,
+                "entities": entities,
+                "summary": summary,
+                "metadata": {
+                    "processing_time_ms": total_ms,
+                    "agent_timings": agent_timings,
+                    "agents_used": [
+                        "classifier",
+                        "safety_analyzer",
+                        "entity_extractor",
+                        "summarizer",
+                    ],
+                    "analyzed_at": datetime.now(timezone.utc).isoformat(),
+                },
+            }
+
+        except QuotaExceededError:
+            # Re-raise quota errors so document_service.py can handle them
+            # with its dedicated QuotaExceededError handler (partial save, not FAILED).
+            raise
+
+        except Exception as e:
+            logger.error(f"Orchestrator failed: {e}", exc_info=True)
+            return {
+                "error": str(e),
+                "metadata": {"failed": True, "error_message": str(e)},
+            }
+
+    async def analyze_for_safety_only(self, text: str) -> Dict[str, Any]:
+        """Quick safety-only analysis for real-time checks."""
+        return await self.safety_analyzer.analyze(text)
+
+    async def classify_only(self, text: str) -> Dict[str, Any]:
+        """Quick classification only."""
+        return await self.classifier.analyze(text)

app/agents/safety_analyzer.py

@@ -0,0 +1,140 @@
+"""
+Safety Analyzer Agent
+Mining safety compliance and hazard detection.
+
+Uses JSON mode (response_mime_type=application/json) + retry for reliable output.
+Processes up to 15000 chars of document content (vs 5000 in v1).
+"""
+
+import logging
+from typing import Any, Dict, List, Optional
+
+from app.agents.base import BaseAgent
+
+logger = logging.getLogger(__name__)
+
+
+class SafetyAnalyzerAgent(BaseAgent):
+    """
+    Safety Compliance Analysis Agent.
+
+    Analyzes mining documents for:
+    - MSHA/OSHA/DGMS compliance issues
+    - Safety hazards and risks
+    - Missing safety requirements
+    - Recommendations for improvement
+    """
+
+    def __init__(self):
+        # magistral-small-latest is a reasoning model, not a standard chat model.
+        # It uses chain-of-thought internally for compliance scoring.
+        super().__init__(model_name="magistral-small-latest", provider="mistral")
+
+    @property
+    def system_prompt(self) -> str:
+        return """You are a mining safety compliance analysis agent.
+
+Your expertise includes:
+- MSHA (Mine Safety and Health Administration) regulations
+- OSHA safety standards
+- DGMS (Directorate General of Mines Safety) regulations
+- Underground and surface mining safety
+- Equipment safety requirements
+- Emergency response protocols
+- Ventilation and air quality standards
+- Ground control and stability
+- Electrical safety in mining
+- Personal protective equipment (PPE)
+- Hazardous materials handling
+
+Analyze documents for:
+1. Compliance with regulations
+2. Potential safety hazards
+3. Missing safety procedures
+4. Risk factors
+5. Areas needing improvement
+
+Be thorough but practical. Focus on actionable findings.
+"""
+
+    async def analyze(
+        self, text: str, context: Optional[Dict] = None
+    ) -> Dict[str, Any]:
+        """
+        Analyze document for safety compliance and hazards.
+
+        Returns:
+            {
+                "score": float (0-100),
+                "status": str ("compliant"|"warning"|"violation"),
+                "hazards": List[Dict],
+                "recommendations": List[str],
+                "compliance_details": Dict,
+                "confidence": float (0-1),
+                "reasoning": Dict (explainability layer)
+            }
+        """
+        category = context.get("category", "unknown") if context else "unknown"
+
+        prompt = (
+            "Analyze this mining document for safety compliance and hazards.\n\n"
+            f"Document Type: {category}\n"
+            f"Document content ({len(text)} chars total, showing up to 15000):\n"
+            f"{self._prepare_text(text)}\n\n"
+            "Evaluate and respond with a JSON object:\n"
+            "{\n"
+            '  "score": <0-100 overall safety score>,\n'
+            '  "status": "<compliant|warning|violation>",\n'
+            '  "confidence": <0.0-1.0>,\n'
+            '  "hazards": [\n'
+            "    {\n"
+            '      "type": "<hazard category>",\n'
+            '      "severity": "<low|medium|high|critical>",\n'
+            '      "description": "<specific hazard details>",\n'
+            '      "regulation": "<relevant MSHA/OSHA/DGMS regulation if applicable>"\n'
+            "    }\n"
+            "  ],\n"
+            '  "recommendations": ["<specific actionable recommendation>"],\n'
+            '  "compliance_details": {\n'
+            '    "msha_compliant": <true|false>,\n'
+            '    "osha_compliant": <true|false>,\n'
+            '    "dgms_compliant": <true|false>,\n'
+            '    "missing_elements": ["<missing safety element>"]\n'
+            "  },\n"
+            '  "reasoning": {\n'
+            '    "score_explanation": "<1-2 sentence explanation of why this specific score was assigned>",\n'
+            '    "positive_factors": ["<safety elements that contributed positively to the score>"],\n'
+            '    "negative_factors": ["<safety gaps or issues that reduced the score>"],\n'
+            '    "evidence": [\n'
+            '      {"text": "<exact quote or paraphrase from document>", "factor": "<positive|negative>", "impact": "<explanation>"}\n'
+            "    ]\n"
+            "  },\n"
+            '  "summary": "<brief safety assessment summary>"\n'
+            "}\n\n"
+            "Scoring guide:\n"
+            "  80-100: Compliant, minimal concerns\n"
+            "  60-79:  Generally compliant with warnings\n"
+            "  40-59:  Significant concerns, needs attention\n"
+            "  0-39:   Critical issues or violations present\n"
+        )
+
+        result = await self._generate_json(prompt)
+
+        return {
+            "score": float(result.get("score") or 50),
+            "status": (result.get("status") or "pending").lower(),
+            "confidence": float(result.get("confidence") or 0.5),
+            "hazards": result.get("hazards", []),
+            "recommendations": result.get("recommendations", []),
+            "compliance_details": result.get("compliance_details", {}),
+            "reasoning": result.get(
+                "reasoning",
+                {
+                    "score_explanation": "",
+                    "positive_factors": [],
+                    "negative_factors": [],
+                    "evidence": [],
+                },
+            ),
+            "summary": result.get("summary", ""),
+        }

app/agents/summarizer.py

@@ -0,0 +1,100 @@
+"""
+Summarizer Agent
+Document summarization for mining documents
+"""
+
+import logging
+from typing import Any, Dict, List, Optional
+
+from app.agents.base import BaseAgent
+
+logger = logging.getLogger(__name__)
+
+
+class SummarizerAgent(BaseAgent):
+    """
+    Document Summarization Agent.
+
+    Creates concise, actionable summaries of mining documents:
+    - Executive summary
+    - Key points extraction
+    - Action items identification
+    """
+
+    def __init__(self):
+        # Shift to Cerebras (gpt-oss-120b) to bypass Gemini API rate limits and avoid Groq parallel execution rate limits
+        super().__init__(model_name="gpt-oss-120b", provider="cerebras")
+
+    @property
+    def system_prompt(self) -> str:
+        return """You are a document summarization agent for the mining industry.
+
+Create clear, actionable summaries that:
+- Highlight critical information first
+- Focus on safety-relevant content
+- Identify action items and deadlines
+- Use plain language accessible to all mining personnel
+- Preserve technical accuracy
+
+Summary structure:
+1. Executive Summary: 2-3 paragraphs covering main purpose and findings
+2. Key Points: 5-7 bullet points of most important information
+3. Action Items: Any required actions or follow-ups (if applicable)
+
+Prioritize:
+- Safety information
+- Compliance requirements
+- Deadlines and schedules
+- Equipment status
+- Personnel responsibilities
+"""
+
+    async def analyze(
+        self, text: str, context: Optional[Dict] = None
+    ) -> Dict[str, Any]:
+        """
+        Generate document summary and key points.
+
+        Returns:
+            {
+                "summary": str,
+                "key_points": List[str],
+                "action_items": List[str],
+                "document_purpose": str,
+                "confidence": float
+            }
+        """
+        category = context.get("category", "unknown") if context else "unknown"
+
+        prompt = f"""Summarize this mining document clearly and concisely.
+
+Document Type: {category}
+Document content ({len(text)} chars total, showing up to 15000):
+{self._prepare_text(text)}
+
+Respond with a JSON object:
+{{
+  "summary": "<2-3 paragraph executive summary covering main purpose and findings>",
+  "key_points": [
+    "<Key point 1>",
+    "<Key point 2>",
+    "<Key point 3>",
+    "<Key point 4>",
+    "<Key point 5>"
+  ],
+  "action_items": ["<required action or follow-up if any>"],
+  "document_purpose": "<one sentence describing the document's main purpose>",
+  "confidence": <0.0-1.0>
+}}
+"""
+        result = await self._generate_json(prompt)
+
+        summary = result.get("summary") or "Summary not available."
+        return {
+            "summary": summary,
+            "key_points": result.get("key_points", []),
+            "action_items": result.get("action_items", []),
+            "document_purpose": result.get("document_purpose", ""),
+            "confidence": float(result.get("confidence") or 0.7),
+            "word_count": len(summary.split()),
+        }

app/api/__init__.py

@@ -0,0 +1,4 @@
+"""
+API Module
+REST API endpoints organized by domain
+"""

app/api/deps.py

@@ -0,0 +1,112 @@
+"""
+API Dependencies
+Shared dependencies for FastAPI endpoints
+"""
+
+import hashlib
+import logging
+from typing import Optional
+
+from fastapi import Depends, Header, Request
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+from sqlalchemy.orm import Session
+
+from app.core.exceptions import AuthenticationError
+from app.core.security import extract_user_email, extract_user_id, verify_jwt_token
+from app.db.session import SessionLocal, get_db
+from app.models.audit import AuditAction, create_audit_log
+from app.models.user import User
+
+logger = logging.getLogger(__name__)
+
+
+def _anonymize_user_id(user_id: str) -> str:
+    """Create a truncated hash of user_id for safe logging."""
+    return hashlib.sha256(user_id.encode()).hexdigest()[:12]
+
+
+# Security scheme
+security = HTTPBearer()
+
+
+async def get_current_user_id(
+    credentials: HTTPAuthorizationCredentials = Depends(security),
+) -> str:
+    """
+    Dependency to extract and verify user ID from JWT.
+    Returns Clerk user ID string.
+    """
+    token = credentials.credentials
+    payload = await verify_jwt_token(token)
+    return extract_user_id(payload)
+
+
+async def get_current_user(
+    user_id: str = Depends(get_current_user_id), db: Session = Depends(get_db)
+) -> User:
+    """
+    Dependency to get current user model from database.
+    Creates user record if it doesn't exist (first login).
+    """
+    user = db.query(User).filter(User.clerk_user_id == user_id).first()
+
+    if not user:
+        # Auto-create user on first access
+        user = User(clerk_user_id=user_id, is_active=True)
+        db.add(user)
+        db.commit()
+        db.refresh(user)
+        logger.info(f"Created new user: {_anonymize_user_id(user_id)}")
+
+    return user
+
+
+async def get_optional_user(
+    credentials: Optional[HTTPAuthorizationCredentials] = Depends(
+        HTTPBearer(auto_error=False)
+    ),
+    db: Session = Depends(get_db),
+) -> Optional[User]:
+    """
+    Dependency for endpoints that work with or without auth.
+    Returns User if authenticated, None otherwise.
+    """
+    if not credentials:
+        return None
+
+    try:
+        token = credentials.credentials
+        payload = await verify_jwt_token(token)
+        user_id = extract_user_id(payload)
+        return db.query(User).filter(User.clerk_user_id == user_id).first()
+    except Exception:
+        return None
+
+
+def get_client_ip(request: Request) -> str:
+    """Extract client IP address from request"""
+    forwarded = request.headers.get("X-Forwarded-For")
+    if forwarded:
+        return forwarded.split(",")[0].strip()
+    return request.client.host if request.client else "unknown"
+
+
+def get_user_agent(request: Request) -> str:
+    """Extract user agent from request"""
+    return request.headers.get("User-Agent", "unknown")
+
+
+async def audit_middleware(
+    request: Request, user_id: str = Depends(get_current_user_id)
+):
+    """
+    Middleware-like dependency to log API access.
+    Add to endpoints that need audit logging.
+    """
+    # This is called after auth, so we have user_id
+    # Actual logging happens in endpoint handlers
+    return {
+        "user_id": user_id,
+        "ip_address": get_client_ip(request),
+        "user_agent": get_user_agent(request),
+    }

app/api/v1/__init__.py

@@ -0,0 +1,8 @@
+"""
+API v1 Module
+Versioned API endpoints
+"""
+
+from app.api.v1.router import api_router
+
+__all__ = ["api_router"]

app/api/v1/analytics.py

@@ -0,0 +1,410 @@
+"""
+Analytics API Endpoints
+Dashboard statistics and mining intelligence metrics
+"""
+
+import logging
+from datetime import datetime, timedelta
+
+from fastapi import APIRouter, Depends, Query
+from sqlalchemy import case, func
+from sqlalchemy.orm import Session
+
+from app.api.deps import get_current_user_id
+from app.db.session import get_db
+from app.models.chat import ChatMessage, ChatSession
+from app.models.document import (
+    ComplianceStatus,
+    Document,
+    DocumentCategory,
+    DocumentStatus,
+)
+from app.schemas.analytics import (
+    CategoryCount,
+    DashboardStats,
+    DocumentAnalytics,
+    SafetyAnalytics,
+    SafetyDistribution,
+    StatusCount,
+)
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+import json
+
+import redis
+
+from app.config import settings
+
+# Optional Redis client
+try:
+    redis_client = redis.Redis.from_url(
+        settings.REDIS_URL, decode_responses=True, socket_timeout=1
+    )
+except Exception:
+    redis_client = None
+
+
+@router.get("/dashboard", response_model=DashboardStats)
+async def get_dashboard_stats(
+    user_id: str = Depends(get_current_user_id), db: Session = Depends(get_db)
+):
+    """
+    Get dashboard statistics for the current user.
+    Provides overview of documents, chats, and safety metrics.
+    """
+    cache_key = f"dashboard_stats:{user_id}"
+
+    if redis_client:
+        try:
+            cached_data = redis_client.get(cache_key)
+            if cached_data:
+                return DashboardStats.model_validate_json(cached_data)
+        except Exception as e:
+            logger.warning(f"Redis cache read error: {e}")
+
+    now = datetime.utcnow()
+    today_start = now.replace(hour=0, minute=0, second=0, microsecond=0)
+    week_start = today_start - timedelta(days=7)
+
+    # Document counts
+    doc_query = db.query(Document).filter(Document.user_id == user_id)
+
+    total_documents = doc_query.count()
+    processed_documents = doc_query.filter(
+        Document.status == DocumentStatus.COMPLETED
+    ).count()
+    pending_documents = doc_query.filter(
+        Document.status.in_(
+            [
+                DocumentStatus.PENDING,
+                DocumentStatus.PROCESSING,
+                DocumentStatus.ANALYZING,
+            ]
+        )
+    ).count()
+    failed_documents = doc_query.filter(
+        Document.status == DocumentStatus.FAILED
+    ).count()
+
+    # Today and week counts
+    docs_today = doc_query.filter(Document.created_at >= today_start).count()
+    docs_week = doc_query.filter(Document.created_at >= week_start).count()
+
+    # Chat counts
+    total_sessions = (
+        db.query(ChatSession).filter(ChatSession.user_id == user_id).count()
+    )
+    total_messages = (
+        db.query(ChatMessage)
+        .join(ChatSession)
+        .filter(ChatSession.user_id == user_id)
+        .count()
+    )
+
+    # Safety metrics
+    safety_stats = (
+        db.query(
+            func.avg(Document.safety_score).label("avg_score"),
+            func.count(case((Document.hazards_detected.isnot(None), 1))).label(
+                "with_hazards"
+            ),
+            func.count(
+                case((Document.compliance_status == ComplianceStatus.VIOLATION, 1))
+            ).label("violations"),
+            func.count(
+                case((Document.compliance_status == ComplianceStatus.WARNING, 1))
+            ).label("warnings"),
+        )
+        .filter(
+            Document.user_id == user_id, Document.status == DocumentStatus.COMPLETED
+        )
+        .first()
+    )
+
+    # Category breakdown
+    category_counts = (
+        db.query(Document.category, func.count(Document.id).label("count"))
+        .filter(Document.user_id == user_id, Document.category.isnot(None))
+        .group_by(Document.category)
+        .all()
+    )
+
+    total_categorized = sum(c.count for c in category_counts)
+    categories = [
+        CategoryCount(
+            category=cat.value if cat else "other",
+            count=count,
+            percentage=(
+                round(count / total_categorized * 100, 1)
+                if total_categorized > 0
+                else 0
+            ),
+        )
+        for cat, count in category_counts
+    ]
+
+    # Last activity
+    last_doc = doc_query.order_by(Document.created_at.desc()).first()
+    last_chat = (
+        db.query(ChatSession)
+        .filter(ChatSession.user_id == user_id)
+        .order_by(ChatSession.updated_at.desc())
+        .first()
+    )
+
+    stats = DashboardStats(
+        total_documents=total_documents,
+        processed_documents=processed_documents,
+        pending_documents=pending_documents,
+        failed_documents=failed_documents,
+        total_chat_sessions=total_sessions,
+        total_messages=total_messages,
+        average_safety_score=(
+            round(safety_stats.avg_score, 1) if safety_stats.avg_score else None
+        ),
+        documents_with_hazards=safety_stats.with_hazards or 0,
+        compliance_violations=safety_stats.violations or 0,
+        compliance_warnings=safety_stats.warnings or 0,
+        documents_processed_today=docs_today,
+        documents_processed_this_week=docs_week,
+        documents_by_category=categories,
+        last_upload_at=last_doc.created_at if last_doc else None,
+        last_chat_at=last_chat.updated_at if last_chat else None,
+    )
+
+    if redis_client:
+        try:
+            # Cache for 60 seconds
+            redis_client.setex(cache_key, 60, stats.model_dump_json())
+        except Exception as e:
+            logger.warning(f"Redis cache write error: {e}")
+
+    return stats
+
+
+@router.get("/documents", response_model=DocumentAnalytics)
+async def get_document_analytics(
+    days: int = Query(30, ge=1, le=365),
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """
+    Get detailed document analytics over time.
+    """
+    start_date = datetime.utcnow() - timedelta(days=days)
+
+    # Uploads by day
+    uploads = (
+        db.query(
+            func.date(Document.created_at).label("date"),
+            func.count(Document.id).label("count"),
+        )
+        .filter(Document.user_id == user_id, Document.created_at >= start_date)
+        .group_by(func.date(Document.created_at))
+        .order_by("date")
+        .all()
+    )
+
+    uploads_by_day = [{"date": str(d), "count": c} for d, c in uploads]
+
+    # Category distribution
+    categories = (
+        db.query(Document.category, func.count(Document.id).label("count"))
+        .filter(Document.user_id == user_id, Document.category.isnot(None))
+        .group_by(Document.category)
+        .all()
+    )
+
+    total = sum(c for _, c in categories)
+    by_category = [
+        CategoryCount(
+            category=cat.value if cat else "other",
+            count=count,
+            percentage=round(count / total * 100, 1) if total > 0 else 0,
+        )
+        for cat, count in categories
+    ]
+
+    # Status distribution
+    statuses = (
+        db.query(Document.status, func.count(Document.id).label("count"))
+        .filter(Document.user_id == user_id)
+        .group_by(Document.status)
+        .all()
+    )
+
+    by_status = [
+        StatusCount(status=s.value if s else "unknown", count=c) for s, c in statuses
+    ]
+
+    # File type distribution
+    file_types = (
+        db.query(Document.file_type, func.count(Document.id).label("count"))
+        .filter(Document.user_id == user_id)
+        .group_by(Document.file_type)
+        .all()
+    )
+
+    by_file_type = [{"type": ft, "count": c} for ft, c in file_types]
+
+    return DocumentAnalytics(
+        uploads_by_day=uploads_by_day,
+        by_category=by_category,
+        by_status=by_status,
+        by_file_type=by_file_type,
+    )
+
+
+@router.get("/safety", response_model=SafetyAnalytics)
+async def get_safety_analytics(
+    user_id: str = Depends(get_current_user_id), db: Session = Depends(get_db)
+):
+    """
+    Get safety compliance analytics.
+    """
+    # Base query for completed documents with safety scores
+    base = db.query(Document).filter(
+        Document.user_id == user_id,
+        Document.status == DocumentStatus.COMPLETED,
+        Document.safety_score.isnot(None),
+    )
+
+    # Score statistics
+    stats = (
+        db.query(
+            func.avg(Document.safety_score).label("avg"),
+            func.min(Document.safety_score).label("min"),
+            func.max(Document.safety_score).label("max"),
+        )
+        .filter(Document.user_id == user_id, Document.safety_score.isnot(None))
+        .first()
+    )
+
+    # Score distribution
+    ranges = [
+        ("0-25", 0, 25),
+        ("26-50", 26, 50),
+        ("51-75", 51, 75),
+        ("76-100", 76, 100),
+    ]
+
+    total_with_scores = base.count()
+    distribution = []
+
+    for label, low, high in ranges:
+        count = base.filter(
+            Document.safety_score >= low, Document.safety_score <= high
+        ).count()
+        distribution.append(
+            SafetyDistribution(
+                range=label,
+                count=count,
+                percentage=(
+                    round(count / total_with_scores * 100, 1)
+                    if total_with_scores > 0
+                    else 0
+                ),
+            )
+        )
+
+    # Compliance counts
+    compliant = (
+        db.query(Document)
+        .filter(
+            Document.user_id == user_id,
+            Document.compliance_status == ComplianceStatus.COMPLIANT,
+        )
+        .count()
+    )
+
+    warnings = (
+        db.query(Document)
+        .filter(
+            Document.user_id == user_id,
+            Document.compliance_status == ComplianceStatus.WARNING,
+        )
+        .count()
+    )
+
+    violations = (
+        db.query(Document)
+        .filter(
+            Document.user_id == user_id,
+            Document.compliance_status == ComplianceStatus.VIOLATION,
+        )
+        .count()
+    )
+
+    return SafetyAnalytics(
+        average_safety_score=round(stats.avg, 1) if stats.avg else 0,
+        min_safety_score=stats.min,
+        max_safety_score=stats.max,
+        score_distribution=distribution,
+        compliant_count=compliant,
+        warning_count=warnings,
+        violation_count=violations,
+    )
+
+
+@router.get("/violations")
+async def get_recent_violations(
+    limit: int = Query(20, ge=1, le=100),
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """
+    Get recent violations and warnings from analyzed documents.
+    Returns documents with detected hazards, ordered by severity and recency.
+    """
+    # Get documents with violations or warnings that have hazards
+    docs = (
+        db.query(Document)
+        .filter(
+            Document.user_id == user_id,
+            Document.status == DocumentStatus.COMPLETED,
+            Document.compliance_status.in_(
+                [ComplianceStatus.VIOLATION, ComplianceStatus.WARNING]
+            ),
+            Document.hazards_detected.isnot(None),
+        )
+        .order_by(Document.created_at.desc())
+        .limit(limit)
+        .all()
+    )
+
+    violations = []
+    for doc in docs:
+        hazards = doc.hazards_detected or []
+        if isinstance(hazards, list):
+            for h in hazards[:3]:  # Max 3 hazards per doc
+                if isinstance(h, dict):
+                    violations.append(
+                        {
+                            "document_id": str(doc.id),
+                            "document_title": doc.title,
+                            "file_name": doc.file_name,
+                            "hazard_type": h.get("type", "Unknown Hazard"),
+                            "severity": h.get("severity", "medium"),
+                            "description": h.get("description", ""),
+                            "regulation": h.get("regulation", ""),
+                            "detected_at": (
+                                doc.processed_at.isoformat()
+                                if doc.processed_at
+                                else doc.created_at.isoformat()
+                            ),
+                            "compliance_status": (
+                                doc.compliance_status.value
+                                if doc.compliance_status
+                                else "warning"
+                            ),
+                        }
+                    )
+
+    return {
+        "violations": violations,
+        "total": len(violations),
+    }

app/api/v1/chat.py

@@ -0,0 +1,331 @@
+"""
+Chat API Endpoints
+Chat sessions and AI conversations with RAG
+"""
+
+import logging
+import uuid
+from datetime import datetime
+from typing import List, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy.orm import Session
+
+from app.api.deps import get_current_user_id
+from app.core.exceptions import NotFoundError
+from app.db.session import get_db
+from app.models.audit import AuditAction, create_audit_log
+from app.models.chat import ChatMessage, ChatSession
+from app.schemas.chat import (
+    ChatMessageResponse,
+    ChatRequest,
+    ChatResponse,
+    ChatSessionCreate,
+    ChatSessionDetailResponse,
+    ChatSessionResponse,
+    ChatSessionUpdateRequest,
+)
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+@router.get("/sessions", response_model=List[ChatSessionResponse])
+async def list_chat_sessions(
+    limit: int = Query(50, ge=1, le=100),
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """
+    List user's chat sessions ordered by most recent.
+    """
+    sessions = (
+        db.query(ChatSession)
+        .filter(ChatSession.user_id == user_id)
+        .order_by(ChatSession.updated_at.desc())
+        .limit(limit)
+        .all()
+    )
+
+    result = []
+    for session in sessions:
+        last_msg = session.messages[-1] if session.messages else None
+        result.append(
+            ChatSessionResponse(
+                id=str(session.id),
+                title=session.title,
+                message_count=len(session.messages),
+                document_context=session.document_context or [],
+                created_at=session.created_at,
+                updated_at=session.updated_at,
+                last_message=last_msg.content[:100] if last_msg else None,
+                last_message_at=last_msg.created_at if last_msg else None,
+            )
+        )
+
+    return result
+
+
+@router.post("/sessions", response_model=ChatSessionResponse)
+async def create_chat_session(
+    request: ChatSessionCreate,
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """
+    Create a new chat session.
+    """
+    session = ChatSession(
+        user_id=user_id,
+        title=request.title or "New Chat",
+        document_context=request.document_ids or [],
+        system_prompt=request.system_prompt,
+    )
+
+    db.add(session)
+    db.commit()
+    db.refresh(session)
+
+    # Audit log
+    audit = create_audit_log(
+        action=AuditAction.CHAT_CREATE.value,
+        user_id=user_id,
+        resource_type="chat_session",
+        resource_id=str(session.id),
+    )
+    db.add(audit)
+    db.commit()
+
+    return ChatSessionResponse(
+        id=str(session.id),
+        title=session.title,
+        message_count=0,
+        document_context=session.document_context,
+        created_at=session.created_at,
+        updated_at=session.updated_at,
+    )
+
+
+@router.get("/sessions/{session_id}", response_model=ChatSessionDetailResponse)
+async def get_chat_session(
+    session_id: uuid.UUID,
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """
+    Get chat session with all messages.
+    """
+    session = (
+        db.query(ChatSession)
+        .filter(ChatSession.id == session_id, ChatSession.user_id == user_id)
+        .first()
+    )
+
+    if not session:
+        raise NotFoundError("Chat session", session_id)
+
+    messages = [
+        ChatMessageResponse(
+            id=str(msg.id),
+            role=msg.role,
+            content=msg.content,
+            sources=msg.sources or [],
+            created_at=msg.created_at,
+            model_used=msg.model_used,
+            response_time_ms=msg.response_time_ms,
+        )
+        for msg in session.messages
+    ]
+
+    return ChatSessionDetailResponse(
+        id=str(session.id),
+        title=session.title,
+        document_context=session.document_context or [],
+        system_prompt=session.system_prompt,
+        messages=messages,
+        created_at=session.created_at,
+        updated_at=session.updated_at,
+    )
+
+
+@router.patch("/sessions/{session_id}", response_model=ChatSessionResponse)
+async def update_chat_session(
+    session_id: uuid.UUID,
+    request: ChatSessionUpdateRequest,
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """
+    Update chat session title or document context.
+    """
+    session = (
+        db.query(ChatSession)
+        .filter(ChatSession.id == session_id, ChatSession.user_id == user_id)
+        .first()
+    )
+
+    if not session:
+        raise NotFoundError("Chat session", session_id)
+
+    if request.title is not None:
+        session.title = request.title
+    if request.document_ids is not None:
+        session.document_context = request.document_ids
+
+    session.updated_at = datetime.utcnow()
+    db.commit()
+    db.refresh(session)
+
+    return ChatSessionResponse(
+        id=str(session.id),
+        title=session.title,
+        message_count=len(session.messages),
+        document_context=session.document_context,
+        created_at=session.created_at,
+        updated_at=session.updated_at,
+    )
+
+
+@router.delete("/sessions/{session_id}")
+async def delete_chat_session(
+    session_id: uuid.UUID,
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """
+    Delete a chat session and all messages.
+    """
+    session = (
+        db.query(ChatSession)
+        .filter(ChatSession.id == session_id, ChatSession.user_id == user_id)
+        .first()
+    )
+
+    if not session:
+        raise NotFoundError("Chat session", session_id)
+
+    # Audit log
+    audit = create_audit_log(
+        action=AuditAction.CHAT_DELETE.value,
+        user_id=user_id,
+        resource_type="chat_session",
+        resource_id=session_id,
+    )
+    db.add(audit)
+
+    db.delete(session)
+    db.commit()
+
+    return {"success": True, "message": "Chat session deleted"}
+
+
+@router.post("/send", response_model=ChatResponse)
+async def send_message(
+    request: ChatRequest,
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """
+    Send a message and get AI response.
+    Uses RAG to find relevant document context.
+    """
+    start_time = datetime.utcnow()
+
+    # Get or create session
+    if request.session_id:
+        session = (
+            db.query(ChatSession)
+            .filter(
+                ChatSession.id == request.session_id, ChatSession.user_id == user_id
+            )
+            .first()
+        )
+        if not session:
+            raise NotFoundError("Chat session", request.session_id)
+    else:
+        # Create new session
+        session = ChatSession(user_id=user_id, title="New Chat")
+        db.add(session)
+        db.commit()
+        db.refresh(session)
+
+    # Save user message (but don't commit yet - wait for successful response)
+    user_message = ChatMessage(
+        session_id=session.id, role="user", content=request.content
+    )
+    db.add(user_message)
+
+    # Generate AI response with RAG
+    from app.services.chat_service import ChatService
+
+    chat_service = ChatService()
+
+    try:
+        ai_response, sources, tokens_used = await chat_service.generate_response(
+            query=request.content,
+            user_id=user_id,
+            document_ids=request.document_ids,
+            db=db,
+        )
+
+        # Calculate response time
+        end_time = datetime.utcnow()
+        response_time_ms = int((end_time - start_time).total_seconds() * 1000)
+
+        # Save assistant message
+        assistant_message = ChatMessage(
+            session_id=session.id,
+            role="assistant",
+            content=ai_response,
+            sources=sources if request.include_sources else [],
+            model_used="gemini-2.5-flash",
+            response_time_ms=response_time_ms,
+            tokens_used=tokens_used,
+        )
+        db.add(assistant_message)
+
+        # Get fresh message count from database
+        message_count = (
+            db.query(ChatMessage).filter(ChatMessage.session_id == session.id).count()
+        )
+
+        # Update session title if first/second message
+        if message_count <= 2:
+            # Auto-generate title from first user message
+            session.title = request.content[:50] + (
+                "..." if len(request.content) > 50 else ""
+            )
+
+        session.updated_at = datetime.utcnow()
+
+        # Audit log
+        audit = create_audit_log(
+            action=AuditAction.CHAT_MESSAGE.value,
+            user_id=user_id,
+            resource_type="chat_session",
+            resource_id=str(session.id),
+            details={"message_length": len(request.content)},
+        )
+        db.add(audit)
+
+        # Commit all changes atomically
+        db.commit()
+        db.refresh(assistant_message)
+    except Exception as e:
+        db.rollback()
+        raise
+
+    return ChatResponse(
+        message=ChatMessageResponse(
+            id=str(assistant_message.id),
+            role="assistant",
+            content=ai_response,
+            sources=sources if request.include_sources else [],
+            created_at=assistant_message.created_at,
+            model_used="gemini-2.5-flash",
+            response_time_ms=response_time_ms,
+        ),
+        session_id=str(session.id),
+        session_title=session.title,
+    )

app/api/v1/chat_stream.py

@@ -0,0 +1,169 @@
+"""
+Streaming Chat Endpoint (SSE)
+Server-Sent Events streaming for real-time AI responses.
+
+Delivers three event types to the client:
+  1. 'sources' — document citations (emitted first so UI renders immediately)
+  2. 'token'   — streamed LLM response tokens
+  3. 'done'    — signals stream completion with metadata
+  4. 'error'   — on failure
+"""
+
+import json
+import logging
+from datetime import datetime
+from typing import List, Optional
+
+from fastapi import APIRouter, Depends, Query
+from fastapi.responses import StreamingResponse
+from sqlalchemy.orm import Session
+
+from app.api.deps import get_current_user_id
+from app.config import settings
+from app.db.session import get_db
+from app.models.chat import ChatMessage, ChatSession
+from app.schemas.chat import ChatRequest
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+@router.post("/stream")
+async def stream_chat(
+    request: ChatRequest,
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """
+    Stream AI response using Server-Sent Events.
+
+    Event format:
+        event: sources
+        data: [{"document_title": ..., "file_name": ..., "page_numbers": [...], ...}]
+
+        event: token
+        data: {"text": "word by word..."}
+
+        event: done
+        data: {"session_id": "...", "sources_count": 3}
+
+    Usage with fetch():
+        const source = new EventSource('/api/v1/chat/stream', {...})
+        source.addEventListener('token', (e) => appendToken(JSON.parse(e.data).text))
+        source.addEventListener('sources', (e) => renderSources(JSON.parse(e.data)))
+        source.addEventListener('done', () => source.close())
+    """
+    from app.services.chat_service import ChatService
+
+    # Get or create session
+    if request.session_id:
+        session = (
+            db.query(ChatSession)
+            .filter(
+                ChatSession.id == request.session_id, ChatSession.user_id == user_id
+            )
+            .first()
+        )
+        if not session:
+            from app.core.exceptions import NotFoundError
+
+            raise NotFoundError("Chat session", request.session_id)
+    else:
+        session = ChatSession(user_id=user_id, title="New Chat")
+        db.add(session)
+        db.commit()
+        db.refresh(session)
+
+    # Save user message immediately
+    user_message = ChatMessage(
+        session_id=session.id,
+        role="user",
+        content=request.content,
+    )
+    db.add(user_message)
+    db.commit()
+
+    chat_service = ChatService()
+
+    async def event_generator():
+        full_response = []
+        sources = []
+        tokens_used = None
+
+        try:
+            async for event in chat_service.generate_response_stream(
+                query=request.content,
+                user_id=user_id,
+                document_ids=request.document_ids,
+                db=db,
+            ):
+                # Forward each SSE event to client
+                yield event
+
+                # Track sources from the sources event for DB persistence
+                if event.startswith("event: sources\n"):
+                    data_line = event.split("data: ", 1)[-1].strip()
+                    try:
+                        sources = json.loads(data_line)
+                    except Exception:
+                        pass
+
+                # Accumulate tokens for DB persistence
+                elif event.startswith("event: token\n"):
+                    data_line = event.split("data: ", 1)[-1].strip()
+                    try:
+                        token_data = json.loads(data_line)
+                        full_response.append(token_data.get("text", ""))
+                    except Exception:
+                        pass
+
+                elif event.startswith("event: done\n"):
+                    data_line = event.split("data: ", 1)[-1].strip()
+                    try:
+                        done_data = json.loads(data_line)
+                        tokens_used = done_data.get("tokens_used")
+                    except Exception:
+                        pass
+
+        except Exception as e:
+            logger.error(f"Streaming error: {e}", exc_info=True)
+            yield f"event: error\ndata: {json.dumps({'message': str(e)})}\n\n"
+
+        finally:
+            # Persist assistant message after stream completes
+            if full_response:
+                response_text = "".join(full_response)
+                assistant_message = ChatMessage(
+                    session_id=session.id,
+                    role="assistant",
+                    content=response_text,
+                    sources=sources if request.include_sources else [],
+                    model_used=settings.GEMINI_MODEL,
+                    tokens_used=tokens_used,
+                )
+                db.add(assistant_message)
+
+                # Auto-title on first message
+                msg_count = (
+                    db.query(ChatMessage)
+                    .filter(ChatMessage.session_id == session.id)
+                    .count()
+                )
+                if msg_count <= 2:
+                    session.title = request.content[:50] + (
+                        "..." if len(request.content) > 50 else ""
+                    )
+
+                session.updated_at = datetime.utcnow()
+                db.commit()
+
+    return StreamingResponse(
+        event_generator(),
+        media_type="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "X-Accel-Buffering": "no",  # Disable nginx buffering
+        },
+    )

app/api/v1/compliance.py

@@ -0,0 +1,265 @@
+"""
+Compliance Audit API Endpoints
+Regulatory compliance auto-auditor: cross-references operational documents
+against regulatory documents to produce per-clause compliance matrices.
+"""
+
+import logging
+from typing import Optional
+from uuid import UUID
+
+from fastapi import APIRouter, Depends, HTTPException, Query, status
+from sqlalchemy.orm import Session
+
+from app.api.deps import get_current_user_id
+from app.db.session import get_db
+from app.models.audit import AuditAction, create_audit_log
+from app.models.compliance import AuditStatus, ComplianceAudit, ComplianceMatrixRow
+from app.models.document import Document, DocumentCategory, DocumentStatus
+from app.schemas.compliance import (
+    ComplianceAuditCreate,
+    ComplianceAuditDetailResponse,
+    ComplianceAuditListResponse,
+    ComplianceAuditResponse,
+    ComplianceMatrixRowResponse,
+)
+from app.services.queue import enqueue_compliance_task
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+@router.get("/audits", response_model=ComplianceAuditListResponse)
+async def list_audits(
+    page: int = Query(default=1, ge=1),
+    page_size: int = Query(default=20, ge=1, le=100),
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """List all compliance audits for the current user."""
+    offset = (page - 1) * page_size
+
+    query = (
+        db.query(ComplianceAudit)
+        .filter(ComplianceAudit.user_id == user_id)
+        .order_by(ComplianceAudit.created_at.desc())
+    )
+
+    total = query.count()
+    audits = query.offset(offset).limit(page_size).all()
+
+    return ComplianceAuditListResponse(
+        audits=[ComplianceAuditResponse.model_validate(a) for a in audits],
+        total=total,
+    )
+
+
+@router.post(
+    "/audits",
+    response_model=ComplianceAuditResponse,
+    status_code=status.HTTP_202_ACCEPTED,
+)
+async def create_audit(
+    data: ComplianceAuditCreate,
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """Create and trigger a new compliance audit."""
+    # Validate regulation document exists and belongs to user
+    reg_doc = (
+        db.query(Document)
+        .filter(
+            Document.id == data.regulation_doc_id,
+            Document.user_id == user_id,
+        )
+        .first()
+    )
+    if not reg_doc:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Regulation document not found",
+        )
+    if reg_doc.status != DocumentStatus.COMPLETED:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Regulation document must be fully processed before auditing",
+        )
+
+    # Validate operational documents
+    op_doc_ids = [str(d) for d in data.operational_doc_ids]
+    op_docs = (
+        db.query(Document)
+        .filter(
+            Document.id.in_(op_doc_ids),
+            Document.user_id == user_id,
+        )
+        .all()
+    )
+
+    if len(op_docs) != len(op_doc_ids):
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="One or more operational documents not found",
+        )
+
+    for doc in op_docs:
+        if doc.status != DocumentStatus.COMPLETED:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=f"Document '{doc.title}' must be fully processed first",
+            )
+
+    # Create audit record
+    audit = ComplianceAudit(
+        user_id=user_id,
+        title=data.title,
+        regulation_doc_id=data.regulation_doc_id,
+        operational_doc_ids=op_doc_ids,
+        status=AuditStatus.PENDING,
+    )
+    db.add(audit)
+    db.commit()
+    db.refresh(audit)
+
+    # Audit log
+    log = create_audit_log(
+        user_id=user_id,
+        action=AuditAction.DOCUMENT_UPLOAD,
+        resource_type="compliance_audit",
+        resource_id=str(audit.id),
+        details={"title": data.title, "regulation_doc": str(data.regulation_doc_id)},
+    )
+    db.add(log)
+    db.commit()
+
+    # Enqueue for background processing
+    await enqueue_compliance_task(str(audit.id))
+
+    logger.info(
+        f"Compliance audit created: {audit.id} — "
+        f"reg_doc={data.regulation_doc_id}, op_docs={len(op_doc_ids)}"
+    )
+
+    return ComplianceAuditResponse.model_validate(audit)
+
+
+@router.get("/audits/{audit_id}", response_model=ComplianceAuditDetailResponse)
+async def get_audit(
+    audit_id: UUID,
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """Get a compliance audit with full matrix rows."""
+    audit = (
+        db.query(ComplianceAudit)
+        .filter(
+            ComplianceAudit.id == audit_id,
+            ComplianceAudit.user_id == user_id,
+        )
+        .first()
+    )
+
+    if not audit:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Audit not found",
+        )
+
+    rows = (
+        db.query(ComplianceMatrixRow)
+        .filter(ComplianceMatrixRow.audit_id == audit.id)
+        .order_by(ComplianceMatrixRow.clause_index)
+        .all()
+    )
+
+    return ComplianceAuditDetailResponse(
+        **ComplianceAuditResponse.model_validate(audit).model_dump(),
+        rows=[ComplianceMatrixRowResponse.model_validate(r) for r in rows],
+    )
+
+
+@router.delete("/audits/{audit_id}")
+async def delete_audit(
+    audit_id: UUID,
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """Delete a compliance audit and all its matrix rows."""
+    audit = (
+        db.query(ComplianceAudit)
+        .filter(
+            ComplianceAudit.id == audit_id,
+            ComplianceAudit.user_id == user_id,
+        )
+        .first()
+    )
+
+    if not audit:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Audit not found",
+        )
+
+    audit_title = audit.title
+    db.delete(audit)
+    db.commit()
+
+    log = create_audit_log(
+        user_id=user_id,
+        action=AuditAction.DOCUMENT_DELETE,
+        resource_type="compliance_audit",
+        resource_id=str(audit_id),
+        details={"title": audit_title},
+    )
+    db.add(log)
+    db.commit()
+
+    return {"message": "Audit deleted"}
+
+
+@router.get("/audits/{audit_id}/export")
+async def export_audit(
+    audit_id: UUID,
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """Export audit results as structured JSON."""
+    audit = (
+        db.query(ComplianceAudit)
+        .filter(
+            ComplianceAudit.id == audit_id,
+            ComplianceAudit.user_id == user_id,
+        )
+        .first()
+    )
+
+    if not audit:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Audit not found",
+        )
+
+    rows = (
+        db.query(ComplianceMatrixRow)
+        .filter(ComplianceMatrixRow.audit_id == audit.id)
+        .order_by(ComplianceMatrixRow.clause_index)
+        .all()
+    )
+
+    return {
+        "audit": audit.to_dict(),
+        "matrix": [
+            {
+                "clause_index": r.clause_index,
+                "clause_text": r.clause_text,
+                "section_title": r.section_title,
+                "status": r.status,
+                "assessment": r.assessment,
+                "confidence": r.confidence,
+                "evidence_chunks": r.evidence_chunks or [],
+                "recommendations": r.recommendations or [],
+            }
+            for r in rows
+        ],
+    }

app/api/v1/documents.py

@@ -0,0 +1,322 @@
+"""
+Document API Endpoints
+Document upload, management, and analysis
+"""
+
+import logging
+import uuid
+from datetime import datetime
+from typing import List, Optional
+
+from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, status
+from sqlalchemy import func
+from sqlalchemy.orm import Session
+
+from app.api.deps import audit_middleware, get_current_user, get_current_user_id
+from app.core.exceptions import NotFoundError
+from app.db.session import get_db
+from app.models.audit import AuditAction, create_audit_log
+from app.models.document import (
+    ComplianceStatus,
+    Document,
+    DocumentCategory,
+    DocumentStatus,
+)
+from app.models.user import User
+from app.schemas.document import (
+    DocumentAnalysisResponse,
+    DocumentCreate,
+    DocumentListResponse,
+    DocumentResponse,
+    DocumentUploadResponse,
+)
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+@router.get("", response_model=DocumentListResponse)
+async def list_documents(
+    page: int = Query(1, ge=1),
+    page_size: int = Query(20, ge=1, le=100),
+    category: Optional[DocumentCategory] = None,
+    status: Optional[DocumentStatus] = None,
+    search: Optional[str] = None,
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """
+    List user's documents with filtering and pagination.
+    """
+    query = db.query(Document).filter(Document.user_id == user_id)
+
+    # Apply filters
+    if category:
+        query = query.filter(Document.category == category)
+    if status:
+        query = query.filter(Document.status == status)
+    if search:
+        # Escape SQL wildcard characters to prevent injection
+        escaped_search = (
+            search.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
+        )
+        safe_pattern = f"%{escaped_search}%"
+        query = query.filter(
+            Document.title.ilike(safe_pattern, escape="\\")
+            | Document.file_name.ilike(safe_pattern, escape="\\")
+        )
+
+    # Get total count
+    total = query.count()
+
+    # Apply pagination
+    offset = (page - 1) * page_size
+    documents = (
+        query.order_by(Document.created_at.desc()).offset(offset).limit(page_size).all()
+    )
+
+    # Calculate stats
+    stats = _calculate_document_stats(db, user_id)
+
+    return DocumentListResponse(
+        documents=[DocumentResponse(**doc.to_dict()) for doc in documents],
+        total=total,
+        page=page,
+        page_size=page_size,
+        stats=stats,
+    )
+
+
+@router.post(
+    "", response_model=DocumentUploadResponse, status_code=status.HTTP_202_ACCEPTED
+)
+async def create_document(
+    request: DocumentCreate,
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """
+    Create a new document from UploadThing URL.
+    Triggers background processing with AI analysis via task queue.
+    """
+    # Create document record
+    document = Document(
+        user_id=user_id,
+        title=request.title or request.file_name.rsplit(".", 1)[0],
+        file_name=request.file_name,
+        file_size=request.file_size,
+        file_type=request.file_type,
+        file_url=request.file_url,
+        status=DocumentStatus.PENDING,
+        tags=request.tags or [],
+    )
+
+    db.add(document)
+    db.flush()  # Flush to get the document.id without committing
+
+    # Create audit log
+    audit = create_audit_log(
+        action=AuditAction.DOCUMENT_UPLOAD.value,
+        user_id=user_id,
+        resource_type="document",
+        resource_id=str(document.id),
+        details={
+            "file_name": document.file_name,
+            "file_size": document.file_size,
+            "file_type": document.file_type,
+        },
+    )
+    db.add(audit)
+
+    # Commit both document and audit atomically
+    db.commit()
+    db.refresh(document)
+
+    # Trigger background processing
+    from app.services.queue import enqueue_document_task
+
+    enqueue_document_task(str(document.id))
+
+    logger.info(f"Document created and enqueued: {document.id} - {document.title}")
+
+    return DocumentUploadResponse(
+        id=str(document.id),
+        title=document.title,
+        file_name=document.file_name,
+        status=DocumentStatus.PENDING,
+        job_id=str(document.id),  # Using doc ID as job ID for now
+        message="Document uploaded successfully. AI analysis queued.",
+    )
+
+
+@router.get("/{document_id}", response_model=DocumentResponse)
+async def get_document(
+    document_id: uuid.UUID,
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """
+    Get document details by ID.
+    """
+    document = (
+        db.query(Document)
+        .filter(Document.id == document_id, Document.user_id == user_id)
+        .first()
+    )
+
+    if not document:
+        raise NotFoundError("Document", document_id)
+
+    return DocumentResponse(**document.to_dict())
+
+
+@router.delete("/{document_id}")
+async def delete_document(
+    document_id: uuid.UUID,
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """
+    Delete a document and all associated data.
+    """
+    document = (
+        db.query(Document)
+        .filter(Document.id == document_id, Document.user_id == user_id)
+        .first()
+    )
+
+    if not document:
+        raise NotFoundError("Document", document_id)
+
+    # Create audit log before deletion
+    audit = create_audit_log(
+        action=AuditAction.DOCUMENT_DELETE.value,
+        user_id=user_id,
+        resource_type="document",
+        resource_id=str(document_id),
+        details={"file_name": document.file_name},
+    )
+    db.add(audit)
+
+    # Delete document (cascade will handle embeddings)
+    db.delete(document)
+    db.commit()
+
+    logger.info(f"Document deleted: {document_id}")
+
+    return {"success": True, "message": "Document deleted successfully"}
+
+
+@router.get("/{document_id}/analysis", response_model=DocumentAnalysisResponse)
+async def get_document_analysis(
+    document_id: uuid.UUID,
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """
+    Get AI analysis results for a document.
+    """
+    document = (
+        db.query(Document)
+        .filter(Document.id == document_id, Document.user_id == user_id)
+        .first()
+    )
+
+    if not document:
+        raise NotFoundError("Document", document_id)
+
+    if document.status != DocumentStatus.COMPLETED:
+        return DocumentAnalysisResponse(
+            document_id=str(document.id), status=document.status.value, analysis=None
+        )
+
+    analysis = {
+        "category": document.category,
+        "subcategory": document.subcategory,
+        "classification_confidence": document.classification_confidence,
+        "summary": document.summary,
+        "key_points": document.key_points or [],
+        "safety_score": document.safety_score,
+        "compliance_status": document.compliance_status,
+        "hazards_detected": document.hazards_detected or [],
+        "safety_recommendations": document.safety_recommendations or [],
+        "entities": {
+            k: v if isinstance(v, list) else []
+            for k, v in (document.entities or {}).items()
+        },
+    }
+
+    return DocumentAnalysisResponse(
+        document_id=str(document.id), status="completed", analysis=analysis
+    )
+
+
+@router.post("/{document_id}/reanalyze", status_code=status.HTTP_202_ACCEPTED)
+async def reanalyze_document(
+    document_id: uuid.UUID,
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """
+    Trigger re-analysis of a document.
+    """
+    document = (
+        db.query(Document)
+        .filter(Document.id == document_id, Document.user_id == user_id)
+        .first()
+    )
+
+    if not document:
+        raise NotFoundError("Document", document_id)
+
+    # Reset status
+    document.status = DocumentStatus.PENDING
+    db.commit()
+
+    # Trigger reprocessing
+    from app.services.queue import enqueue_document_task
+
+    enqueue_document_task(str(document.id))
+
+    return {"success": True, "message": "Document reanalysis queued"}
+
+
+def _calculate_document_stats(db: Session, user_id: str) -> dict:
+    """Calculate aggregated statistics for user's documents"""
+    # Category distribution
+    category_counts = (
+        db.query(Document.category, func.count(Document.id))
+        .filter(Document.user_id == user_id, Document.category.isnot(None))
+        .group_by(Document.category)
+        .all()
+    )
+
+    by_category = {
+        cat.value if cat else "other": count for cat, count in category_counts
+    }
+
+    # Status distribution
+    status_counts = (
+        db.query(Document.status, func.count(Document.id))
+        .filter(Document.user_id == user_id)
+        .group_by(Document.status)
+        .all()
+    )
+
+    by_status = {
+        status.value if status else "unknown": count for status, count in status_counts
+    }
+
+    # Average safety score
+    avg_score = (
+        db.query(func.avg(Document.safety_score))
+        .filter(Document.user_id == user_id, Document.safety_score.isnot(None))
+        .scalar()
+    )
+
+    return {
+        "by_category": by_category,
+        "by_status": by_status,
+        "avg_safety_score": round(avg_score, 2) if avg_score else None,
+    }

app/api/v1/health.py

@@ -0,0 +1,80 @@
+"""
+Health Check Endpoints
+System health and status monitoring
+"""
+
+from datetime import datetime
+
+from fastapi import APIRouter, Depends
+from sqlalchemy.orm import Session
+
+from app.config import settings
+from app.db.session import check_db_connection, get_db
+from app.schemas.common import HealthResponse
+
+router = APIRouter()
+
+
+@router.get("/", response_model=HealthResponse)
+async def root():
+    """Root endpoint - basic health check"""
+    return HealthResponse(
+        status="healthy",
+        version=settings.APP_VERSION,
+        environment=settings.ENVIRONMENT,
+        timestamp=datetime.utcnow(),
+    )
+
+
+@router.get("/health", response_model=HealthResponse)
+async def health_check(db: Session = Depends(get_db)):
+    """
+    Detailed health check with service status.
+    Checks database connectivity and other services.
+    """
+    services = {"database": "unknown", "redis": "unknown", "ai": "unknown"}
+
+    # Check database
+    try:
+        from sqlalchemy import text
+
+        db.execute(text("SELECT 1"))
+        services["database"] = "healthy"
+    except Exception as e:
+        services["database"] = f"unhealthy: {str(e)}"
+
+    # Check Redis (if configured)
+    r = None
+    try:
+        import redis
+
+        r = redis.from_url(settings.REDIS_URL)
+        r.ping()
+        services["redis"] = "healthy"
+    except Exception:
+        services["redis"] = "not_configured"
+    finally:
+        if r is not None:
+            r.close()
+
+    # Check AI service (Gemini)
+    try:
+        import google.generativeai as genai
+
+        genai.configure(api_key=settings.GEMINI_API_KEY)
+        services["ai"] = "healthy"
+    except Exception:
+        services["ai"] = "not_configured"
+
+    # Overall status
+    overall = "healthy"
+    if services["database"] != "healthy":
+        overall = "degraded"
+
+    return HealthResponse(
+        status=overall,
+        version=settings.APP_VERSION,
+        environment=settings.ENVIRONMENT,
+        timestamp=datetime.utcnow(),
+        services=services,
+    )

app/api/v1/jobs.py

@@ -0,0 +1,122 @@
+"""
+Jobs API Endpoints
+Background job status tracking
+"""
+
+import logging
+
+from fastapi import APIRouter, Depends, Query
+from sqlalchemy.orm import Session
+
+from app.api.deps import get_current_user_id
+from app.core.exceptions import NotFoundError
+from app.db.session import get_db
+from app.models.document import Document, DocumentStatus
+from app.schemas.common import JobStatusResponse
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+@router.get("/{job_id}", response_model=JobStatusResponse)
+async def get_job_status(
+    job_id: str,
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """
+    Get status of a background processing job.
+    Currently jobs are tracked via document ID.
+    """
+    # For now, job_id is document_id
+    document = (
+        db.query(Document)
+        .filter(Document.id == job_id, Document.user_id == user_id)
+        .first()
+    )
+
+    if not document:
+        raise NotFoundError("Job", job_id)
+
+    # Map document status to job status
+    status_map = {
+        DocumentStatus.PENDING: "pending",
+        DocumentStatus.PROCESSING: "processing",
+        DocumentStatus.ANALYZING: "processing",
+        DocumentStatus.COMPLETED: "completed",
+        DocumentStatus.FAILED: "failed",
+    }
+
+    # Calculate progress based on status
+    progress_map = {
+        DocumentStatus.PENDING: 0,
+        DocumentStatus.PROCESSING: 30,
+        DocumentStatus.ANALYZING: 70,
+        DocumentStatus.COMPLETED: 100,
+        DocumentStatus.FAILED: 0,
+    }
+
+    result = None
+    if document.status == DocumentStatus.COMPLETED:
+        result = {
+            "document_id": str(document.id),
+            "category": document.category.value if document.category else None,
+            "safety_score": document.safety_score,
+            "summary_preview": document.summary[:200] if document.summary else None,
+        }
+
+    return JobStatusResponse(
+        job_id=str(document.id),
+        status=status_map.get(document.status, "unknown"),
+        progress=progress_map.get(document.status, 0),
+        result=result,
+        error=document.processing_error,
+        created_at=document.created_at,
+        updated_at=document.updated_at,
+        completed_at=document.processed_at,
+    )
+
+
+@router.get("")
+async def list_active_jobs(
+    user_id: str = Depends(get_current_user_id), db: Session = Depends(get_db)
+):
+    """
+    List all active (non-completed) processing jobs.
+    """
+    active_docs = (
+        db.query(Document)
+        .filter(
+            Document.user_id == user_id,
+            Document.status.in_(
+                [
+                    DocumentStatus.PENDING,
+                    DocumentStatus.PROCESSING,
+                    DocumentStatus.ANALYZING,
+                ]
+            ),
+        )
+        .order_by(Document.created_at.desc())
+        .all()
+    )
+
+    jobs = []
+    for doc in active_docs:
+        progress = {
+            DocumentStatus.PENDING: 0,
+            DocumentStatus.PROCESSING: 30,
+            DocumentStatus.ANALYZING: 70,
+        }.get(doc.status, 0)
+
+        jobs.append(
+            {
+                "job_id": str(doc.id),
+                "document_title": doc.title,
+                "status": doc.status.value,
+                "progress": progress,
+                "created_at": doc.created_at.isoformat(),
+            }
+        )
+
+    return {"jobs": jobs, "count": len(jobs)}

app/api/v1/prompts.py

@@ -0,0 +1,200 @@
+"""
+Custom Prompts API Endpoints
+CRUD operations for user-defined AI prompts
+"""
+
+import logging
+from typing import List, Optional
+
+from fastapi import APIRouter, Depends, Query
+from sqlalchemy.orm import Session
+
+from app.api.deps import get_current_user, get_current_user_id
+from app.core.exceptions import NotFoundError
+from app.db.session import get_db
+from app.models.prompt import CustomPrompt
+from app.schemas.prompt import (
+    PromptCreate,
+    PromptListResponse,
+    PromptResponse,
+    PromptUpdate,
+)
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+@router.get("", response_model=List[PromptResponse])
+async def list_prompts(
+    category: Optional[str] = None,
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """
+    List all custom prompts for the current user.
+    """
+    query = db.query(CustomPrompt).filter(CustomPrompt.user_id == user_id)
+
+    if category:
+        query = query.filter(CustomPrompt.category == category)
+
+    prompts = query.order_by(CustomPrompt.created_at.desc()).all()
+
+    return [
+        PromptResponse(
+            id=str(p.id),
+            name=p.name,
+            prompt=p.prompt_text,
+            description=p.description,
+            category=p.category,
+            is_default=p.is_default,
+            created_at=p.created_at,
+            updated_at=p.updated_at,
+        )
+        for p in prompts
+    ]
+
+
+@router.post("", response_model=PromptResponse, status_code=201)
+async def create_prompt(
+    request: PromptCreate,
+    user=Depends(get_current_user),
+    db: Session = Depends(get_db),
+):
+    """
+    Create a new custom prompt.
+    """
+    prompt = CustomPrompt(
+        user_id=user.clerk_user_id,
+        name=request.name,
+        prompt_text=request.prompt,
+        description=request.description,
+        category=request.category,
+        is_default=False,
+    )
+
+    db.add(prompt)
+    db.commit()
+    db.refresh(prompt)
+
+    logger.info(f"Prompt created: {prompt.id} by user {user.clerk_user_id[:12]}")
+
+    return PromptResponse(
+        id=str(prompt.id),
+        name=prompt.name,
+        prompt=prompt.prompt_text,
+        description=prompt.description,
+        category=prompt.category,
+        is_default=prompt.is_default,
+        created_at=prompt.created_at,
+        updated_at=prompt.updated_at,
+    )
+
+
+@router.get("/{prompt_id}", response_model=PromptResponse)
+async def get_prompt(
+    prompt_id: str,
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """
+    Get a specific prompt by ID.
+    """
+    prompt = (
+        db.query(CustomPrompt)
+        .filter(
+            CustomPrompt.id == prompt_id,
+            CustomPrompt.user_id == user_id,
+        )
+        .first()
+    )
+
+    if not prompt:
+        raise NotFoundError("Prompt", prompt_id)
+
+    return PromptResponse(
+        id=str(prompt.id),
+        name=prompt.name,
+        prompt=prompt.prompt_text,
+        description=prompt.description,
+        category=prompt.category,
+        is_default=prompt.is_default,
+        created_at=prompt.created_at,
+        updated_at=prompt.updated_at,
+    )
+
+
+@router.put("/{prompt_id}", response_model=PromptResponse)
+async def update_prompt(
+    prompt_id: str,
+    request: PromptUpdate,
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """
+    Update a custom prompt.
+    """
+    prompt = (
+        db.query(CustomPrompt)
+        .filter(
+            CustomPrompt.id == prompt_id,
+            CustomPrompt.user_id == user_id,
+        )
+        .first()
+    )
+
+    if not prompt:
+        raise NotFoundError("Prompt", prompt_id)
+
+    if request.name is not None:
+        prompt.name = request.name
+    if request.prompt is not None:
+        prompt.prompt_text = request.prompt
+    if request.description is not None:
+        prompt.description = request.description
+    if request.category is not None:
+        prompt.category = request.category
+
+    db.commit()
+    db.refresh(prompt)
+
+    return PromptResponse(
+        id=str(prompt.id),
+        name=prompt.name,
+        prompt=prompt.prompt_text,
+        description=prompt.description,
+        category=prompt.category,
+        is_default=prompt.is_default,
+        created_at=prompt.created_at,
+        updated_at=prompt.updated_at,
+    )
+
+
+@router.delete("/{prompt_id}")
+async def delete_prompt(
+    prompt_id: str,
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """
+    Delete a custom prompt.
+    """
+    prompt = (
+        db.query(CustomPrompt)
+        .filter(
+            CustomPrompt.id == prompt_id,
+            CustomPrompt.user_id == user_id,
+        )
+        .first()
+    )
+
+    if not prompt:
+        raise NotFoundError("Prompt", prompt_id)
+
+    db.delete(prompt)
+    db.commit()
+
+    logger.info(f"Prompt deleted: {prompt_id}")
+
+    return {"success": True, "message": "Prompt deleted successfully"}

app/api/v1/router.py

@@ -0,0 +1,42 @@
+"""
+API v1 Router
+Combines all endpoint routers into single API router
+"""
+
+from fastapi import APIRouter
+
+from app.api.v1 import (
+    analytics,
+    chat,
+    chat_stream,
+    compliance,
+    documents,
+    health,
+    jobs,
+    prompts,
+    search,
+    user,
+)
+
+api_router = APIRouter()
+
+# Include all routers
+api_router.include_router(health.router, tags=["Health"])
+
+api_router.include_router(documents.router, prefix="/documents", tags=["Documents"])
+
+api_router.include_router(chat.router, prefix="/chat", tags=["Chat"])
+
+api_router.include_router(chat_stream.router, prefix="/chat", tags=["Chat"])
+
+api_router.include_router(analytics.router, prefix="/analytics", tags=["Analytics"])
+
+api_router.include_router(jobs.router, prefix="/jobs", tags=["Jobs"])
+
+api_router.include_router(prompts.router, prefix="/prompts", tags=["Prompts"])
+
+api_router.include_router(user.router, prefix="/user", tags=["User"])
+
+api_router.include_router(search.router, prefix="/search", tags=["Search"])
+
+api_router.include_router(compliance.router, prefix="/compliance", tags=["Compliance"])

app/api/v1/search.py

@@ -0,0 +1,138 @@
+"""
+Semantic Search API Endpoint
+Natural language search across all user documents.
+
+Production retrieval pipeline:
+  1. Embed query via Gemini text-embedding-004
+  2. Hybrid search: pgvector cosine + pg_trgm BM25 via RRF
+  3. Cross-encoder reranking for precise relevance
+  4. Results grouped by document with page-level provenance
+"""
+
+import asyncio
+import logging
+from typing import List, Optional
+
+import google.generativeai as genai
+from fastapi import APIRouter, Depends
+from fastapi import Query as FastAPIQuery
+from sqlalchemy.orm import Session
+
+from app.api.deps import get_current_user_id
+from app.config import settings
+from app.db.session import get_db
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+# Configure Gemini for embeddings
+genai.configure(api_key=settings.GEMINI_API_KEY)
+
+
+async def _get_query_embedding(text_input: str) -> List[float]:
+    """Generate query embedding using Gemini text-embedding-004."""
+    try:
+        result = await asyncio.to_thread(
+            genai.embed_content,
+            model=settings.EMBEDDING_MODEL,
+            content=text_input,
+            task_type="retrieval_query",
+        )
+        return result["embedding"]
+    except Exception as e:
+        logger.error(f"Query embedding failed: {e}")
+        return []
+
+
+@router.get("")
+async def semantic_search(
+    q: str = FastAPIQuery(
+        ..., min_length=2, description="Natural language search query"
+    ),
+    limit: int = FastAPIQuery(default=20, ge=1, le=50),
+    category: Optional[str] = FastAPIQuery(default=None),
+    user_id: str = Depends(get_current_user_id),
+    db: Session = Depends(get_db),
+):
+    """
+    Semantic search across all user documents.
+
+    Production pipeline:
+      1. Hybrid search (pgvector + pg_trgm via RRF)
+      2. Cross-encoder reranking
+      3. Results with page-level provenance
+
+    Example queries:
+      - "ventilation rules for underground mines"
+      - "30 CFR 75.323 methane requirements"
+      - "equipment maintenance schedule for Caterpillar D11"
+    """
+    if not q.strip():
+        return {"query": q, "results": [], "total": 0}
+
+    # Generate query embedding
+    embedding = await _get_query_embedding(q.strip())
+    if not embedding:
+        return {
+            "query": q,
+            "results": [],
+            "total": 0,
+            "error": "Could not generate embedding for query",
+        }
+
+    # Step 1: Hybrid search (over-fetch for reranking)
+    from app.services.hybrid_search import hybrid_search
+
+    candidates = await hybrid_search(
+        query_text=q.strip(),
+        query_embedding=embedding,
+        db=db,
+        user_id=user_id,
+        top_k=settings.RERANK_OVER_FETCH,
+    )
+
+    # Step 2: Rerank
+    if settings.ENABLE_RERANKING and len(candidates) > limit:
+        from app.services.reranker import rerank
+
+        candidates = rerank(query=q.strip(), chunks=candidates, top_k=limit)
+    else:
+        candidates = candidates[:limit]
+
+    # Format results
+    results = []
+    for row in candidates:
+        pages = row.get("page_numbers", [])
+        page_str = (
+            f"Pages {pages[0]}\u2013{pages[-1]}"
+            if len(pages) > 1
+            else f"Page {pages[0]}" if pages else "Unknown page"
+        )
+        results.append(
+            {
+                "chunk_id": row["id"],
+                "document_id": row["document_id"],
+                "document_title": row["document_title"],
+                "file_name": row["file_name"],
+                "chunk_text": row["text"],
+                "section_title": row.get("section_title"),
+                "page_numbers": pages,
+                "page_label": page_str,
+                "relevance_score": round(
+                    row.get("rerank_score", row.get("score", 0.0)), 4
+                ),
+                "relevance_percent": round(
+                    row.get("rerank_score", row.get("score", 0.0)) * 100, 1
+                ),
+            }
+        )
+
+    return {
+        "query": q,
+        "results": results,
+        "total": len(results),
+        "filters_applied": {
+            "category": category,
+        },
+    }

app/api/v1/user.py

@@ -0,0 +1,105 @@
+"""
+User Profile API Endpoints
+User profile management endpoints
+"""
+
+import logging
+from typing import List, Optional
+
+from fastapi import APIRouter, Depends
+from pydantic import BaseModel
+from sqlalchemy.orm import Session
+
+from app.api.deps import get_current_user, get_current_user_id
+from app.db.session import get_db
+from app.models.user import User
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+class UserProfileResponse(BaseModel):
+    """User profile response"""
+
+    clerk_user_id: str
+    email: Optional[str] = None
+    full_name: Optional[str] = None
+    avatar_url: Optional[str] = None
+    company_name: Optional[str] = None
+    company_role: Optional[str] = None
+    industry_focus: Optional[List[str]] = None
+    mine_sites: Optional[List[str]] = None
+    is_active: bool = True
+
+    class Config:
+        from_attributes = True
+
+
+class UserProfileUpdate(BaseModel):
+    """User profile update request"""
+
+    full_name: Optional[str] = None
+    company_name: Optional[str] = None
+    company_role: Optional[str] = None
+    industry_focus: Optional[List[str]] = None
+    mine_sites: Optional[List[str]] = None
+
+
+@router.get("/profile", response_model=UserProfileResponse)
+async def get_user_profile(
+    user: User = Depends(get_current_user),
+):
+    """
+    Get the current user's profile.
+    Creates the user record on first access if it doesn't exist.
+    """
+    return UserProfileResponse(
+        clerk_user_id=user.clerk_user_id,
+        email=user.email,
+        full_name=user.full_name,
+        avatar_url=user.avatar_url,
+        company_name=user.company_name,
+        company_role=user.company_role,
+        industry_focus=user.industry_focus,
+        mine_sites=user.mine_sites,
+        is_active=user.is_active,
+    )
+
+
+@router.put("/profile", response_model=UserProfileResponse)
+async def update_user_profile(
+    request: UserProfileUpdate,
+    user: User = Depends(get_current_user),
+    db: Session = Depends(get_db),
+):
+    """
+    Update the current user's profile.
+    """
+    if request.full_name is not None:
+        user.full_name = request.full_name
+    if request.company_name is not None:
+        user.company_name = request.company_name
+    if request.company_role is not None:
+        user.company_role = request.company_role
+    if request.industry_focus is not None:
+        user.industry_focus = request.industry_focus
+    if request.mine_sites is not None:
+        user.mine_sites = request.mine_sites
+
+    db.commit()
+    db.refresh(user)
+
+    logger.info(f"User profile updated: {user.clerk_user_id[:12]}")
+
+    return UserProfileResponse(
+        clerk_user_id=user.clerk_user_id,
+        email=user.email,
+        full_name=user.full_name,
+        avatar_url=user.avatar_url,
+        company_name=user.company_name,
+        company_role=user.company_role,
+        industry_focus=user.industry_focus,
+        mine_sites=user.mine_sites,
+        is_active=user.is_active,
+    )

app/config.py

@@ -0,0 +1,125 @@
+"""
+Application Configuration
+Centralized settings management using Pydantic Settings
+"""
+
+import os
+from functools import lru_cache
+from typing import List, Optional
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    """Application settings loaded from environment variables"""
+
+    # Application
+    APP_NAME: str = "MiningNiti"
+    APP_VERSION: str = "2.0.0"
+    DEBUG: bool = Field(default=False)
+    ENVIRONMENT: str = Field(default="development")
+
+    # API
+    API_V1_PREFIX: str = "/api/v1"
+    CORS_ORIGINS: List[str] = Field(
+        default=["http://localhost:3000", "https://*.vercel.app"]
+    )
+
+    # Database
+    DATABASE_URL: str = Field(..., description="PostgreSQL connection string")
+    DB_POOL_SIZE: int = Field(default=5)
+    DB_MAX_OVERFLOW: int = Field(default=10)
+
+    # Redis
+    REDIS_URL: str = Field(default="redis://localhost:6379/0")
+
+    # AI/ML - Multi-Provider Setup
+    GEMINI_API_KEY: str = Field(..., description="Google Gemini API Key")
+    GROQ_API_KEY: str = Field(
+        ..., description="Groq API Key for Classifier & Entity Extractors"
+    )
+    MISTRAL_API_KEY: str = Field(..., description="Mistral API Key for Safety Analyzer")
+    CEREBRAS_API_KEY: str = Field(default="", description="Cerebras API Key")
+
+    GEMINI_MODEL: str = Field(default="gemini-1.5-flash")
+    EMBEDDING_MODEL: str = Field(default="models/gemini-embedding-001")
+
+    AGENT_PROVIDER_MAP: dict = {
+        "embeddings": {"provider": "gemini", "model": "text-embedding-004"},
+        "chat_service": {"provider": "gemini", "model": "gemini-1.5-flash"},
+        "summarizer_agent": {"provider": "gemini", "model": "gemini-1.5-flash"},
+        "classifier_agent": {"provider": "groq", "model": "llama-3.3-70b-versatile"},
+        "entity_extractor": {"provider": "cerebras", "model": "llama-4-scout"},
+        "safety_analyzer": {"provider": "mistral", "model": "magistral-small-latest"},
+        "fallback": {"provider": "openrouter", "model": "deepseek/deepseek-r1:free"},
+    }
+
+    # Authentication - Clerk
+    CLERK_JWKS_URL: str = Field(..., description="Clerk JWKS URL for JWT verification")
+
+    # Document Processing
+    MAX_FILE_SIZE_MB: int = Field(default=50)
+    ALLOWED_FILE_TYPES: List[str] = Field(
+        default=[
+            "application/pdf",
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+            "text/plain",
+        ]
+    )
+    CHUNK_SIZE: int = Field(default=1000)
+    CHUNK_OVERLAP: int = Field(default=200)
+
+    # Mining AI Settings
+    SAFETY_SCORE_THRESHOLD: float = Field(default=70.0)
+    MAX_EMBEDDINGS_PER_QUERY: int = Field(default=5)
+
+    # RAG Pipeline — Production Retrieval
+    RERANK_MODEL: str = Field(
+        default="cross-encoder/ms-marco-MiniLM-L-6-v2",
+        description="Cross-encoder model for reranking retrieved chunks",
+    )
+    RERANK_OVER_FETCH: int = Field(
+        default=20,
+        description="How many chunks to fetch from vector+BM25 before reranking",
+    )
+    RERANK_TOP_K: int = Field(
+        default=5,
+        description="Final number of chunks after reranking",
+    )
+    SIMILARITY_THRESHOLD: float = Field(
+        default=0.25,
+        description="Minimum cosine similarity to include a chunk (0-1)",
+    )
+    ENABLE_HYBRID_SEARCH: bool = Field(
+        default=True,
+        description="Combine vector search with pg_trgm BM25 via RRF",
+    )
+    ENABLE_RERANKING: bool = Field(
+        default=True,
+        description="Apply cross-encoder reranking after retrieval",
+    )
+    RRF_K: int = Field(
+        default=60,
+        description="Reciprocal Rank Fusion constant (higher = less rank influence)",
+    )
+
+    # SSL
+    SSL_CERT_PATH: Optional[str] = Field(default=None)
+
+    model_config = SettingsConfigDict(
+        env_file=".env", env_file_encoding="utf-8", case_sensitive=True, extra="ignore"
+    )
+
+
+@lru_cache()
+def get_settings() -> Settings:
+    """
+    Get cached settings instance.
+    Uses lru_cache for performance - settings are loaded once.
+    """
+    return Settings()
+
+
+# Convenience export
+settings = get_settings()

app/core/__init__.py

@@ -0,0 +1,3 @@
+"""
+Core module - Security, Exceptions, Utilities
+"""

app/core/exceptions.py

@@ -0,0 +1,115 @@
+"""
+Custom Exception Classes
+Enterprise-grade error handling with proper HTTP status codes
+"""
+
+from typing import Any, Dict, Optional
+
+from fastapi import HTTPException, status
+
+
+class MiningNitiException(Exception):
+    """Base exception for all MiningNiti errors"""
+
+    def __init__(
+        self,
+        message: str,
+        code: str = "INTERNAL_ERROR",
+        details: Optional[Dict[str, Any]] = None,
+    ):
+        self.message = message
+        self.code = code
+        self.details = details or {}
+        super().__init__(self.message)
+
+
+class AuthenticationError(HTTPException):
+    """Raised when authentication fails"""
+
+    def __init__(self, detail: str = "Authentication failed"):
+        import logging
+
+        logging.getLogger("app.core.exceptions").warning(
+            f"AuthenticationError raised: {detail}"
+        )
+        super().__init__(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail=detail,
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+
+class AuthorizationError(HTTPException):
+    """Raised when user lacks permission"""
+
+    def __init__(self, detail: str = "Permission denied"):
+        super().__init__(status_code=status.HTTP_403_FORBIDDEN, detail=detail)
+
+
+class NotFoundError(HTTPException):
+    """Raised when resource is not found"""
+
+    def __init__(self, resource: str = "Resource", resource_id: str = ""):
+        detail = f"{resource} not found"
+        if resource_id:
+            detail = f"{resource} with id '{resource_id}' not found"
+        super().__init__(status_code=status.HTTP_404_NOT_FOUND, detail=detail)
+
+
+class ValidationError(HTTPException):
+    """Raised when request validation fails"""
+
+    def __init__(
+        self, detail: str = "Validation failed", errors: Optional[list] = None
+    ):
+        super().__init__(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail={"message": detail, "errors": errors or []},
+        )
+
+
+class DocumentProcessingError(MiningNitiException):
+    """Raised when document processing fails"""
+
+    def __init__(self, message: str, document_id: Optional[str] = None):
+        super().__init__(
+            message=message,
+            code="DOCUMENT_PROCESSING_ERROR",
+            details={"document_id": document_id} if document_id else {},
+        )
+
+
+class AIServiceError(MiningNitiException):
+    """Raised when AI service (Gemini) fails"""
+
+    def __init__(self, message: str, service: str = "gemini"):
+        super().__init__(
+            message=message, code="AI_SERVICE_ERROR", details={"service": service}
+        )
+
+
+class RateLimitError(HTTPException):
+    """Raised when rate limit is exceeded"""
+
+    def __init__(self, retry_after: int = 60):
+        super().__init__(
+            status_code=status.HTTP_429_TOO_MANY_REQUESTS,
+            detail=f"Rate limit exceeded. Retry after {retry_after} seconds.",
+            headers={"Retry-After": str(retry_after)},
+        )
+
+
+class JobNotFoundError(NotFoundError):
+    """Raised when background job is not found"""
+
+    def __init__(self, job_id: str):
+        super().__init__(resource="Job", resource_id=job_id)
+
+
+class SafetyViolationError(MiningNitiException):
+    """Raised when safety compliance check fails critically"""
+
+    def __init__(self, message: str, violations: list):
+        super().__init__(
+            message=message, code="SAFETY_VIOLATION", details={"violations": violations}
+        )

app/core/security.py

@@ -0,0 +1,148 @@
+"""
+Security Module
+JWT verification, authentication, and authorization utilities
+"""
+
+import logging
+from datetime import datetime, timedelta
+from functools import lru_cache
+from typing import Any, Dict, Optional
+
+import httpx
+from jose import JWTError, jwk, jwt
+from jose.exceptions import ExpiredSignatureError
+
+from app.config import settings
+from app.core.exceptions import AuthenticationError
+
+logger = logging.getLogger(__name__)
+
+
+class JWKSClient:
+    """
+    JWKS (JSON Web Key Set) client for Clerk JWT verification.
+    Caches keys to avoid repeated network requests.
+    """
+
+    def __init__(self, jwks_url: str):
+        self.jwks_url = jwks_url
+        self._keys: Dict[str, Any] = {}
+        self._last_fetch: Optional[datetime] = None
+        self._cache_duration = timedelta(hours=1)
+
+    async def get_signing_key(self, kid: str) -> Optional[Dict[str, Any]]:
+        """Get signing key by key ID (kid)"""
+        await self._refresh_keys_if_needed()
+        return self._keys.get(kid)
+
+    async def _refresh_keys_if_needed(self, force: bool = False):
+        """Refresh keys if cache is stale or force is True"""
+        now = datetime.utcnow()
+
+        if (
+            not force
+            and self._last_fetch
+            and (now - self._last_fetch) < self._cache_duration
+        ):
+            return
+
+        try:
+            async with httpx.AsyncClient() as client:
+                response = await client.get(self.jwks_url, timeout=10.0)
+                response.raise_for_status()
+                jwks_data = response.json()
+
+                self._keys = {key["kid"]: key for key in jwks_data.get("keys", [])}
+                self._last_fetch = now
+                logger.info(f"Refreshed JWKS keys: {len(self._keys)} keys loaded")
+
+        except Exception as e:
+            logger.error(f"Failed to fetch JWKS: {e}")
+            if not self._keys:
+                raise AuthenticationError("Unable to verify authentication")
+
+
+# Global JWKS client instance
+_jwks_client: Optional[JWKSClient] = None
+
+
+def get_jwks_client() -> JWKSClient:
+    """Get or create JWKS client singleton"""
+    global _jwks_client
+    if _jwks_client is None:
+        _jwks_client = JWKSClient(settings.CLERK_JWKS_URL)
+    return _jwks_client
+
+
+async def verify_jwt_token(token: str) -> Dict[str, Any]:
+    """
+    Verify JWT token from Clerk.
+
+    Args:
+        token: JWT token string
+
+    Returns:
+        Decoded token payload with user information
+
+    Raises:
+        AuthenticationError: If token is invalid or expired
+    """
+    try:
+        # Decode header to get key ID
+        unverified_header = jwt.get_unverified_header(token)
+        kid = unverified_header.get("kid")
+
+        if not kid:
+            raise AuthenticationError("Invalid token: missing key ID")
+
+        # Get signing key
+        jwks_client = get_jwks_client()
+        signing_key = await jwks_client.get_signing_key(kid)
+
+        if not signing_key:
+            # Force refresh and try again
+            await jwks_client._refresh_keys_if_needed(force=True)
+            signing_key = await jwks_client.get_signing_key(kid)
+
+            if not signing_key:
+                raise AuthenticationError("Invalid token: unknown signing key")
+
+        # Verify and decode token
+        payload = jwt.decode(
+            token,
+            signing_key,
+            algorithms=["RS256"],
+            options={"verify_aud": False},  # Clerk doesn't always set audience
+        )
+
+        return payload
+
+    except ExpiredSignatureError:
+        raise AuthenticationError("Token has expired")
+    except JWTError as e:
+        logger.warning(f"JWT verification failed: {e}")
+        raise AuthenticationError("Invalid token")
+    except Exception as e:
+        logger.error(f"Authentication error: {e}")
+        raise AuthenticationError("Authentication failed")
+
+
+def extract_user_id(payload: Dict[str, Any]) -> str:
+    """
+    Extract user ID from JWT payload.
+    Clerk uses 'sub' claim for user ID.
+    """
+    user_id = payload.get("sub")
+    if not user_id:
+        raise AuthenticationError("Invalid token: missing user ID")
+    return user_id
+
+
+def extract_user_email(payload: Dict[str, Any]) -> Optional[str]:
+    """Extract email from JWT payload if available"""
+    # Clerk may include email in different claims
+    return (
+        payload.get("email")
+        or payload.get("primary_email")
+        or payload.get("email_addresses", [{}])[0].get("email_address")
+    )

app/db/__init__.py

@@ -0,0 +1,7 @@
+"""
+Database module - Session management and migrations
+"""
+
+from app.db.session import Base, SessionLocal, engine, get_db
+
+__all__ = ["engine", "SessionLocal", "get_db", "Base"]

app/db/session.py

@@ -0,0 +1,117 @@
+"""
+Database Session Management
+SQLAlchemy engine and session configuration with connection pooling
+"""
+
+import logging
+from contextlib import contextmanager
+from typing import Generator
+
+from sqlalchemy import create_engine, event, text
+from sqlalchemy.orm import Session, sessionmaker
+from sqlalchemy.pool import QueuePool
+
+from app.config import settings
+
+# Use the single canonical Base so all models share the same metadata
+from app.models.base import Base  # noqa: F401 - re-exported for convenience
+
+logger = logging.getLogger(__name__)
+
+# Configure engine with connection pooling
+engine_args = {
+    "pool_size": settings.DB_POOL_SIZE,
+    "max_overflow": settings.DB_MAX_OVERFLOW,
+    "pool_pre_ping": True,  # Verify connections before use
+    "pool_recycle": 3600,  # Recycle connections after 1 hour
+    "echo": False,  # Suppress excessive SQL query logging
+}
+
+# Add SSL config if certificate path provided
+connect_args = {}
+if settings.SSL_CERT_PATH:
+    connect_args["sslmode"] = "require"
+    connect_args["sslrootcert"] = settings.SSL_CERT_PATH
+
+engine = create_engine(
+    settings.DATABASE_URL, poolclass=QueuePool, connect_args=connect_args, **engine_args
+)
+
+# Session factory
+SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+
+# Connection event listeners for debugging
+@event.listens_for(engine, "connect")
+def on_connect(dbapi_conn, connection_record):
+    logger.debug("Database connection established")
+
+
+@event.listens_for(engine, "checkout")
+def on_checkout(dbapi_conn, connection_record, connection_proxy):
+    logger.debug("Database connection checked out from pool")
+
+
+def get_db() -> Generator[Session, None, None]:
+    """
+    Dependency for FastAPI endpoints.
+    Yields a database session and ensures cleanup.
+
+    Usage:
+        @app.get("/items")
+        def get_items(db: Session = Depends(get_db)):
+            ...
+    """
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()
+
+
+@contextmanager
+def get_db_context() -> Generator[Session, None, None]:
+    """
+    Context manager for database sessions outside of FastAPI.
+    Useful for background workers and scripts.
+
+    Usage:
+        with get_db_context() as db:
+            db.query(...)
+    """
+    db = SessionLocal()
+    try:
+        yield db
+        db.commit()
+    except Exception:
+        db.rollback()
+        raise
+    finally:
+        db.close()
+
+
+def init_db():
+    """Initialize database tables using the canonical Base from models.base"""
+    # Import all models so they register their tables with Base.metadata
+    from app.models import audit, chat, document, prompt, user  # noqa: F401
+    from app.models.base import Base as ModelBase
+
+    # Ensure pgvector extension exists before creating tables that use VECTOR columns
+    with engine.connect() as conn:
+        conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
+        conn.commit()
+    logger.info("pgvector extension ensured")
+
+    ModelBase.metadata.create_all(bind=engine)
+    logger.info("Database tables created successfully")
+
+
+def check_db_connection() -> bool:
+    """Check if database is reachable"""
+    try:
+        with engine.connect() as conn:
+            conn.execute(text("SELECT 1"))
+        return True
+    except Exception as e:
+        logger.error(f"Database connection failed: {e}")
+        return False

app/main.py

@@ -0,0 +1,283 @@
+"""
+MiningNiti Enterprise Backend
+FastAPI Application Entry Point
+
+AI-Powered Document Intelligence for the Coal Mining Industry
+"""
+
+import logging
+from contextlib import asynccontextmanager
+from datetime import datetime
+
+from fastapi import FastAPI, HTTPException, Request, status
+from fastapi.exceptions import RequestValidationError
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from slowapi import Limiter, _rate_limit_exceeded_handler
+from slowapi.errors import RateLimitExceeded
+from slowapi.middleware import SlowAPIMiddleware
+from slowapi.util import get_remote_address
+
+from app.api.v1 import api_router
+from app.config import settings
+from app.core.exceptions import MiningNitiException
+from app.db.session import check_db_connection, init_db
+
+# Configure logging
+logging.basicConfig(
+    level=logging.DEBUG if settings.DEBUG else logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+# Suppress SQLAlchemy's extremely verbose SQL echo in debug mode —
+# it drowns out real application logs. Set to WARNING to only see errors.
+logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING)
+logging.getLogger("sqlalchemy.pool").setLevel(logging.WARNING)
+logging.getLogger("sqlalchemy.dialects").setLevel(logging.WARNING)
+# Also suppress httpcore connection-level debug spam
+logging.getLogger("httpcore").setLevel(logging.WARNING)
+logging.getLogger("httpx").setLevel(logging.INFO)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Application lifespan handler for startup and shutdown events"""
+    # Startup
+    logger.info(f"Starting {settings.APP_NAME} v{settings.APP_VERSION}")
+    logger.info(f"Environment: {settings.ENVIRONMENT}")
+
+    import asyncio
+
+    from app.services.queue import compliance_worker, document_worker
+
+    worker_task = asyncio.create_task(document_worker())
+    compliance_worker_task = asyncio.create_task(compliance_worker())
+
+    # Check database connection
+    if check_db_connection():
+        logger.info("Database connection verified")
+        # Auto-create tables on startup (idempotent)
+        try:
+            init_db()
+            logger.info("Database tables initialized")
+        except Exception as e:
+            logger.warning(f"Database table creation warning: {e}")
+
+        # Recovery: reset documents stuck in transient states from a previous crash/restart
+        try:
+            from app.db.session import get_db_context
+            from app.models.document import Document, DocumentStatus
+
+            with get_db_context() as db:
+                stuck_docs = (
+                    db.query(Document)
+                    .filter(Document.status.in_(["processing", "analyzing"]))
+                    .all()
+                )
+                if stuck_docs:
+                    for doc in stuck_docs:
+                        doc.status = DocumentStatus.PENDING
+                        doc.processing_error = "Reset after server restart"
+                    db.commit()
+                    logger.info(
+                        f"Recovery: reset {len(stuck_docs)} stuck document(s) to PENDING"
+                    )
+                else:
+                    logger.info("Recovery: no stuck documents found")
+        except Exception as e:
+            logger.warning(f"Document recovery warning: {e}")
+
+        # Recovery: reset compliance audits stuck in running state
+        try:
+            from app.models.compliance import AuditStatus, ComplianceAudit
+
+            with get_db_context() as db:
+                stuck_audits = (
+                    db.query(ComplianceAudit)
+                    .filter(ComplianceAudit.status.in_(["running"]))
+                    .all()
+                )
+                if stuck_audits:
+                    for audit in stuck_audits:
+                        audit.status = AuditStatus.PENDING
+                        audit.processing_error = "Reset after server restart"
+                    db.commit()
+                    logger.info(
+                        f"Recovery: reset {len(stuck_audits)} stuck audit(s) to PENDING"
+                    )
+        except Exception as e:
+            logger.warning(f"Audit recovery warning: {e}")
+    else:
+        logger.warning("Database connection failed - some features may not work")
+
+    yield
+
+    # Shutdown
+    logger.info("Shutting down application")
+    worker_task.cancel()
+    compliance_worker_task.cancel()
+    try:
+        await worker_task
+    except asyncio.CancelledError:
+        pass
+    try:
+        await compliance_worker_task
+    except asyncio.CancelledError:
+        pass
+
+
+# Create FastAPI application
+app = FastAPI(
+    title=settings.APP_NAME,
+    description="""
+## MiningNiti - AI Document Intelligence for Mining
+
+Enterprise-grade document processing and AI chat platform 
+specifically designed for the coal mining industry.
+
+### Features
+- 📄 **Smart Document Processing** - Upload PDF, DOCX, TXT with AI analysis
+- 🤖 **Multi-Agent AI** - Classification, Safety Analysis, Entity Extraction
+- 💬 **RAG Chat** - Context-aware conversations with document citations
+- 📊 **Analytics Dashboard** - Safety metrics, compliance tracking
+- 🔒 **Enterprise Security** - JWT auth, audit logging
+
+### AI Agents
+1. **Classifier Agent** - Categorizes mining documents
+2. **Safety Analyzer** - Detects hazards and compliance issues
+3. **Entity Extractor** - Extracts equipment, chemicals, regulations
+4. **Summarizer** - Creates executive summaries
+    """,
+    version=settings.APP_VERSION,
+    docs_url="/docs",
+    redoc_url="/redoc",
+    openapi_url="/openapi.json",
+    lifespan=lifespan,
+)
+
+# Rate Limiter
+limiter = Limiter(key_func=get_remote_address, default_limits=["120/minute"])
+
+# CORS Configuration
+_EXTRA_ORIGINS = ["http://localhost:3000", "http://localhost:3001"]
+
+app.state.limiter = limiter
+app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
+app.add_middleware(SlowAPIMiddleware)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=settings.CORS_ORIGINS + _EXTRA_ORIGINS,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+# Exception Handlers
+
+
+def _get_cors_headers(request: Request) -> dict:
+    """
+    Build CORS headers to attach to error responses.
+    This is needed because FastAPI's HTTPBearer can short-circuit before
+    CORSMiddleware has a chance to add Access-Control-Allow-Origin headers,
+    causing the browser to report a CORS error instead of the real auth error.
+    """
+    origin = request.headers.get("origin", "")
+    allowed_origins = settings.CORS_ORIGINS + _EXTRA_ORIGINS
+    if origin in allowed_origins or any(
+        origin.endswith(o.lstrip("*")) for o in allowed_origins if "*" in o
+    ):
+        return {
+            "Access-Control-Allow-Origin": origin,
+            "Access-Control-Allow-Credentials": "true",
+        }
+    return {}
+
+
+@app.exception_handler(HTTPException)
+async def http_exception_handler(request: Request, exc: HTTPException):
+    """Handle HTTP exceptions with CORS headers so auth failures are visible to the browser"""
+    headers = {**(exc.headers or {}), **_get_cors_headers(request)}
+    return JSONResponse(
+        status_code=exc.status_code,
+        content={"detail": exc.detail},
+        headers=headers,
+    )
+
+
+@app.exception_handler(MiningNitiException)
+async def miningniti_exception_handler(request: Request, exc: MiningNitiException):
+    """Handle custom application exceptions"""
+    return JSONResponse(
+        status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+        content={
+            "error": exc.message,
+            "code": exc.code,
+            "details": exc.details,
+            "timestamp": datetime.utcnow().isoformat(),
+        },
+        headers=_get_cors_headers(request),
+    )
+
+
+@app.exception_handler(RequestValidationError)
+async def validation_exception_handler(request: Request, exc: RequestValidationError):
+    """Handle Pydantic validation errors"""
+    return JSONResponse(
+        status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+        content={
+            "error": "Validation failed",
+            "code": "VALIDATION_ERROR",
+            "details": exc.errors(),
+            "timestamp": datetime.utcnow().isoformat(),
+        },
+        headers=_get_cors_headers(request),
+    )
+
+
+@app.exception_handler(Exception)
+async def global_exception_handler(request: Request, exc: Exception):
+    """Global exception handler for unhandled errors"""
+    logger.error(f"Unhandled exception: {exc}", exc_info=True)
+    return JSONResponse(
+        status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+        content={
+            "error": "Internal server error",
+            "code": "INTERNAL_SERVER_ERROR",
+            "timestamp": datetime.utcnow().isoformat(),
+        },
+        headers=_get_cors_headers(request),
+    )
+
+
+# Include API router
+app.include_router(api_router, prefix=settings.API_V1_PREFIX)
+
+
+# Root endpoint (without /api/v1 prefix for health checks)
+@app.get("/", tags=["Root"])
+async def root():
+    """Root endpoint - application info"""
+    return {
+        "name": settings.APP_NAME,
+        "version": settings.APP_VERSION,
+        "description": "AI Document Intelligence for Mining Industry",
+        "docs": "/docs",
+        "health": f"{settings.API_V1_PREFIX}/health",
+    }
+
+
+@app.get("/health", tags=["Root"])
+async def health():
+    """Quick health check for load balancers"""
+    return {"status": "healthy", "timestamp": datetime.utcnow().isoformat()}
+
+
+# Run with: uvicorn app.main:app --reload
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run("app.main:app", host="0.0.0.0", port=8000, reload=settings.DEBUG)

app/models/__init__.py

@@ -0,0 +1,30 @@
+"""
+Database Models
+SQLAlchemy ORM models for the MiningNiti platform
+"""
+
+from app.models.audit import AuditAction, AuditLog
+from app.models.base import Base, TimestampMixin, UUIDMixin
+from app.models.chat import ChatMessage, ChatSession
+from app.models.compliance import AuditStatus, ComplianceAudit, ComplianceMatrixRow
+from app.models.document import Document, DocumentCategory, DocumentEmbedding
+from app.models.prompt import CustomPrompt
+from app.models.user import User
+
+__all__ = [
+    "Base",
+    "TimestampMixin",
+    "UUIDMixin",
+    "User",
+    "Document",
+    "DocumentEmbedding",
+    "DocumentCategory",
+    "ChatSession",
+    "ChatMessage",
+    "AuditLog",
+    "AuditAction",
+    "CustomPrompt",
+    "ComplianceAudit",
+    "ComplianceMatrixRow",
+    "AuditStatus",
+]

app/models/audit.py

@@ -0,0 +1,142 @@
+"""
+Audit Log Model
+Enterprise compliance logging for all user actions
+"""
+
+import uuid
+from datetime import datetime
+from enum import Enum
+
+from sqlalchemy import JSON, Column, DateTime, String, Text
+from sqlalchemy.dialects.postgresql import JSONB as PG_JSONB
+from sqlalchemy.dialects.postgresql import UUID
+
+JSONB = JSON().with_variant(PG_JSONB, "postgresql")
+
+from app.models.base import Base, UUIDMixin
+
+
+class AuditAction(str, Enum):
+    """Types of auditable actions"""
+
+    # Document actions
+    DOCUMENT_UPLOAD = "document.upload"
+    DOCUMENT_VIEW = "document.view"
+    DOCUMENT_DELETE = "document.delete"
+    DOCUMENT_PROCESS = "document.process"
+
+    # Chat actions
+    CHAT_CREATE = "chat.create"
+    CHAT_MESSAGE = "chat.message"
+    CHAT_DELETE = "chat.delete"
+
+    # User actions
+    USER_LOGIN = "user.login"
+    USER_LOGOUT = "user.logout"
+    USER_PROFILE_UPDATE = "user.profile_update"
+
+    # Admin actions
+    ADMIN_ACTION = "admin.action"
+
+    # System actions
+    SYSTEM_ERROR = "system.error"
+    AI_ANALYSIS = "ai.analysis"
+
+
+class AuditLog(Base, UUIDMixin):
+    """
+    Immutable audit log for enterprise compliance.
+    Tracks all user actions and system events.
+    """
+
+    __tablename__ = "audit_logs"
+
+    # Who
+    user_id = Column(
+        String(255), nullable=True, index=True
+    )  # Nullable for system events
+    user_email = Column(String(255), nullable=True)
+
+    # What
+    action = Column(String(100), nullable=False, index=True)
+    resource_type = Column(String(50), nullable=True)  # document, chat, user
+    resource_id = Column(String(255), nullable=True)
+
+    # Details
+    description = Column(Text, nullable=True)
+    details = Column(JSONB, default={})
+    # Example details:
+    # {
+    #   "file_name": "safety_manual.pdf",
+    #   "file_size": 1024000,
+    #   "category": "safety_protocol"
+    # }
+
+    # When
+    timestamp = Column(DateTime, default=datetime.utcnow, nullable=False, index=True)
+
+    # Where
+    ip_address = Column(String(50), nullable=True)
+    user_agent = Column(Text, nullable=True)
+
+    # Outcome
+    success = Column(String(10), default="true")  # true, false, partial
+    error_message = Column(Text, nullable=True)
+
+    def __repr__(self):
+        return f"<AuditLog {self.action} by {self.user_id}>"
+
+    def to_dict(self):
+        """Convert to dictionary for API responses"""
+        return {
+            "id": str(self.id),
+            "user_id": self.user_id,
+            "action": self.action,
+            "resource_type": self.resource_type,
+            "resource_id": self.resource_id,
+            "description": self.description,
+            "details": self.details,
+            "timestamp": self.timestamp.isoformat() if self.timestamp else None,
+            "success": self.success,
+        }
+
+
+def create_audit_log(
+    action: str,
+    user_id: str = None,
+    user_email: str = None,
+    resource_type: str = None,
+    resource_id: str = None,
+    description: str = None,
+    details: dict = None,
+    ip_address: str = None,
+    user_agent: str = None,
+    success: str = "true",
+    error_message: str = None,
+) -> AuditLog:
+    """
+    Factory function to create audit log entries.
+
+    Usage:
+        log = create_audit_log(
+            action=AuditAction.DOCUMENT_UPLOAD.value,
+            user_id=current_user.id,
+            resource_type="document",
+            resource_id=str(doc.id),
+            details={"file_name": doc.file_name}
+        )
+        db.add(log)
+    """
+    return AuditLog(
+        action=action,
+        user_id=user_id,
+        user_email=user_email,
+        resource_type=resource_type,
+        resource_id=resource_id,
+        description=description,
+        details=details or {},
+        ip_address=ip_address,
+        user_agent=user_agent,
+        success=success,
+        error_message=error_message,
+    )

app/models/base.py

@@ -0,0 +1,41 @@
+"""
+Base Model Classes
+Mixins and base classes for all SQLAlchemy models
+"""
+
+import uuid
+from datetime import datetime
+
+from sqlalchemy import Column, DateTime
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.ext.declarative import declarative_base, declared_attr
+
+Base = declarative_base()
+
+
+class UUIDMixin:
+    """Mixin that adds a UUID primary key"""
+
+    id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, index=True)
+
+
+class TimestampMixin:
+    """Mixin that adds created_at and updated_at timestamps"""
+
+    created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
+
+    updated_at = Column(
+        DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=True
+    )
+
+
+class TableNameMixin:
+    """Mixin that auto-generates table name from class name"""
+
+    @declared_attr
+    def __tablename__(cls):
+        # Convert CamelCase to snake_case
+        name = cls.__name__
+        return "".join(["_" + c.lower() if c.isupper() else c for c in name]).lstrip(
+            "_"
+        )

app/models/chat.py

@@ -0,0 +1,126 @@
+"""
+Chat Models
+Chat sessions and messages with RAG context
+"""
+
+import uuid
+from datetime import datetime
+
+from sqlalchemy import JSON, Column, DateTime, ForeignKey, Integer, String, Text
+from sqlalchemy.dialects.postgresql import JSONB as PG_JSONB
+from sqlalchemy.dialects.postgresql import UUID
+
+JSONB = JSON().with_variant(PG_JSONB, "postgresql")
+from sqlalchemy.orm import relationship
+
+from app.models.base import Base, TimestampMixin, UUIDMixin
+
+
+class ChatSession(Base, UUIDMixin, TimestampMixin):
+    """
+    Chat session for grouping related messages.
+    Each session maintains context for the conversation.
+    """
+
+    __tablename__ = "chat_sessions"
+
+    # Owner
+    user_id = Column(
+        String(255), ForeignKey("users.clerk_user_id"), nullable=False, index=True
+    )
+
+    # Session info
+    title = Column(String(500), nullable=False, default="New Chat")
+
+    # Context - selected documents for this session
+    document_context = Column(JSONB, default=list)  # List of document IDs
+
+    # Custom prompt if set
+    system_prompt = Column(Text, nullable=True)
+
+    # Metadata
+    metadata_ = Column("metadata", JSONB, default=dict)
+
+    # Relationships
+    user = relationship("User", back_populates="chat_sessions")
+    messages = relationship(
+        "ChatMessage",
+        back_populates="session",
+        cascade="all, delete-orphan",
+        order_by="ChatMessage.created_at",
+    )
+
+    @property
+    def message_count(self) -> int:
+        return len(self.messages)
+
+    def __repr__(self):
+        return f"<ChatSession {self.title[:30]}...>"
+
+    def to_dict(self):
+        """Convert to dictionary for API responses"""
+        return {
+            "id": str(self.id),
+            "title": self.title,
+            "message_count": self.message_count,
+            "document_context": self.document_context,
+            "created_at": self.created_at.isoformat() if self.created_at else None,
+            "updated_at": self.updated_at.isoformat() if self.updated_at else None,
+        }
+
+
+class ChatMessage(Base, UUIDMixin):
+    """
+    Individual chat message with RAG source citations.
+    """
+
+    __tablename__ = "chat_messages"
+
+    # Parent session
+    session_id = Column(
+        UUID(as_uuid=True),
+        ForeignKey("chat_sessions.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+
+    # Message content
+    role = Column(String(20), nullable=False)  # "user" or "assistant"
+    content = Column(Text, nullable=False)
+
+    # RAG sources - documents/chunks used for this response
+    sources = Column(JSONB, default=[])
+    # Structure:
+    # [
+    #   {
+    #     "document_id": "uuid",
+    #     "document_title": "Safety Manual",
+    #     "chunk_text": "...",
+    #     "relevance_score": 0.95,
+    #     "page": 5
+    #   }
+    # ]
+
+    # AI metadata
+    model_used = Column(String(100), nullable=True)
+    tokens_used = Column(JSONB, nullable=True)  # {"input": 100, "output": 50}
+    response_time_ms = Column(Integer, nullable=True)
+
+    # Timestamp
+    created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
+
+    # Relationships
+    session = relationship("ChatSession", back_populates="messages")
+
+    def __repr__(self):
+        return f"<ChatMessage {self.role}: {self.content[:30]}...>"
+
+    def to_dict(self):
+        """Convert to dictionary for API responses"""
+        return {
+            "id": str(self.id),
+            "role": self.role,
+            "content": self.content,
+            "sources": self.sources,
+            "created_at": self.created_at.isoformat() if self.created_at else None,
+        }

app/models/compliance.py

@@ -0,0 +1,156 @@
+"""
+Compliance Audit Models
+Regulatory compliance auto-auditor: cross-references operational documents
+against regulatory documents (MSHA/OSHA/EPA/DGMS) to produce per-clause
+compliance matrices with citations.
+"""
+
+import uuid
+from datetime import datetime, timezone
+from enum import Enum
+
+from sqlalchemy import JSON, Column, DateTime
+from sqlalchemy import Enum as SQLEnum
+from sqlalchemy import Float, ForeignKey, Integer, String, Text
+from sqlalchemy.dialects.postgresql import JSONB as PG_JSONB
+from sqlalchemy.dialects.postgresql import UUID
+
+JSONB = JSON().with_variant(PG_JSONB, "postgresql")
+from sqlalchemy.orm import relationship
+
+from app.models.base import Base, TimestampMixin, UUIDMixin
+
+
+class AuditStatus(str, Enum):
+    """Compliance audit processing status"""
+
+    PENDING = "pending"
+    RUNNING = "running"
+    COMPLETED = "completed"
+    FAILED = "failed"
+
+
+class ComplianceAudit(Base, UUIDMixin, TimestampMixin):
+    """
+    A compliance audit that cross-references one regulatory document
+    against a set of operational documents.
+    """
+
+    __tablename__ = "compliance_audits"
+
+    # Owner
+    user_id = Column(
+        String(255),
+        ForeignKey("users.clerk_user_id"),
+        nullable=False,
+        index=True,
+    )
+
+    title = Column(String(500), nullable=False)
+
+    # The regulatory document being audited against
+    regulation_doc_id = Column(
+        UUID(as_uuid=True),
+        ForeignKey("documents.id"),
+        nullable=False,
+    )
+
+    # List of operational document UUIDs being audited
+    operational_doc_ids = Column(JSONB, nullable=False, default=list)
+
+    # Status
+    status = Column(
+        SQLEnum(AuditStatus),
+        default=AuditStatus.PENDING,
+        nullable=False,
+        index=True,
+    )
+
+    # Aggregate stats
+    total_clauses = Column(Integer, nullable=True)
+    processed_clauses = Column(Integer, default=0, nullable=False)
+    compliant_count = Column(Integer, nullable=True)
+    gap_count = Column(Integer, nullable=True)
+    missing_count = Column(Integer, nullable=True)
+    overall_score = Column(Float, nullable=True)  # 0-100
+
+    processing_error = Column(Text, nullable=True)
+    completed_at = Column(DateTime, nullable=True)
+
+    # Relationships
+    rows = relationship(
+        "ComplianceMatrixRow",
+        back_populates="audit",
+        cascade="all, delete-orphan",
+    )
+    regulation_doc = relationship(
+        "Document",
+        foreign_keys=[regulation_doc_id],
+    )
+
+    def __repr__(self):
+        return f"<ComplianceAudit {self.title[:50]}... status={self.status.value}>"
+
+    def to_dict(self):
+        return {
+            "id": str(self.id),
+            "title": self.title,
+            "regulation_doc_id": str(self.regulation_doc_id),
+            "operational_doc_ids": [str(d) for d in (self.operational_doc_ids or [])],
+            "status": self.status.value if self.status else None,
+            "total_clauses": self.total_clauses,
+            "processed_clauses": self.processed_clauses,
+            "compliant_count": self.compliant_count,
+            "gap_count": self.gap_count,
+            "missing_count": self.missing_count,
+            "overall_score": self.overall_score,
+            "processing_error": self.processing_error,
+            "completed_at": (
+                self.completed_at.replace(tzinfo=timezone.utc).isoformat()
+                if self.completed_at
+                else None
+            ),
+            "created_at": (
+                self.created_at.replace(tzinfo=timezone.utc).isoformat()
+                if self.created_at
+                else None
+            ),
+        }
+
+
+class ComplianceMatrixRow(Base, UUIDMixin):
+    """
+    A single row in the compliance matrix: one regulation clause
+    assessed against operational document evidence.
+    """
+
+    __tablename__ = "compliance_matrix_rows"
+
+    audit_id = Column(
+        UUID(as_uuid=True),
+        ForeignKey("compliance_audits.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+
+    clause_index = Column(Integer, nullable=False)
+    clause_text = Column(Text, nullable=False)
+    section_title = Column(String(500), nullable=True)
+
+    # compliant | gap | missing
+    status = Column(String(50), nullable=False)
+    assessment = Column(Text, nullable=False)  # LLM explanation
+    confidence = Column(Float, nullable=False)  # 0.0-1.0
+
+    # Evidence chunks that informed the assessment
+    evidence_chunks = Column(JSONB, nullable=True)
+    # Structure: [{"chunk_text": "...", "document_title": "...",
+    #              "page_numbers": [12,13], "relevance_score": 0.87}]
+
+    recommendations = Column(JSONB, nullable=True)  # list of strings
+
+    # Relationship
+    audit = relationship("ComplianceAudit", back_populates="rows")
+
+    def __repr__(self):
+        return f"<ComplianceMatrixRow clause={self.clause_index} status={self.status}>"

app/models/document.py

@@ -0,0 +1,230 @@
+"""
+Document Models
+Document storage, classification, and embeddings
+"""
+
+import uuid
+from datetime import datetime, timezone
+from enum import Enum
+
+from sqlalchemy import JSON, Column, DateTime
+from sqlalchemy import Enum as SQLEnum
+from sqlalchemy import Float, ForeignKey, Integer, String, Text
+from sqlalchemy.dialects.postgresql import ARRAY
+from sqlalchemy.dialects.postgresql import JSONB as PG_JSONB
+from sqlalchemy.dialects.postgresql import UUID
+
+JSONB = JSON().with_variant(PG_JSONB, "postgresql")
+from pgvector.sqlalchemy import Vector
+from sqlalchemy.orm import relationship
+
+from app.models.base import Base, TimestampMixin, UUIDMixin
+
+
+class DocumentCategory(str, Enum):
+    """Mining document categories for classification"""
+
+    SAFETY_PROTOCOL = "safety_protocol"
+    EQUIPMENT_MANUAL = "equipment_manual"
+    REGULATORY = "regulatory"
+    INCIDENT_REPORT = "incident_report"
+    GEOLOGICAL = "geological"
+    ENVIRONMENTAL = "environmental"
+    TRAINING = "training"
+    PERMIT = "permit"
+    MAINTENANCE = "maintenance"
+    OTHER = "other"
+
+
+class DocumentStatus(str, Enum):
+    """Document processing status"""
+
+    PENDING = "pending"
+    PROCESSING = "processing"
+    ANALYZING = "analyzing"
+    COMPLETED = "completed"
+    FAILED = "failed"
+
+
+class ComplianceStatus(str, Enum):
+    """Safety compliance status"""
+
+    COMPLIANT = "compliant"
+    WARNING = "warning"
+    VIOLATION = "violation"
+    PENDING = "pending"
+    NOT_APPLICABLE = "not_applicable"
+
+
+class Document(Base, UUIDMixin, TimestampMixin):
+    """
+    Document model with AI-enhanced metadata.
+    Stores file info, classification, safety analysis, and extracted entities.
+    """
+
+    __tablename__ = "documents"
+
+    # Owner
+    user_id = Column(
+        String(255), ForeignKey("users.clerk_user_id"), nullable=False, index=True
+    )
+
+    # File information
+    title = Column(String(500), nullable=False)
+    file_name = Column(String(500), nullable=False)
+    file_size = Column(Integer, nullable=False)  # bytes
+    file_type = Column(String(100), nullable=False)  # MIME type
+    file_url = Column(Text, nullable=False)
+
+    # Processing status
+    status = Column(
+        SQLEnum(DocumentStatus),
+        default=DocumentStatus.PENDING,
+        nullable=False,
+        index=True,
+    )
+    processing_error = Column(Text, nullable=True)
+    processed_at = Column(DateTime, nullable=True)
+
+    # Extracted content
+    content = Column(Text, nullable=True)  # Full text content
+    page_count = Column(Integer, nullable=True)  # deprecated alias — use total_pages
+    total_pages = Column(
+        Integer, nullable=True
+    )  # authoritative page count from extractor
+    word_count = Column(Integer, nullable=True)
+
+    # AI Classification
+    category = Column(
+        SQLEnum(DocumentCategory),
+        default=DocumentCategory.OTHER,
+        nullable=True,
+        index=True,
+    )
+    subcategory = Column(String(100), nullable=True)
+    classification_confidence = Column(Float, nullable=True)  # 0.0 - 1.0
+
+    # AI Summary
+    summary = Column(Text, nullable=True)  # AI-generated summary
+    key_points = Column(JSONB, nullable=True)  # List of key points
+
+    # Safety Analysis
+    safety_score = Column(Float, nullable=True)  # 0-100
+    compliance_status = Column(
+        SQLEnum(ComplianceStatus), default=ComplianceStatus.PENDING, nullable=True
+    )
+    hazards_detected = Column(JSONB, nullable=True)  # List of hazards
+    safety_recommendations = Column(JSONB, nullable=True)
+
+    # Named Entity Recognition
+    entities = Column(JSONB, nullable=True)
+    # Structure:
+    # {
+    #   "equipment": ["Caterpillar D11", "Komatsu PC8000"],
+    #   "chemicals": ["methane", "coal dust"],
+    #   "locations": ["Mine Site A", "Section 4B"],
+    #   "personnel": ["John Smith", "Safety Team"],
+    #   "dates": ["2024-01-15", "Q1 2024"],
+    #   "regulations": ["MSHA 30 CFR 75.400", "OSHA 1910.134"]
+    # }
+
+    # Extra Metadata
+    extra_metadata = Column("metadata", JSONB, default=dict)
+    tags = Column(JSONB, default=list)
+
+    # Relationships
+    user = relationship("User", back_populates="documents")
+    embeddings = relationship(
+        "DocumentEmbedding", back_populates="document", cascade="all, delete-orphan"
+    )
+
+    def __repr__(self):
+        return f"<Document {self.title[:50]}...>"
+
+    def to_dict(self):
+        """Convert to dictionary for API responses"""
+        return {
+            "id": str(self.id),
+            "title": self.title,
+            "file_name": self.file_name,
+            "file_size": self.file_size,
+            "file_type": self.file_type,
+            "file_url": self.file_url,
+            "status": self.status.value if self.status else None,
+            "category": self.category.value if self.category else None,
+            "subcategory": self.subcategory,
+            "classification_confidence": self.classification_confidence,
+            "summary": self.summary,
+            "key_points": self.key_points,
+            "safety_score": self.safety_score,
+            "compliance_status": (
+                self.compliance_status.value if self.compliance_status else None
+            ),
+            "hazards_detected": self.hazards_detected,
+            "entities": {
+                k: v if isinstance(v, list) else []
+                for k, v in (self.entities or {}).items()
+            }
+            or None,
+            "page_count": self.page_count,
+            "word_count": self.word_count,
+            "created_at": (
+                self.created_at.replace(tzinfo=timezone.utc).isoformat()
+                if self.created_at
+                else None
+            ),
+            "processed_at": (
+                self.processed_at.replace(tzinfo=timezone.utc).isoformat()
+                if self.processed_at
+                else None
+            ),
+            "total_pages": self.total_pages or self.page_count,
+        }
+
+
+class DocumentEmbedding(Base, UUIDMixin):
+    """
+    Vector embeddings for document chunks.
+    Used for semantic search and RAG.
+
+    The embedding column uses pgvector's native Vector(768) type with an
+    HNSW index (see migration 001) for sub-5ms approximate nearest-neighbor
+    search instead of brute-force Python cosine similarity.
+    """
+
+    __tablename__ = "document_embeddings"
+
+    # Parent document
+    document_id = Column(
+        UUID(as_uuid=True),
+        ForeignKey("documents.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+
+    # Chunk information
+    chunk_index = Column(Integer, nullable=False)
+    chunk_text = Column(Text, nullable=False)
+
+    # Vector embedding — native pgvector type with HNSW index (see migration 001)
+    # Replaces the old JSONB column for 10-100x faster similarity search.
+    embedding = Column(Vector(768), nullable=False)
+    embedding_model = Column(String(100), default="text-embedding-004")
+
+    # Context metadata — powers context-aware answers with page citations
+    section_title = Column(String(500), nullable=True)  # e.g. "Safety Procedures"
+    page_numbers = Column(
+        JSONB, nullable=True
+    )  # e.g. [12, 13] — pages this chunk spans
+
+    # Legacy page columns (kept for backward compat, use page_numbers instead)
+    start_page = Column(Integer, nullable=True)
+    end_page = Column(Integer, nullable=True)
+
+    extra_metadata = Column("metadata", JSONB, default=dict)
+
+    # Relationships
+    document = relationship("Document", back_populates="embeddings")
+
+    def __repr__(self):
+        return f"<DocumentEmbedding doc={self.document_id} chunk={self.chunk_index}>"

app/models/prompt.py

@@ -0,0 +1,63 @@
+"""
+Custom Prompt Model
+User-defined AI prompts for specialized mining document analysis
+"""
+
+import uuid
+from datetime import datetime
+
+from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String, Text
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.orm import relationship
+
+from app.models.base import Base, TimestampMixin, UUIDMixin
+
+
+class CustomPrompt(Base, UUIDMixin, TimestampMixin):
+    """
+    User-defined custom prompts for AI analysis.
+    Allows users to save specialized prompts for safety reviews,
+    compliance checks, equipment inspections, etc.
+    """
+
+    __tablename__ = "custom_prompts"
+
+    # Owner
+    user_id = Column(
+        String(255), ForeignKey("users.clerk_user_id"), nullable=False, index=True
+    )
+
+    # Prompt info
+    name = Column(String(255), nullable=False)
+    prompt_text = Column(Text, nullable=False)
+    description = Column(Text, nullable=True)
+
+    # Category/type for UI grouping
+    category = Column(
+        String(100), nullable=True
+    )  # e.g., "safety", "compliance", "equipment"
+
+    # Whether this is a default/system prompt
+    is_default = Column(Boolean, default=False, nullable=False)
+
+    # Usage tracking
+    use_count = Column(Integer, default=0, nullable=False)
+
+    # Relationships
+    user = relationship("User", back_populates="custom_prompts")
+
+    def __repr__(self):
+        return f"<CustomPrompt {self.name}>"
+
+    def to_dict(self):
+        """Convert to dictionary for API responses"""
+        return {
+            "id": str(self.id),
+            "name": self.name,
+            "prompt": self.prompt_text,
+            "description": self.description,
+            "category": self.category,
+            "is_default": self.is_default,
+            "created_at": self.created_at.isoformat() if self.created_at else None,
+            "updated_at": self.updated_at.isoformat() if self.updated_at else None,
+        }

app/models/user.py

@@ -0,0 +1,78 @@
+"""
+User Model
+User profile and organization management
+"""
+
+import uuid
+from datetime import datetime
+
+from sqlalchemy import JSON, Boolean, Column, DateTime, String, Text
+from sqlalchemy.dialects.postgresql import JSONB as PG_JSONB
+from sqlalchemy.dialects.postgresql import UUID
+
+JSONB = JSON().with_variant(PG_JSONB, "postgresql")
+from sqlalchemy.orm import relationship
+
+from app.models.base import Base, TimestampMixin, UUIDMixin
+
+
+class User(Base, UUIDMixin, TimestampMixin):
+    """
+    User model linked to Clerk authentication.
+    Stores user profile and preferences.
+    """
+
+    __tablename__ = "users"
+
+    # Clerk integration
+    clerk_user_id = Column(String(255), unique=True, nullable=False, index=True)
+    email = Column(String(255), nullable=True, index=True)
+
+    # Profile
+    full_name = Column(String(255), nullable=True)
+    avatar_url = Column(Text, nullable=True)
+
+    # Organization/Company (for enterprise)
+    company_name = Column(String(255), nullable=True)
+    company_role = Column(
+        String(100), nullable=True
+    )  # Safety Officer, Engineer, Manager
+
+    # Mining-specific
+    industry_focus = Column(JSONB, nullable=True)  # ["coal", "underground", "surface"]
+    mine_sites = Column(JSONB, nullable=True)  # Associated mine sites
+
+    # Preferences
+    preferences = Column(JSONB, default={})
+
+    # Status
+    is_active = Column(Boolean, default=True)
+    last_login = Column(DateTime, nullable=True)
+
+    # Relationships
+    documents = relationship(
+        "Document", back_populates="user", cascade="all, delete-orphan"
+    )
+    chat_sessions = relationship(
+        "ChatSession", back_populates="user", cascade="all, delete-orphan"
+    )
+    custom_prompts = relationship(
+        "CustomPrompt", back_populates="user", cascade="all, delete-orphan"
+    )
+
+    def __repr__(self):
+        return f"<User {self.email or self.clerk_user_id}>"
+
+    def to_dict(self):
+        """Convert to dictionary for API responses"""
+        return {
+            "id": str(self.id),
+            "clerk_user_id": self.clerk_user_id,
+            "email": self.email,
+            "full_name": self.full_name,
+            "company_name": self.company_name,
+            "company_role": self.company_role,
+            "industry_focus": self.industry_focus,
+            "is_active": self.is_active,
+            "created_at": self.created_at.isoformat() if self.created_at else None,
+        }

app/schemas/__init__.py

@@ -0,0 +1,50 @@
+"""
+Pydantic Schemas
+Request and response models for API validation
+"""
+
+from app.schemas.analytics import DashboardStats, DocumentAnalytics, SafetyAnalytics
+from app.schemas.chat import (
+    ChatMessageResponse,
+    ChatRequest,
+    ChatResponse,
+    ChatSessionCreate,
+    ChatSessionResponse,
+)
+from app.schemas.common import (
+    ErrorResponse,
+    HealthResponse,
+    JobStatusResponse,
+    PaginatedResponse,
+)
+from app.schemas.document import (
+    DocumentAnalysisResponse,
+    DocumentCreate,
+    DocumentListResponse,
+    DocumentResponse,
+    DocumentUploadResponse,
+)
+
+__all__ = [
+    # Document
+    "DocumentCreate",
+    "DocumentResponse",
+    "DocumentListResponse",
+    "DocumentUploadResponse",
+    "DocumentAnalysisResponse",
+    # Chat
+    "ChatRequest",
+    "ChatResponse",
+    "ChatSessionCreate",
+    "ChatSessionResponse",
+    "ChatMessageResponse",
+    # Analytics
+    "DashboardStats",
+    "DocumentAnalytics",
+    "SafetyAnalytics",
+    # Common
+    "HealthResponse",
+    "ErrorResponse",
+    "PaginatedResponse",
+    "JobStatusResponse",
+]

app/schemas/analytics.py

@@ -0,0 +1,140 @@
+"""
+Analytics Schemas
+Pydantic models for dashboard and analytics endpoints
+"""
+
+from datetime import date, datetime
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, Field
+
+
+class CategoryCount(BaseModel):
+    """Count by document category"""
+
+    category: str
+    count: int
+    percentage: float
+
+
+class StatusCount(BaseModel):
+    """Count by document status"""
+
+    status: str
+    count: int
+
+
+class SafetyDistribution(BaseModel):
+    """Safety score distribution"""
+
+    range: str  # "0-25", "26-50", "51-75", "76-100"
+    count: int
+    percentage: float
+
+
+class DashboardStats(BaseModel):
+    """Main dashboard statistics"""
+
+    # Document stats
+    total_documents: int = 0
+    processed_documents: int = 0
+    pending_documents: int = 0
+    failed_documents: int = 0
+
+    # Chat stats
+    total_chat_sessions: int = 0
+    total_messages: int = 0
+
+    # Safety stats
+    average_safety_score: Optional[float] = None
+    documents_with_hazards: int = 0
+    compliance_violations: int = 0
+    compliance_warnings: int = 0
+
+    # Processing stats
+    documents_processed_today: int = 0
+    documents_processed_this_week: int = 0
+
+    # Category breakdown
+    documents_by_category: List[CategoryCount] = []
+
+    # Recent activity
+    last_upload_at: Optional[datetime] = None
+    last_chat_at: Optional[datetime] = None
+
+
+class DocumentAnalytics(BaseModel):
+    """Detailed document analytics"""
+
+    # Time series
+    uploads_by_day: List[Dict[str, Any]] = []  # [{"date": "2024-01-15", "count": 5}]
+    processing_times: List[Dict[str, Any]] = (
+        []
+    )  # [{"date": "...", "avg_time_ms": 1500}]
+
+    # Category distribution
+    by_category: List[CategoryCount] = []
+
+    # Status distribution
+    by_status: List[StatusCount] = []
+
+    # File type distribution
+    by_file_type: List[Dict[str, Any]] = []
+
+    # Top documents by views
+    top_documents: List[Dict[str, Any]] = []
+
+
+class SafetyAnalytics(BaseModel):
+    """Safety compliance analytics"""
+
+    # Overall scores
+    average_safety_score: float = 0
+    median_safety_score: Optional[float] = None
+    min_safety_score: Optional[float] = None
+    max_safety_score: Optional[float] = None
+
+    # Distribution
+    score_distribution: List[SafetyDistribution] = []
+
+    # Compliance
+    compliant_count: int = 0
+    warning_count: int = 0
+    violation_count: int = 0
+
+    # Hazards
+    total_hazards_detected: int = 0
+    hazards_by_type: List[Dict[str, Any]] = []  # [{"type": "fall hazard", "count": 10}]
+
+    # Trend
+    safety_trend: List[Dict[str, Any]] = []  # [{"date": "...", "avg_score": 75}]
+
+    # Top issues
+    top_safety_concerns: List[Dict[str, Any]] = []
+
+
+class EntityAnalytics(BaseModel):
+    """Named entity analytics"""
+
+    # Equipment mentioned
+    top_equipment: List[Dict[str, Any]] = (
+        []
+    )  # [{"name": "Caterpillar D11", "mentions": 25}]
+
+    # Locations
+    top_locations: List[Dict[str, Any]] = []
+
+    # Chemicals
+    chemicals_mentioned: List[Dict[str, Any]] = []
+
+    # Regulations referenced
+    regulations_cited: List[Dict[str, Any]] = []
+
+
+class AnalyticsSummary(BaseModel):
+    """Combined analytics summary"""
+
+    documents: DocumentAnalytics
+    safety: SafetyAnalytics
+    entities: EntityAnalytics
+    generated_at: datetime = Field(default_factory=datetime.utcnow)