Ship production RAG hardening: citation extraction, full-library retrieval, auth.

- Citation-grounded extraction pipeline with contradiction audit and OCR/table-aware ingest
- Tenant auth and RAG document manager (list/delete/re-ingest)
- Full-tenant library retrieval (old + new report_source docs) enabled by default
- Frontend: auth gate, citation audit display, document manager
- Regression tests for extraction, ingest normalization, and library widening
- Stop tracking local SQLite WAL files; extend .gitignore

Co-authored-by: Cursor <cursoragent@cursor.com>

.gitignore

@@ -41,6 +41,8 @@ htmlcov/
 
 # Databases
 *.db
+*.db-shm
+*.db-wal
 *.sqlite
 *.sqlite3
 # DB backups (e.g. dev.db.bak-20260424-142846) — must NEVER be committed.

app/api/auth.py

@@ -0,0 +1,184 @@
+"""Tenant authentication: passphrase login + HMAC-signed bearer tokens.
+
+The token's subject (``sub``) is the tenant_id. Because the token is signed with
+a server-side secret, a client cannot forge a token for an arbitrary tenant, so
+``request.state.tenant_id`` derived from a verified token is trustworthy — this
+is what enforces user isolation. Stdlib only (``hashlib``, ``hmac``, ``secrets``).
+"""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import hmac
+import json
+import logging
+import secrets
+import time
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.config import settings
+from app.db.database import get_db
+from app.db.models import Tenant
+from app.models.schemas import AuthRequest, AuthTokenResponse, AuthWhoAmIResponse
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+_PBKDF2_ITERATIONS = 200_000
+_DEFAULT_SECRETS = frozenset({"", "dev-secret-change-me", "change-me-in-production"})
+
+# Process-ephemeral fallback secret, generated once if the operator shipped a
+# sentinel TENANT_SECRET_KEY in production. Tokens signed with it are
+# unforgeable but do not survive a restart (every restart forces re-login).
+_EPHEMERAL_SECRET: str | None = None
+
+
+def _signing_key() -> bytes:
+    global _EPHEMERAL_SECRET
+    configured = (settings.tenant_secret_key or "").strip()
+    if configured and configured not in _DEFAULT_SECRETS:
+        return configured.encode("utf-8")
+    if settings.dev_mode:
+        # Stable across the dev session; fine for local use.
+        return (configured or "dev-secret-change-me").encode("utf-8")
+    if _EPHEMERAL_SECRET is None:
+        _EPHEMERAL_SECRET = secrets.token_urlsafe(48)
+        logger.warning(
+            "TENANT_SECRET_KEY is unset/default in production — using an "
+            "ephemeral signing key. Set TENANT_SECRET_KEY so issued tokens "
+            "survive restarts."
+        )
+    return _EPHEMERAL_SECRET.encode("utf-8")
+
+
+# ── Password hashing ──────────────────────────────────────────────────────────
+
+def hash_password(password: str, *, salt: str | None = None, iterations: int = _PBKDF2_ITERATIONS) -> tuple[str, str, int]:
+    """Return ``(hash_hex, salt_hex, iterations)`` for ``password``."""
+    salt_bytes = bytes.fromhex(salt) if salt else secrets.token_bytes(16)
+    dk = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt_bytes, iterations)
+    return dk.hex(), salt_bytes.hex(), iterations
+
+
+def verify_password(password: str, *, hash_hex: str, salt_hex: str, iterations: int) -> bool:
+    candidate, _, _ = hash_password(password, salt=salt_hex, iterations=iterations)
+    return hmac.compare_digest(candidate, hash_hex)
+
+
+# ── Token mint / verify ─────────────────────────────────────────────────────────
+
+def _b64u_encode(raw: bytes) -> str:
+    return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
+
+
+def _b64u_decode(data: str) -> bytes:
+    pad = "=" * (-len(data) % 4)
+    return base64.urlsafe_b64decode(data + pad)
+
+
+def mint_token(tenant_id: str, *, ttl_seconds: int | None = None) -> tuple[str, int]:
+    """Create a signed token for ``tenant_id``; returns ``(token, expires_at)``."""
+    now = int(time.time())
+    exp = now + int(ttl_seconds if ttl_seconds is not None else settings.auth_token_ttl_seconds)
+    payload = _b64u_encode(json.dumps({"sub": tenant_id, "iat": now, "exp": exp}, separators=(",", ":")).encode("utf-8"))
+    sig = _b64u_encode(hmac.new(_signing_key(), payload.encode("ascii"), hashlib.sha256).digest())
+    return f"{payload}.{sig}", exp
+
+
+def verify_token(token: str) -> str | None:
+    """Return the tenant_id if ``token`` is valid and unexpired, else ``None``."""
+    if not token or "." not in token:
+        return None
+    payload_b64, _, sig_b64 = token.partition(".")
+    expected = _b64u_encode(hmac.new(_signing_key(), payload_b64.encode("ascii"), hashlib.sha256).digest())
+    if not hmac.compare_digest(sig_b64, expected):
+        return None
+    try:
+        payload = json.loads(_b64u_decode(payload_b64))
+    except (ValueError, json.JSONDecodeError):
+        return None
+    sub = payload.get("sub")
+    exp = payload.get("exp")
+    if not isinstance(sub, str) or not isinstance(exp, int):
+        return None
+    if exp < int(time.time()):
+        return None
+    return sub
+
+
+# ── Validation ──────────────────────────────────────────────────────────────────
+
+def _normalise_tenant_id(raw: str) -> str:
+    tid = (raw or "").strip()
+    if not (3 <= len(tid) <= 128):
+        raise HTTPException(status_code=422, detail="Tenant/User ID must be 3–128 characters.")
+    if not all(c.isalnum() or c in "._-" for c in tid):
+        raise HTTPException(
+            status_code=422,
+            detail="Tenant/User ID may only contain letters, digits, '.', '_' and '-'.",
+        )
+    return tid
+
+
+def _validate_passphrase(pw: str) -> None:
+    if len(pw or "") < 8:
+        raise HTTPException(status_code=422, detail="Passphrase must be at least 8 characters.")
+
+
+# ── Endpoints ─────────────────────────────────────────────────────────────────
+
+@router.post("/auth/register", response_model=AuthTokenResponse, status_code=201)
+async def register(body: AuthRequest, db: AsyncSession = Depends(get_db)) -> AuthTokenResponse:
+    """Create a new tenant and return a signed access token."""
+    tenant_id = _normalise_tenant_id(body.tenant_id)
+    _validate_passphrase(body.passphrase)
+
+    existing = await db.get(Tenant, tenant_id)
+    if existing is not None:
+        raise HTTPException(status_code=409, detail="That Tenant/User ID is already taken. Log in instead.")
+
+    h, salt, iters = hash_password(body.passphrase)
+    db.add(Tenant(id=tenant_id, password_hash=h, password_salt=salt, pbkdf2_iterations=iters))
+    await db.commit()
+
+    token, exp = mint_token(tenant_id)
+    logger.info("Registered tenant=%s", tenant_id)
+    return AuthTokenResponse(tenant_id=tenant_id, access_token=token, expires_at=exp)
+
+
+@router.post("/auth/login", response_model=AuthTokenResponse)
+async def login(body: AuthRequest, db: AsyncSession = Depends(get_db)) -> AuthTokenResponse:
+    """Verify a tenant passphrase and return a signed access token."""
+    tenant_id = (body.tenant_id or "").strip()
+    tenant = await db.get(Tenant, tenant_id) if tenant_id else None
+    # Run a dummy verify even when the tenant is unknown to keep timing uniform.
+    if tenant is None:
+        hash_password(body.passphrase or "x")
+        raise HTTPException(status_code=401, detail="Invalid Tenant/User ID or passphrase.")
+
+    ok = verify_password(
+        body.passphrase or "",
+        hash_hex=tenant.password_hash,
+        salt_hex=tenant.password_salt,
+        iterations=tenant.pbkdf2_iterations,
+    )
+    if not ok:
+        raise HTTPException(status_code=401, detail="Invalid Tenant/User ID or passphrase.")
+
+    from datetime import UTC, datetime
+
+    tenant.last_login_at = datetime.now(UTC)
+    await db.commit()
+
+    token, exp = mint_token(tenant_id)
+    return AuthTokenResponse(tenant_id=tenant_id, access_token=token, expires_at=exp)
+
+
+@router.get("/auth/me", response_model=AuthWhoAmIResponse)
+async def whoami(request: Request) -> AuthWhoAmIResponse:
+    """Return the tenant resolved from the current request's verified token/header."""
+    return AuthWhoAmIResponse(tenant_id=request.state.tenant_id)

app/api/generate.py

@@ -523,6 +523,7 @@ async def get_sections(
             interference_level=_meta_interference_level(meta),
             word_count=_meta_int(meta, "word_count"),
             generated_at=meta.get("generated_at") if isinstance(meta.get("generated_at"), str) else None,
+            citation_audit=meta.get("citation_audit") if isinstance(meta.get("citation_audit"), dict) else None,
         )
 
     return SectionsResponse(report_id=report_id, sections=sections)

app/api/middleware.py

@@ -1,4 +1,13 @@
-"""Middleware that enforces a non-empty X-Tenant-ID header on every request."""
+"""Middleware that authenticates every request and pins ``request.state.tenant_id``.
+
+Identity is resolved from a signed bearer token (``Authorization: Bearer <token>``)
+whose subject is the tenant_id. Because the token is HMAC-signed server-side, a
+client cannot claim another tenant's data — this is what enforces isolation.
+
+For local development (``DEV_MODE=true``) a raw ``X-Tenant-ID`` header is still
+accepted as a fallback so existing tooling and tests keep working. In production
+a valid token is mandatory.
+"""
 
 from collections.abc import Awaitable, Callable
 
@@ -6,24 +15,34 @@ from starlette.middleware.base import BaseHTTPMiddleware
 from starlette.requests import Request
 from starlette.responses import JSONResponse, Response
 
-# Paths that do not require a tenant identity (exact match)
+from app.api.auth import verify_token
+from app.config import settings
+
+# Paths that do not require authentication (exact match)
 _EXEMPT: frozenset[str] = frozenset(
-    {"/", "/health", "/docs", "/openapi.json", "/redoc", "/favicon.ico"}
+    {
+        "/",
+        "/health",
+        "/docs",
+        "/openapi.json",
+        "/redoc",
+        "/favicon.ico",
+        "/auth/login",
+        "/auth/register",
+    }
 )
 
-# Path prefixes that do not require a tenant identity
+# Path prefixes that do not require authentication
 _EXEMPT_PREFIXES: tuple[str, ...] = ("/static/",)
 
 
 class TenantAuthMiddleware(BaseHTTPMiddleware):
-    """Reject requests that carry no ``X-Tenant-ID`` header.
-
-    On success the header value is stored on ``request.state.tenant_id``
-    so downstream handlers can read it without re-parsing headers.
+    """Authenticate the caller and expose the verified tenant on request state.
 
     Example::
 
-        curl -H "X-Tenant-ID: tenant_abc" http://localhost:8000/upload ...
+        # Obtain a token, then call protected endpoints with it:
+        curl -H "Authorization: Bearer <token>" http://localhost:8000/documents
     """
 
     async def dispatch(
@@ -35,12 +54,31 @@ class TenantAuthMiddleware(BaseHTTPMiddleware):
         if path in _EXEMPT or path.startswith(_EXEMPT_PREFIXES):
             return await call_next(request)
 
-        tenant_id: str | None = request.headers.get("X-Tenant-ID")
-        if not tenant_id or not tenant_id.strip():
+        tenant_id = self._resolve_tenant(request)
+        if tenant_id is None:
             return JSONResponse(
                 status_code=401,
-                content={"detail": "Missing or empty X-Tenant-ID header"},
+                content={"detail": "Authentication required. Log in to obtain a token."},
             )
 
-        request.state.tenant_id = tenant_id.strip()
+        request.state.tenant_id = tenant_id
         return await call_next(request)
+
+    @staticmethod
+    def _resolve_tenant(request: Request) -> str | None:
+        auth = request.headers.get("Authorization", "")
+        if auth.lower().startswith("bearer "):
+            token = auth[7:].strip()
+            sub = verify_token(token)
+            if sub:
+                return sub
+            # A present-but-invalid token must never silently fall through to
+            # the dev header path.
+            return None
+
+        if settings.dev_mode:
+            raw = request.headers.get("X-Tenant-ID")
+            if raw and raw.strip():
+                return raw.strip()
+
+        return None

app/api/status.py

@@ -23,6 +23,22 @@ from app.models.schemas import (
 router = APIRouter()
 
 
+def _iso_utc(dt: datetime | None) -> str | None:
+    """Serialize a DB timestamp as UTC ISO-8601 with a ``Z`` suffix.
+
+    SQLite returns naive datetimes; we treat those as UTC because ORM
+    defaults use ``datetime.now(UTC)``. Without ``Z``, browsers parse the
+    value as local time and display the wrong clock time.
+    """
+    if dt is None:
+        return None
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=UTC)
+    else:
+        dt = dt.astimezone(UTC)
+    return dt.isoformat().replace("+00:00", "Z")
+
+
 def _utc_age_seconds(now_utc: datetime, then: datetime | None) -> float:
     """Compute age in seconds, tolerant of naive DB datetimes.
 
@@ -68,7 +84,7 @@ async def document_info(
         "filename": doc.filename,
         "status": doc.status.value,
         "chunk_count": chunk_count,
-        "created_at": doc.created_at.isoformat(),
+        "created_at": _iso_utc(doc.created_at),
         "survey_level": doc.survey_level,
     }
 
@@ -130,6 +146,14 @@ async def list_documents(
         .offset(offset)
     )
     rows = result.scalars().all()
+
+    def _file_size(path_str: str) -> int | None:
+        try:
+            p = Path(path_str)
+            return p.stat().st_size if p.is_file() else None
+        except OSError:
+            return None
+
     return {
         "tenant_id": tenant_id,
         "limit": limit,
@@ -139,8 +163,15 @@ async def list_documents(
                 "document_id": d.id,
                 "filename": d.filename,
                 "status": d.status.value,
-                "created_at": d.created_at.isoformat(),
+                "created_at": _iso_utc(d.created_at),
+                "updated_at": _iso_utc(d.updated_at),
                 "survey_level": d.survey_level,
+                "document_purpose": (
+                    d.document_purpose.value
+                    if hasattr(d.document_purpose, "value")
+                    else d.document_purpose
+                ),
+                "file_size": _file_size(d.file_path),
                 "error": d.error_message,
             }
             for d in rows

app/api/upload.py

@@ -10,13 +10,13 @@ from pathlib import Path
 from typing import Annotated
 
 from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile
-from sqlalchemy import delete, func, select
+from sqlalchemy import delete, select, update
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.api.rate_limit import check_read
 from app.config import settings
 from app.db.database import get_db
-from app.db.models import Document, IngestStatus, Report
+from app.db.models import Document, IngestStatus, Report, ReportStatus
 from app.ingest.schedule import schedule_ingest
 from app.ingest.zip_extract import extract_reference_documents
 from app.services.photo_policy_corpus import invalidate_tenant_photo_policy_cache
@@ -24,6 +24,7 @@ from app.models.schemas import (
     BatchUploadItem,
     BulkUploadResponse,
     DocumentDeleteResponse,
+    DocumentReingestResponse,
     DocumentSurveyLevelResponse,
     DocumentSurveyLevelUpdate,
     UploadResponse,
@@ -98,11 +99,17 @@ def _parse_optional_survey_level(raw: str | None) -> int | None:
 async def upload_file(
     request: Request,
     file: UploadFile = File(...),
-    tenant_id: str = Form(...),
+    tenant_id: Annotated[str | None, Form()] = None,
     survey_level: Annotated[str | None, Form()] = None,
     db: AsyncSession = Depends(get_db),
 ) -> UploadResponse:
-    """Accept a single document upload and schedule async ingestion."""
+    """Accept a single document upload and schedule async ingestion.
+
+    The owning tenant is taken from the authenticated request, never from the
+    client-supplied form field — uploading into another tenant's library is not
+    possible.
+    """
+    tenant_id = request.state.tenant_id
     suffix = Path(file.filename or "").suffix.lower()
     if suffix not in _ALLOWED_SUFFIXES:
         raise HTTPException(
@@ -137,7 +144,7 @@ async def upload_file(
 async def upload_batch(
     request: Request,
     files: list[UploadFile] = File(...),
-    tenant_id: str = Form(...),
+    tenant_id: Annotated[str | None, Form()] = None,
     survey_level: Annotated[str | None, Form()] = None,
     db: AsyncSession = Depends(get_db),
 ) -> BulkUploadResponse:
@@ -155,6 +162,7 @@ async def upload_batch(
     For very large libraries (millions of files), split work across multiple
     batch requests and/or several ZIPs — each stays within ``max_upload_batch_files``.
     """
+    tenant_id = request.state.tenant_id
     if not files:
         raise HTTPException(status_code=422, detail="No files uploaded")
 
@@ -300,6 +308,130 @@ async def patch_document_survey_level(
     return DocumentSurveyLevelResponse(document_id=document_id, survey_level=doc.survey_level)
 
 
+async def _requeue_document(db: AsyncSession, doc: Document) -> bool:
+    """Delete a document's stale chunks and re-queue it through the new pipeline.
+
+    Returns True when queued, False when the source file is missing on disk.
+    Clearing the old vectors first prevents duplicate chunks (old flattened +
+    new table-aware) from coexisting in the index.
+    """
+    path = Path(doc.file_path)
+    if not path.is_file():
+        return False
+    try:
+        from app.vectorstore.factory import get_vectorstore
+
+        get_vectorstore().delete_document(doc.id)
+    except Exception as exc:  # noqa: BLE001 — stale-chunk cleanup is best-effort
+        logger.warning("Vector store delete failed during reingest doc=%s: %s", doc.id, exc)
+    doc.status = IngestStatus.pending
+    doc.error_message = None
+    schedule_ingest(doc_id=doc.id, file_path=path)
+    return True
+
+
+async def _active_report_doc_ids(db: AsyncSession, document_ids: list[str]) -> set[str]:
+    """Document ids that currently have a report pending/generating against them."""
+    if not document_ids:
+        return set()
+    res = await db.execute(
+        select(Report.document_id)
+        .where(Report.document_id.in_(document_ids))
+        .where(Report.status.in_((ReportStatus.pending, ReportStatus.generating)))
+    )
+    return {row[0] for row in res.all() if row[0]}
+
+
+@router.post(
+    "/documents/{document_id}/reingest",
+    response_model=DocumentReingestResponse,
+    summary="Re-ingest one document through the current parser/chunker",
+)
+async def reingest_document(
+    document_id: str,
+    request: Request,
+    db: AsyncSession = Depends(get_db),
+    _: None = Depends(check_read),
+) -> DocumentReingestResponse:
+    """Re-process a single uploaded document through the latest ingestion pipeline.
+
+    Use this after pipeline upgrades (e.g. table-aware parsing / chunking) so the
+    document's chunks reflect the new logic. Blocked with HTTP 409 while a report
+    is actively generating from this document.
+    """
+    tenant_id: str = request.state.tenant_id
+    doc = await db.get(Document, document_id)
+    if doc is None or doc.tenant_id != tenant_id:
+        raise HTTPException(status_code=404, detail="Document not found")
+
+    if await _active_report_doc_ids(db, [document_id]):
+        raise HTTPException(
+            status_code=409,
+            detail="A report is still generating from this document. Wait for it to finish before re-ingesting.",
+        )
+
+    queued = await _requeue_document(db, doc)
+    await db.commit()
+    if not queued:
+        return DocumentReingestResponse(
+            queued=0,
+            skipped_missing_file=1,
+            detail="Source file is no longer on disk; cannot re-ingest.",
+        )
+    return DocumentReingestResponse(
+        queued=1,
+        document_ids=[document_id],
+        detail="Document re-queued for ingestion through the current pipeline.",
+    )
+
+
+@router.post(
+    "/documents/reingest",
+    response_model=DocumentReingestResponse,
+    summary="Re-ingest all of the tenant's documents through the current pipeline",
+)
+async def reingest_all_documents(
+    request: Request,
+    db: AsyncSession = Depends(get_db),
+    _: None = Depends(check_read),
+) -> DocumentReingestResponse:
+    """Re-process every uploaded document for the tenant through the latest pipeline.
+
+    Documents with a report actively generating against them are skipped (not
+    blocking the whole batch). This is the one-shot "activate the new parser/
+    chunker on my existing library" action.
+    """
+    tenant_id: str = request.state.tenant_id
+    res = await db.execute(select(Document).where(Document.tenant_id == tenant_id))
+    docs = list(res.scalars().all())
+    if not docs:
+        return DocumentReingestResponse(queued=0, detail="No documents to re-ingest.")
+
+    active = await _active_report_doc_ids(db, [d.id for d in docs])
+    queued_ids: list[str] = []
+    skipped_active = 0
+    skipped_missing = 0
+    for doc in docs:
+        if doc.id in active:
+            skipped_active += 1
+            continue
+        if await _requeue_document(db, doc):
+            queued_ids.append(doc.id)
+        else:
+            skipped_missing += 1
+    await db.commit()
+    return DocumentReingestResponse(
+        queued=len(queued_ids),
+        document_ids=queued_ids,
+        skipped_active=skipped_active,
+        skipped_missing_file=skipped_missing,
+        detail=(
+            f"Re-queued {len(queued_ids)} document(s) through the current pipeline; "
+            f"skipped {skipped_active} actively-generating and {skipped_missing} missing-file."
+        ),
+    )
+
+
 @router.delete(
     "/documents/{document_id}",
     response_model=DocumentDeleteResponse,
@@ -313,25 +445,39 @@ async def delete_uploaded_document(
 ) -> DocumentDeleteResponse:
     """Delete an uploaded file, its DB row, and all vector-index chunks for this document.
 
-    Blocked with HTTP 409 when any report still references the document (FK integrity).
+    Finished reports that used this document are **detached** (their
+    ``document_id`` is set to NULL) so their generated content is preserved while
+    the source file is removed. Deletion is blocked with HTTP 409 only when a
+    report is still actively generating against this document — pulling the
+    source mid-job would corrupt the run.
     """
     tenant_id: str = request.state.tenant_id
     doc = await db.get(Document, document_id)
     if doc is None or doc.tenant_id != tenant_id:
         raise HTTPException(status_code=404, detail="Document not found")
 
-    cnt = await db.execute(
-        select(func.count()).select_from(Report).where(Report.document_id == document_id)
+    active = await db.execute(
+        select(Report.id)
+        .where(Report.document_id == document_id)
+        .where(Report.status.in_((ReportStatus.pending, ReportStatus.generating)))
+        .limit(1)
     )
-    if int(cnt.scalar_one() or 0) > 0:
+    if active.first() is not None:
         raise HTTPException(
             status_code=409,
             detail=(
-                "This file is still linked to one or more reports. "
-                "Finish or abandon those jobs first, or upload replacements under a new document."
+                "A report is still generating from this document. "
+                "Wait for it to finish (or abandon it) before deleting the source file."
             ),
         )
 
+    # Detach finished/failed reports so they survive without the source file.
+    await db.execute(
+        update(Report)
+        .where(Report.document_id == document_id)
+        .values(document_id=None)
+    )
+
     try:
         from app.vectorstore.factory import get_vectorstore
 

app/chunking/splitter.py

@@ -15,18 +15,31 @@ Compared to master branch:
 """
 
 import logging
+import re
 
 import tiktoken
 from langchain_core.documents import Document
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 
 from app.config import settings
+from app.ingest.ocr_normalize import TABLE_CLOSE, TABLE_OPEN
 
 logger = logging.getLogger(__name__)
 
 # Shared tiktoken encoding used for both splitting and prompt token counting
 _ENCODING_NAME = "cl100k_base"
 
+# Sentence-aware separator ladder: the recursive splitter tries these in order,
+# so it prefers paragraph -> sentence boundaries and only falls back to word /
+# character splits when a single sentence exceeds the token budget. This is what
+# keeps chunks from being cut mid-sentence.
+_SEPARATORS = ["\n\n", "\n", ". ", "? ", "! ", "; ", ", ", " ", ""]
+
+_TABLE_BLOCK_RE = re.compile(
+    re.escape(TABLE_OPEN) + r".*?" + re.escape(TABLE_CLOSE),
+    re.DOTALL,
+)
+
 
 def count_tokens(text: str) -> int:
     """Count the number of tiktoken tokens in ``text``.
@@ -70,7 +83,7 @@ def build_splitter(
         encoding_name=_ENCODING_NAME,
         chunk_size=chunk_size or settings.chunk_size,
         chunk_overlap=chunk_overlap or settings.chunk_overlap,
-        separators=["\n\n", "\n", ". ", " ", ""],
+        separators=_SEPARATORS,
     )
 
 
@@ -101,7 +114,9 @@ def split_documents(
         # Each chunk has page_content ≤ settings.chunk_size tokens
     """
     splitter = build_splitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
-    chunks = splitter.split_documents(documents)
+    chunks: list[Document] = []
+    for doc in documents:
+        chunks.extend(_split_one(doc, splitter))
     logger.debug(
         "Split %d document(s) into %d chunks (size=%d, overlap=%d)",
         len(documents),
@@ -110,3 +125,37 @@ def split_documents(
         chunk_overlap or settings.chunk_overlap,
     )
     return chunks
+
+
+def _split_one(
+    doc: Document, splitter: RecursiveCharacterTextSplitter
+) -> list[Document]:
+    """Split one document, keeping each ``[TABLE]…[/TABLE]`` block as one chunk.
+
+    Tables carry row/column meaning that must never be cut across chunk
+    boundaries, so they are emitted intact (tagged ``section_type='table'``)
+    while the surrounding prose is split sentence-aware. Document order is
+    preserved.
+    """
+    content = doc.page_content or ""
+    if TABLE_OPEN not in content:
+        return splitter.split_documents([doc])
+
+    out: list[Document] = []
+    cursor = 0
+    for m in _TABLE_BLOCK_RE.finditer(content):
+        prose = content[cursor:m.start()].strip()
+        if prose:
+            out.extend(
+                splitter.split_documents([Document(page_content=prose, metadata=dict(doc.metadata))])
+            )
+        table_meta = dict(doc.metadata)
+        table_meta["section_type"] = "table"
+        out.append(Document(page_content=m.group(0).strip(), metadata=table_meta))
+        cursor = m.end()
+    tail = content[cursor:].strip()
+    if tail:
+        out.extend(
+            splitter.split_documents([Document(page_content=tail, metadata=dict(doc.metadata))])
+        )
+    return out

app/config.py

@@ -316,6 +316,35 @@ class Settings(BaseSettings):
         description="TTL for cached survey-tier corpus profiles built from knowledge_base_dirs (seconds).",
     )
     chunk_size: int = Field(default=500, ge=100, le=2000)
+
+    # ── Citation-grounded extraction (anti-hallucination layer) ──────────────
+    enable_citation_extraction: bool = Field(
+        default=True,
+        description=(
+            "Run schema-constrained, citation-grounded extraction + contradiction "
+            "audit alongside generation. Findings unsupported by retrieved source "
+            "spans are dropped and surfaced as a per-section audit (confidence, "
+            "contradictions, dropped claims). Adds one deterministic LLM call per "
+            "section; degrades to a no-op when no OpenAI key is configured."
+        ),
+    )
+    extraction_max_tokens: int = Field(
+        default=1500,
+        ge=256,
+        le=8000,
+        description="Max output tokens for the deterministic extraction call.",
+    )
+    rag_use_full_tenant_library: bool = Field(
+        default=True,
+        description=(
+            "When true, report generation retrieves from ALL of the tenant's "
+            "ingested report-source documents (old + new uploads), not only the "
+            "file attached when the report was created. The report's own upload "
+            "is still prioritised. style_corpus (past completed reports) remain "
+            "excluded from factual retrieval, and tenant isolation is preserved. "
+            "Set false to restore strict per-report document isolation."
+        ),
+    )
     max_bullets_per_section: int = Field(
         default=800,
         ge=20,
@@ -665,6 +694,12 @@ class Settings(BaseSettings):
 
     # ── Security ─────────────────────────────────────────────────────────────
     tenant_secret_key: str = Field(default="dev-secret-change-me")
+    auth_token_ttl_seconds: int = Field(
+        default=7 * 24 * 3600,
+        ge=300,
+        le=90 * 24 * 3600,
+        description="Lifetime of an issued tenant bearer token, in seconds (default 7 days).",
+    )
     dev_mode: bool = Field(default=False)
     # CORS: list specific origins in production, e.g. ["https://app.example.com"]
     # The wildcard ["*"] is safe here because allow_credentials=False in main.py

app/db/database.py

@@ -42,8 +42,21 @@ def parallel_section_writes_safe() -> bool:
 
 
 def multi_section_parallel_enabled() -> bool:
-    """Parallel multi-section jobs (in-process or Temporal) when async pipeline is on."""
-    return bool(settings.enable_async_pipeline and parallel_section_writes_safe())
+    """Parallel multi-section jobs (in-process or Temporal).
+
+    Enabled by either the async pipeline OR the dedicated
+    ``enable_parallel_section_generation`` flag, and only when concurrent
+    section writes are safe for the active DB backend (SQLite requires
+    ``allow_sqlite_parallel_sections``). This only governs the Temporal
+    workflow request's ``parallel_sections`` field; the default (non-Temporal)
+    job path is unaffected.
+    """
+    if not parallel_section_writes_safe():
+        return False
+    return bool(
+        settings.enable_async_pipeline
+        or settings.enable_parallel_section_generation
+    )
 
 
 def effective_section_concurrency() -> int:
@@ -198,12 +211,71 @@ def _postgres_add_column_if_missing(
     connection.execute(text(f"ALTER TABLE {table} ADD COLUMN IF NOT EXISTS {ddl_suffix}"))
 
 
+def _sqlite_drop_not_null(connection: object, table: str, column: str) -> None:
+    """Relax a ``NOT NULL`` constraint on one SQLite column via a table rebuild.
+
+    SQLite cannot ``ALTER COLUMN``; the standard recipe is to recreate the table
+    with the relaxed definition and copy rows. Idempotent: no-op when the column
+    is already nullable or absent. FK declarations are intentionally dropped on
+    rebuild (we want lenient deletes), and the ``ix_<table>_tenant_id`` helper
+    index is recreated.
+    """
+    from sqlalchemy import text
+
+    info = connection.execute(text(f"PRAGMA table_info({table})")).fetchall()  # type: ignore[attr-defined]
+    if not info:
+        return
+    target = next((row for row in info if row[1] == column), None)
+    if target is None or int(target[3]) == 0:
+        # column missing, or already nullable
+        return
+
+    col_defs: list[str] = []
+    for _cid, name, ctype, notnull, dflt, pk in info:
+        piece = f'"{name}" {ctype or ""}'.rstrip()
+        if int(pk):
+            piece += " PRIMARY KEY"
+        if int(notnull) and name != column:
+            piece += " NOT NULL"
+        if dflt is not None:
+            piece += f" DEFAULT {dflt}"
+        col_defs.append(piece)
+    col_names = ", ".join(f'"{row[1]}"' for row in info)
+    tmp = f"{table}__migrate_tmp"
+
+    connection.execute(text("PRAGMA foreign_keys=OFF"))  # type: ignore[attr-defined]
+    connection.execute(text(f'DROP TABLE IF EXISTS "{tmp}"'))  # type: ignore[attr-defined]
+    connection.execute(text(f'CREATE TABLE "{tmp}" ({", ".join(col_defs)})'))  # type: ignore[attr-defined]
+    connection.execute(  # type: ignore[attr-defined]
+        text(f'INSERT INTO "{tmp}" ({col_names}) SELECT {col_names} FROM "{table}"')
+    )
+    connection.execute(text(f'DROP TABLE "{table}"'))  # type: ignore[attr-defined]
+    connection.execute(text(f'ALTER TABLE "{tmp}" RENAME TO "{table}"'))  # type: ignore[attr-defined]
+    connection.execute(  # type: ignore[attr-defined]
+        text(f'CREATE INDEX IF NOT EXISTS "ix_{table}_tenant_id" ON "{table}" (tenant_id)')
+    )
+
+
+def _postgres_drop_not_null(connection: object, table: str, column: str) -> None:
+    """Relax a ``NOT NULL`` constraint on PostgreSQL (idempotent)."""
+    from sqlalchemy import inspect, text
+
+    insp = inspect(connection)
+    if not insp.has_table(table):
+        return
+    col = next((c for c in insp.get_columns(table) if c["name"] == column), None)
+    if col is None or col.get("nullable", True):
+        return
+    connection.execute(text(f'ALTER TABLE {table} ALTER COLUMN {column} DROP NOT NULL'))
+
+
 async def migrate_schema() -> None:
     """Apply lightweight schema upgrades when models gain nullable columns."""
     engine = get_engine()
 
     def _upgrade(sync_conn: object) -> None:
         if is_sqlite_database():
+            _sqlite_drop_not_null(sync_conn, "reports", "document_id")
             _sqlite_add_column_if_missing(
                 sync_conn, "documents", "survey_level", "survey_level INTEGER"
             )
@@ -232,6 +304,7 @@ async def migrate_schema() -> None:
                 "document_purpose VARCHAR(32) NOT NULL DEFAULT 'report_source'",
             )
         else:
+            _postgres_drop_not_null(sync_conn, "reports", "document_id")
             _postgres_add_column_if_missing(
                 sync_conn,
                 "reports",

app/db/models.py

@@ -101,8 +101,11 @@ class Report(Base):
         String(36), primary_key=True, default=lambda: str(uuid.uuid4())
     )
     tenant_id: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
-    document_id: Mapped[str] = mapped_column(
-        String(36), ForeignKey("documents.id"), nullable=False
+    # Nullable so a finished report's source document can be deleted: on delete
+    # the referencing terminal reports are detached (document_id -> NULL) rather
+    # than blocking removal. Active jobs (pending/generating) still block deletion.
+    document_id: Mapped[str | None] = mapped_column(
+        String(36), ForeignKey("documents.id"), nullable=True
     )
     status: Mapped[ReportStatus] = mapped_column(
         Enum(ReportStatus), default=ReportStatus.pending
@@ -178,3 +181,23 @@ class ReportSectionPhoto(Base):
     content_type: Mapped[str] = mapped_column(String(128), nullable=False)
 
     created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
+
+
+class Tenant(Base):
+    """An authenticated tenant (user/workspace) for access control + isolation.
+
+    The ``id`` is the tenant identifier carried on every request as the
+    cryptographically-verified subject of a signed bearer token. The passphrase
+    is never stored — only a PBKDF2-HMAC-SHA256 hash and its per-tenant salt.
+    """
+
+    __tablename__ = "tenants"
+
+    id: Mapped[str] = mapped_column(String(128), primary_key=True)
+    password_hash: Mapped[str] = mapped_column(String(256), nullable=False)
+    password_salt: Mapped[str] = mapped_column(String(64), nullable=False)
+    pbkdf2_iterations: Mapped[int] = mapped_column(Integer, nullable=False, default=200_000)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
+    last_login_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True, default=None
+    )

app/extraction/__init__.py

@@ -0,0 +1,47 @@
+"""Citation-grounded extraction & verification core.
+
+This package turns retrieved source spans into schema-constrained, evidence-backed
+survey findings and audits them for hallucination and contradiction. It is the
+deterministic "no invented facts" layer that sits between retrieval and rendering.
+
+Design tenets (do not relax these without a regression test):
+
+* Extraction over summarization — findings are copied/derived from cited spans.
+* Grounding over fluency — every claim must map to a literal source span.
+* Determinism over creativity — temperature=0, top_p=1, fixed seed.
+* Citation validation over freeform generation — unsupported claims are dropped.
+
+The pure-Python validation/contradiction layers (``citation_validator`` and
+``contradiction``) have NO network dependency and are fully unit-testable
+offline. Only :mod:`app.extraction.extractor` calls the LLM.
+"""
+
+from app.extraction.schemas import (
+    AtomicClaim,
+    ClaimEvidence,
+    ClaimType,
+    ClaimVerification,
+    ConditionRating,
+    Contradiction,
+    ContradictionKind,
+    EvidenceSpan,
+    GroundedClaim,
+    SectionExtraction,
+    SupportLevel,
+    SurveyFinding,
+)
+
+__all__ = [
+    "AtomicClaim",
+    "ClaimEvidence",
+    "ClaimType",
+    "ClaimVerification",
+    "ConditionRating",
+    "Contradiction",
+    "ContradictionKind",
+    "EvidenceSpan",
+    "GroundedClaim",
+    "SectionExtraction",
+    "SupportLevel",
+    "SurveyFinding",
+]

app/extraction/citation_validator.py

@@ -0,0 +1,229 @@
+"""Deterministic citation & source-span validation.
+
+This module enforces the core guarantee: **every claim must map to a literal
+source span.** It is pure Python with no network dependency, so it can be unit
+tested offline and run as a hard gate on any LLM output.
+
+Validation policy (audit-safe — bias toward dropping, never inventing):
+
+* A cited ``chunk_id`` must exist in the retrieved evidence pool. Fabricated
+  citations invalidate the span.
+* An evidence span's text must actually appear in its cited chunk (verbatim
+  after normalization, or via high token-containment to tolerate OCR drift).
+* Every NUMBER, MONETARY AMOUNT, and flagged PROPER NOUN in the finding text
+  must be present in the cited evidence. This blocks altered ratings, fabricated
+  totals, and entity substitution ("London plane tree" -> "cedar tree").
+* Escalation/severity words (catastrophic, deadly, unsafe, ...) may only appear
+  if present verbatim in the source.
+
+Anything failing a *critical* check is forced to ``NOT_FOUND`` and dropped.
+"""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+from collections.abc import Iterable, Mapping
+
+from app.extraction.schemas import SupportLevel, SurveyFinding
+
+# Severity / escalation vocabulary that must never be introduced by the model.
+# These may ONLY survive validation if present verbatim in the cited evidence.
+BANNED_SEVERITY_TERMS: frozenset[str] = frozenset({
+    "catastrophic", "deadly", "lethal", "unsafe", "dangerous", "hazardous",
+    "severe health", "health consequences", "environmental disaster", "disaster",
+    "life-threatening", "life threatening", "toxic", "fatal", "collapse imminent",
+})
+
+# Containment thresholds for whole-finding grounding (token overlap with evidence).
+_SUPPORTED_CONTAINMENT = 0.85
+_PARTIAL_CONTAINMENT = 0.55
+# Span-level containment for tolerating OCR/quote drift when a verbatim
+# substring match fails.
+_SPAN_CONTAINMENT = 0.90
+
+_STOPWORDS: frozenset[str] = frozenset({
+    "the", "a", "an", "and", "or", "of", "to", "in", "on", "at", "is", "are",
+    "was", "were", "be", "been", "being", "this", "that", "these", "those",
+    "with", "for", "as", "by", "from", "it", "its", "has", "have", "had",
+    "which", "but", "not", "no", "there", "their", "they", "we", "you", "i",
+    "will", "would", "should", "could", "may", "can", "also", "some", "any",
+})
+
+_WORD_RE = re.compile(r"[A-Za-z0-9£%.\-/]+")
+# Numbers, percentages, money, and ratings like "CR2"/"1"/"£12,500"/"3.5m".
+_NUMBER_RE = re.compile(r"£?\d[\d,]*(?:\.\d+)?%?")
+# TitleCase proper-noun runs of >=2 words, or a single word with >=2 capitals.
+_PROPER_NOUN_RE = re.compile(r"\b(?:[A-Z][a-z]+(?:\s+[A-Z][a-z]+)+)\b")
+
+
+def normalize_for_match(text: str) -> str:
+    """Lowercase, fold accents, normalize quotes/dashes, collapse whitespace."""
+    if not text:
+        return ""
+    text = unicodedata.normalize("NFKD", text)
+    text = "".join(c for c in text if not unicodedata.combining(c))
+    text = (
+        text.replace("\u2019", "'").replace("\u2018", "'")
+        .replace("\u201c", '"').replace("\u201d", '"')
+        .replace("\u2013", "-").replace("\u2014", "-")
+        .replace("\u00a0", " ")
+    )
+    text = text.lower()
+    text = re.sub(r"\s+", " ", text)
+    return text.strip()
+
+
+def content_tokens(text: str) -> list[str]:
+    """Tokens used for containment scoring (stopwords removed)."""
+    norm = normalize_for_match(text)
+    return [t for t in _WORD_RE.findall(norm) if t not in _STOPWORDS and len(t) > 1]
+
+
+def _containment(needle_tokens: Iterable[str], haystack_tokens: set[str]) -> float:
+    needle = list(needle_tokens)
+    if not needle:
+        return 1.0
+    hits = sum(1 for t in needle if t in haystack_tokens)
+    return hits / len(needle)
+
+
+def _coerce_pool(pool: Mapping[str, str] | Iterable[object]) -> dict[str, str]:
+    """Accept a ``{chunk_id: text}`` map or an iterable of SearchResult-like rows."""
+    if isinstance(pool, Mapping):
+        return {str(k): str(v) for k, v in pool.items()}
+    out: dict[str, str] = {}
+    for row in pool:
+        cid = getattr(row, "chunk_id", None)
+        txt = getattr(row, "text", None)
+        if cid is not None and txt is not None:
+            out[str(cid)] = str(txt)
+    return out
+
+
+def span_in_chunk(span_text: str, chunk_text: str) -> bool:
+    """True when ``span_text`` is grounded in ``chunk_text``.
+
+    Verbatim (normalized) substring match first; falls back to high
+    token-containment to tolerate OCR/whitespace/quote drift while still
+    rejecting genuinely absent content.
+    """
+    if not span_text or not chunk_text:
+        return False
+    nspan = normalize_for_match(span_text)
+    nchunk = normalize_for_match(chunk_text)
+    if not nspan:
+        return False
+    if nspan in nchunk:
+        return True
+    chunk_tok = set(_WORD_RE.findall(nchunk))
+    span_tok = _WORD_RE.findall(nspan)
+    return _containment(span_tok, chunk_tok) >= _SPAN_CONTAINMENT
+
+
+def _numbers(text: str) -> set[str]:
+    return {m.replace(",", "") for m in _NUMBER_RE.findall(text or "")}
+
+
+def _proper_nouns(text: str) -> set[str]:
+    return {normalize_for_match(m) for m in _PROPER_NOUN_RE.findall(text or "")}
+
+
+def _banned_terms_present(text: str) -> set[str]:
+    norm = normalize_for_match(text)
+    return {term for term in BANNED_SEVERITY_TERMS if term in norm}
+
+
+def validate_finding(
+    finding: SurveyFinding,
+    pool: Mapping[str, str] | Iterable[object],
+) -> tuple[SupportLevel, list[str]]:
+    """Validate one finding against the retrieved evidence pool.
+
+    Returns the computed :class:`SupportLevel` and a list of human-readable
+    violation reasons (empty when fully supported). Mutates nothing.
+    """
+    pool_map = _coerce_pool(pool)
+    violations: list[str] = []
+
+    # 1. Spans must cite real chunks and actually appear in them.
+    valid_spans = []
+    for span in finding.evidence:
+        chunk_text = pool_map.get(span.chunk_id)
+        if chunk_text is None:
+            violations.append(f"citation to unknown chunk_id={span.chunk_id!r}")
+            continue
+        if not span_in_chunk(span.text, chunk_text):
+            violations.append(f"span not found in chunk {span.chunk_id!r}: {span.text[:60]!r}")
+            continue
+        valid_spans.append((span, chunk_text))
+
+    if not valid_spans:
+        return SupportLevel.NOT_FOUND, violations or ["no valid supporting spans"]
+
+    # Build the union of grounded evidence text from spans that validated.
+    evidence_text = " \n ".join(s.text for s, _ in valid_spans)
+    # Also allow grounding against the FULL cited chunk text (the span is a
+    # window into it; numbers/entities elsewhere in the same chunk still count).
+    evidence_text += " \n " + " \n ".join(ct for _, ct in valid_spans)
+    evidence_norm = normalize_for_match(evidence_text)
+    evidence_tok = set(_WORD_RE.findall(evidence_norm))
+    evidence_numbers = _numbers(evidence_text)
+    evidence_nouns = _proper_nouns(evidence_text)
+
+    # 2. CRITICAL: every number in the finding must be in the evidence.
+    for num in _numbers(finding.finding):
+        if num not in evidence_numbers:
+            violations.append(f"unsupported number/amount {num!r} not in evidence")
+
+    # 3. CRITICAL: severity escalation must exist in the source.
+    finding_banned = _banned_terms_present(finding.finding)
+    evidence_banned = _banned_terms_present(evidence_text)
+    for term in finding_banned - evidence_banned:
+        violations.append(f"unsupported severity term {term!r} not in evidence")
+
+    # 4. CRITICAL: proper nouns (materials, species, places) must not be substituted.
+    for noun in _proper_nouns(finding.finding):
+        if noun not in evidence_nouns and noun not in evidence_norm:
+            violations.append(f"unsupported/altered entity {noun!r} not in evidence")
+
+    has_critical = bool(violations)
+
+    # 5. Whole-finding token containment (catches paraphrased fabrication).
+    containment = _containment(content_tokens(finding.finding), evidence_tok)
+
+    if has_critical:
+        return SupportLevel.NOT_FOUND, violations
+    if containment >= _SUPPORTED_CONTAINMENT:
+        return SupportLevel.SUPPORTED, []
+    if containment >= _PARTIAL_CONTAINMENT:
+        return SupportLevel.PARTIAL, [f"partial grounding (containment={containment:.2f})"]
+    return SupportLevel.NOT_FOUND, [f"insufficient grounding (containment={containment:.2f})"]
+
+
+def validate_findings(
+    findings: list[SurveyFinding],
+    pool: Mapping[str, str] | Iterable[object],
+    *,
+    drop_partial: bool = False,
+) -> tuple[list[SurveyFinding], list[str]]:
+    """Validate every finding; return (kept_findings, dropped_claim_descriptions).
+
+    ``NOT_FOUND`` findings are always dropped. ``PARTIAL`` findings are kept
+    (with their downgraded support level) unless ``drop_partial`` is set, in
+    which case only fully ``SUPPORTED`` findings survive — the strictest,
+    audit-safe mode.
+    """
+    kept: list[SurveyFinding] = []
+    dropped: list[str] = []
+    for f in findings:
+        support, reasons = validate_finding(f, pool)
+        f.support = support
+        if support == SupportLevel.NOT_FOUND:
+            dropped.append(f"{f.element}: {'; '.join(reasons)}")
+            continue
+        if support == SupportLevel.PARTIAL and drop_partial:
+            dropped.append(f"{f.element}: partial grounding dropped under strict mode")
+            continue
+        kept.append(f)
+    return kept, dropped

app/extraction/contradiction.py

@@ -0,0 +1,186 @@
+"""Deterministic contradiction & duplicate audit (STEP 7).
+
+Pure-Python verifier that compares extracted findings and flags logical
+inconsistencies, tracing each back to the findings (and therefore source spans)
+that produced it. No network dependency — fully unit-testable.
+
+Detected classes:
+
+* ``RATING_CONFLICT``  — same element given two different condition ratings.
+* ``CONDITION``        — same element described both satisfactory and defective.
+* ``OPERATIONAL``      — same element described both operational and non-operational.
+* ``DUPLICATE``        — near-identical findings for the same element.
+* ``MUTUALLY_EXCLUSIVE`` — explicit antonym pairs on the same element.
+
+Resolution policy: the verifier never fabricates a merged truth. It keeps the
+finding with the stronger evidence support and records the conflict for audit.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import defaultdict
+
+from app.extraction.citation_validator import normalize_for_match
+from app.extraction.schemas import (
+    Contradiction,
+    ContradictionKind,
+    ConditionRating,
+    SupportLevel,
+    SurveyFinding,
+)
+
+# Lexical polarity cues. Kept conservative to avoid false positives.
+_DEFECTIVE_CUES = (
+    "defective", "defect", "failed", "failing", "broken", "cracked", "leak",
+    "leaking", "damp", "rot", "rotten", "corroded", "deteriorated", "missing",
+    "damaged", "unsafe", "not satisfactory", "poor condition", "in need of repair",
+)
+_SATISFACTORY_CUES = (
+    "satisfactory", "good condition", "sound", "no defect", "no defects",
+    "well maintained", "intact", "serviceable", "in good order", "no repair",
+)
+_NON_OPERATIONAL_CUES = (
+    "not operational", "non-operational", "inoperable", "not working",
+    "not functioning", "out of order", "does not work", "non operational",
+    "not in working order",
+)
+_OPERATIONAL_CUES = (
+    "operational", "working", "functioning", "in working order", "operates correctly",
+    "fully functional", "in good working order",
+)
+
+_WORD_RE = re.compile(r"[a-z0-9]+")
+
+
+def _element_key(f: SurveyFinding) -> str:
+    """Normalized (section, element) key for grouping findings about one thing."""
+    return f"{normalize_for_match(f.section)}::{normalize_for_match(f.element)}"
+
+
+def _has_any(text: str, cues: tuple[str, ...]) -> bool:
+    norm = normalize_for_match(text)
+    return any(cue in norm for cue in cues)
+
+
+def _polarity(text: str) -> tuple[bool, bool, bool, bool]:
+    """Return (defective, satisfactory, operational, non_operational) cue presence."""
+    return (
+        _has_any(text, _DEFECTIVE_CUES),
+        _has_any(text, _SATISFACTORY_CUES),
+        _has_any(text, _OPERATIONAL_CUES),
+        _has_any(text, _NON_OPERATIONAL_CUES),
+    )
+
+
+def _jaccard(a: str, b: str) -> float:
+    ta = set(_WORD_RE.findall(normalize_for_match(a)))
+    tb = set(_WORD_RE.findall(normalize_for_match(b)))
+    if not ta or not tb:
+        return 0.0
+    return len(ta & tb) / len(ta | tb)
+
+
+def _support_rank(level: SupportLevel) -> int:
+    return {SupportLevel.SUPPORTED: 2, SupportLevel.PARTIAL: 1, SupportLevel.NOT_FOUND: 0}[level]
+
+
+def _keep_stronger(a: tuple[int, SurveyFinding], b: tuple[int, SurveyFinding]) -> tuple[int, int]:
+    """Return (keep_index, drop_index) preferring stronger evidence support."""
+    (ia, fa), (ib, fb) = a, b
+    if _support_rank(fa.support) >= _support_rank(fb.support):
+        return ia, ib
+    return ib, ia
+
+
+def audit_contradictions(
+    findings: list[SurveyFinding],
+    *,
+    duplicate_threshold: float = 0.9,
+) -> tuple[list[SurveyFinding], list[Contradiction]]:
+    """Detect contradictions/duplicates and return (resolved_findings, reports).
+
+    Resolution keeps the evidence-stronger finding for hard contradictions and
+    collapses near-duplicates. Every action is recorded in the returned
+    :class:`Contradiction` list for the audit trail. The input list is not
+    mutated; a filtered copy is returned.
+    """
+    contradictions: list[Contradiction] = []
+    drop: set[int] = set()
+
+    groups: dict[str, list[int]] = defaultdict(list)
+    for i, f in enumerate(findings):
+        groups[_element_key(f)].append(i)
+
+    for _key, idxs in groups.items():
+        for pos_a in range(len(idxs)):
+            for pos_b in range(pos_a + 1, len(idxs)):
+                ia, ib = idxs[pos_a], idxs[pos_b]
+                if ia in drop or ib in drop:
+                    continue
+                fa, fb = findings[ia], findings[ib]
+
+                # Rating conflict (both real ratings, different values).
+                real = {ConditionRating.CR1, ConditionRating.CR2, ConditionRating.CR3}
+                if (
+                    fa.condition_rating in real
+                    and fb.condition_rating in real
+                    and fa.condition_rating != fb.condition_rating
+                ):
+                    keep, dropped = _keep_stronger((ia, fa), (ib, fb))
+                    drop.add(dropped)
+                    contradictions.append(Contradiction(
+                        kind=ContradictionKind.RATING_CONFLICT,
+                        element=fa.element,
+                        detail=(
+                            f"Condition rating {fa.condition_rating.value} vs "
+                            f"{fb.condition_rating.value} for the same element."
+                        ),
+                        finding_indices=[ia, ib],
+                        resolution=f"kept finding #{keep} (stronger evidence), dropped #{dropped}",
+                    ))
+                    continue
+
+                da, sa, oa, na = _polarity(fa.finding)
+                db, sb, ob, nb = _polarity(fb.finding)
+
+                # Satisfactory vs defective.
+                if (da and sb) or (sa and db):
+                    keep, dropped = _keep_stronger((ia, fa), (ib, fb))
+                    drop.add(dropped)
+                    contradictions.append(Contradiction(
+                        kind=ContradictionKind.CONDITION,
+                        element=fa.element,
+                        detail="One finding describes the element as defective, the other as satisfactory.",
+                        finding_indices=[ia, ib],
+                        resolution=f"kept finding #{keep} (stronger evidence), dropped #{dropped}",
+                    ))
+                    continue
+
+                # Operational vs non-operational.
+                if (oa and nb) or (na and ob):
+                    keep, dropped = _keep_stronger((ia, fa), (ib, fb))
+                    drop.add(dropped)
+                    contradictions.append(Contradiction(
+                        kind=ContradictionKind.OPERATIONAL,
+                        element=fa.element,
+                        detail="One finding states operational, the other non-operational.",
+                        finding_indices=[ia, ib],
+                        resolution=f"kept finding #{keep} (stronger evidence), dropped #{dropped}",
+                    ))
+                    continue
+
+                # Near-duplicate.
+                if _jaccard(fa.finding, fb.finding) >= duplicate_threshold:
+                    keep, dropped = _keep_stronger((ia, fa), (ib, fb))
+                    drop.add(dropped)
+                    contradictions.append(Contradiction(
+                        kind=ContradictionKind.DUPLICATE,
+                        element=fa.element,
+                        detail="Near-identical findings for the same element.",
+                        finding_indices=[ia, ib],
+                        resolution=f"collapsed to finding #{keep}, dropped #{dropped}",
+                    ))
+
+    resolved = [f for i, f in enumerate(findings) if i not in drop]
+    return resolved, contradictions

app/extraction/domain_scope.py

@@ -0,0 +1,201 @@
+"""Section-domain scoping to prevent cross-section contamination (STEP 6).
+
+Roofing prompts must only see roofing evidence; drainage only drainage, etc.
+This module is a pure, deterministic keyword classifier — no embeddings, no
+network — so the scoping decision is reproducible and unit-testable.
+
+Usage::
+
+    domain = classify_section("Roofing", template_id="E1")
+    roofing_only = scope_chunks(domain, retrieved_chunks)
+
+The classifier is intentionally conservative: a chunk with no clear domain
+signal is treated as ``GENERAL`` and is admissible to any section (so we never
+starve a section of context), but a chunk that clearly belongs to a *different*
+domain is excluded.
+"""
+
+from __future__ import annotations
+
+from app.extraction.citation_validator import normalize_for_match
+
+GENERAL = "general"
+
+# Domain → indicative terms. Order matters for deterministic tie-breaking
+# (earlier domains win ties). Terms are matched as normalized substrings, so
+# multi-word phrases are supported.
+DOMAIN_LEXICON: dict[str, frozenset[str]] = {
+    "roofing": frozenset({
+        "roof", "roofing", "slate", "tile", "tiling", "ridge", "hip", "valley",
+        "verge", "eaves", "soffit", "fascia", "felt", "flashing", "parapet",
+        "covering", "rafter", "purlin", "truss", "flat roof", "pitched roof",
+    }),
+    "chimney": frozenset({
+        "chimney", "chimney stack", "flaunching", "pot", "cowl", "flue",
+        "breast", "corbel", "pointing to the stack",
+    }),
+    "rainwater": frozenset({
+        "gutter", "guttering", "downpipe", "rainwater", "hopper", "rwp",
+        "rainwater goods", "fall pipe",
+    }),
+    "drainage": frozenset({
+        "drain", "drainage", "sewer", "manhole", "inspection chamber", "gully",
+        "soil pipe", "foul", "surface water", "septic", "soakaway", "below ground",
+    }),
+    "walls": frozenset({
+        "wall", "masonry", "brickwork", "blockwork", "render", "rendering",
+        "pointing", "cavity", "spalling", "cracking", "lintel", "dpc",
+        "damp proof course", "external wall", "load bearing",
+    }),
+    "dampness": frozenset({
+        "damp", "dampness", "moisture", "rising damp", "penetrating damp",
+        "condensation", "mould", "mold", "hygroscopic", "salts", "tide mark",
+    }),
+    "timber": frozenset({
+        "timber", "woodworm", "beetle", "wet rot", "dry rot", "joist", "decay",
+        "fungal", "infestation", "rot to the",
+    }),
+    "windows_doors": frozenset({
+        "window", "windows", "door", "doors", "glazing", "double glazing",
+        "frame", "casement", "sash", "joinery", "fenestration", "sill",
+    }),
+    "ceilings_floors": frozenset({
+        "ceiling", "floor", "flooring", "floorboard", "screed", "plaster",
+        "lath", "cornice", "skirting", "subfloor",
+    }),
+    "electrical": frozenset({
+        "electric", "electrical", "wiring", "consumer unit", "fuse box",
+        "rcd", "socket", "circuit", "earthing", "eicr", "fixed wiring",
+        "distribution board",
+    }),
+    "heating": frozenset({
+        "boiler", "heating", "radiator", "central heating", "flue", "gas",
+        "thermostat", "hot water", "cylinder", "underfloor heating", "combi",
+    }),
+    "plumbing": frozenset({
+        "plumbing", "pipework", "water supply", "stopcock", "waste pipe",
+        "mains water", "lead pipe", "tank", "overflow", "sanitary",
+    }),
+    "insulation_energy": frozenset({
+        "insulation", "epc", "energy performance", "sap", "u-value",
+        "thermal", "loft insulation", "cavity insulation", "energy efficiency",
+    }),
+    "grounds": frozenset({
+        "garden", "boundary", "fence", "fencing", "patio", "driveway", "path",
+        "tree", "hedge", "retaining wall", "outbuilding", "grounds",
+    }),
+    "services_other": frozenset({
+        "ventilation", "extractor", "smoke alarm", "carbon monoxide",
+        "asbestos", "fire", "security",
+    }),
+}
+
+# Map common RICS section titles / template hints to a canonical domain.
+_SECTION_ALIASES: dict[str, str] = {
+    "roof": "roofing", "roof coverings": "roofing", "main roof": "roofing",
+    "chimney stacks": "chimney", "chimneys": "chimney",
+    "rainwater pipes and gutters": "rainwater", "gutters": "rainwater",
+    "drainage": "drainage",
+    "main walls": "walls", "external walls": "walls", "walls": "walls",
+    "dampness": "dampness", "damp": "dampness",
+    "windows": "windows_doors", "doors": "windows_doors",
+    "ceilings": "ceilings_floors", "floors": "ceilings_floors",
+    "electricity": "electrical", "electrical": "electrical",
+    "heating": "heating", "gas": "heating",
+    "water": "plumbing", "plumbing": "plumbing",
+    "insulation": "insulation_energy", "energy efficiency": "insulation_energy",
+    "grounds": "grounds", "gardens": "grounds", "boundaries": "grounds",
+}
+
+
+def _score_domains(text: str) -> dict[str, int]:
+    norm = normalize_for_match(text)
+    if not norm:
+        return {}
+    scores: dict[str, int] = {}
+    for domain, terms in DOMAIN_LEXICON.items():
+        hits = 0
+        for term in terms:
+            # Count occurrences; multi-word terms matched as substrings.
+            if " " in term:
+                hits += norm.count(term)
+            else:
+                # word-ish boundary check to avoid 'gas' in 'gasket' etc.
+                hits += _count_word(norm, term)
+        if hits:
+            scores[domain] = hits
+    return scores
+
+
+def _count_word(haystack: str, word: str) -> int:
+    count = 0
+    start = 0
+    n = len(word)
+    while True:
+        idx = haystack.find(word, start)
+        if idx == -1:
+            break
+        before = haystack[idx - 1] if idx > 0 else " "
+        after = haystack[idx + n] if idx + n < len(haystack) else " "
+        if not before.isalnum() and not after.isalnum():
+            count += 1
+        start = idx + n
+    return count
+
+
+def classify_text(text: str) -> str:
+    """Return the dominant domain for ``text`` or :data:`GENERAL`."""
+    scores = _score_domains(text)
+    if not scores:
+        return GENERAL
+    # Highest score wins; ties broken by lexicon insertion order (stable).
+    best = max(DOMAIN_LEXICON.keys(), key=lambda d: scores.get(d, 0))
+    return best if scores.get(best, 0) > 0 else GENERAL
+
+
+def classify_section(section_name: str | None, template_id: str | None = None) -> str:
+    """Resolve a section title/template into a canonical domain.
+
+    Falls back to keyword classification of the section name, then
+    :data:`GENERAL`.
+    """
+    name = normalize_for_match(section_name or "")
+    if name in _SECTION_ALIASES:
+        return _SECTION_ALIASES[name]
+    for alias, domain in _SECTION_ALIASES.items():
+        if alias in name:
+            return domain
+    guessed = classify_text(section_name or "")
+    return guessed
+
+
+def scope_chunks(
+    section_domain: str,
+    chunks: list[object],
+    *,
+    strict: bool = True,
+) -> list[object]:
+    """Filter retrieved chunks to those admissible for ``section_domain``.
+
+    A chunk is admissible when its dominant domain equals ``section_domain`` or
+    is :data:`GENERAL`. Chunks clearly belonging to a *different* domain are
+    excluded. If filtering would remove everything (e.g. weak classification),
+    the original list is returned to avoid starving extraction — unless
+    ``strict`` is False, in which case the filtered (possibly empty) list is
+    always returned.
+
+    ``chunks`` are SearchResult-like objects exposing ``.text``.
+    """
+    if section_domain == GENERAL or not chunks:
+        return chunks
+    kept: list[object] = []
+    for c in chunks:
+        text = getattr(c, "text", "") or ""
+        cdom = classify_text(text)
+        if cdom == section_domain or cdom == GENERAL:
+            kept.append(c)
+    if not kept and strict:
+        # Never starve the extractor on a misclassification; better to extract
+        # from broader context than to silently drop the whole section.
+        return chunks
+    return kept

app/extraction/extractor.py

@@ -0,0 +1,249 @@
+"""Citation-grounded, schema-constrained section extraction (STEP 3/4/8).
+
+Pipeline for one section:
+
+    section-scoped chunks
+      -> deterministic LLM extraction (temperature=0, top_p=1, seed, JSON mode)
+      -> Pydantic schema validation (closed enums, mandatory evidence)
+      -> citation/source-span validation (drop unsupported claims)
+      -> contradiction audit (resolve conflicts, collapse duplicates)
+      -> SectionExtraction (with confidence + dropped-claim audit trail)
+
+The LLM is the only non-deterministic component, and it is pinned as hard as
+the API allows. Everything after it is pure, deterministic Python.
+
+This module is additive: it does not alter the existing generation path. Callers
+opt in (see ``settings.enable_citation_extraction``). It degrades safely to an
+empty extraction when no API key is configured, so import and unit tests never
+require network access.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+
+from app.config import settings
+from app.extraction.citation_validator import validate_findings
+from app.extraction.contradiction import audit_contradictions
+from app.extraction.domain_scope import GENERAL, classify_section, scope_chunks
+from app.extraction.prompts import (
+    EXTRACTION_SYSTEM_PROMPT,
+    EXTRACTION_USER_TEMPLATE,
+    build_chunks_block,
+)
+from app.extraction.output_validator import should_abstain
+from app.extraction.schemas import (
+    AtomicClaim,
+    ClaimEvidence,
+    ClaimType,
+    ClaimVerification,
+    ConditionRating,
+    SectionExtraction,
+    SupportLevel,
+    SurveyFinding,
+)
+
+logger = logging.getLogger(__name__)
+
+# Fixed seed for reproducible decoding across identical inputs.
+_EXTRACTION_SEED = 7
+
+
+def _chunk_tuples(chunks: list[object]) -> list[tuple[str, str, str | None]]:
+    """Adapt SearchResult-like rows into (chunk_id, text, label) tuples."""
+    out: list[tuple[str, str, str | None]] = []
+    for c in chunks:
+        cid = getattr(c, "chunk_id", None)
+        text = getattr(c, "text", None)
+        if not cid or not text:
+            continue
+        label = getattr(c, "section_title", None)
+        out.append((str(cid), str(text), label))
+    return out
+
+
+def _parse_findings(raw_json: str, section: str) -> list[SurveyFinding]:
+    """Parse model JSON into validated SurveyFinding models, skipping malformed rows."""
+    try:
+        data = json.loads(raw_json)
+    except (json.JSONDecodeError, TypeError):
+        logger.warning("extractor: model returned non-JSON for section=%s", section)
+        return []
+    rows = data.get("findings", []) if isinstance(data, dict) else []
+    findings: list[SurveyFinding] = []
+    for row in rows:
+        if not isinstance(row, dict):
+            continue
+        row.setdefault("section", section)
+        try:
+            findings.append(SurveyFinding.model_validate(row))
+        except Exception as exc:  # malformed row -> skip, never fabricate
+            logger.debug("extractor: skipped malformed finding: %s", exc)
+    return findings
+
+
+async def extract_section(
+    *,
+    section: str,
+    chunks: list[object],
+    tenant_id: str | None = None,
+    drop_partial: bool = False,
+    domain: str | None = None,
+    model: str | None = None,
+) -> SectionExtraction:
+    """Extract verified, contradiction-free findings for one section.
+
+    When ``domain`` is provided (STEP 6), chunks clearly belonging to a
+    different domain are excluded before extraction as defense-in-depth — the
+    caller should still scope retrieval, but this guarantees the extractor never
+    sees cross-domain evidence. The supplied (scoped) pool is the ONLY
+    admissible evidence.
+
+    Returns a :class:`SectionExtraction`. With no API key (or no chunks), returns
+    an empty extraction rather than raising.
+    """
+    if domain and domain != GENERAL:
+        chunks = scope_chunks(domain, chunks)
+    pool = _chunk_tuples(chunks)
+    if not pool:
+        return SectionExtraction(section=section)
+    if not (settings.openai_api_key or "").strip():
+        logger.info("extractor: no API key; returning empty extraction for %s", section)
+        return SectionExtraction(section=section)
+
+    user_prompt = EXTRACTION_USER_TEMPLATE.format(
+        section=section,
+        chunks_block=build_chunks_block(pool),
+    )
+
+    from app.llm.openai_chat import chat_completions_create
+
+    raw = await chat_completions_create(
+        messages=[
+            {"role": "system", "content": EXTRACTION_SYSTEM_PROMPT},
+            {"role": "user", "content": user_prompt},
+        ],
+        model=model or settings.chat_model,
+        max_tokens=settings.extraction_max_tokens,
+        temperature=0.0,
+        top_p=1.0,
+        seed=_EXTRACTION_SEED,
+        response_format={"type": "json_object"},
+        phase="extraction",
+        section_id=section,
+        tenant_id=tenant_id,
+    )
+
+    findings = _parse_findings(raw, section)
+
+    # Hard citation gate: drop everything not grounded in the supplied pool.
+    kept, dropped = validate_findings(findings, pool, drop_partial=drop_partial)
+
+    # Contradiction/duplicate audit on the survivors.
+    resolved, contradictions = audit_contradictions(kept)
+
+    logger.info(
+        "extract_section=%s extracted=%d kept=%d dropped=%d contradictions=%d",
+        section, len(findings), len(resolved), len(dropped), len(contradictions),
+    )
+
+    return SectionExtraction(
+        section=section,
+        findings=resolved,
+        contradictions=contradictions,
+        dropped_claims=dropped,
+    )
+
+
+_SUPPORT_CONFIDENCE: dict[SupportLevel, float] = {
+    SupportLevel.SUPPORTED: 1.0,
+    SupportLevel.PARTIAL: 0.5,
+    SupportLevel.NOT_FOUND: 0.0,
+}
+
+
+def findings_to_atomic_claims(
+    findings: list[SurveyFinding],
+    *,
+    min_confidence: float = 1.0,
+) -> list[AtomicClaim]:
+    """Decompose grounded findings into atomic, evidence-bound claim records.
+
+    Each finding yields at most two atomic claims: a ``condition_rating`` claim
+    (only when the source stated a real CR1/CR2/CR3) and an ``observation``
+    claim for the finding text. Every claim is bound to the finding's first
+    valid evidence span and then passed through the abstention gate
+    (:func:`should_abstain`) — any claim that introduces forbidden/fabricated
+    language or falls below ``min_confidence`` is dropped (RETURN NOTHING).
+
+    Pure and deterministic: no network, no synthesis. Findings are assumed to
+    have already passed citation validation.
+    """
+    claims: list[AtomicClaim] = []
+    real_ratings = {ConditionRating.CR1, ConditionRating.CR2, ConditionRating.CR3}
+    for f in findings:
+        if not f.evidence:
+            continue
+        span = f.evidence[0]
+        evidence_text = " \n ".join(e.text for e in f.evidence)
+        confidence = _SUPPORT_CONFIDENCE.get(f.support, 0.0)
+        page = span.page if span.page is not None else 0
+        ev = ClaimEvidence(
+            text=span.text,
+            page=page,
+            section=span.section_label or "",
+            chunk_id=span.chunk_id,
+        )
+
+        if f.condition_rating in real_ratings:
+            rating_claim = f"{f.element} condition rating is {f.condition_rating.value}"
+            if not should_abstain(
+                rating_claim, evidence_text,
+                min_confidence=min_confidence, confidence=confidence,
+            ):
+                claims.append(AtomicClaim(
+                    claim=rating_claim,
+                    claim_type=ClaimType.CONDITION_RATING,
+                    evidence=ev,
+                    verification=ClaimVerification(
+                        supported=True, contradiction_detected=False, confidence=confidence,
+                    ),
+                ))
+
+        if not should_abstain(
+            f.finding, evidence_text,
+            min_confidence=min_confidence, confidence=confidence,
+        ):
+            claims.append(AtomicClaim(
+                claim=f.finding,
+                claim_type=ClaimType.OBSERVATION,
+                evidence=ev,
+                verification=ClaimVerification(
+                    supported=True, contradiction_detected=False, confidence=confidence,
+                ),
+            ))
+    return claims
+
+
+async def audit_section_grounding(
+    *,
+    section_name: str,
+    template_id: str | None,
+    chunks: list[object],
+    tenant_id: str | None = None,
+) -> SectionExtraction:
+    """High-level, non-destructive grounding audit for a generated section.
+
+    Resolves the section's domain, scopes the retrieved evidence to that domain,
+    and runs citation-grounded extraction + contradiction audit. Intended to be
+    called alongside generation (behind ``settings.enable_citation_extraction``)
+    to produce a traceability artifact — it never mutates generated prose.
+    """
+    domain = classify_section(section_name, template_id)
+    return await extract_section(
+        section=section_name or (template_id or "section"),
+        chunks=chunks,
+        tenant_id=tenant_id,
+        domain=domain,
+    )

app/extraction/output_validator.py

@@ -0,0 +1,109 @@
+"""Post-generation output validator & abstention layer.
+
+Deterministic, pure-Python gate that runs AFTER extraction/generation and
+REJECTS any output that smuggles in non-evidence-grounded language:
+
+* forbidden hedging / synthesis phrases (``appears to``, ``likely``, ``overall``…),
+* fabricated metrics (percentages, confidence/authenticity/consistency scores),
+* unsupported severity escalation.
+
+A forbidden token only survives if it is present verbatim in the cited evidence
+(STEP: ABSOLUTE GROUNDING POLICY). The layer never rewrites text — it reports
+violations and the caller abstains (drops the record). No network, fully
+unit-testable.
+"""
+
+from __future__ import annotations
+
+import re
+
+from app.extraction.citation_validator import (
+    BANNED_SEVERITY_TERMS,
+    normalize_for_match,
+)
+
+# Hedging / synthesis / evaluation vocabulary that must never be introduced.
+FORBIDDEN_PHRASES: frozenset[str] = frozenset({
+    "appears to", "seems to", "likely", "suggests", "indicates",
+    "approximately reliable", "overall", "authenticity", "consistency score",
+    "confidence score", "reliability score", "assessment", "evaluation",
+    "in summary", "in conclusion", "executive summary", "we recommend",
+    "it is recommended", "concerning", "potentially hazardous",
+})
+
+# Fabricated-metric patterns: percentages and explicit scoring statements.
+_PERCENT_RE = re.compile(r"\b\d{1,3}(?:\.\d+)?\s?%")
+_SCORE_RE = re.compile(
+    r"\b(?:authenticity|reliability|consistency|confidence|quality)\s+"
+    r"(?:score|rating|level|index)\b",
+    re.IGNORECASE,
+)
+
+
+def _present_in_evidence(term: str, evidence_norm: str) -> bool:
+    return term in evidence_norm
+
+
+def find_violations(text: str, evidence: str = "") -> list[str]:
+    """Return a list of violation descriptions for ``text``.
+
+    A forbidden phrase / severity term is only a violation when it is NOT
+    present verbatim in ``evidence``. Percentages and scoring statements are
+    violations unless the identical token appears in evidence.
+    """
+    if not text:
+        return []
+    text_norm = normalize_for_match(text)
+    evidence_norm = normalize_for_match(evidence or "")
+    violations: list[str] = []
+
+    for phrase in FORBIDDEN_PHRASES:
+        if phrase in text_norm and not _present_in_evidence(phrase, evidence_norm):
+            violations.append(f"forbidden phrase {phrase!r}")
+
+    for term in BANNED_SEVERITY_TERMS:
+        if term in text_norm and not _present_in_evidence(term, evidence_norm):
+            violations.append(f"unsupported severity term {term!r}")
+
+    for m in _PERCENT_RE.findall(text):
+        token = normalize_for_match(m)
+        if token not in evidence_norm:
+            violations.append(f"fabricated percentage {m.strip()!r}")
+
+    for m in _SCORE_RE.findall(text):
+        if normalize_for_match(m) not in evidence_norm:
+            violations.append(f"fabricated metric {m.strip()!r}")
+
+    return violations
+
+
+def is_clean(text: str, evidence: str = "") -> bool:
+    """True when ``text`` introduces no forbidden/fabricated content."""
+    return not find_violations(text, evidence)
+
+
+def should_abstain(
+    text: str,
+    evidence: str = "",
+    *,
+    min_confidence: float = 1.0,
+    confidence: float = 1.0,
+    evidence_aligned: bool = True,
+    source_attributed: bool = True,
+) -> bool:
+    """Abstention logic (STEP: ABSTENTION LOGIC).
+
+    Returns True (drop the record / RETURN NOTHING) when any of:
+
+    * confidence is below ``min_confidence``,
+    * evidence alignment failed,
+    * source attribution failed,
+    * the text contains forbidden/fabricated content not grounded in evidence.
+    """
+    if confidence < min_confidence:
+        return True
+    if not evidence_aligned:
+        return True
+    if not source_attributed:
+        return True
+    return not is_clean(text, evidence)

app/extraction/prompts.py

@@ -0,0 +1,75 @@
+"""Hardened anti-hallucination prompts for citation-grounded extraction.
+
+These prompts implement STEP 5 (anti-hallucination rules) and STEP 4
+(schema-constrained extraction). They instruct the model to behave as a
+deterministic evidence extractor, never a writer.
+"""
+
+from __future__ import annotations
+
+EXTRACTION_SYSTEM_PROMPT = """\
+You are a deterministic evidence-extraction engine for RICS property survey \
+reports. You are NOT a writer, summariser, or assistant. You extract structured \
+findings ONLY from the SOURCE CHUNKS provided. The PDF source is the single \
+authority.
+
+ABSOLUTE RULES (violating any one is a critical failure):
+- Do NOT infer, speculate, generalise, or extrapolate.
+- Do NOT add, upgrade, or downgrade severity. Copy the condition exactly.
+- Do NOT invent risks, remediation actions, materials, locations, or entities.
+- Do NOT substitute proper nouns. If the source says "London plane tree", you \
+write "London plane tree" — never "cedar tree", "mature tree", or "vegetation".
+- Do NOT add safety/health/environmental implications unless they appear \
+verbatim in the source.
+- Do NOT synthesise conclusions that are not directly stated in a cited span.
+- Every finding MUST be backed by at least one verbatim evidence span copied \
+from a source chunk, with that chunk's id.
+- If the source does not support a field, OMIT it or use null. NEVER fill gaps.
+
+CONDITION RATINGS: use ONLY one of "1", "2", "3", "NI" (not inspected), or "NA" \
+(no rating stated). Never invent a rating. If the source gives no rating, use "NA".
+
+OUTPUT: a single JSON object matching the provided schema. No prose outside JSON.
+"""
+
+EXTRACTION_USER_TEMPLATE = """\
+SECTION DOMAIN: {section}
+
+You may ONLY use the SOURCE CHUNKS below. Do not use any outside knowledge.
+Each chunk has an id you MUST cite in the "chunk_id" of every evidence span.
+Evidence span text MUST be copied verbatim from the chunk it cites.
+
+SOURCE CHUNKS:
+{chunks_block}
+
+Return JSON with this exact shape:
+{{
+  "findings": [
+    {{
+      "section": "{section}",
+      "element": "<specific element named in the source>",
+      "condition_rating": "1" | "2" | "3" | "NI" | "NA",
+      "finding": "<description using only words/facts from the cited spans>",
+      "evidence": [
+        {{"chunk_id": "<id from a source chunk>", "text": "<verbatim span>"}}
+      ],
+      "page_refs": [<int page numbers if present in chunk labels>]
+    }}
+  ]
+}}
+
+If a chunk contains no extractable survey finding, do not invent one. Return an \
+empty "findings" list rather than fabricating content.
+"""
+
+
+def build_chunks_block(chunks: list[tuple[str, str, str | None]]) -> str:
+    """Render ``(chunk_id, text, label)`` tuples into a numbered prompt block."""
+    lines: list[str] = []
+    for cid, text, label in chunks:
+        header = f"[chunk_id={cid}"
+        if label:
+            header += f" | {label}"
+        header += "]"
+        lines.append(f"{header}\n{text.strip()}")
+    return "\n\n---\n\n".join(lines)

app/extraction/schemas.py

@@ -0,0 +1,199 @@
+"""Schema-constrained extraction types.
+
+These Pydantic models are the contract the LLM extractor must satisfy. The model
+may NEVER emit a value outside these constraints — condition ratings are a closed
+enum, evidence is mandatory, and unsupported fields are represented explicitly
+(``SupportLevel.NOT_FOUND``) rather than interpolated.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class ConditionRating(str, Enum):
+    """RICS Home Survey condition rating — a closed set.
+
+    The extractor must map any source rating onto exactly one of these. It may
+    never invent an intermediate value, upgrade, or downgrade. ``NI`` / ``NA``
+    are explicit "no rating present" sentinels so a missing rating is never
+    silently coerced to a real one.
+    """
+
+    CR1 = "1"  # No repair currently needed.
+    CR2 = "2"  # Defects requiring repair/replacement but not urgent.
+    CR3 = "3"  # Defects that are serious and/or need urgent attention.
+    NI = "NI"  # Not inspected.
+    NA = "NA"  # Not applicable / no rating stated in source.
+
+    @classmethod
+    def coerce(cls, value: object) -> "ConditionRating":
+        """Map a raw model/source value onto the enum without inventing a rating.
+
+        Anything that is not an exact, recognised rating becomes :attr:`NA`
+        (not a guessed CR1/CR2/CR3). This is deliberate: an unparseable rating
+        must degrade to "no rating", never to a fabricated severity.
+        """
+        if isinstance(value, cls):
+            return value
+        s = str(value or "").strip().upper()
+        direct = {
+            "1": cls.CR1, "CR1": cls.CR1, "CONDITION RATING 1": cls.CR1,
+            "2": cls.CR2, "CR2": cls.CR2, "CONDITION RATING 2": cls.CR2,
+            "3": cls.CR3, "CR3": cls.CR3, "CONDITION RATING 3": cls.CR3,
+            "NI": cls.NI, "NOT INSPECTED": cls.NI,
+            "NA": cls.NA, "N/A": cls.NA, "": cls.NA, "NONE": cls.NA,
+        }
+        return direct.get(s, cls.NA)
+
+
+class SupportLevel(str, Enum):
+    """How well a claim is backed by cited source spans."""
+
+    SUPPORTED = "supported"        # Every token traceable to a cited span.
+    PARTIAL = "partial"            # Core claim supported; minor unverified detail.
+    NOT_FOUND = "not_found"        # No supporting evidence — must be dropped.
+
+
+class EvidenceSpan(BaseModel):
+    """A literal span copied from a retrieved source chunk.
+
+    ``text`` MUST be a verbatim (or near-verbatim) substring of the chunk
+    identified by ``chunk_id``. The citation validator rejects spans that do
+    not actually appear in their cited chunk.
+    """
+
+    chunk_id: str = Field(..., description="ID of the retrieved chunk this span came from.")
+    doc_id: str | None = Field(default=None, description="Source document UUID.")
+    page: int | None = Field(default=None, description="1-based page number when known.")
+    section_label: str | None = Field(default=None, description="Section/page heading label.")
+    text: str = Field(..., min_length=1, description="Verbatim span from the cited chunk.")
+
+    @field_validator("text")
+    @classmethod
+    def _strip(cls, v: str) -> str:
+        v = v.strip()
+        if not v:
+            raise ValueError("evidence span text cannot be blank")
+        return v
+
+
+class GroundedClaim(BaseModel):
+    """An atomic claim and the evidence that supports it."""
+
+    claim: str = Field(..., min_length=1)
+    supporting_spans: list[EvidenceSpan] = Field(default_factory=list)
+    support: SupportLevel = Field(default=SupportLevel.NOT_FOUND)
+    page_refs: list[int] = Field(default_factory=list)
+
+
+class SurveyFinding(BaseModel):
+    """One condition finding for a building element, fully evidence-backed.
+
+    This is the structured-extraction unit from STEP 4 of the spec. ``finding``
+    is descriptive prose derived ONLY from the cited spans; ``condition_rating``
+    is a closed enum; ``evidence`` carries the literal spans; ``page_refs`` the
+    source pages.
+    """
+
+    section: str = Field(..., description="Domain/section, e.g. 'Roofing', 'Drainage'.")
+    element: str = Field(..., description="Specific element, e.g. 'Main roof covering'.")
+    condition_rating: ConditionRating = Field(default=ConditionRating.NA)
+    finding: str = Field(..., min_length=1, description="Evidence-grounded description.")
+    evidence: list[EvidenceSpan] = Field(default_factory=list)
+    page_refs: list[int] = Field(default_factory=list)
+    support: SupportLevel = Field(default=SupportLevel.NOT_FOUND)
+
+    @field_validator("condition_rating", mode="before")
+    @classmethod
+    def _coerce_rating(cls, v: object) -> ConditionRating:
+        return ConditionRating.coerce(v)
+
+
+class ContradictionKind(str, Enum):
+    OPERATIONAL = "operational_vs_non_operational"
+    CONDITION = "satisfactory_vs_defective"
+    RATING_CONFLICT = "inconsistent_condition_rating"
+    DUPLICATE = "duplicate_finding"
+    MUTUALLY_EXCLUSIVE = "mutually_exclusive_statement"
+    UNSUPPORTED_SUMMARY = "unsupported_summary"
+
+
+class Contradiction(BaseModel):
+    """A detected logical inconsistency between two findings (or within one)."""
+
+    kind: ContradictionKind
+    element: str
+    detail: str
+    finding_indices: list[int] = Field(default_factory=list)
+    resolution: str | None = Field(
+        default=None,
+        description="Which side was kept and why (evidence-supported version preserved).",
+    )
+
+
+class ClaimType(str, Enum):
+    """Closed set of atomic claim categories (no free-text types)."""
+
+    CONDITION_RATING = "condition_rating"
+    MEASUREMENT = "measurement"
+    MATERIAL = "material"
+    ENTITY = "entity"            # named product/model/species/person/org
+    LOCATION = "location"
+    DATE = "date"
+    QUANTITY = "quantity"
+    OBSERVATION = "observation"  # plain factual observation
+    OTHER = "other"
+
+
+class ClaimEvidence(BaseModel):
+    """Exact evidence binding for one atomic claim."""
+
+    text: str = Field(..., min_length=1, description="Verbatim span from the cited chunk.")
+    page: int = Field(default=0, description="1-based page number; 0 when unknown.")
+    section: str = Field(default="", description="Section/heading label of the source span.")
+    chunk_id: str = Field(..., min_length=1, description="ID of the cited source chunk.")
+
+
+class ClaimVerification(BaseModel):
+    """Verification outcome for one atomic claim."""
+
+    supported: bool = False
+    contradiction_detected: bool = False
+    confidence: float = Field(default=0.0, ge=0.0, le=1.0)
+
+
+class AtomicClaim(BaseModel):
+    """A single atomic factual claim with mandatory evidence binding.
+
+    This is the forensic unit required by the evidence-comparison contract:
+    one indivisible fact, the exact span that supports it, and the deterministic
+    verification result. Unsupported claims are dropped before output.
+    """
+
+    claim: str = Field(..., min_length=1)
+    claim_type: ClaimType = Field(default=ClaimType.OTHER)
+    evidence: ClaimEvidence
+    verification: ClaimVerification = Field(default_factory=ClaimVerification)
+
+
+class SectionExtraction(BaseModel):
+    """The full schema-constrained extraction for one report section."""
+
+    section: str
+    findings: list[SurveyFinding] = Field(default_factory=list)
+    contradictions: list[Contradiction] = Field(default_factory=list)
+    dropped_claims: list[str] = Field(
+        default_factory=list,
+        description="Claims removed for lack of supporting evidence (audit trail).",
+    )
+
+    @property
+    def confidence(self) -> float:
+        """Fraction of findings that are fully supported (0.0–1.0)."""
+        if not self.findings:
+            return 0.0
+        supported = sum(1 for f in self.findings if f.support == SupportLevel.SUPPORTED)
+        return round(supported / len(self.findings), 3)

app/generator/postprocess.py

@@ -377,6 +377,10 @@ def enforce_verify(
     for key, val in _slots.items():
         text = text.replace(key, val)
 
+    # Collapse degenerate runs of the missing-data placeholder (e.g. a minimum-
+    # interference output that padded sparse notes by repeating the token).
+    text = _collapse_placeholder_runs(text)
+
     return text
 
 
@@ -429,10 +433,29 @@ def verbatim_overlap_ratio(text: str, sources: list[str], n: int = 6) -> float:
     return matched / len(text_grams)
 
 
+_REPEATED_PLACEHOLDER_RE = re.compile(r"(?:\[DATA NOT PROVIDED\]\s*){2,}", re.IGNORECASE)
+
+
+def _collapse_placeholder_runs(text: str) -> str:
+    """Collapse degenerate runs of the missing-data placeholder.
+
+    Minimum-interference mode can pad sparse notes up to the word floor by
+    repeating ``[DATA NOT PROVIDED]`` dozens or hundreds of times. Any run of
+    two or more (optionally whitespace-separated) collapses to a single token,
+    so it still signals a gap without spamming the section.
+    """
+    if not text:
+        return text
+    collapsed = _REPEATED_PLACEHOLDER_RE.sub("[DATA NOT PROVIDED] ", text)
+    collapsed = re.sub(r"\s{2,}", " ", collapsed)
+    return collapsed.strip()
+
+
 def _strip_missing_fact_phrases(text: str) -> str:
     """Remove hard-coded missing-data filler phrases from output text."""
     if not text:
         return text
+    text = _collapse_placeholder_runs(text)
     cleaned = re.sub(
         r"\bInformation not provided in source document\b[.,;:!?]*",
         "",

app/generator/prompts.py

@@ -268,7 +268,9 @@ the STANDARD SOURCE PASSAGES. Use retrieved uploaded-report excerpts only to res
 ambiguous terminology or to confirm section relevance — do NOT copy narrative or \
 property-specific findings from them. Do NOT invent, infer, expand, or editorialize beyond \
 grammar and structural placement. Preserve the surveyor's original wording as closely as \
-professional RICS grammar allows.\
+professional RICS grammar allows. There is no minimum length and no word-count floor: never pad, \
+never repeat a sentence or token, and never emit [DATA NOT PROVIDED] more than once in a row — a \
+sparse note simply yields a sparse section.\
 """
 
 
@@ -325,10 +327,13 @@ structure of [standard_paragraphs].\n\
 Use [uploaded_reports] solely to understand terminology and paragraph context. Do not copy content from them.\n\
 Do NOT add any information, interpretation, opinion, transition phrase, or filler sentence that does not \
 originate directly from the messy notes.\n\
-If a subsection of the standard structure has no corresponding data in the messy notes, leave it blank or \
-insert the exact token [DATA NOT PROVIDED] — do not fill it.\n\
+If a subsection of the standard structure has no corresponding data in the messy notes, omit it. You MAY \
+insert the exact token [DATA NOT PROVIDED] at most ONCE for that subsection — never repeat it, never place \
+it more than once in a row, and never use it (or any phrase) as filler to reach a length target.\n\
 Preserve the user's original wording as closely as possible; clean grammar and structure only.\n\
-Output length: match the density of the input notes — no padding. Stay within {min_words}–{max_words} words.\n\
+Output length is governed SOLELY by the density of the input notes. There is NO minimum word count: short \
+notes produce short output. Never pad, repeat sentences, or duplicate tokens to lengthen the text. Do not \
+exceed {max_words} words.\n\
 """
 
 

app/ingest/hierarchy.py

@@ -113,7 +113,10 @@ def build_hierarchical_documents(
             chunk.metadata["section_title"] = stitle
             chunk.metadata["paragraph_index"] = j
             chunk.metadata["parent_hierarchy"] = "section"
-            chunk.metadata["section_type"] = "paragraph"
+            # Preserve table-aware tagging from the splitter; only default prose
+            # chunks to "paragraph".
+            if chunk.metadata.get("section_type") != "table":
+                chunk.metadata["section_type"] = "paragraph"
             chunk.metadata["proposed_chunk_id"] = f"{doc_id}_hier_{sid}_p{j:04d}"
             chunk.metadata.setdefault("source", stem)
             paragraph_docs.append(chunk)

app/ingest/ocr_normalize.py

@@ -0,0 +1,147 @@
+"""OCR / layout normalization for extracted PDF text (STEP 1).
+
+PDF text extraction routinely corrupts the document in ways that poison
+downstream retrieval and generation:
+
+* sentences are broken by hard line wraps,
+* words are split with end-of-line hyphens ("condi-\ntion"),
+* the same running header / footer / page number repeats on every page,
+* whitespace and newlines are inconsistent.
+
+These functions repair that corruption deterministically before chunking, so
+chunks contain whole sentences and no boilerplate noise. Pure string ops — no
+dependencies, fully unit-testable.
+
+The table sentinel (``[TABLE]…[/TABLE]``) emitted by the parser is treated as
+opaque: normalization never reflows text inside a table block.
+"""
+
+from __future__ import annotations
+
+import re
+
+TABLE_OPEN = "[TABLE]"
+TABLE_CLOSE = "[/TABLE]"
+
+_TABLE_BLOCK_RE = re.compile(
+    re.escape(TABLE_OPEN) + r".*?" + re.escape(TABLE_CLOSE),
+    re.DOTALL,
+)
+# end-of-line hyphenation: "condi-\ntion" -> "condition"
+_HYPHEN_WRAP_RE = re.compile(r"(\w)-\n[ \t]*(\w)")
+# bare page-number / "Page x of y" / "x / y" lines
+_PAGE_NUM_RE = re.compile(
+    r"^\s*(?:page\s+)?\d+\s*(?:of|/)\s*\d+\s*$|^\s*page\s+\d+\s*$|^\s*\d{1,4}\s*$",
+    re.IGNORECASE,
+)
+
+
+def _protect_tables(text: str) -> tuple[str, list[str]]:
+    """Replace table blocks with placeholders so reflow never touches them."""
+    blocks: list[str] = []
+
+    def _stash(m: re.Match[str]) -> str:
+        blocks.append(m.group(0))
+        return f"\x00TBL{len(blocks) - 1}\x00"
+
+    return _TABLE_BLOCK_RE.sub(_stash, text), blocks
+
+
+def _restore_tables(text: str, blocks: list[str]) -> str:
+    for i, block in enumerate(blocks):
+        text = text.replace(f"\x00TBL{i}\x00", block)
+    return text
+
+
+def normalize_text(text: str) -> str:
+    """Repair a single page/section of extracted text.
+
+    Steps: normalize unicode whitespace, de-hyphenate wrapped words, unwrap
+    hard-wrapped sentences within a paragraph (single newline -> space) while
+    preserving paragraph breaks (blank lines), and collapse excess whitespace.
+    Table blocks are preserved verbatim.
+    """
+    if not text:
+        return ""
+
+    protected, blocks = _protect_tables(text)
+
+    # Normalize unicode whitespace / non-breaking spaces.
+    protected = protected.replace("\u00a0", " ").replace("\r\n", "\n").replace("\r", "\n")
+
+    # De-hyphenate words split across a line break.
+    protected = _HYPHEN_WRAP_RE.sub(r"\1\2", protected)
+
+    # Unwrap: within each blank-line-delimited paragraph, join hard-wrapped
+    # lines into a single line. This restores sentence continuity that PDF
+    # extraction destroys by emitting one newline per visual line.
+    paragraphs = re.split(r"\n[ \t]*\n", protected)
+    rebuilt: list[str] = []
+    for para in paragraphs:
+        if "\x00TBL" in para:
+            rebuilt.append(para.strip())
+            continue
+        lines = [ln.strip() for ln in para.split("\n") if ln.strip()]
+        if not lines:
+            continue
+        rebuilt.append(" ".join(lines))
+    out = "\n\n".join(rebuilt)
+
+    # Collapse runs of spaces and excessive blank lines.
+    out = re.sub(r"[ \t]{2,}", " ", out)
+    out = re.sub(r"\n{3,}", "\n\n", out)
+
+    return _restore_tables(out.strip(), blocks)
+
+
+def _candidate_boundary_lines(page_text: str, edge: int = 3) -> set[str]:
+    """First/last ``edge`` non-empty lines of a page (header/footer candidates)."""
+    lines = [ln.strip() for ln in page_text.split("\n") if ln.strip()]
+    if not lines:
+        return set()
+    return set(lines[:edge]) | set(lines[-edge:])
+
+
+def strip_running_headers_footers(pages: list[str], *, edge: int = 3) -> list[str]:
+    """Remove repeated running headers/footers and page numbers across pages.
+
+    A short line appearing in the top/bottom ``edge`` lines of a majority of
+    pages is treated as boilerplate and removed from every page. Bare page
+    numbers are always removed. Single-page documents are returned unchanged
+    (no cross-page signal to safely act on).
+    """
+    if len(pages) < 3:
+        # Still strip bare page numbers even when we can't detect repetition.
+        return [_drop_page_numbers(p) for p in pages]
+
+    freq: dict[str, int] = {}
+    for p in pages:
+        for line in _candidate_boundary_lines(p, edge):
+            if len(line) <= 120:
+                freq[line] = freq.get(line, 0) + 1
+
+    threshold = max(2, int(len(pages) * 0.5))
+    boilerplate = {ln for ln, n in freq.items() if n >= threshold}
+
+    cleaned: list[str] = []
+    for p in pages:
+        kept = []
+        for line in p.split("\n"):
+            s = line.strip()
+            if s and s in boilerplate:
+                continue
+            kept.append(line)
+        cleaned.append(_drop_page_numbers("\n".join(kept)))
+    return cleaned
+
+
+def _drop_page_numbers(text: str) -> str:
+    return "\n".join(
+        ln for ln in text.split("\n") if not _PAGE_NUM_RE.match(ln.strip())
+    )
+
+
+def normalize_pages(pages: list[str]) -> list[str]:
+    """Full document-level normalization: strip boilerplate, then reflow each page."""
+    deboiled = strip_running_headers_footers(pages)
+    return [normalize_text(p) for p in deboiled]

app/ingest/parser_pdf.py

@@ -1,59 +1,155 @@
-"""Parse .pdf files using the LangChain PyMuPDFLoader.
-
-LangChain replaces the manual pymupdf font-size heuristics and line-by-line
-block extraction with ``PyMuPDFLoader`` which returns one LangChain
-``Document`` per page.  Page-level metadata (``page``, ``source``) is
-preserved automatically.
-
-Compared to master branch:
-- ✅ Simpler code — no font-size heading heuristics
-- ✅ Standard LangChain interface with reliable per-page Documents
-- ✅ File handle automatically managed by the loader
-- ⚠️  Heading detection not performed (heading levels not in metadata)
+"""Layout- and table-aware PDF parsing (STEP 1).
+
+Replaces the plain ``PyMuPDFLoader`` (which flattens tables into ambiguous
+text and leaves OCR line-wrap corruption in place) with a hardened parser built
+on PyMuPDF (``fitz``) — already a project dependency, so no new packages.
+
+Per page we:
+
+1. Extract text in reading order (``sort=True``) to reduce multi-column
+   scrambling.
+2. Detect tables with ``page.find_tables()`` and render each as Markdown
+   wrapped in ``[TABLE]…[/TABLE]`` sentinels so condition-rating blocks and
+   financial totals survive chunking with row/column structure intact and are
+   NEVER flattened into prose.
+3. Normalize the prose across pages: strip running headers/footers and page
+   numbers, repair hyphenation, and unwrap hard-wrapped sentences.
+
+Output remains a list of LangChain ``Document`` objects (one per page) with the
+same ``page`` / ``source`` / ``total_pages`` metadata the rest of the pipeline
+expects, so this is a drop-in replacement. If anything fails, we fall back to
+the original ``PyMuPDFLoader`` so ingestion never hard-fails on a quirky PDF.
 """
 
+from __future__ import annotations
+
 import logging
 from pathlib import Path
 
-from langchain_community.document_loaders import PyMuPDFLoader
 from langchain_core.documents import Document
 
+from app.ingest.ocr_normalize import TABLE_CLOSE, TABLE_OPEN, normalize_pages
+
 logger = logging.getLogger(__name__)
 
 
-def parse_pdf(file_path: Path) -> list[Document]:
-    """Load a ``.pdf`` file and return its pages as LangChain Documents.
+def _cell(value: object) -> str:
+    """Render one table cell: strip, collapse newlines, escape pipes."""
+    s = "" if value is None else str(value)
+    return s.replace("\n", " ").replace("|", "\\|").strip()
+
+
+def _table_to_markdown(rows: list[list[object]]) -> str:
+    """Render extracted table rows as a GitHub-flavoured Markdown table."""
+    cleaned = [[_cell(c) for c in row] for row in rows if row]
+    cleaned = [r for r in cleaned if any(c for c in r)]
+    if not cleaned:
+        return ""
+    width = max(len(r) for r in cleaned)
+    cleaned = [r + [""] * (width - len(r)) for r in cleaned]
+    header = cleaned[0]
+    body = cleaned[1:]
+    lines = ["| " + " | ".join(header) + " |", "| " + " | ".join(["---"] * width) + " |"]
+    for r in body:
+        lines.append("| " + " | ".join(r) + " |")
+    return "\n".join(lines)
+
+
+def _extract_page_tables(page: object) -> list[str]:
+    """Return Markdown for each table on ``page`` (best-effort, never raises)."""
+    out: list[str] = []
+    try:
+        finder = page.find_tables()
+    except Exception as exc:  # find_tables can choke on malformed content
+        logger.debug("find_tables failed on a page: %s", exc)
+        return out
+    tables = getattr(finder, "tables", None) or []
+    for tbl in tables:
+        try:
+            md = _table_to_markdown(tbl.extract())
+        except Exception as exc:  # noqa: BLE001 — a bad table must not kill the page
+            logger.debug("table.extract failed: %s", exc)
+            continue
+        if md:
+            out.append(f"{TABLE_OPEN}\n{md}\n{TABLE_CLOSE}")
+    return out
+
 
-    Uses ``langchain_community.document_loaders.PyMuPDFLoader`` which
-    extracts text from each page and attaches ``page``, ``source``,
-    ``total_pages``, and ``file_path`` metadata.
+def _parse_pdf_fitz(file_path: Path) -> list[Document]:
+    """Primary path: layout/table-aware extraction via PyMuPDF."""
+    import fitz  # PyMuPDF — already a dependency
+
+    doc = fitz.open(str(file_path))
+    try:
+        total_pages = doc.page_count
+        prose_pages: list[str] = []
+        page_tables: list[list[str]] = []
+        for page in doc:
+            prose_pages.append(page.get_text("text", sort=True) or "")
+            page_tables.append(_extract_page_tables(page))
+    finally:
+        doc.close()
+
+    normalized = normalize_pages(prose_pages)
+
+    documents: list[Document] = []
+    for i, prose in enumerate(normalized):
+        parts = [prose.strip()] if prose.strip() else []
+        parts.extend(page_tables[i])
+        content = "\n\n".join(parts)
+        documents.append(
+            Document(
+                page_content=content,
+                metadata={
+                    "page": i,
+                    "total_pages": total_pages,
+                    "source": file_path.name,
+                    "file_path": str(file_path),
+                    "has_tables": bool(page_tables[i]),
+                },
+            )
+        )
+    return documents
+
+
+def _parse_pdf_fallback(file_path: Path) -> list[Document]:
+    """Fallback: original LangChain loader (no table structure, no normalization)."""
+    from langchain_community.document_loaders import PyMuPDFLoader
+
+    return PyMuPDFLoader(str(file_path)).load()
+
+
+def parse_pdf(file_path: Path) -> list[Document]:
+    """Load a ``.pdf`` and return one normalized, table-aware Document per page.
 
     Args:
         file_path: Absolute path to a ``.pdf`` file.
 
     Returns:
-        List of :class:`langchain_core.documents.Document` objects, one per
-        PDF page.  Empty pages are included with empty ``page_content``.
+        List of :class:`langchain_core.documents.Document`, one per page, with
+        prose normalized and tables preserved as Markdown inside
+        ``[TABLE]…[/TABLE]`` sentinels.
 
     Raises:
         FileNotFoundError: If ``file_path`` does not exist.
-        ValueError: If the file cannot be opened as a valid PDF.
-
-    Example::
-
-        docs = parse_pdf(Path("/uploads/survey.pdf"))
-        print(f"Loaded {len(docs)} pages")
+        ValueError: If the file cannot be opened as a valid PDF by either path.
     """
     if not file_path.exists():
         raise FileNotFoundError(f"File not found: {file_path}")
 
     try:
-        loader = PyMuPDFLoader(str(file_path))
-        docs = loader.load()
-    except Exception as exc:
-        raise ValueError(f"Cannot open PDF '{file_path}': {exc}") from exc
-
-    logger.debug(
-        "Parsed %d page(s) from %s via LangChain PyMuPDFLoader", len(docs), file_path.name
-    )
-    return docs
+        docs = _parse_pdf_fitz(file_path)
+        logger.debug(
+            "Parsed %d page(s) from %s (table-aware PyMuPDF)", len(docs), file_path.name
+        )
+        return docs
+    except Exception as exc:  # noqa: BLE001 — degrade gracefully, never lose ingestion
+        logger.warning(
+            "Table-aware PDF parse failed for %s (%s); falling back to PyMuPDFLoader",
+            file_path.name,
+            exc,
+        )
+        try:
+            return _parse_pdf_fallback(file_path)
+        except Exception as exc2:
+            raise ValueError(f"Cannot open PDF '{file_path}': {exc2}") from exc2

app/llm/openai_chat.py

@@ -16,6 +16,8 @@ async def chat_completions_create_raw(
     max_tokens: int = 800,
     temperature: float = 0.0,
     response_format: dict[str, str] | None = None,
+    top_p: float | None = None,
+    seed: int | None = None,
     phase: str = "chat",
     section_id: str | None = None,
     api_key: str | None = None,
@@ -35,6 +37,10 @@ async def chat_completions_create_raw(
     }
     if response_format is not None:
         kwargs["response_format"] = response_format
+    if top_p is not None:
+        kwargs["top_p"] = top_p
+    if seed is not None:
+        kwargs["seed"] = seed
 
     if settings.enable_async_pipeline:
         from openai import AsyncOpenAI
@@ -96,6 +102,8 @@ async def chat_completions_create(
     max_tokens: int = 800,
     temperature: float = 0.0,
     response_format: dict[str, str] | None = None,
+    top_p: float | None = None,
+    seed: int | None = None,
     phase: str = "chat",
     section_id: str | None = None,
     api_key: str | None = None,
@@ -108,6 +116,8 @@ async def chat_completions_create(
         max_tokens=max_tokens,
         temperature=temperature,
         response_format=response_format,
+        top_p=top_p,
+        seed=seed,
         api_key=api_key,
         phase=phase,
         section_id=section_id,

app/main.py

@@ -16,6 +16,7 @@ from sqlalchemy import text
 
 from app.api import (
     agentic,
+    auth,
     canonical_rollout,
     catalog,
     content_similarity,
@@ -314,6 +315,7 @@ def create_app() -> FastAPI:
     )
     app.add_middleware(TenantAuthMiddleware)
 
+    app.include_router(auth.router, tags=["auth"])
     app.include_router(upload.router, tags=["upload"])
     app.include_router(survey_level.router, tags=["upload"])
     app.include_router(style_library.router, tags=["style-library"])

app/models/schemas.py

@@ -42,6 +42,30 @@ class WritingStyleProfile(BaseModel):
     )
 
 
+# ── Auth ──────────────────────────────────────────────────────────────────────
+
+class AuthRequest(BaseModel):
+    """Register / login payload: tenant identity + passphrase."""
+
+    tenant_id: str = Field(..., description="Tenant/User ID (workspace identifier)")
+    passphrase: str = Field(..., description="Account passphrase (min 8 chars)")
+
+
+class AuthTokenResponse(BaseModel):
+    """Issued bearer token bound to the verified tenant."""
+
+    tenant_id: str
+    access_token: str
+    token_type: str = "bearer"
+    expires_at: int = Field(..., description="Unix epoch seconds when the token expires")
+
+
+class AuthWhoAmIResponse(BaseModel):
+    """Tenant resolved from the current request's verified credentials."""
+
+    tenant_id: str
+
+
 # ── Upload ────────────────────────────────────────────────────────────────────
 
 class UploadResponse(BaseModel):
@@ -429,6 +453,14 @@ class SectionPayload(BaseModel):
         default=None,
         description="ISO 8601 UTC timestamp when this section was last generated or updated by AI.",
     )
+    citation_audit: dict[str, Any] | None = Field(
+        default=None,
+        description=(
+            "Citation-grounded extraction audit for this section: confidence "
+            "(fraction of fully-supported findings), supported findings count, "
+            "detected contradictions, and claims dropped for lack of evidence."
+        ),
+    )
 
 
 class AILevel(int, Enum):
@@ -1116,6 +1148,20 @@ class DocumentDeleteResponse(BaseModel):
     detail: str = ""
 
 
+class DocumentReingestResponse(BaseModel):
+    """Result of re-queuing one or more documents through the ingestion pipeline."""
+
+    queued: int = Field(..., description="Number of documents re-queued for ingestion.")
+    document_ids: list[str] = Field(default_factory=list)
+    skipped_active: int = Field(
+        default=0, description="Documents skipped because a report is generating against them."
+    )
+    skipped_missing_file: int = Field(
+        default=0, description="Documents skipped because their source file is gone."
+    )
+    detail: str = ""
+
+
 # ── Style Library (per-tenant past reports for style learning) ───────────────
 
 

app/services/document_library.py

@@ -0,0 +1,169 @@
+"""Shared helpers for tenant RAG document list and delete operations."""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+from pathlib import Path
+
+from fastapi import HTTPException
+from sqlalchemy import delete, func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import Document, DocumentPurpose, IngestStatus, Report
+from app.models.schemas import DocumentDeleteResponse, DocumentListItem
+
+logger = logging.getLogger(__name__)
+
+
+def ingest_status_to_display(status: IngestStatus | str) -> str:
+    """Map DB ingest status to UI-friendly labels."""
+    raw = status.value if isinstance(status, IngestStatus) else str(status)
+    if raw in ("pending", "processing"):
+        return "processing"
+    if raw == "complete":
+        return "ready"
+    if raw == "failed":
+        return "failed"
+    return raw
+
+
+def file_size_bytes(file_path: str | None) -> int | None:
+    """Return on-disk byte size when the upload file still exists."""
+    if not file_path:
+        return None
+    try:
+        p = Path(file_path)
+        if p.is_file():
+            return int(p.stat().st_size)
+    except OSError:
+        return None
+    return None
+
+
+def document_list_item(
+    doc: Document,
+    *,
+    chunk_count: int,
+    linked_report_count: int,
+) -> DocumentListItem:
+    purpose = doc.document_purpose
+    purpose_val = purpose.value if isinstance(purpose, DocumentPurpose) else str(purpose)
+    ingest = doc.status.value if isinstance(doc.status, IngestStatus) else str(doc.status)
+    return DocumentListItem(
+        document_id=doc.id,
+        filename=doc.filename,
+        file_size_bytes=file_size_bytes(doc.file_path),
+        upload_timestamp=doc.created_at,
+        updated_at=doc.updated_at,
+        status=ingest_status_to_display(doc.status),
+        ingest_status=ingest,
+        document_purpose=purpose_val,
+        chunk_count=int(chunk_count),
+        survey_level=doc.survey_level,
+        error_message=doc.error_message,
+        storage_path=doc.file_path,
+        linked_report_count=int(linked_report_count),
+    )
+
+
+async def delete_tenant_document(
+    db: AsyncSession,
+    *,
+    tenant_id: str,
+    document_id: str,
+    expected_purpose: DocumentPurpose | None = DocumentPurpose.report_source,
+) -> DocumentDeleteResponse:
+    """Remove a document from vector index, disk, and database.
+
+    Args:
+        expected_purpose: When set, reject deletes for other purposes (e.g.
+            style_corpus must use the style-library delete route).
+    """
+    doc = await db.get(Document, document_id)
+    if doc is None or doc.tenant_id != tenant_id:
+        raise HTTPException(status_code=404, detail="Document not found")
+
+    if expected_purpose is not None and doc.document_purpose != expected_purpose:
+        raise HTTPException(
+            status_code=400,
+            detail=(
+                "This document is not a RAG report-source upload "
+                "(use DELETE /style-library/{id} for style-library items)."
+            ),
+        )
+
+    cnt = await db.execute(
+        select(func.count()).select_from(Report).where(Report.document_id == document_id)
+    )
+    if int(cnt.scalar_one() or 0) > 0:
+        raise HTTPException(
+            status_code=409,
+            detail=(
+                "This file is still linked to one or more reports. "
+                "Finish or abandon those jobs first, or upload replacements under a new document."
+            ),
+        )
+
+    warnings: list[str] = []
+    chunks_before = 0
+    chunks_after = 0
+    vector_deleted = False
+
+    try:
+        from app.vectorstore.factory import get_vectorstore
+
+        vs = get_vectorstore()
+        chunks_before = int(vs.count_for_doc(document_id))
+        try:
+            vs.delete_document(document_id)
+            vector_deleted = True
+            chunks_after = int(vs.count_for_doc(document_id))
+            if chunks_after > 0:
+                warnings.append(
+                    f"Vector index still reports {chunks_after} chunk(s) for this document."
+                )
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Vector store delete failed for doc=%s: %s", document_id, exc)
+            warnings.append(f"Vector store deletion error: {exc}")
+            try:
+                chunks_after = int(vs.count_for_doc(document_id))
+            except Exception:  # noqa: BLE001
+                chunks_after = chunks_before
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("Vector store unavailable during delete doc=%s: %s", document_id, exc)
+        warnings.append(f"Vector store unavailable: {exc}")
+
+    file_removed = False
+    fp = Path(doc.file_path)
+    try:
+        if fp.is_file():
+            fp.unlink()
+            file_removed = True
+    except OSError as exc:
+        logger.warning("Could not remove file %s: %s", fp, exc)
+        warnings.append(f"Could not remove file from disk: {exc}")
+
+    await db.execute(delete(Document).where(Document.id == document_id))
+    await db.commit()
+
+    from app.retrieval.semantic_cache import invalidate_semantic_cache_for_tenant
+    from app.services.photo_policy import invalidate_tenant_photo_policy_cache
+
+    await invalidate_semantic_cache_for_tenant(tenant_id)
+    invalidate_tenant_photo_policy_cache(tenant_id)
+
+    removed = vector_deleted and (chunks_after == 0) and file_removed
+    detail = "Document removed from disk, database, and search index."
+    if warnings:
+        detail = "Document removed with warnings: " + "; ".join(warnings)
+
+    return DocumentDeleteResponse(
+        document_id=document_id,
+        removed=removed or not warnings,
+        detail=detail,
+        vector_chunks_removed=max(0, chunks_before - chunks_after),
+        file_removed=file_removed,
+        database_removed=True,
+        warnings=warnings,
+    )

app/services/generation.py

@@ -1426,6 +1426,38 @@ def _mode_controls(ai_level: int, mode: str, ai_percent: int | None = None) -> t
     return t, h
 
 
+async def _tenant_report_source_doc_ids(
+    db: AsyncSession | None,
+    tenant_id: str,
+) -> list[str]:
+    """Return all of the tenant's successfully-ingested report-source doc IDs.
+
+    Used to widen RAG retrieval to the tenant's whole library (old + new
+    uploads) when ``settings.rag_use_full_tenant_library`` is enabled.
+    ``style_corpus`` documents are excluded so past-report observations never
+    enter the factual evidence pool. Best-effort: returns ``[]`` on any error so
+    generation degrades to per-report isolation rather than failing.
+    """
+    if db is None:
+        return []
+    try:
+        from sqlalchemy import select as _select
+
+        from app.db.models import Document, DocumentPurpose, IngestStatus
+
+        result = await db.execute(
+            _select(Document.id).where(
+                Document.tenant_id == tenant_id,
+                Document.status == IngestStatus.complete,
+                Document.document_purpose == DocumentPurpose.report_source,
+            )
+        )
+        return [str(row[0]) for row in result.all() if row[0]]
+    except Exception as exc:  # noqa: BLE001 — never break generation on this widening step
+        logger.warning("Full-library doc id lookup failed tenant=%s: %s", tenant_id, exc)
+        return []
+
+
 async def _generate_section_text(
     tenant_id: str,
     template_id: str,
@@ -1627,6 +1659,32 @@ async def _generate_section_text(
         universe.add(str(primary_document_id))
     universe.update(ref_ids)
     universe.update(runtime_extra)
+
+    # Whole-library retrieval (old + new uploads). When enabled and we are NOT
+    # in strict per-report isolation, widen the admissible document set to every
+    # ingested report-source document for this tenant. The report's own upload
+    # (primary_document_id) and explicit references stay prioritised by the
+    # retriever's ordering; style_corpus is excluded by the query above and by
+    # the report-source filter in the lookup helper. Runtime section vectors are
+    # always kept.
+    library_doc_ids: list[str] = []
+    if (
+        getattr(settings, "rag_use_full_tenant_library", True)
+        and not strict_uploaded_only
+    ):
+        library_doc_ids = await _tenant_report_source_doc_ids(db, tenant_id)
+        if library_doc_ids:
+            universe.update(library_doc_ids)
+            logger.info(
+                "Full-library retrieval for section=%s tenant=%s: %d report-source "
+                "doc(s) admissible (primary=%s, explicit_refs=%d)",
+                template_id,
+                tenant_id,
+                len(library_doc_ids),
+                (primary_document_id[:8] if primary_document_id else None),
+                len(ref_ids),
+            )
+
     allowed = frozenset(universe) if universe else None
 
     if not allowed:
@@ -2288,6 +2346,61 @@ async def _generate_section_text(
     return text, provenance, confidence, top_results, doc_ctx_results, metrics
 
 
+async def _attach_citation_audit(
+    *,
+    metrics: dict[str, Any],
+    template_id: str,
+    survey_level: int | None,
+    evidence: list[SearchResult],
+    tenant_id: str,
+) -> None:
+    """Run the deterministic grounding audit and record it in ``metrics``.
+
+    Best-effort and fully isolated: any failure here must never affect the
+    generated section, so the audit is wrapped and only logged. This is an
+    additive traceability artifact, not part of the generation contract.
+    """
+    try:
+        from app.extraction.extractor import audit_section_grounding
+        from app.templates.registry import get_template
+
+        tmpl = get_template(template_id, survey_level)
+        section_name = tmpl.title if tmpl else template_id
+
+        extraction = await audit_section_grounding(
+            section_name=section_name,
+            template_id=template_id,
+            chunks=evidence,
+            tenant_id=tenant_id,
+        )
+        metrics["citation_audit"] = {
+            "section": extraction.section,
+            "confidence": extraction.confidence,
+            "findings": len(extraction.findings),
+            "contradictions": [c.model_dump() for c in extraction.contradictions],
+            "dropped_claims": extraction.dropped_claims,
+        }
+        if extraction.contradictions or extraction.dropped_claims:
+            logger.warning(
+                "Citation audit section=%s confidence=%.2f findings=%d "
+                "contradictions=%d dropped=%d",
+                extraction.section,
+                extraction.confidence,
+                len(extraction.findings),
+                len(extraction.contradictions),
+                len(extraction.dropped_claims),
+            )
+        else:
+            logger.info(
+                "Citation audit section=%s confidence=%.2f findings=%d (clean)",
+                extraction.section,
+                extraction.confidence,
+                len(extraction.findings),
+            )
+    except Exception as exc:  # noqa: BLE001 — audit must never break generation
+        logger.warning("Citation audit skipped for section=%s: %s", template_id, exc)
+
+
 # ── GENERATE mode ──────────────────────────────────────────────────────────────
 
 async def _run_generate(
@@ -2356,6 +2469,7 @@ async def _run_generate(
                 fallback_used=bool(cached.get("fallback_used", fallback_used)),
                 interference_level=_tier_cached or interference_level,
                 mark_report_complete=mark_report_complete,
+                citation_audit=cached.get("citation_audit") if isinstance(cached.get("citation_audit"), dict) else None,
             )
             return
 
@@ -2404,6 +2518,20 @@ async def _run_generate(
         merged_for_meta.append(r)
     provenance = attach_snippet_metadata(provenance, merged_for_meta, filenames)
 
+    # 7b. Optional citation-grounded audit (non-destructive). Runs the
+    # deterministic extraction + contradiction layer over the SAME retrieved
+    # evidence the section was built from, scoped to this section's domain.
+    # It never mutates `text`; it only records traceability (confidence,
+    # contradictions, dropped/ungrounded claims) into metrics for auditing.
+    if settings.enable_citation_extraction and top_results:
+        await _attach_citation_audit(
+            metrics=metrics,
+            template_id=template_id,
+            survey_level=report.survey_level,
+            evidence=list(top_results),
+            tenant_id=tenant_id,
+        )
+
     # 8. Cache & persist (ai_level stored for auditability; it is already part of the key)
     section_cache.set(cache_key, {
         "text": text,
@@ -2417,6 +2545,7 @@ async def _run_generate(
         "pipeline": str(pipeline),
         "fallback_used": bool(fallback_used),
         "interference_level": interference_level,
+        "citation_audit": metrics.get("citation_audit") if isinstance(metrics.get("citation_audit"), dict) else None,
     })
     _bullet_clamps_meta = (
         metrics.get("bullet_clamps")
@@ -2454,6 +2583,7 @@ async def _run_generate(
         mark_report_complete=mark_report_complete,
         bullet_clamps=_bullet_clamps_meta,
         coverage_metrics=_coverage_meta,
+        citation_audit=metrics.get("citation_audit") if isinstance(metrics.get("citation_audit"), dict) else None,
     )
     logger.info(
         "Generated section=%s for report=%s (confidence=%.3f, style=%s, ai=%s%% level=%s)",
@@ -3010,6 +3140,7 @@ async def _persist_section(
     mark_report_complete: bool = True,
     bullet_clamps: dict[str, Any] | None = None,
     coverage_metrics: dict[str, Any] | None = None,
+    citation_audit: dict[str, Any] | None = None,
 ) -> None:
     """Upsert a section row and optionally mark the report as complete.
 
@@ -3070,6 +3201,8 @@ async def _persist_section(
         meta["bullet_clamps"] = bullet_clamps
     if coverage_metrics:
         meta["notes_coverage"] = coverage_metrics
+    if citation_audit:
+        meta["citation_audit"] = citation_audit
     provenance_json = json.dumps({"sources": provenance, "meta": meta})
 
     if section_orm is None and not is_sqlite_database():

app/services/photo_vision.py

@@ -68,7 +68,7 @@ async def _vision_batch_call_async(
     return await chat_completions_create_raw(
         messages=[
             {"role": "system", "content": "You are a careful building inspection assistant."},
-            {"role": "user", "content": [{"type": "input_text", "text": prompt}, *images]},
+            {"role": "user", "content": [{"type": "text", "text": prompt}, *images]},
         ],
         model=model,
         max_tokens=800,
@@ -103,7 +103,9 @@ async def _vision_batches_async(
         images: list[dict] = []
         for p, ct in chunk:
             try:
-                images.append({"type": "input_image", "image_url": _data_url_for_image(p, ct)})
+                images.append(
+                    {"type": "image_url", "image_url": {"url": _data_url_for_image(p, ct)}}
+                )
             except Exception as exc:  # noqa: BLE001
                 logger.debug("Could not read photo for vision: %s", exc)
         if not images:
@@ -177,7 +179,9 @@ def _vision_batches_sync(
         images: list[dict] = []
         for p, ct in chunk:
             try:
-                images.append({"type": "input_image", "image_url": _data_url_for_image(p, ct)})
+                images.append(
+                    {"type": "image_url", "image_url": {"url": _data_url_for_image(p, ct)}}
+                )
             except Exception as exc:  # noqa: BLE001
                 logger.debug("Could not read photo for vision: %s", exc)
         if not images:
@@ -210,7 +214,7 @@ def _vision_batches_sync(
             model=model,
             messages=[
                 {"role": "system", "content": "You are a careful building inspection assistant."},
-                {"role": "user", "content": [{"type": "input_text", "text": prompt}, *images]},
+                {"role": "user", "content": [{"type": "text", "text": prompt}, *images]},
             ],
             temperature=0.1,
         )

app/tests/test_extraction_grounding.py

@@ -0,0 +1,412 @@
+"""Regression tests for the citation-grounded extraction layer.
+
+These tests are the executable success criteria for the anti-hallucination
+guarantees. They are fully deterministic and require NO OpenAI key — they
+exercise the pure-Python validation/contradiction core that gates the LLM.
+
+Covered failure modes (from the spec's TESTING REQUIREMENTS):
+- altered condition ratings
+- fabricated materials
+- invented locations / entity substitution
+- unsupported risk / severity claims
+- contradiction generation (rating / condition / operational / duplicate)
+- malformed / fabricated sentences (paraphrase with no grounding)
+- dropped warranties (positive findings preserved when grounded)
+- fabricated monetary totals
+- citation to a non-existent chunk
+"""
+
+from __future__ import annotations
+
+from app.extraction.citation_validator import (
+    span_in_chunk,
+    validate_finding,
+    validate_findings,
+)
+from app.extraction.contradiction import audit_contradictions
+from app.extraction.schemas import (
+    ConditionRating,
+    ContradictionKind,
+    EvidenceSpan,
+    SupportLevel,
+    SurveyFinding,
+)
+
+
+def _finding(element: str, rating, text: str, *, chunk_id="c1", span=None) -> SurveyFinding:
+    return SurveyFinding(
+        section="Roofing",
+        element=element,
+        condition_rating=rating,
+        finding=text,
+        evidence=[EvidenceSpan(chunk_id=chunk_id, text=span or text)],
+    )
+
+
+# ── ConditionRating enum: closed set, never fabricated ──────────────────────
+
+def test_rating_enum_coerces_unknown_to_na_not_a_guess():
+    assert ConditionRating.coerce("2") is ConditionRating.CR2
+    assert ConditionRating.coerce("CR3") is ConditionRating.CR3
+    # Garbage must degrade to NA, never to a fabricated severity.
+    assert ConditionRating.coerce("urgent") is ConditionRating.NA
+    assert ConditionRating.coerce("") is ConditionRating.NA
+    assert ConditionRating.coerce(None) is ConditionRating.NA
+
+
+def test_finding_rating_is_schema_constrained():
+    f = SurveyFinding(section="Roofing", element="Ridge", condition_rating="severe", finding="x")
+    assert f.condition_rating is ConditionRating.NA
+
+
+# ── span_in_chunk: verbatim + OCR-drift tolerance, but rejects absent text ──
+
+def test_span_match_verbatim_and_drift():
+    chunk = "The main roof covering is natural slate, generally in sound condition."
+    assert span_in_chunk("natural slate", chunk)
+    assert span_in_chunk("The main roof covering is natural slate", chunk)
+    # whitespace/case drift still matches
+    assert span_in_chunk("NATURAL   slate", chunk)
+
+
+def test_span_absent_is_rejected():
+    chunk = "The main roof covering is natural slate."
+    assert not span_in_chunk("concrete interlocking tiles", chunk)
+
+
+# ── Altered condition ratings are caught by the contradiction audit ─────────
+
+def test_altered_rating_conflict_detected_and_resolved():
+    strong = _finding("Main roof", ConditionRating.CR2, "Slate covering shows slipped tiles.")
+    strong.support = SupportLevel.SUPPORTED
+    weak = _finding("Main roof", ConditionRating.CR1, "Slate covering shows slipped tiles.")
+    weak.support = SupportLevel.PARTIAL
+    resolved, reports = audit_contradictions([strong, weak])
+    assert len(resolved) == 1
+    assert resolved[0].condition_rating is ConditionRating.CR2  # stronger evidence kept
+    assert any(r.kind is ContradictionKind.RATING_CONFLICT for r in reports)
+
+
+# ── Fabricated materials / invented locations: entity substitution ──────────
+
+def test_entity_substitution_is_dropped():
+    pool = {"c1": "A London plane tree is located near the rear boundary."}
+    # Model swapped the species — must be rejected as unsupported entity.
+    bad = SurveyFinding(
+        section="Grounds", element="Tree", condition_rating="NA",
+        finding="A Lombardy Poplar is located near the rear boundary.",
+        evidence=[EvidenceSpan(chunk_id="c1", text="located near the rear boundary")],
+    )
+    support, violations = validate_finding(bad, pool)
+    assert support is SupportLevel.NOT_FOUND
+    assert any("entity" in v for v in violations)
+
+
+def test_correct_entity_is_supported():
+    pool = {"c1": "A London plane tree is located near the rear boundary."}
+    good = SurveyFinding(
+        section="Grounds", element="Tree", condition_rating="NA",
+        finding="A London plane tree is located near the rear boundary.",
+        evidence=[EvidenceSpan(chunk_id="c1", text="A London plane tree is located near the rear boundary")],
+    )
+    support, violations = validate_finding(good, pool)
+    assert support is SupportLevel.SUPPORTED
+    assert violations == []
+
+
+# ── Fabricated monetary totals / numbers ────────────────────────────────────
+
+def test_fabricated_total_is_dropped():
+    pool = {"c1": "Repairs to the parapet are recommended."}
+    bad = SurveyFinding(
+        section="Roofing", element="Parapet", condition_rating="2",
+        finding="Repairs to the parapet are recommended at a cost of £12,500.",
+        evidence=[EvidenceSpan(chunk_id="c1", text="Repairs to the parapet are recommended")],
+    )
+    support, violations = validate_finding(bad, pool)
+    assert support is SupportLevel.NOT_FOUND
+    assert any("number" in v or "amount" in v for v in violations)
+
+
+def test_grounded_total_is_preserved():
+    pool = {"c1": "Repairs to the parapet are recommended at a cost of £12,500."}
+    good = SurveyFinding(
+        section="Roofing", element="Parapet", condition_rating="2",
+        finding="Repairs to the parapet are recommended at a cost of £12,500.",
+        evidence=[EvidenceSpan(chunk_id="c1", text="Repairs to the parapet are recommended at a cost of £12,500")],
+    )
+    support, _ = validate_finding(good, pool)
+    assert support is SupportLevel.SUPPORTED
+
+
+# ── Unsupported risk / severity escalation ──────────────────────────────────
+
+def test_unsupported_severity_is_dropped():
+    pool = {"c1": "There is minor surface staining to the ceiling."}
+    bad = SurveyFinding(
+        section="Interior", element="Ceiling", condition_rating="2",
+        finding="There is minor surface staining to the ceiling, a catastrophic and unsafe defect.",
+        evidence=[EvidenceSpan(chunk_id="c1", text="There is minor surface staining to the ceiling")],
+    )
+    support, violations = validate_finding(bad, pool)
+    assert support is SupportLevel.NOT_FOUND
+    assert any("severity" in v for v in violations)
+
+
+def test_severity_allowed_when_in_source():
+    pool = {"c1": "The boiler flue is unsafe and must not be used."}
+    good = SurveyFinding(
+        section="Services", element="Boiler flue", condition_rating="3",
+        finding="The boiler flue is unsafe and must not be used.",
+        evidence=[EvidenceSpan(chunk_id="c1", text="The boiler flue is unsafe and must not be used")],
+    )
+    support, _ = validate_finding(good, pool)
+    assert support is SupportLevel.SUPPORTED
+
+
+# ── Citation to a non-existent chunk ────────────────────────────────────────
+
+def test_citation_to_unknown_chunk_is_rejected():
+    pool = {"c1": "Slate covering is sound."}
+    bad = SurveyFinding(
+        section="Roofing", element="Covering", condition_rating="1",
+        finding="Slate covering is sound.",
+        evidence=[EvidenceSpan(chunk_id="ghost", text="Slate covering is sound")],
+    )
+    support, violations = validate_finding(bad, pool)
+    assert support is SupportLevel.NOT_FOUND
+    assert any("unknown chunk_id" in v for v in violations)
+
+
+# ── Malformed / paraphrased fabrication with no grounding ───────────────────
+
+def test_ungrounded_paraphrase_is_dropped():
+    pool = {"c1": "The flat roof is covered in felt."}
+    bad = SurveyFinding(
+        section="Roofing", element="Flat roof", condition_rating="2",
+        finding="Extensive structural movement threatens imminent failure of the dwelling.",
+        evidence=[EvidenceSpan(chunk_id="c1", text="The flat roof is covered in felt")],
+    )
+    support, _ = validate_finding(bad, pool)
+    assert support is SupportLevel.NOT_FOUND
+
+
+# ── Positive findings / warranties preserved when grounded ──────────────────
+
+def test_warranty_preserved():
+    pool = {"c1": "The replacement boiler was fitted in 2021 and carries a 10 year manufacturer warranty."}
+    good = SurveyFinding(
+        section="Services", element="Boiler", condition_rating="1",
+        finding="The replacement boiler was fitted in 2021 and carries a 10 year manufacturer warranty.",
+        evidence=[EvidenceSpan(
+            chunk_id="c1",
+            text="The replacement boiler was fitted in 2021 and carries a 10 year manufacturer warranty",
+        )],
+    )
+    support, _ = validate_finding(good, pool)
+    assert support is SupportLevel.SUPPORTED
+
+
+# ── Contradiction: satisfactory vs defective ────────────────────────────────
+
+def test_satisfactory_vs_defective_contradiction():
+    a = _finding("Gutters", ConditionRating.NA, "The gutters are in good condition and sound.")
+    a.support = SupportLevel.SUPPORTED
+    b = _finding("Gutters", ConditionRating.NA, "The gutters are defective and leaking badly.")
+    b.support = SupportLevel.NOT_FOUND
+    resolved, reports = audit_contradictions([a, b])
+    assert len(resolved) == 1
+    assert resolved[0] is a  # evidence-stronger side kept
+    assert any(r.kind is ContradictionKind.CONDITION for r in reports)
+
+
+# ── Contradiction: operational vs non-operational ───────────────────────────
+
+def test_operational_contradiction():
+    a = _finding("Heating", ConditionRating.NA, "The heating system is fully operational and working.")
+    a.support = SupportLevel.SUPPORTED
+    b = _finding("Heating", ConditionRating.NA, "The heating system is not operational and out of order.")
+    b.support = SupportLevel.PARTIAL
+    resolved, reports = audit_contradictions([a, b])
+    assert len(resolved) == 1
+    assert any(r.kind is ContradictionKind.OPERATIONAL for r in reports)
+
+
+# ── Duplicate collapse ──────────────────────────────────────────────────────
+
+def test_duplicate_findings_collapsed():
+    a = _finding("Chimney", ConditionRating.CR2, "The chimney stack shows perished pointing requiring repair.")
+    b = _finding("Chimney", ConditionRating.CR2, "The chimney stack shows perished pointing requiring repair.")
+    resolved, reports = audit_contradictions([a, b])
+    assert len(resolved) == 1
+    assert any(r.kind is ContradictionKind.DUPLICATE for r in reports)
+
+
+# ── End-to-end gate: mixed batch keeps only grounded, non-conflicting ───────
+
+def test_validate_findings_batch_drops_unsupported():
+    pool = {
+        "c1": "The main roof is natural slate in sound condition.",
+        "c2": "The rear addition roof is covered in felt with no visible defects.",
+    }
+    findings = [
+        SurveyFinding(section="Roofing", element="Main roof", condition_rating="1",
+                      finding="The main roof is natural slate in sound condition.",
+                      evidence=[EvidenceSpan(chunk_id="c1", text="The main roof is natural slate in sound condition")]),
+        SurveyFinding(section="Roofing", element="Rear roof", condition_rating="3",
+                      finding="The rear addition roof has collapsed and is deadly.",
+                      evidence=[EvidenceSpan(chunk_id="c2", text="The rear addition roof is covered in felt")]),
+    ]
+    kept, dropped = validate_findings(findings, pool)
+    assert len(kept) == 1
+    assert kept[0].element == "Main roof"
+    assert len(dropped) == 1
+
+
+"""Section-domain scoping (STEP 6) — prevent cross-section contamination."""
+
+
+def test_classify_section_aliases_and_keywords():
+    from app.extraction.domain_scope import classify_section
+
+    assert classify_section("Roof coverings") == "roofing"
+    assert classify_section("Chimney stacks") == "chimney"
+    assert classify_section("Rainwater pipes and gutters") == "rainwater"
+    assert classify_section("Electricity") == "electrical"
+    assert classify_section("About the property") == "general"
+
+
+def test_classify_text_dominant_domain():
+    from app.extraction.domain_scope import classify_text
+
+    assert classify_text("The natural slate roof covering has slipped tiles at the ridge.") == "roofing"
+    assert classify_text("The consumer unit lacks RCD protection on the circuits.") == "electrical"
+    assert classify_text("This paragraph is generic boilerplate with no domain.") == "general"
+
+
+def test_scope_chunks_excludes_foreign_domain():
+    from app.extraction.domain_scope import scope_chunks
+
+    class _Row:
+        def __init__(self, text):
+            self.text = text
+
+    chunks = [
+        _Row("The slate roof covering is sound at the ridge and eaves."),
+        _Row("The drainage manhole and inspection chamber were inspected."),
+        _Row("General introductory text about the inspection."),
+    ]
+    kept = scope_chunks("roofing", chunks)
+    texts = [c.text for c in kept]
+    assert any("slate roof" in t for t in texts)
+    assert any("introductory" in t for t in texts)  # general is admissible
+    assert not any("manhole" in t for t in texts)   # drainage excluded
+
+
+def test_scope_chunks_never_starves_under_strict():
+    from app.extraction.domain_scope import scope_chunks
+
+    class _Row:
+        def __init__(self, text):
+            self.text = text
+
+    # All chunks belong to a different domain -> strict returns originals
+    # rather than an empty pool.
+    chunks = [_Row("The consumer unit and wiring circuits were inspected.")]
+    kept = scope_chunks("roofing", chunks, strict=True)
+    assert len(kept) == 1
+    kept_nonstrict = scope_chunks("roofing", chunks, strict=False)
+    assert kept_nonstrict == []
+
+
+def test_section_extraction_confidence():
+    pool = {"c1": "The main roof is natural slate in sound condition."}
+    f = SurveyFinding(section="Roofing", element="Main roof", condition_rating="1",
+                      finding="The main roof is natural slate in sound condition.",
+                      evidence=[EvidenceSpan(chunk_id="c1", text="The main roof is natural slate in sound condition")])
+    kept, _ = validate_findings([f], pool)
+    from app.extraction.schemas import SectionExtraction
+    sec = SectionExtraction(section="Roofing", findings=kept)
+    assert sec.confidence == 1.0
+
+
+"""Post-generation output validator + abstention (forbidden phrases / metrics)."""
+
+
+def test_output_validator_flags_forbidden_phrases():
+    from app.extraction.output_validator import find_violations, is_clean
+
+    bad = "The roof appears to be defective and overall reliability is questionable."
+    v = find_violations(bad, evidence="")
+    assert any("appears to" in x for x in v)
+    assert any("overall" in x for x in v)
+    assert not is_clean(bad)
+
+
+def test_output_validator_flags_fabricated_percentage_and_score():
+    from app.extraction.output_validator import find_violations
+
+    v = find_violations("Authenticity score is high with 87% confidence.", evidence="")
+    assert any("percentage" in x for x in v)
+    assert any("metric" in x for x in v)
+
+
+def test_output_validator_allows_terms_present_in_evidence():
+    from app.extraction.output_validator import is_clean
+
+    # "unsafe" is permitted because it is verbatim in the evidence.
+    text = "The flue is unsafe."
+    evidence = "The boiler flue is unsafe and must not be used."
+    assert is_clean(text, evidence)
+
+
+def test_output_validator_clean_text_passes():
+    from app.extraction.output_validator import is_clean
+
+    assert is_clean("The main roof covering is natural slate.", evidence="")
+
+
+def test_abstain_on_low_confidence_and_attribution_failure():
+    from app.extraction.output_validator import should_abstain
+
+    assert should_abstain("clean text", "", confidence=0.5, min_confidence=1.0)
+    assert should_abstain("clean text", "", evidence_aligned=False)
+    assert should_abstain("clean text", "", source_attributed=False)
+    assert not should_abstain("The roof is slate.", "", confidence=1.0)
+
+
+def test_findings_to_atomic_claims_grounded():
+    from app.extraction.extractor import findings_to_atomic_claims
+    from app.extraction.schemas import ClaimType
+
+    f = SurveyFinding(
+        section="Roofing", element="Main roof", condition_rating="2",
+        finding="The main roof covering is natural slate with slipped tiles.",
+        evidence=[EvidenceSpan(chunk_id="c1", page=12, section_label="Page 12",
+                               text="The main roof covering is natural slate with slipped tiles")],
+    )
+    f.support = SupportLevel.SUPPORTED
+    claims = findings_to_atomic_claims([f])
+    types = {c.claim_type for c in claims}
+    assert ClaimType.CONDITION_RATING in types
+    assert ClaimType.OBSERVATION in types
+    rating = next(c for c in claims if c.claim_type is ClaimType.CONDITION_RATING)
+    assert "is 2" in rating.claim
+    assert rating.evidence.chunk_id == "c1"
+    assert rating.evidence.page == 12
+    assert rating.verification.supported is True
+    assert rating.verification.confidence == 1.0
+
+
+def test_findings_to_atomic_claims_abstains_below_confidence():
+    from app.extraction.extractor import findings_to_atomic_claims
+
+    f = SurveyFinding(
+        section="Roofing", element="Main roof", condition_rating="2",
+        finding="The main roof covering is natural slate.",
+        evidence=[EvidenceSpan(chunk_id="c1", text="The main roof covering is natural slate")],
+    )
+    f.support = SupportLevel.PARTIAL  # confidence 0.5 < default min 1.0
+    assert findings_to_atomic_claims([f], min_confidence=1.0) == []
+    # Lowering the bar admits them.
+    assert findings_to_atomic_claims([f], min_confidence=0.5)

app/tests/test_full_library_retrieval.py

@@ -0,0 +1,253 @@
+"""Regression tests: report generation must use old + new tenant RAG uploads.
+
+When ``rag_use_full_tenant_library`` is enabled (default), every user's report
+generation widens retrieval to all ingested ``report_source`` documents for
+that tenant — not only the file attached when the report was created.
+"""
+
+from __future__ import annotations
+
+import uuid
+from unittest.mock import MagicMock, patch
+
+import pytest
+import pytest_asyncio
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+
+from app.db.database import Base
+from app.db.models import Document, DocumentPurpose, IngestStatus, Report, ReportStatus
+from app.generator.adapter import MockLLMAdapter
+from app.models.schemas import WritingStyleProfile
+from app.services.generation import (
+    _generate_section_text,
+    _tenant_report_source_doc_ids,
+)
+
+_MOCK_PROFILE = WritingStyleProfile(
+    tone="formal",
+    formality_level="professional",
+    avg_sentence_complexity="moderate",
+    vocabulary_level="technical",
+    common_phrases=[],
+    structural_patterns=[],
+    writing_style_summary="Professional RICS surveyor style.",
+)
+
+
+@pytest_asyncio.fixture
+async def memory_session() -> AsyncSession:
+    engine = create_async_engine("sqlite+aiosqlite:///:memory:", echo=False)
+    async with engine.begin() as conn:
+        await conn.run_sync(Base.metadata.create_all)
+    factory = async_sessionmaker(engine, expire_on_commit=False)
+    async with factory() as session:
+        yield session
+    await engine.dispose()
+
+
+def _doc(
+    *,
+    tenant_id: str,
+    status: IngestStatus = IngestStatus.complete,
+    purpose: DocumentPurpose = DocumentPurpose.report_source,
+) -> Document:
+    return Document(
+        id=str(uuid.uuid4()),
+        tenant_id=tenant_id,
+        filename="survey.pdf",
+        file_path="/tmp/survey.pdf",
+        status=status,
+        document_purpose=purpose,
+    )
+
+
+@pytest.mark.asyncio
+async def test_tenant_report_source_doc_ids_includes_all_complete_report_source(
+    memory_session: AsyncSession,
+) -> None:
+    tenant = "tenant_alpha"
+    old = _doc(tenant_id=tenant)
+    new = _doc(tenant_id=tenant)
+    pending = _doc(tenant_id=tenant, status=IngestStatus.pending)
+    style = _doc(tenant_id=tenant, purpose=DocumentPurpose.style_corpus)
+    memory_session.add_all([old, new, pending, style])
+    await memory_session.commit()
+
+    ids = await _tenant_report_source_doc_ids(memory_session, tenant)
+
+    assert set(ids) == {old.id, new.id}
+    assert pending.id not in ids
+    assert style.id not in ids
+
+
+@pytest.mark.asyncio
+async def test_tenant_report_source_doc_ids_isolated_per_tenant(
+    memory_session: AsyncSession,
+) -> None:
+    doc_a = _doc(tenant_id="tenant_A")
+    doc_b = _doc(tenant_id="tenant_B")
+    memory_session.add_all([doc_a, doc_b])
+    await memory_session.commit()
+
+    ids_a = await _tenant_report_source_doc_ids(memory_session, "tenant_A")
+    ids_b = await _tenant_report_source_doc_ids(memory_session, "tenant_B")
+
+    assert ids_a == [doc_a.id]
+    assert ids_b == [doc_b.id]
+
+
+@pytest.mark.asyncio
+async def test_tenant_report_source_doc_ids_returns_empty_without_db() -> None:
+    assert await _tenant_report_source_doc_ids(None, "any") == []
+
+
+@pytest.mark.asyncio
+async def test_generate_section_text_search_includes_old_library_doc(
+    memory_session: AsyncSession,
+) -> None:
+    """Paragraph retrieval must pass old + new doc IDs in doc_id_in."""
+    tenant = "tenant_lib"
+    primary = _doc(tenant_id=tenant)
+    old_upload = _doc(tenant_id=tenant)
+    memory_session.add_all([primary, old_upload])
+    report = Report(
+        id=str(uuid.uuid4()),
+        tenant_id=tenant,
+        document_id=primary.id,
+        status=ReportStatus.generating,
+    )
+    memory_session.add(report)
+    await memory_session.commit()
+
+    captured: list[frozenset[str] | None] = []
+
+    def _search(*_args, **kwargs):  # noqa: ANN002, ANN003
+        captured.append(kwargs.get("doc_id_in"))
+        return []
+
+    fake_vs = MagicMock(search=_search)
+
+    with (
+        patch("app.services.generation.settings.rag_use_full_tenant_library", True),
+        patch("app.services.generation.settings.hierarchical_rag_enabled", True),
+        patch("app.services.generation.get_vectorstore", return_value=fake_vs),
+        patch("app.services.generation.rerank", return_value=[]),
+        patch("app.llm.generation_facade.get_llm_adapter", return_value=MockLLMAdapter()),
+        patch(
+            "app.services.generation.get_template",
+            return_value=MagicMock(skeleton="[D]: [content]."),
+        ),
+    ):
+        await _generate_section_text(
+            tenant_id=tenant,
+            template_id="D",
+            bullets=["Semi-detached property"],
+            style_profile=_MOCK_PROFILE,
+            primary_document_id=primary.id,
+            reference_document_ids=[],
+            retrieval_level="paragraph",
+            db=memory_session,
+            strict_uploaded_only=False,
+        )
+
+    assert captured, "vectorstore.search should have been called"
+    allowed = captured[0]
+    assert allowed is not None
+    assert primary.id in allowed
+    assert old_upload.id in allowed
+
+
+@pytest.mark.asyncio
+async def test_generate_section_text_strict_mode_excludes_library_widening(
+    memory_session: AsyncSession,
+) -> None:
+    tenant = "tenant_strict"
+    primary = _doc(tenant_id=tenant)
+    old_upload = _doc(tenant_id=tenant)
+    memory_session.add_all([primary, old_upload])
+    await memory_session.commit()
+
+    captured: list[frozenset[str] | None] = []
+
+    def _search(*_args, **kwargs):  # noqa: ANN002, ANN003
+        captured.append(kwargs.get("doc_id_in"))
+        return []
+
+    fake_vs = MagicMock(search=_search)
+
+    with (
+        patch("app.services.generation.settings.rag_use_full_tenant_library", True),
+        patch("app.services.generation.settings.hierarchical_rag_enabled", True),
+        patch("app.services.generation.get_vectorstore", return_value=fake_vs),
+        patch("app.services.generation.rerank", return_value=[]),
+        patch("app.llm.generation_facade.get_llm_adapter", return_value=MockLLMAdapter()),
+        patch(
+            "app.services.generation.get_template",
+            return_value=MagicMock(skeleton="[D]: [content]."),
+        ),
+    ):
+        await _generate_section_text(
+            tenant_id=tenant,
+            template_id="D",
+            bullets=["Test"],
+            style_profile=_MOCK_PROFILE,
+            primary_document_id=primary.id,
+            reference_document_ids=[],
+            retrieval_level="paragraph",
+            db=memory_session,
+            strict_uploaded_only=True,
+        )
+
+    assert captured
+    allowed = captured[0]
+    assert allowed is not None
+    assert primary.id in allowed
+    assert old_upload.id not in allowed
+
+
+@pytest.mark.asyncio
+async def test_generate_section_text_flag_off_excludes_library_widening(
+    memory_session: AsyncSession,
+) -> None:
+    tenant = "tenant_flag_off"
+    primary = _doc(tenant_id=tenant)
+    old_upload = _doc(tenant_id=tenant)
+    memory_session.add_all([primary, old_upload])
+    await memory_session.commit()
+
+    captured: list[frozenset[str] | None] = []
+
+    def _search(*_args, **kwargs):  # noqa: ANN002, ANN003
+        captured.append(kwargs.get("doc_id_in"))
+        return []
+
+    fake_vs = MagicMock(search=_search)
+
+    with (
+        patch("app.services.generation.settings.rag_use_full_tenant_library", False),
+        patch("app.services.generation.settings.hierarchical_rag_enabled", True),
+        patch("app.services.generation.get_vectorstore", return_value=fake_vs),
+        patch("app.services.generation.rerank", return_value=[]),
+        patch("app.llm.generation_facade.get_llm_adapter", return_value=MockLLMAdapter()),
+        patch(
+            "app.services.generation.get_template",
+            return_value=MagicMock(skeleton="[D]: [content]."),
+        ),
+    ):
+        await _generate_section_text(
+            tenant_id=tenant,
+            template_id="D",
+            bullets=["Test"],
+            style_profile=_MOCK_PROFILE,
+            primary_document_id=primary.id,
+            reference_document_ids=[],
+            retrieval_level="paragraph",
+            db=memory_session,
+            strict_uploaded_only=False,
+        )
+
+    assert captured
+    allowed = captured[0]
+    assert allowed is not None
+    assert primary.id in allowed
+    assert old_upload.id not in allowed

app/tests/test_generator.py

@@ -139,7 +139,9 @@ def test_enforce_verify_wraps_invented_number() -> None:
         snippets=[],
     )
     assert "120" not in result
-    assert result == "The property has  floor area."
+    # enforce_verify collapses the whitespace left where the invented number
+    # was removed (\s{2,} -> single space), so no double space survives.
+    assert result == "The property has floor area."
 
 
 def test_enforce_verify_wraps_invented_entity() -> None:

app/tests/test_ingest_normalize.py

@@ -0,0 +1,135 @@
+"""Regression tests for OCR normalization, table preservation, and chunking.
+
+Covers the spec's remaining failure modes:
+- OCR contamination (running headers/footers, page numbers, hyphenation)
+- malformed sentences (hard-wrap unwrap)
+- missing tables / chunk-boundary corruption (tables kept intact, never split)
+
+All deterministic and dependency-light (no live PDF / network needed).
+"""
+
+from __future__ import annotations
+
+from langchain_core.documents import Document
+
+from app.chunking.splitter import split_documents
+from app.ingest.ocr_normalize import (
+    TABLE_CLOSE,
+    TABLE_OPEN,
+    normalize_pages,
+    normalize_text,
+    strip_running_headers_footers,
+)
+from app.ingest.parser_pdf import _table_to_markdown
+
+
+# ── OCR normalization ───────────────────────────────────────────────────────
+
+def test_dehyphenation_repairs_wrapped_words():
+    out = normalize_text("The roof is in poor condi-\ntion at the ridge.")
+    assert "condition" in out
+    assert "condi-" not in out
+
+
+def test_unwrap_restores_sentence_continuity():
+    raw = "The main roof covering\nis natural slate and\nremains in sound condition."
+    out = normalize_text(raw)
+    assert out == "The main roof covering is natural slate and remains in sound condition."
+
+
+def test_paragraph_breaks_preserved():
+    raw = "First paragraph line one\nline two.\n\nSecond paragraph here."
+    out = normalize_text(raw)
+    assert out == "First paragraph line one line two.\n\nSecond paragraph here."
+
+
+def test_table_block_preserved_verbatim_through_normalization():
+    table = f"{TABLE_OPEN}\n| A | B |\n| --- | --- |\n| 1 | 2 |\n{TABLE_CLOSE}"
+    raw = f"Some intro text that is\nhard wrapped.\n\n{table}\n\nTrailing note."
+    out = normalize_text(raw)
+    assert table in out  # untouched
+    assert "intro text that is hard wrapped." in out  # prose still reflowed
+
+
+def test_running_headers_and_page_numbers_stripped():
+    pages = [
+        "ACME Surveyors Ltd\nRoof section content for page one.\nPage 1 of 3",
+        "ACME Surveyors Ltd\nDrainage content for page two.\nPage 2 of 3",
+        "ACME Surveyors Ltd\nElectrical content for page three.\nPage 3 of 3",
+    ]
+    cleaned = strip_running_headers_footers(pages)
+    joined = "\n".join(cleaned)
+    assert "ACME Surveyors Ltd" not in joined  # repeated header removed
+    assert "Page 1 of 3" not in joined  # page numbers removed
+    assert "Roof section content" in joined  # real content kept
+
+
+def test_normalize_pages_single_page_keeps_content():
+    pages = ["Only one page here.\nWith a wrapped line."]
+    out = normalize_pages(pages)
+    assert out == ["Only one page here. With a wrapped line."]
+
+
+# ── Table rendering ─────────────────────────────────────────────────────────
+
+def test_table_to_markdown_handles_none_and_pipes():
+    rows = [["Element", "Rating"], ["Roof | main", None], ["Drains", "2"]]
+    md = _table_to_markdown(rows)
+    lines = md.splitlines()
+    assert lines[0] == "| Element | Rating |"
+    assert lines[1] == "| --- | --- |"
+    assert r"Roof \| main" in md  # pipe escaped
+    assert "| Drains | 2 |" in lines[-1]
+
+
+def test_table_to_markdown_pads_ragged_rows():
+    md = _table_to_markdown([["A", "B", "C"], ["1"]])
+    # ragged body row padded to header width
+    assert "| 1 |  |  |" in md
+
+
+# ── Table-aware chunking ────────────────────────────────────────────────────
+
+def _table(rows: int) -> str:
+    body = "\n".join(f"| Element{i} | Rating{i} | Cost{i} |" for i in range(rows))
+    return f"{TABLE_OPEN}\n| Element | Rating | Cost |\n| --- | --- | --- |\n{body}\n{TABLE_CLOSE}"
+
+
+def test_table_kept_as_single_chunk():
+    table = _table(rows=3)
+    doc = Document(page_content=f"Intro prose.\n\n{table}\n\nOutro prose.", metadata={"doc": "d1"})
+    chunks = split_documents([doc], chunk_size=200, chunk_overlap=0)
+    table_chunks = [c for c in chunks if c.metadata.get("section_type") == "table"]
+    assert len(table_chunks) == 1
+    assert table_chunks[0].page_content.startswith(TABLE_OPEN)
+    assert table_chunks[0].page_content.rstrip().endswith(TABLE_CLOSE)
+
+
+def test_large_table_not_split_across_chunks():
+    # A table far larger than chunk_size must still emit exactly one chunk.
+    table = _table(rows=60)
+    doc = Document(page_content=f"Heading.\n\n{table}", metadata={})
+    chunks = split_documents([doc], chunk_size=100, chunk_overlap=0)
+    table_chunks = [c for c in chunks if c.metadata.get("section_type") == "table"]
+    assert len(table_chunks) == 1
+    assert table_chunks[0].page_content.count(TABLE_OPEN) == 1
+    assert table_chunks[0].page_content.count(TABLE_CLOSE) == 1
+
+
+def test_prose_without_table_unaffected():
+    doc = Document(page_content="A simple paragraph of prose with no tables.", metadata={})
+    chunks = split_documents([doc], chunk_size=200, chunk_overlap=0)
+    assert len(chunks) == 1
+    assert chunks[0].metadata.get("section_type") != "table"
+
+
+def test_table_and_prose_order_preserved():
+    table = _table(rows=2)
+    doc = Document(page_content=f"Before text.\n\n{table}\n\nAfter text.", metadata={})
+    chunks = split_documents([doc], chunk_size=300, chunk_overlap=0)
+    kinds = ["table" if c.metadata.get("section_type") == "table" else "prose" for c in chunks]
+    assert "table" in kinds
+    # the table chunk is between prose chunks
+    t_idx = kinds.index("table")
+    assert any(k == "prose" for k in kinds[:t_idx])
+    assert any(k == "prose" for k in kinds[t_idx + 1:])

frontend/index.html

@@ -274,6 +274,22 @@
     .cite-chip:hover { border-color: var(--navy); background: #f0f4fa; }
     .cite-chip kbd { font-size: .68rem; opacity: .75; }
 
+    /* ── Auth gate ─────────────────────────────────────────────────────── */
+    .auth-overlay { position: fixed; inset: 0; z-index: 9800; display: flex; align-items: center; justify-content: center; padding: 1.5rem; background: linear-gradient(135deg, #0d1a28, #14304a); }
+    .auth-overlay.hidden-gate { display: none; }
+    body.auth-gate { overflow: hidden; }
+    .auth-card { width: min(420px, 100%); background: var(--card); border: 1.5px solid var(--border); border-radius: 14px; box-shadow: 0 18px 60px rgba(0,0,0,.35); padding: 2rem 1.8rem; }
+    .auth-brand { font-size: 1.4rem; font-weight: 800; color: var(--navy); text-align: center; }
+    .auth-brand span { color: var(--teal); }
+    .auth-logo { font-size: 1.5rem; }
+    .auth-tagline { font-size: .85rem; color: var(--muted); text-align: center; margin: .5rem 0 1.4rem; line-height: 1.5; }
+    .auth-tabs { display: flex; gap: .4rem; background: #eef2f8; border-radius: 8px; padding: .25rem; margin-bottom: 1.2rem; }
+    .auth-tab { flex: 1; border: none; background: transparent; padding: .55rem; border-radius: 6px; font-size: .85rem; font-weight: 600; color: var(--muted); cursor: pointer; transition: all var(--transition); }
+    .auth-tab.active { background: var(--card); color: var(--navy); box-shadow: 0 1px 4px rgba(0,0,0,.08); }
+    .auth-submit { width: 100%; margin-top: .4rem; justify-content: center; }
+    html.dark .auth-tabs { background: #18202c; }
+    html.dark .auth-card { background: #0f1c2b; }
+
     .app-toast { position: fixed; bottom: 1.5rem; right: 1.5rem; max-width: 22rem; padding: .75rem 1rem; border-radius: 8px; color: #fff; font-size: .88rem; z-index: 9999; box-shadow: 0 4px 20px rgba(0,0,0,.15); transition: opacity .25s ease; }
     .app-toast.hidden { opacity: 0; pointer-events: none; }
     .app-toast.toast-ok { background: #1a5f4a; }
@@ -294,6 +310,39 @@
     .similar-modal-close { border: none; background: transparent; font-size: 1.5rem; line-height: 1; cursor: pointer; color: var(--muted); padding: 0 .25rem; }
     .similar-modal-close:hover { color: var(--danger); }
     .similar-modal-body { padding: 1rem 1.2rem 1.4rem; overflow-y: auto; flex: 1; }
+
+    /* ── RAG Document Manager ─────────────────────────────────────────── */
+    .docmgr-toolbar { display: flex; align-items: center; justify-content: space-between; gap: .75rem; margin-bottom: .75rem; flex-wrap: wrap; }
+    .docmgr-count { font-size: .8rem; color: var(--muted); font-weight: 600; }
+    .docmgr-table { width: 100%; border-collapse: collapse; font-size: .82rem; }
+    .docmgr-table th { text-align: left; font-size: .68rem; text-transform: uppercase; letter-spacing: .05em; color: var(--muted); font-weight: 700; padding: .4rem .55rem; border-bottom: 1.5px solid var(--border); white-space: nowrap; }
+    .docmgr-table td { padding: .55rem .55rem; border-bottom: 1px solid var(--border); vertical-align: middle; }
+    .docmgr-table tr:last-child td { border-bottom: none; }
+    .docmgr-table tr.row-deleting { opacity: .5; pointer-events: none; }
+    .docmgr-fname { font-weight: 600; color: var(--navy); word-break: break-word; max-width: 22rem; }
+    .docmgr-id { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: .68rem; color: var(--muted); }
+    .docmgr-meta { font-size: .72rem; color: var(--muted); white-space: nowrap; }
+    .docmgr-badge { display: inline-block; font-size: .66rem; font-weight: 700; padding: .12rem .5rem; border-radius: 10px; text-transform: uppercase; letter-spacing: .03em; white-space: nowrap; }
+    .docmgr-badge.b-ready { background: #e3f4ec; color: #1a5f4a; }
+    .docmgr-badge.b-processing { background: #fef6e0; color: #8a6d3b; }
+    .docmgr-badge.b-pending { background: #eef2f8; color: var(--navy-lt); }
+    .docmgr-badge.b-failed { background: #fdecea; color: #b71c1c; }
+    .docmgr-badge.b-session { background: var(--teal); color: #fff; margin-left: .35rem; }
+    .docmgr-badge.b-style { background: #ece3f7; color: #5b3a8a; }
+    .docmgr-del { font-size: .76rem; font-weight: 600; padding: .32rem .7rem; border-radius: 6px; border: 1.5px solid var(--danger); background: #fff; color: var(--danger); cursor: pointer; white-space: nowrap; transition: all var(--transition); }
+    .docmgr-del:hover { background: var(--danger); color: #fff; }
+    .docmgr-del:disabled { opacity: .5; cursor: default; }
+    .docmgr-empty { text-align: center; padding: 2.5rem 1rem; color: var(--muted); }
+    .docmgr-empty .icon { font-size: 2.2rem; display: block; margin-bottom: .6rem; }
+    .docmgr-spinner { width: 1.05rem; height: 1.05rem; border: 2.5px solid rgba(0,0,0,.15); border-top-color: var(--navy); border-radius: 50%; display: inline-block; animation: docmgr-spin .7s linear infinite; vertical-align: middle; }
+    @keyframes docmgr-spin { to { transform: rotate(360deg); } }
+    .docmgr-loading { text-align: center; padding: 2rem 1rem; color: var(--muted); }
+    html.dark .docmgr-badge.b-ready { background: #14352a; color: #6fdab2; }
+    html.dark .docmgr-badge.b-processing { background: #3a2f17; color: #e6c878; }
+    html.dark .docmgr-badge.b-pending { background: #18202c; color: #9db4d0; }
+    html.dark .docmgr-badge.b-failed { background: #3a1715; color: #f0a39d; }
+    html.dark .docmgr-badge.b-style { background: #2a1f3a; color: #c5a8e8; }
+    html.dark .similar-modal-head[style*="fdecea"], html.dark #doc-delete-modal .similar-modal-head { background: #3a1715 !important; }
     .similar-intro { font-size: .86rem; color: var(--muted); margin: 0 0 1rem; line-height: 1.5; }
     .similar-h4 { margin: 1.2rem 0 .6rem; font-size: .88rem; color: var(--navy); text-transform: uppercase; letter-spacing: .04em; }
     .similar-h4:first-of-type { margin-top: 0; }
@@ -554,7 +603,7 @@
     .divider { border: none; border-top: 1.5px solid var(--border); margin: 1.2rem 0; }
   </style>
 </head>
-<body>
+<body class="auth-gate">
 
 <header>
   <div class="logo">
@@ -564,12 +613,46 @@
   <span class="subtitle">RICS Survey Report Generator</span>
   <div class="spacer"></div>
   <div class="tenant-badge" id="tenant-badge">Loading…</div>
+  <button class="theme-toggle" id="btn-doc-manager" onclick="openDocManager()" title="View and delete your uploaded RAG documents">
+    <span aria-hidden="true">🗂️</span>
+    <span>My Documents</span>
+  </button>
+  <button class="theme-toggle" id="btn-logout" onclick="logout()" title="Sign out" style="display:none">
+    <span aria-hidden="true">🚪</span>
+    <span>Log out</span>
+  </button>
   <button class="theme-toggle" id="theme-toggle-btn" onclick="toggleTheme()" title="Switch between light and dark mode">
     <span class="th-icon" id="theme-icon">☀️</span>
     <span id="theme-label">Light</span>
   </button>
 </header>
 
+<!-- ── Auth gate ─────────────────────────────────────────────────────────── -->
+<div id="auth-overlay" class="auth-overlay" aria-hidden="false">
+  <div class="auth-card">
+    <div class="auth-brand"><span class="auth-logo">📋</span> Report Genius <span>AI</span></div>
+    <p class="auth-tagline">Sign in to your private workspace. Your documents and reports are isolated to your account.</p>
+    <div class="auth-tabs">
+      <button type="button" class="auth-tab active" id="auth-tab-login" onclick="setAuthMode('login')">Log in</button>
+      <button type="button" class="auth-tab" id="auth-tab-register" onclick="setAuthMode('register')">Create account</button>
+    </div>
+    <form id="auth-form" onsubmit="submitAuth(event)" autocomplete="on">
+      <div class="form-group">
+        <label for="auth-tenant">Tenant / User ID</label>
+        <input type="text" id="auth-tenant" name="username" autocomplete="username" placeholder="e.g. acme_surveys" />
+        <div class="form-hint">3–128 chars · letters, digits, '.', '_', '-'. This is your private workspace key.</div>
+      </div>
+      <div class="form-group">
+        <label for="auth-pass">Passphrase</label>
+        <input type="password" id="auth-pass" name="password" autocomplete="current-password" placeholder="At least 8 characters" />
+        <div class="form-hint" id="auth-pass-hint">Use a strong, memorable passphrase. There is no recovery — keep it safe.</div>
+      </div>
+      <div id="auth-alert" class="hidden"></div>
+      <button type="submit" class="btn btn-primary auth-submit" id="auth-submit-btn">Log in</button>
+    </form>
+  </div>
+</div>
+
 <div class="steps" id="step-indicator">
   <div class="step-item active" id="si-1"><div class="step-num">1</div> Upload</div>
   <div class="step-sep">›</div>
@@ -819,6 +902,41 @@
   </div>
 </div>
 
+<!-- ── RAG Document Manager modal ────────────────────────────────────────── -->
+<div id="doc-manager-modal" class="similar-modal hidden" role="dialog" aria-modal="true" aria-labelledby="doc-manager-title">
+  <div class="similar-modal-backdrop" onclick="closeDocManager()"></div>
+  <div class="similar-modal-panel" style="width:min(900px,100%)">
+    <div class="similar-modal-head">
+      <h3 id="doc-manager-title">🗂️ RAG Document Manager</h3>
+      <button type="button" class="similar-modal-close" onclick="closeDocManager()" aria-label="Close">×</button>
+    </div>
+    <div class="similar-modal-body">
+      <p class="similar-intro" id="doc-manager-intro">
+        All documents indexed under your tenant. Deleting removes the file, its database record, and every
+        vector embedding for that document from the search index. This cannot be undone.
+      </p>
+      <div class="docmgr-toolbar">
+        <span id="docmgr-count" class="docmgr-count"></span>
+        <button type="button" class="btn-similar" id="docmgr-refresh" onclick="loadDocManagerList()">↻ Refresh</button>
+        <button type="button" class="btn-similar" id="docmgr-reingest" onclick="reingestAllDocuments()" title="Re-process every document through the latest table-aware parser and chunker">⟳ Re-ingest library</button>
+      </div>
+      <div id="doc-manager-list"></div>
+    </div>
+  </div>
+</div>
+
+<!-- ── Delete confirmation modal ─────────────────────────────────────────── -->
+<div id="doc-delete-modal" class="similar-modal hidden" role="alertdialog" aria-modal="true" aria-labelledby="doc-delete-title" style="z-index:9500">
+  <div class="similar-modal-backdrop" onclick="cancelDocDelete()"></div>
+  <div class="similar-modal-panel" style="max-width:480px">
+    <div class="similar-modal-head" style="background:#fdecea;border-bottom:2px solid var(--danger)">
+      <h3 id="doc-delete-title" style="color:var(--danger)">Delete document?</h3>
+      <button type="button" class="similar-modal-close" onclick="cancelDocDelete()" aria-label="Close">×</button>
+    </div>
+    <div class="similar-modal-body" id="doc-delete-body" style="padding:1.2rem"></div>
+  </div>
+</div>
+
 <!-- ── Tier mismatch warning modal ──────────────────────────────────────── -->
 <div id="tier-mismatch-modal" class="similar-modal hidden" role="alertdialog" aria-modal="true" aria-labelledby="tier-mismatch-title">
   <div class="similar-modal-backdrop"></div>
@@ -862,8 +980,10 @@
 
 const state = {
   tenantId: null,
+  token: null,
   docId: null,
   docIds: [],    // all accepted document UUIDs from last batch upload
+  sessionDocIds: [], // every doc uploaded during THIS browser session (for "current" vs "previous" tagging)
   reportId: null,
   surveyLevel: 3,
   file: null,
@@ -1015,7 +1135,10 @@ function apiUrl(path) {
 }
 
 async function apiFetch(method, path, body, headers = {}, options = {}) {
-  const opts = { method, headers: { 'X-Tenant-ID': state.tenantId, ...headers } };
+  const baseHeaders = { ...headers };
+  if (state.token) baseHeaders['Authorization'] = `Bearer ${state.token}`;
+  if (state.tenantId) baseHeaders['X-Tenant-ID'] = state.tenantId; // dev fallback
+  const opts = { method, headers: baseHeaders };
   if (body instanceof FormData) { opts.body = body; }
   else if (body) { opts.headers['Content-Type'] = 'application/json'; opts.body = JSON.stringify(body); }
   // Optional AbortSignal — used by _saveSectionEdit to cancel an in-flight save
@@ -1023,6 +1146,10 @@ async function apiFetch(method, path, body, headers = {}, options = {}) {
   // callers can ignore this; behaviour is unchanged when signal is omitted.
   if (options && options.signal) opts.signal = options.signal;
   const res = await fetch(apiUrl(path), opts);
+  if (res.status === 401 && !(options && options.noAuthRedirect)) {
+    handleAuthExpired();
+    throw new Error('Your session expired. Please log in again.');
+  }
   if (res.status === 204) {
     if (!res.ok) throw new Error(`HTTP ${res.status}`);
     return null;
@@ -1055,6 +1182,123 @@ async function apiFetch(method, path, body, headers = {}, options = {}) {
   return data;
 }
 
+/* ── Authentication (must run before protected API calls) ─────────────── */
+const AUTH_KEY = 'rics_auth';
+let _authMode = 'login';
+
+function setAuthMode(mode) {
+  _authMode = mode;
+  $('auth-tab-login').classList.toggle('active', mode === 'login');
+  $('auth-tab-register').classList.toggle('active', mode === 'register');
+  $('auth-submit-btn').textContent = mode === 'register' ? 'Create account & sign in' : 'Log in';
+  $('auth-pass').setAttribute('autocomplete', mode === 'register' ? 'new-password' : 'current-password');
+  $('auth-pass-hint').textContent = mode === 'register'
+    ? 'At least 8 characters. There is no recovery — store it safely.'
+    : 'Enter the passphrase you registered with this Tenant/User ID.';
+  hideAlert('auth-alert');
+}
+
+function showAuthOverlay() {
+  const el = $('auth-overlay');
+  if (el) {
+    el.classList.remove('hidden-gate');
+    el.setAttribute('aria-hidden', 'false');
+  }
+  document.body.classList.add('auth-gate');
+  setTimeout(() => { const t = $('auth-tenant'); if (t) t.focus(); }, 50);
+}
+
+function hideAuthOverlay() {
+  const el = $('auth-overlay');
+  if (el) {
+    el.classList.add('hidden-gate');
+    el.setAttribute('aria-hidden', 'true');
+  }
+  document.body.classList.remove('auth-gate');
+}
+
+function _persistAuth(sess) {
+  localStorage.setItem(AUTH_KEY, JSON.stringify(sess));
+}
+
+function applyAuthSession(sess) {
+  state.token = sess.access_token;
+  state.tenantId = sess.tenant_id;
+  _persistAuth(sess);
+  $('tenant-badge').textContent = `🔑 ${sess.tenant_id}`;
+  $('tenant-badge').classList.add('visible');
+  $('btn-logout').style.display = '';
+  const ti = $('tenant-input');
+  if (ti) { ti.value = sess.tenant_id; ti.readOnly = true; ti.title = 'Logged in as this tenant'; }
+  hideAuthOverlay();
+}
+
+async function submitAuth(event) {
+  event.preventDefault();
+  const tenant = $('auth-tenant').value.trim();
+  const pass = $('auth-pass').value;
+  if (!tenant || !pass) { showAlert('auth-alert', 'warn', 'Enter both a Tenant/User ID and a passphrase.'); return; }
+  const btn = $('auth-submit-btn');
+  btn.disabled = true;
+  const original = btn.textContent;
+  btn.textContent = _authMode === 'register' ? 'Creating account…' : 'Signing in…';
+  try {
+    const path = _authMode === 'register' ? '/auth/register' : '/auth/login';
+    const res = await apiFetch('POST', path, { tenant_id: tenant, passphrase: pass }, {}, { noAuthRedirect: true });
+    applyAuthSession(res);
+    $('auth-pass').value = '';
+    showToast(_authMode === 'register' ? 'Account created — welcome.' : 'Signed in.', 'ok');
+  } catch (err) {
+    showAlert('auth-alert', 'error', escapeHtml(err.message || 'Authentication failed.'));
+  } finally {
+    btn.disabled = false;
+    btn.textContent = original;
+  }
+}
+
+function logout() {
+  localStorage.removeItem(AUTH_KEY);
+  state.token = null;
+  state.tenantId = null;
+  $('btn-logout').style.display = 'none';
+  $('tenant-badge').classList.remove('visible');
+  if (typeof closeDocManager === 'function') closeDocManager();
+  const ti = $('tenant-input');
+  if (ti) { ti.readOnly = false; ti.value = ''; ti.title = ''; }
+  showAuthOverlay();
+  showToast('Signed out.', 'ok');
+}
+
+function handleAuthExpired() {
+  localStorage.removeItem(AUTH_KEY);
+  state.token = null;
+  state.tenantId = null;
+  $('btn-logout').style.display = 'none';
+  $('tenant-badge').classList.remove('visible');
+  const ti = $('tenant-input');
+  if (ti) { ti.readOnly = false; ti.value = ''; ti.title = ''; }
+  showAuthOverlay();
+  showAlert('auth-alert', 'warn', 'Your session expired. Please log in again.');
+}
+
+async function bootstrapAuth() {
+  setAuthMode('login');
+  let sess = null;
+  try { sess = JSON.parse(localStorage.getItem(AUTH_KEY) || 'null'); } catch { sess = null; }
+  const stillValid = sess && sess.access_token && (!sess.expires_at || sess.expires_at * 1000 > Date.now() + 30000);
+  if (!stillValid) { showAuthOverlay(); return; }
+  state.token = sess.access_token;
+  state.tenantId = sess.tenant_id;
+  try {
+    const me = await apiFetch('GET', '/auth/me', null, {}, { noAuthRedirect: true });
+    applyAuthSession({ ...sess, tenant_id: me.tenant_id });
+  } catch {
+    handleAuthExpired();
+  }
+}
+
+bootstrapAuth();
+
 /* ── Tier mismatch modal ─────────────────────────────────────────────── */
 let _mismatchResolve = null;
 let _mismatchReject  = null;
@@ -1276,9 +1520,8 @@ $('btn-upload').addEventListener('click', handleUpload);
 
 async function handleUpload() {
   if (!state.files.length) { showAlert('upload-alert', 'warn', 'Please choose at least one file.'); return; }
-  let tid = $('tenant-input').value.trim();
-  if (!tid) { tid = 'tenant_' + Math.random().toString(36).slice(2, 10); $('tenant-input').value = tid; }
-  state.tenantId = tid;
+  if (!state.token || !state.tenantId) { showAuthOverlay(); return; }
+  const tid = state.tenantId; // identity comes from the verified token, not the input
   $('tenant-badge').textContent = `🔑 ${tid}`;
   $('tenant-badge').classList.add('visible');
   showStep(2);
@@ -1302,6 +1545,7 @@ async function handleUpload() {
 
     state.docIds = up.items.filter(it => it.document_id).map(it => it.document_id);
     if (!state.docIds.length) throw new Error('Batch upload returned no document IDs');
+    state.sessionDocIds = Array.from(new Set([...(state.sessionDocIds || []), ...state.docIds]));
 
     $('proc-title').textContent = 'Indexing reference documents…';
     $('proc-detail').textContent = `Queued ${up.accepted} file(s) for parsing & embedding${up.rejected ? ` (${up.rejected} rejected)` : ''}`;
@@ -2397,6 +2641,8 @@ function buildResultCard(sec, result, isKept) {
         </div>`).join('')}
     </details>` : '';
 
+  const auditHtml = buildCitationAudit(result.citation_audit);
+
   const trx = result.ai_transparency;
   const req = trx && typeof trx.requested_ai_involvement_percent === 'number' ? trx.requested_ai_involvement_percent : null;
   const meas = trx && typeof trx.measured_ai_involvement_percent === 'number' ? trx.measured_ai_involvement_percent : null;
@@ -2451,6 +2697,7 @@ function buildResultCard(sec, result, isKept) {
       >${textHtml}</div>
       ${notesHtml}
       ${transHtml}
+      ${auditHtml}
       ${citeChips}
       ${provHtml}
       <div class="result-actions">
@@ -2666,6 +2913,43 @@ function bindResultTextEditing(code) {
   });
 }
 
+function buildCitationAudit(audit) {
+  if (!audit || typeof audit !== 'object') return '';
+  const conf = typeof audit.confidence === 'number' ? Math.round(audit.confidence * 100) : null;
+  const findings = typeof audit.findings === 'number' ? audit.findings : 0;
+  const contradictions = Array.isArray(audit.contradictions) ? audit.contradictions : [];
+  const dropped = Array.isArray(audit.dropped_claims) ? audit.dropped_claims : [];
+  // Nothing meaningful to show (e.g. no API key / no findings) — stay quiet.
+  if (conf === null && !findings && !contradictions.length && !dropped.length) return '';
+
+  const clean = !contradictions.length && !dropped.length;
+  const tone = clean
+    ? 'border:1px solid #bfe3c6;background:#f1faf3;color:#1f6b35'
+    : 'border:1px solid #f0d6b4;background:#fff7ea;color:#7a4a00';
+  const icon = clean ? '✅' : '⚠️';
+  const confStr = conf !== null ? `${conf}%` : 'n/a';
+
+  let body = `<strong>${icon} Evidence audit:</strong> ${findings} grounded finding(s), `
+    + `confidence ${confStr}.`;
+
+  if (contradictions.length) {
+    body += `<details style="margin-top:.4rem"><summary style="cursor:pointer">`
+      + `${contradictions.length} contradiction(s) resolved</summary>`
+      + `<ul style="margin:.35rem 0 0 1.1rem;list-style:disc">`
+      + contradictions.map(c => `<li>${escapeHtml((c.element || '') + ': ' + (c.detail || ''))}`
+          + (c.resolution ? ` <em>(${escapeHtml(c.resolution)})</em>` : '') + `</li>`).join('')
+      + `</ul></details>`;
+  }
+  if (dropped.length) {
+    body += `<details style="margin-top:.4rem"><summary style="cursor:pointer">`
+      + `${dropped.length} unsupported claim(s) dropped</summary>`
+      + `<ul style="margin:.35rem 0 0 1.1rem;list-style:disc">`
+      + dropped.map(d => `<li>${escapeHtml(String(d))}</li>`).join('')
+      + `</ul></details>`;
+  }
+  return `<div class="section-transparency" style="${tone}">${body}</div>`;
+}
+
 function buildCitationChips(prov) {
   if (!prov || !prov.length) return '';
   const chips = prov.map((p, i) => {
@@ -2720,6 +3004,185 @@ function escapeHtml(t) {
   return t.replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;').replace(/"/g,'&quot;');
 }
 
+/* ── RAG Document Manager ──────────────────────────────────────────────── */
+function _docMgrResolveTenant() {
+  if (state.tenantId) return state.tenantId;
+  const typed = ($('tenant-input') && $('tenant-input').value || '').trim();
+  if (typed) { state.tenantId = typed; }
+  return state.tenantId;
+}
+
+function formatBytes(n) {
+  if (n == null || isNaN(n)) return '—';
+  if (n < 1024) return `${n} B`;
+  const units = ['KB', 'MB', 'GB'];
+  let val = n / 1024, i = 0;
+  while (val >= 1024 && i < units.length - 1) { val /= 1024; i++; }
+  return `${val.toFixed(val >= 10 ? 0 : 1)} ${units[i]}`;
+}
+
+function formatDateTime(iso) {
+  if (!iso) return '—';
+  let s = String(iso).trim();
+  // Server timestamps are UTC; SQLite often omits the offset in JSON.
+  if (s && !/Z$|[+-]\d{2}:\d{2}$/.test(s)) s += 'Z';
+  const d = new Date(s);
+  if (isNaN(d.getTime())) return '—';
+  return d.toLocaleString(undefined, {
+    year: 'numeric',
+    month: 'short',
+    day: 'numeric',
+    hour: '2-digit',
+    minute: '2-digit',
+  });
+}
+
+const DOC_STATUS_BADGE = {
+  complete: { cls: 'b-ready', label: 'Ready' },
+  processing: { cls: 'b-processing', label: 'Processing' },
+  pending: { cls: 'b-pending', label: 'Pending' },
+  failed: { cls: 'b-failed', label: 'Failed' },
+};
+
+function openDocManager() {
+  const tid = _docMgrResolveTenant();
+  $('doc-manager-modal').classList.remove('hidden');
+  if (tid) {
+    $('tenant-badge').textContent = `🔑 ${tid}`;
+    $('tenant-badge').classList.add('visible');
+    loadDocManagerList();
+  } else {
+    $('doc-manager-list').innerHTML =
+      '<div class="docmgr-empty"><span class="icon">🔑</span>Enter your Tenant / User ID on the upload screen (or upload a file) to view your documents.</div>';
+    $('docmgr-count').textContent = '';
+  }
+}
+
+function closeDocManager() {
+  $('doc-manager-modal').classList.add('hidden');
+}
+
+async function reingestAllDocuments() {
+  const btn = $('docmgr-reingest');
+  if (!confirm('Re-ingest every document through the latest parser/chunker? '
+      + 'Existing chunks are replaced. Documents with a report currently generating are skipped.')) {
+    return;
+  }
+  const original = btn ? btn.textContent : '';
+  if (btn) { btn.disabled = true; btn.textContent = '⟳ Re-ingesting…'; }
+  try {
+    const res = await apiFetch('POST', '/documents/reingest', {});
+    showToast((res && res.detail) || `Re-queued ${res ? res.queued : 0} document(s).`, 'ok');
+    await loadDocManagerList();
+  } catch (err) {
+    showToast('Re-ingest failed: ' + (err && err.message ? err.message : 'request failed'), 'err');
+  } finally {
+    if (btn) { btn.disabled = false; btn.textContent = original; }
+  }
+}
+
+async function loadDocManagerList() {
+  const listEl = $('doc-manager-list');
+  const countEl = $('docmgr-count');
+  listEl.innerHTML = '<div class="docmgr-loading"><span class="docmgr-spinner"></span> Loading your documents…</div>';
+  countEl.textContent = '';
+  try {
+    const data = await apiFetch('GET', '/documents?limit=500', null);
+    const docs = (data && data.documents) || [];
+    if (!docs.length) {
+      listEl.innerHTML =
+        '<div class="docmgr-empty"><span class="icon">📭</span>No documents uploaded yet under this tenant.</div>';
+      return;
+    }
+    const sessionSet = new Set(state.sessionDocIds || []);
+    _docMgrNames = {};
+    const rows = docs.map(d => {
+      _docMgrNames[d.document_id] = d.filename || 'untitled';
+      const badge = DOC_STATUS_BADGE[d.status] || DOC_STATUS_BADGE.pending;
+      const isSession = sessionSet.has(d.document_id);
+      const isStyle = d.document_purpose === 'style_corpus';
+      const sessionTag = isSession
+        ? '<span class="docmgr-badge b-session" title="Uploaded during this session">This session</span>'
+        : '';
+      const styleTag = isStyle
+        ? '<span class="docmgr-badge b-style" title="Past report kept for style learning only">Style</span>'
+        : '';
+      const errAttr = d.error ? ` title="${escapeHtml(String(d.error))}"` : '';
+      return `<tr id="docrow-${escapeHtml(d.document_id)}">
+        <td>
+          <div class="docmgr-fname">${escapeHtml(d.filename || 'untitled')}${sessionTag}${styleTag}</div>
+          <div class="docmgr-id">${escapeHtml(d.document_id)}</div>
+        </td>
+        <td class="docmgr-meta">${formatDateTime(d.created_at)}</td>
+        <td class="docmgr-meta">${formatBytes(d.file_size)}</td>
+        <td><span class="docmgr-badge ${badge.cls}"${errAttr}>${badge.label}</span></td>
+        <td style="text-align:right">
+          <button type="button" class="docmgr-del" onclick="confirmDocDelete('${escapeHtml(d.document_id)}')">🗑 Delete</button>
+        </td>
+      </tr>`;
+    }).join('');
+    listEl.innerHTML = `<table class="docmgr-table">
+      <thead><tr><th>Document</th><th>Uploaded</th><th>Size</th><th>Status</th><th></th></tr></thead>
+      <tbody>${rows}</tbody>
+    </table>`;
+    const sessionCount = docs.filter(d => sessionSet.has(d.document_id)).length;
+    countEl.textContent = `${docs.length} document${docs.length === 1 ? '' : 's'}` +
+      (sessionCount ? ` · ${sessionCount} from this session` : '');
+  } catch (err) {
+    listEl.innerHTML =
+      `<div class="docmgr-empty"><span class="icon">⚠️</span>Could not load documents: ${escapeHtml(err.message || 'request failed')}</div>`;
+  }
+}
+
+let _pendingDeleteDocId = null;
+let _docMgrNames = {};
+
+function confirmDocDelete(docId) {
+  _pendingDeleteDocId = docId;
+  const filename = _docMgrNames[docId] || 'this document';
+  $('doc-delete-body').innerHTML = `
+    <p style="margin:0 0 .8rem;line-height:1.5">
+      Permanently delete <strong>${escapeHtml(filename)}</strong>?
+    </p>
+    <p style="margin:0 0 1.1rem;font-size:.82rem;color:var(--muted);line-height:1.5">
+      This removes the file, its database record, and all vector embeddings for this document from the search index.
+      Reports already generated are unaffected. <strong>This cannot be undone.</strong>
+    </p>
+    <div style="display:flex;justify-content:flex-end;gap:.6rem">
+      <button type="button" class="btn-similar" onclick="cancelDocDelete()">Cancel</button>
+      <button type="button" class="docmgr-del" id="doc-delete-confirm-btn" onclick="executeDocDelete()">🗑 Delete permanently</button>
+    </div>`;
+  $('doc-delete-modal').classList.remove('hidden');
+}
+
+function cancelDocDelete() {
+  _pendingDeleteDocId = null;
+  $('doc-delete-modal').classList.add('hidden');
+}
+
+async function executeDocDelete() {
+  const docId = _pendingDeleteDocId;
+  if (!docId) return;
+  const confirmBtn = $('doc-delete-confirm-btn');
+  if (confirmBtn) { confirmBtn.disabled = true; confirmBtn.innerHTML = '<span class="docmgr-spinner"></span> Deleting…'; }
+  const row = $(`docrow-${docId}`);
+  if (row) row.classList.add('row-deleting');
+  try {
+    const res = await apiFetch('DELETE', `/documents/${encodeURIComponent(docId)}`, null);
+    cancelDocDelete();
+    state.docIds = (state.docIds || []).filter(id => id !== docId);
+    state.sessionDocIds = (state.sessionDocIds || []).filter(id => id !== docId);
+    showToast((res && res.detail) || 'Document deleted.', 'ok');
+    await loadDocManagerList();
+  } catch (err) {
+    cancelDocDelete();
+    if (row) row.classList.remove('row-deleting');
+    const msg = err.message || 'Delete failed';
+    // 409 = still linked to a report (FK guard) — surface the server's guidance.
+    showToast(msg.length > 160 ? msg.slice(0, 157) + '…' : msg, 'err');
+  }
+}
+
 window._lastLibraryMatches = [];
 
 function peerSectionsMap(excludeCode) {
@@ -3096,15 +3559,18 @@ $('btn-download').addEventListener('click', () => {
 
 /* ── Restart ─────────────────────────────────────────────────────────────── */
 $('btn-restart').addEventListener('click', () => {
-  Object.assign(state, { tenantId:null, docId:null, docIds:[], reportId:null, file:null, files:[], sections:{}, results:{}, styleProfile:null });
+  // Preserve the authenticated identity (token + tenant) across a new report.
+  Object.assign(state, { docId:null, docIds:[], reportId:null, file:null, files:[], sections:{}, results:{}, styleProfile:null });
   fileInput.value = '';
-  $('tenant-input').value = '';
   $('file-display').classList.add('hidden');
   zone.classList.remove('has-file','drag-over');
   $('btn-upload').disabled = true;
-  $('tenant-badge').classList.remove('visible');
   $('style-profile-panel').classList.add('hidden');
   hideAlert('upload-alert');
+  if (state.tenantId) {
+    const ti = $('tenant-input');
+    if (ti) { ti.value = state.tenantId; ti.readOnly = true; }
+  }
   showStep(1);
 });