app/config.py

"""Application configuration loaded from environment variables or a .env file."""

import os
import tempfile
from pathlib import Path
from typing import Self

from pydantic import Field, model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict


class Settings(BaseSettings):
    """Central settings object.

    All values can be overridden via environment variables or a ``.env`` file
    in the project root.  See ``.env.example`` for the full reference.
    """

    model_config = SettingsConfigDict(
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    # ── LLM ──────────────────────────────────────────────────────────────────
    openai_api_key: str = Field(default="", description="OpenAI API key")
    embedding_model: str = Field(default="text-embedding-3-small", description="OpenAI embedding model name (legacy, only used if prefer_local_embeddings is False)")
    local_embedding_model: str = Field(
        default="all-MiniLM-L6-v2",
        description="sentence-transformers model used for free, local embeddings (preferred default)",
    )
    prefer_local_embeddings: bool = Field(
        default=True,
        description=(
            "When True (default), always use the free HuggingFace sentence-transformers model "
            "for embeddings even if OPENAI_API_KEY is set. Set to False to fall back to OpenAI "
            "embeddings (incurs API cost)."
        ),
    )
    chat_model: str = Field(default="gpt-4o-mini")
    inspector_body_model: str = Field(
        default="gpt-4o",
        description=(
            "Primary model for inspector-loop body drafting (submit_inspection_section). "
            "Kept separate from chat_model so utility passes can stay on cheaper models."
        ),
    )

    # ── Vector store (FAISS default; optional Qdrant for async pipeline) ─────
    faiss_index_path: str = Field(
        default_factory=lambda: str(Path.home() / ".report_genius" / "faiss_index"),
        description="Directory for persisted FAISS index files (index.faiss, etc.)",
    )
    vectorstore_backend: str = Field(
        default="faiss",
        description="Vector store backend: 'faiss' (local) or 'qdrant' (async retrieval path).",
    )
    qdrant_url: str = Field(
        default="http://localhost:6333",
        description="Qdrant HTTP endpoint when vectorstore_backend=qdrant.",
    )
    qdrant_api_key: str | None = Field(
        default=None,
        description="Optional Qdrant API key.",
    )
    qdrant_collection: str = Field(
        default="rics_chunks",
        description="Main Qdrant collection for tenant-scoped chunks.",
    )
    qdrant_cache_collection: str = Field(
        default="rics_semantic_cache",
        description="Qdrant collection for semantic retrieval cache entries.",
    )
    enable_hybrid_retrieval: bool = Field(
        default=True,
        description="When true with VECTORSTORE_BACKEND=qdrant, merge vector + BM25 via RRF.",
    )
    semantic_cache_enabled: bool = Field(
        default=True,
        description="When true with VECTORSTORE_BACKEND=qdrant, cache retrieval results by query embedding.",
    )
    semantic_cache_ttl_hours: int = Field(
        default=24,
        ge=1,
        le=168,
        description="TTL for semantic cache entries (hours).",
    )
    semantic_cache_similarity_threshold: float = Field(
        default=0.92,
        ge=0.5,
        le=1.0,
        description="Minimum cosine similarity for a semantic cache hit.",
    )

    # ── Database ─────────────────────────────────────────────────────────────
    database_url: str = Field(default="sqlite+aiosqlite:///./dev.db")

    # ── File storage ─────────────────────────────────────────────────────────
    upload_dir: Path = Field(
        default_factory=lambda: Path.home() / ".report_genius" / "uploads"
    )
    max_single_upload_bytes: int = Field(
        default=50 * 1024 * 1024,
        ge=1_048_576,
        le=500 * 1024 * 1024,
        description="Max size per regular .docx/.pdf upload (bytes)",
    )
    max_archive_upload_bytes: int = Field(
        default=500 * 1024 * 1024,
        ge=10 * 1024 * 1024,
        le=5 * 1024 * 1024 * 1024,
        description="Max compressed size for a .zip in batch uploads (bytes)",
    )
    max_upload_batch_files: int = Field(
        default=2000,
        ge=1,
        le=100_000,
        description="Max logical documents per batch request after expanding ZIPs",
    )
    max_zip_members: int = Field(
        default=50_000,
        ge=1,
        le=500_000,
        description="Max file entries inside one ZIP",
    )
    max_zip_uncompressed_bytes: int = Field(
        default=5 * 1024 * 1024 * 1024,
        ge=100 * 1024 * 1024,
        description="Max declared uncompressed total size for one ZIP",
    )
    max_concurrent_ingests: int = Field(
        default=4,
        ge=1,
        le=64,
        description=(
            "Parallel ingestion workers. FAISS uses one process-wide index guarded by a lock; "
            "raising this mainly increases how many files parse/embed in parallel before indexing."
        ),
    )
    max_batch_status_document_ids: int = Field(
        default=10_000,
        ge=100,
        le=100_000,
        description="Max document IDs per /documents/batch-status call",
    )
    documents_list_max_limit: int = Field(
        default=500,
        ge=10,
        le=2000,
        description="Max rows returned by GET /documents",
    )

    # ── Cache ─────────────────────────────────────────────────────────────────
    cache_dir: Path = Field(
        default_factory=lambda: Path(tempfile.gettempdir()) / "section_cache"
    )

    # ── Report section photos ────────────────────────────────────────────────
    max_section_photo_bytes: int = Field(
        default=8 * 1024 * 1024,
        ge=256 * 1024,
        le=50 * 1024 * 1024,
        description="Max size per uploaded section photo (bytes).",
    )
    max_section_photos_per_section: int = Field(
        default=8,
        ge=0,
        le=50,
        description="Max number of photos stored per report section.",
    )
    section_photo_vision_enabled: bool = Field(
        default=True,
        description=(
            "If true and OPENAI_API_KEY is set, the server may analyze uploaded photos with a vision-capable model "
            "to produce additional observations for generation."
        ),
    )
    section_photo_vision_batch_size: int = Field(
        default=6,
        ge=1,
        le=20,
        description=(
            "Max images per vision API request; all uploaded section photos are covered by sequential batches."
        ),
    )
    section_photo_vision_max_observations: int = Field(
        default=48,
        ge=12,
        le=120,
        description="Upper cap on merged bullet observations after analyzing all photos for a section.",
    )
    section_photo_vision_model: str = Field(
        default="",
        description=(
            "OpenAI model for section photo vision analysis. "
            "When empty, uses chat_model if set, otherwise gpt-4o."
        ),
    )
    section_photo_analyze_on_upload: bool = Field(
        default=True,
        description=(
            "When true and OPENAI_API_KEY is set, run vision analysis after section photo upload "
            "(cached per section) so generate does not wait on first vision call."
        ),
    )
    section_photo_policy_data_driven: bool = Field(
        default=True,
        description=(
            "If true, the server scans each tenant's completed PDF/DOCX uploads (the same indexed files used for "
            "tenant RAG) for embedded images per RICS section. No letter-based heuristic; knowledge_base_dirs is "
            "not used for this product feature."
        ),
    )
    section_photo_policy_cache_seconds: int = Field(
        default=6 * 60 * 60,
        ge=60,
        le=7 * 24 * 60 * 60,
        description="TTL for cached photo-policy corpus statistics (seconds).",
    )
    section_photo_policy_min_examples: int = Field(
        default=3,
        ge=1,
        le=1_000_000,
        description=(
            "Minimum number of tenant indexed uploads in which a RICS section heading must appear before "
            "photo-upload rules are inferred for that section."
        ),
    )
    section_photo_policy_tenant_consensus_ratio: float = Field(
        default=1.0,
        ge=0.0,
        le=1.0,
        description=(
            "Among tenant uploads where a section heading is detected, minimum fraction that must also contain "
            "an image in that section before enabling photo upload (1.0 = every such upload)."
        ),
    )
    section_photo_policy_overrides_json: str = Field(
        default="",
        description=(
            "Optional JSON object mapping section_code (e.g. \"E2\") to REQUIRES_IMAGE, OPTIONAL_IMAGE, "
            "or NO_IMAGE_NEEDED. When set for a section, it replaces tenant-library-derived policy for that code."
        ),
    )

    # ── Generation limits ────────────────────────────────────────────────────
    max_context_tokens: int = Field(default=400, ge=100, le=2000)
    max_output_tokens: int = Field(default=300, ge=50, le=1000)
    retrieval_top_k: int = Field(default=10, ge=1, le=50)
    rerank_top_n: int = Field(default=3, ge=1, le=10)
    rag_doc_context_max_chunks: int = Field(
        default=6,
        ge=0,
        le=24,
        description="Total cap for document-level RAG excerpts (whole-PDF narrative).",
    )
    rag_doc_chunks_primary: int = Field(
        default=4,
        ge=0,
        le=16,
        description="Max document-level chunks taken from the report's primary survey PDF.",
    )
    rag_doc_chunks_per_reference: int = Field(
        default=2,
        ge=0,
        le=8,
        description="Max document-level chunks per exemplar/reference PDF (e.g. past RICS report).",
    )
    hierarchical_rag_enabled: bool = Field(
        default=True,
        description=(
            "Use coarse→fine retrieval (document → section → paragraph) when the index "
            "contains hierarchy_level metadata; falls back to flat retrieval if needed."
        ),
    )
    hierarchical_k_document: int = Field(default=4, ge=0, le=16)
    hierarchical_k_section: int = Field(default=8, ge=0, le=32)
    hierarchical_k_paragraph_pool: int = Field(default=36, ge=4, le=120)
    rics_exemplar_document_ids: str = Field(
        default="",
        description=(
            "Optional comma-separated upload UUIDs (e.g. Behrang / template RICS PDFs) "
            "always considered as reference documents for hierarchical routing."
        ),
    )

    # ── Knowledge base (local standards corpus, optional) ────────────────────
    knowledge_base_enabled: bool = Field(
        default=True,
        description=(
            "If true, the server can index local RICS standards/exemplar documents "
            "from knowledge_base_dirs into the vector store under a reserved tenant."
        ),
    )
    knowledge_base_tenant_id: str = Field(
        default="__rics_kb__",
        description="Reserved tenant_id used for the local knowledge base corpus.",
    )
    knowledge_base_dirs: str = Field(
        default="Behrang RICS Documents,RAW Context",
        description=(
            "Comma-separated folder names/paths (relative to repo root or absolute) "
            "to scan for local standards/exemplar PDFs/DOCX."
        ),
    )
    rics_standard_paragraphs_docx: str = Field(
        default="Behrang RICS Documents/HB-BS STANDARD PARAS v6 Sept 2015.doc",
        description=(
            "Path to the master standard-paragraphs Word file (repo-relative or absolute): "
            ".docx / .docm parsed directly; legacy .doc converted via pandoc, LibreOffice, or "
            "Microsoft Word+pywin32 (Windows). When non-empty and resolved, **only** this file loads. "
            "If the path is missing, the basename is also tried at the repo root. "
            "When empty, discovery uses standard_paragraphs_docx_globs under knowledge_base_dirs."
        ),
    )
    standard_paragraphs_docx_globs: str = Field(
        default=(
            "*standard*paragraph*.docx,*Standard*Paragraph*.docx,"
            "*HB-BS*STANDARD*PARAS*.doc,*HB-BS*STANDARD*PARAS*.docx"
        ),
        description=(
            "Used only when rics_standard_paragraphs_docx is empty: comma-separated filename glob patterns "
            "(case-insensitive) scanned under each knowledge_base_dir root for .docx / .docm / .doc."
        ),
    )
    survey_level_corpus_cache_seconds: int = Field(
        default=6 * 60 * 60,
        ge=60,
        le=7 * 24 * 60 * 60,
        description="TTL for cached survey-tier corpus profiles built from knowledge_base_dirs (seconds).",
    )
    chunk_size: int = Field(default=500, ge=100, le=2000)
    chunk_overlap: int = Field(default=75, ge=0, le=200)

    # ── Autonomous RICS inspector (OpenAI tool-calling) ───────────────────────
    inspector_tool_agent: bool = Field(
        default=True,
        description=(
            "When true and OPENAI_API_KEY is set, agentic section generation uses an OpenAI "
            "tool-calling loop so the model chooses retrieval/similarity tools; otherwise the "
            "legacy fixed pipeline runs (including mock adapter in tests)."
        ),
    )
    inspector_max_tool_rounds: int = Field(
        default=12,
        ge=3,
        le=40,
        description="Max assistant turns (each may include multiple tool calls) for the inspector agent.",
    )

    # ── Generation pipeline selection ─────────────────────────────────────────
    primary_generate_pipeline: str = Field(
        default="agentic",
        description=(
            "Which pipeline powers POST /reports/{report_id}/generate for mode=generate. "
            "'agentic' = use the inspector HeadAgent first (tool-calling when live), then fall back to the standard "
            "fixed RAG generator on error. "
            "'standard' = always use the fixed RAG generator."
        ),
    )

    # ── Notes-only generation (anti-leak) ────────────────────────────────────
    notes_only_generation: bool = Field(
        default=True,
        description=(
            "When true, generated report content must be grounded ONLY in the user-supplied bullets/draft text "
            "(uploaded messy notes). Tenant RAG / exemplar documents may be used as *reference-only* guidance to "
            "help interpret messy notes and follow structure/tone, but are not allowed to contribute property-specific "
            "facts. In this mode, provenance/citations are suppressed so no RAG content is exposed to the user."
        ),
    )
    agentic_inspector_when_notes_only: bool = Field(
        default=True,
        description=(
            "When true with notes_only_generation, POST /generate still uses the inspector HeadAgent "
            "(tool-calling, photo vision enrichment, Phase 2 speculation when enabled). "
            "Set false only if you need the lighter standard notes-only pipeline."
        ),
    )

    # ── LLM compliance validation (generate mode) ────────────────────────────
    llm_section_validator_enabled: bool = Field(
        default=False,
        description=(
            "If true and OPENAI_API_KEY is set, run an additional LLM-based compliance validator after "
            "generation (survey-level behavioural checks). On FAIL, regenerate once with feedback. "
            "Deterministic guards (non-invention + heuristic tier validator) still run regardless."
        ),
    )
    llm_section_validator_max_retries: int = Field(
        default=1,
        ge=0,
        le=3,
        description="Max regenerate attempts when LLM validator returns FAIL.",
    )

    # ── Backend scale profile (AI Phase 3 flags + job queue; not for HF Spaces) ─
    scale_optimization_profile: bool = Field(
        default=False,
        description=(
            "When true (non-HF), enables AI Phase 3 flags (async pipeline, speculation, "
            "prompt cache) and ENABLE_JOB_QUEUE when REDIS_URL is set. "
            "See docs/AI_FEATURES_PHASES.md. Use with docker compose --profile redis --profile jobs."
        ),
    )

    # ── Async pipeline / concurrency guards (latency optimisation) ─────────
    enable_async_pipeline: bool = Field(
        default=False,
        description=(
            "When true, use the async LLM pipeline (async OpenAI calls, parallel "
            "multi-section generation, async retrieval when Qdrant is enabled)."
        ),
    )
    max_concurrent_llm_calls: int = Field(
        default=10,
        ge=1,
        le=2000,
        description=(
            "Global upper bound on concurrent in-flight LLM calls to avoid "
            "rate-limit storms. Throttles async OpenAI paths (async pipeline, inspector loop, "
            "and throttled chat completions) with structured cache_hit logging."
        ),
    )

    # ── AI Phase 3 + backend Temporal (speculation/prompt cache are AI; Temporal is not) ─
    enable_speculative_executor: bool = Field(
        default=False,
        description=(
            "When true with enable_async_pipeline, prefetch inspector tools using "
            "PatternRegistry before the LLM requests them."
        ),
    )
    speculative_context_window: int = Field(default=3, ge=1, le=12)
    speculative_probability_threshold: float = Field(default=0.75, ge=0.5, le=1.0)
    speculative_learn_min_observations: int = Field(
        default=5,
        ge=3,
        le=100,
        description=(
            "Minimum times a context→tool sequence must be observed before "
            "PatternRegistry auto-registers a learned speculative pattern."
        ),
    )
    allow_sqlite_parallel_sections: bool = Field(
        default=False,
        description=(
            "Allow parallel multi-section generation against SQLite. Default false "
            "because SQLite serializes writers and can cause database locked errors."
        ),
    )
    enable_prompt_caching: bool = Field(
        default=False,
        description=(
            "When true with enable_async_pipeline, stabilise system prefixes and "
            "pass OpenAI prompt_cache_key; log cache_hit_rate from usage."
        ),
    )
    prompt_cache_min_system_tokens: int = Field(
        default=1024,
        ge=512,
        le=8192,
        description="Minimum system-prompt tokens to target OpenAI automatic prefix caching.",
    )
    enable_temporal_workflow: bool = Field(
        default=False,
        description=(
            "When true, POST /generate starts a Temporal ReportGenerationWorkflow "
            "instead of an in-process asyncio task (requires temporal-worker)."
        ),
    )
    temporal_host: str = Field(
        default="localhost:7233",
        description="Temporal frontend gRPC address (host:port).",
    )
    temporal_namespace: str = Field(default="default")
    temporal_task_queue: str = Field(default="reports")
    generation_sla_seconds: int = Field(
        default=600,
        ge=120,
        le=3600,
        description=(
            "Product target: full multi-section report generation should finish within this "
            "wall-clock budget when async parallel sections are enabled (default 10 minutes)."
        ),
    )
    generation_timeout_seconds: int = Field(
        default=720,
        ge=300,
        le=24 * 60 * 60,
        description=(
            "Hard limit: reports in generating longer than this are marked failed by the sweeper. "
            "Defaults to generation_sla_seconds + 2 minutes grace."
        ),
    )
    generation_stale_sweep_seconds: int = Field(
        default=120,
        ge=0,
        le=3600,
        description="Interval for stale generating-report sweeper (0 disables).",
    )

    # ── Backend: Redis (rate limits + job queue — not an AI phase) ───────────
    redis_url: str = Field(
        default="",
        description=(
            "Redis URL for distributed rate limiting and optional generation job queue. "
            "Example: redis://localhost:6379/0"
        ),
    )
    enable_job_queue: bool = Field(
        default=False,
        description=(
            "When true with redis_url set, POST /generate and /agentic/generate enqueue "
            "work to Redis for jobs_worker.py instead of in-process asyncio tasks."
        ),
    )
    job_queue_key: str = Field(
        default="rics:jobs:generation",
        description="Redis list key for generation job payloads (JSON).",
    )
    job_queue_block_seconds: int = Field(
        default=5,
        ge=1,
        le=60,
        description="BRPOP timeout for the jobs worker loop.",
    )
    job_queue_max_concurrent: int = Field(
        default=2,
        ge=1,
        le=32,
        description="Max generation jobs processed in parallel per jobs worker process.",
    )

    ingest_timeout_seconds: int = Field(
        default=900,
        ge=30,
        le=24 * 60 * 60,
        description=(
            "Max time allowed for a single document ingest (parse/split/embed/index). "
            "Stale 'processing' documents beyond this are marked failed so the UI can continue."
        ),
    )

    # ── Production / HF Spaces (prioritise user-facing AI over dev infra) ─────
    production_ai_profile: bool = Field(
        default=False,
        description=(
            "When true (or when running on Hugging Face Spaces via SPACE_ID), ingest-time "
            "LLM sanitisation is disabled (regex redaction still runs when "
            "ENABLE_RAG_UPLOAD_SANITISATION=true) so OpenAI quota is reserved for generation, "
            "inspector, and photo vision."
        ),
    )

    # ── Personalised style RAG (private tenant library; not OpenAI fine-tuning) ─
    personalised_style_rag_enabled: bool = Field(
        default=True,
        description=(
            "When true: uploads are sanitised and indexed per tenant; generation retrieves "
            "only from that tenant's ingested report library (not shared KB). KB/style fallback "
            "applies only before the tenant has completed uploads."
        ),
    )

    # ── RAG upload sanitisation (PII/confidential stripping before indexing) ─
    enable_rag_upload_sanitisation: bool = Field(
        default=True,
        description=(
            "When true, tenant uploads are sanitised before embedding into the vector index. "
            "Use RAG_SANITISATION_USE_LLM=true for LLM chunks (dev/staging); regex-only is faster "
            "and does not compete with report generation. On-disk files are not rewritten."
        ),
    )
    rag_sanitisation_use_llm: bool = Field(
        default=False,
        description=(
            "When true with ENABLE_RAG_UPLOAD_SANITISATION, run LLM sanitisation per chunk. "
            "When false, regex-only redaction (recommended for production/HF)."
        ),
    )
    rag_sanitisation_skip_kb_tenant: bool = Field(
        default=True,
        description="Skip sanitisation for the reserved knowledge-base tenant (e.g. __rics_kb__).",
    )
    rag_sanitisation_chunk_chars: int = Field(
        default=20_000,
        ge=2_000,
        le=50_000,
        description="Max characters per LLM sanitisation call for long reports.",
    )
    rag_sanitisation_max_output_tokens: int = Field(
        default=4096,
        ge=512,
        le=16_384,
        description="Max tokens per sanitisation LLM response chunk.",
    )
    rag_sanitisation_fail_closed: bool = Field(
        default=True,
        description=(
            "If true, ingestion/runtime RAG sync fails when sanitisation would store empty text. "
            "When false, regex fallback is used even when the LLM returns nothing."
        ),
    )

    # ── Rate limiting ────────────────────────────────────────────────────────
    rate_limit_generate_rpm: int = Field(
        default=20,
        ge=1,
        le=10_000,
        description="Max POST /generate requests per tenant per minute",
    )
    rate_limit_read_rpm: int = Field(
        default=120,
        ge=1,
        le=10_000,
        description="Max read-endpoint requests per tenant per minute",
    )

    # ── Security ─────────────────────────────────────────────────────────────
    tenant_secret_key: str = Field(default="dev-secret-change-me")
    dev_mode: bool = Field(default=False)
    # CORS: list specific origins in production, e.g. ["https://app.example.com"]
    # The wildcard ["*"] is safe here because allow_credentials=False in main.py
    allowed_origins: list[str] = Field(default=["*"])

    @model_validator(mode="after")
    def _sync_openai_key_from_process_env(self) -> Self:
        """HF Space secrets inject OPENAI_API_KEY at runtime (not in the image)."""
        if not os.environ.get("SPACE_ID"):
            return self
        env_key = (os.environ.get("OPENAI_API_KEY") or "").strip()
        if env_key:
            self.openai_api_key = env_key
        return self

    @model_validator(mode="after")
    def _apply_deployment_profile(self) -> Self:
        """HF Spaces and production_ai_profile reserve OpenAI for report AI features."""
        on_hf_space = bool(os.environ.get("SPACE_ID"))
        if self.personalised_style_rag_enabled:
            self.enable_rag_upload_sanitisation = True
        if self.production_ai_profile or on_hf_space:
            # Regex sanitisation stays on (privacy); only disable per-chunk LLM ingest calls.
            self.rag_sanitisation_use_llm = False
            # Meet ~10m full-report SLA: parallel multi-section + async OpenAI (no Redis required).
            if not self.scale_optimization_profile:
                self.enable_async_pipeline = True
                self.allow_sqlite_parallel_sections = True
        if self.generation_timeout_seconds < self.generation_sla_seconds + 60:
            self.generation_timeout_seconds = int(self.generation_sla_seconds) + 120
        if self.scale_optimization_profile and not on_hf_space:
            self.enable_async_pipeline = True
            self.enable_speculative_executor = True
            self.enable_prompt_caching = True
            self.allow_sqlite_parallel_sections = True
            if (self.redis_url or "").strip():
                self.enable_job_queue = True
        return self


settings = Settings()


def effective_openai_api_key() -> str:
    """Resolved API key from settings and process env (HF secrets use the latter)."""
    return (settings.openai_api_key or os.environ.get("OPENAI_API_KEY") or "").strip()


def get_settings() -> Settings:
    """Return the application-wide settings singleton."""
    return settings