backend/core/ingest.py

"""Operator bundle ingest: report template (PDF) + standard paragraphs (Word).

* **Report template (PDF)** — schema discovery only (section order, titles, ratings).
  Defines how the finished report is structured.
* **Standard paragraphs (Word)** — firm-approved boilerplate per section, ingested
  into the MASTER RAG tier (not scrubbed; gated by ``assert_no_pii``).
* **Reference uploads** — optional past completed reports, scrubbed, style only.
"""

from __future__ import annotations

import logging
import re
from pathlib import Path

from backend.config import settings
from backend.core import hybrid_discovery_compiler, pii_scrubber, section_bridge, template_discoverer
from backend.core.rics_canonical_l3 import PARENT_SECTION_COUNT
from backend.core.rag_store import TIER_MASTER, TIER_REFERENCE, Chunk, get_rag_store
from backend.utils import doc_extractor

logger = logging.getLogger(__name__)


def _chunk_text(text: str) -> list[str]:
    min_c = settings.paragraph_min_chars
    max_c = settings.paragraph_max_chars
    overlap = settings.chunk_overlap
    paras = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
    if not paras:
        return []

    merged: list[str] = []
    buf = ""
    for para in paras:
        if not buf:
            buf = para
        else:
            buf = f"{buf}\n\n{para}"
        if len(buf) >= min_c:
            merged.append(buf)
            buf = ""
    if buf:
        if merged and len(buf) < min_c:
            merged[-1] = f"{merged[-1]}\n\n{buf}"
        else:
            merged.append(buf)

    chunks: list[str] = []
    for block in merged:
        if len(block) <= max_c:
            chunks.append(block)
            continue
        step = max(max_c - overlap, 1)
        for i in range(0, len(block), step):
            piece = block[i : i + max_c].strip()
            if piece:
                chunks.append(piece)
    return [c for c in chunks if c.strip()]


def _chunk_reference_text(text: str) -> list[str]:
    """Split REFERENCE past-report text into long-form chunks (heading-aware)."""
    from backend.config import settings

    min_c = settings.paragraph_min_chars
    max_c = settings.reference_paragraph_max_chars
    overlap = settings.reference_chunk_overlap
    paras = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
    if not paras:
        return []

    merged: list[str] = []
    buf = ""
    for para in paras:
        if not buf:
            buf = para
        else:
            buf = f"{buf}\n\n{para}"
        if len(buf) >= min_c:
            merged.append(buf)
            buf = ""
    if buf:
        if merged and len(buf) < min_c:
            merged[-1] = f"{merged[-1]}\n\n{buf}"
        else:
            merged.append(buf)

    chunks: list[str] = []
    for block in merged:
        if len(block) <= max_c:
            chunks.append(block)
            continue
        step = max(max_c - overlap, 1)
        for i in range(0, len(block), step):
            piece = block[i : i + max_c].strip()
            if piece:
                chunks.append(piece)
    return [c for c in chunks if c.strip()]


def _operator_filenames_lower() -> frozenset[str]:
    return frozenset({
        settings.report_template_filename.lower(),
        settings.standard_paragraphs_filename.lower(),
        settings.master_template_filename.lower(),
    })


def ingest_report_template(tenant_id: str, path: Path) -> dict:
    """Discover and persist schema from the report template PDF/DOCX."""
    if not path.is_file():
        raise FileNotFoundError(f"Report template not found: {path}")

    schema = template_discoverer.discover_report_template_schema(path)
    schema = hybrid_discovery_compiler.enrich_schema_with_hybrid_compiler(schema, path)
    schema.report_template_source = path.name
    schema.source_filename = path.name
    template_discoverer.save_schema(tenant_id, schema)

    logger.info(
        "Report template ingested for tenant=%s: %d parent sections from %s",
        tenant_id, PARENT_SECTION_COUNT, path.name,
    )
    return {
        "sections": PARENT_SECTION_COUNT,
        "schema_version": schema.version,
        "report_template": path.name,
    }


def ingest_standard_paragraphs(tenant_id: str, path: Path) -> dict:
    """Ingest standard paragraph wording into the MASTER RAG tier."""
    if not path.is_file():
        raise FileNotFoundError(f"Standard paragraphs file not found: {path}")

    full_text = doc_extractor.extract_text(path)
    pii_scrubber.assert_no_pii(full_text, context=f"standard paragraphs {path.name}")

    chunks_raw = template_discoverer.discover_standard_paragraph_chunks(path)
    store = get_rag_store()
    store.clear_tier(tenant_id, TIER_MASTER)
    total = 0
    for dc in chunks_raw:
        chunks = [
            Chunk(
                text=t,
                section_id=dc.section_id,
                tier=TIER_MASTER,
                is_scrubbed=False,
                source_filename=path.name,
            )
            for t in _chunk_text(dc.text)
        ]
        total += store.ingest_document(
            tenant_id,
            doc_id=f"paragraphs:{path.name}",
            chunks=chunks,
            tier=TIER_MASTER,
            source_filename=path.name,
        )

    # Record paragraphs source + build PDF->Word section alias map.
    schema = template_discoverer.load_schema(tenant_id)
    alias_count = 0
    if schema is not None:
        schema.standard_paragraphs_source = path.name
        ptitles = section_bridge.paragraph_titles_from_word(path)
        schema.paragraph_section_titles = ptitles
        schema.section_alias_map = section_bridge.build_section_alias_map(
            schema.sections, ptitles
        )
        alias_count = sum(
            1 for k, v in schema.section_alias_map.items() if k != v
        )
        template_discoverer.save_schema(tenant_id, schema)

    logger.info(
        "Standard paragraphs ingested for tenant=%s: %d chunks, %d aliases from %s",
        tenant_id, total, alias_count, path.name,
    )
    return {
        "chunks": total,
        "standard_paragraphs": path.name,
        "section_aliases": alias_count,
    }


def ingest_operator_bundle(tenant_id: str) -> dict:
    """Ingest report template (schema) + standard paragraphs (MASTER RAG)."""
    template_res = ingest_report_template(tenant_id, settings.report_template_path)
    paragraphs_res = ingest_standard_paragraphs(tenant_id, settings.standard_paragraphs_path)
    return {
        "sections": template_res["sections"],
        "schema_version": template_res["schema_version"],
        "report_template": template_res["report_template"],
        "standard_paragraphs": paragraphs_res["standard_paragraphs"],
        "paragraph_chunks": paragraphs_res["chunks"],
    }


def ingest_master(tenant_id: str, path: Path) -> dict:
    """Legacy/admin helper: treat a single file as both template and paragraphs."""
    schema, discovered = template_discoverer.discover_schema(path)
    full_text = doc_extractor.extract_text(path)
    pii_scrubber.assert_no_pii(full_text, context=f"master template {path.name}")
    template_discoverer.save_schema(tenant_id, schema)

    store = get_rag_store()
    store.clear_tier(tenant_id, TIER_MASTER)
    total = 0
    for dc in discovered:
        chunks = [
            Chunk(
                text=t,
                section_id=dc.section_id,
                tier=TIER_MASTER,
                is_scrubbed=False,
                source_filename=path.name,
            )
            for t in _chunk_text(dc.text)
        ]
        total += store.ingest_document(
            tenant_id,
            doc_id=f"master:{schema.source_filename}",
            chunks=chunks,
            tier=TIER_MASTER,
            source_filename=path.name,
        )
    return {
        "sections": len(schema.sections),
        "chunks": total,
        "schema_version": schema.version,
        "source": schema.source_filename,
    }


def ingest_reference(tenant_id: str, path: Path) -> int:
    """Ingest a past report into REFERENCE tier; PII scrubbed per chunk in RagStore."""
    from backend.core.photo_layout import merge_layout_from_reference_docx

    text = doc_extractor.extract_text(path)
    schema = template_discoverer.load_schema(tenant_id)
    valid = set(schema.section_ids()) if schema else None
    from backend.core.reference_chunker import build_reference_chunks

    chunks = build_reference_chunks(
        text,
        source_filename=path.name,
        valid_section_ids=valid,
    )
    count = get_rag_store().ingest_document(
        tenant_id,
        doc_id=f"reference:{path.name}",
        chunks=chunks,
        tier=TIER_REFERENCE,
        source_filename=path.name,
    )
    if path.suffix.lower() in (".docx", ".docm"):
        section_ids = set(schema.section_ids()) if schema else None
        merge_layout_from_reference_docx(tenant_id, path, valid_section_ids=section_ids)
    return count


def auto_ingest_reference_dir(tenant_id: str) -> dict:
    """Pre-seed optional reference docs; never ingests the operator bundle files."""
    folder = settings.reference_dir_path
    if not folder.is_dir():
        return {"documents": 0, "chunks": 0}

    globs = [g.strip() for g in settings.reference_auto_ingest_globs.split(",") if g.strip()]
    skip = _operator_filenames_lower()
    seen: set[Path] = set()
    docs = 0
    total = 0
    for pattern in globs:
        for f in sorted(folder.glob(pattern)):
            if f.name.lower() in skip or f in seen:
                continue
            seen.add(f)
            try:
                total += ingest_reference(tenant_id, f)
                docs += 1
            except Exception as exc:  # noqa: BLE001
                logger.warning("Reference auto-ingest failed for %s (%s).", f.name, exc)

    logger.info("Reference auto-ingest for tenant=%s: %d docs, %d chunks", tenant_id, docs, total)
    return {"documents": docs, "chunks": total}