app/generator/notes_router.py

"""Route a flat list of messy inspector notes into per-section RICS buckets.

Workflow (user-visible):
  1. Surveyor uploads/pastes a single dump of field notes (often spanning
     multiple RICS sections — roof, services, grounds, signing all in one
     stream).
  2. ``/extract-notes`` parses that file into a flat ``lines`` list.
  3. This module then maps each line to the most-relevant RICS section
     code for the active product tier (L1/L2/L3) so the downstream
     generator gets *focused* bullets per section.

Without this routing every section would receive the same blob and the
LLM would have to re-classify per call — slow, inconsistent, and prone
to dropping detail (the failure mode users report when asking for "the
report to contain the complete data that is present in the messy notes").

Implementation:
  - Tier 1 (deterministic): a per-section keyword index built from the
    SectionTemplate title + expected_fields + a hand-curated synonyms map
    covering the most common RICS inspection vocabulary. Pure regex,
    works offline, no LLM cost.
  - Tier 2 (optional LLM refinement): when ``use_llm=True`` and an API
    key is configured, the LLM re-assigns ambiguous lines using the
    section context. The deterministic router always runs first so the
    LLM only adjudicates ties / low-confidence matches.
"""

from __future__ import annotations

import json
import logging
import re
from collections import defaultdict
from dataclasses import dataclass, field
from typing import Iterable

from app.config import settings
from app.templates.registry import get_survey_pack
from app.templates.rics_templates import SectionTemplate

logger = logging.getLogger(__name__)


# ---------------------------------------------------------------------------
# Section -> synonym index
# ---------------------------------------------------------------------------

# Hand-curated synonyms map. Each entry attaches a list of section code
# *prefixes* (L3 groupings) and a list of phrase fragments. We use prefixes
# rather than exact codes so the same map works across L1/L2/L3 packs (their
# code letters line up: A=intro, E=outside, F=inside, G=services, H=grounds).
# Where a synonym only applies to one specific subcode we use the full code.
#
# Keys are case-insensitive substrings matched against the lowercased
# bullet line. Order in the value list represents priority — earlier entries
# get higher scores when more than one keyword from the same section hits.
_SECTION_SYNONYMS: dict[str, list[str]] = {
    # ── A / B / C / D (intro, inspection, summary, property) ────────────
    "A": [
        "introduction", "report reference", "client name", "client names",
        "surveyor name", "rics number", "company name", "instructing",
        "scope of report", "terms of engagement", "client",
    ],
    "B": [
        "inspection date", "weather", "weather conditions", "inspected on",
        "date of inspection", "limitations", "limitation to inspection",
        "limits of inspection", "areas not inspected", "access restricted",
        "extent of inspection", "trace and access", "loft hatch",
        "occupied", "tenants present", "furniture", "carpets",
    ],
    "C": [
        "overall opinion", "overall assessment", "summary of condition",
        "condition rating", "ratings summary", "summary table",
        "executive summary", "valuation", "reinstatement",
    ],
    "D": [
        "about the property", "type of property", "property type",
        "construction", "year built", "approximate age", "circa",
        "tenure", "freehold", "leasehold", "council tax", "epc rating",
        "accommodation", "rooms", "address",  # generic address mentions
        "postcode",
    ],
    # ── E: Outside ───────────────────────────────────────────────────────
    "E1": [
        "chimney", "chimney stack", "chimney pot", "flue", "flashing",
    ],
    "E2": [
        "roof", "roof covering", "tile", "tiles", "slate", "slates",
        "felt", "ridge", "valley", "verge", "underlay", "lead flashing",
        "soffit", "fascia", "barge",
    ],
    "E3": [
        "rainwater goods", "gutter", "gutters", "downpipe", "downpipes",
        "rwp", "hopper",
    ],
    "E4": [
        "wall", "walls", "external wall", "external walls", "elevation",
        "render", "rendering", "pebbledash", "brickwork", "pointing",
        "repointing", "cavity wall", "cavity insulation",
    ],
    "E5": [
        "window", "windows", "frame", "frames", "double glazing", "dg",
        "triple glazing", "tg", "single glazing", "sg", "sealed unit",
        "misted", "misting", "casement", "sash",
    ],
    "E6": [
        "door", "doors", "external door", "front door", "back door",
        "rear door", "patio door", "french door",
    ],
    "E7": [
        "outbuilding", "outbuildings", "garage", "shed", "garden room",
        "summer house", "conservatory",
    ],
    "E8": [
        "boundaries", "boundary wall", "fence", "fences", "gate", "gates",
    ],
    "E9": [
        "other external", "external lighting", "satellite dish", "aerial",
    ],
    # ── F: Inside ────────────────────────────────────────────────────────
    "F1": [
        "roof structure", "roof void", "roof timber", "rafter", "purlin",
        "loft", "loft insulation", "joist",
    ],
    "F2": [
        "ceiling", "ceilings", "lath and plaster", "artex",
    ],
    "F3": [
        "internal wall", "internal walls", "partition", "stud wall",
    ],
    "F4": [
        "floor", "floors", "floorboard", "floorboards", "subfloor",
        "screed", "joists",
    ],
    "F5": [
        "fireplace", "fireplaces", "chimney breast", "hearth",
    ],
    "F6": [
        "built-in", "fitted furniture", "fitted kitchen", "fitted wardrobe",
        "kitchen units",
    ],
    "F7": [
        "woodwork", "skirting", "architrave", "internal joinery",
        "internal door", "internal doors",
    ],
    "F8": [
        "bathroom fittings", "kitchen fittings", "sanitary ware", "wc",
        "basin", "bath", "shower", "sink",
    ],
    "F9": [
        "dampness", "damp", "rising damp", "penetrating damp", "condensation",
        "mould", "mold", "wet rot", "dry rot", "woodworm", "rot",
    ],
    # ── G: Services ──────────────────────────────────────────────────────
    "G1": [
        "electrical", "electric", "consumer unit", "fuse board", "wiring",
        "rcd", "mcb", "socket", "sockets", "eicr", "electrics", "earthing",
    ],
    "G2": [
        "gas", "boiler", "central heating", "radiator", "radiators",
        "gas safe", "flue", "carbon monoxide", "heating system",
    ],
    "G3": [
        "water supply", "stopcock", "pipework", "lead pipe", "lead piping",
        "copper pipe", "plastic pipe", "mains water", "water tank",
    ],
    "G4": [
        "hot water", "hot water cylinder", "immersion heater", "calorifier",
    ],
    "G5": [
        "drainage", "drain", "drains", "soil pipe", "manhole", "gully",
        "septic tank", "cesspit",
    ],
    "G6": [
        "ventilation", "extract fan", "extractor", "air brick",
        "ventilator", "mvhr",
    ],
    "G7": [
        "other services", "fire alarm", "smoke alarm", "burglar alarm",
        "intruder alarm", "tv aerial", "telephone",
    ],
    "G8": [
        "renewables", "solar panel", "solar pv", "solar thermal", "heat pump",
        "biomass",
    ],
    # ── H: Grounds ───────────────────────────────────────────────────────
    "H1": [
        "garage doors", "garage door", "garage roof",
    ],
    "H2": [
        "grounds", "garden", "patio", "path", "paths", "driveway",
        "tarmac", "block paving", "decking",
    ],
    "H3": [
        "shared areas", "communal", "common parts", "stairwell",
    ],
    # ── I / J / K / L ────────────────────────────────────────────────────
    "I1": [
        "legal advisers", "legal adviser", "lease", "leasehold", "freehold",
        "guarantees", "warranty", "warranties", "building regulations",
        "planning permission", "planning consent",
    ],
    "J1": [
        "risk to people", "asbestos", "lead paint", "radon",
    ],
    "J2": [
        "risk to building", "flooding", "subsidence",
    ],
    "J3": [
        "energy efficiency risk", "epc",
    ],
    "K1": [
        "energy efficiency", "epc", "insulation", "loft insulation",
        "cavity insulation", "u-value",
    ],
    "L": [
        "surveyor declaration", "declaration", "signature",
    ],
}

# Boost weights: matching the section's *title* words is stronger evidence
# than matching the synonym map. The numeric weight is added to the line's
# score for that section.
_TITLE_TOKEN_WEIGHT: float = 2.4
_SYNONYM_WEIGHT_FIRST: float = 3.0  # first synonym in the list is "primary"
_SYNONYM_WEIGHT_REST: float = 1.6
_EXPECTED_FIELD_WEIGHT: float = 1.4

# Minimum keyword confidence required to OVERRIDE an explicitly-typed section
# code. Set high: only re-label when the description unambiguously matches a
# different RICS section (e.g. "E1 roof structure" → F1). A weak/ambiguous
# description leaves the surveyor's typed code intact.
_EXPLICIT_OVERRIDE_FLOOR: float = 0.55


_SECTION_HEADING_RE = re.compile(
    r"""
    ^                              # start of line
    \s*
    (?:                            # optional bullet glyph
        [-•*]\s*
    )?
    (?:section\s+)?                # optional "section" prefix
    (?P<code>[A-L]\d{0,2})         # code like A, E2, G1
    [\s:.\-]+
    (?P<rest>.*)$
    """,
    re.IGNORECASE | re.VERBOSE,
)


# ---------------------------------------------------------------------------
# Index building
# ---------------------------------------------------------------------------


def _normalise(text: str) -> str:
    return re.sub(r"\s+", " ", (text or "").strip().lower())


@dataclass(slots=True, frozen=True)
class _SectionIndex:
    code: str
    title: str
    title_tokens: tuple[str, ...]
    synonyms: tuple[str, ...]
    primary_synonyms: tuple[str, ...]
    expected_fields_tokens: tuple[str, ...]


def _build_section_index(template: SectionTemplate) -> _SectionIndex:
    title_clean = _normalise(template.title)
    title_tokens = tuple(
        t for t in re.findall(r"[A-Za-z]{4,}", title_clean) if t not in _STOPWORDS
    )
    # Resolve synonyms by exact code, then by top-letter fallback. For L1
    # packs (which only have single-letter codes like ``E`` / ``F`` / ``G``),
    # we also merge in every subcode synonym that starts with that letter
    # so the line "roof tiles slipped" still hits "E" — without this the
    # L1 sections would have empty synonyms tuples and routing would
    # collapse to "unknown" for every L1 input.
    syn: list[str] = list(_SECTION_SYNONYMS.get(template.code) or [])
    if not syn and len(template.code) == 1:
        for k, v in _SECTION_SYNONYMS.items():
            if k.startswith(template.code):
                syn.extend(v)
    if not syn:
        syn = list(_SECTION_SYNONYMS.get(template.code[0]) or [])
    primary = (syn[0],) if syn else ()
    expected_tokens: list[str] = []
    for f in template.expected_fields[:30]:
        for tok in re.findall(r"[a-z_]{3,}", f.lower()):
            tok = tok.replace("_", " ").strip()
            if tok and tok not in _STOPWORDS:
                expected_tokens.append(tok)
    return _SectionIndex(
        code=template.code,
        title=template.title,
        title_tokens=title_tokens,
        synonyms=tuple(_normalise(s) for s in syn),
        primary_synonyms=tuple(_normalise(s) for s in primary),
        expected_fields_tokens=tuple(expected_tokens),
    )


_STOPWORDS: frozenset[str] = frozenset(
    {
        "the", "and", "for", "with", "from", "into", "onto", "your", "their",
        "about", "this", "that", "these", "those", "report",
    }
)


_INDEX_CACHE: dict[int, list[_SectionIndex]] = {}


def _section_index_for_survey(survey_level: int) -> list[_SectionIndex]:
    if survey_level in _INDEX_CACHE:
        return _INDEX_CACHE[survey_level]
    pack = get_survey_pack(survey_level)
    out = [_build_section_index(t) for t in pack._by_code.values()]
    _INDEX_CACHE[survey_level] = out
    return out


# ---------------------------------------------------------------------------
# Scoring
# ---------------------------------------------------------------------------


@dataclass(slots=True)
class _LineScore:
    code: str | None
    confidence: float
    matched_terms: list[str] = field(default_factory=list)


def _explicit_section_prefix(line: str, codes: set[str]) -> str | None:
    """If the line begins with an explicit section code (e.g. ``E2: …``), trust it."""
    m = _SECTION_HEADING_RE.match(line)
    if not m:
        return None
    code = (m.group("code") or "").upper()
    return code if code in codes else None


def _score_line(line: str, indexes: list[_SectionIndex]) -> _LineScore:
    """Return the best-matching section + confidence for one line."""
    norm = _normalise(line)
    if not norm:
        return _LineScore(code=None, confidence=0.0, matched_terms=[])

    scores: dict[str, float] = defaultdict(float)
    matched: dict[str, list[str]] = defaultdict(list)

    for idx in indexes:
        for tok in idx.title_tokens:
            if tok and re.search(rf"\b{re.escape(tok)}\b", norm):
                scores[idx.code] += _TITLE_TOKEN_WEIGHT
                matched[idx.code].append(tok)
        for i, syn in enumerate(idx.synonyms):
            if not syn:
                continue
            if re.search(rf"(?<![a-z])({re.escape(syn)})(?![a-z])", norm):
                w = _SYNONYM_WEIGHT_FIRST if i == 0 else _SYNONYM_WEIGHT_REST
                scores[idx.code] += w
                matched[idx.code].append(syn)
        for tok in idx.expected_fields_tokens:
            if re.search(rf"\b{re.escape(tok)}\b", norm):
                scores[idx.code] += _EXPECTED_FIELD_WEIGHT
                matched[idx.code].append(tok)

    if not scores:
        return _LineScore(code=None, confidence=0.0, matched_terms=[])

    best_code, best_score = max(scores.items(), key=lambda kv: kv[1])
    # Confidence: ratio of best score to total accumulated score, with a
    # floor on absolute strength so a single weak match doesn't count as
    # high-confidence routing.
    total = sum(scores.values())
    ratio = best_score / total if total > 0 else 0.0
    strength = min(1.0, best_score / 6.0)  # 6 is a "two strong synonyms" target
    confidence = round(0.35 * ratio + 0.65 * strength, 4)
    return _LineScore(
        code=best_code,
        confidence=confidence,
        matched_terms=list(dict.fromkeys(matched[best_code]))[:6],
    )


# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------


@dataclass(slots=True)
class _RoutedLine:
    line: str
    code: str | None
    confidence: float
    matched_terms: list[str]


def _route_deterministic(
    lines: list[str], *, survey_level: int, confidence_floor: float = 0.30
) -> list[_RoutedLine]:
    """Per-line deterministic routing (no LLM)."""
    indexes = _section_index_for_survey(survey_level)
    code_set = {idx.code for idx in indexes}
    out: list[_RoutedLine] = []
    for raw in lines:
        line = str(raw or "").strip()
        if not line:
            continue
        # Honour explicit section codes if the user typed them — UNLESS the
        # description strongly contradicts the RICS definition for that code.
        # Surveyors use their own informal numbering (e.g. "E1 roof structure"
        # where RICS E1 = Chimney stacks and roof structure is F1). Blindly
        # trusting the typed code mis-files content across the whole report.
        forced = _explicit_section_prefix(line, code_set)
        if forced:
            descr_score = _score_line(line, indexes)
            if (
                descr_score.code
                and descr_score.code != forced
                and descr_score.confidence >= _EXPLICIT_OVERRIDE_FLOOR
            ):
                out.append(
                    _RoutedLine(
                        line=line,
                        code=descr_score.code,
                        confidence=descr_score.confidence,
                        matched_terms=(
                            [f"[relabelled from {forced}]", *descr_score.matched_terms]
                        ),
                    )
                )
                continue
            out.append(
                _RoutedLine(
                    line=line, code=forced, confidence=0.99, matched_terms=["[explicit code]"]
                )
            )
            continue
        score = _score_line(line, indexes)
        if score.code and score.confidence >= confidence_floor:
            out.append(
                _RoutedLine(
                    line=line,
                    code=score.code,
                    confidence=score.confidence,
                    matched_terms=score.matched_terms,
                )
            )
        else:
            out.append(
                _RoutedLine(
                    line=line,
                    code=None,
                    confidence=score.confidence,
                    matched_terms=score.matched_terms,
                )
            )
    return out


async def _llm_refine(
    routed: list[_RoutedLine], *, survey_level: int
) -> list[_RoutedLine]:
    """LLM refinement pass for low-confidence lines only.

    Sends only the unmatched / low-confidence lines to the LLM along with
    the section title list; high-confidence deterministic matches are
    preserved without paying the LLM cost.
    """
    if not (settings.openai_api_key or "").strip():
        return routed
    pack = get_survey_pack(survey_level)
    section_list = "\n".join(
        f"- {t.code}: {t.title}" for t in pack._by_code.values()
    )

    candidates = [
        (i, r)
        for i, r in enumerate(routed)
        if r.code is None or r.confidence < 0.55
    ]
    if not candidates:
        return routed

    payload_lines = [{"idx": i, "text": r.line} for i, r in candidates]
    system = (
        "You are a UK RICS Home Survey routing assistant. Given a list of raw inspector "
        "field-note lines and the active RICS report's section codes, return a JSON array "
        "of objects {idx, code, confidence} mapping each line to the most relevant section. "
        "Use ONLY codes from the provided list. If a line is truly off-topic (e.g. signature "
        "or unrelated chatter), return code=null. Output strict JSON only — no commentary."
    )
    user = (
        "RICS sections for this report (Level "
        f"{pack.level} — {pack.product_label}):\n"
        + section_list
        + "\n\nLines to route:\n"
        + json.dumps(payload_lines, ensure_ascii=False)
    )
    try:
        from app.llm.openai_chat import chat_completions_create

        raw = await chat_completions_create(
            messages=[
                {"role": "system", "content": system},
                {"role": "user", "content": user},
            ],
            model=settings.chat_model,
            max_tokens=min(4096, 80 * max(1, len(candidates))),
            temperature=0.0,
            phase="notes_route",
            section_id=None,
        )
        parsed = json.loads(raw or "[]")
    except Exception as exc:  # noqa: BLE001
        logger.warning("Notes-router LLM refinement failed (%s); keeping deterministic", exc)
        return routed

    if not isinstance(parsed, list):
        return routed
    by_idx = {int(p.get("idx", -1)): p for p in parsed if isinstance(p, dict)}
    code_set = {t.code for t in pack._by_code.values()}
    for i, r in candidates:
        p = by_idx.get(i)
        if not p:
            continue
        new_code = p.get("code")
        if new_code is None:
            continue
        if new_code not in code_set:
            continue
        try:
            new_conf = float(p.get("confidence", 0.6) or 0.6)
        except Exception:  # noqa: BLE001
            new_conf = 0.6
        new_conf = max(0.0, min(1.0, new_conf))
        # Only adopt the LLM's choice when its confidence beats the
        # deterministic match (or the deterministic match was null).
        if r.code is None or new_conf > r.confidence:
            r.code = new_code
            r.confidence = round(new_conf, 4)
            r.matched_terms = (r.matched_terms or []) + ["[llm-refined]"]
    return routed


@dataclass(slots=True)
class NotesRoutingResult:
    bullets_by_section: dict[str, list[str]]
    unrouted_lines: list[str]
    routing_details: list[dict]
    used_llm: bool


async def route_notes(
    lines: Iterable[str],
    *,
    survey_level: int,
    use_llm: bool = False,
    duplicate_to_unmatched: bool = False,
    confidence_floor: float = 0.30,
) -> NotesRoutingResult:
    """Route a flat list of messy notes lines into per-section bullets.

    Args:
        lines: Raw notes lines (typically the output of ``/extract-notes``).
        survey_level: RICS product tier (1/2/3) — drives the section universe.
        use_llm: Pass true to invoke the LLM refinement pass for low-confidence
            lines. Requires ``OPENAI_API_KEY``.
        duplicate_to_unmatched: When true, lines that don't match any section
            get a synthetic ``__unrouted__`` bucket so the caller can still
            ingest them somewhere instead of dropping them silently.
        confidence_floor: Minimum routing confidence to commit a line to a
            specific section. Lower values are more aggressive (route more
            lines) but produce more false routings.
    """
    lvl = max(1, min(3, int(survey_level or 3)))
    line_list = [str(ln).strip() for ln in lines if str(ln).strip()]
    routed = _route_deterministic(line_list, survey_level=lvl, confidence_floor=confidence_floor)
    if use_llm:
        routed = await _llm_refine(routed, survey_level=lvl)

    buckets: dict[str, list[str]] = {}
    unrouted: list[str] = []
    details: list[dict] = []
    for r in routed:
        details.append(
            {
                "line": r.line,
                "section": r.code,
                "confidence": float(r.confidence),
                "matched_terms": list(r.matched_terms or []),
            }
        )
        if r.code is None:
            unrouted.append(r.line)
            if duplicate_to_unmatched:
                buckets.setdefault("__unrouted__", []).append(r.line)
            continue
        buckets.setdefault(r.code, []).append(r.line)

    return NotesRoutingResult(
        bullets_by_section=buckets,
        unrouted_lines=unrouted,
        routing_details=details,
        used_llm=bool(use_llm and (settings.openai_api_key or "").strip()),
    )