app/generator/notes_coverage.py

"""Notes-coverage guard — ensures generated section text preserves the facts
in the inspector's RAW NOTES.

The pipeline already enforces *non-invention* (output cannot add facts that
aren't in notes/RAG). This module enforces the *converse*: output must
**preserve** the facts that ARE in the notes. Without it, the LLM happily
compresses 600 words of dense field notes into a 140-word tier-target
paragraph and silently drops half the observations — the exact failure
mode the user reports.

We tokenise each bullet into "fact tokens" (numbers + multi-word noun
phrases + low-frequency content terms), compute the overlap ratio against
the generated text, and surface any bullet whose key tokens never appear
in the output. The caller (``_generate_section_text`` /
``run_inspector_tool_loop``) uses ``coverage_report`` to decide whether to
regenerate with a stricter hint listing the missing facts.

This module is deliberately deterministic (no LLM, no embeddings). Coverage
checking inside a regenerate loop must be fast and cheap, otherwise the
"regenerate up to 3 times" knob doubles latency for every section.
"""

from __future__ import annotations

import logging
import re
from dataclasses import dataclass, field

logger = logging.getLogger(__name__)


# ---------------------------------------------------------------------------
# Tokenisation
# ---------------------------------------------------------------------------

# Words ignored when extracting fact tokens — these are not factual content.
# Kept intentionally narrow; we WANT "damp", "crack", "mould", "boiler" etc.
# to be treated as facts. The list contains pure function words plus a few
# RICS-grade fillers that appear in nearly every bullet and would otherwise
# inflate the false-coverage score.
_STOPWORDS: frozenset[str] = frozenset(
    {
        "a", "an", "the", "and", "or", "but", "if", "then", "of", "to", "in",
        "on", "at", "by", "for", "with", "from", "into", "onto", "as", "is",
        "are", "was", "were", "be", "been", "being", "this", "that", "these",
        "those", "it", "its", "their", "them", "they", "we", "our", "you",
        "your", "i", "me", "my", "any", "some", "all", "no", "not", "so",
        "do", "does", "did", "has", "have", "had", "will", "would", "should",
        "may", "might", "can", "could", "must", "shall", "also", "very", "more",
        "less", "much", "many", "few", "such", "etc", "ie", "eg", "vs",
        "than", "then", "now", "yet", "still", "just", "only", "even", "ever",
        "ok", "okay", "fine", "good", "bad",
    }
)

# Compass / location modifiers — we want these to count as facts so dropping
# "north elevation" actually triggers coverage failure. Hence they are NOT in
# _STOPWORDS even though they're short.

# Token splitter: words (incl. apostrophes/hyphens), numbers with optional
# units, postcodes.
_TOKEN_RE = re.compile(
    r"""
    (?P<postcode>[A-Z]{1,2}\d[A-Z\d]?\s*\d[A-Z]{2})    # UK postcode
    |
    (?P<money>£\s*\d[\d,]*(?:\.\d+)?)                  # money
    |
    (?P<number>
        \d[\d,]*(?:\.\d+)?                             # base number
        (?:\s*(?:sq\s*m|sqm|m²|sq\s*ft|sqft|mm|cm|m|km|ft|in|%|kg|kw|hrs?|hr|yrs?|yr|years?|months?|bedrooms?|bedroom|beds?|bathrooms?|baths?|storeys?|stories?|floors?))?
    )
    |
    (?P<word>[A-Za-z][A-Za-z'\-]{1,})                  # word (>=2 chars after lead letter, so "a" is 1 char and skipped)
    """,
    re.VERBOSE | re.IGNORECASE,
)


def _tokenise(text: str) -> list[str]:
    """Lowercased token stream from arbitrary text."""
    out: list[str] = []
    if not text:
        return out
    for m in _TOKEN_RE.finditer(text):
        if m.group("postcode"):
            out.append(re.sub(r"\s+", "", m.group("postcode")).upper())
        elif m.group("money"):
            out.append(re.sub(r"\s+", "", m.group("money")).lower())
        elif m.group("number"):
            out.append(re.sub(r"\s+", "", m.group("number")).lower())
        else:
            w = m.group("word").strip().lower()
            if w:
                out.append(w)
    return out


def _is_factish(token: str) -> bool:
    """Return True if ``token`` carries factual information worth tracking.

    A token is fact-bearing if it is:
      - a number, money, or postcode (has a digit or starts with £); OR
      - a content word that isn't a stopword and is at least 3 chars.

    We deliberately keep "damp", "crack", "leak", "rust" etc. (4-5 chars)
    because dropping them is a meaningful detail loss.
    """
    if not token:
        return False
    if any(ch.isdigit() for ch in token) or token.startswith("£"):
        return True
    if token in _STOPWORDS:
        return False
    if len(token) < 3:
        return False
    return True


@dataclass(frozen=True, slots=True)
class BulletFacts:
    """Per-bullet fact summary used by the coverage check."""

    bullet: str
    tokens: tuple[str, ...]
    key_tokens: tuple[str, ...]


def extract_bullet_facts(bullets: list[str]) -> list[BulletFacts]:
    """Tokenise each bullet into a ``BulletFacts`` summary.

    ``key_tokens`` is the deduplicated subset of ``tokens`` that are
    fact-bearing (per ``_is_factish``). When ``key_tokens`` is empty (the
    bullet is purely stopwords / very short), it is dropped from the
    returned list because there is nothing to verify coverage for.
    """
    out: list[BulletFacts] = []
    for raw in bullets or []:
        b = str(raw or "").strip()
        if not b:
            continue
        toks = tuple(_tokenise(b))
        keys: list[str] = []
        seen: set[str] = set()
        for t in toks:
            if not _is_factish(t):
                continue
            if t in seen:
                continue
            seen.add(t)
            keys.append(t)
        if not keys:
            continue
        out.append(BulletFacts(bullet=b, tokens=toks, key_tokens=tuple(keys)))
    return out


# ---------------------------------------------------------------------------
# Coverage scoring
# ---------------------------------------------------------------------------


@dataclass(frozen=True, slots=True)
class BulletCoverage:
    """How well a single bullet's facts appear in the generated text."""

    bullet: str
    coverage_ratio: float
    matched_tokens: tuple[str, ...]
    missing_tokens: tuple[str, ...]


@dataclass(frozen=True, slots=True)
class CoverageReport:
    """Aggregate coverage summary across all bullets."""

    coverage_ratio: float
    bullets_total: int
    bullets_well_covered: int
    bullets_poorly_covered: int
    poor_bullets: tuple[BulletCoverage, ...]
    missing_tokens: tuple[str, ...] = field(default_factory=tuple)

    @property
    def needs_regenerate(self) -> bool:
        """Default threshold: regenerate when ratio < 0.70 OR ≥3 poor bullets."""
        return self.coverage_ratio < 0.70 or self.bullets_poorly_covered >= 3


_OUTPUT_TOKEN_CACHE_LIMIT = 16


def _output_token_set(text: str) -> set[str]:
    """Build a set of normalised tokens from the generated text.

    Numbers / money / postcodes match exactly (after whitespace
    normalisation). Words match case-insensitively. We also collapse
    common derivational endings so "cracked" / "cracking" / "cracks"
    all hit the same bucket as "crack" — otherwise the LLM's perfectly
    valid rewording of "minor cracking to the rear" looks like a missing
    fact for the bullet that said "cracks at rear".
    """
    tokens = set(_tokenise(text))
    expanded: set[str] = set()
    for t in tokens:
        expanded.add(t)
        if any(ch.isdigit() for ch in t) or t.startswith("£"):
            continue
        for suf in ("ing", "ed", "es", "s", "ly"):
            if t.endswith(suf) and len(t) > len(suf) + 2:
                expanded.add(t[: -len(suf)])
        # Stem singular/plural roughly: "houses" -> "house"
        if t.endswith("y") and len(t) > 3:
            expanded.add(t[:-1] + "ies")
    return expanded


def _bullet_coverage(
    facts: BulletFacts, output_tokens: set[str]
) -> BulletCoverage:
    matched: list[str] = []
    missing: list[str] = []
    for k in facts.key_tokens:
        if k in output_tokens:
            matched.append(k)
            continue
        # Loosen for stem-collisions: try stripping common suffixes from the
        # bullet token too. The LLM may write "crack" when bullet says
        # "cracks", which the output_token expansion already covers, but
        # we also reverse-stem in case the LLM uses an even shorter root.
        hit = False
        if not any(ch.isdigit() for ch in k):
            for suf in ("ing", "ed", "es", "s", "ly"):
                if k.endswith(suf) and len(k) > len(suf) + 2:
                    root = k[: -len(suf)]
                    if root in output_tokens:
                        matched.append(k)
                        hit = True
                        break
        if not hit:
            missing.append(k)
    total = len(facts.key_tokens) or 1
    ratio = len(matched) / total
    return BulletCoverage(
        bullet=facts.bullet,
        coverage_ratio=ratio,
        matched_tokens=tuple(matched),
        missing_tokens=tuple(missing),
    )


def coverage_report(
    bullets: list[str],
    text: str,
    *,
    well_covered_threshold: float = 0.6,
    poor_bullet_threshold: float = 0.34,
    extra_trusted_text: list[str] | None = None,
) -> CoverageReport:
    """Compute coverage of ``bullets`` inside the generated ``text``.

    ``extra_trusted_text`` lets the caller mark additional strings (e.g.
    other sections of the same report) as "already covered" so a fact
    that legitimately belongs to another section isn't double-counted as
    missing here.

    Thresholds:
      * ``well_covered_threshold``  — a bullet is "well covered" when its
        key-token coverage ratio is at least this value.
      * ``poor_bullet_threshold``   — a bullet is "poorly covered" when
        its ratio is *below* this value. Bullets between the two
        thresholds are partial — neither well nor poor.
    """
    facts = extract_bullet_facts(bullets)
    if not facts:
        return CoverageReport(
            coverage_ratio=1.0,
            bullets_total=0,
            bullets_well_covered=0,
            bullets_poorly_covered=0,
            poor_bullets=(),
            missing_tokens=(),
        )
    output_tokens = _output_token_set(text or "")
    if extra_trusted_text:
        for s in extra_trusted_text:
            output_tokens |= _output_token_set(s or "")
    well = 0
    poor: list[BulletCoverage] = []
    ratios: list[float] = []
    missing_aggregate: list[str] = []
    seen_missing: set[str] = set()
    for f in facts:
        bc = _bullet_coverage(f, output_tokens)
        ratios.append(bc.coverage_ratio)
        if bc.coverage_ratio >= well_covered_threshold:
            well += 1
        if bc.coverage_ratio < poor_bullet_threshold:
            poor.append(bc)
        for m in bc.missing_tokens:
            if m in seen_missing:
                continue
            seen_missing.add(m)
            missing_aggregate.append(m)
    overall = sum(ratios) / len(ratios) if ratios else 1.0
    return CoverageReport(
        coverage_ratio=overall,
        bullets_total=len(facts),
        bullets_well_covered=well,
        bullets_poorly_covered=len(poor),
        poor_bullets=tuple(poor),
        missing_tokens=tuple(missing_aggregate[:120]),
    )


def coverage_regenerate_hint(report: CoverageReport, *, max_bullets: int = 6) -> str:
    """Render a strict regenerate hint listing the dropped facts/bullets.

    The hint is meant to be appended to ``creativity_hint`` for the
    one-more-attempt path. We list the most poorly covered bullets first,
    plus a flat list of missing tokens, so the LLM has both shapes to
    re-anchor against.
    """
    if not report.poor_bullets and not report.missing_tokens:
        return ""
    lines: list[str] = [
        "RAW NOTES COVERAGE VIOLATION — your previous draft dropped observations.",
        (
            "You MUST regenerate so every observation from the RAW NOTES surfaces in the output. "
            "Do not summarise away facts. Do not compress two bullets into a single generic clause. "
            "If a fact is genuinely off-topic for this section, leave a one-line professional "
            "acknowledgment rather than dropping it silently."
        ),
        f"Coverage measured: {int(round(report.coverage_ratio * 100))}% "
        f"(target ≥ 70%; {report.bullets_poorly_covered} of {report.bullets_total} bullets poorly covered).",
    ]
    if report.poor_bullets:
        lines.append("POORLY COVERED BULLETS (re-anchor every one of these):")
        for i, bc in enumerate(report.poor_bullets[:max_bullets], 1):
            preview = bc.bullet.strip().replace("\n", " ")
            if len(preview) > 260:
                preview = preview[:259] + "…"
            missing_preview = ", ".join(bc.missing_tokens[:8])
            lines.append(f"  {i}. \"{preview}\" — missing tokens: {missing_preview}")
    if report.missing_tokens:
        flat = ", ".join(report.missing_tokens[:24])
        lines.append(f"DROPPED FACT TOKENS (must appear in the regenerated text): {flat}")
    return "\n".join(lines)