app/generator/term_glossary.py

"""Note pre-processing: typo normalisation + unverified-term flagging.

Runs BEFORE the LLM sees the notes. Two jobs:

1. Normalise a small set of known transcription errors (lather -> lath, etc.).
2. Flag genuinely unrecognised technical terms so the model does not publish a
   transcription artefact ("astratin support") as if it were valid RICS
   terminology.

Design choice: we deliberately do NOT spell-check every word against a general
English dictionary. Surveying prose is full of legitimate low-frequency terms
(efflorescence, flaunching, bressummer) that a generic dictionary flags as
errors, which would be worse than the disease. Instead we flag conservatively:
a long, non-approved word is only flagged when it is used in a *structural-term
position* (immediately followed by a structural element noun such as
"support", "beam", "lintel"), which is exactly the "astratin support" pattern,
or when it is an obvious non-word (no vowels, repeated triples). This keeps the
false-positive rate near zero.
"""

from __future__ import annotations

import json
import re
from functools import lru_cache
from pathlib import Path

_DATA_PATH = Path(__file__).resolve().parent.parent / "data" / "surveying_approved_terms.json"

# Structural-element nouns that, when preceded by an unknown long word, strongly
# imply the unknown word was meant to be a (mis-transcribed) technical term.
_STRUCTURAL_NOUNS: frozenset[str] = frozenset(
    {
        "support", "supports", "beam", "beams", "lintel", "lintels", "joist",
        "joists", "prop", "props", "member", "members", "bracket", "brackets",
        "truss", "trusses", "purlin", "purlins", "rafter", "rafters",
        "padstone", "wallplate", "plate", "girder", "column", "post",
    }
)

# Known transcription errors -> canonical form. Applied case-insensitively as
# whole words. Keep this list tight and unambiguous.
_TYPO_NORMALISATIONS: dict[str, str] = {
    "lather": "lath",
    "wasteness": "deterioration",
}

_WORD_RE = re.compile(r"[A-Za-z][A-Za-z'\-]*")


@lru_cache(maxsize=1)
def approved_terms() -> frozenset[str]:
    """Load the approved surveying glossary (lowercased tokens)."""
    try:
        raw = json.loads(_DATA_PATH.read_text(encoding="utf-8"))
        terms = raw.get("terms", []) if isinstance(raw, dict) else []
        return frozenset(str(t).strip().lower() for t in terms if str(t).strip())
    except Exception:  # noqa: BLE001 — never block generation on a bad glossary
        return frozenset()


def _is_obvious_nonword(token: str) -> bool:
    """Heuristic for a token that cannot be a real English/technical word."""
    low = token.lower()
    if len(low) < 6:
        return False
    if not re.search(r"[aeiou]", low):  # no vowels at all
        return True
    if re.search(r"(.)\1\1", low):  # three identical letters in a row
        return True
    return False


def is_unverified_term(token: str, *, next_token: str | None = None) -> bool:
    """Return True if ``token`` should be flagged as unverified terminology."""
    low = token.lower().strip("-'")
    if len(low) <= 5 or not low.isalpha():
        return False
    if low in approved_terms():
        return False
    if _is_obvious_nonword(low):
        return True
    if next_token and next_token.lower().strip(".,;:-'") in _STRUCTURAL_NOUNS:
        return True
    return False


def preprocess_notes(text: str) -> str:
    """Normalise known typos and wrap unverified terms with a flag token.

    The flag format matches the UNKNOWN TERM RULE in the generation prompt:
    ``[UNVERIFIED_TERM: "astratin"]``.
    """
    if not text or not text.strip():
        return text

    # 1. Typo normalisation (whole-word, case-insensitive, preserve nothing
    #    fancy — these are unambiguous corrections).
    def _norm(m: re.Match[str]) -> str:
        return _TYPO_NORMALISATIONS[m.group(0).lower()]

    if _TYPO_NORMALISATIONS:
        pattern = re.compile(
            r"\b(" + "|".join(re.escape(k) for k in _TYPO_NORMALISATIONS) + r")\b",
            re.IGNORECASE,
        )
        text = pattern.sub(_norm, text)

    # 2. Unverified-term flagging. Walk tokens with lookahead at the next token.
    matches = list(_WORD_RE.finditer(text))
    if not matches:
        return text
    out: list[str] = []
    cursor = 0
    for i, m in enumerate(matches):
        tok = m.group(0)
        nxt = matches[i + 1].group(0) if i + 1 < len(matches) else None
        if is_unverified_term(tok, next_token=nxt):
            out.append(text[cursor : m.start()])
            out.append(f'[UNVERIFIED_TERM: "{tok}"]')
            cursor = m.end()
    out.append(text[cursor:])
    return "".join(out)


def preprocess_bullets(bullets: list[str]) -> list[str]:
    """Apply :func:`preprocess_notes` to each bullet."""
    return [preprocess_notes(b) if isinstance(b, str) else b for b in bullets]