"""Heuristic classifier for chunk role: boilerplate vs reference vs exemplar. At assembly tier (``ai_percent <= 12``) the system should preferentially quote *boilerplate* text — generic professional phrasing without property-specific facts. *Reference* chunks are example reports for other properties; their wording is reusable but their facts (postcodes, addresses, prices, names) must never bleed into the generated output. This module provides a single classification entry point that ingestion code can call to stamp the chunk_role on each Document's metadata before it is written to the vector store. """ from __future__ import annotations import re # Patterns that indicate a chunk contains property-specific facts (=> reference). _POSTCODE_RE = re.compile(r"\b[A-Z]{1,2}\d[A-Z\d]?\s*\d[A-Z]{2}\b", re.IGNORECASE) _ADDRESS_RE = re.compile( r"\b\d{1,4}\s+[A-Za-z][A-Za-z'\-]*(?:\s+[A-Za-z][A-Za-z'\-]*){0,5}\s+" r"(?:Road|Rd|Street|St|Avenue|Ave|Lane|Ln|Drive|Dr|Crescent|Close|Place|Way|Gardens|Gdns|Court|Ct|Terrace|Terr)\b", re.IGNORECASE, ) _MONEY_RE = re.compile(r"£\s*\d[\d,]*(?:\.\d+)?") _DATE_RE = re.compile( r"\b(?:\d{1,2}[/-]\d{1,2}[/-]\d{2,4}|\d{1,2}\s+(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\s+\d{2,4})\b", re.IGNORECASE, ) _LONG_NUMBER_RE = re.compile(r"\b\d{5,}\b") _NAME_TITLE_RE = re.compile(r"\b(MRICS|FRICS|AssocRICS)\b") # Tokens that suggest a chunk is approved boilerplate / generic professional wording. _BOILERPLATE_HINTS = ( "we recommend", "we advise", "should be", "typically", "is of", "are of", "construction", "no evidence", "satisfactory", "fair condition", "good condition", "poor condition", "the surveyor", "this report", "where possible", "appears to be", "found to be", ) def classify_chunk_role(text: str) -> str: """Return ``"boilerplate"``, ``"reference"``, or ``"exemplar"``. Heuristic: - **reference**: contains property-specific facts (postcode/address/money/ date/long-number/name-title). The *wording* may be reusable but the facts are not. - **boilerplate**: short, generic professional sentence with phrasing hints typical of approved RICS templates and no property-specific facts. - **exemplar**: anything else — useful for context but not preferentially quoted at assembly tier. """ t = (text or "").strip() if not t: return "exemplar" if ( _POSTCODE_RE.search(t) or _ADDRESS_RE.search(t) or _MONEY_RE.search(t) or _DATE_RE.search(t) or _LONG_NUMBER_RE.search(t) or _NAME_TITLE_RE.search(t) ): return "reference" lower = t.lower() hint_hits = sum(1 for h in _BOILERPLATE_HINTS if h in lower) if hint_hits >= 1 and len(t.split()) <= 60: return "boilerplate" return "exemplar" def role_priority(role: str | None) -> int: """Sort key: boilerplate > exemplar > reference (lower is better).""" if role == "boilerplate": return 0 if role == "exemplar": return 1 return 2