Spaces:
Sleeping
Sleeping
| """Heuristic classifier for chunk role: boilerplate vs reference vs exemplar. | |
| At assembly tier (``ai_percent <= 12``) the system should preferentially quote | |
| *boilerplate* text — generic professional phrasing without property-specific | |
| facts. *Reference* chunks are example reports for other properties; their | |
| wording is reusable but their facts (postcodes, addresses, prices, names) | |
| must never bleed into the generated output. | |
| This module provides a single classification entry point that ingestion code | |
| can call to stamp the chunk_role on each Document's metadata before it is | |
| written to the vector store. | |
| """ | |
| from __future__ import annotations | |
| import re | |
| # Patterns that indicate a chunk contains property-specific facts (=> reference). | |
| _POSTCODE_RE = re.compile(r"\b[A-Z]{1,2}\d[A-Z\d]?\s*\d[A-Z]{2}\b", re.IGNORECASE) | |
| _ADDRESS_RE = re.compile( | |
| r"\b\d{1,4}\s+[A-Za-z][A-Za-z'\-]*(?:\s+[A-Za-z][A-Za-z'\-]*){0,5}\s+" | |
| r"(?:Road|Rd|Street|St|Avenue|Ave|Lane|Ln|Drive|Dr|Crescent|Close|Place|Way|Gardens|Gdns|Court|Ct|Terrace|Terr)\b", | |
| re.IGNORECASE, | |
| ) | |
| _MONEY_RE = re.compile(r"£\s*\d[\d,]*(?:\.\d+)?") | |
| _DATE_RE = re.compile( | |
| r"\b(?:\d{1,2}[/-]\d{1,2}[/-]\d{2,4}|\d{1,2}\s+(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\s+\d{2,4})\b", | |
| re.IGNORECASE, | |
| ) | |
| _LONG_NUMBER_RE = re.compile(r"\b\d{5,}\b") | |
| _NAME_TITLE_RE = re.compile(r"\b(MRICS|FRICS|AssocRICS)\b") | |
| # Tokens that suggest a chunk is approved boilerplate / generic professional wording. | |
| _BOILERPLATE_HINTS = ( | |
| "we recommend", | |
| "we advise", | |
| "should be", | |
| "typically", | |
| "is of", | |
| "are of", | |
| "construction", | |
| "no evidence", | |
| "satisfactory", | |
| "fair condition", | |
| "good condition", | |
| "poor condition", | |
| "the surveyor", | |
| "this report", | |
| "where possible", | |
| "appears to be", | |
| "found to be", | |
| ) | |
| def classify_chunk_role(text: str) -> str: | |
| """Return ``"boilerplate"``, ``"reference"``, or ``"exemplar"``. | |
| Heuristic: | |
| - **reference**: contains property-specific facts (postcode/address/money/ | |
| date/long-number/name-title). The *wording* may be reusable but the | |
| facts are not. | |
| - **boilerplate**: short, generic professional sentence with phrasing | |
| hints typical of approved RICS templates and no property-specific facts. | |
| - **exemplar**: anything else — useful for context but not preferentially | |
| quoted at assembly tier. | |
| """ | |
| t = (text or "").strip() | |
| if not t: | |
| return "exemplar" | |
| if ( | |
| _POSTCODE_RE.search(t) | |
| or _ADDRESS_RE.search(t) | |
| or _MONEY_RE.search(t) | |
| or _DATE_RE.search(t) | |
| or _LONG_NUMBER_RE.search(t) | |
| or _NAME_TITLE_RE.search(t) | |
| ): | |
| return "reference" | |
| lower = t.lower() | |
| hint_hits = sum(1 for h in _BOILERPLATE_HINTS if h in lower) | |
| if hint_hits >= 1 and len(t.split()) <= 60: | |
| return "boilerplate" | |
| return "exemplar" | |
| def role_priority(role: str | None) -> int: | |
| """Sort key: boilerplate > exemplar > reference (lower is better).""" | |
| if role == "boilerplate": | |
| return 0 | |
| if role == "exemplar": | |
| return 1 | |
| return 2 | |