Spaces:
Sleeping
Sleeping
File size: 3,132 Bytes
0b42403 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 | """Heuristic classifier for chunk role: boilerplate vs reference vs exemplar.
At assembly tier (``ai_percent <= 12``) the system should preferentially quote
*boilerplate* text — generic professional phrasing without property-specific
facts. *Reference* chunks are example reports for other properties; their
wording is reusable but their facts (postcodes, addresses, prices, names)
must never bleed into the generated output.
This module provides a single classification entry point that ingestion code
can call to stamp the chunk_role on each Document's metadata before it is
written to the vector store.
"""
from __future__ import annotations
import re
# Patterns that indicate a chunk contains property-specific facts (=> reference).
_POSTCODE_RE = re.compile(r"\b[A-Z]{1,2}\d[A-Z\d]?\s*\d[A-Z]{2}\b", re.IGNORECASE)
_ADDRESS_RE = re.compile(
r"\b\d{1,4}\s+[A-Za-z][A-Za-z'\-]*(?:\s+[A-Za-z][A-Za-z'\-]*){0,5}\s+"
r"(?:Road|Rd|Street|St|Avenue|Ave|Lane|Ln|Drive|Dr|Crescent|Close|Place|Way|Gardens|Gdns|Court|Ct|Terrace|Terr)\b",
re.IGNORECASE,
)
_MONEY_RE = re.compile(r"£\s*\d[\d,]*(?:\.\d+)?")
_DATE_RE = re.compile(
r"\b(?:\d{1,2}[/-]\d{1,2}[/-]\d{2,4}|\d{1,2}\s+(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\s+\d{2,4})\b",
re.IGNORECASE,
)
_LONG_NUMBER_RE = re.compile(r"\b\d{5,}\b")
_NAME_TITLE_RE = re.compile(r"\b(MRICS|FRICS|AssocRICS)\b")
# Tokens that suggest a chunk is approved boilerplate / generic professional wording.
_BOILERPLATE_HINTS = (
"we recommend",
"we advise",
"should be",
"typically",
"is of",
"are of",
"construction",
"no evidence",
"satisfactory",
"fair condition",
"good condition",
"poor condition",
"the surveyor",
"this report",
"where possible",
"appears to be",
"found to be",
)
def classify_chunk_role(text: str) -> str:
"""Return ``"boilerplate"``, ``"reference"``, or ``"exemplar"``.
Heuristic:
- **reference**: contains property-specific facts (postcode/address/money/
date/long-number/name-title). The *wording* may be reusable but the
facts are not.
- **boilerplate**: short, generic professional sentence with phrasing
hints typical of approved RICS templates and no property-specific facts.
- **exemplar**: anything else — useful for context but not preferentially
quoted at assembly tier.
"""
t = (text or "").strip()
if not t:
return "exemplar"
if (
_POSTCODE_RE.search(t)
or _ADDRESS_RE.search(t)
or _MONEY_RE.search(t)
or _DATE_RE.search(t)
or _LONG_NUMBER_RE.search(t)
or _NAME_TITLE_RE.search(t)
):
return "reference"
lower = t.lower()
hint_hits = sum(1 for h in _BOILERPLATE_HINTS if h in lower)
if hint_hits >= 1 and len(t.split()) <= 60:
return "boilerplate"
return "exemplar"
def role_priority(role: str | None) -> int:
"""Sort key: boilerplate > exemplar > reference (lower is better)."""
if role == "boilerplate":
return 0
if role == "exemplar":
return 1
return 2
|