Spaces:
Sleeping
Sleeping
| """Unified, deterministic text normalization for ingest + retrieval queries. | |
| A single normalization pass keeps the *indexed* reference text and the *query* | |
| text in the same canonical form, so cosmetic Unicode differences (smart quotes, | |
| en/em dashes, ligatures, non-breaking spaces, PDF line-break hyphenation) never | |
| cause a lexical/semantic miss. It is intentionally conservative: | |
| * **Structure preserving** — paragraph breaks (blank lines) survive, so the | |
| reference chunker's section/paragraph splitting is unaffected. Only *runs* of | |
| intra-line whitespace collapse to a single space. | |
| * **Code preserving** — RICS section codes (``D4``, ``E2``, ``F1`` …) are plain | |
| alphanumerics and are never altered by any transform here. | |
| * **No semantic edits** — no stemming, casing, stop-word or PII handling. PII | |
| scrubbing happens on its own path and is untouched. | |
| Gated by ``settings.text_normalize_enabled`` at every call site (default off); | |
| this module itself is a pure function with no global state. | |
| """ | |
| from __future__ import annotations | |
| import re | |
| import unicodedata | |
| # Smart quotes / primes -> straight ASCII; dashes -> hyphen; ellipsis -> "..." | |
| _FOLD: dict[str, str] = { | |
| "\u2018": "'", | |
| "\u2019": "'", | |
| "\u201a": "'", | |
| "\u201b": "'", | |
| "\u2032": "'", | |
| "\u02bc": "'", | |
| "\u201c": '"', | |
| "\u201d": '"', | |
| "\u201e": '"', | |
| "\u201f": '"', | |
| "\u2033": '"', | |
| "\u2010": "-", | |
| "\u2011": "-", | |
| "\u2012": "-", | |
| "\u2013": "-", | |
| "\u2014": "-", | |
| "\u2015": "-", | |
| "\u2212": "-", | |
| "\u2026": "...", | |
| } | |
| _TRANSLATION = str.maketrans(_FOLD) | |
| # Newline canonicalization: CR/CRLF and Unicode line/paragraph separators -> \n | |
| _NEWLINE_RE = re.compile(r"\r\n?|\u2028|\u2029") | |
| # Soft-hyphen or hyphen immediately before a newline that splits a word (PDF | |
| # extraction artifact): join the two word halves back together. | |
| _DEHYPHEN_RE = re.compile(r"(\w)[\u00ad\-]\n[ \t]*(\w)") | |
| # Strip stray soft hyphens left anywhere else. | |
| _SOFT_HYPHEN_RE = re.compile(r"\u00ad") | |
| # Any run of horizontal whitespace (incl. NBSP / thin / figure spaces) -> one space | |
| _HSPACE_RE = re.compile(r"[ \t\u00a0\u2000-\u200a\u202f\u205f\u3000]+") | |
| _TRAILING_HSPACE_RE = re.compile(r"[ \t]+\n") | |
| _LEADING_HSPACE_RE = re.compile(r"\n[ \t]+") | |
| _MULTI_NEWLINE_RE = re.compile(r"\n{3,}") | |
| def normalize_text(text: str) -> str: | |
| """Canonicalize Unicode, fold quotes/dashes, de-hyphenate, collapse whitespace. | |
| Preserves blank-line paragraph boundaries; collapses only intra-line | |
| whitespace runs. Returns ``""`` for falsy input. Idempotent. | |
| """ | |
| if not text: | |
| return "" | |
| text = _NEWLINE_RE.sub("\n", text) | |
| text = _DEHYPHEN_RE.sub(r"\1\2", text) | |
| text = _SOFT_HYPHEN_RE.sub("", text) | |
| text = text.translate(_TRANSLATION) | |
| # NFKC folds ligatures (fi -> fi), full-width forms and compatibility chars. | |
| text = unicodedata.normalize("NFKC", text) | |
| text = _HSPACE_RE.sub(" ", text) | |
| text = _TRAILING_HSPACE_RE.sub("\n", text) | |
| text = _LEADING_HSPACE_RE.sub("\n", text) | |
| text = _MULTI_NEWLINE_RE.sub("\n\n", text) | |
| return text.strip() | |
| def normalize_query(text: str) -> str: | |
| """Single-line normalization for a retrieval query (newlines -> spaces).""" | |
| normalized = normalize_text(text) | |
| return _HSPACE_RE.sub(" ", normalized.replace("\n", " ")).strip() | |