|
|
| """
|
| The one text normaliser. Every metric in this repo must use it.
|
|
|
| THE BUG THIS MODULE EXISTS TO FIX
|
| ---------------------------------
|
| The obvious way to strip punctuation is `re.sub(r"[^\\w\\s]", " ", s)`. In
|
| Python's `re`, `\\w` means "alphanumeric" as decided by `str.isalnum()`, and
|
| `str.isalnum()` is False for Unicode category **Mn/Mc** -- the combining marks
|
| that carry the VOWELS in every Brahmic script.
|
|
|
| So `[^\\w\\s]` deletes the vowel signs and the virama, and because it replaces
|
| them with a SPACE it also splits the word at each one:
|
|
|
| ফ্ৰেংক গিফৰ্ডে তিনিগৰাকী মহিলাক বিয়া কৰাইছিল 6 words
|
| ফ ৰ ক গ ফৰ ড ত ন গৰ ক মহ ল ক ব য কৰ ইছ ল 18 fragments
|
|
|
| Latin and Arabic script survive untouched, so the bug is invisible in English
|
| and in Urdu, and silently destroys the other 13 languages. Every token-level
|
| number computed through it -- token-F1, exact match, subsequence, overlap80,
|
| the reader's own IDF span scorer -- was measuring consonant fragments rather
|
| than words.
|
|
|
| THE FIX
|
| -------
|
| Classify by Unicode category instead of by `\\w`:
|
|
|
| P*, S* punctuation and symbols -> replaced with a space (word boundary)
|
| Cf ZWJ / ZWNJ / bidi marks -> DELETED, not spaced (they sit INSIDE
|
| a word; spacing them re-splits it)
|
| L*, N*, M* -> kept (M* is the whole point)
|
|
|
| `str.translate` with a self-populating table gives C-speed lookup after the
|
| first few hundred characters, so this is not slower than the regex in practice.
|
| """
|
| from __future__ import annotations
|
|
|
| import re
|
| import unicodedata
|
|
|
| _WS = re.compile(r"\s+")
|
|
|
|
|
| class _PunctTable(dict):
|
| """ord -> replacement, filled in on first sight of each codepoint."""
|
|
|
| def __missing__(self, o: int) -> str | None:
|
| cat = unicodedata.category(chr(o))
|
| if cat[0] in ("P", "S"):
|
| v: str | None = " "
|
| elif cat == "Cf":
|
| v = None
|
| else:
|
| v = chr(o)
|
| self[o] = v
|
| return v
|
|
|
|
|
| _PUNCT_TABLE = _PunctTable()
|
|
|
|
|
| def strip_punct(s: str) -> str:
|
| """Punctuation and symbols -> space; format controls -> gone. Marks kept."""
|
| return s.translate(_PUNCT_TABLE)
|
|
|
|
|
| def normalise(s: str, drop_punct: bool = False) -> str:
|
| """NFKC + casefold + whitespace collapse, optionally punctuation-stripped."""
|
| s = unicodedata.normalize("NFKC", s).lower()
|
| if drop_punct:
|
| s = strip_punct(s)
|
| return _WS.sub(" ", s).strip()
|
|
|
|
|
| _EVENT = re.compile(r"[\[(][^\[\]()]{0,40}[\])]")
|
|
|
|
|
| def strip_audio_events(s: str) -> tuple[str, list[str]]:
|
| """Drop a speech-to-text provider's non-speech annotations.
|
|
|
| ElevenLabs scribe_v1 tags what it hears but cannot transcribe inline, as
|
| "[background noise]", "[music]", "(laughter)". Those tags are a faithful
|
| description of the audio and useless as a query: they are English, they are
|
| absent from an Indic corpus, and they pull retrieval toward whatever
|
| passage happens to share their words. A live query came back as "Solar
|
| system based [background noise]" and retrieved a chunk about weather
|
| feeds.
|
|
|
| Bracketed spans are removed wholesale rather than matched against a
|
| vocabulary of known tags. Providers invent new ones, the list would rot,
|
| and neither a spoken question nor an MS MARCO query contains brackets --
|
| so the general rule costs nothing the specific one would have saved.
|
| The length cap keeps a genuinely bracketed clause from vanishing.
|
|
|
| Returns the cleaned text and the tags removed, so the caller can show what
|
| was dropped instead of silently changing what the user said.
|
| """
|
| dropped = _EVENT.findall(s)
|
| return _WS.sub(" ", _EVENT.sub(" ", s)).strip(), dropped
|
|
|
|
|
| def tokens(s: str) -> list[str]:
|
| return normalise(s, True).split()
|
|
|
