#!/usr/bin/env python3
"""
The one text normaliser. Every metric in this repo must use it.

THE BUG THIS MODULE EXISTS TO FIX
---------------------------------
The obvious way to strip punctuation is `re.sub(r"[^\\w\\s]", " ", s)`. In
Python's `re`, `\\w` means "alphanumeric" as decided by `str.isalnum()`, and
`str.isalnum()` is False for Unicode category **Mn/Mc** -- the combining marks
that carry the VOWELS in every Brahmic script.

So `[^\\w\\s]` deletes the vowel signs and the virama, and because it replaces
them with a SPACE it also splits the word at each one:

    ফ্ৰেংক গিফৰ্ডে তিনিগৰাকী মহিলাক বিয়া কৰাইছিল      6 words
    ফ ৰ ক গ ফৰ ড ত ন গৰ ক মহ ল ক ব য কৰ ইছ ল          18 fragments

Latin and Arabic script survive untouched, so the bug is invisible in English
and in Urdu, and silently destroys the other 13 languages. Every token-level
number computed through it -- token-F1, exact match, subsequence, overlap80,
the reader's own IDF span scorer -- was measuring consonant fragments rather
than words.

THE FIX
-------
Classify by Unicode category instead of by `\\w`:

    P*, S*   punctuation and symbols  -> replaced with a space (word boundary)
    Cf       ZWJ / ZWNJ / bidi marks  -> DELETED, not spaced (they sit INSIDE
                                         a word; spacing them re-splits it)
    L*, N*, M*                        -> kept (M* is the whole point)

`str.translate` with a self-populating table gives C-speed lookup after the
first few hundred characters, so this is not slower than the regex in practice.
"""
from __future__ import annotations

import re
import unicodedata

_WS = re.compile(r"\s+")


class _PunctTable(dict):
    """ord -> replacement, filled in on first sight of each codepoint."""

    def __missing__(self, o: int) -> str | None:
        cat = unicodedata.category(chr(o))
        if cat[0] in ("P", "S"):
            v: str | None = " "
        elif cat == "Cf":
            v = None          # str.translate deletes on None
        else:
            v = chr(o)
        self[o] = v
        return v


_PUNCT_TABLE = _PunctTable()


def strip_punct(s: str) -> str:
    """Punctuation and symbols -> space; format controls -> gone. Marks kept."""
    return s.translate(_PUNCT_TABLE)


def normalise(s: str, drop_punct: bool = False) -> str:
    """NFKC + casefold + whitespace collapse, optionally punctuation-stripped."""
    s = unicodedata.normalize("NFKC", s).lower()
    if drop_punct:
        s = strip_punct(s)
    return _WS.sub(" ", s).strip()


_EVENT = re.compile(r"[\[(][^\[\]()]{0,40}[\])]")


def strip_audio_events(s: str) -> tuple[str, list[str]]:
    """Drop a speech-to-text provider's non-speech annotations.

    ElevenLabs scribe_v1 tags what it hears but cannot transcribe inline, as
    "[background noise]", "[music]", "(laughter)". Those tags are a faithful
    description of the audio and useless as a query: they are English, they are
    absent from an Indic corpus, and they pull retrieval toward whatever
    passage happens to share their words. A live query came back as "Solar
    system based [background noise]" and retrieved a chunk about weather
    feeds.

    Bracketed spans are removed wholesale rather than matched against a
    vocabulary of known tags. Providers invent new ones, the list would rot,
    and neither a spoken question nor an MS MARCO query contains brackets --
    so the general rule costs nothing the specific one would have saved.
    The length cap keeps a genuinely bracketed clause from vanishing.

    Returns the cleaned text and the tags removed, so the caller can show what
    was dropped instead of silently changing what the user said.
    """
    dropped = _EVENT.findall(s)
    return _WS.sub(" ", _EVENT.sub(" ", s)).strip(), dropped


def tokens(s: str) -> list[str]:
    return normalise(s, True).split()