scripts/text_norm.py

#!/usr/bin/env python3
"""Canonical Hindi/Hinglish transcript normalizer for codepoint-level CTC.

Rules (locked 2026-07-20):
  1. Unicode NFC.
  2. Remove non-audio tags: strip everything inside <...> incl. brackets.
  3. Code-mix: `hindiword [English]` -> `English` (drop word before [X], keep X, drop brackets).
  4. Decompose precomposed nukta letters (U+0958..095F) -> base + U+093C for consistency.
  5. Map Devanagari digits ०-९ -> ASCII 0-9.
  6. Drop danda ।/॥ (punctuation).
  7. Remove zero-width marks (ZWJ/ZWNJ/ZWSP/BOM) WITHOUT inserting space (preserve conjuncts).
  8. Keep only: Devanagari block letters/signs, ASCII a-z (lowercased), digits 0-9. Everything
     else (punctuation, foreign scripts, symbols) -> space.
  9. Collapse whitespace, trim.

Two outputs:
  normalize(text)  -> human-readable cleaned text with normal spaces (used for WER refs).
  to_tokens(text)  -> same but spaces replaced by U+2581 '▁' (word-boundary token) for CTC
                      supervision. Each remaining char (incl ▁) is one token.
"""
import re
import unicodedata

WB = "▁"  # ▁ word-boundary token
_TAG = re.compile(r"<[^>]*>")
_BRACKET = re.compile(r"\S+\s*\[([^\]]*)\]")
_WS = re.compile(r"\s+")
_ZERO_WIDTH = {"​", "‌", "‍", ""}
# precomposed nukta -> base + nukta
_NUKTA = {
    "क़": "क़", "ख़": "ख़", "ग़": "ग़",
    "ज़": "ज़", "ड़": "ड़", "ढ़": "ढ़",
    "फ़": "फ़", "य़": "य़",
}
_DEV_DIGITS = {chr(0x0966 + i): str(i) for i in range(10)}
_DROP_DEV = {"।", "॥"}  # danda, double danda


def normalize(text: str) -> str:
    if not text:
        return ""
    t = unicodedata.normalize("NFC", text)
    t = _TAG.sub(" ", t)
    t = _BRACKET.sub(lambda m: " " + m.group(1) + " ", t)
    out = []
    for ch in t:
        if ch in _ZERO_WIDTH:
            continue
        if ch in _NUKTA:
            out.append(_NUKTA[ch]); continue
        if ch in _DEV_DIGITS:
            out.append(_DEV_DIGITS[ch]); continue
        o = ord(ch)
        if 0x0900 <= o <= 0x097F:
            out.append(" " if ch in _DROP_DEV else ch)
        elif "a" <= ch <= "z" or "A" <= ch <= "Z":
            out.append(ch.lower())
        elif ch in "0123456789":
            out.append(ch)
        else:
            out.append(" ")
    t = "".join(out)
    return _WS.sub(" ", t).strip()


def to_tokens(text: str) -> str:
    """Cleaned text with spaces -> ▁ (for CTC supervision). Leading ▁ omitted."""
    n = normalize(text)
    if not n:
        return ""
    return n.replace(" ", WB)


if __name__ == "__main__":
    ex = "<Persistent-noise-start> और जैसे कि पनीर [Paneer] पकौड़ा हुआ कई तरह के पकौड़े हैं बना <Persistent-noise-end>"
    print("IN :", ex)
    print("NRM:", normalize(ex))
    print("TOK:", to_tokens(ex))
    expect = "और जैसे कि paneer पकौड़ा हुआ कई तरह के पकौड़े हैं बना"
    assert normalize(ex) == expect, f"GOT: {normalize(ex)!r}"
    print("OK")