File size: 3,096 Bytes
e146811 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 | #!/usr/bin/env python3
"""Canonical Hindi/Hinglish transcript normalizer for codepoint-level CTC.
Rules (locked 2026-07-20):
1. Unicode NFC.
2. Remove non-audio tags: strip everything inside <...> incl. brackets.
3. Code-mix: `hindiword [English]` -> `English` (drop word before [X], keep X, drop brackets).
4. Decompose precomposed nukta letters (U+0958..095F) -> base + U+093C for consistency.
5. Map Devanagari digits ०-९ -> ASCII 0-9.
6. Drop danda ।/॥ (punctuation).
7. Remove zero-width marks (ZWJ/ZWNJ/ZWSP/BOM) WITHOUT inserting space (preserve conjuncts).
8. Keep only: Devanagari block letters/signs, ASCII a-z (lowercased), digits 0-9. Everything
else (punctuation, foreign scripts, symbols) -> space.
9. Collapse whitespace, trim.
Two outputs:
normalize(text) -> human-readable cleaned text with normal spaces (used for WER refs).
to_tokens(text) -> same but spaces replaced by U+2581 '▁' (word-boundary token) for CTC
supervision. Each remaining char (incl ▁) is one token.
"""
import re
import unicodedata
WB = "▁" # ▁ word-boundary token
_TAG = re.compile(r"<[^>]*>")
_BRACKET = re.compile(r"\S+\s*\[([^\]]*)\]")
_WS = re.compile(r"\s+")
_ZERO_WIDTH = {"", "", "", ""}
# precomposed nukta -> base + nukta
_NUKTA = {
"क़": "क़", "ख़": "ख़", "ग़": "ग़",
"ज़": "ज़", "ड़": "ड़", "ढ़": "ढ़",
"फ़": "फ़", "य़": "य़",
}
_DEV_DIGITS = {chr(0x0966 + i): str(i) for i in range(10)}
_DROP_DEV = {"।", "॥"} # danda, double danda
def normalize(text: str) -> str:
if not text:
return ""
t = unicodedata.normalize("NFC", text)
t = _TAG.sub(" ", t)
t = _BRACKET.sub(lambda m: " " + m.group(1) + " ", t)
out = []
for ch in t:
if ch in _ZERO_WIDTH:
continue
if ch in _NUKTA:
out.append(_NUKTA[ch]); continue
if ch in _DEV_DIGITS:
out.append(_DEV_DIGITS[ch]); continue
o = ord(ch)
if 0x0900 <= o <= 0x097F:
out.append(" " if ch in _DROP_DEV else ch)
elif "a" <= ch <= "z" or "A" <= ch <= "Z":
out.append(ch.lower())
elif ch in "0123456789":
out.append(ch)
else:
out.append(" ")
t = "".join(out)
return _WS.sub(" ", t).strip()
def to_tokens(text: str) -> str:
"""Cleaned text with spaces -> ▁ (for CTC supervision). Leading ▁ omitted."""
n = normalize(text)
if not n:
return ""
return n.replace(" ", WB)
if __name__ == "__main__":
ex = "<Persistent-noise-start> और जैसे कि पनीर [Paneer] पकौड़ा हुआ कई तरह के पकौड़े हैं बना <Persistent-noise-end>"
print("IN :", ex)
print("NRM:", normalize(ex))
print("TOK:", to_tokens(ex))
expect = "और जैसे कि paneer पकौड़ा हुआ कई तरह के पकौड़े हैं बना"
assert normalize(ex) == expect, f"GOT: {normalize(ex)!r}"
print("OK")
|