Download tiny_voca/text.py from RL10x/tiny-voca-hindi: direct link, hf CLI and curl.
- Browser
- Download file 14.7 kB
-
https://huggingface.co/RL10x/tiny-voca-hindi/resolve/main/tiny_voca/text.py
- Command line
-
hf download hf://RL10x/tiny-voca-hindi/tiny_voca/text.py
-
curl -L -o text.py https://huggingface.co/RL10x/tiny-voca-hindi/resolve/main/tiny_voca/text.py
14.7 kB
| """Hindi front end: Devanagari text -> phoneme tokens with a nasal feature. | |
| Replaces tiny-voca's English front end (eSpeak / dizione). The same code runs during annotation, training-set preparation | |
| and inference, so tokens can never drift between them. | |
| Input is Devanagari after our normalizer (small_tts.normalize_hi with english=False): numbers, dates and money are | |
| already words. Latin-script words are skipped (this model reads Devanagari only). | |
| * A token is one Hindi phoneme (IPA). Aspiration and breathy voice are part of the phoneme (kʰ, bʱ), as are vowel length | |
| (aː, iː). Phones the forced aligner has no label for map to the nearest label for alignment only (align.py). | |
| * Nasalization (anusvara before a vowel/sonorant, chandrabindu) is the parallel ``stress`` feature: 1 = the vowel is | |
| nasalized. It is not a token, so it never needs audio frames of its own. (tiny-voca's field name is kept.) | |
| * Inherent vowel /ə/ and schwa deletion: word-final schwa is dropped (except in one-akshara words), and a medial schwa | |
| is dropped in the context V C _ C V when that does not make a three-consonant cluster (right to left). Words that | |
| break the rule (compounds) are a known source of error. | |
| * Punctuation, word boundaries and the utterance edges are explicit tokens that may last zero frames (pauses). | |
| """ | |
| from dataclasses import dataclass | |
| import hashlib | |
| import json | |
| import os | |
| import re | |
| import unicodedata | |
| FRONTEND_VERSION = 1 | |
| FRONTENDS = ("hindi",) | |
| FRONTEND_VARIABLE = "TINY_VOCA_FRONTEND" | |
| PAD, BOS, EOS, SP, UNK = "<pad>", "<bos>", "<eos>", "<sp>", "<unk>" | |
| SPECIALS = (PAD, BOS, EOS, SP, UNK) | |
| PUNCTUATION = (",", ".", "?", "!", ";", ":", '"', "'", "(", ")", "—", "…") | |
| CONSONANTS = ("k", "kʰ", "ɡ", "ɡʱ", "ŋ", "tʃ", "tʃʰ", "dʒ", "dʒʱ", "ɲ", "ʈ", "ʈʰ", "ɖ", "ɖʱ", "ɳ", | |
| "t̪", "t̪ʰ", "d̪", "d̪ʱ", "n", "p", "pʰ", "b", "bʱ", "m", "j", "ɾ", "l", "ʋ", "ʃ", "ʂ", "s", "ɦ", | |
| "ɽ", "ɽʱ", "q", "x", "ɣ", "z", "f", "ʒ") | |
| VOWELS = ("ə", "aː", "ɪ", "iː", "ʊ", "uː", "eː", "ɛː", "oː", "ɔː", "æ", "ɔ") | |
| PHONES = CONSONANTS + VOWELS | |
| # Append-only: changing the order or removing a symbol invalidates checkpoints. | |
| SYMBOLS = SPECIALS + PUNCTUATION + PHONES | |
| SYMBOL_TO_ID = {symbol: index for index, symbol in enumerate(SYMBOLS)} | |
| PHONE_SET = frozenset(PHONES) | |
| PHONE_IDS = frozenset(SYMBOL_TO_ID[p] for p in PHONES) | {SYMBOL_TO_ID[UNK]} | |
| VOICELESS = frozenset(("k", "kʰ", "tʃ", "tʃʰ", "ʈ", "ʈʰ", "t̪", "t̪ʰ", "p", "pʰ", "ʃ", "ʂ", "s", "q", "x", "f")) | |
| VOWEL_SET = frozenset(VOWELS) | |
| SHORT_VOWELS = frozenset(("ə", "ɪ", "ʊ", "æ", "ɔ")) | |
| # ---------------------------------------------------------------- Devanagari tables | |
| CONSONANT_CHARS = { | |
| "क": "k", "ख": "kʰ", "ग": "ɡ", "घ": "ɡʱ", "ङ": "ŋ", | |
| "च": "tʃ", "छ": "tʃʰ", "ज": "dʒ", "झ": "dʒʱ", "ञ": "ɲ", | |
| "ट": "ʈ", "ठ": "ʈʰ", "ड": "ɖ", "ढ": "ɖʱ", "ण": "ɳ", | |
| "त": "t̪", "थ": "t̪ʰ", "द": "d̪", "ध": "d̪ʱ", "न": "n", | |
| "प": "p", "फ": "pʰ", "ब": "b", "भ": "bʱ", "म": "m", | |
| "य": "j", "र": "ɾ", "ल": "l", "व": "ʋ", "श": "ʃ", "ष": "ʂ", "स": "s", "ह": "ɦ", | |
| "ळ": "l", "ऱ": "ɾ", "ऴ": "l", | |
| } | |
| NUKTA_FORMS = {"क": "q", "ख": "x", "ग": "ɣ", "ज": "z", "फ": "f", "ड": "ɽ", "ढ": "ɽʱ", "झ": "ʒ", "न": "n", "र": "ɾ", | |
| "य": "j", "ल": "l"} | |
| INDEPENDENT_VOWELS = { | |
| "अ": "ə", "आ": "aː", "इ": "ɪ", "ई": "iː", "उ": "ʊ", "ऊ": "uː", "ऋ": ("ɾ", "ɪ"), "ॠ": ("ɾ", "iː"), | |
| "ए": "eː", "ऐ": "ɛː", "ओ": "oː", "औ": "ɔː", "ऍ": "æ", "ऑ": "ɔ", "ऎ": "eː", "ऒ": "oː", "ऌ": ("l", "ɪ"), | |
| } | |
| VOWEL_SIGNS = { | |
| "ा": "aː", "ि": "ɪ", "ी": "iː", "ु": "ʊ", "ू": "uː", "ृ": ("ɾ", "ɪ"), "ॄ": ("ɾ", "iː"), | |
| "े": "eː", "ै": "ɛː", "ो": "oː", "ौ": "ɔː", "ॅ": "æ", "ॉ": "ɔ", "ॆ": "eː", "ॊ": "oː", "ॢ": ("l", "ɪ"), | |
| } | |
| VIRAMA, NUKTA, ANUSVARA, CHANDRABINDU, VISARGA, AVAGRAHA = "्", "़", "ं", "ँ", "ः", "ऽ" | |
| # Anusvara before a stop becomes the nasal of the stop's place of articulation. | |
| HOMORGANIC = {} | |
| for _nasal, _stops in (("ŋ", "k kʰ ɡ ɡʱ q x ɣ"), ("ɲ", "tʃ tʃʰ dʒ dʒʱ"), ("ɳ", "ʈ ʈʰ ɖ ɖʱ"), | |
| ("n", "t̪ t̪ʰ d̪ d̪ʱ"), ("m", "p pʰ b bʱ")): | |
| for _s in _stops.split(): | |
| HOMORGANIC[_s] = _nasal | |
| PUNCT_MAP = {"।": ".", "॥": ".", "‘": "'", "’": "'", "“": '"', "”": '"', "–": "—", "-": None, "…": "…"} | |
| _DEVANAGARI = re.compile(r"[ऀ-ॿ]") | |
| _LATIN = re.compile(r"[A-Za-z]") | |
| class NothingToPronounce(ValueError): | |
| """The text has no Devanagari word to say: only marks, Latin script, emoji and the like.""" | |
| class Phonemized: | |
| symbols: tuple | |
| ids: tuple | |
| stress: tuple # here: 1 = nasalized vowel, 0 otherwise | |
| unknown: tuple # characters that had no mapping | |
| def __len__(self): | |
| return len(self.ids) | |
| def frontend(): | |
| return "hindi" | |
| def set_frontend(name): | |
| if name not in FRONTENDS: | |
| raise ValueError(f"front end must be one of {FRONTENDS}, not {name!r}") | |
| def frontend_identity(): | |
| return {"frontend_version": FRONTEND_VERSION, "phonemizer": "tts_hi devanagari rules", "symbols": list(SYMBOLS)} | |
| def frontend_runtime(): | |
| return {"phonemizer": frontend_fingerprint()} | |
| def frontend_fingerprint(): | |
| return hashlib.sha256(json.dumps(frontend_identity(), sort_keys=True, ensure_ascii=False).encode()).hexdigest() | |
| def normalize_text(text): | |
| text = unicodedata.normalize("NFC", text) | |
| text = "".join(c for c in text if unicodedata.category(c) != "Cf") # ZWJ / ZWNJ | |
| out = [] | |
| for c in text: | |
| if c in PUNCT_MAP: | |
| if PUNCT_MAP[c] is not None: | |
| out.append(PUNCT_MAP[c]) | |
| else: | |
| out.append(" ") | |
| else: | |
| out.append(c) | |
| text = "".join(out) | |
| text = re.sub(r"\.{3,}", "…", text) | |
| return re.sub(r"\s+", " ", text).strip() | |
| # ---------------------------------------------------------------- one word | |
| def _aksharas(word, unknown): | |
| """Units: ('C', phone, vowel_or_None, nasal, explicit) for a consonant (+vowel sign), ('V', vowel, nasal) for an | |
| independent vowel. vowel None = inherent schwa (subject to deletion) unless the consonant carries a virama.""" | |
| units, i, n = [], 0, len(word) | |
| while i < n: | |
| c = word[i] | |
| if c in CONSONANT_CHARS: | |
| phone = CONSONANT_CHARS[c] | |
| j = i + 1 | |
| if j < n and word[j] == NUKTA: | |
| phone = NUKTA_FORMS.get(c, phone) | |
| j += 1 | |
| vowel, explicit, virama = None, False, False | |
| if j < n and word[j] == VIRAMA: | |
| virama = True | |
| j += 1 | |
| elif j < n and word[j] in VOWEL_SIGNS: | |
| vowel, explicit = VOWEL_SIGNS[word[j]], True | |
| j += 1 | |
| nasal = 0 | |
| while j < n and word[j] in (ANUSVARA, CHANDRABINDU, VISARGA): | |
| if word[j] == VISARGA: | |
| units.append(("C", phone, vowel, nasal, explicit, virama)) | |
| phone, vowel, explicit, virama, nasal = "ɦ", None, False, True, 0 | |
| else: | |
| nasal = max(nasal, 2 if word[j] == CHANDRABINDU else 1) | |
| j += 1 | |
| units.append(("C", phone, vowel, nasal, explicit, virama)) | |
| i = j | |
| elif c in INDEPENDENT_VOWELS: | |
| vowel = INDEPENDENT_VOWELS[c] | |
| j, nasal = i + 1, 0 | |
| while j < n and word[j] in (ANUSVARA, CHANDRABINDU, VISARGA): | |
| if word[j] == VISARGA: | |
| units.append(("V", vowel, nasal)) | |
| vowel, nasal = None, False | |
| units.append(("C", "ɦ", None, False, False, True)) | |
| else: | |
| nasal = max(nasal, 2 if word[j] == CHANDRABINDU else 1) | |
| j += 1 | |
| if vowel is not None: | |
| units.append(("V", vowel, nasal)) | |
| i = j | |
| elif c in (AVAGRAHA,): | |
| i += 1 | |
| elif c in VOWEL_SIGNS or c in (ANUSVARA, CHANDRABINDU, NUKTA, VIRAMA, VISARGA): | |
| # a sign without a base (typo): attach the vowel / nasal to the previous unit if possible | |
| if c in VOWEL_SIGNS: | |
| units.append(("V", VOWEL_SIGNS[c], False)) | |
| elif c in (ANUSVARA, CHANDRABINDU) and units: | |
| u = units[-1] | |
| kind = 2 if c == CHANDRABINDU else 1 | |
| units[-1] = u[:3] + (kind,) + u[4:] if u[0] == "C" else (u[0], u[1], kind) | |
| i += 1 | |
| else: | |
| unknown.append(c) | |
| i += 1 | |
| return units | |
| def _word_phones(word, unknown): | |
| """List of (phone, nasal) for one Devanagari word.""" | |
| units = _aksharas(word, unknown) | |
| if not units: | |
| return [] | |
| # Inherent schwas: positions of consonant units with vowel None and no virama. | |
| schwa = [u[0] == "C" and u[2] is None and not u[5] for u in units] | |
| keep = list(schwa) | |
| syllables = sum(1 for k, u in enumerate(units) if u[0] == "V" or (u[0] == "C" and (u[2] is not None or schwa[k]))) | |
| last_c = max((k for k, u in enumerate(units) if u[0] == "C"), default=-1) | |
| if last_c == len(units) - 1 and schwa[last_c] and syllables > 1 and not units[last_c][3]: | |
| keep[last_c] = False # word-final schwa | |
| def has_vowel(k): | |
| u = units[k] | |
| return u[0] == "V" or (u[0] == "C" and (u[2] is not None or keep[k])) | |
| def longest_cluster(): | |
| run = best = 0 | |
| for m in range(len(units)): | |
| run = 0 if has_vowel(m) else run + 1 | |
| best = max(best, run) | |
| return best | |
| # Medial schwa: V C ə C V -> V C C V, right to left; never in the first akshara, never a 3-consonant cluster, | |
| # never on a nasalized schwa. | |
| for k in range(len(units) - 2, 0, -1): | |
| if not keep[k] or units[k][3]: | |
| continue | |
| if not has_vowel(k - 1) or units[k + 1][0] != "C" or not has_vowel(k + 1): | |
| continue | |
| keep[k] = False | |
| if longest_cluster() >= 3: | |
| keep[k] = True | |
| phones = [] | |
| for k, u in enumerate(units): | |
| if u[0] == "V": | |
| v = u[1] | |
| for p in (v if isinstance(v, tuple) else (v,)): | |
| phones.append([p, 0]) | |
| phones[-1][1] = u[2] | |
| continue | |
| _, cons, vowel, nasal, _, _ = u | |
| phones.append([cons, False]) | |
| if vowel is not None: | |
| for p in (vowel if isinstance(vowel, tuple) else (vowel,)): | |
| phones.append([p, False]) | |
| elif keep[k]: | |
| phones.append(["ə", False]) | |
| if nasal: | |
| phones[-1][1] = nasal # mark; resolved below | |
| # Nasal marks: chandrabindu always nasalizes the vowel; anusvara after a SHORT vowel before a stop becomes the | |
| # stop's nasal consonant (हिंदी ɦɪnd̪iː), otherwise it nasalizes the vowel (जाएंगे dʒaːeː̃ɡeː, मैं mɛː̃). | |
| out = [] | |
| for idx, (p, nasal) in enumerate(phones): | |
| nxt = phones[idx + 1][0] if idx + 1 < len(phones) else None | |
| if nasal and p in VOWEL_SET: | |
| if nasal == 1 and p in SHORT_VOWELS and nxt in HOMORGANIC: | |
| out.append((p, 0)) | |
| out.append((HOMORGANIC[nxt], 0)) | |
| else: | |
| out.append((p, 1)) | |
| elif nasal: # nasal on a bare consonant (no vowel kept) | |
| out.append((p, 0)) | |
| if nxt in HOMORGANIC: | |
| out.append((HOMORGANIC[nxt], 0)) | |
| else: | |
| out.append((p, 0)) | |
| # ज्ञ is said /ɡj/ in Hindi | |
| fixed, i = [], 0 | |
| while i < len(out): | |
| if i + 1 < len(out) and out[i][0] == "dʒ" and out[i + 1][0] == "ɲ": | |
| fixed += [("ɡ", 0), ("j", 0)] | |
| i += 2 | |
| else: | |
| fixed.append(out[i]) | |
| i += 1 | |
| return fixed | |
| def chunks(text): | |
| """Yield (is_punctuation, piece) over the normalized text.""" | |
| for token in re.findall(r"[ऀ-ॣ०-ॿ\w]+|[^\s\w]", text): | |
| if token in PUNCTUATION: | |
| yield True, token | |
| elif _DEVANAGARI.search(token): | |
| yield False, token | |
| # Latin words, digits left over, emoji: not spoken by this model | |
| def tokenize(pieces): | |
| """pieces: (is_punctuation, value) where a text value is a list of words, each a list of (phone, nasal).""" | |
| symbols, stress, unknown = [BOS], [0], [] | |
| for punctuation, value in pieces: | |
| if punctuation: | |
| if symbols[-1] != value: | |
| symbols.append(value) | |
| stress.append(0) | |
| continue | |
| for word in value: | |
| if not word: | |
| continue | |
| if symbols[-1] in PHONE_SET or symbols[-1] == UNK: | |
| symbols.append(SP) | |
| stress.append(0) | |
| for phone, nasal in word: | |
| symbols.append(phone if phone in PHONE_SET else UNK) | |
| stress.append(int(nasal)) | |
| symbols.append(EOS) | |
| stress.append(0) | |
| if not any(s in PHONE_SET or s == UNK for s in symbols): | |
| raise NothingToPronounce("the text has nothing to pronounce") | |
| return Phonemized(tuple(symbols), tuple(SYMBOL_TO_ID[s] for s in symbols), tuple(stress), tuple(unknown)) | |
| def phonemize(text): | |
| if not isinstance(text, str) or not text.strip(): | |
| raise ValueError("text must not be empty") | |
| unknown, pieces = [], [] | |
| for punctuation, piece in chunks(normalize_text(text)): | |
| pieces.append((True, piece) if punctuation else (False, [_word_phones(piece, unknown)])) | |
| result = tokenize(pieces) | |
| return Phonemized(result.symbols, result.ids, result.stress, tuple(unknown)) | |
| def to_ipa(text): | |
| """Readable form for checks: words joined by spaces, nasalized vowels with a tilde.""" | |
| p = phonemize(text) | |
| out = [] | |
| for s, n in zip(p.symbols, p.stress): | |
| if s == SP: | |
| out.append(" ") | |
| elif s in PHONE_SET: | |
| out.append(s + ("̃" if n else "")) | |
| elif s in PUNCTUATION: | |
| out.append(s) | |
| return "".join(out) | |
| def is_phone(token_ids): | |
| return [int(i) in PHONE_IDS for i in token_ids] | |