RL10x's picture
Add inference code (Python + C++ engine + web page)
ab1b068 verified
Raw History Blame Contribute Delete
14.7 kB
"""Hindi front end: Devanagari text -> phoneme tokens with a nasal feature.
Replaces tiny-voca's English front end (eSpeak / dizione). The same code runs during annotation, training-set preparation
and inference, so tokens can never drift between them.
Input is Devanagari after our normalizer (small_tts.normalize_hi with english=False): numbers, dates and money are
already words. Latin-script words are skipped (this model reads Devanagari only).
* A token is one Hindi phoneme (IPA). Aspiration and breathy voice are part of the phoneme (kʰ, bʱ), as are vowel length
(aː, iː). Phones the forced aligner has no label for map to the nearest label for alignment only (align.py).
* Nasalization (anusvara before a vowel/sonorant, chandrabindu) is the parallel ``stress`` feature: 1 = the vowel is
nasalized. It is not a token, so it never needs audio frames of its own. (tiny-voca's field name is kept.)
* Inherent vowel /ə/ and schwa deletion: word-final schwa is dropped (except in one-akshara words), and a medial schwa
is dropped in the context V C _ C V when that does not make a three-consonant cluster (right to left). Words that
break the rule (compounds) are a known source of error.
* Punctuation, word boundaries and the utterance edges are explicit tokens that may last zero frames (pauses).
"""
from dataclasses import dataclass
import hashlib
import json
import os
import re
import unicodedata
FRONTEND_VERSION = 1
FRONTENDS = ("hindi",)
FRONTEND_VARIABLE = "TINY_VOCA_FRONTEND"
PAD, BOS, EOS, SP, UNK = "<pad>", "<bos>", "<eos>", "<sp>", "<unk>"
SPECIALS = (PAD, BOS, EOS, SP, UNK)
PUNCTUATION = (",", ".", "?", "!", ";", ":", '"', "'", "(", ")", "—", "…")
CONSONANTS = ("k", "kʰ", "ɡ", "ɡʱ", "ŋ", "tʃ", "tʃʰ", "dʒ", "dʒʱ", "ɲ", "ʈ", "ʈʰ", "ɖ", "ɖʱ", "ɳ",
"t̪", "t̪ʰ", "d̪", "d̪ʱ", "n", "p", "pʰ", "b", "bʱ", "m", "j", "ɾ", "l", "ʋ", "ʃ", "ʂ", "s", "ɦ",
"ɽ", "ɽʱ", "q", "x", "ɣ", "z", "f", "ʒ")
VOWELS = ("ə", "aː", "ɪ", "iː", "ʊ", "uː", "eː", "ɛː", "oː", "ɔː", "æ", "ɔ")
PHONES = CONSONANTS + VOWELS
# Append-only: changing the order or removing a symbol invalidates checkpoints.
SYMBOLS = SPECIALS + PUNCTUATION + PHONES
SYMBOL_TO_ID = {symbol: index for index, symbol in enumerate(SYMBOLS)}
PHONE_SET = frozenset(PHONES)
PHONE_IDS = frozenset(SYMBOL_TO_ID[p] for p in PHONES) | {SYMBOL_TO_ID[UNK]}
VOICELESS = frozenset(("k", "kʰ", "tʃ", "tʃʰ", "ʈ", "ʈʰ", "t̪", "t̪ʰ", "p", "pʰ", "ʃ", "ʂ", "s", "q", "x", "f"))
VOWEL_SET = frozenset(VOWELS)
SHORT_VOWELS = frozenset(("ə", "ɪ", "ʊ", "æ", "ɔ"))
# ---------------------------------------------------------------- Devanagari tables
CONSONANT_CHARS = {
"क": "k", "ख": "kʰ", "ग": "ɡ", "घ": "ɡʱ", "ङ": "ŋ",
"च": "tʃ", "छ": "tʃʰ", "ज": "dʒ", "झ": "dʒʱ", "ञ": "ɲ",
"ट": "ʈ", "ठ": "ʈʰ", "ड": "ɖ", "ढ": "ɖʱ", "ण": "ɳ",
"त": "t̪", "थ": "t̪ʰ", "द": "d̪", "ध": "d̪ʱ", "न": "n",
"प": "p", "फ": "pʰ", "ब": "b", "भ": "bʱ", "म": "m",
"य": "j", "र": "ɾ", "ल": "l", "व": "ʋ", "श": "ʃ", "ष": "ʂ", "स": "s", "ह": "ɦ",
"ळ": "l", "ऱ": "ɾ", "ऴ": "l",
}
NUKTA_FORMS = {"क": "q", "ख": "x", "ग": "ɣ", "ज": "z", "फ": "f", "ड": "ɽ", "ढ": "ɽʱ", "झ": "ʒ", "न": "n", "र": "ɾ",
"य": "j", "ल": "l"}
INDEPENDENT_VOWELS = {
"अ": "ə", "आ": "aː", "इ": "ɪ", "ई": "iː", "उ": "ʊ", "ऊ": "uː", "ऋ": ("ɾ", "ɪ"), "ॠ": ("ɾ", "iː"),
"ए": "eː", "ऐ": "ɛː", "ओ": "oː", "औ": "ɔː", "ऍ": "æ", "ऑ": "ɔ", "ऎ": "eː", "ऒ": "oː", "ऌ": ("l", "ɪ"),
}
VOWEL_SIGNS = {
"ा": "aː", "ि": "ɪ", "ी": "iː", "ु": "ʊ", "ू": "uː", "ृ": ("ɾ", "ɪ"), "ॄ": ("ɾ", "iː"),
"े": "eː", "ै": "ɛː", "ो": "oː", "ौ": "ɔː", "ॅ": "æ", "ॉ": "ɔ", "ॆ": "eː", "ॊ": "oː", "ॢ": ("l", "ɪ"),
}
VIRAMA, NUKTA, ANUSVARA, CHANDRABINDU, VISARGA, AVAGRAHA = "्", "़", "ं", "ँ", "ः", "ऽ"
# Anusvara before a stop becomes the nasal of the stop's place of articulation.
HOMORGANIC = {}
for _nasal, _stops in (("ŋ", "k kʰ ɡ ɡʱ q x ɣ"), ("ɲ", "tʃ tʃʰ dʒ dʒʱ"), ("ɳ", "ʈ ʈʰ ɖ ɖʱ"),
("n", "t̪ t̪ʰ d̪ d̪ʱ"), ("m", "p pʰ b bʱ")):
for _s in _stops.split():
HOMORGANIC[_s] = _nasal
PUNCT_MAP = {"।": ".", "॥": ".", "‘": "'", "’": "'", "“": '"', "”": '"', "–": "—", "-": None, "…": "…"}
_DEVANAGARI = re.compile(r"[ऀ-ॿ]")
_LATIN = re.compile(r"[A-Za-z]")
class NothingToPronounce(ValueError):
"""The text has no Devanagari word to say: only marks, Latin script, emoji and the like."""
@dataclass(frozen=True)
class Phonemized:
symbols: tuple
ids: tuple
stress: tuple # here: 1 = nasalized vowel, 0 otherwise
unknown: tuple # characters that had no mapping
def __len__(self):
return len(self.ids)
def frontend():
return "hindi"
def set_frontend(name):
if name not in FRONTENDS:
raise ValueError(f"front end must be one of {FRONTENDS}, not {name!r}")
def frontend_identity():
return {"frontend_version": FRONTEND_VERSION, "phonemizer": "tts_hi devanagari rules", "symbols": list(SYMBOLS)}
def frontend_runtime():
return {"phonemizer": frontend_fingerprint()}
def frontend_fingerprint():
return hashlib.sha256(json.dumps(frontend_identity(), sort_keys=True, ensure_ascii=False).encode()).hexdigest()
def normalize_text(text):
text = unicodedata.normalize("NFC", text)
text = "".join(c for c in text if unicodedata.category(c) != "Cf") # ZWJ / ZWNJ
out = []
for c in text:
if c in PUNCT_MAP:
if PUNCT_MAP[c] is not None:
out.append(PUNCT_MAP[c])
else:
out.append(" ")
else:
out.append(c)
text = "".join(out)
text = re.sub(r"\.{3,}", "…", text)
return re.sub(r"\s+", " ", text).strip()
# ---------------------------------------------------------------- one word
def _aksharas(word, unknown):
"""Units: ('C', phone, vowel_or_None, nasal, explicit) for a consonant (+vowel sign), ('V', vowel, nasal) for an
independent vowel. vowel None = inherent schwa (subject to deletion) unless the consonant carries a virama."""
units, i, n = [], 0, len(word)
while i < n:
c = word[i]
if c in CONSONANT_CHARS:
phone = CONSONANT_CHARS[c]
j = i + 1
if j < n and word[j] == NUKTA:
phone = NUKTA_FORMS.get(c, phone)
j += 1
vowel, explicit, virama = None, False, False
if j < n and word[j] == VIRAMA:
virama = True
j += 1
elif j < n and word[j] in VOWEL_SIGNS:
vowel, explicit = VOWEL_SIGNS[word[j]], True
j += 1
nasal = 0
while j < n and word[j] in (ANUSVARA, CHANDRABINDU, VISARGA):
if word[j] == VISARGA:
units.append(("C", phone, vowel, nasal, explicit, virama))
phone, vowel, explicit, virama, nasal = "ɦ", None, False, True, 0
else:
nasal = max(nasal, 2 if word[j] == CHANDRABINDU else 1)
j += 1
units.append(("C", phone, vowel, nasal, explicit, virama))
i = j
elif c in INDEPENDENT_VOWELS:
vowel = INDEPENDENT_VOWELS[c]
j, nasal = i + 1, 0
while j < n and word[j] in (ANUSVARA, CHANDRABINDU, VISARGA):
if word[j] == VISARGA:
units.append(("V", vowel, nasal))
vowel, nasal = None, False
units.append(("C", "ɦ", None, False, False, True))
else:
nasal = max(nasal, 2 if word[j] == CHANDRABINDU else 1)
j += 1
if vowel is not None:
units.append(("V", vowel, nasal))
i = j
elif c in (AVAGRAHA,):
i += 1
elif c in VOWEL_SIGNS or c in (ANUSVARA, CHANDRABINDU, NUKTA, VIRAMA, VISARGA):
# a sign without a base (typo): attach the vowel / nasal to the previous unit if possible
if c in VOWEL_SIGNS:
units.append(("V", VOWEL_SIGNS[c], False))
elif c in (ANUSVARA, CHANDRABINDU) and units:
u = units[-1]
kind = 2 if c == CHANDRABINDU else 1
units[-1] = u[:3] + (kind,) + u[4:] if u[0] == "C" else (u[0], u[1], kind)
i += 1
else:
unknown.append(c)
i += 1
return units
def _word_phones(word, unknown):
"""List of (phone, nasal) for one Devanagari word."""
units = _aksharas(word, unknown)
if not units:
return []
# Inherent schwas: positions of consonant units with vowel None and no virama.
schwa = [u[0] == "C" and u[2] is None and not u[5] for u in units]
keep = list(schwa)
syllables = sum(1 for k, u in enumerate(units) if u[0] == "V" or (u[0] == "C" and (u[2] is not None or schwa[k])))
last_c = max((k for k, u in enumerate(units) if u[0] == "C"), default=-1)
if last_c == len(units) - 1 and schwa[last_c] and syllables > 1 and not units[last_c][3]:
keep[last_c] = False # word-final schwa
def has_vowel(k):
u = units[k]
return u[0] == "V" or (u[0] == "C" and (u[2] is not None or keep[k]))
def longest_cluster():
run = best = 0
for m in range(len(units)):
run = 0 if has_vowel(m) else run + 1
best = max(best, run)
return best
# Medial schwa: V C ə C V -> V C C V, right to left; never in the first akshara, never a 3-consonant cluster,
# never on a nasalized schwa.
for k in range(len(units) - 2, 0, -1):
if not keep[k] or units[k][3]:
continue
if not has_vowel(k - 1) or units[k + 1][0] != "C" or not has_vowel(k + 1):
continue
keep[k] = False
if longest_cluster() >= 3:
keep[k] = True
phones = []
for k, u in enumerate(units):
if u[0] == "V":
v = u[1]
for p in (v if isinstance(v, tuple) else (v,)):
phones.append([p, 0])
phones[-1][1] = u[2]
continue
_, cons, vowel, nasal, _, _ = u
phones.append([cons, False])
if vowel is not None:
for p in (vowel if isinstance(vowel, tuple) else (vowel,)):
phones.append([p, False])
elif keep[k]:
phones.append(["ə", False])
if nasal:
phones[-1][1] = nasal # mark; resolved below
# Nasal marks: chandrabindu always nasalizes the vowel; anusvara after a SHORT vowel before a stop becomes the
# stop's nasal consonant (हिंदी ɦɪnd̪iː), otherwise it nasalizes the vowel (जाएंगे dʒaːeː̃ɡeː, मैं mɛː̃).
out = []
for idx, (p, nasal) in enumerate(phones):
nxt = phones[idx + 1][0] if idx + 1 < len(phones) else None
if nasal and p in VOWEL_SET:
if nasal == 1 and p in SHORT_VOWELS and nxt in HOMORGANIC:
out.append((p, 0))
out.append((HOMORGANIC[nxt], 0))
else:
out.append((p, 1))
elif nasal: # nasal on a bare consonant (no vowel kept)
out.append((p, 0))
if nxt in HOMORGANIC:
out.append((HOMORGANIC[nxt], 0))
else:
out.append((p, 0))
# ज्ञ is said /ɡj/ in Hindi
fixed, i = [], 0
while i < len(out):
if i + 1 < len(out) and out[i][0] == "dʒ" and out[i + 1][0] == "ɲ":
fixed += [("ɡ", 0), ("j", 0)]
i += 2
else:
fixed.append(out[i])
i += 1
return fixed
def chunks(text):
"""Yield (is_punctuation, piece) over the normalized text."""
for token in re.findall(r"[ऀ-ॣ०-ॿ\w]+|[^\s\w]", text):
if token in PUNCTUATION:
yield True, token
elif _DEVANAGARI.search(token):
yield False, token
# Latin words, digits left over, emoji: not spoken by this model
def tokenize(pieces):
"""pieces: (is_punctuation, value) where a text value is a list of words, each a list of (phone, nasal)."""
symbols, stress, unknown = [BOS], [0], []
for punctuation, value in pieces:
if punctuation:
if symbols[-1] != value:
symbols.append(value)
stress.append(0)
continue
for word in value:
if not word:
continue
if symbols[-1] in PHONE_SET or symbols[-1] == UNK:
symbols.append(SP)
stress.append(0)
for phone, nasal in word:
symbols.append(phone if phone in PHONE_SET else UNK)
stress.append(int(nasal))
symbols.append(EOS)
stress.append(0)
if not any(s in PHONE_SET or s == UNK for s in symbols):
raise NothingToPronounce("the text has nothing to pronounce")
return Phonemized(tuple(symbols), tuple(SYMBOL_TO_ID[s] for s in symbols), tuple(stress), tuple(unknown))
def phonemize(text):
if not isinstance(text, str) or not text.strip():
raise ValueError("text must not be empty")
unknown, pieces = [], []
for punctuation, piece in chunks(normalize_text(text)):
pieces.append((True, piece) if punctuation else (False, [_word_phones(piece, unknown)]))
result = tokenize(pieces)
return Phonemized(result.symbols, result.ids, result.stress, tuple(unknown))
def to_ipa(text):
"""Readable form for checks: words joined by spaces, nasalized vowels with a tilde."""
p = phonemize(text)
out = []
for s, n in zip(p.symbols, p.stress):
if s == SP:
out.append(" ")
elif s in PHONE_SET:
out.append(s + ("̃" if n else ""))
elif s in PUNCTUATION:
out.append(s)
return "".join(out)
def is_phone(token_ids):
return [int(i) in PHONE_IDS for i in token_ids]