File size: 14,703 Bytes
ab1b068
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
"""Hindi front end: Devanagari text -> phoneme tokens with a nasal feature.



Replaces tiny-voca's English front end (eSpeak / dizione). The same code runs during annotation, training-set preparation

and inference, so tokens can never drift between them.



Input is Devanagari after our normalizer (small_tts.normalize_hi with english=False): numbers, dates and money are

already words. Latin-script words are skipped (this model reads Devanagari only).



* A token is one Hindi phoneme (IPA). Aspiration and breathy voice are part of the phoneme (kʰ, bʱ), as are vowel length

  (aː, iː). Phones the forced aligner has no label for map to the nearest label for alignment only (align.py).

* Nasalization (anusvara before a vowel/sonorant, chandrabindu) is the parallel ``stress`` feature: 1 = the vowel is

  nasalized. It is not a token, so it never needs audio frames of its own. (tiny-voca's field name is kept.)

* Inherent vowel /ə/ and schwa deletion: word-final schwa is dropped (except in one-akshara words), and a medial schwa

  is dropped in the context V C _ C V when that does not make a three-consonant cluster (right to left). Words that

  break the rule (compounds) are a known source of error.

* Punctuation, word boundaries and the utterance edges are explicit tokens that may last zero frames (pauses).

"""

from dataclasses import dataclass
import hashlib
import json
import os
import re
import unicodedata

FRONTEND_VERSION = 1
FRONTENDS = ("hindi",)
FRONTEND_VARIABLE = "TINY_VOCA_FRONTEND"

PAD, BOS, EOS, SP, UNK = "<pad>", "<bos>", "<eos>", "<sp>", "<unk>"
SPECIALS = (PAD, BOS, EOS, SP, UNK)
PUNCTUATION = (",", ".", "?", "!", ";", ":", '"', "'", "(", ")", "—", "…")
CONSONANTS = ("k", "kʰ", "ɡ", "ɡʱ", "ŋ", "tʃ", "tʃʰ", "dʒ", "dʒʱ", "ɲ", "ʈ", "ʈʰ", "ɖ", "ɖʱ", "ɳ",
              "t̪", "t̪ʰ", "d̪", "d̪ʱ", "n", "p", "pʰ", "b", "bʱ", "m", "j", "ɾ", "l", "ʋ", "ʃ", "ʂ", "s", "ɦ",
              "ɽ", "ɽʱ", "q", "x", "ɣ", "z", "f", "ʒ")
VOWELS = ("ə", "aː", "ɪ", "iː", "ʊ", "uː", "eː", "ɛː", "oː", "ɔː", "æ", "ɔ")
PHONES = CONSONANTS + VOWELS
# Append-only: changing the order or removing a symbol invalidates checkpoints.
SYMBOLS = SPECIALS + PUNCTUATION + PHONES
SYMBOL_TO_ID = {symbol: index for index, symbol in enumerate(SYMBOLS)}
PHONE_SET = frozenset(PHONES)
PHONE_IDS = frozenset(SYMBOL_TO_ID[p] for p in PHONES) | {SYMBOL_TO_ID[UNK]}
VOICELESS = frozenset(("k", "kʰ", "tʃ", "tʃʰ", "ʈ", "ʈʰ", "t̪", "t̪ʰ", "p", "pʰ", "ʃ", "ʂ", "s", "q", "x", "f"))
VOWEL_SET = frozenset(VOWELS)
SHORT_VOWELS = frozenset(("ə", "ɪ", "ʊ", "æ", "ɔ"))

# ---------------------------------------------------------------- Devanagari tables
CONSONANT_CHARS = {
    "क": "k", "ख": "kʰ", "ग": "ɡ", "घ": "ɡʱ", "ङ": "ŋ",
    "च": "tʃ", "छ": "tʃʰ", "ज": "dʒ", "झ": "dʒʱ", "ञ": "ɲ",
    "ट": "ʈ", "ठ": "ʈʰ", "ड": "ɖ", "ढ": "ɖʱ", "ण": "ɳ",
    "त": "t̪", "थ": "t̪ʰ", "द": "d̪", "ध": "d̪ʱ", "न": "n",
    "प": "p", "फ": "pʰ", "ब": "b", "भ": "bʱ", "म": "m",
    "य": "j", "र": "ɾ", "ल": "l", "व": "ʋ", "श": "ʃ", "ष": "ʂ", "स": "s", "ह": "ɦ",
    "ळ": "l", "ऱ": "ɾ", "ऴ": "l",
}
NUKTA_FORMS = {"क": "q", "ख": "x", "ग": "ɣ", "ज": "z", "फ": "f", "ड": "ɽ", "ढ": "ɽʱ", "झ": "ʒ", "न": "n", "र": "ɾ",
               "य": "j", "ल": "l"}
INDEPENDENT_VOWELS = {
    "अ": "ə", "आ": "aː", "इ": "ɪ", "ई": "iː", "उ": "ʊ", "ऊ": "uː", "ऋ": ("ɾ", "ɪ"), "ॠ": ("ɾ", "iː"),
    "ए": "eː", "ऐ": "ɛː", "ओ": "oː", "औ": "ɔː", "ऍ": "æ", "ऑ": "ɔ", "ऎ": "eː", "ऒ": "oː", "ऌ": ("l", "ɪ"),
}
VOWEL_SIGNS = {
    "ा": "aː", "ि": "ɪ", "ी": "iː", "ु": "ʊ", "ू": "uː", "ृ": ("ɾ", "ɪ"), "ॄ": ("ɾ", "iː"),
    "े": "eː", "ै": "ɛː", "ो": "oː", "ौ": "ɔː", "ॅ": "æ", "ॉ": "ɔ", "ॆ": "eː", "ॊ": "oː", "ॢ": ("l", "ɪ"),
}
VIRAMA, NUKTA, ANUSVARA, CHANDRABINDU, VISARGA, AVAGRAHA = "्", "़", "ं", "ँ", "ः", "ऽ"
# Anusvara before a stop becomes the nasal of the stop's place of articulation.
HOMORGANIC = {}
for _nasal, _stops in (("ŋ", "k kʰ ɡ ɡʱ q x ɣ"), ("ɲ", "tʃ tʃʰ dʒ dʒʱ"), ("ɳ", "ʈ ʈʰ ɖ ɖʱ"),
                       ("n", "t̪ t̪ʰ d̪ d̪ʱ"), ("m", "p pʰ b bʱ")):
    for _s in _stops.split():
        HOMORGANIC[_s] = _nasal
PUNCT_MAP = {"।": ".", "॥": ".", "‘": "'", "’": "'", "“": '"', "”": '"', "–": "—", "-": None, "…": "…"}
_DEVANAGARI = re.compile(r"[ऀ-ॿ]")
_LATIN = re.compile(r"[A-Za-z]")


class NothingToPronounce(ValueError):
    """The text has no Devanagari word to say: only marks, Latin script, emoji and the like."""


@dataclass(frozen=True)
class Phonemized:
    symbols: tuple
    ids: tuple
    stress: tuple       # here: 1 = nasalized vowel, 0 otherwise
    unknown: tuple      # characters that had no mapping

    def __len__(self):
        return len(self.ids)


def frontend():
    return "hindi"


def set_frontend(name):
    if name not in FRONTENDS:
        raise ValueError(f"front end must be one of {FRONTENDS}, not {name!r}")


def frontend_identity():
    return {"frontend_version": FRONTEND_VERSION, "phonemizer": "tts_hi devanagari rules", "symbols": list(SYMBOLS)}


def frontend_runtime():
    return {"phonemizer": frontend_fingerprint()}


def frontend_fingerprint():
    return hashlib.sha256(json.dumps(frontend_identity(), sort_keys=True, ensure_ascii=False).encode()).hexdigest()


def normalize_text(text):
    text = unicodedata.normalize("NFC", text)
    text = "".join(c for c in text if unicodedata.category(c) != "Cf")      # ZWJ / ZWNJ
    out = []
    for c in text:
        if c in PUNCT_MAP:
            if PUNCT_MAP[c] is not None:
                out.append(PUNCT_MAP[c])
            else:
                out.append(" ")
        else:
            out.append(c)
    text = "".join(out)
    text = re.sub(r"\.{3,}", "…", text)
    return re.sub(r"\s+", " ", text).strip()


# ---------------------------------------------------------------- one word
def _aksharas(word, unknown):
    """Units: ('C', phone, vowel_or_None, nasal, explicit) for a consonant (+vowel sign), ('V', vowel, nasal) for an

    independent vowel. vowel None = inherent schwa (subject to deletion) unless the consonant carries a virama."""
    units, i, n = [], 0, len(word)
    while i < n:
        c = word[i]
        if c in CONSONANT_CHARS:
            phone = CONSONANT_CHARS[c]
            j = i + 1
            if j < n and word[j] == NUKTA:
                phone = NUKTA_FORMS.get(c, phone)
                j += 1
            vowel, explicit, virama = None, False, False
            if j < n and word[j] == VIRAMA:
                virama = True
                j += 1
            elif j < n and word[j] in VOWEL_SIGNS:
                vowel, explicit = VOWEL_SIGNS[word[j]], True
                j += 1
            nasal = 0
            while j < n and word[j] in (ANUSVARA, CHANDRABINDU, VISARGA):
                if word[j] == VISARGA:
                    units.append(("C", phone, vowel, nasal, explicit, virama))
                    phone, vowel, explicit, virama, nasal = "ɦ", None, False, True, 0
                else:
                    nasal = max(nasal, 2 if word[j] == CHANDRABINDU else 1)
                j += 1
            units.append(("C", phone, vowel, nasal, explicit, virama))
            i = j
        elif c in INDEPENDENT_VOWELS:
            vowel = INDEPENDENT_VOWELS[c]
            j, nasal = i + 1, 0
            while j < n and word[j] in (ANUSVARA, CHANDRABINDU, VISARGA):
                if word[j] == VISARGA:
                    units.append(("V", vowel, nasal))
                    vowel, nasal = None, False
                    units.append(("C", "ɦ", None, False, False, True))
                else:
                    nasal = max(nasal, 2 if word[j] == CHANDRABINDU else 1)
                j += 1
            if vowel is not None:
                units.append(("V", vowel, nasal))
            i = j
        elif c in (AVAGRAHA,):
            i += 1
        elif c in VOWEL_SIGNS or c in (ANUSVARA, CHANDRABINDU, NUKTA, VIRAMA, VISARGA):
            # a sign without a base (typo): attach the vowel / nasal to the previous unit if possible
            if c in VOWEL_SIGNS:
                units.append(("V", VOWEL_SIGNS[c], False))
            elif c in (ANUSVARA, CHANDRABINDU) and units:
                u = units[-1]
                kind = 2 if c == CHANDRABINDU else 1
                units[-1] = u[:3] + (kind,) + u[4:] if u[0] == "C" else (u[0], u[1], kind)
            i += 1
        else:
            unknown.append(c)
            i += 1
    return units


def _word_phones(word, unknown):
    """List of (phone, nasal) for one Devanagari word."""
    units = _aksharas(word, unknown)
    if not units:
        return []
    # Inherent schwas: positions of consonant units with vowel None and no virama.
    schwa = [u[0] == "C" and u[2] is None and not u[5] for u in units]
    keep = list(schwa)
    syllables = sum(1 for k, u in enumerate(units) if u[0] == "V" or (u[0] == "C" and (u[2] is not None or schwa[k])))
    last_c = max((k for k, u in enumerate(units) if u[0] == "C"), default=-1)
    if last_c == len(units) - 1 and schwa[last_c] and syllables > 1 and not units[last_c][3]:
        keep[last_c] = False                                     # word-final schwa
    def has_vowel(k):
        u = units[k]
        return u[0] == "V" or (u[0] == "C" and (u[2] is not None or keep[k]))

    def longest_cluster():
        run = best = 0
        for m in range(len(units)):
            run = 0 if has_vowel(m) else run + 1
            best = max(best, run)
        return best

    # Medial schwa: V C ə C V -> V C C V, right to left; never in the first akshara, never a 3-consonant cluster,
    # never on a nasalized schwa.
    for k in range(len(units) - 2, 0, -1):
        if not keep[k] or units[k][3]:
            continue
        if not has_vowel(k - 1) or units[k + 1][0] != "C" or not has_vowel(k + 1):
            continue
        keep[k] = False
        if longest_cluster() >= 3:
            keep[k] = True
    phones = []
    for k, u in enumerate(units):
        if u[0] == "V":
            v = u[1]
            for p in (v if isinstance(v, tuple) else (v,)):
                phones.append([p, 0])
            phones[-1][1] = u[2]
            continue
        _, cons, vowel, nasal, _, _ = u
        phones.append([cons, False])
        if vowel is not None:
            for p in (vowel if isinstance(vowel, tuple) else (vowel,)):
                phones.append([p, False])
        elif keep[k]:
            phones.append(["ə", False])
        if nasal:
            phones[-1][1] = nasal                                 # mark; resolved below
    # Nasal marks: chandrabindu always nasalizes the vowel; anusvara after a SHORT vowel before a stop becomes the
    # stop's nasal consonant (हिंदी ɦɪnd̪iː), otherwise it nasalizes the vowel (जाएंगे dʒaːeː̃ɡeː, मैं mɛː̃).
    out = []
    for idx, (p, nasal) in enumerate(phones):
        nxt = phones[idx + 1][0] if idx + 1 < len(phones) else None
        if nasal and p in VOWEL_SET:
            if nasal == 1 and p in SHORT_VOWELS and nxt in HOMORGANIC:
                out.append((p, 0))
                out.append((HOMORGANIC[nxt], 0))
            else:
                out.append((p, 1))
        elif nasal:                                               # nasal on a bare consonant (no vowel kept)
            out.append((p, 0))
            if nxt in HOMORGANIC:
                out.append((HOMORGANIC[nxt], 0))
        else:
            out.append((p, 0))
    # ज्ञ is said /ɡj/ in Hindi
    fixed, i = [], 0
    while i < len(out):
        if i + 1 < len(out) and out[i][0] == "dʒ" and out[i + 1][0] == "ɲ":
            fixed += [("ɡ", 0), ("j", 0)]
            i += 2
        else:
            fixed.append(out[i])
            i += 1
    return fixed


def chunks(text):
    """Yield (is_punctuation, piece) over the normalized text."""
    for token in re.findall(r"[ऀ-ॣ०-ॿ\w]+|[^\s\w]", text):
        if token in PUNCTUATION:
            yield True, token
        elif _DEVANAGARI.search(token):
            yield False, token
        # Latin words, digits left over, emoji: not spoken by this model


def tokenize(pieces):
    """pieces: (is_punctuation, value) where a text value is a list of words, each a list of (phone, nasal)."""
    symbols, stress, unknown = [BOS], [0], []
    for punctuation, value in pieces:
        if punctuation:
            if symbols[-1] != value:
                symbols.append(value)
                stress.append(0)
            continue
        for word in value:
            if not word:
                continue
            if symbols[-1] in PHONE_SET or symbols[-1] == UNK:
                symbols.append(SP)
                stress.append(0)
            for phone, nasal in word:
                symbols.append(phone if phone in PHONE_SET else UNK)
                stress.append(int(nasal))
    symbols.append(EOS)
    stress.append(0)
    if not any(s in PHONE_SET or s == UNK for s in symbols):
        raise NothingToPronounce("the text has nothing to pronounce")
    return Phonemized(tuple(symbols), tuple(SYMBOL_TO_ID[s] for s in symbols), tuple(stress), tuple(unknown))


def phonemize(text):
    if not isinstance(text, str) or not text.strip():
        raise ValueError("text must not be empty")
    unknown, pieces = [], []
    for punctuation, piece in chunks(normalize_text(text)):
        pieces.append((True, piece) if punctuation else (False, [_word_phones(piece, unknown)]))
    result = tokenize(pieces)
    return Phonemized(result.symbols, result.ids, result.stress, tuple(unknown))


def to_ipa(text):
    """Readable form for checks: words joined by spaces, nasalized vowels with a tilde."""
    p = phonemize(text)
    out = []
    for s, n in zip(p.symbols, p.stress):
        if s == SP:
            out.append(" ")
        elif s in PHONE_SET:
            out.append(s + ("̃" if n else ""))
        elif s in PUNCTUATION:
            out.append(s)
    return "".join(out)


def is_phone(token_ids):
    return [int(i) in PHONE_IDS for i in token_ids]