File size: 23,825 Bytes
ab1b068
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
#include "g2p.h"

#include <algorithm>
#include <map>
#include <set>
#include <unordered_map>

namespace {
using U = std::u32string;

// ------------------------------------------------------------------ UTF-8
U decode(const std::string& s) {
    U out;
    for (size_t i = 0; i < s.size();) {
        const unsigned char c = s[i];
        char32_t cp; int n;
        if (c < 0x80) { cp = c; n = 1; }
        else if ((c >> 5) == 6) { cp = c & 0x1F; n = 2; }
        else if ((c >> 4) == 14) { cp = c & 0x0F; n = 3; }
        else if ((c >> 3) == 30) { cp = c & 0x07; n = 4; }
        else { ++i; out.push_back(0xFFFD); continue; }
        if (i + n > s.size()) { out.push_back(0xFFFD); break; }
        bool ok = true;
        for (int k = 1; k < n; ++k) {
            const unsigned char d = s[i + k];
            if ((d >> 6) != 2) { ok = false; break; }
            cp = (cp << 6) | (d & 0x3F);
        }
        if (!ok) { out.push_back(0xFFFD); ++i; continue; }
        out.push_back(cp); i += n;
    }
    return out;
}
std::string encode(const U& u) {
    std::string out;
    for (char32_t cp : u) {
        if (cp < 0x80) out += char(cp);
        else if (cp < 0x800) { out += char(0xC0 | (cp >> 6)); out += char(0x80 | (cp & 0x3F)); }
        else if (cp < 0x10000) { out += char(0xE0 | (cp >> 12)); out += char(0x80 | ((cp >> 6) & 0x3F)); out += char(0x80 | (cp & 0x3F)); }
        else { out += char(0xF0 | (cp >> 18)); out += char(0x80 | ((cp >> 12) & 0x3F)); out += char(0x80 | ((cp >> 6) & 0x3F)); out += char(0x80 | (cp & 0x3F)); }
    }
    return out;
}

// ------------------------------------------------------------------ character classes (Python semantics)
bool is_space(char32_t c) {        // Python str.isspace / re \s (Unicode)
    return c == ' ' || (c >= 0x09 && c <= 0x0D) || (c >= 0x1C && c <= 0x1F) || c == 0x85 || c == 0xA0 || c == 0x1680 ||
           (c >= 0x2000 && c <= 0x200A) || c == 0x2028 || c == 0x2029 || c == 0x202F || c == 0x205F || c == 0x3000;
}
bool is_format(char32_t c) {       // unicodedata.category == "Cf" (the common ones)
    return c == 0xAD || (c >= 0x600 && c <= 0x605) || c == 0x61C || c == 0x6DD || c == 0x70F || c == 0x180E ||
           (c >= 0x200B && c <= 0x200F) || (c >= 0x202A && c <= 0x202E) || (c >= 0x2060 && c <= 0x2064) ||
           (c >= 0x2066 && c <= 0x206F) || c == 0xFEFF || (c >= 0xFFF9 && c <= 0xFFFB);
}
bool is_devanagari(char32_t c) { return c >= 0x900 && c <= 0x97F; }
bool is_devanagari_mark(char32_t c) {   // Devanagari combining marks (Mn/Mc): not \w, not alnum
    return (c >= 0x900 && c <= 0x903) || (c >= 0x93A && c <= 0x93C) || (c >= 0x93E && c <= 0x94F) ||
           (c >= 0x951 && c <= 0x957) || c == 0x962 || c == 0x963;
}
bool is_mark(char32_t c) {         // combining marks outside Devanagari that Python's \w rejects
    return (c >= 0x300 && c <= 0x36F) || (c >= 0x1AB0 && c <= 0x1AFF) || (c >= 0x1DC0 && c <= 0x1DFF) ||
           (c >= 0x20D0 && c <= 0x20FF) || (c >= 0xFE00 && c <= 0xFE0F) || (c >= 0xFE20 && c <= 0xFE2F);
}
bool is_symbol_or_punct(char32_t c) {   // non-word characters above ASCII
    return (c >= 0xA1 && c <= 0xBF && c != 0xAA && c != 0xB2 && c != 0xB3 && c != 0xB5 && c != 0xB9 && c != 0xBA &&
            c != 0xBC && c != 0xBD && c != 0xBE) || c == 0xD7 || c == 0xF7 || (c >= 0x2010 && c <= 0x2027) ||
           (c >= 0x2030 && c <= 0x205E) || (c >= 0x20A0 && c <= 0x20CF) || (c >= 0x2190 && c <= 0x2BFF) || (c >= 0x2E00 && c <= 0x2E7F) || (c >= 0x3000 && c <= 0x303F) ||
           (c >= 0xFF01 && c <= 0xFF0F) || (c >= 0x1F000 && c <= 0x1FAFF) || c == 0x964 || c == 0x965 || c == 0x970 ||
           c == 0xFFFD;
}
bool is_word(char32_t c) {         // Python re \w (Unicode): alphanumerics and underscore
    if (c < 0x80) return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_';
    if (is_space(c) || is_format(c) || is_mark(c) || is_devanagari_mark(c) || is_symbol_or_punct(c)) return false;
    if (c == 0xA0) return false;
    return true;
}
bool is_alnum(char32_t c) { return c != '_' && is_word(c); }   // str.isalnum, close enough for our texts

// ------------------------------------------------------------------ text.normalize_text
// NFC for Devanagari: U+0958..U+095F are composition exclusions (NFC splits them into base + nukta); NA/RA/LLA + nukta
// compose to U+0929/U+0931/U+0934. Nukta (ccc 7) sorts before virama (ccc 9).
U nfc_devanagari(const U& in) {
    U d;
    static const char32_t base958[8] = {0x915, 0x916, 0x917, 0x91C, 0x921, 0x922, 0x92B, 0x92F};
    for (char32_t c : in) {
        if (c >= 0x958 && c <= 0x95F) { d.push_back(base958[c - 0x958]); d.push_back(0x93C); }
        else if (c == 0x929) { d.push_back(0x928); d.push_back(0x93C); }
        else if (c == 0x931) { d.push_back(0x930); d.push_back(0x93C); }
        else if (c == 0x934) { d.push_back(0x933); d.push_back(0x93C); }
        else d.push_back(c);
    }
    for (size_t i = 1; i < d.size(); ++i)                       // canonical order: nukta before virama
        if (d[i] == 0x93C && d[i - 1] == 0x94D) std::swap(d[i], d[i - 1]);
    U out;
    for (size_t i = 0; i < d.size(); ++i) {
        if (i + 1 < d.size() && d[i + 1] == 0x93C && (d[i] == 0x928 || d[i] == 0x930 || d[i] == 0x933)) {
            out.push_back(d[i] == 0x928 ? 0x929 : d[i] == 0x930 ? 0x931 : 0x934);
            ++i;
        } else out.push_back(d[i]);
    }
    return out;
}

U normalize_u(const U& text_in) {
    U text = nfc_devanagari(text_in), t;
    for (char32_t c : text) if (!is_format(c)) t.push_back(c);
    U out;
    for (char32_t c : t) {
        switch (c) {
            case 0x964: case 0x965: out.push_back('.'); break;          // । ॥
            case 0x2018: case 0x2019: out.push_back('\''); break;        // ‘ ’
            case 0x201C: case 0x201D: out.push_back('"'); break;         // “ ”
            case 0x2013: out.push_back(0x2014); break;                   // – -> —
            case '-': out.push_back(' '); break;
            default: out.push_back(c);
        }
    }
    U dots;                                                              // \.{3,} -> …
    for (size_t i = 0; i < out.size();) {
        if (out[i] == '.') {
            size_t j = i; while (j < out.size() && out[j] == '.') ++j;
            if (j - i >= 3) dots.push_back(0x2026); else dots.append(out, i, j - i);
            i = j;
        } else dots.push_back(out[i++]);
    }
    U res;                                                               // \s+ -> " ", strip
    bool space = false;
    for (char32_t c : dots) {
        if (is_space(c)) { space = true; continue; }
        if (space && !res.empty()) res.push_back(' ');
        space = false;
        res.push_back(c);
    }
    return res;
}

// ------------------------------------------------------------------ tables
const std::map<char32_t, const char*> CONSONANT_CHARS = {
    {U'क', "k"}, {U'ख', "kʰ"}, {U'ग', "ɡ"}, {U'घ', "ɡʱ"}, {U'ङ', "ŋ"},
    {U'च', "tʃ"}, {U'छ', "tʃʰ"}, {U'ज', "dʒ"}, {U'झ', "dʒʱ"}, {U'ञ', "ɲ"},
    {U'ट', "ʈ"}, {U'ठ', "ʈʰ"}, {U'ड', "ɖ"}, {U'ढ', "ɖʱ"}, {U'ण', "ɳ"},
    {U'त', "t̪"}, {U'थ', "t̪ʰ"}, {U'द', "d̪"}, {U'ध', "d̪ʱ"}, {U'न', "n"},
    {U'प', "p"}, {U'फ', "pʰ"}, {U'ब', "b"}, {U'भ', "bʱ"}, {U'म', "m"},
    {U'य', "j"}, {U'र', "ɾ"}, {U'ल', "l"}, {U'व', "ʋ"}, {U'श', "ʃ"}, {U'ष', "ʂ"}, {U'स', "s"}, {U'ह', "ɦ"},
    {U'ळ', "l"}, {U'ऱ', "ɾ"}, {U'ऴ', "l"},
};
const std::map<char32_t, const char*> NUKTA_FORMS = {
    {U'क', "q"}, {U'ख', "x"}, {U'ग', "ɣ"}, {U'ज', "z"}, {U'फ', "f"}, {U'ड', "ɽ"}, {U'ढ', "ɽʱ"}, {U'झ', "ʒ"},
    {U'न', "n"}, {U'र', "ɾ"}, {U'य', "j"}, {U'ल', "l"},
};
using Vow = std::vector<std::string>;          // a vowel value: one phone or a pair (ऋ -> ɾ ɪ)
const std::map<char32_t, Vow> INDEPENDENT_VOWELS = {
    {U'अ', {"ə"}}, {U'आ', {"aː"}}, {U'इ', {"ɪ"}}, {U'ई', {"iː"}}, {U'उ', {"ʊ"}}, {U'ऊ', {"uː"}}, {U'ऋ', {"ɾ", "ɪ"}},
    {U'ॠ', {"ɾ", "iː"}}, {U'ए', {"eː"}}, {U'ऐ', {"ɛː"}}, {U'ओ', {"oː"}}, {U'औ', {"ɔː"}}, {U'ऍ', {"æ"}}, {U'ऑ', {"ɔ"}},
    {U'ऎ', {"eː"}}, {U'ऒ', {"oː"}}, {U'ऌ', {"l", "ɪ"}},
};
const std::map<char32_t, Vow> VOWEL_SIGNS = {
    {U'ा', {"aː"}}, {U'ि', {"ɪ"}}, {U'ी', {"iː"}}, {U'ु', {"ʊ"}}, {U'ू', {"uː"}}, {U'ृ', {"ɾ", "ɪ"}}, {U'ॄ', {"ɾ", "iː"}},
    {U'े', {"eː"}}, {U'ै', {"ɛː"}}, {U'ो', {"oː"}}, {U'ौ', {"ɔː"}}, {U'ॅ', {"æ"}}, {U'ॉ', {"ɔ"}}, {U'ॆ', {"eː"}},
    {U'ॊ', {"oː"}}, {U'ॢ', {"l", "ɪ"}},
};
const char32_t VIRAMA = U'्', NUKTA = U'़', ANUSVARA = U'ं', CHANDRABINDU = U'ँ', VISARGA = U'ः', AVAGRAHA = U'ऽ';
const std::set<std::string> VOWEL_SET = {"ə", "aː", "ɪ", "iː", "ʊ", "uː", "eː", "ɛː", "oː", "ɔː", "æ", "ɔ"};
const std::set<std::string> SHORT_VOWELS = {"ə", "ɪ", "ʊ", "æ", "ɔ"};
const std::set<std::string> PUNCTUATION = {",", ".", "?", "!", ";", ":", "\"", "'", "(", ")", "—", "…"};
const std::unordered_map<std::string, std::string> HOMORGANIC = [] {
    std::unordered_map<std::string, std::string> m;
    const std::vector<std::pair<std::string, std::vector<std::string>>> groups = {
        {"ŋ", {"k", "kʰ", "ɡ", "ɡʱ", "q", "x", "ɣ"}}, {"ɲ", {"tʃ", "tʃʰ", "dʒ", "dʒʱ"}}, {"ɳ", {"ʈ", "ʈʰ", "ɖ", "ɖʱ"}},
        {"n", {"t̪", "t̪ʰ", "d̪", "d̪ʱ"}}, {"m", {"p", "pʰ", "b", "bʱ"}}};
    for (auto& [nasal, stops] : groups) for (auto& s : stops) m[s] = nasal;
    return m;
}();

// ------------------------------------------------------------------ text._aksharas
struct Unit {
    bool consonant;     // 'C' or 'V'
    std::string phone;  // consonant phone (C)
    Vow vowel;          // C: vowel sign (empty = none / inherent schwa); V: the vowel (empty = None)
    int nasal;          // 0, 1 anusvara, 2 chandrabindu
    bool explicit_;
    bool virama;
};

std::vector<Unit> aksharas(const U& w) {
    std::vector<Unit> units;
    size_t i = 0;
    const size_t n = w.size();
    auto in = [](const auto& m, char32_t c) { return m.find(c) != m.end(); };
    while (i < n) {
        const char32_t c = w[i];
        if (in(CONSONANT_CHARS, c)) {
            std::string phone = CONSONANT_CHARS.at(c);
            size_t j = i + 1;
            if (j < n && w[j] == NUKTA) {
                auto it = NUKTA_FORMS.find(c);
                if (it != NUKTA_FORMS.end()) phone = it->second;
                ++j;
            }
            Vow vowel; bool explicit_ = false, virama = false;
            if (j < n && w[j] == VIRAMA) { virama = true; ++j; }
            else if (j < n && in(VOWEL_SIGNS, w[j])) { vowel = VOWEL_SIGNS.at(w[j]); explicit_ = true; ++j; }
            int nasal = 0;
            while (j < n && (w[j] == ANUSVARA || w[j] == CHANDRABINDU || w[j] == VISARGA)) {
                if (w[j] == VISARGA) {
                    units.push_back({true, phone, vowel, nasal, explicit_, virama});
                    phone = "ɦ"; vowel.clear(); explicit_ = false; virama = true; nasal = 0;
                } else nasal = std::max(nasal, w[j] == CHANDRABINDU ? 2 : 1);
                ++j;
            }
            units.push_back({true, phone, vowel, nasal, explicit_, virama});
            i = j;
        } else if (in(INDEPENDENT_VOWELS, c)) {
            Vow vowel = INDEPENDENT_VOWELS.at(c);
            size_t j = i + 1; int nasal = 0;
            while (j < n && (w[j] == ANUSVARA || w[j] == CHANDRABINDU || w[j] == VISARGA)) {
                if (w[j] == VISARGA) {
                    units.push_back({false, "", vowel, nasal, false, false});
                    vowel.clear(); nasal = 0;
                    units.push_back({true, "ɦ", {}, 0, false, true});
                } else nasal = std::max(nasal, w[j] == CHANDRABINDU ? 2 : 1);
                ++j;
            }
            if (!vowel.empty()) units.push_back({false, "", vowel, nasal, false, false});
            i = j;
        } else if (c == AVAGRAHA) {
            ++i;
        } else if (in(VOWEL_SIGNS, c) || c == ANUSVARA || c == CHANDRABINDU || c == NUKTA || c == VIRAMA || c == VISARGA) {
            if (in(VOWEL_SIGNS, c)) units.push_back({false, "", VOWEL_SIGNS.at(c), 0, false, false});
            else if ((c == ANUSVARA || c == CHANDRABINDU) && !units.empty()) units.back().nasal = c == CHANDRABINDU ? 2 : 1;
            ++i;
        } else {
            ++i;                                                         // unknown character
        }
    }
    return units;
}

// ------------------------------------------------------------------ text._word_phones
std::vector<std::pair<std::string, int>> word_phones(const U& word) {
    std::vector<Unit> u = aksharas(word);
    std::vector<std::pair<std::string, int>> none;
    if (u.empty()) return none;
    const int n = int(u.size());
    std::vector<char> schwa(n), keep(n);
    for (int k = 0; k < n; ++k) schwa[k] = u[k].consonant && u[k].vowel.empty() && !u[k].virama;
    keep = schwa;
    int syllables = 0, last_c = -1;
    for (int k = 0; k < n; ++k) {
        if (!u[k].consonant || !u[k].vowel.empty() || schwa[k]) ++syllables;
        if (u[k].consonant) last_c = k;
    }
    if (last_c == n - 1 && schwa[last_c] && syllables > 1 && !u[last_c].nasal) keep[last_c] = 0;
    auto has_vowel = [&](int k) { return !u[k].consonant || !u[k].vowel.empty() || keep[k]; };
    auto longest_cluster = [&]() {
        int run = 0, best = 0;
        for (int m = 0; m < n; ++m) { run = has_vowel(m) ? 0 : run + 1; best = std::max(best, run); }
        return best;
    };
    for (int k = n - 2; k > 0; --k) {
        if (!keep[k] || u[k].nasal) continue;
        if (!has_vowel(k - 1) || !u[k + 1].consonant || !has_vowel(k + 1)) continue;
        keep[k] = 0;
        if (longest_cluster() >= 3) keep[k] = 1;
    }
    std::vector<std::pair<std::string, int>> phones;
    for (int k = 0; k < n; ++k) {
        if (!u[k].consonant) {
            for (auto& p : u[k].vowel) phones.push_back({p, 0});
            phones.back().second = u[k].nasal;
            continue;
        }
        phones.push_back({u[k].phone, 0});
        if (!u[k].vowel.empty()) for (auto& p : u[k].vowel) phones.push_back({p, 0});
        else if (keep[k]) phones.push_back({"ə", 0});
        if (u[k].nasal) phones.back().second = u[k].nasal;
    }
    std::vector<std::pair<std::string, int>> out;
    for (size_t idx = 0; idx < phones.size(); ++idx) {
        const auto& [p, nasal] = phones[idx];
        const std::string nxt = idx + 1 < phones.size() ? phones[idx + 1].first : std::string();
        const auto h = HOMORGANIC.find(nxt);
        if (nasal && VOWEL_SET.count(p)) {
            if (nasal == 1 && SHORT_VOWELS.count(p) && h != HOMORGANIC.end()) { out.push_back({p, 0}); out.push_back({h->second, 0}); }
            else out.push_back({p, 1});
        } else if (nasal) {
            out.push_back({p, 0});
            if (h != HOMORGANIC.end()) out.push_back({h->second, 0});
        } else out.push_back({p, 0});
    }
    std::vector<std::pair<std::string, int>> fixed;                     // ज्ञ -> ɡ j
    for (size_t i = 0; i < out.size();) {
        if (i + 1 < out.size() && out[i].first == "dʒ" && out[i + 1].first == "ɲ") { fixed.push_back({"ɡ", 0}); fixed.push_back({"j", 0}); i += 2; }
        else fixed.push_back(out[i++]);
    }
    return fixed;
}

// ------------------------------------------------------------------ text.chunks: [ऀ-ॣ०-ॿ\w]+ | [^\s\w]
struct Chunk { bool punct; U piece; };
std::vector<Chunk> chunks(const U& text) {
    std::vector<Chunk> out;
    auto in_run = [](char32_t c) { return (c >= 0x900 && c <= 0x963) || (c >= 0x966 && c <= 0x97F) || is_word(c); };
    for (size_t i = 0; i < text.size();) {
        const char32_t c = text[i];
        if (in_run(c)) {
            size_t j = i; while (j < text.size() && in_run(text[j])) ++j;
            U tok = text.substr(i, j - i);
            if (std::any_of(tok.begin(), tok.end(), is_devanagari)) out.push_back({false, tok});
            i = j;
        } else if (!is_space(c)) {
            const std::string s = encode(U(1, c));
            if (PUNCTUATION.count(s)) out.push_back({true, U(1, c)});
            ++i;
        } else ++i;
    }
    return out;
}

bool ends_with_abbreviation(const U& t) {           // infer._NOT_AN_END (ASCII, case-insensitive)
    if (t.empty() || t.back() != '.') return false;
    const U body = t.substr(0, t.size() - 1);
    auto word_ascii = [](char32_t c) { return c < 0x80 && ((c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_'); };
    static const char* ABBR[] = {"mr", "mrs", "ms", "dr", "prof", "sr", "jr", "st", "vs", "etc", "no", "inc", "ltd", "co",
                                 "e.g", "i.e", "a.m", "p.m"};
    for (const char* a : ABBR) {
        const size_t L = std::char_traits<char>::length(a);
        if (body.size() < L) continue;
        bool ok = true;
        for (size_t k = 0; k < L; ++k) {
            char32_t c = body[body.size() - L + k];
            if (c < 0x80) c = char32_t(std::tolower(int(c)));
            if (c != char32_t(a[k])) { ok = false; break; }
        }
        if (!ok) continue;
        const size_t start = body.size() - L;
        if (start == 0 || !word_ascii(body[start - 1])) return true;       // \b before the abbreviation
    }
    if (!body.empty()) {                                                   // \b[A-Z]\. (any single ASCII letter)
        const char32_t c = body.back();
        const bool letter = c < 0x80 && ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'));
        if (letter && (body.size() == 1 || !word_ascii(body[body.size() - 2]))) return true;
    }
    return false;
}
bool space_or_bom(char32_t c) { return is_space(c) || c == 0xFEFF; }
}  // namespace

// ------------------------------------------------------------------ public
std::string normalize_text(const std::string& text) { return encode(normalize_u(decode(text))); }

HindiG2P::HindiG2P(const std::vector<std::string>& symbols) : symbols_(symbols) {}

int HindiG2P::id_of(const std::string& s) const {
    for (size_t i = 0; i < symbols_.size(); ++i) if (symbols_[i] == s) return int(i);
    return id_of("<unk>");
}

Phonemized HindiG2P::phonemize(const std::string& text) const {
    const U norm = normalize_u(decode(text));
    std::set<std::string> phone_set(symbols_.begin() + 17, symbols_.end());   // specials (5) + punctuation (12) first
    Phonemized r;
    r.symbols.push_back("<bos>"); r.stress.push_back(0);
    for (const Chunk& ch : chunks(norm)) {
        if (ch.punct) {
            const std::string p = encode(ch.piece);
            if (r.symbols.back() != p) { r.symbols.push_back(p); r.stress.push_back(0); }
            continue;
        }
        const auto word = word_phones(ch.piece);
        if (word.empty()) continue;
        if (phone_set.count(r.symbols.back()) || r.symbols.back() == "<unk>") { r.symbols.push_back("<sp>"); r.stress.push_back(0); }
        for (const auto& [phone, nasal] : word) {
            r.symbols.push_back(phone_set.count(phone) ? phone : "<unk>");
            r.stress.push_back(nasal ? 1 : 0);
        }
    }
    r.symbols.push_back("<eos>"); r.stress.push_back(0);
    bool any = false;
    for (auto& s : r.symbols) if (phone_set.count(s) || s == "<unk>") { any = true; break; }
    if (!any) throw NothingToPronounce("the text has nothing to pronounce");
    for (auto& s : r.symbols) r.ids.push_back(id_of(s));
    return r;
}

std::vector<std::string> split_sentences(const std::string& text) {
    const U in = decode(text);
    U t;                                                                  // [\s]+ -> " ", strip
    bool sp = false;
    for (char32_t c : in) {
        if (space_or_bom(c)) { sp = true; continue; }
        if (sp && !t.empty()) t.push_back(' ');
        sp = false; t.push_back(c);
    }
    // _SENTENCE_END: whitespace after [.!?…] (optionally + one closing quote/bracket) before an optional opening
    // quote/bracket and [A-Z0-9] or Devanagari
    auto end_mark = [](char32_t c) { return c == '.' || c == '!' || c == '?' || c == 0x2026; };
    auto closing = [](char32_t c) { return c == '"' || c == '\'' || c == ')' || c == ']'; };
    auto opening = [](char32_t c) { return c == '"' || c == '\'' || c == '(' || c == '['; };
    auto starter = [](char32_t c) { return (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || is_devanagari(c); };
    std::vector<U> pieces;
    size_t start = 0;
    for (size_t i = 0; i < t.size(); ++i) {
        if (t[i] != ' ') continue;
        const bool before = (i >= 1 && end_mark(t[i - 1])) || (i >= 2 && closing(t[i - 1]) && end_mark(t[i - 2]));
        size_t k = i + 1;
        if (k < t.size() && opening(t[k])) ++k;
        const bool after = k < t.size() && starter(t[k]);
        if (before && after) { pieces.push_back(t.substr(start, i - start)); start = i + 1; }
    }
    pieces.push_back(t.substr(start));
    std::vector<std::string> parts;
    U current;
    for (const U& p : pieces) {
        current = current.empty() ? p : current + U(1, ' ') + p;
        if (!ends_with_abbreviation(current)) { parts.push_back(encode(current)); current.clear(); }
    }
    if (!current.empty()) parts.push_back(encode(current));
    std::vector<std::string> out;
    for (auto& p : parts) { const U u = decode(p); if (std::any_of(u.begin(), u.end(), is_alnum)) out.push_back(p); }
    return out;
}

std::string finished(const std::string& text) {
    U t = decode(text);
    while (!t.empty() && space_or_bom(t.back())) t.pop_back();
    if (t.empty()) return text;
    size_t s = t.size();
    while (s > 0 && !space_or_bom(t[s - 1])) --s;
    U last = t.substr(s);
    bool marked;
    if (std::any_of(last.begin(), last.end(), is_alnum)) {
        static const U CLOSE = U"\"'”’)]»";
        U r = last;
        while (!r.empty() && CLOSE.find(r.back()) != U::npos) r.pop_back();
        static const U MARKS = U".!?…,;:";
        marked = !r.empty() && MARKS.find(r.back()) != U::npos;
    } else {
        marked = std::any_of(last.begin(), last.end(), [](char32_t c) { return c == '.' || c == '!' || c == '?' || c == 0x2026; }) ||
                 last == U"," || last == U";" || last == U":";
    }
    return marked ? text : encode(t) + ".";
}

std::vector<std::string> shorter(const std::string& text) {
    const U t = decode(text);
    std::vector<U> pieces;                                               // split at whitespace runs after , ; : —
    size_t start = 0;
    for (size_t i = 0; i < t.size(); ++i) {
        if (!is_space(t[i])) continue;
        if (i == 0) continue;
        const char32_t p = t[i - 1];
        if (p == ',' || p == ';' || p == ':' || p == 0x2014) {
            size_t j = i; while (j < t.size() && is_space(t[j])) ++j;
            pieces.push_back(t.substr(start, i - start));
            start = j; i = j - 1;
        }
    }
    pieces.push_back(t.substr(start));
    std::vector<std::string> out;
    if (pieces.size() > 1) { for (auto& p : pieces) out.push_back(encode(p)); return out; }
    std::vector<U> words; start = 0;
    for (size_t i = 0; i <= t.size(); ++i)
        if (i == t.size() || t[i] == ' ') { words.push_back(t.substr(start, i - start)); start = i + 1; }
    if (words.size() < 2) return out;
    const size_t half = (words.size() + 1) / 2;
    auto join = [&](size_t from, size_t to) {                            // " ".join(words[from:to])
        U s;
        for (size_t i = from; i < to; ++i) { if (i > from) s.push_back(' '); s += words[i]; }
        return encode(s);
    };
    out.push_back(join(0, half)); out.push_back(join(half, words.size()));
    return out;
}