Download cpp/src/g2p.cpp from RL10x/tiny-voca-hindi: direct link, hf CLI and curl.
- Browser
- Download file 23.8 kB
-
https://huggingface.co/RL10x/tiny-voca-hindi/resolve/main/cpp/src/g2p.cpp
- Command line
-
hf download hf://RL10x/tiny-voca-hindi/cpp/src/g2p.cpp
-
curl -L -o g2p.cpp https://huggingface.co/RL10x/tiny-voca-hindi/resolve/main/cpp/src/g2p.cpp
23.8 kB
| namespace { | |
| using U = std::u32string; | |
| // ------------------------------------------------------------------ UTF-8 | |
| U decode(const std::string& s) { | |
| U out; | |
| for (size_t i = 0; i < s.size();) { | |
| const unsigned char c = s[i]; | |
| char32_t cp; int n; | |
| if (c < 0x80) { cp = c; n = 1; } | |
| else if ((c >> 5) == 6) { cp = c & 0x1F; n = 2; } | |
| else if ((c >> 4) == 14) { cp = c & 0x0F; n = 3; } | |
| else if ((c >> 3) == 30) { cp = c & 0x07; n = 4; } | |
| else { ++i; out.push_back(0xFFFD); continue; } | |
| if (i + n > s.size()) { out.push_back(0xFFFD); break; } | |
| bool ok = true; | |
| for (int k = 1; k < n; ++k) { | |
| const unsigned char d = s[i + k]; | |
| if ((d >> 6) != 2) { ok = false; break; } | |
| cp = (cp << 6) | (d & 0x3F); | |
| } | |
| if (!ok) { out.push_back(0xFFFD); ++i; continue; } | |
| out.push_back(cp); i += n; | |
| } | |
| return out; | |
| } | |
| std::string encode(const U& u) { | |
| std::string out; | |
| for (char32_t cp : u) { | |
| if (cp < 0x80) out += char(cp); | |
| else if (cp < 0x800) { out += char(0xC0 | (cp >> 6)); out += char(0x80 | (cp & 0x3F)); } | |
| else if (cp < 0x10000) { out += char(0xE0 | (cp >> 12)); out += char(0x80 | ((cp >> 6) & 0x3F)); out += char(0x80 | (cp & 0x3F)); } | |
| else { out += char(0xF0 | (cp >> 18)); out += char(0x80 | ((cp >> 12) & 0x3F)); out += char(0x80 | ((cp >> 6) & 0x3F)); out += char(0x80 | (cp & 0x3F)); } | |
| } | |
| return out; | |
| } | |
| // ------------------------------------------------------------------ character classes (Python semantics) | |
| bool is_space(char32_t c) { // Python str.isspace / re \s (Unicode) | |
| return c == ' ' || (c >= 0x09 && c <= 0x0D) || (c >= 0x1C && c <= 0x1F) || c == 0x85 || c == 0xA0 || c == 0x1680 || | |
| (c >= 0x2000 && c <= 0x200A) || c == 0x2028 || c == 0x2029 || c == 0x202F || c == 0x205F || c == 0x3000; | |
| } | |
| bool is_format(char32_t c) { // unicodedata.category == "Cf" (the common ones) | |
| return c == 0xAD || (c >= 0x600 && c <= 0x605) || c == 0x61C || c == 0x6DD || c == 0x70F || c == 0x180E || | |
| (c >= 0x200B && c <= 0x200F) || (c >= 0x202A && c <= 0x202E) || (c >= 0x2060 && c <= 0x2064) || | |
| (c >= 0x2066 && c <= 0x206F) || c == 0xFEFF || (c >= 0xFFF9 && c <= 0xFFFB); | |
| } | |
| bool is_devanagari(char32_t c) { return c >= 0x900 && c <= 0x97F; } | |
| bool is_devanagari_mark(char32_t c) { // Devanagari combining marks (Mn/Mc): not \w, not alnum | |
| return (c >= 0x900 && c <= 0x903) || (c >= 0x93A && c <= 0x93C) || (c >= 0x93E && c <= 0x94F) || | |
| (c >= 0x951 && c <= 0x957) || c == 0x962 || c == 0x963; | |
| } | |
| bool is_mark(char32_t c) { // combining marks outside Devanagari that Python's \w rejects | |
| return (c >= 0x300 && c <= 0x36F) || (c >= 0x1AB0 && c <= 0x1AFF) || (c >= 0x1DC0 && c <= 0x1DFF) || | |
| (c >= 0x20D0 && c <= 0x20FF) || (c >= 0xFE00 && c <= 0xFE0F) || (c >= 0xFE20 && c <= 0xFE2F); | |
| } | |
| bool is_symbol_or_punct(char32_t c) { // non-word characters above ASCII | |
| return (c >= 0xA1 && c <= 0xBF && c != 0xAA && c != 0xB2 && c != 0xB3 && c != 0xB5 && c != 0xB9 && c != 0xBA && | |
| c != 0xBC && c != 0xBD && c != 0xBE) || c == 0xD7 || c == 0xF7 || (c >= 0x2010 && c <= 0x2027) || | |
| (c >= 0x2030 && c <= 0x205E) || (c >= 0x20A0 && c <= 0x20CF) || (c >= 0x2190 && c <= 0x2BFF) || (c >= 0x2E00 && c <= 0x2E7F) || (c >= 0x3000 && c <= 0x303F) || | |
| (c >= 0xFF01 && c <= 0xFF0F) || (c >= 0x1F000 && c <= 0x1FAFF) || c == 0x964 || c == 0x965 || c == 0x970 || | |
| c == 0xFFFD; | |
| } | |
| bool is_word(char32_t c) { // Python re \w (Unicode): alphanumerics and underscore | |
| if (c < 0x80) return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_'; | |
| if (is_space(c) || is_format(c) || is_mark(c) || is_devanagari_mark(c) || is_symbol_or_punct(c)) return false; | |
| if (c == 0xA0) return false; | |
| return true; | |
| } | |
| bool is_alnum(char32_t c) { return c != '_' && is_word(c); } // str.isalnum, close enough for our texts | |
| // ------------------------------------------------------------------ text.normalize_text | |
| // NFC for Devanagari: U+0958..U+095F are composition exclusions (NFC splits them into base + nukta); NA/RA/LLA + nukta | |
| // compose to U+0929/U+0931/U+0934. Nukta (ccc 7) sorts before virama (ccc 9). | |
| U nfc_devanagari(const U& in) { | |
| U d; | |
| static const char32_t base958[8] = {0x915, 0x916, 0x917, 0x91C, 0x921, 0x922, 0x92B, 0x92F}; | |
| for (char32_t c : in) { | |
| if (c >= 0x958 && c <= 0x95F) { d.push_back(base958[c - 0x958]); d.push_back(0x93C); } | |
| else if (c == 0x929) { d.push_back(0x928); d.push_back(0x93C); } | |
| else if (c == 0x931) { d.push_back(0x930); d.push_back(0x93C); } | |
| else if (c == 0x934) { d.push_back(0x933); d.push_back(0x93C); } | |
| else d.push_back(c); | |
| } | |
| for (size_t i = 1; i < d.size(); ++i) // canonical order: nukta before virama | |
| if (d[i] == 0x93C && d[i - 1] == 0x94D) std::swap(d[i], d[i - 1]); | |
| U out; | |
| for (size_t i = 0; i < d.size(); ++i) { | |
| if (i + 1 < d.size() && d[i + 1] == 0x93C && (d[i] == 0x928 || d[i] == 0x930 || d[i] == 0x933)) { | |
| out.push_back(d[i] == 0x928 ? 0x929 : d[i] == 0x930 ? 0x931 : 0x934); | |
| ++i; | |
| } else out.push_back(d[i]); | |
| } | |
| return out; | |
| } | |
| U normalize_u(const U& text_in) { | |
| U text = nfc_devanagari(text_in), t; | |
| for (char32_t c : text) if (!is_format(c)) t.push_back(c); | |
| U out; | |
| for (char32_t c : t) { | |
| switch (c) { | |
| case 0x964: case 0x965: out.push_back('.'); break; // । ॥ | |
| case 0x2018: case 0x2019: out.push_back('\''); break; // ‘ ’ | |
| case 0x201C: case 0x201D: out.push_back('"'); break; // “ ” | |
| case 0x2013: out.push_back(0x2014); break; // – -> — | |
| case '-': out.push_back(' '); break; | |
| default: out.push_back(c); | |
| } | |
| } | |
| U dots; // \.{3,} -> … | |
| for (size_t i = 0; i < out.size();) { | |
| if (out[i] == '.') { | |
| size_t j = i; while (j < out.size() && out[j] == '.') ++j; | |
| if (j - i >= 3) dots.push_back(0x2026); else dots.append(out, i, j - i); | |
| i = j; | |
| } else dots.push_back(out[i++]); | |
| } | |
| U res; // \s+ -> " ", strip | |
| bool space = false; | |
| for (char32_t c : dots) { | |
| if (is_space(c)) { space = true; continue; } | |
| if (space && !res.empty()) res.push_back(' '); | |
| space = false; | |
| res.push_back(c); | |
| } | |
| return res; | |
| } | |
| // ------------------------------------------------------------------ tables | |
| const std::map<char32_t, const char*> CONSONANT_CHARS = { | |
| {U'क', "k"}, {U'ख', "kʰ"}, {U'ग', "ɡ"}, {U'घ', "ɡʱ"}, {U'ङ', "ŋ"}, | |
| {U'च', "tʃ"}, {U'छ', "tʃʰ"}, {U'ज', "dʒ"}, {U'झ', "dʒʱ"}, {U'ञ', "ɲ"}, | |
| {U'ट', "ʈ"}, {U'ठ', "ʈʰ"}, {U'ड', "ɖ"}, {U'ढ', "ɖʱ"}, {U'ण', "ɳ"}, | |
| {U'त', "t̪"}, {U'थ', "t̪ʰ"}, {U'द', "d̪"}, {U'ध', "d̪ʱ"}, {U'न', "n"}, | |
| {U'प', "p"}, {U'फ', "pʰ"}, {U'ब', "b"}, {U'भ', "bʱ"}, {U'म', "m"}, | |
| {U'य', "j"}, {U'र', "ɾ"}, {U'ल', "l"}, {U'व', "ʋ"}, {U'श', "ʃ"}, {U'ष', "ʂ"}, {U'स', "s"}, {U'ह', "ɦ"}, | |
| {U'ळ', "l"}, {U'ऱ', "ɾ"}, {U'ऴ', "l"}, | |
| }; | |
| const std::map<char32_t, const char*> NUKTA_FORMS = { | |
| {U'क', "q"}, {U'ख', "x"}, {U'ग', "ɣ"}, {U'ज', "z"}, {U'फ', "f"}, {U'ड', "ɽ"}, {U'ढ', "ɽʱ"}, {U'झ', "ʒ"}, | |
| {U'न', "n"}, {U'र', "ɾ"}, {U'य', "j"}, {U'ल', "l"}, | |
| }; | |
| using Vow = std::vector<std::string>; // a vowel value: one phone or a pair (ऋ -> ɾ ɪ) | |
| const std::map<char32_t, Vow> INDEPENDENT_VOWELS = { | |
| {U'अ', {"ə"}}, {U'आ', {"aː"}}, {U'इ', {"ɪ"}}, {U'ई', {"iː"}}, {U'उ', {"ʊ"}}, {U'ऊ', {"uː"}}, {U'ऋ', {"ɾ", "ɪ"}}, | |
| {U'ॠ', {"ɾ", "iː"}}, {U'ए', {"eː"}}, {U'ऐ', {"ɛː"}}, {U'ओ', {"oː"}}, {U'औ', {"ɔː"}}, {U'ऍ', {"æ"}}, {U'ऑ', {"ɔ"}}, | |
| {U'ऎ', {"eː"}}, {U'ऒ', {"oː"}}, {U'ऌ', {"l", "ɪ"}}, | |
| }; | |
| const std::map<char32_t, Vow> VOWEL_SIGNS = { | |
| {U'ा', {"aː"}}, {U'ि', {"ɪ"}}, {U'ी', {"iː"}}, {U'ु', {"ʊ"}}, {U'ू', {"uː"}}, {U'ृ', {"ɾ", "ɪ"}}, {U'ॄ', {"ɾ", "iː"}}, | |
| {U'े', {"eː"}}, {U'ै', {"ɛː"}}, {U'ो', {"oː"}}, {U'ौ', {"ɔː"}}, {U'ॅ', {"æ"}}, {U'ॉ', {"ɔ"}}, {U'ॆ', {"eː"}}, | |
| {U'ॊ', {"oː"}}, {U'ॢ', {"l", "ɪ"}}, | |
| }; | |
| const char32_t VIRAMA = U'्', NUKTA = U'़', ANUSVARA = U'ं', CHANDRABINDU = U'ँ', VISARGA = U'ः', AVAGRAHA = U'ऽ'; | |
| const std::set<std::string> VOWEL_SET = {"ə", "aː", "ɪ", "iː", "ʊ", "uː", "eː", "ɛː", "oː", "ɔː", "æ", "ɔ"}; | |
| const std::set<std::string> SHORT_VOWELS = {"ə", "ɪ", "ʊ", "æ", "ɔ"}; | |
| const std::set<std::string> PUNCTUATION = {",", ".", "?", "!", ";", ":", "\"", "'", "(", ")", "—", "…"}; | |
| const std::unordered_map<std::string, std::string> HOMORGANIC = [] { | |
| std::unordered_map<std::string, std::string> m; | |
| const std::vector<std::pair<std::string, std::vector<std::string>>> groups = { | |
| {"ŋ", {"k", "kʰ", "ɡ", "ɡʱ", "q", "x", "ɣ"}}, {"ɲ", {"tʃ", "tʃʰ", "dʒ", "dʒʱ"}}, {"ɳ", {"ʈ", "ʈʰ", "ɖ", "ɖʱ"}}, | |
| {"n", {"t̪", "t̪ʰ", "d̪", "d̪ʱ"}}, {"m", {"p", "pʰ", "b", "bʱ"}}}; | |
| for (auto& [nasal, stops] : groups) for (auto& s : stops) m[s] = nasal; | |
| return m; | |
| }(); | |
| // ------------------------------------------------------------------ text._aksharas | |
| struct Unit { | |
| bool consonant; // 'C' or 'V' | |
| std::string phone; // consonant phone (C) | |
| Vow vowel; // C: vowel sign (empty = none / inherent schwa); V: the vowel (empty = None) | |
| int nasal; // 0, 1 anusvara, 2 chandrabindu | |
| bool explicit_; | |
| bool virama; | |
| }; | |
| std::vector<Unit> aksharas(const U& w) { | |
| std::vector<Unit> units; | |
| size_t i = 0; | |
| const size_t n = w.size(); | |
| auto in = [](const auto& m, char32_t c) { return m.find(c) != m.end(); }; | |
| while (i < n) { | |
| const char32_t c = w[i]; | |
| if (in(CONSONANT_CHARS, c)) { | |
| std::string phone = CONSONANT_CHARS.at(c); | |
| size_t j = i + 1; | |
| if (j < n && w[j] == NUKTA) { | |
| auto it = NUKTA_FORMS.find(c); | |
| if (it != NUKTA_FORMS.end()) phone = it->second; | |
| ++j; | |
| } | |
| Vow vowel; bool explicit_ = false, virama = false; | |
| if (j < n && w[j] == VIRAMA) { virama = true; ++j; } | |
| else if (j < n && in(VOWEL_SIGNS, w[j])) { vowel = VOWEL_SIGNS.at(w[j]); explicit_ = true; ++j; } | |
| int nasal = 0; | |
| while (j < n && (w[j] == ANUSVARA || w[j] == CHANDRABINDU || w[j] == VISARGA)) { | |
| if (w[j] == VISARGA) { | |
| units.push_back({true, phone, vowel, nasal, explicit_, virama}); | |
| phone = "ɦ"; vowel.clear(); explicit_ = false; virama = true; nasal = 0; | |
| } else nasal = std::max(nasal, w[j] == CHANDRABINDU ? 2 : 1); | |
| ++j; | |
| } | |
| units.push_back({true, phone, vowel, nasal, explicit_, virama}); | |
| i = j; | |
| } else if (in(INDEPENDENT_VOWELS, c)) { | |
| Vow vowel = INDEPENDENT_VOWELS.at(c); | |
| size_t j = i + 1; int nasal = 0; | |
| while (j < n && (w[j] == ANUSVARA || w[j] == CHANDRABINDU || w[j] == VISARGA)) { | |
| if (w[j] == VISARGA) { | |
| units.push_back({false, "", vowel, nasal, false, false}); | |
| vowel.clear(); nasal = 0; | |
| units.push_back({true, "ɦ", {}, 0, false, true}); | |
| } else nasal = std::max(nasal, w[j] == CHANDRABINDU ? 2 : 1); | |
| ++j; | |
| } | |
| if (!vowel.empty()) units.push_back({false, "", vowel, nasal, false, false}); | |
| i = j; | |
| } else if (c == AVAGRAHA) { | |
| ++i; | |
| } else if (in(VOWEL_SIGNS, c) || c == ANUSVARA || c == CHANDRABINDU || c == NUKTA || c == VIRAMA || c == VISARGA) { | |
| if (in(VOWEL_SIGNS, c)) units.push_back({false, "", VOWEL_SIGNS.at(c), 0, false, false}); | |
| else if ((c == ANUSVARA || c == CHANDRABINDU) && !units.empty()) units.back().nasal = c == CHANDRABINDU ? 2 : 1; | |
| ++i; | |
| } else { | |
| ++i; // unknown character | |
| } | |
| } | |
| return units; | |
| } | |
| // ------------------------------------------------------------------ text._word_phones | |
| std::vector<std::pair<std::string, int>> word_phones(const U& word) { | |
| std::vector<Unit> u = aksharas(word); | |
| std::vector<std::pair<std::string, int>> none; | |
| if (u.empty()) return none; | |
| const int n = int(u.size()); | |
| std::vector<char> schwa(n), keep(n); | |
| for (int k = 0; k < n; ++k) schwa[k] = u[k].consonant && u[k].vowel.empty() && !u[k].virama; | |
| keep = schwa; | |
| int syllables = 0, last_c = -1; | |
| for (int k = 0; k < n; ++k) { | |
| if (!u[k].consonant || !u[k].vowel.empty() || schwa[k]) ++syllables; | |
| if (u[k].consonant) last_c = k; | |
| } | |
| if (last_c == n - 1 && schwa[last_c] && syllables > 1 && !u[last_c].nasal) keep[last_c] = 0; | |
| auto has_vowel = [&](int k) { return !u[k].consonant || !u[k].vowel.empty() || keep[k]; }; | |
| auto longest_cluster = [&]() { | |
| int run = 0, best = 0; | |
| for (int m = 0; m < n; ++m) { run = has_vowel(m) ? 0 : run + 1; best = std::max(best, run); } | |
| return best; | |
| }; | |
| for (int k = n - 2; k > 0; --k) { | |
| if (!keep[k] || u[k].nasal) continue; | |
| if (!has_vowel(k - 1) || !u[k + 1].consonant || !has_vowel(k + 1)) continue; | |
| keep[k] = 0; | |
| if (longest_cluster() >= 3) keep[k] = 1; | |
| } | |
| std::vector<std::pair<std::string, int>> phones; | |
| for (int k = 0; k < n; ++k) { | |
| if (!u[k].consonant) { | |
| for (auto& p : u[k].vowel) phones.push_back({p, 0}); | |
| phones.back().second = u[k].nasal; | |
| continue; | |
| } | |
| phones.push_back({u[k].phone, 0}); | |
| if (!u[k].vowel.empty()) for (auto& p : u[k].vowel) phones.push_back({p, 0}); | |
| else if (keep[k]) phones.push_back({"ə", 0}); | |
| if (u[k].nasal) phones.back().second = u[k].nasal; | |
| } | |
| std::vector<std::pair<std::string, int>> out; | |
| for (size_t idx = 0; idx < phones.size(); ++idx) { | |
| const auto& [p, nasal] = phones[idx]; | |
| const std::string nxt = idx + 1 < phones.size() ? phones[idx + 1].first : std::string(); | |
| const auto h = HOMORGANIC.find(nxt); | |
| if (nasal && VOWEL_SET.count(p)) { | |
| if (nasal == 1 && SHORT_VOWELS.count(p) && h != HOMORGANIC.end()) { out.push_back({p, 0}); out.push_back({h->second, 0}); } | |
| else out.push_back({p, 1}); | |
| } else if (nasal) { | |
| out.push_back({p, 0}); | |
| if (h != HOMORGANIC.end()) out.push_back({h->second, 0}); | |
| } else out.push_back({p, 0}); | |
| } | |
| std::vector<std::pair<std::string, int>> fixed; // ज्ञ -> ɡ j | |
| for (size_t i = 0; i < out.size();) { | |
| if (i + 1 < out.size() && out[i].first == "dʒ" && out[i + 1].first == "ɲ") { fixed.push_back({"ɡ", 0}); fixed.push_back({"j", 0}); i += 2; } | |
| else fixed.push_back(out[i++]); | |
| } | |
| return fixed; | |
| } | |
| // ------------------------------------------------------------------ text.chunks: [ऀ-ॣ०-ॿ\w]+ | [^\s\w] | |
| struct Chunk { bool punct; U piece; }; | |
| std::vector<Chunk> chunks(const U& text) { | |
| std::vector<Chunk> out; | |
| auto in_run = [](char32_t c) { return (c >= 0x900 && c <= 0x963) || (c >= 0x966 && c <= 0x97F) || is_word(c); }; | |
| for (size_t i = 0; i < text.size();) { | |
| const char32_t c = text[i]; | |
| if (in_run(c)) { | |
| size_t j = i; while (j < text.size() && in_run(text[j])) ++j; | |
| U tok = text.substr(i, j - i); | |
| if (std::any_of(tok.begin(), tok.end(), is_devanagari)) out.push_back({false, tok}); | |
| i = j; | |
| } else if (!is_space(c)) { | |
| const std::string s = encode(U(1, c)); | |
| if (PUNCTUATION.count(s)) out.push_back({true, U(1, c)}); | |
| ++i; | |
| } else ++i; | |
| } | |
| return out; | |
| } | |
| bool ends_with_abbreviation(const U& t) { // infer._NOT_AN_END (ASCII, case-insensitive) | |
| if (t.empty() || t.back() != '.') return false; | |
| const U body = t.substr(0, t.size() - 1); | |
| auto word_ascii = [](char32_t c) { return c < 0x80 && ((c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_'); }; | |
| static const char* ABBR[] = {"mr", "mrs", "ms", "dr", "prof", "sr", "jr", "st", "vs", "etc", "no", "inc", "ltd", "co", | |
| "e.g", "i.e", "a.m", "p.m"}; | |
| for (const char* a : ABBR) { | |
| const size_t L = std::char_traits<char>::length(a); | |
| if (body.size() < L) continue; | |
| bool ok = true; | |
| for (size_t k = 0; k < L; ++k) { | |
| char32_t c = body[body.size() - L + k]; | |
| if (c < 0x80) c = char32_t(std::tolower(int(c))); | |
| if (c != char32_t(a[k])) { ok = false; break; } | |
| } | |
| if (!ok) continue; | |
| const size_t start = body.size() - L; | |
| if (start == 0 || !word_ascii(body[start - 1])) return true; // \b before the abbreviation | |
| } | |
| if (!body.empty()) { // \b[A-Z]\. (any single ASCII letter) | |
| const char32_t c = body.back(); | |
| const bool letter = c < 0x80 && ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')); | |
| if (letter && (body.size() == 1 || !word_ascii(body[body.size() - 2]))) return true; | |
| } | |
| return false; | |
| } | |
| bool space_or_bom(char32_t c) { return is_space(c) || c == 0xFEFF; } | |
| } // namespace | |
| // ------------------------------------------------------------------ public | |
| std::string normalize_text(const std::string& text) { return encode(normalize_u(decode(text))); } | |
| HindiG2P::HindiG2P(const std::vector<std::string>& symbols) : symbols_(symbols) {} | |
| int HindiG2P::id_of(const std::string& s) const { | |
| for (size_t i = 0; i < symbols_.size(); ++i) if (symbols_[i] == s) return int(i); | |
| return id_of("<unk>"); | |
| } | |
| Phonemized HindiG2P::phonemize(const std::string& text) const { | |
| const U norm = normalize_u(decode(text)); | |
| std::set<std::string> phone_set(symbols_.begin() + 17, symbols_.end()); // specials (5) + punctuation (12) first | |
| Phonemized r; | |
| r.symbols.push_back("<bos>"); r.stress.push_back(0); | |
| for (const Chunk& ch : chunks(norm)) { | |
| if (ch.punct) { | |
| const std::string p = encode(ch.piece); | |
| if (r.symbols.back() != p) { r.symbols.push_back(p); r.stress.push_back(0); } | |
| continue; | |
| } | |
| const auto word = word_phones(ch.piece); | |
| if (word.empty()) continue; | |
| if (phone_set.count(r.symbols.back()) || r.symbols.back() == "<unk>") { r.symbols.push_back("<sp>"); r.stress.push_back(0); } | |
| for (const auto& [phone, nasal] : word) { | |
| r.symbols.push_back(phone_set.count(phone) ? phone : "<unk>"); | |
| r.stress.push_back(nasal ? 1 : 0); | |
| } | |
| } | |
| r.symbols.push_back("<eos>"); r.stress.push_back(0); | |
| bool any = false; | |
| for (auto& s : r.symbols) if (phone_set.count(s) || s == "<unk>") { any = true; break; } | |
| if (!any) throw NothingToPronounce("the text has nothing to pronounce"); | |
| for (auto& s : r.symbols) r.ids.push_back(id_of(s)); | |
| return r; | |
| } | |
| std::vector<std::string> split_sentences(const std::string& text) { | |
| const U in = decode(text); | |
| U t; // [\s]+ -> " ", strip | |
| bool sp = false; | |
| for (char32_t c : in) { | |
| if (space_or_bom(c)) { sp = true; continue; } | |
| if (sp && !t.empty()) t.push_back(' '); | |
| sp = false; t.push_back(c); | |
| } | |
| // _SENTENCE_END: whitespace after [.!?…] (optionally + one closing quote/bracket) before an optional opening | |
| // quote/bracket and [A-Z0-9] or Devanagari | |
| auto end_mark = [](char32_t c) { return c == '.' || c == '!' || c == '?' || c == 0x2026; }; | |
| auto closing = [](char32_t c) { return c == '"' || c == '\'' || c == ')' || c == ']'; }; | |
| auto opening = [](char32_t c) { return c == '"' || c == '\'' || c == '(' || c == '['; }; | |
| auto starter = [](char32_t c) { return (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || is_devanagari(c); }; | |
| std::vector<U> pieces; | |
| size_t start = 0; | |
| for (size_t i = 0; i < t.size(); ++i) { | |
| if (t[i] != ' ') continue; | |
| const bool before = (i >= 1 && end_mark(t[i - 1])) || (i >= 2 && closing(t[i - 1]) && end_mark(t[i - 2])); | |
| size_t k = i + 1; | |
| if (k < t.size() && opening(t[k])) ++k; | |
| const bool after = k < t.size() && starter(t[k]); | |
| if (before && after) { pieces.push_back(t.substr(start, i - start)); start = i + 1; } | |
| } | |
| pieces.push_back(t.substr(start)); | |
| std::vector<std::string> parts; | |
| U current; | |
| for (const U& p : pieces) { | |
| current = current.empty() ? p : current + U(1, ' ') + p; | |
| if (!ends_with_abbreviation(current)) { parts.push_back(encode(current)); current.clear(); } | |
| } | |
| if (!current.empty()) parts.push_back(encode(current)); | |
| std::vector<std::string> out; | |
| for (auto& p : parts) { const U u = decode(p); if (std::any_of(u.begin(), u.end(), is_alnum)) out.push_back(p); } | |
| return out; | |
| } | |
| std::string finished(const std::string& text) { | |
| U t = decode(text); | |
| while (!t.empty() && space_or_bom(t.back())) t.pop_back(); | |
| if (t.empty()) return text; | |
| size_t s = t.size(); | |
| while (s > 0 && !space_or_bom(t[s - 1])) --s; | |
| U last = t.substr(s); | |
| bool marked; | |
| if (std::any_of(last.begin(), last.end(), is_alnum)) { | |
| static const U CLOSE = U"\"'”’)]»"; | |
| U r = last; | |
| while (!r.empty() && CLOSE.find(r.back()) != U::npos) r.pop_back(); | |
| static const U MARKS = U".!?…,;:"; | |
| marked = !r.empty() && MARKS.find(r.back()) != U::npos; | |
| } else { | |
| marked = std::any_of(last.begin(), last.end(), [](char32_t c) { return c == '.' || c == '!' || c == '?' || c == 0x2026; }) || | |
| last == U"," || last == U";" || last == U":"; | |
| } | |
| return marked ? text : encode(t) + "."; | |
| } | |
| std::vector<std::string> shorter(const std::string& text) { | |
| const U t = decode(text); | |
| std::vector<U> pieces; // split at whitespace runs after , ; : — | |
| size_t start = 0; | |
| for (size_t i = 0; i < t.size(); ++i) { | |
| if (!is_space(t[i])) continue; | |
| if (i == 0) continue; | |
| const char32_t p = t[i - 1]; | |
| if (p == ',' || p == ';' || p == ':' || p == 0x2014) { | |
| size_t j = i; while (j < t.size() && is_space(t[j])) ++j; | |
| pieces.push_back(t.substr(start, i - start)); | |
| start = j; i = j - 1; | |
| } | |
| } | |
| pieces.push_back(t.substr(start)); | |
| std::vector<std::string> out; | |
| if (pieces.size() > 1) { for (auto& p : pieces) out.push_back(encode(p)); return out; } | |
| std::vector<U> words; start = 0; | |
| for (size_t i = 0; i <= t.size(); ++i) | |
| if (i == t.size() || t[i] == ' ') { words.push_back(t.substr(start, i - start)); start = i + 1; } | |
| if (words.size() < 2) return out; | |
| const size_t half = (words.size() + 1) / 2; | |
| auto join = [&](size_t from, size_t to) { // " ".join(words[from:to]) | |
| U s; | |
| for (size_t i = from; i < to; ++i) { if (i > from) s.push_back(' '); s += words[i]; } | |
| return encode(s); | |
| }; | |
| out.push_back(join(0, half)); out.push_back(join(half, words.size())); | |
| return out; | |
| } | |