tiny-voca-hindi / cpp /src /g2p.cpp
RL10x's picture
Add inference code (Python + C++ engine + web page)
ab1b068 verified
Raw History Blame Contribute Delete
23.8 kB
#include "g2p.h"
#include <algorithm>
#include <map>
#include <set>
#include <unordered_map>
namespace {
using U = std::u32string;
// ------------------------------------------------------------------ UTF-8
U decode(const std::string& s) {
U out;
for (size_t i = 0; i < s.size();) {
const unsigned char c = s[i];
char32_t cp; int n;
if (c < 0x80) { cp = c; n = 1; }
else if ((c >> 5) == 6) { cp = c & 0x1F; n = 2; }
else if ((c >> 4) == 14) { cp = c & 0x0F; n = 3; }
else if ((c >> 3) == 30) { cp = c & 0x07; n = 4; }
else { ++i; out.push_back(0xFFFD); continue; }
if (i + n > s.size()) { out.push_back(0xFFFD); break; }
bool ok = true;
for (int k = 1; k < n; ++k) {
const unsigned char d = s[i + k];
if ((d >> 6) != 2) { ok = false; break; }
cp = (cp << 6) | (d & 0x3F);
}
if (!ok) { out.push_back(0xFFFD); ++i; continue; }
out.push_back(cp); i += n;
}
return out;
}
std::string encode(const U& u) {
std::string out;
for (char32_t cp : u) {
if (cp < 0x80) out += char(cp);
else if (cp < 0x800) { out += char(0xC0 | (cp >> 6)); out += char(0x80 | (cp & 0x3F)); }
else if (cp < 0x10000) { out += char(0xE0 | (cp >> 12)); out += char(0x80 | ((cp >> 6) & 0x3F)); out += char(0x80 | (cp & 0x3F)); }
else { out += char(0xF0 | (cp >> 18)); out += char(0x80 | ((cp >> 12) & 0x3F)); out += char(0x80 | ((cp >> 6) & 0x3F)); out += char(0x80 | (cp & 0x3F)); }
}
return out;
}
// ------------------------------------------------------------------ character classes (Python semantics)
bool is_space(char32_t c) { // Python str.isspace / re \s (Unicode)
return c == ' ' || (c >= 0x09 && c <= 0x0D) || (c >= 0x1C && c <= 0x1F) || c == 0x85 || c == 0xA0 || c == 0x1680 ||
(c >= 0x2000 && c <= 0x200A) || c == 0x2028 || c == 0x2029 || c == 0x202F || c == 0x205F || c == 0x3000;
}
bool is_format(char32_t c) { // unicodedata.category == "Cf" (the common ones)
return c == 0xAD || (c >= 0x600 && c <= 0x605) || c == 0x61C || c == 0x6DD || c == 0x70F || c == 0x180E ||
(c >= 0x200B && c <= 0x200F) || (c >= 0x202A && c <= 0x202E) || (c >= 0x2060 && c <= 0x2064) ||
(c >= 0x2066 && c <= 0x206F) || c == 0xFEFF || (c >= 0xFFF9 && c <= 0xFFFB);
}
bool is_devanagari(char32_t c) { return c >= 0x900 && c <= 0x97F; }
bool is_devanagari_mark(char32_t c) { // Devanagari combining marks (Mn/Mc): not \w, not alnum
return (c >= 0x900 && c <= 0x903) || (c >= 0x93A && c <= 0x93C) || (c >= 0x93E && c <= 0x94F) ||
(c >= 0x951 && c <= 0x957) || c == 0x962 || c == 0x963;
}
bool is_mark(char32_t c) { // combining marks outside Devanagari that Python's \w rejects
return (c >= 0x300 && c <= 0x36F) || (c >= 0x1AB0 && c <= 0x1AFF) || (c >= 0x1DC0 && c <= 0x1DFF) ||
(c >= 0x20D0 && c <= 0x20FF) || (c >= 0xFE00 && c <= 0xFE0F) || (c >= 0xFE20 && c <= 0xFE2F);
}
bool is_symbol_or_punct(char32_t c) { // non-word characters above ASCII
return (c >= 0xA1 && c <= 0xBF && c != 0xAA && c != 0xB2 && c != 0xB3 && c != 0xB5 && c != 0xB9 && c != 0xBA &&
c != 0xBC && c != 0xBD && c != 0xBE) || c == 0xD7 || c == 0xF7 || (c >= 0x2010 && c <= 0x2027) ||
(c >= 0x2030 && c <= 0x205E) || (c >= 0x20A0 && c <= 0x20CF) || (c >= 0x2190 && c <= 0x2BFF) || (c >= 0x2E00 && c <= 0x2E7F) || (c >= 0x3000 && c <= 0x303F) ||
(c >= 0xFF01 && c <= 0xFF0F) || (c >= 0x1F000 && c <= 0x1FAFF) || c == 0x964 || c == 0x965 || c == 0x970 ||
c == 0xFFFD;
}
bool is_word(char32_t c) { // Python re \w (Unicode): alphanumerics and underscore
if (c < 0x80) return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_';
if (is_space(c) || is_format(c) || is_mark(c) || is_devanagari_mark(c) || is_symbol_or_punct(c)) return false;
if (c == 0xA0) return false;
return true;
}
bool is_alnum(char32_t c) { return c != '_' && is_word(c); } // str.isalnum, close enough for our texts
// ------------------------------------------------------------------ text.normalize_text
// NFC for Devanagari: U+0958..U+095F are composition exclusions (NFC splits them into base + nukta); NA/RA/LLA + nukta
// compose to U+0929/U+0931/U+0934. Nukta (ccc 7) sorts before virama (ccc 9).
U nfc_devanagari(const U& in) {
U d;
static const char32_t base958[8] = {0x915, 0x916, 0x917, 0x91C, 0x921, 0x922, 0x92B, 0x92F};
for (char32_t c : in) {
if (c >= 0x958 && c <= 0x95F) { d.push_back(base958[c - 0x958]); d.push_back(0x93C); }
else if (c == 0x929) { d.push_back(0x928); d.push_back(0x93C); }
else if (c == 0x931) { d.push_back(0x930); d.push_back(0x93C); }
else if (c == 0x934) { d.push_back(0x933); d.push_back(0x93C); }
else d.push_back(c);
}
for (size_t i = 1; i < d.size(); ++i) // canonical order: nukta before virama
if (d[i] == 0x93C && d[i - 1] == 0x94D) std::swap(d[i], d[i - 1]);
U out;
for (size_t i = 0; i < d.size(); ++i) {
if (i + 1 < d.size() && d[i + 1] == 0x93C && (d[i] == 0x928 || d[i] == 0x930 || d[i] == 0x933)) {
out.push_back(d[i] == 0x928 ? 0x929 : d[i] == 0x930 ? 0x931 : 0x934);
++i;
} else out.push_back(d[i]);
}
return out;
}
U normalize_u(const U& text_in) {
U text = nfc_devanagari(text_in), t;
for (char32_t c : text) if (!is_format(c)) t.push_back(c);
U out;
for (char32_t c : t) {
switch (c) {
case 0x964: case 0x965: out.push_back('.'); break; // । ॥
case 0x2018: case 0x2019: out.push_back('\''); break; // ‘ ’
case 0x201C: case 0x201D: out.push_back('"'); break; // “ ”
case 0x2013: out.push_back(0x2014); break; // – -> —
case '-': out.push_back(' '); break;
default: out.push_back(c);
}
}
U dots; // \.{3,} -> …
for (size_t i = 0; i < out.size();) {
if (out[i] == '.') {
size_t j = i; while (j < out.size() && out[j] == '.') ++j;
if (j - i >= 3) dots.push_back(0x2026); else dots.append(out, i, j - i);
i = j;
} else dots.push_back(out[i++]);
}
U res; // \s+ -> " ", strip
bool space = false;
for (char32_t c : dots) {
if (is_space(c)) { space = true; continue; }
if (space && !res.empty()) res.push_back(' ');
space = false;
res.push_back(c);
}
return res;
}
// ------------------------------------------------------------------ tables
const std::map<char32_t, const char*> CONSONANT_CHARS = {
{U'क', "k"}, {U'ख', "kʰ"}, {U'ग', "ɡ"}, {U'घ', "ɡʱ"}, {U'ङ', "ŋ"},
{U'च', "tʃ"}, {U'छ', "tʃʰ"}, {U'ज', "dʒ"}, {U'झ', "dʒʱ"}, {U'ञ', "ɲ"},
{U'ट', "ʈ"}, {U'ठ', "ʈʰ"}, {U'ड', "ɖ"}, {U'ढ', "ɖʱ"}, {U'ण', "ɳ"},
{U'त', "t̪"}, {U'थ', "t̪ʰ"}, {U'द', "d̪"}, {U'ध', "d̪ʱ"}, {U'न', "n"},
{U'प', "p"}, {U'फ', "pʰ"}, {U'ब', "b"}, {U'भ', "bʱ"}, {U'म', "m"},
{U'य', "j"}, {U'र', "ɾ"}, {U'ल', "l"}, {U'व', "ʋ"}, {U'श', "ʃ"}, {U'ष', "ʂ"}, {U'स', "s"}, {U'ह', "ɦ"},
{U'ळ', "l"}, {U'ऱ', "ɾ"}, {U'ऴ', "l"},
};
const std::map<char32_t, const char*> NUKTA_FORMS = {
{U'क', "q"}, {U'ख', "x"}, {U'ग', "ɣ"}, {U'ज', "z"}, {U'फ', "f"}, {U'ड', "ɽ"}, {U'ढ', "ɽʱ"}, {U'झ', "ʒ"},
{U'न', "n"}, {U'र', "ɾ"}, {U'य', "j"}, {U'ल', "l"},
};
using Vow = std::vector<std::string>; // a vowel value: one phone or a pair (ऋ -> ɾ ɪ)
const std::map<char32_t, Vow> INDEPENDENT_VOWELS = {
{U'अ', {"ə"}}, {U'आ', {"aː"}}, {U'इ', {"ɪ"}}, {U'ई', {"iː"}}, {U'उ', {"ʊ"}}, {U'ऊ', {"uː"}}, {U'ऋ', {"ɾ", "ɪ"}},
{U'ॠ', {"ɾ", "iː"}}, {U'ए', {"eː"}}, {U'ऐ', {"ɛː"}}, {U'ओ', {"oː"}}, {U'औ', {"ɔː"}}, {U'ऍ', {"æ"}}, {U'ऑ', {"ɔ"}},
{U'ऎ', {"eː"}}, {U'ऒ', {"oː"}}, {U'ऌ', {"l", "ɪ"}},
};
const std::map<char32_t, Vow> VOWEL_SIGNS = {
{U'ा', {"aː"}}, {U'ि', {"ɪ"}}, {U'ी', {"iː"}}, {U'ु', {"ʊ"}}, {U'ू', {"uː"}}, {U'ृ', {"ɾ", "ɪ"}}, {U'ॄ', {"ɾ", "iː"}},
{U'े', {"eː"}}, {U'ै', {"ɛː"}}, {U'ो', {"oː"}}, {U'ौ', {"ɔː"}}, {U'ॅ', {"æ"}}, {U'ॉ', {"ɔ"}}, {U'ॆ', {"eː"}},
{U'ॊ', {"oː"}}, {U'ॢ', {"l", "ɪ"}},
};
const char32_t VIRAMA = U'्', NUKTA = U'़', ANUSVARA = U'ं', CHANDRABINDU = U'ँ', VISARGA = U'ः', AVAGRAHA = U'ऽ';
const std::set<std::string> VOWEL_SET = {"ə", "aː", "ɪ", "iː", "ʊ", "uː", "eː", "ɛː", "oː", "ɔː", "æ", "ɔ"};
const std::set<std::string> SHORT_VOWELS = {"ə", "ɪ", "ʊ", "æ", "ɔ"};
const std::set<std::string> PUNCTUATION = {",", ".", "?", "!", ";", ":", "\"", "'", "(", ")", "—", "…"};
const std::unordered_map<std::string, std::string> HOMORGANIC = [] {
std::unordered_map<std::string, std::string> m;
const std::vector<std::pair<std::string, std::vector<std::string>>> groups = {
{"ŋ", {"k", "kʰ", "ɡ", "ɡʱ", "q", "x", "ɣ"}}, {"ɲ", {"tʃ", "tʃʰ", "dʒ", "dʒʱ"}}, {"ɳ", {"ʈ", "ʈʰ", "ɖ", "ɖʱ"}},
{"n", {"t̪", "t̪ʰ", "d̪", "d̪ʱ"}}, {"m", {"p", "pʰ", "b", "bʱ"}}};
for (auto& [nasal, stops] : groups) for (auto& s : stops) m[s] = nasal;
return m;
}();
// ------------------------------------------------------------------ text._aksharas
struct Unit {
bool consonant; // 'C' or 'V'
std::string phone; // consonant phone (C)
Vow vowel; // C: vowel sign (empty = none / inherent schwa); V: the vowel (empty = None)
int nasal; // 0, 1 anusvara, 2 chandrabindu
bool explicit_;
bool virama;
};
std::vector<Unit> aksharas(const U& w) {
std::vector<Unit> units;
size_t i = 0;
const size_t n = w.size();
auto in = [](const auto& m, char32_t c) { return m.find(c) != m.end(); };
while (i < n) {
const char32_t c = w[i];
if (in(CONSONANT_CHARS, c)) {
std::string phone = CONSONANT_CHARS.at(c);
size_t j = i + 1;
if (j < n && w[j] == NUKTA) {
auto it = NUKTA_FORMS.find(c);
if (it != NUKTA_FORMS.end()) phone = it->second;
++j;
}
Vow vowel; bool explicit_ = false, virama = false;
if (j < n && w[j] == VIRAMA) { virama = true; ++j; }
else if (j < n && in(VOWEL_SIGNS, w[j])) { vowel = VOWEL_SIGNS.at(w[j]); explicit_ = true; ++j; }
int nasal = 0;
while (j < n && (w[j] == ANUSVARA || w[j] == CHANDRABINDU || w[j] == VISARGA)) {
if (w[j] == VISARGA) {
units.push_back({true, phone, vowel, nasal, explicit_, virama});
phone = "ɦ"; vowel.clear(); explicit_ = false; virama = true; nasal = 0;
} else nasal = std::max(nasal, w[j] == CHANDRABINDU ? 2 : 1);
++j;
}
units.push_back({true, phone, vowel, nasal, explicit_, virama});
i = j;
} else if (in(INDEPENDENT_VOWELS, c)) {
Vow vowel = INDEPENDENT_VOWELS.at(c);
size_t j = i + 1; int nasal = 0;
while (j < n && (w[j] == ANUSVARA || w[j] == CHANDRABINDU || w[j] == VISARGA)) {
if (w[j] == VISARGA) {
units.push_back({false, "", vowel, nasal, false, false});
vowel.clear(); nasal = 0;
units.push_back({true, "ɦ", {}, 0, false, true});
} else nasal = std::max(nasal, w[j] == CHANDRABINDU ? 2 : 1);
++j;
}
if (!vowel.empty()) units.push_back({false, "", vowel, nasal, false, false});
i = j;
} else if (c == AVAGRAHA) {
++i;
} else if (in(VOWEL_SIGNS, c) || c == ANUSVARA || c == CHANDRABINDU || c == NUKTA || c == VIRAMA || c == VISARGA) {
if (in(VOWEL_SIGNS, c)) units.push_back({false, "", VOWEL_SIGNS.at(c), 0, false, false});
else if ((c == ANUSVARA || c == CHANDRABINDU) && !units.empty()) units.back().nasal = c == CHANDRABINDU ? 2 : 1;
++i;
} else {
++i; // unknown character
}
}
return units;
}
// ------------------------------------------------------------------ text._word_phones
std::vector<std::pair<std::string, int>> word_phones(const U& word) {
std::vector<Unit> u = aksharas(word);
std::vector<std::pair<std::string, int>> none;
if (u.empty()) return none;
const int n = int(u.size());
std::vector<char> schwa(n), keep(n);
for (int k = 0; k < n; ++k) schwa[k] = u[k].consonant && u[k].vowel.empty() && !u[k].virama;
keep = schwa;
int syllables = 0, last_c = -1;
for (int k = 0; k < n; ++k) {
if (!u[k].consonant || !u[k].vowel.empty() || schwa[k]) ++syllables;
if (u[k].consonant) last_c = k;
}
if (last_c == n - 1 && schwa[last_c] && syllables > 1 && !u[last_c].nasal) keep[last_c] = 0;
auto has_vowel = [&](int k) { return !u[k].consonant || !u[k].vowel.empty() || keep[k]; };
auto longest_cluster = [&]() {
int run = 0, best = 0;
for (int m = 0; m < n; ++m) { run = has_vowel(m) ? 0 : run + 1; best = std::max(best, run); }
return best;
};
for (int k = n - 2; k > 0; --k) {
if (!keep[k] || u[k].nasal) continue;
if (!has_vowel(k - 1) || !u[k + 1].consonant || !has_vowel(k + 1)) continue;
keep[k] = 0;
if (longest_cluster() >= 3) keep[k] = 1;
}
std::vector<std::pair<std::string, int>> phones;
for (int k = 0; k < n; ++k) {
if (!u[k].consonant) {
for (auto& p : u[k].vowel) phones.push_back({p, 0});
phones.back().second = u[k].nasal;
continue;
}
phones.push_back({u[k].phone, 0});
if (!u[k].vowel.empty()) for (auto& p : u[k].vowel) phones.push_back({p, 0});
else if (keep[k]) phones.push_back({"ə", 0});
if (u[k].nasal) phones.back().second = u[k].nasal;
}
std::vector<std::pair<std::string, int>> out;
for (size_t idx = 0; idx < phones.size(); ++idx) {
const auto& [p, nasal] = phones[idx];
const std::string nxt = idx + 1 < phones.size() ? phones[idx + 1].first : std::string();
const auto h = HOMORGANIC.find(nxt);
if (nasal && VOWEL_SET.count(p)) {
if (nasal == 1 && SHORT_VOWELS.count(p) && h != HOMORGANIC.end()) { out.push_back({p, 0}); out.push_back({h->second, 0}); }
else out.push_back({p, 1});
} else if (nasal) {
out.push_back({p, 0});
if (h != HOMORGANIC.end()) out.push_back({h->second, 0});
} else out.push_back({p, 0});
}
std::vector<std::pair<std::string, int>> fixed; // ज्ञ -> ɡ j
for (size_t i = 0; i < out.size();) {
if (i + 1 < out.size() && out[i].first == "dʒ" && out[i + 1].first == "ɲ") { fixed.push_back({"ɡ", 0}); fixed.push_back({"j", 0}); i += 2; }
else fixed.push_back(out[i++]);
}
return fixed;
}
// ------------------------------------------------------------------ text.chunks: [ऀ-ॣ०-ॿ\w]+ | [^\s\w]
struct Chunk { bool punct; U piece; };
std::vector<Chunk> chunks(const U& text) {
std::vector<Chunk> out;
auto in_run = [](char32_t c) { return (c >= 0x900 && c <= 0x963) || (c >= 0x966 && c <= 0x97F) || is_word(c); };
for (size_t i = 0; i < text.size();) {
const char32_t c = text[i];
if (in_run(c)) {
size_t j = i; while (j < text.size() && in_run(text[j])) ++j;
U tok = text.substr(i, j - i);
if (std::any_of(tok.begin(), tok.end(), is_devanagari)) out.push_back({false, tok});
i = j;
} else if (!is_space(c)) {
const std::string s = encode(U(1, c));
if (PUNCTUATION.count(s)) out.push_back({true, U(1, c)});
++i;
} else ++i;
}
return out;
}
bool ends_with_abbreviation(const U& t) { // infer._NOT_AN_END (ASCII, case-insensitive)
if (t.empty() || t.back() != '.') return false;
const U body = t.substr(0, t.size() - 1);
auto word_ascii = [](char32_t c) { return c < 0x80 && ((c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_'); };
static const char* ABBR[] = {"mr", "mrs", "ms", "dr", "prof", "sr", "jr", "st", "vs", "etc", "no", "inc", "ltd", "co",
"e.g", "i.e", "a.m", "p.m"};
for (const char* a : ABBR) {
const size_t L = std::char_traits<char>::length(a);
if (body.size() < L) continue;
bool ok = true;
for (size_t k = 0; k < L; ++k) {
char32_t c = body[body.size() - L + k];
if (c < 0x80) c = char32_t(std::tolower(int(c)));
if (c != char32_t(a[k])) { ok = false; break; }
}
if (!ok) continue;
const size_t start = body.size() - L;
if (start == 0 || !word_ascii(body[start - 1])) return true; // \b before the abbreviation
}
if (!body.empty()) { // \b[A-Z]\. (any single ASCII letter)
const char32_t c = body.back();
const bool letter = c < 0x80 && ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'));
if (letter && (body.size() == 1 || !word_ascii(body[body.size() - 2]))) return true;
}
return false;
}
bool space_or_bom(char32_t c) { return is_space(c) || c == 0xFEFF; }
} // namespace
// ------------------------------------------------------------------ public
std::string normalize_text(const std::string& text) { return encode(normalize_u(decode(text))); }
HindiG2P::HindiG2P(const std::vector<std::string>& symbols) : symbols_(symbols) {}
int HindiG2P::id_of(const std::string& s) const {
for (size_t i = 0; i < symbols_.size(); ++i) if (symbols_[i] == s) return int(i);
return id_of("<unk>");
}
Phonemized HindiG2P::phonemize(const std::string& text) const {
const U norm = normalize_u(decode(text));
std::set<std::string> phone_set(symbols_.begin() + 17, symbols_.end()); // specials (5) + punctuation (12) first
Phonemized r;
r.symbols.push_back("<bos>"); r.stress.push_back(0);
for (const Chunk& ch : chunks(norm)) {
if (ch.punct) {
const std::string p = encode(ch.piece);
if (r.symbols.back() != p) { r.symbols.push_back(p); r.stress.push_back(0); }
continue;
}
const auto word = word_phones(ch.piece);
if (word.empty()) continue;
if (phone_set.count(r.symbols.back()) || r.symbols.back() == "<unk>") { r.symbols.push_back("<sp>"); r.stress.push_back(0); }
for (const auto& [phone, nasal] : word) {
r.symbols.push_back(phone_set.count(phone) ? phone : "<unk>");
r.stress.push_back(nasal ? 1 : 0);
}
}
r.symbols.push_back("<eos>"); r.stress.push_back(0);
bool any = false;
for (auto& s : r.symbols) if (phone_set.count(s) || s == "<unk>") { any = true; break; }
if (!any) throw NothingToPronounce("the text has nothing to pronounce");
for (auto& s : r.symbols) r.ids.push_back(id_of(s));
return r;
}
std::vector<std::string> split_sentences(const std::string& text) {
const U in = decode(text);
U t; // [\s]+ -> " ", strip
bool sp = false;
for (char32_t c : in) {
if (space_or_bom(c)) { sp = true; continue; }
if (sp && !t.empty()) t.push_back(' ');
sp = false; t.push_back(c);
}
// _SENTENCE_END: whitespace after [.!?…] (optionally + one closing quote/bracket) before an optional opening
// quote/bracket and [A-Z0-9] or Devanagari
auto end_mark = [](char32_t c) { return c == '.' || c == '!' || c == '?' || c == 0x2026; };
auto closing = [](char32_t c) { return c == '"' || c == '\'' || c == ')' || c == ']'; };
auto opening = [](char32_t c) { return c == '"' || c == '\'' || c == '(' || c == '['; };
auto starter = [](char32_t c) { return (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || is_devanagari(c); };
std::vector<U> pieces;
size_t start = 0;
for (size_t i = 0; i < t.size(); ++i) {
if (t[i] != ' ') continue;
const bool before = (i >= 1 && end_mark(t[i - 1])) || (i >= 2 && closing(t[i - 1]) && end_mark(t[i - 2]));
size_t k = i + 1;
if (k < t.size() && opening(t[k])) ++k;
const bool after = k < t.size() && starter(t[k]);
if (before && after) { pieces.push_back(t.substr(start, i - start)); start = i + 1; }
}
pieces.push_back(t.substr(start));
std::vector<std::string> parts;
U current;
for (const U& p : pieces) {
current = current.empty() ? p : current + U(1, ' ') + p;
if (!ends_with_abbreviation(current)) { parts.push_back(encode(current)); current.clear(); }
}
if (!current.empty()) parts.push_back(encode(current));
std::vector<std::string> out;
for (auto& p : parts) { const U u = decode(p); if (std::any_of(u.begin(), u.end(), is_alnum)) out.push_back(p); }
return out;
}
std::string finished(const std::string& text) {
U t = decode(text);
while (!t.empty() && space_or_bom(t.back())) t.pop_back();
if (t.empty()) return text;
size_t s = t.size();
while (s > 0 && !space_or_bom(t[s - 1])) --s;
U last = t.substr(s);
bool marked;
if (std::any_of(last.begin(), last.end(), is_alnum)) {
static const U CLOSE = U"\"'”’)]»";
U r = last;
while (!r.empty() && CLOSE.find(r.back()) != U::npos) r.pop_back();
static const U MARKS = U".!?…,;:";
marked = !r.empty() && MARKS.find(r.back()) != U::npos;
} else {
marked = std::any_of(last.begin(), last.end(), [](char32_t c) { return c == '.' || c == '!' || c == '?' || c == 0x2026; }) ||
last == U"," || last == U";" || last == U":";
}
return marked ? text : encode(t) + ".";
}
std::vector<std::string> shorter(const std::string& text) {
const U t = decode(text);
std::vector<U> pieces; // split at whitespace runs after , ; : —
size_t start = 0;
for (size_t i = 0; i < t.size(); ++i) {
if (!is_space(t[i])) continue;
if (i == 0) continue;
const char32_t p = t[i - 1];
if (p == ',' || p == ';' || p == ':' || p == 0x2014) {
size_t j = i; while (j < t.size() && is_space(t[j])) ++j;
pieces.push_back(t.substr(start, i - start));
start = j; i = j - 1;
}
}
pieces.push_back(t.substr(start));
std::vector<std::string> out;
if (pieces.size() > 1) { for (auto& p : pieces) out.push_back(encode(p)); return out; }
std::vector<U> words; start = 0;
for (size_t i = 0; i <= t.size(); ++i)
if (i == t.size() || t[i] == ' ') { words.push_back(t.substr(start, i - start)); start = i + 1; }
if (words.size() < 2) return out;
const size_t half = (words.size() + 1) / 2;
auto join = [&](size_t from, size_t to) { // " ".join(words[from:to])
U s;
for (size_t i = from; i < to; ++i) { if (i > from) s.push_back(' '); s += words[i]; }
return encode(s);
};
out.push_back(join(0, half)); out.push_back(join(half, words.size()));
return out;
}