File size: 7,671 Bytes
8f8b0fa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Any Latin or Cyrillic text -> the model's 29-class romanized alphabet.

The model expects uroman's romanization conventions. Three engines are available:

* ``table`` (default): a small context-aware table plus Unicode decomposition. Deterministic
  and dependency-free; `kotlin/Romanizer.kt` implements the same table. It matches uroman on
  Latin-script languages and Russian.
* ``uroman``: the reference convention. Requires the `uroman` package.
* ``icu``: ICU ``Any-Latin; Latin-ASCII``. Requires PyICU. Output can differ between ICU
  versions, so it is provided for comparison only.

Romanization runs per word, after `align.expand_text`, so word indices are unchanged. A word
that romanizes to nothing (e.g. a lone "—") becomes the `*` wildcard (`<unk>`).
"""
from __future__ import annotations

import unicodedata
from functools import lru_cache

from .align import Expansion

# ISO 639-1 -> the ISO 639-3 code uroman takes.
LANG3 = {"en": "eng", "de": "deu", "fr": "fra", "es": "spa", "it": "ita", "pt": "por",
         "nl": "nld", "pl": "pol", "ru": "rus"}

ALLOWED = set("abcdefghijklmnopqrstuvwxyz'")

# Latin letters whose romanization is not "strip the accent" (uroman's values).
LATIN_SPECIAL = {
    "ä": "ae", "å": "aa", "ö": "oe", "ø": "oe", "ü": "ue", "œ": "oe", "æ": "ae", "ß": "ss",
    "ç": "s", "ş": "sh", "ș": "sh", "ț": "ts", "þ": "th", "ð": "th", "ł": "l", "đ": "d",
    "ı": "i", "ŀ": "l", "ij": "ij",
}

# Cyrillic, following uroman for Russian. `е` depends on context: "ye" at word start and after
# а/и/о/у/ъ/ь, otherwise "e"; at word start before е/ё it is "e"; after е it alternates
# ("ее" -> "eye"). Word-final "ий" is "y" ("синий" -> "siny").
RU = {"а": "a", "б": "b", "в": "v", "г": "g", "д": "d", "е": "e", "ё": "yo", "ж": "zh",
      "з": "z", "и": "i", "й": "y", "к": "k", "л": "l", "м": "m", "н": "n", "о": "o", "п": "p",
      "р": "r", "с": "s", "т": "t", "у": "u", "ф": "f", "х": "kh", "ц": "ts", "ч": "ch",
      "ш": "sh", "щ": "shch", "ъ": "", "ы": "y", "ь": "", "э": "e", "ю": "yu", "я": "ya",
      "і": "i", "ї": "yi", "є": "ye", "ґ": "g", "ў": "w"}
RU_YE_AFTER = set("аиоуъь")


def _ru_e(prev: str, prev_out: str, nxt: str) -> str:
    if not prev:
        return "e" if nxt in ("е", "ё") else "ye"
    if prev == "е":
        return "e" if prev_out == "ye" else "ye"
    return "ye" if prev in RU_YE_AFTER else "e"


APOSTROPHES = {"’": "'", "‘": "'", "ʼ": "'", "`": "'"}


def _fold_latin(ch: str) -> str:
    if ch in LATIN_SPECIAL:
        return LATIN_SPECIAL[ch]
    # strip combining marks (Unicode category Mn)
    base = "".join(c for c in unicodedata.normalize("NFKD", ch)
                   if unicodedata.category(c) != "Mn")
    return base


@lru_cache(maxsize=200_000)
def _table_word(word: str) -> str:
    w = [APOSTROPHES.get(c, c) for c in word.lower()]
    out = []
    prev = ""
    final_iy = len(w) >= 2 and w[-2] == "и" and w[-1] == "й"
    for i, ch in enumerate(w):
        if final_iy and i == len(w) - 2:
            out.append("")                  # word-final "ий" -> "y": the й supplies it
        elif ch == "е":
            out.append(_ru_e(prev, out[-1] if out else "", w[i + 1] if i + 1 < len(w) else ""))
        elif ch in RU:
            out.append(RU[ch])
        else:
            out.append(_fold_latin(ch))
        prev = ch
    return "".join(c for c in "".join(out) if c in ALLOWED)


_UROMAN = None
_ICU = None


def _uroman_word(word: str, lang: str) -> str:
    global _UROMAN
    if _UROMAN is None:
        import uroman
        _UROMAN = uroman.Uroman()
    r = _UROMAN.romanize_string(word, lcode=LANG3.get(lang))
    return "".join(c for c in r.lower() if c in ALLOWED)


def _icu_word(word: str, lang: str) -> str:
    global _ICU
    if _ICU is None:
        import icu
        _ICU = icu.Transliterator.createInstance("Any-Latin; Latin-ASCII")
    return "".join(c for c in _ICU.transliterate(word).lower() if c in ALLOWED)


def romanize_word(word: str, lang: str = "en", engine: str = "table") -> str:
    """One word (no spaces) -> characters of the romanized alphabet. May return ""."""
    if engine == "table":
        return _table_word(word)
    if engine == "uroman":
        return _uroman_word(word, lang)
    if engine == "icu":
        return _icu_word(word, lang)
    raise ValueError(f"unknown romanizer {engine!r}")


def _romanize_piece(piece: str, lang: str, engine: str) -> str:
    """A run of one word's expanded characters, possibly with internal spaces ("eighteen twelve")."""
    parts = [romanize_word(p, lang, engine) for p in piece.split(" ")]
    return " ".join(p for p in parts if p)


def number_readings_lang(digits: str, lang: str) -> list[str]:
    """Spoken forms of a digit run in `lang`, likeliest first (requires num2words).

    For 4-digit values between 1000 and 2100 the year reading comes first where num2words has one.
    Languages num2words does not support keep the digits, which romanize to the `*` wildcard.
    """
    from num2words import num2words
    n = int(digits)
    out = []
    try:
        card = num2words(n, lang=lang)
    except NotImplementedError:
        card = digits
    if len(digits) == 4 and 1000 <= n <= 2100:
        try:
            yr = num2words(n, lang=lang, to="year")
            if yr and yr != card:
                out.append(yr)
        except (NotImplementedError, TypeError, AttributeError):
            pass
    out.append(card)
    return [o.replace("-", " ").replace(",", "") for o in out]


def romanize_expansion(exp: Expansion, lang: str = "en", engine: str = "table") -> Expansion:
    """Map an [Expansion] onto the romanized alphabet, keeping words, word indices and groups.

    Each word is romanized as one unit (romanization is context-dependent) and `char_to_word`
    is rebuilt so every output character still points at its source word.
    """
    src, c2w = exp.text, exp.char_to_word
    gstart = {g[0]: g for g in exp.groups}
    out: list[str] = []
    o_c2w: list[int] = []
    groups: list[tuple[int, int, tuple[str, ...]]] = []

    def put(chars: str, wi: int) -> None:
        out.extend(chars)
        o_c2w.extend([wi] * len(chars))

    i, n = 0, len(src)
    while i < n:
        wi = c2w[i]
        j = i
        while j < n and c2w[j] == wi:
            j += 1
        if wi < 0:
            put(src[i:j], -1)
            i = j
            continue
        start_len = len(out)
        k = i
        while k < j:
            if k in gstart:
                a, b, readings = gstart[k]
                rom = tuple(r for r in (_romanize_piece(x, lang, engine) for x in readings) if r)
                rom = tuple(dict.fromkeys(rom)) or ("*",)
                s0 = len(out)
                put(rom[0], wi)
                if len(rom) > 1:
                    groups.append((s0, len(out), rom))
                k = b
                continue
            nxt = min([g for g in gstart if k < g < j] or [j])
            put(_romanize_piece(src[k:nxt], lang, engine), wi)
            k = nxt
        if len(out) == start_len:
            put("*", wi)                    # unromanizable word: a wildcard, never a hole
        i = j

    return Expansion(text="".join(out), char_to_word=o_c2w, words=list(exp.words), groups=groups)


def prepare_text(text: str, lang: str = "en", engine: str = "table") -> Expansion:
    """Text -> romanized [Expansion] ready for `expansion_to_tokens`."""
    from .align import expand_text
    return romanize_expansion(expand_text(text, lang=lang), lang=lang, engine=engine)