Download kotlin/Romanizer.kt from cava-reader/cavaalign-v1: direct link, hf CLI and curl.
- Browser
- Download file 4.14 kB
-
https://huggingface.co/cava-reader/cavaalign-v1/resolve/main/kotlin/Romanizer.kt
- Command line
-
hf download hf://cava-reader/cavaalign-v1/kotlin/Romanizer.kt
-
curl -L -o Romanizer.kt https://huggingface.co/cava-reader/cavaalign-v1/resolve/main/kotlin/Romanizer.kt
4.14 kB
| // SPDX-License-Identifier: AGPL-3.0-or-later | |
| package dev.micr0.cavaalign | |
| import java.text.Normalizer | |
| /** | |
| * Latin or Cyrillic text -> the model's 29-class romanized alphabet. | |
| * | |
| * Port of the `table` engine in `cavaalign/romanize.py`, character for character. It does not | |
| * use `android.icu.text.Transliterator`, whose output can change with the Android version. | |
| */ | |
| object Romanizer { | |
| private const val ALLOWED = "abcdefghijklmnopqrstuvwxyz'" | |
| private val LATIN_SPECIAL = mapOf( | |
| 'ä' to "ae", 'å' to "aa", 'ö' to "oe", 'ø' to "oe", 'ü' to "ue", 'œ' to "oe", 'æ' to "ae", | |
| 'ß' to "ss", 'ç' to "s", 'ş' to "sh", 'ș' to "sh", 'ț' to "ts", 'þ' to "th", 'ð' to "th", | |
| 'ł' to "l", 'đ' to "d", 'ı' to "i", 'ŀ' to "l", 'ij' to "ij", | |
| ) | |
| private val RU = mapOf( | |
| 'а' to "a", 'б' to "b", 'в' to "v", 'г' to "g", 'д' to "d", 'е' to "e", 'ё' to "yo", | |
| 'ж' to "zh", 'з' to "z", 'и' to "i", 'й' to "y", 'к' to "k", 'л' to "l", 'м' to "m", | |
| 'н' to "n", 'о' to "o", 'п' to "p", 'р' to "r", 'с' to "s", 'т' to "t", 'у' to "u", | |
| 'ф' to "f", 'х' to "kh", 'ц' to "ts", 'ч' to "ch", 'ш' to "sh", 'щ' to "shch", 'ъ' to "", | |
| 'ы' to "y", 'ь' to "", 'э' to "e", 'ю' to "yu", 'я' to "ya", 'і' to "i", 'ї' to "yi", | |
| 'є' to "ye", 'ґ' to "g", 'ў' to "w", | |
| ) | |
| // `е` is context-dependent; see `_ru_e` in romanize.py. | |
| private val RU_YE_AFTER = "аиоуъь".toSet() | |
| private fun ruE(prev: Char?, prevOut: String, next: Char?): String = when { | |
| prev == null -> if (next == 'е' || next == 'ё') "e" else "ye" | |
| prev == 'е' -> if (prevOut == "ye") "e" else "ye" | |
| prev in RU_YE_AFTER -> "ye" | |
| else -> "e" | |
| } | |
| private val APOSTROPHES = mapOf('’' to '\'', '‘' to '\'', 'ʼ' to '\'', '`' to '\'') | |
| private fun foldLatin(ch: Char): String { | |
| LATIN_SPECIAL[ch]?.let { return it } | |
| val d = Normalizer.normalize(ch.toString(), Normalizer.Form.NFKD) | |
| val sb = StringBuilder() | |
| for (c in d) if (Character.getType(c) != Character.NON_SPACING_MARK.toInt()) sb.append(c) | |
| return sb.toString() | |
| } | |
| /** One word (no spaces) -> romanized characters. May return "". */ | |
| fun word(word: String): String { | |
| val w = word.lowercase().map { APOSTROPHES[it] ?: it } | |
| val out = ArrayList<String>(w.size) | |
| var prev: Char? = null | |
| // word-final "ий" -> "y" ("синий" -> "siny"); "ий" elsewhere is "iy" | |
| val finalIy = w.size >= 2 && w[w.size - 2] == 'и' && w[w.size - 1] == 'й' | |
| for ((i, ch) in w.withIndex()) { | |
| out.add(when { | |
| finalIy && i == w.size - 2 -> "" | |
| ch == 'е' -> ruE(prev, out.lastOrNull() ?: "", w.getOrNull(i + 1)) | |
| RU.containsKey(ch) -> RU.getValue(ch) | |
| else -> foldLatin(ch) | |
| }) | |
| prev = ch | |
| } | |
| val res = StringBuilder() | |
| for (part in out) for (c in part) if (ALLOWED.indexOf(c) >= 0) res.append(c) | |
| return res.toString() | |
| } | |
| /** A run of one word's characters, possibly with spaces ("eighteen twelve"). */ | |
| private fun piece(s: String): String = | |
| s.split(' ').map { word(it) }.filter { it.isNotEmpty() }.joinToString(" ") | |
| /** | |
| * Romanize an [Expansion] word by word, keeping its words and word indices. | |
| * | |
| * Inter-word spaces are kept; a word that romanizes to nothing becomes `*` (`<unk>`). | |
| */ | |
| fun expansion(exp: Expansion): Expansion { | |
| val src = exp.text | |
| val c2w = exp.charToWord | |
| val out = StringBuilder() | |
| val oc2w = ArrayList<Int>(src.length) | |
| var i = 0 | |
| while (i < src.length) { | |
| val wi = c2w[i] | |
| var j = i | |
| while (j < src.length && c2w[j] == wi) j++ | |
| val chars = if (wi < 0) src.substring(i, j) else piece(src.substring(i, j)).ifEmpty { "*" } | |
| out.append(chars) | |
| repeat(chars.length) { oc2w.add(wi) } | |
| i = j | |
| } | |
| return Expansion(out.toString(), oc2w.toIntArray(), exp.words) | |
| } | |
| } | |