cavaalign-v1 / kotlin /Romanizer.kt
micr0-dev's picture
CavaAlign v1
8f8b0fa
Raw History Blame Contribute Delete
4.14 kB
// SPDX-License-Identifier: AGPL-3.0-or-later
package dev.micr0.cavaalign
import java.text.Normalizer
/**
* Latin or Cyrillic text -> the model's 29-class romanized alphabet.
*
* Port of the `table` engine in `cavaalign/romanize.py`, character for character. It does not
* use `android.icu.text.Transliterator`, whose output can change with the Android version.
*/
object Romanizer {
private const val ALLOWED = "abcdefghijklmnopqrstuvwxyz'"
private val LATIN_SPECIAL = mapOf(
'ä' to "ae", 'å' to "aa", 'ö' to "oe", 'ø' to "oe", 'ü' to "ue", 'œ' to "oe", 'æ' to "ae",
'ß' to "ss", 'ç' to "s", 'ş' to "sh", 'ș' to "sh", 'ț' to "ts", 'þ' to "th", 'ð' to "th",
'ł' to "l", 'đ' to "d", 'ı' to "i", 'ŀ' to "l", 'ij' to "ij",
)
private val RU = mapOf(
'а' to "a", 'б' to "b", 'в' to "v", 'г' to "g", 'д' to "d", 'е' to "e", 'ё' to "yo",
'ж' to "zh", 'з' to "z", 'и' to "i", 'й' to "y", 'к' to "k", 'л' to "l", 'м' to "m",
'н' to "n", 'о' to "o", 'п' to "p", 'р' to "r", 'с' to "s", 'т' to "t", 'у' to "u",
'ф' to "f", 'х' to "kh", 'ц' to "ts", 'ч' to "ch", 'ш' to "sh", 'щ' to "shch", 'ъ' to "",
'ы' to "y", 'ь' to "", 'э' to "e", 'ю' to "yu", 'я' to "ya", 'і' to "i", 'ї' to "yi",
'є' to "ye", 'ґ' to "g", 'ў' to "w",
)
// `е` is context-dependent; see `_ru_e` in romanize.py.
private val RU_YE_AFTER = "аиоуъь".toSet()
private fun ruE(prev: Char?, prevOut: String, next: Char?): String = when {
prev == null -> if (next == 'е' || next == 'ё') "e" else "ye"
prev == 'е' -> if (prevOut == "ye") "e" else "ye"
prev in RU_YE_AFTER -> "ye"
else -> "e"
}
private val APOSTROPHES = mapOf('’' to '\'', '‘' to '\'', 'ʼ' to '\'', '`' to '\'')
private fun foldLatin(ch: Char): String {
LATIN_SPECIAL[ch]?.let { return it }
val d = Normalizer.normalize(ch.toString(), Normalizer.Form.NFKD)
val sb = StringBuilder()
for (c in d) if (Character.getType(c) != Character.NON_SPACING_MARK.toInt()) sb.append(c)
return sb.toString()
}
/** One word (no spaces) -> romanized characters. May return "". */
fun word(word: String): String {
val w = word.lowercase().map { APOSTROPHES[it] ?: it }
val out = ArrayList<String>(w.size)
var prev: Char? = null
// word-final "ий" -> "y" ("синий" -> "siny"); "ий" elsewhere is "iy"
val finalIy = w.size >= 2 && w[w.size - 2] == 'и' && w[w.size - 1] == 'й'
for ((i, ch) in w.withIndex()) {
out.add(when {
finalIy && i == w.size - 2 -> ""
ch == 'е' -> ruE(prev, out.lastOrNull() ?: "", w.getOrNull(i + 1))
RU.containsKey(ch) -> RU.getValue(ch)
else -> foldLatin(ch)
})
prev = ch
}
val res = StringBuilder()
for (part in out) for (c in part) if (ALLOWED.indexOf(c) >= 0) res.append(c)
return res.toString()
}
/** A run of one word's characters, possibly with spaces ("eighteen twelve"). */
private fun piece(s: String): String =
s.split(' ').map { word(it) }.filter { it.isNotEmpty() }.joinToString(" ")
/**
* Romanize an [Expansion] word by word, keeping its words and word indices.
*
* Inter-word spaces are kept; a word that romanizes to nothing becomes `*` (`<unk>`).
*/
fun expansion(exp: Expansion): Expansion {
val src = exp.text
val c2w = exp.charToWord
val out = StringBuilder()
val oc2w = ArrayList<Int>(src.length)
var i = 0
while (i < src.length) {
val wi = c2w[i]
var j = i
while (j < src.length && c2w[j] == wi) j++
val chars = if (wi < 0) src.substring(i, j) else piece(src.substring(i, j)).ifEmpty { "*" }
out.append(chars)
repeat(chars.length) { oc2w.add(wi) }
i = j
}
return Expansion(out.toString(), oc2w.toIntArray(), exp.words)
}
}