File size: 8,425 Bytes
9aba9fd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 | """Casual / chat-romanization keys (AksharaLP v2).
Two lossy keys are computed for a roman token and for a Devanagari word; a
casual spelling matches the intended word when the keys collide.
K1: consonant classes (aspiration, nukta, sibilants merged), vowel classes
(length merged, ai->e, au->o), every 'a'/schwa removed except word
initial, nasal dropped, repeats collapsed.
K2: K1 with all vowels removed (chat consonant-only spellings: bht, thk).
The index is built at load time from the lexicon (0 bytes stored).
Same algorithm is ported 1:1 to JS (release/runtime) - keep in sync.
"""
import re
ASP = set("kgjtdpbc")
# Devanagari -> key class (nukta ignored except ड़/ढ़ -> R)
_DC = {}
for chars, k in [("कख", "k"), ("गघ", "g"), ("चछ", "c"), ("जझ", "j"), ("टठतथ", "t"),
("डढदध", "d"), ("णनऩ", "n"), ("पफ", "p"), ("बभ", "b"), ("म", "m"),
("य", "y"), ("रऱ", "r"), ("लळ", "l"), ("व", "v"), ("शषस", "s"), ("ह", "h")]:
for c in chars: _DC[c] = k
for c, k in zip("\u0958\u0959\u095a\u095b\u095c\u095d\u095e\u095f", "kkgjRRpy"): _DC[c] = k
_DV = {}
for chars, k in [("अआा", "a"), ("इईिी", "i"), ("उऊुू", "u"), ("एऐेैॅऍ", "e"), ("ओऔऑोौॉ", "o"), ("ऋृ", "r")]:
for c in chars: _DV[c] = k
VOW = set("aeiou")
def _fin(s):
# drop non-initial a, collapse repeats
out = []
for i, ch in enumerate(s):
if ch == "a" and i > 0: continue
if out and out[-1] == ch: continue
out.append(ch)
return "".join(out)
def k2_of(k1):
s = k1[0] + "".join(c for c in k1[1:] if c not in VOW) if k1 else ""
if s and s[0] in VOW: s = s[1:] or s
out = []
for ch in s:
if out and out[-1] == ch: continue
out.append(ch)
return "".join(out)
def dev_keys(w):
"""K1 keys of a Devanagari word (2 if it has ड़/ढ़: as d and as r)."""
raw = []
for i, ch in enumerate(w):
if ch == "\u093c":
if raw and raw[-1] == "d" and i > 0 and w[i - 1] in "डढ": raw[-1] = "R"
continue
if ch in _DC: raw.append(_DC[ch])
elif ch in _DV: raw.append(_DV[ch])
s = "".join(raw)
if not s: return []
ks = {_fin(s.replace("R", "d")), _fin(s.replace("R", "r"))}
return sorted(k for k in ks if k)
def _norm(t):
t = t.lower()
t = re.sub(r"[^a-z]", "", t)
return t
def rom_keys(t, cap=12):
"""K1 keys of a roman token, expanded over ambiguous casual rules."""
t = _norm(t)
if not t: return [], []
t = re.sub(r"(.)\1{2,}", r"\1", t) # emphasis repeats
# deterministic letter merges
t = t.replace("chh", "c").replace("cch", "c").replace("ch", "c").replace("sh", "s")
t = t.replace("x", "ks").replace("q", "k").replace("z", "j").replace("f", "p").replace("w", "v")
# (unit list) each position: list of alternatives
alts = []
i = 0; L = len(t)
while i < L:
c = t[i]; nx = t[i + 1] if i + 1 < L else ""
prev = t[i - 1] if i > 0 else ""
if c == "h" and prev in ASP and i > 0:
alts.append(("", "h")); i += 1; continue # aspiration / separate h
if nx and c in "ae" and nx in "iy" and (i + 2 >= L or t[i + 2] not in VOW):
if c == "a" and nx == "i": alts.append(("e", "ai")); i += 2; continue
if c == "a" and nx == "y" and i + 2 >= L: alts.append(("e",)); i += 2; continue
if c == "e" and nx == "y" and i + 2 >= L: alts.append(("e",)); i += 2; continue
if nx and c in "ao" and nx in "uvw" and (i + 2 >= L or t[i + 2] not in VOW) and c + nx != "av":
alts.append(("o", c + "u")); i += 2; continue
if c == "o" and nx == "u": alts.append(("o", "u")); i += 2; continue
if c == "e" and nx == "e": alts.append(("i",)); i += 2; continue
if c == "o" and nx == "o": alts.append(("u", "o")); i += 2; continue
if c == "e" and nx == "i" and (i + 2 >= L or t[i + 2] not in VOW): alts.append(("e",)); i += 2; continue
if c == "e" and nx == "h" and i + 2 < L and t[i + 2] not in VOW and i > 0: alts.append(("e", "")); i += 1; continue
if nx and c == "i" and nx in "aeou" and i > 0: alts.append(("y", "i")); i += 1; continue
if c == "y" and i == L - 1 and prev and prev not in VOW: alts.append(("e", "i")); i += 1; continue
if c == "n" and prev in VOW and i > 0 and (i == L - 1 or (nx and nx not in VOW and nx != "y")):
alts.append(("n", "")); i += 1; continue
if c in "iu" and prev == "r" and i >= 2 and t[i - 2] not in VOW:
alts.append((c, "")); i += 1; continue
alts.append((c,)); i += 1
outs = [""]
for a in alts:
nxt = []
for o in outs:
for x in a: nxt.append(o + x)
outs = list(dict.fromkeys(nxt))[:64]
k1 = []; seen = set()
for o in outs:
k = _fin(o)
if k and k not in seen: seen.add(k); k1.append(k)
k1 = k1[:cap]
k2 = list(dict.fromkeys(k2_of(k) for k in k1 if k2_of(k)))
return k1, k2
# Irregular chat abbreviations (not derivable by rules). Values: candidate natives in prior order.
ABBR = {
"h": ["है", "हैं"], "hy": ["है"], "hn": ["हाँ", "हैं"], "hm": ["हम", "हम्म"],
"k": ["के", "की", "ओके"], "ki": ["की", "कि"], "n": ["ना", "न"], "ni": ["नहीं", "नी"], "nai": ["नहीं", "नई"],
"nhi": ["नहीं"], "nhe": ["नहीं"], "nahi": ["नहीं"], "nai": ["नहीं", "नई"],
"kr": ["कर"], "krna": ["करना"], "kro": ["करो"], "tm": ["तुम"], "tu": ["तू"], "mai": ["मैं", "में"],
"me": ["में", "मैं"], "m": ["मैं", "में"], "thk": ["ठीक"], "tk": ["ठीक", "तक"],
"yr": ["यार"], "yrr": ["यार"], "plz": ["प्लीज़"], "pls": ["प्लीज़"], "plzz": ["प्लीज़"],
"tq": ["थैंक यू"], "ty": ["थैंक यू"], "thx": ["थैंक्स"], "thnx": ["थैंक्स"], "thnks": ["थैंक्स"],
"gn": ["गुड नाइट"], "gm": ["गुड मॉर्निंग"], "ok": ["ओके"], "okk": ["ओके"], "kk": ["ओके"],
"sry": ["सॉरी"], "bro": ["ब्रो"], "msg": ["मैसेज"], "bday": ["बर्थडे"], "clg": ["कॉलेज"],
"ofc": ["ऑफ़िस"], "pic": ["पिक"], "bhai": ["भाई"], "bht": ["बहुत"], "bhot": ["बहुत"], "bohot": ["बहुत"],
"pta": ["पता"], "kb": ["कब"], "jb": ["जब"], "sb": ["सब"], "mtlb": ["मतलब"], "kch": ["कुछ"], "kuch": ["कुछ"],
}
class CasualIndex:
"""K1/K2 -> lexicon ids sorted by frequency (top `top_n` lexicon words only)."""
def __init__(self, words, uni=None, top_n=300000, per_key=8):
order = range(len(words))
if uni is not None:
u = uni if callable(uni) else uni.__getitem__
order = sorted(order, key=lambda i: (-u(i), i))
self.k1 = {}; self.k2 = {}
for r, i in enumerate(order):
if r >= top_n: break
for k in dev_keys(words[i]):
b = self.k1.setdefault(k, [])
if len(b) < per_key and i not in b: b.append(i)
k2 = k2_of(k)
if k2:
b2 = self.k2.setdefault(k2, [])
if len(b2) < per_key and i not in b2: b2.append(i)
def lookup(self, roman, n1=6, n2=4):
k1, k2 = rom_keys(roman)
r1 = []; r2 = []
for k in k1:
for i in self.k1.get(k, ()):
if i not in r1: r1.append(i)
for k in k2:
for i in self.k2.get(k, ()):
if i not in r2: r2.append(i)
return r1[:n1 * 2], r2[:n2 * 2]
def squeeze_in(t, mode=0):
"""Emphasis-repeat normalization of the model input. mode 0: off; 1: 3+ -> 1; 2: 3+ -> 2 for a/e/o, 1 otherwise."""
if not mode: return t
if mode == 1: return re.sub(r"([a-z])\1{2,}", r"\1", t)
return re.sub(r"([a-z])\1{2,}", lambda m: m.group(1) * (2 if m.group(1) in "aeo" else 1), t)
def squeeze(t):
"""Normalize emphasis repeats (3+ same letters -> keep 1, e.g. pleaseee->please)."""
return re.sub(r"([a-zA-Z])\1{2,}", r"\1", t)
|