| """Consonant-skeleton index over the lexicon: a storage-free 'romanization memory'. |
| Both the Devanagari lexicon word and the romanized input are reduced to a coarse consonant-class string |
| (vowels, y, aspiration, nukta, halant dropped; n/m/anusvara merged; repeats collapsed). The index is built at load time |
| from lex.bin (no extra bytes shipped). At query time, lexicon words sharing the input's skeleton (top-K by frequency) |
| become candidates and are scored by the model (teacher-forced). The JS runtime (release/space/js/aksharalp.mjs) mirrors |
| these tables exactly -- keep them in sync.""" |
| DEV = {} |
| for chars, c in (("कखक़ख़", "K"), ("गघग़", "G"), ("चछ", "C"), ("जझज़", "J"), ("टठतथ", "T"), ("डढदधड़ढ़", "D"), |
| ("ङञणनऩमंँ", "N"), ("पफफ़", "P"), ("बभ", "B"), ("रऱऋृ", "R"), ("लळ", "L"), ("व", "V"), ("शषस", "S"), |
| ("हः", "H")): |
| for ch in chars: |
| if ch != "\u093c": DEV[ch] = c |
| DEV.pop("\u093c", None) |
| ROM1 = dict(k="K", q="K", g="G", j="J", z="J", t="T", d="D", n="N", m="N", p="P", f="P", b="B", r="R", l="L", v="V", w="V", |
| s="S", h="H") |
| ASP = set("kgcjtdpbsz") |
|
|
|
|
| def dev_skel(w): |
| out = [] |
| for ch in w: |
| c = DEV.get(ch) |
| if c and (not out or out[-1] != c): out.append(c) |
| return "".join(out) |
|
|
|
|
| def rom_skels(w, maxk=4): |
| """Skeletons of a roman word, ambiguity expanded (c -> K|S, ch -> C|K, tion -> SN|TN), at most maxk.""" |
| w = w.lower(); outs = [""]; i = 0; n = len(w) |
| while i < n: |
| ch = w[i]; step = 1 |
| if ch in "ts" and w.startswith("ion", i + 1): |
| opts = ["SN", "TN"] if ch == "t" else ["SN"]; step = 4 |
| elif ch == "c": |
| nx = w[i + 1] if i + 1 < n else "" |
| if nx == "h": |
| opts = ["C", "K"]; step = 2 |
| while i + step < n and w[i + step] in "ch": step += 1 |
| elif nx in ("e", "i", "y"): opts = ["S", "K"] |
| elif nx == "k": opts = ["K"]; step = 2 |
| else: opts = ["K", "S"] |
| elif ch == "x": opts = ["KS"] |
| elif ch in ASP and i + 1 < n and w[i + 1] == "h": |
| opts = [ROM1.get(ch, "")]; step = 2 |
| while i + step < n and w[i + step] == "h": step += 1 |
| else: opts = [ROM1.get(ch, "")] |
| new = [] |
| for o in outs: |
| for p in opts: |
| s = o |
| for c in p: |
| if not s or s[-1] != c: s += c |
| new.append(s) |
| outs = list(dict.fromkeys(new))[:maxk]; i += step |
| return outs |
|
|
|
|
| class SkelIndex: |
| def __init__(self, words, uni): |
| """words: lexicon list; uni(i) -> unigram log-prob. Buckets keep ids sorted by frequency (desc).""" |
| b = {} |
| for i, w in enumerate(words): |
| k = dev_skel(w) |
| if k: b.setdefault(k, []).append(i) |
| for k, v in b.items(): v.sort(key=lambda i: (-uni(i), i)) |
| self.b = b |
|
|
| def lookup(self, roman, topk=24): |
| seen = []; S = set() |
| for k in rom_skels(roman): |
| for i in self.b.get(k, ())[:topk]: |
| if i not in S: S.add(i); seen.append(i) |
| return seen |
|
|
|
|
| |
| _C = dict(zip("कखगघङचछजझञटठडढणतथदधनपफबभमयरलवशषसहळ", |
| "k kh g gh n ch chh j jh n t th d dh n t th d dh n p ph b bh m y r l v sh sh s h l".split())) |
| _V = dict(zip("अआइईउऊऋएऐओऔऑ", "a aa i ee u oo ri e ai o au o".split())) |
| _M = dict(zip("ािीुूृेैोौॉॅ", "a i ee u oo ri e ai o au o e".split())) |
|
|
|
|
| def dev_rom(w): |
| out = []; L = len(w) |
| for j, ch in enumerate(w): |
| if ch in _C: |
| out.append(_C[ch]); nx = w[j + 1] if j + 1 < L else "" |
| if nx == "\u093c": nx = w[j + 2] if j + 2 < L else "" |
| if nx and nx not in _M and nx != "\u094d" and j + 1 < L - 1: out.append("a") |
| elif ch in _V: out.append(_V[ch]) |
| elif ch in _M: out.append(_M[ch]) |
| elif ch in "ंँ": out.append("n") |
| elif ch == "ः": out.append("h") |
| elif ch == "\u093c" and out: |
| out[-1] = {"k": "q", "kh": "kh", "g": "g", "j": "z", "ph": "f", "d": "r", "dh": "rh"}.get(out[-1], out[-1]) |
| return "".join(out) |
|
|
|
|
| def lev(a, b): |
| if len(a) < len(b): a, b = b, a |
| prev = list(range(len(b) + 1)) |
| for i, ca in enumerate(a, 1): |
| cur = [i] |
| for j, cb in enumerate(b, 1): |
| cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb))) |
| prev = cur |
| return prev[-1] |
|
|
