Blame for python/bhc/skel.py

"""Consonant-skeleton index over the lexicon: a storage-free 'romanization memory'.
Both the Devanagari lexicon word and the romanized input are reduced to a coarse consonant-class string
(vowels, y, aspiration, nukta, halant dropped; n/m/anusvara merged; repeats collapsed). The index is built at load time
from lex.bin (no extra bytes shipped). At query time, lexicon words sharing the input's skeleton (top-K by frequency)
become candidates and are scored by the model (teacher-forced). The JS runtime (release/space/js/aksharalp.mjs) mirrors
these tables exactly -- keep them in sync."""
DEV = {}
for chars, c in (("कखक़ख़", "K"), ("गघग़", "G"), ("चछ", "C"), ("जझज़", "J"), ("टठतथ", "T"), ("डढदधड़ढ़", "D"),
                 ("ङञणनऩमंँ", "N"), ("पफफ़", "P"), ("बभ", "B"), ("रऱऋृ", "R"), ("लळ", "L"), ("व", "V"), ("शषस", "S"),
                 ("हः", "H")):
    for ch in chars:
        if ch != "\u093c": DEV[ch] = c
DEV.pop("\u093c", None)   # nukta: dropped (NFC keeps क़ etc. decomposed)
ROM1 = dict(k="K", q="K", g="G", j="J", z="J", t="T", d="D", n="N", m="N", p="P", f="P", b="B", r="R", l="L", v="V", w="V",
            s="S", h="H")
ASP = set("kgcjtdpbsz")   # consonant + h -> same class (h dropped); 'sh' -> S, 'ch' -> C


def dev_skel(w):
    out = []
    for ch in w:
        c = DEV.get(ch)
        if c and (not out or out[-1] != c): out.append(c)
    return "".join(out)


def rom_skels(w, maxk=4):
    """Skeletons of a roman word, ambiguity expanded (c -> K|S, ch -> C|K, tion -> SN|TN), at most maxk."""
    w = w.lower(); outs = [""]; i = 0; n = len(w)
    while i < n:
        ch = w[i]; step = 1
        if ch in "ts" and w.startswith("ion", i + 1):
            opts = ["SN", "TN"] if ch == "t" else ["SN"]; step = 4
        elif ch == "c":
            nx = w[i + 1] if i + 1 < n else ""
            if nx == "h":
                opts = ["C", "K"]; step = 2
                while i + step < n and w[i + step] in "ch": step += 1      # chh, chch
            elif nx in ("e", "i", "y"): opts = ["S", "K"]
            elif nx == "k": opts = ["K"]; step = 2
            else: opts = ["K", "S"]
        elif ch == "x": opts = ["KS"]
        elif ch in ASP and i + 1 < n and w[i + 1] == "h":
            opts = [ROM1.get(ch, "")]; step = 2
            while i + step < n and w[i + step] == "h": step += 1
        else: opts = [ROM1.get(ch, "")]
        new = []
        for o in outs:
            for p in opts:
                s = o
                for c in p:
                    if not s or s[-1] != c: s += c
                new.append(s)
        outs = list(dict.fromkeys(new))[:maxk]; i += step
    return outs


class SkelIndex:
    def __init__(self, words, uni):
        """words: lexicon list; uni(i) -> unigram log-prob. Buckets keep ids sorted by frequency (desc)."""
        b = {}
        for i, w in enumerate(words):
            k = dev_skel(w)
            if k: b.setdefault(k, []).append(i)
        for k, v in b.items(): v.sort(key=lambda i: (-uni(i), i))
        self.b = b

    def lookup(self, roman, topk=24):
        seen = []; S = set()
        for k in rom_skels(roman):
            for i in self.b.get(k, ())[:topk]:
                if i not in S: S.add(i); seen.append(i)
        return seen


# ---- cheap rule romanizer + edit distance prefilter (rank skeleton-bucket words before model scoring)
_C = dict(zip("कखगघङचछजझञटठडढणतथदधनपफबभमयरलवशषसहळ",
              "k kh g gh n ch chh j jh n t th d dh n t th d dh n p ph b bh m y r l v sh sh s h l".split()))
_V = dict(zip("अआइईउऊऋएऐओऔऑ", "a aa i ee u oo ri e ai o au o".split()))
_M = dict(zip("ािीुूृेैोौॉॅ", "a i ee u oo ri e ai o au o e".split()))


def dev_rom(w):
    out = []; L = len(w)
    for j, ch in enumerate(w):
        if ch in _C:
            out.append(_C[ch]); nx = w[j + 1] if j + 1 < L else ""
            if nx == "\u093c": nx = w[j + 2] if j + 2 < L else ""
            if nx and nx not in _M and nx != "\u094d" and j + 1 < L - 1: out.append("a")
        elif ch in _V: out.append(_V[ch])
        elif ch in _M: out.append(_M[ch])
        elif ch in "ंँ": out.append("n")
        elif ch == "ः": out.append("h")
        elif ch == "\u093c" and out:
            out[-1] = {"k": "q", "kh": "kh", "g": "g", "j": "z", "ph": "f", "d": "r", "dh": "rh"}.get(out[-1], out[-1])
    return "".join(out)


def lev(a, b):
    if len(a) < len(b): a, b = b, a
    prev = list(range(len(b) + 1))
    for i, ca in enumerate(a, 1):
        cur = [i]
        for j, cb in enumerate(b, 1):
            cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb)))
        prev = cur
    return prev[-1]