fix: ship bhc/casual.py (translit2.py imports it; the python/ tree could not be imported)

python/bhc/casual.py

@@ -0,0 +1,174 @@
+"""Casual / chat-romanization keys (AksharaLP v2).
+
+Two lossy keys are computed for a roman token and for a Devanagari word; a
+casual spelling matches the intended word when the keys collide.
+  K1: consonant classes (aspiration, nukta, sibilants merged), vowel classes
+      (length merged, ai->e, au->o), every 'a'/schwa removed except word
+      initial, nasal dropped, repeats collapsed.
+  K2: K1 with all vowels removed (chat consonant-only spellings: bht, thk).
+The index is built at load time from the lexicon (0 bytes stored).
+Same algorithm is ported 1:1 to JS (release/runtime) - keep in sync.
+"""
+import re
+
+ASP = set("kgjtdpbc")
+# Devanagari -> key class (nukta ignored except ड़/ढ़ -> R)
+_DC = {}
+for chars, k in [("कख", "k"), ("गघ", "g"), ("चछ", "c"), ("जझ", "j"), ("टठतथ", "t"),
+                 ("डढदध", "d"), ("णनऩ", "n"), ("पफ", "p"), ("बभ", "b"), ("म", "m"),
+                 ("य", "y"), ("रऱ", "r"), ("लळ", "l"), ("व", "v"), ("शषस", "s"), ("ह", "h")]:
+    for c in chars: _DC[c] = k
+for c, k in zip("\u0958\u0959\u095a\u095b\u095c\u095d\u095e\u095f", "kkgjRRpy"): _DC[c] = k
+_DV = {}
+for chars, k in [("अआा", "a"), ("इईिी", "i"), ("उऊुू", "u"), ("एऐेैॅऍ", "e"), ("ओऔऑोौॉ", "o"), ("ऋृ", "r")]:
+    for c in chars: _DV[c] = k
+VOW = set("aeiou")
+
+
+def _fin(s):
+    # drop non-initial a, collapse repeats
+    out = []
+    for i, ch in enumerate(s):
+        if ch == "a" and i > 0: continue
+        if out and out[-1] == ch: continue
+        out.append(ch)
+    return "".join(out)
+
+
+def k2_of(k1):
+    s = k1[0] + "".join(c for c in k1[1:] if c not in VOW) if k1 else ""
+    if s and s[0] in VOW: s = s[1:] or s
+    out = []
+    for ch in s:
+        if out and out[-1] == ch: continue
+        out.append(ch)
+    return "".join(out)
+
+
+def dev_keys(w):
+    """K1 keys of a Devanagari word (2 if it has ड़/ढ़: as d and as r)."""
+    raw = []
+    for i, ch in enumerate(w):
+        if ch == "\u093c":
+            if raw and raw[-1] == "d" and i > 0 and w[i - 1] in "डढ": raw[-1] = "R"
+            continue
+        if ch in _DC: raw.append(_DC[ch])
+        elif ch in _DV: raw.append(_DV[ch])
+    s = "".join(raw)
+    if not s: return []
+    ks = {_fin(s.replace("R", "d")), _fin(s.replace("R", "r"))}
+    return sorted(k for k in ks if k)
+
+
+def _norm(t):
+    t = t.lower()
+    t = re.sub(r"[^a-z]", "", t)
+    return t
+
+
+def rom_keys(t, cap=12):
+    """K1 keys of a roman token, expanded over ambiguous casual rules."""
+    t = _norm(t)
+    if not t: return [], []
+    t = re.sub(r"(.)\1{2,}", r"\1", t)          # emphasis repeats
+    # deterministic letter merges
+    t = t.replace("chh", "c").replace("cch", "c").replace("ch", "c").replace("sh", "s")
+    t = t.replace("x", "ks").replace("q", "k").replace("z", "j").replace("f", "p").replace("w", "v")
+    # (unit list) each position: list of alternatives
+    alts = []
+    i = 0; L = len(t)
+    while i < L:
+        c = t[i]; nx = t[i + 1] if i + 1 < L else ""
+        prev = t[i - 1] if i > 0 else ""
+        if c == "h" and prev in ASP and i > 0:
+            alts.append(("", "h")); i += 1; continue          # aspiration / separate h
+        if nx and c in "ae" and nx in "iy" and (i + 2 >= L or t[i + 2] not in VOW):
+            if c == "a" and nx == "i": alts.append(("e", "ai")); i += 2; continue
+            if c == "a" and nx == "y" and i + 2 >= L: alts.append(("e",)); i += 2; continue
+            if c == "e" and nx == "y" and i + 2 >= L: alts.append(("e",)); i += 2; continue
+        if nx and c in "ao" and nx in "uvw" and (i + 2 >= L or t[i + 2] not in VOW) and c + nx != "av":
+            alts.append(("o", c + "u")); i += 2; continue
+        if c == "o" and nx == "u": alts.append(("o", "u")); i += 2; continue
+        if c == "e" and nx == "e": alts.append(("i",)); i += 2; continue
+        if c == "o" and nx == "o": alts.append(("u", "o")); i += 2; continue
+        if c == "e" and nx == "i" and (i + 2 >= L or t[i + 2] not in VOW): alts.append(("e",)); i += 2; continue
+        if c == "e" and nx == "h" and i + 2 < L and t[i + 2] not in VOW and i > 0: alts.append(("e", "")); i += 1; continue
+        if nx and c == "i" and nx in "aeou" and i > 0: alts.append(("y", "i")); i += 1; continue
+        if c == "y" and i == L - 1 and prev and prev not in VOW: alts.append(("e", "i")); i += 1; continue
+        if c == "n" and prev in VOW and i > 0 and (i == L - 1 or (nx and nx not in VOW and nx != "y")):
+            alts.append(("n", "")); i += 1; continue
+        if c in "iu" and prev == "r" and i >= 2 and t[i - 2] not in VOW:
+            alts.append((c, "")); i += 1; continue
+        alts.append((c,)); i += 1
+    outs = [""]
+    for a in alts:
+        nxt = []
+        for o in outs:
+            for x in a: nxt.append(o + x)
+        outs = list(dict.fromkeys(nxt))[:64]
+    k1 = []; seen = set()
+    for o in outs:
+        k = _fin(o)
+        if k and k not in seen: seen.add(k); k1.append(k)
+    k1 = k1[:cap]
+    k2 = list(dict.fromkeys(k2_of(k) for k in k1 if k2_of(k)))
+    return k1, k2
+
+
+# Irregular chat abbreviations (not derivable by rules). Values: candidate natives in prior order.
+ABBR = {
+    "h": ["है", "हैं"], "hy": ["है"], "hn": ["हाँ", "हैं"], "hm": ["हम", "हम्म"],
+    "k": ["के", "की", "ओके"], "ki": ["की", "कि"], "n": ["ना", "न"], "ni": ["नहीं", "नी"], "nai": ["नहीं", "नई"],
+    "nhi": ["नहीं"], "nhe": ["नहीं"], "nahi": ["नहीं"], "nai": ["नहीं", "नई"],
+    "kr": ["कर"], "krna": ["करना"], "kro": ["करो"], "tm": ["तुम"], "tu": ["तू"], "mai": ["मैं", "में"],
+    "me": ["में", "मैं"], "m": ["मैं", "में"], "thk": ["ठीक"], "tk": ["ठीक", "तक"],
+    "yr": ["यार"], "yrr": ["यार"], "plz": ["प्लीज़"], "pls": ["प्लीज़"], "plzz": ["प्लीज़"],
+    "tq": ["थैंक यू"], "ty": ["थैंक यू"], "thx": ["थैंक्स"], "thnx": ["थैंक्स"], "thnks": ["थैंक्स"],
+    "gn": ["गुड नाइट"], "gm": ["गुड मॉर्निंग"], "ok": ["ओके"], "okk": ["ओके"], "kk": ["ओके"],
+    "sry": ["सॉरी"], "bro": ["ब्रो"], "msg": ["मैसेज"], "bday": ["बर्थडे"], "clg": ["कॉलेज"],
+    "ofc": ["ऑफ़िस"], "pic": ["पिक"], "bhai": ["भाई"], "bht": ["बहुत"], "bhot": ["बहुत"], "bohot": ["बहुत"],
+    "pta": ["पता"], "kb": ["कब"], "jb": ["जब"], "sb": ["सब"], "mtlb": ["मतलब"], "kch": ["कुछ"], "kuch": ["कुछ"],
+}
+
+
+class CasualIndex:
+    """K1/K2 -> lexicon ids sorted by frequency (top `top_n` lexicon words only)."""
+
+    def __init__(self, words, uni=None, top_n=300000, per_key=8):
+        order = range(len(words))
+        if uni is not None:
+            u = uni if callable(uni) else uni.__getitem__
+            order = sorted(order, key=lambda i: (-u(i), i))
+        self.k1 = {}; self.k2 = {}
+        for r, i in enumerate(order):
+            if r >= top_n: break
+            for k in dev_keys(words[i]):
+                b = self.k1.setdefault(k, [])
+                if len(b) < per_key and i not in b: b.append(i)
+                k2 = k2_of(k)
+                if k2:
+                    b2 = self.k2.setdefault(k2, [])
+                    if len(b2) < per_key and i not in b2: b2.append(i)
+
+    def lookup(self, roman, n1=6, n2=4):
+        k1, k2 = rom_keys(roman)
+        r1 = []; r2 = []
+        for k in k1:
+            for i in self.k1.get(k, ()):
+                if i not in r1: r1.append(i)
+        for k in k2:
+            for i in self.k2.get(k, ()):
+                if i not in r2: r2.append(i)
+        return r1[:n1 * 2], r2[:n2 * 2]
+
+
+def squeeze_in(t, mode=0):
+    """Emphasis-repeat normalization of the model input. mode 0: off; 1: 3+ -> 1; 2: 3+ -> 2 for a/e/o, 1 otherwise."""
+    if not mode: return t
+    if mode == 1: return re.sub(r"([a-z])\1{2,}", r"\1", t)
+    return re.sub(r"([a-z])\1{2,}", lambda m: m.group(1) * (2 if m.group(1) in "aeo" else 1), t)
+
+
+def squeeze(t):
+    """Normalize emphasis repeats (3+ same letters -> keep 1, e.g. pleaseee->please)."""
+    return re.sub(r"([a-zA-Z])\1{2,}", r"\1", t)