src/scripts/syl_align.py

"""Split a Thai word into surface syllables by aligning the writing to the pronunciation.

The unit of กลอน is the syllable, so a grid cell must be one. No surface segmenter we have gets this
right: pythainlp's han_solo, dict and ssg engines and TLTK's syl_segment all return ถวิล, พันธนา and
ปรารถนา whole, and each invents splits of its own (ถวิ|ล, ปรา|รถ, แก้|มหอม). Only the pronunciation
is correct, because the count follows the sound — ถวิล is ถะ-หวิน.

TLTK gives the pronunciation one syllable at a time (ถวิล -> tha1~win4), so each syllable's ONSET is
known. This walks the written form left to right and cuts wherever the next syllable's onset letter
appears, carrying any leading vowel (เ แ โ ใ ไ) back to the syllable it belongs to.
"""
import sys, re, functools
sys.path.insert(0, ".")
from klon8 import prosody as P

LEAD_VOWEL = "เแโใไ"
# Thai consonant letters by the onset phone TLTK writes for them
ONSET = {
    "ก": "k", "ข": "kh", "ฃ": "kh", "ค": "kh", "ฅ": "kh", "ฆ": "kh", "ง": "ng",
    "จ": "c", "ฉ": "ch", "ช": "ch", "ฌ": "ch", "ซ": "s", "ศ": "s", "ษ": "s", "ส": "s",
    "ญ": "j", "ย": "j", "ฎ": "d", "ด": "d", "ฏ": "t", "ต": "t",
    "ฐ": "th", "ฑ": "th", "ฒ": "th", "ถ": "th", "ท": "th", "ธ": "th",
    "ณ": "n", "น": "n", "บ": "b", "ป": "p", "ผ": "ph", "พ": "ph", "ภ": "ph",
    "ฝ": "f", "ฟ": "f", "ม": "m", "ร": "r", "ล": "l", "ฬ": "l", "ว": "w",
    "ห": "h", "ฮ": "h", "อ": "?",
}
# ห as a silent tone marker, and the true clusters
DIGRAPH = {"หน": "n", "หม": "m", "หญ": "j", "หย": "j", "หง": "ng", "หล": "l", "หว": "w", "หร": "r",
           "กร": "kr", "กล": "kl", "กว": "kw", "ขร": "khr", "ขล": "khl", "ขว": "khw",
           "คร": "khr", "คล": "khl", "คว": "khw", "ปร": "pr", "ปล": "pl", "ผล": "phl",
           "พร": "phr", "พล": "phl", "ตร": "tr", "ทร": "s", "ดร": "dr", "บร": "br", "บล": "bl"}


def onset_at(w, i):
    """Every onset the letters at position i could spell, longest first."""
    out = []
    if i + 1 < len(w) and w[i:i + 2] in DIGRAPH: out.append((DIGRAPH[w[i:i + 2]], 2))
    if w[i] in ONSET: out.append((ONSET[w[i]], 1))
    return out


@functools.lru_cache(maxsize=200000)
def split_word(w):
    """Surface syllables of one word, count-matched to its pronunciation. () if alignment fails."""
    # the onset comes from prosody's own parser, not from a regex over the phone string:
    # "tha1" is th + a + tone, and [a-zA-Z?]+ grabs "tha", which matches no letter.
    syls = P.g2p_line(w)
    if not syls: return ()
    onsets = [x.onset for x in syls]
    codas = [x.coda for x in syls]
    if len(onsets) <= 1: return (w,)
    starts, k = [0], 1
    i = 1
    while i < len(w) and k < len(onsets):
        for cand, n in onset_at(w, i):
            if cand != onsets[k]: continue
            begin = i - 1 if (i and w[i - 1] in LEAD_VOWEL) else i
            if begin <= starts[-1]: continue
            # a syllable that is pronounced with a coda needs at least an onset letter and a coda
            # letter written, so a one-character piece cannot be it. Without this, ออดอ้อน (?OOt~?OOn)
            # cuts at the second อ - which is the vowel of syllable 1, not the onset of syllable 2.
            if codas[k - 1] and begin - starts[-1] < 2: continue
            starts.append(begin); k += 1; i = begin + n
            break
        else:
            i += 1
            continue
    if len(starts) != len(onsets): return ()
    return tuple(w[a:b] for a, b in zip(starts, starts[1:] + [len(w)]))


def syls_aligned(line, limit=None):
    """Surface syllables of one วรรค. Falls back to the whole word when alignment fails."""
    from pythainlp.tokenize import word_tokenize
    out = []
    for w in word_tokenize(line, engine="newmm"):
        if not re.search(r"[ก-๙]", w): continue
        got = split_word(w)
        out.extend(got if got else [w])
    return out[:limit] if limit else out


if __name__ == "__main__":
    tests = [("มาเหนี่ยวรั้งร้อยขวัญพันธนา", "มา|เหนี่ยว|รั้ง|ร้อย|ขวัญ|พัน|ธ|นา"),
             ("หอมกลิ่นคำร่ำรินถวิลนัก", "หอม|กลิ่น|คำ|ร่ำ|ริน|ถ|วิล|นัก"),
             ("ปรารถนาก็เวียนวกเกินยกพ้น", "ปราร|ถ|นา|ก็|เวียน|วก|เกิน|ยก|พ้น"),
             ("ไม่ได้จูบแก้มหอมดอมเสนอ", "ไม่|ได้|จูบ|แก้ม|หอม|ดอม|เส|นอ"),
             ("เผลอ-เผยความอาวรณ์ออดอ้อนหา", "เผลอ|เผย|ความ|อา|วรณ์|ออด|อ้อน|หา"),
             ("อารมณ์ผู้เพ็ญถวิล ก็รินไหล", "อา|รมณ์|ผู้|เพ็ญ|ถ|วิล|ก็|ริน|ไหล")]
    nc = ns = 0
    for t, hand in tests:
        got = syls_aligned(t)
        want = hand.split("|")
        cnt = len(got) == len(P.g2p_line(t))
        exact = got == tuple(want) or list(got) == want
        nc += cnt; ns += exact
        print(f"  {'EXACT' if exact else ('count' if cnt else '     '):5s} {len(got)} vs {len(want)}   " + " | ".join(got))
        if not exact: print(f"        hand: {' | '.join(want)}")
    print(f"\n  count correct {nc}/6, split identical to the hand segmentation {ns}/6")
    P.save_cache()