File size: 5,858 Bytes
7d3eb8f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 | """Split a Thai word into surface syllables by aligning the writing to the pronunciation.
The unit of กลอน is the syllable, so a grid cell must be one. No surface segmenter we have gets this
right: pythainlp's han_solo, dict and ssg engines and TLTK's syl_segment all return ถวิล, พันธนา and
ปรารถนา whole, and each invents splits of its own (ถวิ|ล, ปรา|รถ, แก้|มหอม). Only the pronunciation
is correct, because the count follows the sound — ถวิล is ถะ-หวิน.
TLTK gives the pronunciation one syllable at a time (ถวิล -> tha1~win4), so each syllable's ONSET is
known. This walks the written form left to right and cuts wherever the next syllable's onset letter
appears, carrying any leading vowel (เ แ โ ใ ไ) back to the syllable it belongs to.
"""
import sys, re, functools
sys.path.insert(0, ".")
from klon8 import prosody as P
LEAD_VOWEL = "เแโใไ"
# Thai consonant letters by the onset phone TLTK writes for them
ONSET = {
"ก": "k", "ข": "kh", "ฃ": "kh", "ค": "kh", "ฅ": "kh", "ฆ": "kh", "ง": "ng",
"จ": "c", "ฉ": "ch", "ช": "ch", "ฌ": "ch", "ซ": "s", "ศ": "s", "ษ": "s", "ส": "s",
"ญ": "j", "ย": "j", "ฎ": "d", "ด": "d", "ฏ": "t", "ต": "t",
"ฐ": "th", "ฑ": "th", "ฒ": "th", "ถ": "th", "ท": "th", "ธ": "th",
"ณ": "n", "น": "n", "บ": "b", "ป": "p", "ผ": "ph", "พ": "ph", "ภ": "ph",
"ฝ": "f", "ฟ": "f", "ม": "m", "ร": "r", "ล": "l", "ฬ": "l", "ว": "w",
"ห": "h", "ฮ": "h", "อ": "?",
}
# ห as a silent tone marker, and the true clusters
DIGRAPH = {"หน": "n", "หม": "m", "หญ": "j", "หย": "j", "หง": "ng", "หล": "l", "หว": "w", "หร": "r",
"กร": "kr", "กล": "kl", "กว": "kw", "ขร": "khr", "ขล": "khl", "ขว": "khw",
"คร": "khr", "คล": "khl", "คว": "khw", "ปร": "pr", "ปล": "pl", "ผล": "phl",
"พร": "phr", "พล": "phl", "ตร": "tr", "ทร": "s", "ดร": "dr", "บร": "br", "บล": "bl"}
def onset_at(w, i):
"""Every onset the letters at position i could spell, longest first."""
out = []
if i + 1 < len(w) and w[i:i + 2] in DIGRAPH: out.append((DIGRAPH[w[i:i + 2]], 2))
if w[i] in ONSET: out.append((ONSET[w[i]], 1))
return out
@functools.lru_cache(maxsize=200000)
def split_word(w):
"""Surface syllables of one word, count-matched to its pronunciation. () if alignment fails."""
# the onset comes from prosody's own parser, not from a regex over the phone string:
# "tha1" is th + a + tone, and [a-zA-Z?]+ grabs "tha", which matches no letter.
syls = P.g2p_line(w)
if not syls: return ()
onsets = [x.onset for x in syls]
codas = [x.coda for x in syls]
if len(onsets) <= 1: return (w,)
starts, k = [0], 1
i = 1
while i < len(w) and k < len(onsets):
for cand, n in onset_at(w, i):
if cand != onsets[k]: continue
begin = i - 1 if (i and w[i - 1] in LEAD_VOWEL) else i
if begin <= starts[-1]: continue
# a syllable that is pronounced with a coda needs at least an onset letter and a coda
# letter written, so a one-character piece cannot be it. Without this, ออดอ้อน (?OOt~?OOn)
# cuts at the second อ - which is the vowel of syllable 1, not the onset of syllable 2.
if codas[k - 1] and begin - starts[-1] < 2: continue
starts.append(begin); k += 1; i = begin + n
break
else:
i += 1
continue
if len(starts) != len(onsets): return ()
return tuple(w[a:b] for a, b in zip(starts, starts[1:] + [len(w)]))
def syls_aligned(line, limit=None):
"""Surface syllables of one วรรค. Falls back to the whole word when alignment fails."""
from pythainlp.tokenize import word_tokenize
out = []
for w in word_tokenize(line, engine="newmm"):
if not re.search(r"[ก-๙]", w): continue
got = split_word(w)
out.extend(got if got else [w])
return out[:limit] if limit else out
if __name__ == "__main__":
tests = [("มาเหนี่ยวรั้งร้อยขวัญพันธนา", "มา|เหนี่ยว|รั้ง|ร้อย|ขวัญ|พัน|ธ|นา"),
("หอมกลิ่นคำร่ำรินถวิลนัก", "หอม|กลิ่น|คำ|ร่ำ|ริน|ถ|วิล|นัก"),
("ปรารถนาก็เวียนวกเกินยกพ้น", "ปราร|ถ|นา|ก็|เวียน|วก|เกิน|ยก|พ้น"),
("ไม่ได้จูบแก้มหอมดอมเสนอ", "ไม่|ได้|จูบ|แก้ม|หอม|ดอม|เส|นอ"),
("เผลอ-เผยความอาวรณ์ออดอ้อนหา", "เผลอ|เผย|ความ|อา|วรณ์|ออด|อ้อน|หา"),
("อารมณ์ผู้เพ็ญถวิล ก็รินไหล", "อา|รมณ์|ผู้|เพ็ญ|ถ|วิล|ก็|ริน|ไหล")]
nc = ns = 0
for t, hand in tests:
got = syls_aligned(t)
want = hand.split("|")
cnt = len(got) == len(P.g2p_line(t))
exact = got == tuple(want) or list(got) == want
nc += cnt; ns += exact
print(f" {'EXACT' if exact else ('count' if cnt else ' '):5s} {len(got)} vs {len(want)} " + " | ".join(got))
if not exact: print(f" hand: {' | '.join(want)}")
print(f"\n count correct {nc}/6, split identical to the hand segmentation {ns}/6")
P.save_cache()
|