Spaces:
Sleeping
Sleeping
File size: 2,300 Bytes
c7f3ffb 0196c78 c7f3ffb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 | import re
import sys
import nltk
# Ensure NLTK data is available
try:
nltk.data.find('taggers/averaged_perceptron_tagger_eng')
except LookupError:
print("[g2p] Downloading missing NLTK resource: averaged_perceptron_tagger_eng", file=sys.stderr)
nltk.download('averaged_perceptron_tagger_eng', quiet=True)
try:
nltk.data.find('corpora/cmudict')
except LookupError:
print("[g2p] Downloading missing NLTK resource: cmudict", file=sys.stderr)
nltk.download('cmudict', quiet=True)
import ToJyutping
from g2pM import G2pM
from g2p_en import G2p as G2pE
_EN_WORD_RE = re.compile(r"^[A-Za-z]+(?:'[A-Za-z]+)*$")
_ZH_WORD_RE = re.compile(r"[\u4e00-\u9fff]")
EN_FLAG = "en_"
YUE_FLAG = "yue_"
ZH_FLAG = "zh_"
g2p_zh = G2pM()
g2p_en = G2pE()
def is_chinese_char(word: str) -> bool:
if len(word) != 1:
return False
return bool(_ZH_WORD_RE.fullmatch(word))
def is_english_word(word: str) -> bool:
if not word:
return False
return bool(_EN_WORD_RE.fullmatch(word))
def g2p_cantonese(sent):
return ToJyutping.get_jyutping_list(sent) # with tone
def g2p_mandarin(sent):
return g2p_zh(sent, tone=True, char_split=False)
def g2p_english(word):
return g2p_en(word)
def g2p_transform(words, lang):
zh_words = []
transformed_words = [0] * len(words)
for idx, w in enumerate(words):
if w == "<SP>":
transformed_words[idx] = w
continue
w = w.replace("?", "").replace(".", "").replace("!", "").replace(",", "")
if is_chinese_char(w):
zh_words.append([idx, w])
else:
if is_english_word(w):
w = EN_FLAG + "-".join(g2p_english(w.lower()))
else:
w = "<SP>"
transformed_words[idx] = w
sent = "".join([k[1] for k in zh_words])
# zh (zh and yue) transformer to g2p
if len(sent) > 0:
if lang == "Cantonese":
g2pm_rst = g2p_cantonese(sent) # with tone
g2pm_rst = [YUE_FLAG + k[1] for k in g2pm_rst]
else:
g2pm_rst = g2p_mandarin(sent)
g2pm_rst = [ZH_FLAG + k for k in g2pm_rst]
for p, w in zip([k[0] for k in zh_words], g2pm_rst):
transformed_words[p] = w
return transformed_words
|