script/pyPengIm.py

from . import utils
import jieba
import types
import yaml
import os

jieba.load_userdict('./dict_data/word_dict/jieba_cut.txt')

class pyPengIm():
    def __init__(self, history=False) -> None:
        self._dict_paths = {
            "vocab": "./dict_data/vocab/origin_vocab.txt",
            "vocab_extension": "./dict_data/vocab/vocab_extension.txt",
            "word_dict": "./dict_data/word_dict/dict.txt",
            "teochew_word_dict": "./dict_data/word_dict/teochew_local_dict.txt",
            "translation_dict": "./dict_data/word_dict/madr_to_tch.txt",
            "surname_dict": "./dict_data/vocab/Surname.txt",
            "IPA_dict": "./dict_data/vocab/IPA_lexicon.txt",
            # "phoneme_dict": "./dict_data/vocab/phone.txt",
            "low_fre_dict": "./dict_data/vocab/low_fre.txt"
        }

        self.accent_dict = self._load_accent()

        self._loaded_dicts = {}
        
        # 是否启用中国历史词典，以支持古代年号、政权、官职、人名、民族等
        if history:
            self.word_dict.update(utils.load_dict("./dict_data/word_dict/history.txt"))
            self.word_dict.update(utils.load_dict("./dict_data/word_dict/reign_title.txt"))

        jieba.cut('')# 预热

    def __getattr__(self, name):
        if name in self._dict_paths:
            if name not in self._loaded_dicts:
                self._loaded_dicts[name] = utils.load_dict(self._dict_paths[name])
            return self._loaded_dicts[name]

        raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{name}'")

    def _load_accent(self,accent_config_path="./dict_data/accent_convert/accent.yaml"):
        with open(accent_config_path, 'r', encoding='utf-8') as file:
            accent_config = yaml.safe_load(file)
            accent_dict = {}
            for k,v in accent_config.items():
                accent_dict[k] = (
                        utils.load_dict(os.path.join("./dict_data/accent_convert",v['path'])),
                        v['name']
                    )

        return accent_dict
    
    def pinyin(self, text, heteronym=False, accent='', auto_split=True):
        text = text.upper()
        if heteronym:
            pinyin_list = self._pinyin_heteronym(text)
        else:
            if auto_split:
                pinyin_list = self.pinyin_optimize(utils.preprocess_generator(self.sentence_cut(text)))
            else:
                pinyin_list = self.pinyin_optimize(text.split(' '))

        if accent in self.accent_dict:
            pinyin_list = self.convert_accent(pinyin_list, accent)

        surname_list = self._surname_notice(text)

        return {
            'result': pinyin_list,
            'pinyin_seq': self._to_pinyin_sequence(pinyin_list),
            'surname_notice': surname_list
        }

    def sentence_cut(self, text):
        new_text_list = []
        for ch in text:
            if ch not in self.vocab and ch not in self.vocab_extension:
                new_text_list.append(' {} '.format(ch))
            else:
                new_text_list.append(ch)
        
        return jieba.cut("".join(new_text_list))


    def _to_pinyin_sequence(self, pinyin_list):
        result = []
        for item in pinyin_list:
            for pinyin in item[1:]:
                if pinyin != ['None']:
                    result.append('|'.join([py.replace('*', '') for py in pinyin]))
                else:
                    result.append(item[0]) ## 非法字符，原样输出

        return ' '.join(result)

    def _surname_notice(self, text):
        result = []
        for ch in text:
            if ch in self.surname_dict.keys():
                item = [ch, self._to_pinyin_list(self.surname_dict[ch])]
                result.append(item)
        return result

    def _pinyin_heteronym(self, text):
        result = []
        text = text.replace('#', '')
        for zh_char in text:
            item = []
            if zh_char in self.vocab.keys():
                item.extend(self._to_pinyin_list(self.vocab[zh_char]))
            if zh_char in self.vocab_extension.keys():
                item.extend(self._to_pinyin_list(self.vocab_extension[zh_char]))
            if zh_char in self.low_fre_dict.keys():
                item.extend(self._to_pinyin_list(self.low_fre_dict[zh_char]))
            result.append([zh_char, item])

        return result

    def _to_pinyin_list(self, pinyin_item):
        return pinyin_item.split('|') if '|' in pinyin_item else [pinyin_item]

    def _word_to_pinyin(self, item_word):
        if ' ' in item_word:
            return [self._to_pinyin_list(pinyin) for pinyin in item_word.split(' ')]
        return [[item_word]]

    def pinyin_optimize(self, word_list):
        result = []
        for word in word_list:
            if not word.strip():
                continue

            word_translate_flag = word.endswith('#')
            word = word.rstrip('#')
            word_found_flag = False

            item = [word]
            if word in self.teochew_word_dict.keys():
                item.extend(self._word_to_pinyin(self.teochew_word_dict[word]))
                word_found_flag = True

            if not word_translate_flag and word in self.word_dict:
                item = [word]
                item.extend(self._word_to_pinyin(self.word_dict[word]))  # 在非翻译模式下，清空之前的拼音，仅保留 word_dict 结果，也就是只保存普通话语义
                word_found_flag = True

            elif not word_translate_flag or not word_found_flag:
                if word in self.word_dict:
                    item.extend(self._word_to_pinyin(self.word_dict[word]))
                    word_found_flag = True


            if not word_found_flag:
                for zh_char in word:
                    if zh_char in self.vocab.keys():
                        item.append(self._to_pinyin_list(self.vocab[zh_char]))
                    elif zh_char in self.vocab_extension.keys():
                        item.append(self._to_pinyin_list(self.vocab_extension[zh_char]))
                    else:
                        item.append(self._to_pinyin_list('None'))

            result.append(item)
        return result

    def convert_accent(self, pinyin_list, accent):
        target_vocab = self.accent_dict[accent][0]

        result = []
        for one_pair in pinyin_list:
            word, pinyins = one_pair[0], list(one_pair[1:])
            item = [word]
            for i, hanzi in enumerate(word):
                pronunciations = []
                for pronunciation in pinyins[i]:
                    query_item = f'{hanzi}_{pronunciation}'
                    target_accent = target_vocab.get(query_item, pronunciation)
                    pronunciations.extend(self._to_pinyin_list(target_accent))
                item.append(list(dict.fromkeys(pronunciations)))# 去重
            result.append(item)
        return result

    def to_IPA(self, pinyin_seq, blank=True):
        if blank:
            split_char = ' '
        else:
            split_char = ''

        result = []
        for pinyin in pinyin_seq.split(' '):
            if '|' in pinyin:
                ipa_item = []
                for py in pinyin.split('|'):
                    ph_list = utils.pinyin_to_phoneme_list(py)
                    ipa_item.append(split_char.join([self.IPA_dict[ph] if ph in self.IPA_dict else ph for ph in ph_list]))
                result.append("|".join(ipa_item))
            else:
                ph_list = utils.pinyin_to_phoneme_list(pinyin)
                result.append(split_char.join(self.IPA_dict[ph] if ph in self.IPA_dict else ph for ph in ph_list))

        return result

    def to_phoneme(self, pinyin_seq):
        return ['|'.join([utils.pinyin_to_phoneme(py) for py in pinyin.split('|')]) if '|' in pinyin else utils.pinyin_to_phoneme(pinyin) for pinyin in pinyin_seq.split(' ')]

    def to_oral(self, text, auto_split=True):
        if isinstance(text, list) or isinstance(text, types.GeneratorType):
            word_list = text
        elif isinstance(text, str):
            word_list = jieba.cut(text) if auto_split else text.split(' ')
        else:
            return None

        return ' '.join([self.translation_dict.get(word, word) + '#' if word in self.translation_dict else word for word in word_list])

    def add_word_mapping(self, user_mapping: dict):
        self.translation_dict.update(user_mapping)

    # 查询单个字在不同地区的口音
    def single_query(self, single_char):
        if single_char not in self.vocab and single_char not in self.vocab_extension:
            return None

        if len(single_char) > 1:
            return None

        result_dict = {}
        pinyin_list = self._pinyin_heteronym(single_char)
        result_dict['府城'] = pinyin_list[0]

        for k,v in self.accent_dict.items():
            result_dict[v[1]] = self.convert_accent(pinyin_list, accent=k)[0]

        return result_dict