Spaces:
Sleeping
Sleeping
| from underthesea import word_tokenize, ner | |
| import re | |
| from symspellpy import SymSpell, Verbosity | |
| class Probability: | |
| def __init__(self): | |
| self.sym_spell_2gram = self.sym_spell_3gram = self.sym_spell = SymSpell( | |
| max_dictionary_edit_distance=2, | |
| prefix_length=7, | |
| count_threshold=1 | |
| ) | |
| self.sym_spell.load_dictionary("dictionary/frequency_vi_test.txt", 0, 1, separator="$") | |
| self.sym_spell_2gram.load_dictionary("dictionary/dic_2_gram_test.txt", 0, 1, separator="$") | |
| self.sym_spell_3gram .load_dictionary("dictionary/dic_3_gram_test.txt", 0, 1, separator="$") | |
| def fix_spelling(self, text): | |
| word_tokenizer_format = word_tokenize(text, format="text") | |
| word_tokenizer = [token for token in word_tokenizer_format.split()] | |
| list_suggestions = self.word_tokenizer_suggestions(word_tokenizer) | |
| list_suggestions.insert(0, [["", -1, -1]]) | |
| list_suggestions.append([["", -1, -1]]) | |
| # Lọc danh sách ứng viên | |
| for i, suggestion in enumerate(list_suggestions): | |
| if i < len(list_suggestions) - 1 and i > 0: | |
| if len(suggestion) > 1: | |
| max ,sum_count = self.fix_spelling_word(suggestion, | |
| list_suggestions[i - 1][0][0] if list_suggestions[i - 1][0][1] != -1 else list_suggestions[i - 2][0][0], | |
| list_suggestions[i + 1]) | |
| if max[0] == "right": | |
| word = max[1] | |
| list_suggestions[i] = [[word, 0, 99999]] | |
| list_suggestions.remove(list_suggestions[i + 1]) | |
| else: | |
| word = max[1] | |
| list_suggestions[i] = [[word, 0, 99999]] | |
| list_suggestions.remove(list_suggestions[i - 1]) | |
| result_list = [suggestions_text[0][0].replace("_", " ") for suggestions_text in list_suggestions if len(suggestions_text) !=0] | |
| result = " ".join(result_list).strip() | |
| return result | |
| def word_tokenizer_suggestions(self, word_tokenizer): | |
| list_suggestions = [] | |
| # Tìm danh sách từ ứng viên | |
| for i, word in enumerate(word_tokenizer): | |
| if self.is_valid_token(word): | |
| if len(word.split()) == 1: | |
| suggestions = self.sym_spell.lookup( | |
| word.lower(), Verbosity.CLOSEST, max_edit_distance=2 | |
| ) | |
| else: | |
| suggestions = self.sym_spell.lookup_compound( | |
| word.lower(), 2 | |
| ) | |
| suggestion = [[s.term, s.distance, s.count] for s in suggestions] | |
| if len(suggestion) == 0: | |
| word_split = word.replace("_", " ").split() | |
| for split in word_split: | |
| suggestion_split = self.sym_spell.lookup( | |
| split.lower(), Verbosity.CLOSEST, max_edit_distance=2 | |
| ) | |
| suggestion = [[s.term, s.distance, s.count] for s in suggestion_split] | |
| if len(suggestion) != 0: | |
| list_suggestions.append(suggestion) | |
| else: | |
| list_suggestions.append([[split, 0, 0]]) | |
| else: | |
| list_suggestions.append(suggestion) | |
| else: | |
| list_suggestions.append([[word, 0, -1]]) | |
| return list_suggestions | |
| def fix_spelling_word(self, suggestion, left_word, right_suggestions): | |
| max = ["", "", -1] | |
| sum_count = 0 | |
| list_probability = [] | |
| for s in suggestion: | |
| right_corresponds = ["{}_{}".format(s[0].replace(" ", "_"), left_text[0].replace(" ", "_")) for left_text in right_suggestions] | |
| left_correspond = ["{}_{}".format(left_word.replace(" ", "_"), s[0].replace(" ", "_"))] | |
| tri_correspond = ["{}_{}".format(left_word.replace(" ", "_"), right_correspond) for right_correspond in right_corresponds] | |
| left_probability = self.probability_2gram(left_correspond) | |
| right_probability = self.probability_2gram(right_corresponds) | |
| tri_probability = self.probability_3gram(tri_correspond) | |
| list_probability.append([left_probability, "left"]) | |
| list_probability.append([right_probability, "right"]) | |
| list_probability.append([tri_probability, "tri"]) | |
| sum_count += left_probability[2] + right_probability[2] + tri_probability[2] | |
| for proba in list_probability: | |
| P = proba[0][1] / sum_count if proba[0][1] > 0 else 0 | |
| if P > max[2]: | |
| max = [proba[1], proba[0][0], P] | |
| return max, sum_count | |
| def probability_2gram(self, words): | |
| max = -1 | |
| best_word = "" | |
| sum_count = 0 | |
| for word in words: | |
| suggestions = self.sym_spell_2gram.lookup( | |
| word, Verbosity.CLOSEST, max_edit_distance=2 | |
| ) | |
| sum_count += self.count_word(suggestions) | |
| if self.count_word(suggestions) > max: | |
| max = self.count_word(suggestions) | |
| best_word = word | |
| result = [best_word, max, sum_count] | |
| return result | |
| def probability_3gram(self, words): | |
| max = -1 | |
| best_word = "" | |
| sum_count = 0 | |
| for word in words: | |
| suggestions = self.sym_spell_3gram.lookup( | |
| word, Verbosity.CLOSEST, max_edit_distance=2 | |
| ) | |
| sum_count += self.count_word(suggestions) | |
| if self.count_word(suggestions) > max: | |
| max = self.count_word(suggestions) | |
| best_word = word | |
| result = [best_word, max, sum_count] | |
| return result | |
| def count_word(self, suggestions): | |
| for suggestion in suggestions: | |
| if suggestion.distance == 0: | |
| return suggestion.count | |
| return 0 | |
| def is_valid_token(self, token): | |
| if re.match(r'^\d+(\.\d+)?$', token): | |
| return False | |
| if "_" in token: | |
| if ner(token.replace("_"," "))[0][1] == "Np": | |
| return False | |
| if re.match(r"^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$", token): | |
| return False | |
| elif re.match(r"^(0[3|5|7|8|9])([0-9]{8})$", token): | |
| return False | |
| return all(c.isalpha() or c == '_' for c in token) | |
| if __name__ == "__main__": | |
| probability = Probability() | |
| result = probability.fix_spelling("Có kinh nghiệm lý đội nhóm gổm 20 nhan vien cấp dưởi. Tính tổ chửc và kỷ luật cao, không ngửng đổi mởi để cải thiện sản phẩm và dịch vụ đi kèm Luôn đặt trải nghiệm của khách hàng lên trên cùng và nỗ lực đưa tên tuổi thuong hiệu tiếp can đưạc tệp người rộng hon. rong quản dung ") | |
| print(result) | |