import numpy as np import pandas as pd import re import string from underthesea import word_tokenize, text_normalize def remove_trailing_letters(word): """A function to remove duplicated ending letters in a word (For example: from "okkk" to "ok") Parameters ---------- word : str A sentence with duplicated ending letters Returns ---------- str A sentence with no duplicated ending letter """ pattern = r"(.)\1+$" replaced_word = re.sub(pattern, r"\1", word, re.UNICODE) if len(replaced_word) <= 1: return replaced_word if (replaced_word[-1] == "e") & (replaced_word[-2] in ["ẻ", "ẹ", "è", "ẻ"]): replaced_word = replaced_word[:-1] elif (replaced_word[-1] == "i") & (replaced_word[-2] in ["í", "ị", "ì", "ỉ"]): replaced_word = replaced_word[:-1] elif (replaced_word[-1] == "a") & (replaced_word[-2] in ["á", "ạ", "à", "ả"]): replaced_word = replaced_word[:-1] elif (replaced_word[-1] == "y") & (replaced_word[-2] in ["ý", "ỵ", "ỳ", "ỷ"]): replaced_word = replaced_word[:-1] elif (replaced_word[-1] == "u") & (replaced_word[-2] in ["ủ", "ụ", "ù", "ủ"]): replaced_word = replaced_word[:-1] elif (replaced_word[-1] == "o") & (replaced_word[-2] in ["ỏ", "ọ", "ò", "ỏ"]): replaced_word = replaced_word[:-1] return replaced_word remove_trailing_letters(" ") def remove_ending_letters_in_sentence(sentence): """A function to remove duplicated ending letters in a sentence (For example: from "Quạt yếu lắmm nhưng màaa lạaa làaa sản phẩm này rẻeee okkk lắmmmm" to "Quạt yếu lắm nhưng mà lạ là sản phẩm này rẻ ok lắm") Parameters ---------- word : str A sentence with duplicated ending letters Returns ---------- str A sentence with no duplicated ending letter """ words = sentence.split() result_words = [remove_trailing_letters(word) for word in words] return " ".join(result_words) def replace_all_numbers(sentence): """Remove all numbers in a sentence Parameters ---------- sentence : str A sentence with numbers Returns ---------- str Sentence with replaced number """ return re.sub(r'\d+', 'number_', sentence) def remove_links(text): """Removes URLs from a string. Parameters ---------- text: The input string containing URLs. Returns ---------- The modified string with URLs removed. """ # Define a regular expression pattern to match URLs url_pattern = r"https?://\S+|\bwww\.\S+" return re.sub(url_pattern, "LINK", text) replace_list = { 'ầ': 'ầ', 'ỏ': 'ỏ', 'ề': 'ề','ễ': 'ễ', 'ắ': 'ắ', 'ủ': 'ủ', 'ế': 'ế', 'ở': 'ở', 'ỉ': 'ỉ', 'ẻ': 'ẻ', 'àk': u' à ','aˋ': 'à', 'iˋ': 'ì', 'ă´': 'ắ','ử': 'ử', 'e˜': 'ẽ', 'y˜': 'ỹ', 'a´': 'á', #Chuẩn hóa 1 số sentiment words/English words ':))': ' positive ', ':)': ' positive ', 'ô kêi': ' ok ', 'okie': ' ok ', ' o kê ': ' ok ', ' okey ': ' ok ', ' ôkê ': ' ok ', ' oki ': ' ok ', ' oke ': ' ok ',' okay ':' ok ',' okê ':' ok ', ' tks ': u' cám ơn ', ' thks ': u' cám ơn ', ' thanks ': u' cám ơn ', ' ths ': u' cám ơn ', ' thank ': u' cám ơn ', '⭐': 'star', '*': 'star', '🌟': 'star', "bik": " bị ",'not': u' không ', u' kg ': u' không ', ' k ': u' không ',' kô ':u' không ','hok':u' không ',' kp ': u' không phải ',u' kô ': u' không ', '"ko ': u' không ', u' ko ': u' không ', u' k ': u' không ', 'khong': u' không ', u' hok ': u' không ','cute': u' dễ thương ',' huhu ': ' negative ', ' vs ': u' với ', ' wa ': ' quá ', 'wá': u' quá', ' j ': u' gì ', '“': ' ', ' sz ': u' cỡ ', 'size': u' cỡ ', u' đx ': u' được ', 'dk': u' được ', ' dc ': u' được ', 'đk': u' được ', ' đc ': u' được ',' authentic ': u' chính hãng ',u' aut ': u' chính hãng ', u' auth ': u' chính hãng ', ' store ': u' cửa hàng ', ' shop ': u' cửa hàng ', ' sp ': u' sản phẩm ', ' gud ': u' tốt ',' god ': u' tốt ',' wel done ':' tốt ', ' good ': u' tốt ', ' gút ': u' tốt ', ' sấu ': u' xấu ',' gut ': u' tốt ', u' tot ': u' tốt ', u' nice ': u' tốt ', ' perfect ': 'rất tốt', 'time': u' thời gian ', 'qá': u' quá ', u' ship ': u' giao hàng ', u' m ': u' mình ', u' mik ': u' mình ', ' ể ': 'ể', ' product ': 'sản phẩm', ' quality ': 'chất lượng', ' excelent ': 'tốt', ' bad ': ' tệ ',' fresh ': ' tươi ','sad': ' tệ ', ' date ': u' hạn sử dụng ', ' hsd ': u' hạn sử dụng ',' quickly ': u' nhanh ', ' quick ': u' nhanh ',' fast ': u' nhanh ',' delivery ': u' giao hàng ',u' síp ': u' giao hàng ', ' beautiful ': u' đẹp ', u' tl ': u' trả lời ', u' r ': u' rồi ', u' shope ': u' cửa hàng ',u' order ': u' đặt hàng ', ' chất lg ': u' chất lượng ',u' sd ': u' sử dụng ',u' dt ': u' điện thoại ',u' nt ': u' nhắn tin ',u' tl ': u' trả lời ',u' sài ': u' xài ',u' bjo ':u' bao giờ ', ' thik ': u' thích ',u' sop ': u' cửa hàng ', ' fb ': ' facebook ', ' face ': ' facebook ', " zl ": " zalo ", ' very ': u' rất ', ' dep ': u' đẹp ',u' xau ': u' xấu ',' delicious ': u' ngon ', u' hàg ': u' hàng ', u' qủa ': u' quả ', ' iu ': u' yêu ','fake': u' giả mạo ', ' trl ': 'trả lời', '><': u' positive ', ' por ': u' tệ ',' poor ': u' tệ ', ' ib ':u' nhắn tin ', ' rep ':u' trả lời ',u' fback ':' feedback ',' fedback ':' feedback ', " đag ": " đang "} def replace_sent(text, replace_list = replace_list): """Remove elements in a text according to the replacement dictionary Parameters ---------- text : str Text that needs replacement Returns ---------- str Text with all elements in the replacement dictionary replaced """ for k, v in replace_list.items(): text = text.replace(k, v) return text def remove_non_alphanumeric(text): """Removes all characters except alphanumeric characters and spaces from a string. Args: text: The input string. Returns: The modified string with only alphanumeric characters and spaces. """ pattern = r"[^\w\s]" # Matches any character except alphanumeric and space return re.sub(pattern, "", text) def remove_all_tag(sentence): """Remove tags added from reading files Parameters ---------- text : str Text that has tags Returns ---------- str Text with no tags """ return re.sub(r'\n|\t|\r', ' ', sentence) def cleaning(sentence): """ A function to process a sentence in following order: 1. lowercase all character 2. replace numbers with tags 3. replace links with tags 4. normalize vietnamese text 5. remove formatting tags 6. remove punctuation 7. replace shorthand 8. remove trailing letters in words ---------- sentence : str input string Returns ---------- str processed string """ sentence = sentence.lower() sentence = replace_all_numbers(sentence) sentence = remove_links(sentence) sentence = text_normalize(sentence) sentence = remove_all_tag(sentence) sentence = remove_non_alphanumeric(sentence) sentence = replace_sent(sentence, replace_list) sentence = remove_ending_letters_in_sentence(sentence) return sentence def cleaning_for_phobert(sentence): sentence = text_normalize(sentence) sentence = remove_all_tag(sentence) sentence = remove_ending_letters_in_sentence(sentence) return sentence