File size: 7,914 Bytes
400427c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 | import numpy as np
import pandas as pd
import re
import string
from underthesea import word_tokenize, text_normalize
def remove_trailing_letters(word):
"""A function to remove duplicated ending letters in a word (For example: from "okkk" to "ok")
Parameters
----------
word : str
A sentence with duplicated ending letters
Returns
----------
str
A sentence with no duplicated ending letter
"""
pattern = r"(.)\1+$"
replaced_word = re.sub(pattern, r"\1", word, re.UNICODE)
if len(replaced_word) <= 1:
return replaced_word
if (replaced_word[-1] == "e") & (replaced_word[-2] in ["ẻ", "ẹ", "è", "ẻ"]):
replaced_word = replaced_word[:-1]
elif (replaced_word[-1] == "i") & (replaced_word[-2] in ["í", "ị", "ì", "ỉ"]):
replaced_word = replaced_word[:-1]
elif (replaced_word[-1] == "a") & (replaced_word[-2] in ["á", "ạ", "à", "ả"]):
replaced_word = replaced_word[:-1]
elif (replaced_word[-1] == "y") & (replaced_word[-2] in ["ý", "ỵ", "ỳ", "ỷ"]):
replaced_word = replaced_word[:-1]
elif (replaced_word[-1] == "u") & (replaced_word[-2] in ["ủ", "ụ", "ù", "ủ"]):
replaced_word = replaced_word[:-1]
elif (replaced_word[-1] == "o") & (replaced_word[-2] in ["ỏ", "ọ", "ò", "ỏ"]):
replaced_word = replaced_word[:-1]
return replaced_word
remove_trailing_letters(" ")
def remove_ending_letters_in_sentence(sentence):
"""A function to remove duplicated ending letters in a sentence (For example: from "Quạt yếu lắmm nhưng màaa lạaa làaa sản phẩm này rẻeee okkk lắmmmm" to "Quạt yếu lắm nhưng mà lạ là sản phẩm này rẻ ok lắm")
Parameters
----------
word : str
A sentence with duplicated ending letters
Returns
----------
str
A sentence with no duplicated ending letter
"""
words = sentence.split()
result_words = [remove_trailing_letters(word) for word in words]
return " ".join(result_words)
def replace_all_numbers(sentence):
"""Remove all numbers in a sentence
Parameters
----------
sentence : str
A sentence with numbers
Returns
----------
str
Sentence with replaced number
"""
return re.sub(r'\d+', 'number_', sentence)
def remove_links(text):
"""Removes URLs from a string.
Parameters
----------
text: The input string containing URLs.
Returns
----------
The modified string with URLs removed.
"""
# Define a regular expression pattern to match URLs
url_pattern = r"https?://\S+|\bwww\.\S+"
return re.sub(url_pattern, "LINK", text)
replace_list = {
'ầ': 'ầ', 'ỏ': 'ỏ', 'ề': 'ề','ễ': 'ễ', 'ắ': 'ắ', 'ủ': 'ủ', 'ế': 'ế', 'ở': 'ở', 'ỉ': 'ỉ',
'ẻ': 'ẻ', 'àk': u' à ','aˋ': 'à', 'iˋ': 'ì', 'ă´': 'ắ','ử': 'ử', 'e˜': 'ẽ', 'y˜': 'ỹ', 'a´': 'á',
#Chuẩn hóa 1 số sentiment words/English words
':))': ' positive ', ':)': ' positive ', 'ô kêi': ' ok ', 'okie': ' ok ', ' o kê ': ' ok ',
' okey ': ' ok ', ' ôkê ': ' ok ', ' oki ': ' ok ', ' oke ': ' ok ',' okay ':' ok ',' okê ':' ok ',
' tks ': u' cám ơn ', ' thks ': u' cám ơn ', ' thanks ': u' cám ơn ', ' ths ': u' cám ơn ', ' thank ': u' cám ơn ',
'⭐': 'star', '*': 'star', '🌟': 'star', "bik": " bị ",'not': u' không ', u' kg ': u' không ', ' k ': u' không ',' kô ':u' không ','hok':u' không ',' kp ': u' không phải ',u' kô ': u' không ', '"ko ': u' không ', u' ko ': u' không ', u' k ': u' không ', 'khong': u' không ', u' hok ': u' không ','cute': u' dễ thương ',' huhu ': ' negative ', ' vs ': u' với ', ' wa ': ' quá ', 'wá': u' quá', ' j ': u' gì ', '“': ' ',
' sz ': u' cỡ ', 'size': u' cỡ ', u' đx ': u' được ', 'dk': u' được ', ' dc ': u' được ', 'đk': u' được ',
' đc ': u' được ',' authentic ': u' chính hãng ',u' aut ': u' chính hãng ', u' auth ': u' chính hãng ', ' store ': u' cửa hàng ',
' shop ': u' cửa hàng ', ' sp ': u' sản phẩm ', ' gud ': u' tốt ',' god ': u' tốt ',' wel done ':' tốt ', ' good ': u' tốt ', ' gút ': u' tốt ',
' sấu ': u' xấu ',' gut ': u' tốt ', u' tot ': u' tốt ', u' nice ': u' tốt ', ' perfect ': 'rất tốt',
'time': u' thời gian ', 'qá': u' quá ', u' ship ': u' giao hàng ', u' m ': u' mình ', u' mik ': u' mình ',
' ể ': 'ể', ' product ': 'sản phẩm', ' quality ': 'chất lượng', ' excelent ': 'tốt', ' bad ': ' tệ ',' fresh ': ' tươi ','sad': ' tệ ',
' date ': u' hạn sử dụng ', ' hsd ': u' hạn sử dụng ',' quickly ': u' nhanh ', ' quick ': u' nhanh ',' fast ': u' nhanh ',' delivery ': u' giao hàng ',u' síp ': u' giao hàng ',
' beautiful ': u' đẹp ', u' tl ': u' trả lời ', u' r ': u' rồi ', u' shope ': u' cửa hàng ',u' order ': u' đặt hàng ',
' chất lg ': u' chất lượng ',u' sd ': u' sử dụng ',u' dt ': u' điện thoại ',u' nt ': u' nhắn tin ',u' tl ': u' trả lời ',u' sài ': u' xài ',u' bjo ':u' bao giờ ',
' thik ': u' thích ',u' sop ': u' cửa hàng ', ' fb ': ' facebook ', ' face ': ' facebook ', " zl ": " zalo ", ' very ': u' rất ',
' dep ': u' đẹp ',u' xau ': u' xấu ',' delicious ': u' ngon ', u' hàg ': u' hàng ', u' qủa ': u' quả ',
' iu ': u' yêu ','fake': u' giả mạo ', ' trl ': 'trả lời', '><': u' positive ',
' por ': u' tệ ',' poor ': u' tệ ', ' ib ':u' nhắn tin ', ' rep ':u' trả lời ',u' fback ':' feedback ',' fedback ':' feedback ', " đag ": " đang "}
def replace_sent(text, replace_list = replace_list):
"""Remove elements in a text according to the replacement dictionary
Parameters
----------
text : str
Text that needs replacement
Returns
----------
str
Text with all elements in the replacement dictionary replaced
"""
for k, v in replace_list.items():
text = text.replace(k, v)
return text
def remove_non_alphanumeric(text):
"""Removes all characters except alphanumeric characters and spaces from a string.
Args:
text: The input string.
Returns:
The modified string with only alphanumeric characters and spaces.
"""
pattern = r"[^\w\s]" # Matches any character except alphanumeric and space
return re.sub(pattern, "", text)
def remove_all_tag(sentence):
"""Remove tags added from reading files
Parameters
----------
text : str
Text that has tags
Returns
----------
str
Text with no tags
"""
return re.sub(r'\n|\t|\r', ' ', sentence)
def cleaning(sentence):
""" A function to process a sentence in following order:
1. lowercase all character
2. replace numbers with tags
3. replace links with tags
4. normalize vietnamese text
5. remove formatting tags
6. remove punctuation
7. replace shorthand
8. remove trailing letters in words
----------
sentence : str
input string
Returns
----------
str
processed string
"""
sentence = sentence.lower()
sentence = replace_all_numbers(sentence)
sentence = remove_links(sentence)
sentence = text_normalize(sentence)
sentence = remove_all_tag(sentence)
sentence = remove_non_alphanumeric(sentence)
sentence = replace_sent(sentence, replace_list)
sentence = remove_ending_letters_in_sentence(sentence)
return sentence
def cleaning_for_phobert(sentence):
sentence = text_normalize(sentence)
sentence = remove_all_tag(sentence)
sentence = remove_ending_letters_in_sentence(sentence)
return sentence |