File size: 2,379 Bytes
339e7f7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 | import torch
import re
import string
from underthesea import word_tokenize
def remove_stopwords_vietnamese(text):
"""
Remove stopwords from the given Vietnamese text.
Args:
text (str): The input text to remove stopwords from.
Returns:
str: The filtered text with stopwords removed.
"""
tokens = word_tokenize(text)
with open('../data/vietnamese_stopwords.txt', 'r', encoding='utf-8') as f:
stopwords = set([line.strip() for line in f.readlines()])
filtered_tokens = [token for token in tokens if token.lower() not in stopwords]
filtered_text = ' '.join(filtered_tokens)
return filtered_text
def remove_footnotes(text):
"""
Removes footnotes from the given text.
Parameters:
text (str): The text from which footnotes need to be removed.
Returns:
str: The text with footnotes removed.
"""
return re.sub(r'\[\d+\]', '', text)
def remove_punctuation(text):
"""
Removes punctuation characters from the given text.
Args:
text (str): The input text.
Returns:
str: The cleaned text with punctuation characters removed.
"""
return text.translate(str.maketrans('', '', string.punctuation))
def lowercase_text(text):
"""
Converts the given text to lowercase.
Args:
text (str): The input text.
Returns:
str: The text converted to lowercase.
"""
return text.lower()
def tokenize(text):
"""
Tokenizes the given text into words.
Args:
text (str): The input text.
Returns:
list: The list of words from the input text.
"""
return word_tokenize(text)
def custom_transform(x, w2v_model, TX=80):
"""
Define preprocess steps for DeepRNN, BiGRU, BiLSTM, BiLSTM + Attention, TextCNN
Steps:
- Lowercase
- Remove footnotes
- Remove punctuation
- Tokenize
- Pad or truncate to fixed length
- Convert to word index
"""
x = x.lower()
x = remove_footnotes(x)
x = remove_punctuation(x)
x = x.split()
if len(x) < TX:
x = x + ['<PAD>'] * (TX - len(x))
elif len(x) > TX:
x = x[:TX]
x = [w2v_model.key_to_index[word] if word in w2v_model.key_to_index else w2v_model.key_to_index['<UNK>'] for word in x]
x = torch.tensor(x, dtype=torch.long) # Ensure the tensor is of type long
return x
|