File size: 3,877 Bytes
360fc74 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 |
import inspect
# Патч для pymorphy2, чтобы использовать getfullargspec вместо getargspec
if not hasattr(inspect, 'getargspec'):
def getargspec(func):
specs = inspect.getfullargspec(func)
return specs.args, specs.varargs, specs.varkw, specs.defaults
inspect.getargspec = getargspec
import re
import string
import numpy as np
import torch
from torch import Tensor
import pymorphy2
from nltk.corpus import stopwords
import nltk
nltk.download('stopwords')
import spacy
import subprocess
# Попытка загрузки модели, если она не установлена
try:
nlp = spacy.load("ru_core_news_sm", disable=["parser", "ner"])
except OSError:
# Установка модели с помощью команды Spacy
subprocess.run(["python", "-m", "spacy", "download", "ru_core_news_sm"])
nlp = spacy.load("ru_core_news_sm", disable=["parser", "ner"])
# Загрузка стоп-слов для русского языка
stop_words = set(stopwords.words('russian'))
# Загрузка модели spacy для русского языка
nlp = spacy.load("ru_core_news_sm", disable=["parser", "ner"])
# Инициализация pymorphy2
morph = pymorphy2.MorphAnalyzer()
def data_preprocessing(text: str) -> str:
# Приведение к нижнему регистру
text = text.lower()
# Удаление HTML-тегов
text = re.sub(r'<.*?>', '', text)
# Удаление символов переноса строки и неразрывного пробела
text = text.replace('\n', ' ').replace('\xa0', ' ')
# Удаление пунктуации и цифр в одном шаге
text = ''.join([c for c in text if c not in string.punctuation and not c.isdigit()])
# Удаление стоп-слов и лемматизация
doc = nlp(text)
text = ' '.join([morph.parse(token.text)[0].normal_form for token in doc if token.text not in stop_words and not token.is_digit])
return text
def get_words_by_freq(sorted_words: list[tuple[str, int]], n: int = 10) -> list:
return list(filter(lambda x: x[1] > n, sorted_words))
def padding(review_int: list, seq_len: int) -> np.array:
"""Make left-sided padding for input list of tokens
Args:
review_int (list): input list of tokens
seq_len (int): max length of sequence, it len(review_int[i]) > seq_len it will be trimmed, else it will be padded by zeros
Returns:
np.array: padded sequences
"""
features = np.zeros((len(review_int), seq_len), dtype=int)
for i, review in enumerate(review_int):
if len(review) <= seq_len:
zeros = list(np.zeros(seq_len - len(review)))
new = zeros + review
else:
new = review[: seq_len]
features[i, :] = np.array(new)
return features
def preprocess_single_string(
input_string: str,
seq_len: int,
vocab_to_int: dict,
verbose: bool = False
) -> Tensor:
"""Function for all preprocessing steps on a single string
Args:
input_string (str): input single string for preprocessing
seq_len (int): max length of sequence, it len(review_int[i]) > seq_len it will be trimmed, else it will be padded by zeros
vocab_to_int (dict, optional): word corpus {'word' : int index}. Defaults to vocab_to_int.
Returns:
list: preprocessed string
"""
preprocessed_string = data_preprocessing(input_string)
result_list = []
for word in preprocessed_string.split():
try:
result_list.append(vocab_to_int[word])
except KeyError as e:
if verbose:
print(f'{e}: not in dictionary!')
pass
result_padded = padding([result_list], seq_len)[0]
return Tensor(result_padded)
|