File size: 1,559 Bytes
76005f9 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 |
import re
import string
import pandas as pd
import nltk
import pymorphy2
from nltk.corpus import stopwords
nltk.download('stopwords')
from sentence_transformers import util
stop_words = set(stopwords.words('russian'))
morph = pymorphy2.MorphAnalyzer()
def data_preprocessing_hard(text: str) -> str:
text = str(text)
text = text.lower()
text = re.sub('<.*?>', '', text)
text = re.sub(r'[^а-яА-Я\s]', '', text)
text = ''.join([c for c in text if c not in string.punctuation])
text = ' '.join([word for word in text.split() if word not in stop_words])
# text = ''.join([char for char in text if not char.isdigit()])
text = ' '.join([morph.parse(word)[0].normal_form for word in text.split()])
return text
def find_rows_with_genres(df, genres_list):
# df['ganres'].fillna('', inplace=True)
genres_pattern = '|'.join(genres_list)
mask = df['ganres'].str.contains(genres_pattern, regex=True)
return mask
def get_mask_in_range(df, range_values):
min_year, max_year = range_values
return (df['year'] >= min_year) & (df['year'] <= max_year)
def recommend(model, text: str, embeddings, top_k):
query_embeddings = model.encode([text], convert_to_tensor=True)
embeddings = embeddings.to("cpu")
# embeddings = util.normalize_embeddings(embeddings)
query_embeddings = query_embeddings.to("cpu")
# query_embeddings = util.normalize_embeddings(query_embeddings)
hits = util.semantic_search(query_embeddings, embeddings, top_k=top_k)#, score_function=util.dot_score)
return hits |