resources/functions.py

import re
import string
import pandas as pd
import nltk
import pymorphy2
from nltk.corpus import stopwords
nltk.download('stopwords')
from sentence_transformers import util

stop_words = set(stopwords.words('russian'))
morph = pymorphy2.MorphAnalyzer()

def data_preprocessing_hard(text: str) -> str:
    text = str(text)
    text = text.lower()
    text = re.sub('<.*?>', '', text)
    text = re.sub(r'[^а-яА-Я\s]', '', text)
    text = ''.join([c for c in text if c not in string.punctuation])
    text = ' '.join([word for word in text.split() if word not in stop_words])
    # text = ''.join([char for char in text if not char.isdigit()])
    text = ' '.join([morph.parse(word)[0].normal_form for word in text.split()])
    return text

def find_rows_with_genres(df, genres_list):
    # df['ganres'].fillna('', inplace=True)
    genres_pattern = '|'.join(genres_list)
    mask = df['ganres'].str.contains(genres_pattern, regex=True)
    return mask

def get_mask_in_range(df, range_values):
    min_year, max_year = range_values
    return (df['year'] >= min_year) & (df['year'] <= max_year)

def recommend(model, text: str, embeddings, top_k):
    query_embeddings = model.encode([text], convert_to_tensor=True)
    embeddings = embeddings.to("cpu")
    # embeddings = util.normalize_embeddings(embeddings)

    query_embeddings = query_embeddings.to("cpu")
    # query_embeddings = util.normalize_embeddings(query_embeddings)
    hits = util.semantic_search(query_embeddings, embeddings, top_k=top_k)#, score_function=util.dot_score)
    return hits