Upload functions.py

resources/functions.py

@@ -0,0 +1,42 @@
+import re
+import string
+import pandas as pd
+import nltk
+import pymorphy2
+from nltk.corpus import stopwords
+nltk.download('stopwords')
+from sentence_transformers import util
+
+stop_words = set(stopwords.words('russian'))
+morph = pymorphy2.MorphAnalyzer()
+
+def data_preprocessing_hard(text: str) -> str:
+    text = str(text)
+    text = text.lower()
+    text = re.sub('<.*?>', '', text)
+    text = re.sub(r'[^а-яА-Я\s]', '', text)
+    text = ''.join([c for c in text if c not in string.punctuation])
+    text = ' '.join([word for word in text.split() if word not in stop_words])
+    # text = ''.join([char for char in text if not char.isdigit()])
+    text = ' '.join([morph.parse(word)[0].normal_form for word in text.split()])
+    return text
+
+def find_rows_with_genres(df, genres_list):
+    # df['ganres'].fillna('', inplace=True)
+    genres_pattern = '|'.join(genres_list)
+    mask = df['ganres'].str.contains(genres_pattern, regex=True)
+    return mask
+
+def get_mask_in_range(df, range_values):
+    min_year, max_year = range_values
+    return (df['year'] >= min_year) & (df['year'] <= max_year)
+
+def recommend(model, text: str, embeddings, top_k):
+    query_embeddings = model.encode([text], convert_to_tensor=True)
+    embeddings = embeddings.to("cpu")
+    # embeddings = util.normalize_embeddings(embeddings)
+
+    query_embeddings = query_embeddings.to("cpu")
+    # query_embeddings = util.normalize_embeddings(query_embeddings)
+    hits = util.semantic_search(query_embeddings, embeddings, top_k=top_k)#, score_function=util.dot_score)
+    return hits