Upload 9 files

app.py

@@ -0,0 +1,68 @@
+import streamlit as st
+from construction_prediction.constants import load_w2v
+from construction_prediction.construction_calculator import get_collocates_for_word_type
+
+st.title('Калькулятор')
+form = st.form('Form')
+target_word = form.text_input(label='Введите целевое слово:',
+                              placeholder='Введите целевое слово',
+                              label_visibility='collapsed'
+                              )
+target_word_pos = form.selectbox(label='Укажите часть речи целевого слова:',
+                                 options=['ADJ', 'ADVB', 'COMP', 'CONJ', 'GRND',
+                                          'INFN', 'INTJ', 'NOUN', 'NPRO', 'NUMR',
+                                          'None', 'PRCL', 'PRED', 'PREP', 'PRTF',
+                                          'PRTS', 'VERB'],
+                                 index=None,
+                                 placeholder='Укажите часть речи целевого слова',
+                                 label_visibility='collapsed'
+                                 )
+current_model = form.selectbox(label='MODEL',
+                               options=['MODEL 1: nplus', 'MODEL 2: fontanka',
+                                        'MODEL 3: librusec', 'MODEL 4: stihi_ru'],
+                               index=None,
+                               placeholder='Выберите модель подбора коллокатов',
+                               label_visibility='collapsed'
+                               )
+restrict_vocab = form.text_area(label='Restrict vocab',
+                                value='',
+                                placeholder='Restrict vocab',
+                                label_visibility='collapsed'
+                                )
+collocate_number = form.number_input(label='Количество коллокатов в выдаче:',
+                                     min_value=1,
+                                     step=1,
+                                     value=10,
+                                     format='%i',
+                                     placeholder='Количество коллокатов в выдаче',
+                                     # label_visibility='collapsed'
+                                     )
+form_button = form.form_submit_button('Запустить')
+
+if form_button:
+    if not target_word:
+        st.error('Вы не ввели целевое слово')
+        st.stop()
+    if not target_word_pos:
+        st.error('Вы не указали часть речи целевого слова')
+        st.stop()
+    if not current_model:
+        st.error('Вы не выбрали модель')
+        st.stop()
+
+    if current_model == 'MODEL 1: nplus':
+        model = load_w2v('models/nplus1_word2vec.bin')
+    elif current_model == 'MODEL 2: fontanka':
+        model = load_w2v('models/fontanka_word2vec.bin')
+    elif current_model == 'MODEL 3: librusec':
+        model = load_w2v('models/librusec_word2vec.bin')
+    else:
+        model = load_w2v('models/stihi_ru_word2vec.bin')
+
+    output = get_collocates_for_word_type(model=model,
+                                          word=target_word,
+                                          target_pos=target_word_pos,
+                                          topn=collocate_number,
+                                          restrict_vocab=restrict_vocab.split()
+                                          )
+    st.write(output)

construction_prediction/constants.py

@@ -0,0 +1,16 @@
+import streamlit as st
+from pymorphy2 import MorphAnalyzer
+from gensim.models import KeyedVectors
+
+
+@st.cache_resource(show_spinner="Загружаю модель поиска коллокатов")
+def load_w2v(model_path):
+    _w2v_model = KeyedVectors.load_word2vec_format(model_path, binary=True)
+    return _w2v_model
+
+
+@st.cache_resource(show_spinner="Загружаю морфологический анализатор")
+def load_morph():
+    _morph = MorphAnalyzer(lang='ru')
+    return _morph
+

construction_prediction/construction_calculator.py

@@ -0,0 +1,41 @@
+from construction_prediction.constants import load_morph
+
+
+morph = load_morph()
+
+
+def filter_results(model, word, target_pos, collocate_pos, topn, restrict_vocab):
+    collocates = []
+    target_word = '_'.join((word, target_pos))
+    for coll, similarity in model.similar_by_word(target_word, topn=topn, restrict_vocab=restrict_vocab):
+        try:
+            coll_word, pos = coll.split('_')
+            if pos == collocate_pos:
+                collocates.append((coll_word, similarity))
+            if len(collocates) == topn:
+                break
+        except ValueError:
+            continue
+    return collocates
+
+
+def get_collocates_for_word_type(model, word, target_pos, topn, restrict_vocab):
+    collocate_pos = 'NOUN' if target_pos == 'ADJ' else 'ADJ'
+
+    collocates = filter_results(model, word, target_pos, collocate_pos, topn * 100, restrict_vocab)
+    output = ''
+    for collocate_with_score in collocates[:topn]:
+        collocate = collocate_with_score[0]
+        similarity_score = round(collocate_with_score[1], 3)
+        noun = word if target_pos == 'NOUN' else collocate
+        adj = word if target_pos == 'ADJ' else collocate
+        try:
+            # Чтобы была конструкция, в которой один элемент склоняется
+            adj = morph.parse(adj)[0].inflect({morph.parse(noun)[0].tag.gender}).word
+            # Чтобы исключить результаты типа 'человечный человек'
+            if not adj[:3] == noun[:3]:
+                # if noun == 'день' and not 'днев' in adj and not 'недел' in adj:
+                output += f'\t{adj} {noun}: {similarity_score}\n\n'
+        except AttributeError:
+            continue
+    return output

models/fontanka_word2vec.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df598857b57857eff856a002e7ec76d01e1f77d6bb8702e3418012376530536c
+size 55259519

models/librusec_word2vec.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd5c7a940159eee9960923ab96b64677527cc71d9b7a83d10fc77c060b6a9efb
+size 32123951

models/nplus1_word2vec.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df63f6f39f0142ab8c21e46e12e2c2f410847e838315171e0690b6d130e2ffa8
+size 5735163

models/stihi_ru_word2vec.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea532b947e9a562aeabcc85157d14b8f660b13412af4b6a50f81128efaca5a4c
+size 19397974