Prepare for experiment and add morphology to bert

pages/2_👨‍🏫_Начало_работы.py

@@ -128,13 +128,13 @@ if st.session_state.get('-LOGGED_IN_BOOL-'):
             key='-TARGET_WORDS_MODE-', horizontal=True)
         DISTRACTOR_MODEL = DISTRACTOR_MODEL_COL.radio(
             label='**Модель для выбора неправильных вариантов**',
-            options=['Худ. лит-ра', 'Новости', 'Все вместе', 'BERT'],
+            options=['Модель-1', 'Модель-2'],
             key='-DISTRACTOR_MODEL_MODE-', horizontal=True)
         CEFR_NUM_DISTRACTORS_COL, UTW_COL = FORM.columns([2, 2])
         with CEFR_NUM_DISTRACTORS_COL:
             CEFR_TEXT_LEVEL = custom_select_box(
                 'Укажите уровень по CEFR:',
-                ['A1', 'A2', 'B1', 'B2', 'C1', 'C2', 'Нет'],
+                ['Без уровня', 'A1', 'A2', 'B1', 'B2', 'C1', 'C2'],
                 no_selection_label='-Выберите языковой уровень-')
             st.session_state['-CEFR_TEXT_LEVEL-'] = CEFR_TEXT_LEVEL
         NUMBER_DISTRACTORS = CEFR_NUM_DISTRACTORS_COL.number_input(
@@ -186,7 +186,7 @@ if st.session_state.get('-LOGGED_IN_BOOL-'):
             PROGRESS_BAR_S = st.progress(0)
 
             # Start generation process. Everything happens inside main_workflow func
-            if DISTRACTOR_MODEL == 'BERT':
+            if DISTRACTOR_MODEL == 'Модель-2':
                 from utilities_language_bert.esp_main_workflow_bert import main_workflow
                 __TASK_DATA__ = main_workflow(
                     file=UPLOAD_FILE,

utilities_language_bert/esp_main_workflow_bert.py

@@ -122,7 +122,7 @@ def main_workflow(
     elif level == 'C2':
         target_minimum = esp_constants.c2_target_set
         distractor_minimum = esp_constants.c2_distractor_set
-    elif level == 'Нет':
+    elif level == 'Без уровня':
         target_minimum = None
         distractor_minimum = None
     else:
@@ -150,8 +150,8 @@ def main_workflow(
                                      target_minimum=target_minimum,
                                      user_target_words=USER_TARGET_WORDS,
                                      frequency_dict=FREQ_DICT)
-        progress.progress(int(30 + (j * (30 / len(workflow)))))
-    progress_s.progress(60)
+        progress.progress(int(30 + (j * (20 / len(workflow)))))
+    progress_s.progress(50)
     DUPLICATE_TARGET_WORDS = defaultdict(list)
     for sentence in workflow:
         for target_word in sentence.target_words:
@@ -164,7 +164,7 @@ def main_workflow(
             if target_word not in RESULT_TW:
                 global_bad_target_words.append(target_word['original_text'])
                 sentence.target_words.remove(target_word)
-    progress_s.progress(65)
+    progress_s.progress(55)
     logs.success('Выбрали слова-пропуски!')
 
     for sentence in workflow:
@@ -176,7 +176,7 @@ def main_workflow(
 
     for sentence in workflow:
         sentence.filter_target_words(target_words_automatic_mode=tw_mode_automatic_mode)
-    progress_s.progress(65)
+    progress_s.progress(60)
 
     RESULT_TASKS = []
     for sentence in workflow:
@@ -193,9 +193,14 @@ def main_workflow(
             f'Обработали {num}/{len(RESULT_TASKS)} целевых слов!')
     logs_d.success(
         f'Обработали {len(RESULT_TASKS)}/{len(RESULT_TASKS)} целевых слов!')
+    progress_s.progress(65)
+    logs.success('Подобрали неправильные варианты!')
 
+    for task in RESULT_TASKS:
+        task.inflect_distractors()
     progress_s.progress(70)
-    logs.success('Подобрали неправильные варианты!')
+    logs.success('Просклоняли и проспрягали неправильные варианты!')
+
     for task in RESULT_TASKS:
         task.sample_distractors(num_distractors=num_distractors)
     progress_s.progress(75)

utilities_language_bert/esp_sentence_bert.py

@@ -2,6 +2,7 @@ import string
 from random import random
 from random import sample
 from utilities_language_general.esp_constants import nlp
+from utilities_language_general.morphology import inflect
 from utilities_language_general.esp_constants import PHRASES
 from utilities_language_general.esp_utils import check_token_bert
 from utilities_language_general.esp_utils import fix_irregular_lemma
@@ -169,7 +170,9 @@ class TASK:
         self.distractors = None
         self.distractors_number = 0
         self.bad_target_word = False
+        self.inflected_distractors = None
         self.pos = task_data['pos']
+        self.tags = task_data['tags']
         self.lemma = task_data['lemma']
         self.gender = task_data['gender']
         self.max_num_distractors = max_num_distractors
@@ -208,20 +211,40 @@ class TASK:
             self.distractors = [d[0] for i, d in enumerate(distractors_sentence) if i < 15]
             self.distractors_number = len(distractors_sentence) if distractors_sentence is not None else 0
 
+    def inflect_distractors(self):
+        inflected_distractors = []
+        for distractor_lemma, distractor_similarity in self.distractors:
+            if distractor_lemma.count('_') > 1:
+                if distractor_lemma.startswith('haber_'):
+                    distractor_lemma = distractor_lemma.split('_')[-2]
+                    inflected = inflect(lemma=distractor_lemma, target_pos=self.pos, target_tags=self.tags)
+                else:
+                    continue
+            else:
+                inflected = inflect(lemma=distractor_lemma, target_pos=self.pos, target_tags=self.tags)
+            if inflected is not None:
+                inflected_distractors.append(inflected)
+        num_distractors = min(4, self.max_num_distractors) if self.max_num_distractors >= 4 \
+            else self.max_num_distractors
+        if len(inflected_distractors) < num_distractors:
+            self.bad_target_word = True
+        else:
+            self.inflected_distractors = inflected_distractors
+
     def sample_distractors(self, num_distractors):
         if not self.bad_target_word:
             num_distractors = min(self.distractors_number, num_distractors) if num_distractors >= 4 else num_distractors
             self.distractors = sample(self.distractors[:min(self.distractors_number, 10)], num_distractors)
 
     def compile_task(self, max_num_distractors):
-        len_distractors = len(self.distractors)
+        len_distractors = len(self.inflected_distractors)
         len_variants = min(len_distractors, max_num_distractors) if max_num_distractors > 4 \
             else max_num_distractors
         letters = (f'({letter})' for letter in string.ascii_lowercase[:len_variants + 1])
         try:
-            distractors = sample(self.distractors, len_variants) + [self.original_text, ]
+            distractors = sample(self.inflected_distractors, len_variants) + [self.original_text, ]
         except ValueError:
-            distractors = self.distractors + [self.original_text, ]
+            distractors = self.inflected_distractors + [self.original_text, ]
         tmp_vars = [f'{item[0]} {item[1].replace("_", " ")}'
                     for item in zip(letters, sorted(distractors, key=lambda _: random()))]
         self.variants.append((self.original_text, tmp_vars))

utilities_language_general/esp_constants.py

@@ -23,9 +23,7 @@ def load_bert():
 
 
 nlp = load_spacy()
-news_model_path = r'news_phrases_s300_cw10_mc50_w4_negative5-075_mean_e10_notshr.bin.gz'
-all_model_path = r'ALL_annot_all_pos_spell_g_h_new_phrases_s300_cw10_mc100_w4_negative5-075_mean_e10_shr.bin.gz'
-lit_model_path = r'LITERATURA_annot_all_pos_spell_g_h_phrases_s300_cw10_mc50_w4_negative_5-075_mean_e20_shr.bin.gz'
+w2v_model_path = r'ALL_annot_all_pos_spell_g_h_new_phrases_s300_cw10_mc100_w4_negative5-075_mean_e10_shr.bin.gz'
 
 # Upload minimums
 a1_path, a1_target_set = r'lexical_minimums/A1_MINIMUM.txt', set()

utilities_language_general/esp_utils.py

@@ -190,23 +190,22 @@ def get_distractors_from_model_bert(model, text_with_masked_task: str, lemma: st
                                                         targets=list(distractor_minimum))]
         else:
             bert_candidates = [token for token in model(text_with_masked_task, top_k=max_num_distractors + 100)]
-        inflected_candidates = []
+        candidates = []
         for candidate in bert_candidates:
             if isinstance(candidate, list):
                 bert_candidates = candidate
                 continue
             if candidate['token_str'].isalpha():
                 candidate_morph = nlp(candidate['token_str'])[0]
-                inflected_candidates.append((f"{candidate_morph.lemma_}_{candidate_morph.text}_{candidate_morph.pos_}",
-                                             candidate['score']))
+                candidates.append((f"{candidate_morph.lemma_}_{candidate_morph.pos_}", candidate['score']))
     except KeyError:
         return None
-    for candidate_distractor in inflected_candidates:
+    for candidate_distractor in candidates:
         if '_' in candidate_distractor[0]:
-            distractor_lemma, distractor_text, distractor_pos,  = candidate_distractor[0].split('_')
+            distractor_lemma, distractor_pos = candidate_distractor[0].split('_')
         else:
-            distractor_lemma, distractor_text, distractor_pos = \
-                nlp(candidate_distractor[0])[0].lemma_, candidate_distractor[0], nlp(candidate_distractor[0])[0].pos_
+            candidate_morph = nlp(candidate_distractor[0])[0]
+            distractor_lemma, distractor_pos = candidate_morph.lemma_, candidate_morph.pos_
         distractor_similarity = candidate_distractor[1]
         candidate_gender = get_tags(distractor_lemma).get('Gender')
         length_ratio = abs(len(lemma) - len(distractor_lemma))
@@ -222,10 +221,10 @@ def get_distractors_from_model_bert(model, text_with_masked_task: str, lemma: st
                      / ((len(lemma) + len(distractor_lemma)) / 2) > min_edit_distance_ratio)):
             if distractor_minimum is not None:
                 if distractor_lemma in distractor_minimum:
-                    _distractors.append((distractor_text, candidate_distractor[1]))
+                    _distractors.append((distractor_lemma, candidate_distractor[1]))
                     global_distractors.add(distractor_lemma)
             else:
-                _distractors.append((distractor_text, candidate_distractor[1]))
+                _distractors.append((distractor_lemma, candidate_distractor[1]))
     num_distractors = min(4, max_num_distractors) if max_num_distractors >= 4 else max_num_distractors
     if len(_distractors) < num_distractors:
         return None

utilities_language_w2v/esp_main_workflow_w2v.py

@@ -11,9 +11,7 @@ from utilities_language_general.esp_constants import load_w2v
 from utilities_language_general.esp_utils import prepare_tasks
 from streamlit.runtime.uploaded_file_manager import UploadedFile
 import utilities_language_general.esp_constants as esp_constants
-from utilities_language_general.esp_constants import all_model_path
-from utilities_language_general.esp_constants import lit_model_path
-from utilities_language_general.esp_constants import news_model_path
+from utilities_language_general.esp_constants import w2v_model_path
 from utilities_language_general.esp_utils import prepare_target_words
 from utilities_language_general.esp_utils import compute_frequency_dict
 from utilities_language_general.esp_constants import BAD_USER_TARGET_WORDS
@@ -65,12 +63,7 @@ def main_workflow(
     MAX_FREQUENCY = 0
 
     # Define which model is used for distractor generation
-    if distractor_model == 'Худ. лит-ра':
-        mask_filler = load_w2v(lit_model_path)
-    elif distractor_model == 'Новости':
-        mask_filler = load_w2v(news_model_path)
-    else:
-        mask_filler = load_w2v(all_model_path)
+    mask_filler = load_w2v(w2v_model_path)
 
     # Get input text
     if file is not None:
@@ -136,7 +129,7 @@ def main_workflow(
     elif level == 'C2':
         target_minimum = esp_constants.c2_target_set
         distractor_minimum = esp_constants.c2_distractor_set
-    elif level == 'Нет':
+    elif level == 'Без уровня':
         target_minimum = None
         distractor_minimum = None
     else: