Fix bert bugs

utilities_language_bert/rus_main_workflow_bert.py

@@ -106,7 +106,7 @@ def main_workflow(
 
     # Get summary. May choose between round_summary_length and summary_length
     SUMMARY = summarization(current_text, num_sentences=round_summary_length)
-    logs.update('Нашли интересные предложения. Пригодятся!')
+    logs.success('Нашли интересные предложения. Пригодятся!')
     progress.progress(25)
 
     for sentence in workflow:
@@ -174,7 +174,7 @@ def main_workflow(
     logs.update(label='Подобрали неправильные варианты!', state='running')
 
     for task in RESULT_TASKS:
-        task.inflect_distractors()
+        task.inflect_distractors(level_name=level)
     progress.progress(80)
     logs.update(label='Просклоняли и проспрягали неправильные варианты!', state='running')
 

utilities_language_bert/rus_sentence_bert.py

@@ -150,6 +150,7 @@ class TASK:
         self.tags = task_data['tags']
         self.lemma = task_data['lemma']
         self.gender = task_data['gender']
+        self.in_summary = task_data['in_summary']
         self.max_num_distractors = max_num_distractors
         self.original_text = task_data['original_text']
         self.sentence_text = task_data['sentence_text']
@@ -180,13 +181,13 @@ class TASK:
             self.distractors = [d[0] for i, d in enumerate(distractors_sentence) if i < 15]
             self.distractors_number = len(distractors_sentence) if distractors_sentence is not None else 0
 
-    def inflect_distractors(self):
+    def inflect_distractors(self, level_name):
         inflected_distractors = []
         if self.distractors is None:
             self.bad_target_word = True
             return
         for distractor_lemma in self.distractors:
-            inflected = make_inflection(text=distractor_lemma, pos=self.pos[1], tags=self.tags)
+            inflected = make_inflection(text=distractor_lemma, pos=self.pos[1], tags=self.tags, level=level_name)
             if inflected is not None:
                 inflected_distractors.append(inflected)
         num_distractors = min(4, self.max_num_distractors) if self.max_num_distractors >= 4 \

utilities_language_general/rus_constants.py

@@ -34,7 +34,7 @@ def load_spacy():
 @st.cache_resource
 def load_bert():
     with st.spinner('Загружаю языковую модель'):
-        _pipeline = pipeline(task="fill-mask", model="a-v-white/bert-base-spanish-wwm-cased-finetuned-literature-pro")
+        _pipeline = pipeline(task="fill-mask", model="a-v-bely/ruBert-base-finetuned-russian-moshkov-child-corpus-pro")
     return _pipeline
 
 
@@ -113,6 +113,7 @@ COMBINE_POS = {
         'B2': {'VERB': ['AUX']},
         'C1': {'VERB': ['AUX']},
         'C2': {'VERB': ['AUX']},
+        'Без уровня': {'VERB': ['AUX']}
     },
     'phrase': 
     {
@@ -122,5 +123,6 @@ COMBINE_POS = {
         'B2': {'VERB': ['AUX']},
         'C1': {'VERB': ['AUX']},
         'C2': {'VERB': ['AUX']},
+        'Без уровня': {'VERB': ['AUX']}
     },
 }

utilities_language_general/rus_utils.py

@@ -41,7 +41,7 @@ def compute_frequency_dict(text: str) -> dict:
 
 
 def convert_gender(gender_spacy):
-    genders = {'Masc': 'masc', 'Fem': 'femn', 'Neut': 'neut'}
+    genders = {'Masc': 'masc', 'Fem': 'femn', 'Neut': 'neut', None: False}
     return genders[gender_spacy]
 
 
@@ -359,21 +359,23 @@ def get_distractors_from_model_bert(model, scaler, classifier, pos_dict:dict, le
             distractor_lemma, distractor_pos = candidate_morph.lemma_, candidate_morph.pos_
         distractor_similarity = candidate_distractor[1]
         candidate_gender = define_gender(distractor_lemma)
+        # print(distractor_lemma, candidate_gender, distractor_pos, pos)
         length_ratio = abs(len(lemma) - len(distractor_lemma))
         decision = make_decision(doc=None, model_type='bert', scaler=scaler, classifier=classifier, pos_dict=pos_dict, level=level_name, 
                                  target_lemma=lemma, target_text=None, target_pos=pos, target_position=None, 
                                  substitute_lemma=distractor_lemma, substitute_pos=distractor_pos, bert_score=distractor_similarity)
-        if (((distractor_pos == pos)
-             or (COMBINE_POS['phrase'][level_name].get(pos) is not None and COMBINE_POS['phrase'][level_name].get(distractor_pos) is not None
-                and distractor_pos in COMBINE_POS['phrase'][level_name][pos] and pos in COMBINE_POS['phrase'][level_name][distractor_pos]))
-                and decision
-                and distractor_lemma != lemma
-                and (len(_distractors) < max_num_distractors + 10)
-                and (candidate_gender == gender and level_name in ('B1', 'B2', 'C1', 'C2'))
-                and (length_ratio <= max_length_ratio)  # May be changed if case of phrases
-                and (distractor_lemma not in global_distractors)
-                and (edit_distance(lemma, distractor_lemma)  # May be changed if case of phrases
-                     / ((len(lemma) + len(distractor_lemma)) / 2) > min_edit_distance_ratio)):
+        condition = (((distractor_pos == pos)
+                      or (COMBINE_POS['phrase'][level_name].get(pos) is not None and COMBINE_POS['phrase'][level_name].get(distractor_pos) is not None
+                          and distractor_pos in COMBINE_POS['phrase'][level_name][pos] and pos in COMBINE_POS['phrase'][level_name][distractor_pos]))
+                          and decision
+                          and distractor_lemma != lemma
+                          and (len(_distractors) < max_num_distractors + 10)
+                          and (candidate_gender == gender and level_name in ('B1', 'B2', 'C1', 'C2'))
+                          and (length_ratio <= max_length_ratio)  # May be changed if case of phrases
+                          and (distractor_lemma not in global_distractors)
+                          and (edit_distance(lemma, distractor_lemma)  # May be changed if case of phrases
+                               / ((len(lemma) + len(distractor_lemma)) / 2) > min_edit_distance_ratio))
+        if condition:
             if distractor_minimum is not None:
                 if distractor_lemma in distractor_minimum:
                     _distractors.append((distractor_lemma, candidate_distractor[1]))

utilities_language_general/similarity_measures.py

@@ -185,6 +185,8 @@ def get_context_linked_words(doc, target_position, target_text):
  
 
 def compute_all_necessary_metrics(target_lemma, target_text, target_position, substitute_lemma, doc, model_type:str, model=None):
+    if model_type == 'bert':
+        return
 
     target_vector = get_vector_for_token(model, target_lemma)
     substitute_vector = get_vector_for_token(model, substitute_lemma)
@@ -246,9 +248,11 @@ def make_decision(doc, model_type, scaler, classifier, pos_dict, level, target_l
     metrics = compute_all_necessary_metrics(target_lemma=target_lemma, target_text=target_text, target_position=target_position, 
                                             substitute_lemma=substitute_lemma, doc=doc, model_type=model_type, model=model)
     target_multiword, substitute_multiword = target_lemma.count('_') > 2, substitute_lemma.count('_') > 2
-    data = [LEVEL_NUMBERS.get(level), pos_dict.get(target_pos), target_multiword, pos_dict.get(substitute_pos), substitute_multiword] + scaler.transform([metrics]).tolist()[0]
     if model_type == 'bert':
-        data = [LEVEL_NUMBERS.get(level), pos_dict.get(target_pos), target_multiword, pos_dict.get(substitute_pos), substitute_multiword, bert_score]
+        scaled_data = scaler.transform([[bert_score]]).tolist()[0]
+    else:
+        scaled_data = scaler.transform([metrics]).tolist()[0]
+    data = [LEVEL_NUMBERS.get(level), pos_dict.get(target_pos), target_multiword, pos_dict.get(substitute_pos), substitute_multiword] + scaled_data 
     predict = classifier.predict(data)
     return bool(predict)