distractor classification for bert

language_data/model3_with_wn_catboost_classifier.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d27b12b7d7c7aa81da02aba229941ffef9e51879be6673c4f389bea10cd1a2db
+size 2425245

language_data/model3_with_wn_minmaxscaler.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c415fb5e8b4258876b11043f43593fde8026456202629c5280cc59a1a5c5351b
+size 1404

utilities_cookies/cookie_manager.py

@@ -1,10 +1,8 @@
 import streamlit as st
 from pathlib import Path
-from typing import Mapping
-from datetime import datetime
-from datetime import timedelta
 from urllib.parse import unquote
-from typing import MutableMapping
+from datetime import datetime, timedelta
+from typing import Mapping, MutableMapping
 from streamlit.components.v1 import components
 
 

utilities_cookies/encrypted_cookie_manager.py

@@ -1,10 +1,8 @@
 import os
 import base64
 import streamlit as st
-from typing import Tuple
-from typing import Optional
+from typing import Tuple, Optional, MutableMapping
 from cryptography import fernet
-from typing import MutableMapping
 from cryptography.fernet import Fernet
 from cryptography.hazmat.primitives import hashes
 from utilities_cookies.cookie_manager import CookieManager

utilities_database/user_database_utils.py

@@ -1,9 +1,9 @@
-import re
-import json
-import secrets
 import pandas as pd
 import streamlit as st
+from json import loads
+from re import search, compile
 from trycourier import Courier
+from secrets import token_urlsafe
 from argon2 import PasswordHasher
 from argon2.exceptions import VerifyMismatchError
 
@@ -37,7 +37,7 @@ def check_valid_name(name_sign_up: str) -> bool:
     name_regex_eng = r'^[A-Za-z_]\w *'
     name_regex_rus = r'^[А-Яа-я_][А-Яа-я0-9_] *'
 
-    if re.search(name_regex_eng, name_sign_up) or re.search(name_regex_rus, name_sign_up):
+    if search(name_regex_eng, name_sign_up) or search(name_regex_rus, name_sign_up):
         return True
     return False
 
@@ -46,12 +46,8 @@ def check_valid_email(email_sign_up: str) -> bool:
     """
     Checks if the user entered a valid email while creating the account.
     """
-    regex = re.compile(r'([A-Za-z0-9]+[.-_])*[A-Za-z0-9]+@[A-Za-z0-9-]+(\.[A-Z|a-z]{2,})+')
+    regex = compile(r'([A-Za-z0-9]+[.-_])*[A-Za-z0-9]+@[A-Za-z0-9-]+(\.[A-Z|a-z]{2,})+')
     return True
-    
-    # if re.fullmatch(regex, email_sign_up):
-    #     return True
-    # return False
 
 
 def check_unique_email(user_log_in_database, email_sign_up: str) -> bool:
@@ -133,7 +129,7 @@ def generate_random_passwd() -> str:
     Generates a random password to be sent in email.
     """
     password_length = 10
-    return secrets.token_urlsafe(password_length)
+    return token_urlsafe(password_length)
 
 
 def send_passwd_in_email(auth_token: str, user_name_forgot_passwd: str, email_forgot_passwd: str, company_name: str,
@@ -266,5 +262,5 @@ def load_users_particular_task(user_task_database, load_mode, creator_name, save
                                                 .eq('save_name', save_name)\
                                                 .eq('save_type', load_mode)\
                                                 .eq('cefr_level',cefr_level).execute().data[0]['generated_result']
-    return_data = json.loads(return_data.replace("'", '"'), strict=False)
+    return_data = loads(return_data.replace("'", '"'), strict=False)
     return return_data

utilities_database/user_database_widgets.py

@@ -3,7 +3,6 @@ from datetime import datetime
 from supabase import create_client, Client
 from utilities_option_menu.option_menu import option_menu
 import utilities_database.user_database_utils as db_utils
-from utilities_database.user_database_utils import check_usr_pass
 from utilities_cookies.encrypted_cookie_manager import EncryptedCookieManager
 
 DB_URL = st.secrets['SUPABASE_URL']
@@ -91,7 +90,7 @@ class LogIn:
                 login_submit_button = st.form_submit_button(label='Войти')
 
                 if login_submit_button:
-                    authenticate_user_check = check_usr_pass(user_log_in_database=user_login_table,
+                    authenticate_user_check = db_utils.check_usr_pass(user_log_in_database=user_login_table,
                                                              user_name=user_name,
                                                              password=password)
 

utilities_language_bert/esp_main_workflow_bert.py

@@ -1,29 +1,20 @@
 import datetime
 from io import StringIO
+from typing import Union
 from random import sample
 from collections import defaultdict
-from streamlit import progress as st_progress
-from streamlit.elements import WIDGETS as ST_WIDGETS
-from utilities_language_general.esp_constants import st
-from utilities_language_bert.esp_sentence_bert import TASK
-from utilities_language_bert.esp_sentence_bert import SENTENCE
-from utilities_language_general.esp_utils import prepare_tasks
-from utilities_language_general.esp_constants import load_bert
 from streamlit.runtime.uploaded_file_manager import UploadedFile
-import utilities_language_general.esp_constants as esp_constants
-from utilities_language_general.esp_constants import summarization
-from utilities_language_general.esp_utils import prepare_target_words
-from utilities_language_general.esp_utils import compute_frequency_dict
-from utilities_language_general.esp_constants import BAD_USER_TARGET_WORDS
-
+from utilities_language_bert.esp_sentence_bert import TASK, SENTENCE
+from utilities_language_general.esp_utils import prepare_tasks, prepare_target_words, compute_frequency_dict
+from utilities_language_general.esp_constants import st, load_bert, load_classifiers, nlp, summarization, BAD_USER_TARGET_WORDS, MINIMUM_SETS
 
 
 def main_workflow(
-        file: UploadedFile or None,
+        file: Union[UploadedFile, None],
         text: str,
-        logs: ST_WIDGETS,
-        progress: st_progress,
-        progress_d: st_progress,
+        logs,
+        progress,
+        progress_d,
         level: str,
         tw_mode_automatic_mode: str,
         target_words: str,
@@ -58,6 +49,7 @@ def main_workflow(
     MAX_FREQUENCY = 0
 
     logs.update(label='Загружаем языковые модели и другие данные', state='running')
+    pos_dict, scaler, classifier = load_classifiers('model3')
     mask_filler = load_bert()
 
     # Get input text
@@ -67,15 +59,15 @@ def main_workflow(
     elif text != '':
         current_text = text
     else:
-        esp_constants.st.warning('Вы и текст не вставили, и файл не выбрали 😢')
+        st.warning('Вы и текст не вставили, и файл не выбрали 😢')
         current_text = ''
-        esp_constants.st.stop()
+        st.stop()
 
     # Process target words
     if tw_mode_automatic_mode == 'Самостоятельно':
         if target_words == '':
-            esp_constants.st.warning('Вы не ввели целевые слова')
-            esp_constants.st.stop()
+            st.warning('Вы не ввели целевые слова')
+            st.stop()
         # Cannot make up paradigm, so only USER_TARGET_WORDS is used
         USER_TARGET_WORDS = prepare_target_words(target_words)
         tw_mode_automatic_mode = False
@@ -89,7 +81,7 @@ def main_workflow(
                     .replace('  ', ' ').replace('…', '...').replace('…', '...')
                     .replace('—', '-').replace('\u2014', '-').replace('—', '-')
                     .replace('-\n', '').replace('\n', '%^&*'))
-    current_text_sentences = [sent.text.strip() for sent in esp_constants.nlp(current_text).sents]
+    current_text_sentences = [sent.text.strip() for sent in nlp(current_text).sents]
     logs.update(label='Получили Ваш текст!', state='running')
     progress.progress(10)
 
@@ -106,27 +98,8 @@ def main_workflow(
     progress.progress(15)
 
     # Choose necessary language minimum according to user's input
-    if level == 'A1':
-        target_minimum = esp_constants.a1_target_set
-        distractor_minimum = esp_constants.a1_distractor_set
-    elif level == 'A2':
-        target_minimum = esp_constants.a2_target_set
-        distractor_minimum = esp_constants.a2_distractor_set
-    elif level == 'B1':
-        target_minimum = esp_constants.b1_target_set
-        distractor_minimum = esp_constants.b1_distractor_set
-    elif level == 'B2':
-        target_minimum = esp_constants.b2_target_set
-        distractor_minimum = esp_constants.b2_distractor_set
-    elif level == 'C1':
-        target_minimum = esp_constants.c1_target_set
-        distractor_minimum = esp_constants.c1_distractor_set
-    elif level == 'C2':
-        target_minimum = esp_constants.c2_target_set
-        distractor_minimum = esp_constants.c2_distractor_set
-    elif level == 'Без уровня':
-        target_minimum = None
-        distractor_minimum = None
+    if level:
+        target_minimum, distractor_minimum = MINIMUM_SETS[level]
     else:
         target_minimum = None
         distractor_minimum = None
@@ -204,7 +177,11 @@ def main_workflow(
             RESULT_TASKS.append(task)
 
     for num, task in enumerate(RESULT_TASKS):
-        task.attach_distractors_to_target_word(model=mask_filler, level_name=level,
+        task.attach_distractors_to_target_word(model=mask_filler, 
+                                               scaler=scaler,
+                                               classifier=classifier,
+                                               pos_dict=pos_dict,
+                                               level_name=level,
                                                global_distractors=GLOBAL_DISTRACTORS,
                                                distractor_minimum=distractor_minimum,
                                                max_frequency=MAX_FREQUENCY)
@@ -240,8 +217,8 @@ def main_workflow(
                 NUMBER_TASKS = 10
             else:
                 NUMBER_TASKS = len(RESULT_TASKS)
-    RESULT_TASKS_in_summary = filter(lambda task: task.in_summary, RESULT_TASKS)
-    RESULT_TASTS_not_in_summary = filter(lambda task: not task.in_summary, RESULT_TASKS)
+    RESULT_TASKS_in_summary = list(filter(lambda task: task.in_summary, RESULT_TASKS))
+    RESULT_TASTS_not_in_summary = list(filter(lambda task: not task.in_summary, RESULT_TASKS))
     if len(RESULT_TASKS_in_summary) >= NUMBER_TASKS:
         RESULT_TASKS = RESULT_TASKS_in_summary
     else:

utilities_language_bert/esp_sentence_bert.py

@@ -43,6 +43,7 @@ class SENTENCE:
                 if not previous_was_phrase:
                     self.sentence_phrases.append(self.sentence_lemma_pos[i][1])
                 previous_was_phrase = False
+        self.sentence_phrases.append(self.sentence_lemma_pos[-1][1])
 
     def search_target_words_automatically(self, target_minimum: set, frequency_dict: dict = None, summary:list=None):
         for token in self.sentence_phrases:
@@ -188,11 +189,11 @@ class TASK:
     def __repr__(self):
         return '\n'.join([f'{key}\t=\t{value}' for key, value in self.__dict__.items()])
 
-    def attach_distractors_to_target_word(self, model, global_distractors, distractor_minimum,
-                                          level_name, max_frequency):
+    def attach_distractors_to_target_word(self, model, scaler, classifier, pos_dict, 
+                                          global_distractors, distractor_minimum, level_name, max_frequency):
         pos = self.pos[0] if self.pos[0] == 'phrase' else self.pos[1]
-        distractors_sentence = get_distractors_from_model_bert(model=model, lemma=self.lemma, pos=pos,
-                                                               gender=self.gender, level_name=level_name,
+        distractors_sentence = get_distractors_from_model_bert(model=model, scaler=scaler, classifier=classifier, pos_dict=pos_dict,
+                                                               level_name=level_name, lemma=self.lemma, pos=pos, gender=self.gender, 
                                                                text_with_masked_task=self.masked_sentence,
                                                                global_distractors=global_distractors,
                                                                distractor_minimum=distractor_minimum,

utilities_language_general/esp_constants.py

@@ -2,7 +2,6 @@ import json
 import spacy
 import gensim
 import streamlit as st
-
 from pickle import load
 from transformers import pipeline
 from summarizer import Summarizer

utilities_language_general/esp_utils.py

@@ -192,9 +192,8 @@ def get_distractors_from_model(doc, model, scaler, classifier, pos_dict:dict, ta
         return distractors
 
 
-def get_distractors_from_model_bert(doc, model, scaler, classifier, text_with_masked_task: str, lemma: str, pos: str, gender: str, lemma_index:int,
-                                    global_distractors: set, distractor_minimum: set, level_name: str, pos_dict:dict,
-                                    max_num_distractors: int, max_length_ratio=5, min_edit_distance_ratio=0.5):
+def get_distractors_from_model_bert(model, scaler, classifier, pos_dict:dict, level_name: str, lemma: str, pos: str, gender: str, text_with_masked_task: str, 
+                                    global_distractors: set, distractor_minimum: set, max_num_distractors: int, max_length_ratio=5, min_edit_distance_ratio=0.5):
     _distractors = []
     try:
         bert_candidates = [token for token in model(text_with_masked_task, top_k=max_num_distractors + 100)]
@@ -217,9 +216,9 @@ def get_distractors_from_model_bert(doc, model, scaler, classifier, text_with_ma
         distractor_similarity = candidate_distractor[1]
         candidate_gender = get_tags(distractor_lemma).get('Gender')
         length_ratio = abs(len(lemma) - len(distractor_lemma))
-        decision = make_decision(doc, model_type='bert', scaler=scaler, classifier=classifier, pos_dict=pos_dict, 
-                                    level=level_name, target_text=lemma, target_pos=pos, target_position=lemma_index, 
-                                    substitute_text=distractor_lemma, substitute_pos=distractor_pos)
+        decision = make_decision(doc=None, model_type='bert', scaler=scaler, classifier=classifier, pos_dict=pos_dict, level=level_name, 
+                                 target_lemma=lemma, target_text=None, target_pos=pos, target_position=None, 
+                                 substitute_lemma=distractor_lemma, substitute_pos=distractor_pos, bert_score=distractor_similarity)
         if ((distractor_pos == pos 
              or (COMBINE_POS['simple'][level_name].get(pos) is not None and COMBINE_POS['simple'][level_name].get(distractor_pos) is not None
                  and distractor_pos in COMBINE_POS['simple'][level_name][pos] and pos in COMBINE_POS['simple'][level_name][distractor_pos]))

utilities_language_w2v/esp_main_workflow_w2v.py

@@ -3,16 +3,12 @@ from io import StringIO
 from typing import Union
 from random import sample
 from collections import defaultdict
-from streamlit import progress as st_progress
-from streamlit.elements import WIDGETS as ST_WIDGETS
 from streamlit.runtime.uploaded_file_manager import UploadedFile
-import utilities_language_general.esp_constants as esp_constants
 from utilities_language_w2v.esp_sentence_w2v import TASK, SENTENCE
 from utilities_language_general.esp_utils import prepare_tasks, prepare_target_words, compute_frequency_dict
 from utilities_language_general.esp_constants import st, load_w2v, load_classifiers, nlp, summarization, BAD_USER_TARGET_WORDS, MINIMUM_SETS
 
 
-
 def main_workflow(
         file: Union[UploadedFile, None],
         text: str,
@@ -84,7 +80,7 @@ def main_workflow(
                     .replace('  ', ' ').replace('…', '...').replace('…', '...')
                     .replace('—', '-').replace('\u2014', '-').replace('—', '-')
                     .replace('-\n', '').replace('\n', '%^&*'))
-    current_text_sentences = [sent.text.strip() for sent in esp_constants.nlp(current_text).sents]
+    current_text_sentences = [sent.text.strip() for sent in nlp(current_text).sents]
     logs.update(label='Получили Ваш текст!', state='running')
     progress.progress(10)