requiremtn fix

app.py

@@ -1,6 +1,7 @@
 import streamlit as st
 from transformers import AutoTokenizer
 from fastT5 import OnnxT5,get_onnx_runtime_sessions
+from keywords import tokenize_sentence, get_multipartiterank_topics,get_topicrank_topics,get_yake_topics
 from annotated_text import annotated_text
 import nltk
 nltk.download('stopwords')
@@ -8,15 +9,14 @@ nltk.download('wordnet')
 nltk.download('punkt')
 from nltk.corpus import stopwords,wordnet
 from nltk.tokenize import sent_tokenize
-from flashtext import KeywordProcessor
-import regex as re
 import string
 import subprocess
-from PIL import Image
 import logging
 import multiprocessing
 total_threads=multiprocessing.cpu_count()
 import onnxruntime as ort
+# from bertopic import BERTopic
+from sklearn.feature_extraction.text import CountVectorizer
 try:
     import pke
     logging.error("importing pke info")
@@ -84,7 +84,7 @@ model_t5=OnnxT5(model_or_model_path=t5_chkpt,onnx_model_sessions=model_session)
 tokenizer_t5=AutoTokenizer.from_pretrained(t5_tokenizer)
 
 def create_question_t5(model,tokenizer,context,answer,max_length=64):
-    input = "context: %s answer: %s " % (context, answer)
+    input = "context: %s answer: %s </s>" % (context, answer)
     features=tokenizer([input],return_tensors='pt')
     output=model.generate(input_ids=features['input_ids'],
                           attention_mask=features['attention_mask'],
@@ -94,7 +94,7 @@ def create_question_t5(model,tokenizer,context,answer,max_length=64):
     return tokenizer.decode(output.squeeze(), skip_special_tokens=True)
 
 def create_answers_t5(model,tokenizer,context,question,max_length=128):
-    input = "context: %s question: %s " % (context, question)
+    input = "context: %s question: %s </s>" % (context, question)
     features=tokenizer([input],return_tensors='pt')
     output=model.generate(input_ids=features['input_ids'],
                           attention_mask=features['attention_mask'],
@@ -124,10 +124,8 @@ c1,c2,c3=st.columns(3)
 with c1:
     create_usingkeyword = st.button("Create Questions using Keywords")
     if create_usingkeyword:
-
-        from keywords import tokenize_sentence,get_noun_adj_verb
         tokenized_sent = tokenize_sentence(input_context)
-        keywords_noun_adj_verb = get_noun_adj_verb(input_context)
+        keywords_noun_adj_verb = get_multipartiterank_topics(input_context)
         t5_questions=[]
 
         with st.spinner("Creating Questionsssss"):
@@ -144,6 +142,6 @@ with c1:
             st.markdown("---")
 
 with c2:
-    create_usingtopics = st.button("Create Questions using Topic Modelling")
-    if create_usingtopics:
+    create_usinglongformer = st.button("Create Questions using Longformer")
+    if create_usinglongformer:
         pass

extract_config.py

@@ -0,0 +1,8 @@
+from transformers import BertConfig,BertForMaskedLM
+
+config=BertConfig()
+model=BertForMaskedLM(config)
+
+print(config)
+
+print(model.config)

keywords.py

@@ -4,8 +4,6 @@ nltk.download('wordnet')
 nltk.download('punkt')
 from nltk.corpus import stopwords,wordnet
 from nltk.tokenize import sent_tokenize
-from flashtext import KeywordProcessor
-import regex as re
 import string
 import subprocess
 import logging
@@ -19,24 +17,40 @@ except:
     subprocess.run(['python3' ,'-m' ,'spacy' ,'download' ,'en'])
     import pke
 
+stoplist = list(string.punctuation)
+stoplist += pke.lang.stopwords.get('en')
+stoplist += ['-lrb-', '-rrb-', '-lcb-', '-rcb-', '-lsb-', '-rsb-']
+stoplist += stopwords.words('english')
 
 def tokenize_sentence(text):
     sentences=sent_tokenize(text)
     sentences=[s.strip().lstrip().rstrip() for s in sentences if len(s) > 20]
     return sentences
 
-def get_noun_adj_verb(text):
+def get_multipartiterank_topics(text):
     output = []
     try:
         extractor = pke.unsupervised.MultipartiteRank()
-        extractor.load_document(input=text, language='en',normalization=None)
+        extractor.load_document(input=text, language='en',normalization=None,stoplist=stoplist)
         # keyphrase candidate selection #'ADJ' 'ADP' 'ADV' 'AUX' 'DET' 'NOUN' 'NUM' 'PART' 'PROPN' 'PUNCT' 'VERB'
-        extractor.candidate_selection(pos={'NOUN', 'VERB', 'ADJ'})
+        extractor.candidate_selection(pos={'NOUN','VERB','ADJ'})
+        extractor.candidate_weighting(threshold=0.7,method='average',alpha=1.1)
+        keyphrases = extractor.get_n_best(n=5)
 
-        # candidate weighting,
-        extractor.candidate_weighting(threshold=0.9,method='average',alpha=1.1)
+        for val in keyphrases:
+            output.append(val[0])
+    except Exception as e:
+        print("found exception",e)
+    return list(set(output))
 
-        #extract top n
+def get_topicrank_topics(text):
+    output = []
+    try:
+        extractor = pke.unsupervised.TopicRank()
+        extractor.load_document(input=text, language='en',normalization=None,stoplist=stoplist)
+        # keyphrase candidate selection #'ADJ' 'ADP' 'ADV' 'AUX' 'DET' 'NOUN' 'NUM' 'PART' 'PROPN' 'PUNCT' 'VERB'
+        extractor.candidate_selection(pos={'NOUN', 'ADJ'})
+        extractor.candidate_weighting(threshold=0.7,method='average')
         keyphrases = extractor.get_n_best(n=5)
 
         for val in keyphrases:
@@ -45,3 +59,19 @@ def get_noun_adj_verb(text):
         print("found exception",e)
     return list(set(output))
 
+def get_yake_topics(text):
+    #statistics model --very poor performance
+    output = []
+    try:
+        extractor = pke.unsupervised.YAKE()
+        extractor.load_document(input=text, language='en',normalization=None,stoplist=stoplist)
+        extractor.candidate_selection(n=3)
+        extractor.candidate_weighting(window=2)
+        keyphrases = extractor.get_n_best(n=5,threshold=0.9)
+
+        for val in keyphrases:
+            output.append(val[0])
+    except Exception as e:
+        print("found exception",e)
+    return list(set(output))
+

requirements.txt

@@ -25,4 +25,6 @@ tokenizers~=0.12.1
 flatbuffers~=1.12
 filelock~=3.6.0
 sacremoses~=0.0.53
-fastT5~=0.1.4
+fastT5~=0.1.4
+nltk~=3.6
+st-annotated-text~=3.0.0