adding the options to choose between input text and upload pdf

app.py

@@ -25,12 +25,16 @@ from sentence_transformers import SentenceTransformer, util
 import textstat
 from spellchecker import SpellChecker
 from transformers import pipeline
-
+import re
+import pymupdf
 print("***************************************************************")
 
 st.set_page_config(
     page_title="Question Generator",
     initial_sidebar_state="auto",
+    menu_items={
+        "About" : "#Hi this our project."
+    }
 )
 
 # Initialize Wikipedia API with a user agent
@@ -64,7 +68,15 @@ def load_qa_models():
 nlp, s2v = load_nlp_models()
 model, tokenizer = load_model()
 similarity_model, spell = load_qa_models()
-
+context_model = similarity_model
+
+def get_pdf_text(pdf_file):
+    doc = pymupdf.open(stream=pdf_file.read(), filetype="pdf")
+    text = ""
+    for page_num in range(doc.page_count):
+        page = doc.load_page(page_num)
+        text += page.get_text()
+    return text
 def save_feedback(question, answer,rating):
     feedback_file = 'question_feedback.json'
     if os.path.exists(feedback_file):
@@ -83,6 +95,31 @@ def save_feedback(question, answer,rating):
     with open(feedback_file, 'w') as f:
         json.dump(feedback_data, f)
 
+
+# Function to clean text
+def clean_text(text):
+    text = re.sub(r"[^\x00-\x7F]", " ", text)
+    return text
+
+# Function to create text chunks
+def segment_text(text, max_segment_length=1000):
+    """Segment the text into smaller chunks."""
+    sentences = sent_tokenize(text)
+    segments = []
+    current_segment = ""
+    
+    for sentence in sentences:
+        if len(current_segment) + len(sentence) <= max_segment_length:
+            current_segment += sentence + " "
+        else:
+            segments.append(current_segment.strip())
+            current_segment = sentence + " "
+    
+    if current_segment:
+        segments.append(current_segment.strip())
+    print(f"\n\nSegement Chunks: {segments}\n\n")
+    return segments
+
 # Function to extract keywords using combined techniques
 def extract_keywords(text, extract_all):
     doc = nlp(text)
@@ -140,6 +177,17 @@ def get_synonyms(word, n=3):
 def generate_options(answer, context, n=3):
     options = [answer]
     
+
+    # Add contextually relevant words using a pre-trained model
+    context_embedding = context_model.encode(context)
+    answer_embedding = context_model.encode(answer)
+    context_words = [token.text for token in nlp(context) if token.is_alpha and token.text.lower() != answer.lower()]
+
+    # Compute similarity scores and sort context words
+    similarity_scores = [util.pytorch_cos_sim(context_model.encode(word), answer_embedding).item() for word in context_words]
+    sorted_context_words = [word for _, word in sorted(zip(similarity_scores, context_words), reverse=True)]
+    options.extend(sorted_context_words[:n])
+
     # Try to get similar words based on sense2vec
     similar_words = get_similar_words_sense2vec(answer, n)
     options.extend(similar_words)
@@ -159,7 +207,7 @@ def generate_options(answer, context, n=3):
     if len(options) < n + 1:
         context_words = [token.text for token in nlp(context) if token.is_alpha and token.text.lower() != answer.lower()]
         options.extend(random.sample(context_words, min(n - len(options) + 1, len(context_words))))
-    
+    print(f"\n\nAll Possible Options: {options}\n\n")    
     # Ensure we have the correct number of unique options
     options = list(dict.fromkeys(options))[:n+1]
     
@@ -172,6 +220,7 @@ def generate_options(answer, context, n=3):
 def map_keywords_to_sentences(text, keywords, context_window_size):
     sentences = sent_tokenize(text)
     keyword_sentence_mapping = {}
+    print(f"\n\nSentences: {sentences}\n\n")
     for keyword in keywords:
         for i, sentence in enumerate(sentences):
             if keyword in sentence:
@@ -270,11 +319,10 @@ def main():
     if 'generated_questions' not in st.session_state:
         st.session_state.generated_questions = []
 
-    text = st.text_area("Enter text here:", value="Joe Biden, the current US president is on a weak wicket going in for his reelection later this November against former President Donald Trump.")
-
     with st.sidebar:
         st.subheader("Customization Options")
         # Customization options
+        input_type = st.radio("Select Input Preference", ("Text Input","Upload PDF"))
         num_beams = st.slider("Select number of beams for question generation", min_value=1, max_value=10, value=5)
         context_window_size = st.slider("Select context window size (number of sentences before and after)", min_value=1, max_value=5, value=1)
         num_questions = st.slider("Select number of questions to generate", min_value=1, max_value=1000, value=5)
@@ -289,31 +337,45 @@ def main():
             extract_all_keywords = st.toggle("Extract Max Keywords",value=False)
         with col2:
             enable_feedback_mode = st.toggle("Enable Feedback Mode",False)
-
+    text = None
+    if input_type == "Text Input":
+        text = st.text_area("Enter text here:", value="Joe Biden, the current US president is on a weak wicket going in for his reelection later this November against former President Donald Trump.")
+    elif input_type == "Upload PDF":
+        file = st.file_uploader("Upload PDF Files")
+        if file is not None:
+            text = get_pdf_text(file)
+    if text:
+        text = clean_text(text)
+        segments = segment_text(text)
     generate_questions_button = st.button("Generate Questions")
     if generate_questions_button and text:
         st.session_state.generated_questions = []
-        keywords = extract_keywords(text, extract_all_keywords)
-        print(f"\n\nFinal Keywords in Main Function: {keywords}\n\n")
-        keyword_sentence_mapping = map_keywords_to_sentences(text, keywords, context_window_size)
-        for i, (keyword, context) in enumerate(keyword_sentence_mapping.items()):
-            if i >= num_questions:
-                break
-            question = generate_question(context, keyword, num_beams=num_beams)
-            options = generate_options(keyword,context)
-            overall_score, relevance_score, complexity_score, spelling_correctness = assess_question_quality(context,question,keyword)
-            tpl = {
-                "question" : question,
-                "context" : context,
-                "answer" : keyword,
-                "options" : options,
-                "overall_score" : overall_score,
-                "relevance_score" : relevance_score,
-                "complexity_score" : complexity_score,
-                "spelling_correctness" : spelling_correctness,
-            }
-            st.session_state.generated_questions.append(tpl)
-
+        for text in segments:
+            keywords = extract_keywords(text, extract_all_keywords)
+            print(f"\n\nFinal Keywords in Main Function: {keywords}\n\n")
+            keyword_sentence_mapping = map_keywords_to_sentences(text, keywords, context_window_size)
+            for i, (keyword, context) in enumerate(keyword_sentence_mapping.items()):
+                if i >= num_questions:
+                    break
+                question = generate_question(context, keyword, num_beams=num_beams)
+                options = generate_options(keyword,context)
+                overall_score, relevance_score, complexity_score, spelling_correctness = assess_question_quality(context,question,keyword)
+                if overall_score < 0.5:
+                    continue
+                tpl = {
+                    "question" : question,
+                    "context" : context,
+                    "answer" : keyword,
+                    "options" : options,
+                    "overall_score" : overall_score,
+                    "relevance_score" : relevance_score,
+                    "complexity_score" : complexity_score,
+                    "spelling_correctness" : spelling_correctness,
+                }
+                st.session_state.generated_questions.append(tpl)
+    
+    # sort question based on their quality score
+    st.session_state.generated_questions = sorted(st.session_state.generated_questions,key = lambda x: x['overall_score'], reverse=True)
     # Display generated questions
     if st.session_state.generated_questions:
         st.header("Generated Questions:",divider='blue')