Upload 56 files

app/backend/chromadb_utils.py

@@ -0,0 +1,61 @@
+import chromadb
+from chromadb.utils import embedding_functions
+from .text_processing import text_chunking
+
+
+def initialize_chromadb(EMBEDDING_MODEL, local_model_path=None):
+    """
+    Initialize ChromaDB client and embedding function, using a local model path if provided.
+    """
+    client = chromadb.Client()
+
+    if local_model_path:
+        embedding_func = embedding_functions.SentenceTransformerEmbeddingFunction(
+            model_name=local_model_path
+        )
+    else:
+        embedding_func = embedding_functions.SentenceTransformerEmbeddingFunction(
+            model_name=EMBEDDING_MODEL
+        )
+
+    return client, embedding_func
+
+
+def initialize_collection(client, embedding_func, collection_name):
+    """
+    Initialize a collection in ChromaDB.
+    """
+    collection = client.get_or_create_collection(
+        name=collection_name,
+        embedding_function=embedding_func,
+        metadata={"hnsw:space": "cosine"},
+    )
+
+    return collection
+
+
+def update_collection(
+    collection,
+    text,
+    max_words=200,
+    min_words=100,
+    overlap_sentences=3,
+):
+    """
+    Update the ChromaDB collection with text chunks.
+    Args:
+        collection: ChromaDB collection object.
+        text (str): The text to be chunked and added.
+        max_words (int): Maximum number of words per chunk.
+        min_words (int): Minimum number of words per chunk.
+        overlap_sentences (int): Number of sentences to overlap between chunks.
+    Returns:
+        None
+    """
+    chunks = text_chunking(text, max_words=max_words, min_words=min_words, overlap_sentences=overlap_sentences)
+    collection.add(
+        documents=chunks,
+        ids=[f"chunk_{j:04d}" for j in range(len(chunks))],
+        metadatas=[{"chunk_index": j} for j in range(len(chunks))]
+    )
+

app/backend/chunks_processing.py

@@ -0,0 +1,32 @@
+import random
+import streamlit as st
+
+
+def query_collection(collection, query='', nresults=3, context_multiplier=2, sim_th=None):
+    """Get relevant text from a collection for a given query"""
+
+    query_result = collection.query(query_texts=query, n_results=nresults*context_multiplier)
+    docs = query_result.get('documents')[0]
+
+    if sim_th is not None:
+        similarities = [1 - d for d in query_result.get("distances")[0]]
+        relevant_docs = [d for d, s in zip(docs, similarities) if s >= sim_th]
+        return ''.join(relevant_docs)
+    return docs
+
+
+def get_chapter_context(chapters, chapter_number, n_questions):
+    chapter = chapters[chapter_number]
+    print(chapter.keys())
+    if chapter is None:
+        raise ValueError(f"Chapter {chapter_number} not found in the chapters list.")
+    if 'chunks' not in chapter:
+        raise ValueError(f"Chapter {chapter_number} does not contain 'text' key.")
+    
+    n_chunks = len(chapter['chunks'])
+    if n_chunks == 0:
+        raise ValueError(f"Chapter {chapter_number} has no chunks to process.")
+    
+    chunks_indices = random.sample(range(n_chunks), min(n_questions, n_chunks))
+    st.session_state['chapter_selected_chunks'] = [chapter['chunks'][i] for i in chunks_indices]
+    

app/backend/get_requests.py

@@ -0,0 +1,37 @@
+from app.backend.runpod_client import format_messages_as_prompt, run_prompt, clean_and_parse_json
+from app.backend.messages_templates import toc_prompt, chapter_prompt, chapter_prompt_edgecase
+import streamlit as st
+
+
+def extract_chapters_from_toc(toc_text: str):
+    prompt = toc_prompt(toc_text)
+    # prompt = format_messages_as_prompt(messages) get rid of this
+    print("use prompt optimized for gemma3")
+    raw_output = run_prompt(prompt)
+    st.session_state['chapters_dict'] = clean_and_parse_json(raw_output)
+
+
+def generate_questions_from_chapter(chunks, num_questions, max_questions=5):
+    prompt = chapter_prompt(contexts=chunks, num_questions=num_questions, max_questions=max_questions)
+    # prompt = format_messages_as_prompt(messages) get rid of this
+    print("use prompt optimized for gemma3")
+    raw_output = run_prompt(prompt)
+    try:
+        generated_questions = clean_and_parse_json(raw_output)
+        st.success("Questions generated successfully!")
+        return generated_questions
+    except:
+        print("Error parsing JSON")
+
+
+def generate_questions_from_chapter_edgecase(chunks, num_questions, max_questions=5):
+    prompt = chapter_prompt_edgecase(grouped_chunks=chunks, num_questions=num_questions, max_questions=max_questions)
+    # prompt = format_messages_as_prompt(messages) get rid of this
+    print("use prompt optimized for gemma3")
+    raw_output = run_prompt(prompt)
+    try:
+        generated_questions = clean_and_parse_json(raw_output)
+        st.success("Questions generated successfully!")
+        return generated_questions
+    except:
+        print("Error parsing JSON")

app/backend/messages_templates.py

@@ -0,0 +1,265 @@
+# def get_toc_extraction_messages(toc_text: str):
+#     return [
+#         {
+#             "role": "system",
+#             "content": "You are a precise document parser that extracts structured information from table of contents. You NEVER hallucinate You NEVER hallucinate, invent, or make up information. You ONLY extract what is explicitly present in the provided text. If you cannot find clear chapter information, you return an empty array. You do not guess chapter titles or page numbers."
+#         },
+#         {
+#             "role": "user",
+#             "content": "I need to extract main chapter information from this table of contents. Only extract numbered chapters, ignore subsections. Do not make up any information."
+#         },
+#         {
+#             "role": "assistant",
+#             "content": "I understand. I will extract ONLY the main chapters that are explicitly shown in your table of contents. I will not invent, guess, or hallucinate any chapter titles or page numbers. I will only use the exact information present in the document."
+#         },
+#         {
+#             "role": "user",
+#             "content": f"""Here is the table of contents:
+
+#         {toc_text}
+
+#         WARNING: DO NOT HALLUCINATE OR INVENT INFORMATION
+#         - Do NOT make up chapter titles like "Probability", "Statistical Inference", "Linear Regression"
+#         - Do NOT guess page numbers
+#         - Do NOT create generic textbook chapters
+#         - ONLY extract what you can clearly see in the provided text
+
+#         CRITICAL RULES:
+#         1. Extract ONLY main chapters that start with a number (1, 2, 3, etc.)
+#         2. Do NOT extract subsections (like 1.1, 1.2, 2.1, etc.)
+#         3. Use the EXACT chapter titles shown in the document
+#         4. Use the EXACT page numbers shown in the document
+#         5. Handle both roman numerals (i, ii, iii, v, x) and arabic numerals (1, 25, 100)
+#         6. Calculate end pages as: next chapter's start page minus 1
+#         7. Return ONLY valid JSON - no explanations, no markdown formatting
+#         8. If you cannot clearly identify chapters, return empty array []
+
+#         Look for patterns like:
+#         - "1 Probability Theory . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 1"
+#         - "2 Distribution Theory and Statistical Models . . . . . . . . . . . . . . . . 155"
+#         - "3 Basic Statistical Theory . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 205"
+
+#         DO NOT extract lines like:
+#         - "1.1 Some Important Music Concepts . . . . . . . . . . . 3"
+#         - "Preface . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . v"
+
+#         Use ONLY the exact titles from the document. Do not shorten or modify them.
+
+#         Return JSON array: [{{"chapter_number": "X", "chapter_title": "...", "start_page": X, "end_page": X}}]
+
+#         REMEMBER: Extract only what is explicitly visible in the text. Do not hallucinate. Be complete and extract all chapters that are clearly numbered.                  y chapters, return an empty array []."""
+#                 },
+#                 {
+#                     "role": "assistant",
+#                     "content": "I will carefully examine the table of contents and extract only the main chapters that are explicitly shown, using their exact titles and page numbers. I will not invent or hallucinate any information."
+#                 }
+#     ]
+
+
+def toc_prompt(toc_text: str):
+    # Convert to Gemma 3 format - single string with proper turn markers
+    prompt = f"""<start_of_turn>user
+You are a precise document parser that extracts structured information from table of contents. You NEVER hallucinate, invent, or make up information. You ONLY extract what is explicitly present in the provided text. If you cannot find clear chapter information, you return an empty array.
+
+I need to extract main chapter information from this table of contents. Only extract numbered chapters, ignore subsections. Do not make up any information.
+
+Here is the table of contents:
+
+{toc_text}
+
+WARNING: DO NOT HALLUCINATE OR INVENT INFORMATION
+- Do NOT make up chapter titles like "Probability", "Statistical Inference", "Linear Regression"
+- Do NOT guess page numbers
+- Do NOT create generic textbook chapters
+- ONLY extract what you can clearly see in the provided text
+
+CRITICAL RULES:
+1. Extract ONLY main chapters that start with a number (1, 2, 3, etc.)
+2. Do NOT extract subsections (like 1.1, 1.2, 2.1, etc.)
+3. Use the EXACT chapter titles shown in the document
+4. Use the EXACT page numbers shown in the document
+5. Handle both roman numerals (i, ii, iii, v, x) and arabic numerals (1, 25, 100)
+6. Calculate end pages as: next chapter's start page minus 1
+7. Return ONLY valid JSON - no explanations, no markdown formatting
+8. If you cannot clearly identify chapters, return empty array []
+
+Look for patterns like:
+- "1 Probability Theory . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 1"
+- "2 Distribution Theory and Statistical Models . . . . . . . . . . . . . . . . 155"
+- "3 Basic Statistical Theory . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 205"
+
+DO NOT extract lines like:
+- "1.1 Some Important Music Concepts . . . . . . . . . . . 3"
+- "Preface . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . v"
+
+Use ONLY the exact titles from the document. Do not shorten or modify them.
+
+Return JSON array: [{{"chapter_number": "X", "chapter_title": "...", "start_page": X, "end_page": X}}]
+
+Extract only what is explicitly visible in the text. Do not hallucinate. Be complete and extract all chapters that are clearly numbered. If no clear main chapters, return an empty array [].<end_of_turn>
+<start_of_turn>model
+I will carefully examine the table of contents and extract only the main chapters that are explicitly shown, using their exact titles and page numbers. I will not invent or hallucinate any information.
+
+Looking at the provided table of contents, I will now extract the main chapters:<end_of_turn>
+<start_of_turn>user
+Perfect. Now provide the JSON array with the extracted chapters.<end_of_turn>
+<start_of_turn>model
+"""
+    
+    return prompt
+
+
+def chapter_prompt(contexts, num_questions, max_questions=5):
+    """
+    Create a prompt formatted for Gemma 3 12B-IT model.
+    This prompt is designed to generate diverse questions based on provided text contexts.
+    Args:
+        contexts (list): List of text contexts to base questions on.
+        num_questions (int): Number of questions to generate.
+        max_questions (int): Maximum number of questions allowed.  
+    Returns:
+        str: Formatted prompt string for Gemma 3 model.
+    """
+    
+    # Gemma uses special tokens for instruction tuning
+    prompt = """<start_of_turn>user
+You are a question generation expert. Generate exactly {num_questions} diverse questions based on the provided text contexts.
+
+IMPORTANT REQUIREMENTS:
+1. Output MUST be valid JSON format
+2. Generate EXACTLY {num_questions} questions
+3. Each question must have a complete answer from the contexts
+4. Vary question types (what, why, how, when, explain, compare)
+5. Do not generate yes/no questions
+6. Answers should be 1-3 sentences long
+
+CONTEXTS:
+{contexts}
+
+OUTPUT FORMAT - Return ONLY valid JSON array:
+[
+{{"question": "Your question here?", "answer": "Complete answer from the context"}},
+{{"question": "Another question?", "answer": "Another answer"}}
+]
+
+Generate the questions now:<end_of_turn>
+<start_of_turn>model
+""".format(
+        num_questions=min(num_questions, max_questions),
+        contexts=format_contexts(contexts)
+    )
+    
+    return prompt
+
+def chapter_prompt_edgecase(grouped_chunks, num_questions, max_questions=5):
+    """
+    Create a prompt formatted for Gemma 3 12B-IT model.
+    This prompt is designed to handle edge cases where contexts retrieved are less than the number of questions requested.
+    Args:
+        contexts (list): List of text contexts to format.
+    Returns:
+        str: Formatted string of contexts.
+    """
+    
+    prompt = """<start_of_turn>user
+Generate {num_questions} questions from the following contexts. You may:
+- Generate one or more questions from each context
+- Use multiple contexts for a single question
+- Skip contexts if they don't contain meaningful information
+
+REQUIREMENTS:
+1. Output valid JSON array format
+2. Generate EXACTLY {num_questions} questions
+3. Each answer must be found in the provided contexts
+4. Create diverse question types
+5. Reference which context group(s) you used
+
+CONTEXT GROUPS:
+{context_groups}
+
+OUTPUT FORMAT - Return ONLY this JSON structure:
+[
+{{"question": "Question text?", "answer": "Answer text", "context_used": [1, 2]}},
+{{"question": "Question text?", "answer": "Answer text", "context_used": [1]}}
+]
+
+Generate the questions:<end_of_turn>
+<start_of_turn>model
+""".format(
+        num_questions=min(num_questions, max_questions),
+        context_groups=format_contexts(grouped_chunks)
+    )
+    
+    return prompt
+
+
+def book_prompt(contexts, num_questions, user_query=None, max_questions=5):
+    """
+    Create a prompt formatted for Gemma 3 12B-IT model with topic awareness.
+    
+    Args:
+        contexts (list): List of text contexts retrieved based on user query
+        num_questions (int): Number of questions to generate
+        user_query (str): The original user query/topic
+        max_questions (int): Maximum number of questions allowed
+    
+    Returns:
+        str: Formatted prompt string for Gemma 3 model
+    """
+    
+    num_questions = min(num_questions, max_questions)
+    
+    # Build topic context section if query provided
+    topic_context = ""
+    if user_query:
+        topic_context = f"""
+TOPIC FOCUS: {user_query}
+The following contexts were retrieved based on this topic. Generate questions that:
+- Relate to the main topic: "{user_query}"
+- Explore different aspects of this topic found in the contexts
+- Connect the topic to broader concepts when relevant
+
+"""
+    
+    prompt = """<start_of_turn>user
+You are a question generation expert. Generate exactly {num_questions} diverse questions based on the provided text contexts.
+{topic_context}
+IMPORTANT REQUIREMENTS:
+1. Output MUST be valid JSON format
+2. Generate EXACTLY {num_questions} questions
+3. Each question must have a complete answer from the contexts
+4. Vary question types (what, why, how, when, explain, compare)
+5. Do not generate yes/no questions
+6. Answers should be 1-3 sentences long
+7. Questions should explore different aspects of the topic
+
+CONTEXTS (Retrieved based on topic: "{query}"):
+{contexts}
+
+OUTPUT FORMAT - Return ONLY valid JSON array:
+[
+{{"question": "Your question here?", "answer": "Complete answer from the context"}},
+{{"question": "Another question?", "answer": "Another answer"}}
+]
+
+Generate the questions now:<end_of_turn>
+<start_of_turn>model
+""".format(
+        num_questions=num_questions,
+        topic_context=topic_context,
+        query=user_query if user_query else "the provided content",
+        contexts=format_contexts(contexts)
+    )
+    
+    return prompt
+
+
+def format_contexts(contexts):
+    """
+    Format contexts for better readability.
+    """
+    formatted = ""
+    for i, context in enumerate(contexts, 1):
+        formatted += f"Context {i}:\n{context.strip()}\n\n"
+    return formatted.strip()
+

app/backend/raw_text_processing.py

@@ -0,0 +1,167 @@
+import fitz  # PyMuPDF
+import warnings
+import streamlit as st
+
+
+def extract_page_data_fitz(doc):
+    """
+    Extracts page numbers and text from a PDF file using PyMuPDF.
+    The function looks for page numbers in the top and bottom 15% of each page.
+    It returns a list of dictionaries, each containing the page index, page number,
+    and the full text of the page.
+    """
+    pages_data = []
+
+    for i, page in enumerate(doc):
+        height = page.rect.height
+        width = page.rect.width
+
+        top_rect = fitz.Rect(0, 0, width, height * 0.15)
+        bottom_rect = fitz.Rect(0, height * 0.85, width, height)
+
+        top_text = page.get_text("text", clip=top_rect).split()
+        bottom_text = page.get_text("text", clip=bottom_rect).split()
+
+        found_number = next((int(text) for text in top_text + bottom_text if text.isdigit()), None)
+        full_text = page.get_text("text")
+
+        pages_data.append({
+            "index": i,
+            "number": found_number,
+            "content": full_text
+        })
+
+    return pages_data
+
+
+def correct_page_numbers(pages_data, sequence_length=10):
+    """
+    Corrects page numbers by finding the first sequence of consecutive values, 
+    filling gaps forward and backward, and setting values < 1 to None. 
+    Returns the index of the first page numbered 1, or None if no sequence is found.
+    """
+    try:
+        seen = [(i, d["number"]) for i, d in enumerate(pages_data) if isinstance(d["number"], int)]
+
+        for start in range(len(seen) - sequence_length + 1):
+            if all(seen[start + j][1] == seen[start][1] + j for j in range(sequence_length)):
+                base_index, base_number = seen[start]
+                break
+        else:
+            return None
+
+        for offset, page in enumerate(pages_data[base_index:], start=0):
+            page["number"] = base_number + offset
+
+        for offset in range(1, base_index + 1):
+            page = pages_data[base_index - offset]
+            page["number"] = base_number - offset
+
+        for page in pages_data:
+            if page["number"] < 1:
+                page["number"] = None
+
+        return next((page['index'] for page in pages_data if page["number"] == 1), None)
+
+    except Exception:
+        return None
+
+
+def extract_text(doc, start_chapter=None):
+    """
+    Extracts the text of the book starting from the specified page index.
+    If no start_chapter is provided, it returns the whole doc.
+    """
+    if start_chapter is not None:
+        all_pages_text = [
+            doc[page_range].get_text("text")
+            for page_range in range(start_chapter, len(doc))
+        ]
+        return "\n".join(all_pages_text) 
+    else:
+        warnings.warn(
+            "No chapter start has been detected: extracting text from the entire PDF.",
+            UserWarning
+        )
+        return "\n".join(page.get_text("text") for page in doc)
+    
+
+def process_pdf():
+    """
+    Processes a PDF file to extract text starting from the first chapter.
+    """
+    pdf_bytes = st.session_state.get("uploaded_pdf_bytes")
+    if not pdf_bytes:
+        st.error("No PDF uploaded.")
+        return
+
+    with st.spinner("Processing uploaded file..."):
+        with fitz.open(stream=pdf_bytes, filetype="pdf") as doc:
+            pages_data_infos = extract_page_data_fitz(doc)
+            chapters_starting_page = correct_page_numbers(pages_data_infos)
+            full_text = extract_text(doc, chapters_starting_page)
+
+        st.session_state['full_text'] = full_text
+        st.session_state['pages_data_infos'] = pages_data_infos
+        st.session_state['chapters_starting_page'] = chapters_starting_page
+
+
+def extract_toc(page_range):
+    """
+    Extracts text from specific pages in a PDF file using PyMuPDF.
+    This is used to extract TOC based on a given range of page numbers indicated by the user.
+    """
+    pdf_bytes = st.session_state.get("uploaded_pdf_bytes")
+    if pdf_bytes is None:
+        st.error("No PDF uploaded.")
+        return ""
+
+    chapters_content_list = []
+    with fitz.open(stream=pdf_bytes, filetype="pdf") as doc:
+        for page_num in page_range:
+            if 0 <= page_num < len(doc):
+                text = doc[page_num].get_text("text")
+                chapters_content_list.append(text)
+            else:
+                print(f"Warning: Page number {page_num} is out of bounds.")
+
+    toc_text = "\n".join(chapters_content_list)
+    st.session_state["toc"] = toc_text
+
+
+def extract_chapters(chapters_dict, pages_data_corrected):
+    """
+    Extract chapters from the provided JSON and pages data.
+    Args:
+        chapters_json (list): List of chapter dictionaries from the TOC.
+        pages_data_corrected (list): List of page data dictionaries with content.
+    Returns:
+        list: List of dictionaries, each containing chapter details and content.
+    """
+    # Initialize an empty list to hold chapter dictionaries
+    chapters = []
+    
+    # Iterate through each chapter in the JSON
+    for chapter in chapters_dict:
+        start_page = chapter['start_page']
+        end_page = chapter['end_page']
+        chapter_text = []
+
+        # Extract content for the chapter from the pages data
+        for chapter_range in range(start_page-1, end_page):
+            chapter_text.append(pages_data_corrected[chapter_range]['content'])
+
+        chapter_text = ' '.join(chapter_text)
+
+        # Create a dictionary for the chapter
+        chapter_dict = {
+            'chapter_number': chapter['chapter_number'],
+            'chapter_title': chapter['chapter_title'],
+            'start_page': start_page,
+            'end_page': end_page,
+            'content': chapter_text
+        }
+
+        chapters.append(chapter_dict)
+    
+    st.session_state['chapters_extracted'] = chapters

app/backend/runpod_client.py

@@ -0,0 +1,68 @@
+import os
+import time
+import requests
+from dotenv import load_dotenv
+import json
+import codecs
+from pathlib import Path
+
+# Load .env from project root
+load_dotenv(dotenv_path=Path(__file__).resolve().parents[2] / ".env")
+
+API_KEY = os.getenv("RUNPOD_API_KEY")
+ENDPOINT = os.getenv("RUNPOD_ENDPOINT")
+
+HEADERS = {
+    "Authorization": f"Bearer {API_KEY}",
+    "Content-Type": "application/json"
+}
+
+
+def format_messages_as_prompt(messages):
+    """Convert messages list to a single prompt string for the model."""
+    parts = []
+    for message in messages:
+        parts.append(f"{message['role'].capitalize()}: {message['content']}")
+    parts.append("Assistant:")
+    return "\n\n".join(parts)
+
+
+def run_prompt(prompt: str) -> str:
+    """Submit a prompt to the RunPod endpoint and get back a response string."""
+    payload = {"input": 
+               {"prompt": prompt}
+               }
+
+    # Start job
+    response = requests.post(f"{ENDPOINT}/run", headers=HEADERS, json=payload)
+    job_id = response.json().get("id")
+    print(f"[RunPod] Job started: {job_id}")
+
+    # Poll for status
+    while True:
+        status_res = requests.get(f"{ENDPOINT}/status/{job_id}", headers=HEADERS).json()
+        status = status_res.get("status")
+        print(f"[RunPod] Status: {status}")
+        if status in ("COMPLETED", "FAILED"):
+            break
+        time.sleep(3)
+
+    if status == "COMPLETED":
+        return status_res["output"]["response"]
+    else:
+        raise RuntimeError("RunPod job failed.")
+
+
+def clean_and_parse_json(raw_text: str):
+    """Clean and parse model output into JSON."""
+    cleaned = raw_text.strip().strip("```json").strip("```").strip("'")
+    try:
+        return json.loads(cleaned)
+    except json.JSONDecodeError:
+        try:
+            # Handle escaped quotes
+            unescaped = codecs.decode(cleaned, 'unicode_escape')
+            return json.loads(unescaped)
+        except Exception as e:
+            raise ValueError("Could not parse JSON output") from e
+        

app/backend/text_processing.py

@@ -0,0 +1,95 @@
+from nltk.tokenize import sent_tokenize
+import nltk
+import streamlit as st
+
+try:
+    nltk.data.find("tokenizers/punkt")
+except LookupError:
+    nltk.download("punkt")
+
+def text_chunking(text, max_words=750, min_words=400, overlap_sentences=5):
+    """
+    Creates text chunks up to max_words using sentences as undivisible units.
+    Each chunk can overlap with the next one by overlap_sentences.
+    Chunks smaller than min_words are merged with the next chunk.
+    """
+    sentences = sent_tokenize(text)
+    word_counts = [len(sentence.split()) for sentence in sentences]
+    
+    chunks = []
+    i = 0
+    
+    while i < len(sentences):
+        chunk_sentences = []
+        word_count = 0
+        chunk_start = i
+        
+        # Build chunk
+        while i < len(sentences):
+            if word_count + word_counts[i] > max_words and chunk_sentences:
+                break
+            chunk_sentences.append(sentences[i])
+            word_count += word_counts[i]
+            i += 1
+        
+        if chunk_sentences:
+            chunks.append(" ".join(chunk_sentences))
+            
+            # Add overlap for next chunk
+            if i < len(sentences):
+                chunk_size = len(chunk_sentences)
+                overlap = min(overlap_sentences, chunk_size - 1)
+                i = max(i - overlap, chunk_start + 1)
+    
+    # Merge small chunks with next chunk
+    merged_chunks = []
+    i = 0
+    while i < len(chunks):
+        current_chunk = chunks[i]
+        current_words = len(current_chunk.split())
+        
+        # If current chunk is too small and there's a next chunk, merge them
+        if current_words < min_words and i + 1 < len(chunks):
+            next_chunk = chunks[i + 1]
+            next_words = len(next_chunk.split())
+            
+            # Only merge if combined size won't be too large
+            if current_words + next_words <= max_words:
+                merged_chunk = current_chunk + " " + next_chunk
+                merged_chunks.append(merged_chunk)
+                i += 2  # Skip next chunk since we merged it
+            else:
+                # Keep small chunk as-is if merging would be too large
+                merged_chunks.append(current_chunk)
+                i += 1
+        else:
+            merged_chunks.append(current_chunk)
+            i += 1
+    
+    # Remove chunks that are too long (likely data blocks or malformed content)
+    final_chunks = []
+    for chunk in merged_chunks:
+        if len(chunk.split()) <= 1000:
+            final_chunks.append(chunk)
+    
+    return final_chunks
+
+
+def chapters_chunking(chapters, max_words=500, min_words=300, overlap_sentences=5):
+    """
+    Chunk the chapters into smaller parts based on word count and overlap.
+    
+    :param chapters: List of chapter dictionaries.
+    :param max_words: Maximum number of words per chunk.
+    :param min_words: Minimum number of words per chunk.
+    :param overlap_sentences: Number of sentences to overlap between chunks.
+    :return: List of dictionaries with chapter information and their respective chunks.
+    """
+    st.session_state['chapters_chunked'] = [
+        {
+            'chapter_number': chapter['chapter_number'],
+            'chapter_title': chapter['chapter_title'],
+            'chunks': text_chunking(chapter['content'], max_words, min_words, overlap_sentences)
+        }
+        for chapter in chapters
+    ]

app/chromadb_model/1_Pooling/config.json

@@ -0,0 +1,10 @@
+{
+  "word_embedding_dimension": 384,
+  "pooling_mode_cls_token": false,
+  "pooling_mode_mean_tokens": true,
+  "pooling_mode_max_tokens": false,
+  "pooling_mode_mean_sqrt_len_tokens": false,
+  "pooling_mode_weightedmean_tokens": false,
+  "pooling_mode_lasttoken": false,
+  "include_prompt": true
+}

app/chromadb_model/README.md

@@ -0,0 +1,173 @@
+---
+language: en
+license: apache-2.0
+library_name: sentence-transformers
+tags:
+- sentence-transformers
+- feature-extraction
+- sentence-similarity
+- transformers
+datasets:
+- s2orc
+- flax-sentence-embeddings/stackexchange_xml
+- ms_marco
+- gooaq
+- yahoo_answers_topics
+- code_search_net
+- search_qa
+- eli5
+- snli
+- multi_nli
+- wikihow
+- natural_questions
+- trivia_qa
+- embedding-data/sentence-compression
+- embedding-data/flickr30k-captions
+- embedding-data/altlex
+- embedding-data/simple-wiki
+- embedding-data/QQP
+- embedding-data/SPECTER
+- embedding-data/PAQ_pairs
+- embedding-data/WikiAnswers
+pipeline_tag: sentence-similarity
+---
+
+
+# all-MiniLM-L6-v2
+This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 384 dimensional dense vector space and can be used for tasks like clustering or semantic search.
+
+## Usage (Sentence-Transformers)
+Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed:
+
+```
+pip install -U sentence-transformers
+```
+
+Then you can use the model like this:
+```python
+from sentence_transformers import SentenceTransformer
+sentences = ["This is an example sentence", "Each sentence is converted"]
+
+model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
+embeddings = model.encode(sentences)
+print(embeddings)
+```
+
+## Usage (HuggingFace Transformers)
+Without [sentence-transformers](https://www.SBERT.net), you can use the model like this: First, you pass your input through the transformer model, then you have to apply the right pooling-operation on-top of the contextualized word embeddings.
+
+```python
+from transformers import AutoTokenizer, AutoModel
+import torch
+import torch.nn.functional as F
+
+#Mean Pooling - Take attention mask into account for correct averaging
+def mean_pooling(model_output, attention_mask):
+    token_embeddings = model_output[0] #First element of model_output contains all token embeddings
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+
+# Sentences we want sentence embeddings for
+sentences = ['This is an example sentence', 'Each sentence is converted']
+
+# Load model from HuggingFace Hub
+tokenizer = AutoTokenizer.from_pretrained('sentence-transformers/all-MiniLM-L6-v2')
+model = AutoModel.from_pretrained('sentence-transformers/all-MiniLM-L6-v2')
+
+# Tokenize sentences
+encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt')
+
+# Compute token embeddings
+with torch.no_grad():
+    model_output = model(**encoded_input)
+
+# Perform pooling
+sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask'])
+
+# Normalize embeddings
+sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1)
+
+print("Sentence embeddings:")
+print(sentence_embeddings)
+```
+
+------
+
+## Background
+
+The project aims to train sentence embedding models on very large sentence level datasets using a self-supervised 
+contrastive learning objective. We used the pretrained [`nreimers/MiniLM-L6-H384-uncased`](https://huggingface.co/nreimers/MiniLM-L6-H384-uncased) model and fine-tuned in on a 
+1B sentence pairs dataset. We use a contrastive learning objective: given a sentence from the pair, the model should predict which out of a set of randomly sampled other sentences, was actually paired with it in our dataset.
+
+We developed this model during the 
+[Community week using JAX/Flax for NLP & CV](https://discuss.huggingface.co/t/open-to-the-community-community-week-using-jax-flax-for-nlp-cv/7104), 
+organized by Hugging Face. We developed this model as part of the project:
+[Train the Best Sentence Embedding Model Ever with 1B Training Pairs](https://discuss.huggingface.co/t/train-the-best-sentence-embedding-model-ever-with-1b-training-pairs/7354). We benefited from efficient hardware infrastructure to run the project: 7 TPUs v3-8, as well as intervention from Googles Flax, JAX, and Cloud team member about efficient deep learning frameworks.
+
+## Intended uses
+
+Our model is intended to be used as a sentence and short paragraph encoder. Given an input text, it outputs a vector which captures 
+the semantic information. The sentence vector may be used for information retrieval, clustering or sentence similarity tasks.
+
+By default, input text longer than 256 word pieces is truncated.
+
+
+## Training procedure
+
+### Pre-training 
+
+We use the pretrained [`nreimers/MiniLM-L6-H384-uncased`](https://huggingface.co/nreimers/MiniLM-L6-H384-uncased) model. Please refer to the model card for more detailed information about the pre-training procedure.
+
+### Fine-tuning 
+
+We fine-tune the model using a contrastive objective. Formally, we compute the cosine similarity from each possible sentence pairs from the batch.
+We then apply the cross entropy loss by comparing with true pairs.
+
+#### Hyper parameters
+
+We trained our model on a TPU v3-8. We train the model during 100k steps using a batch size of 1024 (128 per TPU core).
+We use a learning rate warm up of 500. The sequence length was limited to 128 tokens. We used the AdamW optimizer with
+a 2e-5 learning rate. The full training script is accessible in this current repository: `train_script.py`.
+
+#### Training data
+
+We use the concatenation from multiple datasets to fine-tune our model. The total number of sentence pairs is above 1 billion sentences.
+We sampled each dataset given a weighted probability which configuration is detailed in the `data_config.json` file.
+
+
+| Dataset                                                  | Paper                                    | Number of training tuples  |
+|--------------------------------------------------------|:----------------------------------------:|:--------------------------:|
+| [Reddit comments (2015-2018)](https://github.com/PolyAI-LDN/conversational-datasets/tree/master/reddit) | [paper](https://arxiv.org/abs/1904.06472) | 726,484,430 |
+| [S2ORC](https://github.com/allenai/s2orc) Citation pairs (Abstracts) | [paper](https://aclanthology.org/2020.acl-main.447/) | 116,288,806 |
+| [WikiAnswers](https://github.com/afader/oqa#wikianswers-corpus) Duplicate question pairs | [paper](https://doi.org/10.1145/2623330.2623677) | 77,427,422 |
+| [PAQ](https://github.com/facebookresearch/PAQ) (Question, Answer) pairs | [paper](https://arxiv.org/abs/2102.07033) | 64,371,441 |
+| [S2ORC](https://github.com/allenai/s2orc) Citation pairs (Titles) | [paper](https://aclanthology.org/2020.acl-main.447/) | 52,603,982 |
+| [S2ORC](https://github.com/allenai/s2orc) (Title, Abstract) | [paper](https://aclanthology.org/2020.acl-main.447/) | 41,769,185 |
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) (Title, Body) pairs  | - | 25,316,456 |
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) (Title+Body, Answer) pairs  | - | 21,396,559 |
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) (Title, Answer) pairs  | - | 21,396,559 |
+| [MS MARCO](https://microsoft.github.io/msmarco/) triplets | [paper](https://doi.org/10.1145/3404835.3462804) | 9,144,553 |
+| [GOOAQ: Open Question Answering with Diverse Answer Types](https://github.com/allenai/gooaq) | [paper](https://arxiv.org/pdf/2104.08727.pdf) | 3,012,496 |
+| [Yahoo Answers](https://www.kaggle.com/soumikrakshit/yahoo-answers-dataset) (Title, Answer) | [paper](https://proceedings.neurips.cc/paper/2015/hash/250cf8b51c773f3f8dc8b4be867a9a02-Abstract.html) | 1,198,260 |
+| [Code Search](https://huggingface.co/datasets/code_search_net) | - | 1,151,414 |
+| [COCO](https://cocodataset.org/#home) Image captions | [paper](https://link.springer.com/chapter/10.1007%2F978-3-319-10602-1_48) | 828,395|
+| [SPECTER](https://github.com/allenai/specter) citation triplets | [paper](https://doi.org/10.18653/v1/2020.acl-main.207) | 684,100 |
+| [Yahoo Answers](https://www.kaggle.com/soumikrakshit/yahoo-answers-dataset) (Question, Answer) | [paper](https://proceedings.neurips.cc/paper/2015/hash/250cf8b51c773f3f8dc8b4be867a9a02-Abstract.html) | 681,164 |
+| [Yahoo Answers](https://www.kaggle.com/soumikrakshit/yahoo-answers-dataset) (Title, Question) | [paper](https://proceedings.neurips.cc/paper/2015/hash/250cf8b51c773f3f8dc8b4be867a9a02-Abstract.html) | 659,896 |
+| [SearchQA](https://huggingface.co/datasets/search_qa) | [paper](https://arxiv.org/abs/1704.05179) | 582,261 |
+| [Eli5](https://huggingface.co/datasets/eli5) | [paper](https://doi.org/10.18653/v1/p19-1346) | 325,475 |
+| [Flickr 30k](https://shannon.cs.illinois.edu/DenotationGraph/) | [paper](https://transacl.org/ojs/index.php/tacl/article/view/229/33) | 317,695 |
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) Duplicate questions (titles) | | 304,525 |
+| AllNLI ([SNLI](https://nlp.stanford.edu/projects/snli/) and [MultiNLI](https://cims.nyu.edu/~sbowman/multinli/) | [paper SNLI](https://doi.org/10.18653/v1/d15-1075), [paper MultiNLI](https://doi.org/10.18653/v1/n18-1101) | 277,230 | 
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) Duplicate questions (bodies) | | 250,519 |
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) Duplicate questions (titles+bodies) | | 250,460 |
+| [Sentence Compression](https://github.com/google-research-datasets/sentence-compression) | [paper](https://www.aclweb.org/anthology/D13-1155/) | 180,000 |
+| [Wikihow](https://github.com/pvl/wikihow_pairs_dataset) | [paper](https://arxiv.org/abs/1810.09305) | 128,542 |
+| [Altlex](https://github.com/chridey/altlex/) | [paper](https://aclanthology.org/P16-1135.pdf) | 112,696 |
+| [Quora Question Triplets](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) | - | 103,663 |
+| [Simple Wikipedia](https://cs.pomona.edu/~dkauchak/simplification/) | [paper](https://www.aclweb.org/anthology/P11-2117/) | 102,225 |
+| [Natural Questions (NQ)](https://ai.google.com/research/NaturalQuestions) | [paper](https://transacl.org/ojs/index.php/tacl/article/view/1455) | 100,231 |
+| [SQuAD2.0](https://rajpurkar.github.io/SQuAD-explorer/) | [paper](https://aclanthology.org/P18-2124.pdf) | 87,599 |
+| [TriviaQA](https://huggingface.co/datasets/trivia_qa) | - | 73,346 |
+| **Total** | | **1,170,060,424** |

app/chromadb_model/config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "sentence-transformers/all-MiniLM-L6-v2",
+  "architectures": [
+    "BertModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 384,
+  "initializer_range": 0.02,
+  "intermediate_size": 1536,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 6,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.48.3",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

app/chromadb_model/config_sentence_transformers.json

@@ -0,0 +1,10 @@
+{
+  "__version__": {
+    "sentence_transformers": "4.1.0",
+    "transformers": "4.48.3",
+    "pytorch": "2.6.0"
+  },
+  "prompts": {},
+  "default_prompt_name": null,
+  "similarity_fn_name": "cosine"
+}

app/chromadb_model/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1377e9af0ca0b016a9f2aa584d6fc71ab3ea6804fae21ef9fb1416e2944057ac
+size 90864192

app/chromadb_model/modules.json

@@ -0,0 +1,20 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "2_Normalize",
+    "type": "sentence_transformers.models.Normalize"
+  }
+]

app/chromadb_model/sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 256,
+  "do_lower_case": false
+}

app/chromadb_model/special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

app/chromadb_model/tokenizer_config.json

@@ -0,0 +1,65 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "max_length": 128,
+  "model_max_length": 256,
+  "never_split": null,
+  "pad_to_multiple_of": null,
+  "pad_token": "[PAD]",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "sep_token": "[SEP]",
+  "stride": 0,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "[UNK]"
+}

app/download_questions.py

@@ -0,0 +1,23 @@
+import streamlit as st
+from docx import Document
+from io import BytesIO
+from datetime import datetime
+
+
+def create_docx_from_data(data):
+    doc = Document()
+    doc.add_heading("Questions", 0)
+    doc.add_paragraph(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
+
+    for chapter, qas in data.items():
+        doc.add_heading(chapter, level=1)
+        doc.add_paragraph("")  # Spacing
+        for idx, qa in enumerate(qas, 1):
+            doc.add_paragraph(f"Q{idx}: {qa['question']}", style='List Number')
+            doc.add_paragraph(f"A: {qa['answer']}", style='Normal')
+            doc.add_paragraph("")  # Spacing
+
+    buffer = BytesIO()
+    doc.save(buffer)
+    buffer.seek(0)
+    return buffer

app/main.py

@@ -0,0 +1,170 @@
+import streamlit as st
+from utils import *
+from main_IO import *
+from download_questions import create_docx_from_data
+from backend.raw_text_processing import *
+from backend.chromadb_utils import *
+import os
+import sys
+import logging
+
+
+# Add the root folder (one level above 'app') to sys.path
+root_path = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+if root_path not in sys.path:
+    sys.path.insert(0, root_path)
+
+# Configuration
+configure_page()
+initialise_session_state()
+apply_style()
+
+# add_sidebar_header()
+st.sidebar.html("""
+<div style='position: fixed; top: 10px; left: 20px; z-index: 999; padding: 10px;'>
+    <h3>Menu</h3>
+</div>
+""")
+
+# Initialize chromadb variables
+EMBEDDING_MODEL = "all-MiniLM-L6-v2"
+model_path = "./chromadb_model"
+
+# Set-up Logger
+st.session_state.use_logger = False
+if st.session_state.use_logger: 
+    level = st.selectbox("Logging level", ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
+    logging.getLogger().setLevel(getattr(logging, level))
+
+# Set default page if not specified
+if "page" not in st.query_params:
+    st.query_params.page = "main"
+
+# Navigation handling
+if st.query_params.page == "topic":
+    st.switch_page("pages/2_topic_questions.py")
+elif st.query_params.page == "chapter":
+    st.switch_page("pages/1_chapter_questions.py")
+elif st.query_params.page == "inspect":
+    st.switch_page("pages/3_inspect_pdf.py")
+else:
+    # Welcome message
+    st.title("Welcome to Text2Test!")
+    st.divider()
+    st.markdown("""
+    Welcome! This app helps you transform your PDFs or texts into interactive study materials by generating meaningful questions.  
+    You can either:
+
+    - Generate questions based on specific topics or keywords
+    - Generate questions from a selected chapter
+
+    Start by uploading your PDF file, then choose your preferred way to generate questions using the options below.  
+    Let’s make studying smarter and more engaging!
+    """)
+    st.divider()
+
+    # Upload PDF file
+    st.subheader("Upload your PDF file")
+    upload_pdf()
+    st.divider()
+
+    # Check if PDF has changed or needs processing
+    if st.session_state.get("pdf_changed") or (
+        st.session_state.get("full_text") is None and 
+        st.session_state.get("uploaded_pdf_bytes") is not None
+    ):
+        process_pdf()  # Extract text from PDF
+
+        with st.spinner("Extracting information from the text..."):
+            client, embedding_func = initialize_chromadb(EMBEDDING_MODEL)  
+            whole_text_collection = initialize_collection(client, embedding_func, "whole_text_chunks")  
+            update_collection(
+                whole_text_collection, 
+                st.session_state.get("full_text"), 
+                max_words=200, 
+                min_words=100, 
+                overlap_sentences=3
+            )
+            st.session_state["pdf_changed"] = False  # Reset flag after processing
+
+    try:
+        uploaded_pdf_name = st.session_state.get('uploaded_pdf_name', None)
+        if uploaded_pdf_name:
+            st.info(f"Uploaded PDF: {uploaded_pdf_name}")
+            debug_log(f"book title: {uploaded_pdf_name}")
+        else:
+            pass
+
+        show_pdf_preview()
+
+    except Exception as e:
+        debug_log(f"Error displaying PDF info or preview: {e}")
+
+    # Main content buttons
+    st.subheader("Generate Questions")
+    st.write("Please choose an option to generate questions:")
+    breaks(1)  
+    cols = st.columns(2)
+    st.html("""
+    <style>
+    div.stButton {
+        display: flex;
+        justify-content: center;
+        margin: 10px 0;
+    }
+
+    div.stButton > button:first-child {
+        width: 80%;
+        padding: 40px 0;
+        background-color: #f0f0f0 !important;
+        border: none !important;
+        border-radius: 10px !important;
+        color: #333 !important;
+        font-family: 'Work Sans', sans-serif !important;
+        font-weight: 600 !important;
+        transition: all 0.3s ease;
+    }
+
+    /* Target the button text directly */
+    div.stButton > button:first-child p,
+    div.stButton > button:first-child span,
+    div.stButton > button:first-child div,
+    div.stButton > button:first-child {
+        font-size: 24px !important;
+        line-height: 1.2 !important;
+    }
+
+    div.stButton > button:first-child:hover {
+        background-color: #e0e0e0 !important;
+        transform: translateY(-2px);
+        box-shadow: 0 4px 8px rgba(0,0,0,0.1);
+    }
+    </style>
+    """)
+
+    with cols[0]:
+        if st.button("Generate Questions on a Topic", key="main_topic"):
+            st.query_params.page = "topic"
+            st.rerun()
+    with cols[1]:
+        if st.button("Generate Questions from a Chapter", key="main_chapter"):
+            st.query_params.page = "chapter"
+            st.rerun()
+
+    if st.session_state.get('questions_to_download'): 
+        with st.sidebar:
+            st.markdown("---")  # Divider
+            st.markdown("**Download Questions**")  # Spacing
+
+            docx_file = create_docx_from_data(st.session_state.get('questions_to_download', {}))
+
+            st.download_button(
+                label="📄 Download as Word (.docx)",
+                data=docx_file,
+                file_name="questions.docx",
+                mime="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+                on_click="ignore"
+            )
+    else:
+        with st.sidebar:
+            st.markdown("---")

app/main_IO.py

@@ -0,0 +1,112 @@
+import streamlit as st
+import fitz  # PyMuPDF
+from PIL import Image
+import io
+
+
+DEFAULT_SESSION_STATE = {
+    # PDF Upload
+    'doc': None,
+    'uploaded_pdf_name': None,
+    'pdf_changed': False,
+    'uploaded_pdf_bytes': None,
+    'page_range_set' : False,
+    'page_range_updated' : False,
+    'full_text': None,
+    'pages_data_infos': None,
+
+    # TOC
+    'page_choice': None,
+    'toc_page_range': None,
+    'toc': None,
+
+    # Chapters
+    'chapters_starting_page': None,
+    'chapters_dict': None,
+    'chapters_extracted': None,
+    'chapters_chunked': None,
+    'selected_chapter_idx': None,
+    'selected_chapter_title': None,
+    'num_questions': None,
+    'chapter_selected_chunks': None,
+    'chapter_prompt': None,
+
+    # Topics
+    'query': None,
+    'questions_ready_topic': False,
+
+    # Questions
+    'questions_dict_chapter': None,
+    'questions_dict_topic': None,
+    'raw_output': None,  # remove this (only for debug)
+    'questions_ready_chapter': False,
+    'questions_to_download' : {}
+}
+
+
+def initialise_session_state():
+    """
+    Initializes the session state variables if not already set.
+    """
+    for key, default_val in DEFAULT_SESSION_STATE.items():
+        if key not in st.session_state:
+            st.session_state[key] = default_val
+
+
+def reset_session_state_on_upload():
+    """
+    Resets session state variables to their default values.
+    """
+    for key, default_val in DEFAULT_SESSION_STATE.items():
+        if key != 'questions_to_download':
+            st.session_state[key] = default_val
+
+
+def upload_pdf():
+    uploaded_file = st.file_uploader("", type=["pdf"])
+    
+    if uploaded_file is not None:
+        prev_file = st.session_state.get('uploaded_pdf_name')
+        if uploaded_file.name != prev_file:
+            # New file detected
+            reset_session_state_on_upload()
+            st.session_state['pdf_changed'] = True
+        else:
+            st.session_state['pdf_changed'] = False
+
+        pdf_bytes = uploaded_file.read()
+
+        if pdf_bytes:
+            st.session_state['uploaded_pdf_bytes'] = pdf_bytes
+            st.session_state['uploaded_pdf_name'] = uploaded_file.name
+            st.success(f"File '{uploaded_file.name}' uploaded successfully!")
+        else:
+            st.error("Uploaded file is empty!")
+                
+    elif uploaded_file is None and st.session_state.get('uploaded_pdf_bytes') is not None:
+        st.success("File uploaded successfully!")
+    else:
+        st.info("Please upload a PDF file to proceed.")
+
+
+def show_pdf_preview():
+    if 'uploaded_pdf_bytes' in st.session_state:
+        pdf_bytes = st.session_state['uploaded_pdf_bytes']
+        doc = None
+        try:
+            doc = fitz.open(stream=pdf_bytes, filetype="pdf")
+            if doc.page_count < 1:
+                st.sidebar.error("PDF has no pages!")
+                return
+            page = doc.load_page(0)
+            pix = page.get_pixmap()
+            img = Image.open(io.BytesIO(pix.tobytes("png")))
+            st.sidebar.image(img, caption="First page preview", use_container_width=True)
+        except Exception as e:
+            st.sidebar.error(f"Failed to open PDF: {e}")
+        finally:
+            if doc is not None:
+                doc.close()
+    else:
+        st.sidebar.write("Upload a PDF to see a preview here.")
+

app/pages/1_chapter_questions.py

@@ -0,0 +1,90 @@
+import streamlit as st
+from app.utils import *
+from app.main_IO import *
+from app.pages.utils_chapter.display_pages import *
+from app.pages.utils_chapter.display_questions import *
+from app.pages.utils_chapter.chapter_extraction import *
+from app.pages.utils_chapter.chapter_selection import *
+from app.download_questions import create_docx_from_data
+
+# Set up logger
+if st.session_state.use_logger: 
+    level = st.selectbox("Logging level", ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
+    logging.getLogger().setLevel(getattr(logging, level))
+else:
+    logging.getLogger().setLevel(logging.CRITICAL + 1)
+
+# Initialise
+apply_style()
+
+# add_sidebar_header()
+st.sidebar.html("""
+<div style='position: fixed; top: 10px; left: 20px; z-index: 999; padding: 10px;'>
+    <h3>Menu</h3>
+</div>
+""")
+
+show_pdf_preview()
+st.title("Generate Questions from a Chapter")
+st.divider()
+st.write("""
+        Here you can generate questions based on a specific chapter.  
+        To do this, please first select the page range that includes the Table of Contents (TOC) — sometimes called the index or contents page — which lists the chapters and their page numbers.  
+
+        This step is important because it helps the app automatically identify and locate chapters, so you can easily choose the exact chapter to generate questions from.
+        """)
+
+# Display the page range selector
+breaks(1)
+display_scrollable_pages()
+
+# UI and Interaction
+set_clicked, start_page, end_page = page_range_selector_ui()
+
+if set_clicked:
+    updated = handle_page_range_submission(start_page, end_page)
+    st.session_state["page_range_updated"] = updated
+
+if st.session_state.get("page_range_updated", False):
+    extract_content_if_needed()
+    st.session_state["page_range_updated"] = False
+
+# Gate rest of app
+if st.session_state.get("page_range_set", False):
+
+    # Call the form in your main app code to generate questions
+    result = chapter_question_form()
+    if result:
+        st.session_state.questions_dict_chapter = result
+    debug_log(f"questions: {st.session_state.get('questions_dict_chapter', 'None')}")
+    
+    if st.session_state.get("questions_ready_chapter"):
+        breaks(2)
+        st.subheader("Generated Questions")
+        st.divider()
+        # Visualize generated questions and store them
+        show_questions(st.session_state.get('questions_dict_chapter'))
+        breaks(1)
+        show_download_controls(st.session_state.get('selected_chapter_title'), st.session_state.get('questions_dict_chapter', 'None'))
+        debug_show_selected_questions()
+
+        with st.sidebar:
+            st.markdown("---")  # Divider
+            st.markdown("**Download Questions**")  # Spacing
+
+            docx_file = create_docx_from_data(st.session_state['questions_to_download'])
+
+            st.download_button(
+                label="📄 Download as Word (.docx)",
+                data=docx_file,
+                file_name="questions.docx",
+                mime="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+                on_click="ignore"
+        )
+            
+else:
+    st.info("Please set a valid page range to continue.")
+
+
+
+

app/pages/2_topic_questions.py

@@ -0,0 +1,106 @@
+import streamlit as st
+import chromadb
+from app.utils import *
+from app.main_IO import *
+from app.download_questions import create_docx_from_data
+from app.pages.utils_chapter.display_questions import *
+from app.pages.utils_chapter.chapter_selection import select_num_questions
+from app.backend.chunks_processing import query_collection
+from app.backend.messages_templates import book_prompt
+from app.backend.runpod_client import run_prompt, clean_and_parse_json
+
+
+# Initialise
+apply_style()
+
+# add_sidebar_header()
+st.sidebar.html("""
+<div style='position: fixed; top: 10px; left: 20px; z-index: 999; padding: 10px;'>
+    <h3>Menu</h3>
+</div>
+""")
+
+show_pdf_preview()
+st.title("Generate Questions on a Topic")
+st.divider()
+st.write("""Here, you can generate questions based on a specific topic.  
+        You can enter a topic or keyword, and the app will generate questions based on the content of the uploaded PDF.""")
+
+breaks(1)
+
+# Set up logger
+if st.session_state.use_logger: 
+    level = st.selectbox("Logging level", ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
+    logging.getLogger().setLevel(getattr(logging, level))
+
+if st.session_state.get("full_text", None) is not None:
+    client = chromadb.Client()  # Use same client init/config
+    whole_text_collection = client.get_collection("whole_text_chunks")
+    debug_log(f"Collection name: {whole_text_collection.name}")
+    debug_log(f"Number of documents: {whole_text_collection.count()}")
+    results = whole_text_collection.get(limit=1)
+
+    documents = results['documents']  # List of text chunks
+    metadatas = results['metadatas']  # List of metadata dicts, e.g. chunk indexes
+
+    for i, (doc, meta) in enumerate(zip(documents, metadatas)):
+        debug_log(f"Chunk {i}:")
+        debug_log(doc)
+        debug_log(f"Metadata: {meta}")
+
+
+with st.form("query_form"):
+    st.subheader("Enter a Topic or Keyword")
+    query = st.text_input("Enter your query:")
+    col1, _ = st.columns([2, 6])
+    with col1:
+        num_questions = select_num_questions()
+    breaks(1)
+    submitted = st.form_submit_button("Submit")
+
+    if submitted and query:
+        with st.spinner("Generating questions..."):
+            # Generate questions based on the query
+            query_context = query_collection(whole_text_collection, query=query, nresults=3, context_multiplier=2)
+            prompt = book_prompt(query_context, num_questions=num_questions, user_query=query)
+            questions_json = run_prompt(prompt)
+            st.session_state.questions_dict_topic = clean_and_parse_json(questions_json)
+            st.session_state['query'] = query
+            st.session_state['questions_ready_topic'] = True
+
+if st.session_state.get("questions_ready_topic"):
+    breaks(2)
+    st.subheader("Generated Questions")
+    st.divider()
+    debug_log(f"Generated questions: {st.session_state.get('questions_dict_topic', 'None')}")
+
+    # Visualize generated questions and store them
+    show_questions(st.session_state['questions_dict_topic'])
+    breaks(1)
+    show_download_controls(st.session_state.get('query'), st.session_state.get('questions_dict_topic', 'None'))
+    debug_show_selected_questions()
+
+    with st.sidebar:
+        st.divider()  # Divider
+        st.markdown("**Download Questions**")  # Spacing
+
+        docx_file = create_docx_from_data(st.session_state.get('questions_to_download', {}))
+
+        st.download_button(
+            label="📄 Download as Word (.docx)",
+            data=docx_file,
+            file_name="questions.docx",
+            mime="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+            on_click="ignore"
+        )
+
+
+
+
+        # query_context = query_collection(whole_text_collection, query=query, nresults=3, context_multiplier=2)
+        # out3 = book_prompt(query_context, num_questions=3, user_query=query)
+        # questions = run_prompt(out)
+
+
+# use https://docs.streamlit.io/develop/api-reference/chat/st.chat_input
+# or https://docs.streamlit.io/develop/api-reference/widgets/st.text_input

app/pages/3_inspect_pdf.py

@@ -0,0 +1,8 @@
+import streamlit as st
+from app.utils import *
+
+# Initialise
+apply_style()
+
+st.title("Work in Progress: Inspect PDF")
+