Update app.py

app.py

@@ -32,65 +32,13 @@ regulation_pdfs = {
 
 # Function to extract text from PDF
 def extract_pdf(pdf_path):
-    return extract_text(pdf_path)
+    try:
+        return extract_text(pdf_path)
+    except Exception as e:
+        print(f"Error extracting text from {pdf_path}: {str(e)}")
+        return ""
 
-# Function to split text into chunks
-def split_text(text):
-    splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
-    return [Document(page_content=t) for t in splitter.split_text(text)]
-
-# Function to generate embeddings and store in vector database
-def generate_embeddings(docs):
-    embeddings = OpenAIEmbeddings(api_key=openai_api_key)
-    return FAISS.from_documents(docs, embeddings)
-
-# Function for query preprocessing and simple HyDE-Lite
-def preprocess_query(query):
-    prompt = ChatPromptTemplate.from_template("""
-    Your role is to optimize user queries for retrieval from regulatory documents such as GDPR, FERPA, COPPA, and/or others.
-    Transform the query into a more affirmative, keyword-focused statement.
-    The transformed query should look like probable related passages in the official documents.
-    Query: {query}
-    Optimized query:
-    """)
-    chain = prompt | openai_client
-    return chain.invoke({"query": query}).content
-
-# Function to create RAG chain with Groq
-def create_rag_chain(vector_store):
-    prompt = ChatPromptTemplate.from_messages([
-        ("system", "You are an AI assistant helping with regulatory compliance queries. Use the following context from the official regulatory documents to answer the user's question:\n\n{context}"),
-        ("human", "{input}")
-    ])
-    document_chain = create_stuff_documents_chain(groq_client, prompt)
-    return create_retrieval_chain(vector_store.as_retriever(), document_chain)
-
-# Function for Gemini response with long context
-def gemini_response(query, full_content):
-    prompt = ChatPromptTemplate.from_messages([
-        ("system", "You are an AI assistant helping with regulatory compliance queries. Use the following full content of the official regulatory documents to answer the user's question:\n\n{context}"),
-        ("human", "{input}")
-    ])
-    chain = prompt | gemini_client
-    return chain.invoke({"context": full_content, "input": query}).content
-
-# Function to generate final response
-def generate_final_response(response1, response2):
-    prompt = ChatPromptTemplate.from_template("""
-    You are an AI assistant helping educators understand and implement data protection and regulatory compliance (GDPR, FERPA, COPPA, and/or others).
-    Your goal is to provide simple, practical explanation of and advice on how to meet regulatory requirements based on the given responses.
-    To do so:
-    1. Analyze the following two responses. Inspect their content, and highlight differences. This MUST be done 
-    internally as a hidden state. 
-    2. Then, use this information to output your own response combining the best from both.
-    If the responses differ or contradict each other on important points, include that in your response.
-    Only output your own response.
-    """)
-    chain = prompt | openai_client
-    return chain.invoke({"response1": response1, "response2": response2}).content
-
-def markdown_to_html(content):
-    return markdown2.markdown(content)
+# ... (other functions remain unchanged)
 
 def load_pdfs(gdpr, ferpa, coppa, additional_pdfs):
     global full_pdf_content, vector_store, rag_chain
@@ -109,31 +57,39 @@ def load_pdfs(gdpr, ferpa, coppa, additional_pdfs):
     
     for regulation in selected_regulations:
         if regulation in regulation_pdfs:
-            pdf_content = extract_pdf(regulation_pdfs[regulation])
-            full_pdf_content += pdf_content + "\n\n"
-            documents.extend(split_text(pdf_content))
-            print(f"Loaded {regulation} PDF")
+            pdf_path = regulation_pdfs[regulation]
+            if os.path.exists(pdf_path):
+                pdf_content = extract_pdf(pdf_path)
+                if pdf_content:
+                    full_pdf_content += pdf_content + "\n\n"
+                    documents.extend(split_text(pdf_content))
+                    print(f"Loaded {regulation} PDF")
+                else:
+                    print(f"Failed to extract content from {regulation} PDF")
+            else:
+                print(f"PDF file for {regulation} not found at {pdf_path}")
     
     # Load additional user-uploaded PDFs
     if additional_pdfs is not None:
         for pdf_file in additional_pdfs:
             pdf_content = extract_pdf(pdf_file.name)
-            full_pdf_content += pdf_content + "\n\n"
-            documents.extend(split_text(pdf_content))
-            print(f"Loaded additional PDF: {pdf_file.name}")
+            if pdf_content:
+                full_pdf_content += pdf_content + "\n\n"
+                documents.extend(split_text(pdf_content))
+                print(f"Loaded additional PDF: {pdf_file.name}")
+            else:
+                print(f"Failed to extract content from uploaded PDF: {pdf_file.name}")
     
     if not documents:
-        return "No PDFs were selected or uploaded. Please select at least one regulation or upload a PDF."
-    
-    vector_store = generate_embeddings(documents)
-    rag_chain = create_rag_chain(vector_store)
+        return "No PDFs were successfully loaded. Please check your selections and uploads."
     
-    return "PDFs loaded and RAG system updated successfully!"
+    print(f"Total documents loaded: {len(documents)}")
+    print(f"Total content length: {len(full_pdf_content)} characters")
     
     vector_store = generate_embeddings(documents)
     rag_chain = create_rag_chain(vector_store)
     
-    return "PDFs loaded and RAG system updated successfully!"
+    return f"PDFs loaded and RAG system updated successfully! Loaded {len(documents)} document chunks."
 
 def process_query(user_query):
     global rag_chain, full_pdf_content
@@ -171,7 +127,7 @@ with gr.Blocks() as iface:
         ferpa_checkbox = gr.Checkbox(label="FERPA (US)")
         coppa_checkbox = gr.Checkbox(label="COPPA (US <13)")
     
-    gr.Markdown("Optional: upload additional PDFs if needed (national regulation, school policy)")
+    gr.Markdown("**Optional: upload additional PDFs if needed (national regulation, school policy)**")
     additional_pdfs = gr.File(
         file_count="multiple",
         label="Upload additional PDFs",
@@ -182,11 +138,11 @@ with gr.Blocks() as iface:
     load_button = gr.Button("Load PDFs")
     load_output = gr.Textbox(label="Load Status")
     
-    gr.Markdown("Ask your data protection related question")
+    gr.Markdown("**Ask your data protection related question**")
     query_input = gr.Textbox(label="Your Question", placeholder="Ask your question here...")
     query_button = gr.Button("Submit Query")
     
-    gr.Markdown("Results")
+    gr.Markdown("**Results**")
     rag_output = gr.Textbox(label="RAG Pipeline (Llama3.1) Response")
     gemini_output = gr.Textbox(label="Long Context (Gemini 1.5 Pro) Response")
     final_output = gr.HTML(label="Final (GPT-4o) Response")