Update app.py

app.py

@@ -57,17 +57,25 @@ check_poppler_installed()
 
 def load_docs(document_path):
     try:
-        # Load the entire PDF content
-        loader = PyMuPDFLoader(document_path)
-        documents = loader.load()
+        import fitz  # PyMuPDF for text extraction
 
-        # Combine all pages into a single string
-        full_text = "\n".join([doc.page_content for doc in documents])
+        # Step 1: Extract plain text from PDF
+        doc = fitz.open(document_path)
+        extracted_text = []
 
-        # Debug: Verify total text size
-        st.write(f"📄 Total Text Length: {len(full_text)} characters")
+        for page_num, page in enumerate(doc):
+            page_text = page.get_text("text")  # Extract text
+            clean_page_text = clean_extracted_text(page_text)
+            if clean_page_text:  # Keep only non-empty cleaned text
+                extracted_text.append(clean_page_text)
 
-        # Split the text into meaningful chunks
+        doc.close()
+
+        # Step 2: Combine cleaned text
+        full_text = "\n".join(extracted_text)
+        st.write(f"📄 Total Cleaned Text Length: {len(full_text)} characters")
+
+        # Step 3: Chunk the cleaned text
         text_splitter = RecursiveCharacterTextSplitter(
             chunk_size=1000,
             chunk_overlap=100,
@@ -78,7 +86,7 @@ def load_docs(document_path):
         # Debug: Show filtered chunks
         st.write(f"🔍 Total Chunks After Splitting: {len(split_docs)}")
         for i, doc in enumerate(split_docs[:5]):  # Show first 5 chunks
-            st.write(f"Chunk {i + 1}: {doc.page_content[:200]}...")
+            st.write(f"Chunk {i + 1}: {doc.page_content[:300]}...")
 
         return split_docs
     except Exception as e:
@@ -86,6 +94,31 @@ def load_docs(document_path):
         st.stop()
 
 
+def clean_extracted_text(text):
+    """
+    Cleans extracted text to remove metadata, headers, and irrelevant content.
+    """
+    lines = text.split("\n")
+    cleaned_lines = []
+
+    for line in lines:
+        line = line.strip()
+
+        # Filter out lines with metadata patterns
+        if (
+            re.match(r"^(U\.S\.|United States|Sheet|Figure|References|Patent No|Date of Patent)", line)
+            or re.match(r"^\(?\d+\)?$", line)  # Matches single numbers (page numbers)
+            or "Examiner" in line
+            or "Attorney" in line
+            or len(line) < 30  # Skip very short lines
+        ):
+            continue
+
+        cleaned_lines.append(line)
+
+    return "\n".join(cleaned_lines)
+
+
 def already_indexed(vectordb, file_name):
     indexed_sources = set(
         x["source"] for x in vectordb.get(include=["metadatas"])["metadatas"]
@@ -236,15 +269,17 @@ if __name__ == "__main__":
         else:
             st.write("✅ File already downloaded.")
 
-        # Generate PDF preview
-        st.write("🖼️ Generating PDF preview...")
-        preview_image_path = preview_pdf(pdf_path)
-        if preview_image_path:
-            st.session_state.pdf_preview = preview_image_path
-            st.image(preview_image_path, caption="First Page Preview", use_container_width=True)
-        else:
-            st.warning("Failed to generate PDF preview.")
-            st.session_state.pdf_preview = None
+        # Generate PDF preview only if not already displayed
+        if not st.session_state.get("pdf_preview_displayed", False):
+            st.write("🖼️ Generating PDF preview...")
+            preview_image_path = preview_pdf(pdf_path)
+            if preview_image_path:
+                st.session_state.pdf_preview = preview_image_path
+                st.image(preview_image_path, caption="First Page Preview", use_container_width=True)
+                st.session_state["pdf_preview_displayed"] = True
+            else:
+                st.warning("Failed to generate PDF preview.")
+                st.session_state.pdf_preview = None
 
         # Load the document into the system
         st.write("🔄 Loading document into the system...")
@@ -258,10 +293,6 @@ if __name__ == "__main__":
             st.error(f"Failed to load the document: {e}")
             st.stop()
 
-    # Display the PDF preview if available
-    if st.session_state.pdf_preview:
-        st.image(st.session_state.pdf_preview, caption="First Page Preview", use_container_width=True)
-
     # Display previous chat messages
     if st.session_state.messages:
         for message in st.session_state.messages: