Update app.py

app.py

@@ -36,6 +36,8 @@ from langchain.vectorstores import Chroma
 from langchain.embeddings import HuggingFaceEmbeddings
 from langchain.text_splitter import NLTKTextSplitter
 from patent_downloader import PatentDownloader
+from langchain.document_loaders import PyMuPDFLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 PERSISTED_DIRECTORY = tempfile.mkdtemp()
 
@@ -55,41 +57,30 @@ check_poppler_installed()
 
 def load_docs(document_path):
     try:
-        loader = UnstructuredPDFLoader(
-            document_path,
-            mode="elements",
-            strategy="fast",
-            ocr_languages=None
-        )
+        # Load the entire PDF content
+        loader = PyMuPDFLoader(document_path)
         documents = loader.load()
-        text_splitter = NLTKTextSplitter(chunk_size=1000)
-        split_docs = text_splitter.split_documents(documents)
-
-        # Filter out metadata, very short, or redundant chunks
-        filtered_docs = []
-        seen_chunks = set()
-
-        for doc in split_docs:
-            content = doc.page_content.strip()
-
-            # Filter conditions: Ignore short chunks, common metadata, or duplicates
-            if (
-                len(content) < 50 or
-                "United States Patent" in content or
-                re.match(r"^\(?\d+\)?$", content) or  # Matches lines like "(12)" or "10"
-                content in seen_chunks
-            ):
-                continue
-
-            filtered_docs.append(doc)
-            seen_chunks.add(content)
-
-        # Debugging: Show filtered chunks
-        st.write(f"🔍 Filtered Documents: {len(filtered_docs)}")
-        for i, doc in enumerate(filtered_docs[:5]):  # Show first 5 chunks
-            st.write(f"Filtered Chunk {i + 1}: {doc.page_content[:200]}...")
-
-        return filtered_docs
+
+        # Combine all pages into a single string
+        full_text = "\n".join([doc.page_content for doc in documents])
+
+        # Debug: Verify total text size
+        st.write(f"📄 Total Text Length: {len(full_text)} characters")
+
+        # Split the text into meaningful chunks
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=1000,
+            chunk_overlap=100,
+            separators=["\n\n", "\n", " ", ""]
+        )
+        split_docs = text_splitter.create_documents([full_text])
+
+        # Debug: Show filtered chunks
+        st.write(f"🔍 Total Chunks After Splitting: {len(split_docs)}")
+        for i, doc in enumerate(split_docs[:5]):  # Show first 5 chunks
+            st.write(f"Chunk {i + 1}: {doc.page_content[:200]}...")
+
+        return split_docs
     except Exception as e:
         st.error(f"Failed to load and process PDF: {e}")
         st.stop()