Update app.py

app.py

@@ -70,25 +70,29 @@ def gen_splits(folder_name):
     new_file_paths = [os.path.join(os.getcwd(), folder_name, file) for file in file_paths]
     
     splits = []
+    empty_pages = 0
+
     for file_path in new_file_paths:
         if not file_path.lower().endswith(".pdf"):
             continue
             
-        # Open document using fitz
         doc = fitz.open(file_path)
         file_name = os.path.basename(file_path)
         
         for page_num in range(len(doc)):
             page = doc.load_page(page_num)
-            text = page.get_text("text") # "text" maintains logical flow; "blocks" is better for tables
-            
-            # Creating a LangChain Document object for each page
-            # This replaces the need for RecursiveCharacterTextSplitter
+            text = page.get_text("text").strip()   # ← strip whitespace
+
+            # ── Skip empty/image-only pages ────────────────────────────────
+            if not text or len(text) < 20:         # ← 20 chars minimum threshold
+                empty_pages += 1
+                continue
+
             page_doc = Document(
                 page_content=text,
                 metadata={
                     "source": file_name,
-                    "page": page_num + 1,  # 1-indexed for user readability
+                    "page": page_num + 1,
                     "total_pages": len(doc),
                     "format": "PDF",
                     "extraction_method": "PyMuPDF"
@@ -97,18 +101,33 @@ def gen_splits(folder_name):
             splits.append(page_doc)
             
         doc.close()
-        
+
+    print(f"✓ Loaded {len(splits)} pages | Skipped {empty_pages} empty/image-only pages")
     return splits
 
 splits = gen_splits(DESTINATION_FOLDER)
 embedding_func = HuggingFaceEmbeddings(model_name='all-MiniLM-L6-v2')
 
 def vectordb_from_splits(splits):
-    vectordb = Chroma.from_documents(documents=splits, persist_directory=PERSIST_DIR, embedding=embedding_func)
+    # ── Reuse existing ChromaDB if persist dir already populated ──────────────
+    if os.path.exists(PERSIST_DIR) and os.listdir(PERSIST_DIR):
+        print("✓ Loading existing ChromaDB from disk — skipping re-embedding.")
+        return Chroma(persist_directory=PERSIST_DIR, embedding_function=embedding_func)
+
+    if not splits:
+        raise ValueError("No text content extracted. Check if PDFs are scanned images.")
+
+    print(f"Building ChromaDB from {len(splits)} chunks...")
+    vectordb = Chroma.from_documents(
+        documents=splits,
+        persist_directory=PERSIST_DIR,
+        embedding=embedding_func
+    )
+    print(f"✓ ChromaDB built successfully.")
     return vectordb
 
-vectordb = vectordb_from_splits(splits)
 
+vectordb = vectordb_from_splits(splits)
 
 
 # RAG Chain