Merge branch 'main' of https://github.com/Deepesh70/Book_Model into deep

.gitignore

@@ -24,5 +24,8 @@ __pycache__/
 # faiss_store/
 vector_store/
 
+# Keep raw source books local only
+Data/
+
 # LangGraph
 .langgraph_api/

Data/Abraham-Silberschatz-Henry-F.-Korth-S.-Sudarshan-Database-System-Concepts-McGraw-Hill-Education-2019.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:dfbdff8a0b2eb32035f68105cf823ab788743d08036416fe81e2e19214e44aaf
-size 17163073

Data/Fundamentals_of_Database_Systems_6th_Edition-1.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e3f0d7d7cd8bfe6973d12acc9301a9b175932c281aac12e452b1100a0ccbfcaa
-size 9118471

Data/Hands On Machine Learning with Scikit Learn and TensorFlow.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:bcbfb1a52ec75e8d248f54de2d8de74fed3985dbd99a9e15c057361fb2398e7a
-size 7551093

Data/dbms-peter-rob.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:59fe9d07abffc730b68dfcd851652299ec437fc0821852bb5c1fb221382c7062
-size 56581782

Data/machine_learning.txt

@@ -1,15 +0,0 @@
-Machine Learning Basics
-
-Machine learning is a subset of artificial intelligence that enables systems to learn and improve
-from experience without being explicitly programmed. It focuses on developing computer programs
-that can access data and use it to learn for themselves.
-
-Types of Machine Learning:
-1. Supervised Learning: Learning with labeled data
-2. Unsupervised Learning: Finding patterns in unlabeled data
-3. Reinforcement Learning: Learning through rewards and penalties
-
-Applications include image recognition, speech processing, and recommendation systems
-
-
-    

Data/python_intro.txt

@@ -1,13 +0,0 @@
-Python Programming Introduction
-
-Python is a high-level, interpreted programming language known for its simplicity and readability.
-Created by Guido van Rossum and first released in 1991, Python has become one of the most popular
-programming languages in the world.
-
-Key Features:
-- Easy to learn and use
-- Extensive standard library
-- Cross-platform compatibility
-- Strong community support
-
-Python is widely used in web development, data science, artificial intelligence, and automation.

quick_test.py

@@ -0,0 +1,86 @@
+"""Quick test to verify the updated embedding pipeline works correctly."""
+import sys
+import os
+sys.path.insert(0, os.getcwd())
+
+from langchain_core.documents import Document
+from src.embedding import EmbeddingPipeline
+from src.vectorstore import FaissVectorStore
+from src.cleaner import DocumentCleaner
+import numpy as np
+
+def test_pipeline():
+    print("=" * 60)
+    print("QUICK PIPELINE VERIFICATION TEST")
+    print("=" * 60)
+    
+    # 1. Test DocumentCleaner
+    print("\n--- Test 1: DocumentCleaner ---")
+    cleaner = DocumentCleaner(min_length=10)
+    
+    test_docs = [
+        Document(page_content="This is a valid document with real content about databases."),
+        Document(page_content="Short"),  # too short
+        Document(page_content=""),       # empty
+        Document(page_content=None),     # None - this would crash old code
+        Document(page_content="Another valid document explaining SQL queries and joins."),
+        Document(page_content="   \n\n\t  "),  # whitespace only
+    ]
+    
+    cleaned = cleaner.clean_documents(test_docs)
+    print(f"  Input: {len(test_docs)} docs -> Cleaned: {len(cleaned)} docs")
+    assert len(cleaned) == 2, f"Expected 2 valid docs, got {len(cleaned)}"
+    print("  ✅ DocumentCleaner works correctly!")
+    
+    # 2. Test EmbeddingPipeline with cleaning + batching
+    print("\n--- Test 2: EmbeddingPipeline (clean + chunk + embed) ---")
+    
+    # Create realistic documents like what PyPDFLoader returns
+    docs = [
+        Document(page_content="Database Management Systems provide an organized way to store and manage data. " * 5),
+        Document(page_content="SQL is a standard language for accessing and manipulating databases. " * 5),
+        Document(page_content=None),  # Simulates a bad PDF page
+        Document(page_content="Machine Learning is a branch of artificial intelligence focused on algorithms. " * 5),
+        Document(page_content=""),    # Empty page
+        Document(page_content="x"),   # Too short - should be cleaned
+    ]
+    
+    pipe = EmbeddingPipeline(chunk_size=200, chunk_overlap=50)
+    chunks = pipe.chunk_documents(docs)
+    print(f"  Chunks created: {len(chunks)}")
+    
+    embeddings, valid_chunks = pipe.embed_chunks(chunks, batch_size=2)
+    print(f"  Embeddings shape: {embeddings.shape}")
+    print(f"  Valid chunks: {len(valid_chunks)}")
+    assert embeddings.shape[0] == len(valid_chunks), "Embeddings and chunks count must match!"
+    assert embeddings.shape[0] > 0, "Should have some embeddings!"
+    print("  ✅ EmbeddingPipeline works correctly!")
+    
+    # 3. Test FaissVectorStore integration
+    print("\n--- Test 3: FaissVectorStore build + query ---")
+    test_store_dir = "test_faiss_store"
+    store = FaissVectorStore(persist_dir=test_store_dir, chunk_size=200, chunk_overlap=50)
+    store.build_from_documents(docs)
+    
+    if store.index is not None:
+        results = store.query("What is a database?", top_k=2)
+        print(f"  Query returned {len(results)} results")
+        for r in results:
+            snippet = r['metadata']['texts'][:80] if r.get('metadata') and r['metadata'].get('texts') else "None"
+            print(f"    Distance: {r['distance']:.4f} | {snippet}...")
+        print("  ✅ FaissVectorStore works correctly!")
+    else:
+        print("  ⚠️ Vector store index is empty")
+    
+    # Cleanup
+    import shutil
+    if os.path.exists(test_store_dir):
+        shutil.rmtree(test_store_dir)
+    
+    print("\n" + "=" * 60)
+    print("🎉 ALL TESTS PASSED! Pipeline is working correctly.")
+    print("=" * 60)
+    print("\nYou can now run: python main.py")
+
+if __name__ == "__main__":
+    test_pipeline()

src/cleaner.py

@@ -0,0 +1,50 @@
+import re
+from typing import List, Any
+import logging
+
+logger = logging.getLogger(__name__)
+
+class DocumentCleaner:
+    def __init__(self, min_length: int = 50):
+        self.min_length = min_length
+        # Regex for common PDF artifacts or excessive whitespace
+        self.whitespace_pattern = re.compile(r'\s+')
+        self.control_chars_pattern = re.compile(r'[\x00-\x1f\x7f-\x9f]')
+
+    def clean_text(self, text: str) -> str:
+        if not text:
+            return ""
+            
+        # Safely remove surrogate code points that crash Windows terminals
+        text = text.encode('utf-8', 'ignore').decode('utf-8')
+        
+        # Remove control characters
+        text = self.control_chars_pattern.sub('', text)
+        
+        # Normalize whitespace (replace newlines/tabs with space and collapse)
+        text = self.whitespace_pattern.sub(' ', text).strip()
+        
+        return text
+
+    def clean_documents(self, documents: List[Any]) -> List[Any]:
+        """
+        Cleans a list of LangChain Document objects.
+        Filters out documents that are too short after cleaning.
+        """
+        cleaned_docs = []
+        for doc in documents:
+            if not hasattr(doc, 'page_content') or doc.page_content is None:
+                continue
+            
+            cleaned_text = self.clean_text(str(doc.page_content))
+            
+            if len(cleaned_text) >= self.min_length:
+                # Update the document content with cleaned version
+                doc.page_content = cleaned_text
+                cleaned_docs.append(doc)
+            
+        print(f"[INFO] Data Cleaning: {len(documents)} -> {len(cleaned_docs)} documents (filtered {len(documents) - len(cleaned_docs)})")
+        return cleaned_docs
+
+def get_default_cleaner():
+    return DocumentCleaner()

src/data_loader.py

@@ -118,7 +118,22 @@ def load_all_documents(data_dir: str) -> List[Any] :
         
         
     print(f"[DEBUG] Total Loaded Documents : {len(document)}")
-    return document
+    
+    # Filter out documents with None or non-string page_content
+    valid_documents = []
+    invalid_count = 0
+    for doc in document:
+        content = getattr(doc, 'page_content', None)
+        if content is not None and isinstance(content, str):
+            valid_documents.append(doc)
+        else:
+            invalid_count += 1
+    
+    if invalid_count > 0:
+        print(f"[WARNING] Filtered out {invalid_count} documents with invalid page_content (None or non-string)")
+    
+    print(f"[DEBUG] Valid Documents after filtering : {len(valid_documents)}")
+    return valid_documents
 
 
 if __name__ == '__main__':

src/embedding.py

@@ -2,7 +2,7 @@ from typing import List, Any, Optional
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 from sentence_transformers import SentenceTransformer
 import numpy as np
-from src.data_loader import load_all_documents
+from src.cleaner import get_default_cleaner
 
 class EmbeddingPipeline:
     def __init__(self, model_name: str = 'all-MiniLM-L6-v2', chunk_size: int = 1000, chunk_overlap: int = 200):
@@ -10,26 +10,75 @@ class EmbeddingPipeline:
         self.chunk_size = chunk_size
         self.chunk_overlap = chunk_overlap
         self.model = SentenceTransformer(model_name)
+        self.cleaner = get_default_cleaner()
         print(f"[INFO] Loaded embedding model: {model_name}")
         
     def chunk_documents(self, documents: List[Any]) -> List[Any]:
+        # 1. Clean documents first
+        cleaned_docs = self.cleaner.clean_documents(documents)
+        
+        # 2. Split into chunks
         splitter = RecursiveCharacterTextSplitter(
             chunk_size=self.chunk_size,
             chunk_overlap=self.chunk_overlap,
             length_function=len,
-            separators=["\n\n", "\n", " ", ""]
+            separators=["\n\n", "\n", " "]
         )
         
-        chunks = splitter.split_documents(documents)
-        print(f"[INFO] Split {len(documents)} documents into {len(chunks)} chunks.")
+        chunks = splitter.split_documents(cleaned_docs)
+        print(f"[INFO] Split {len(cleaned_docs)} documents into {len(chunks)} chunks.")
         return chunks
     
-    def embed_chunks(self, chunks: List[Any]) -> np.ndarray:
-        texts = [chunk.page_content for chunk in chunks]
-        print(f"[INFO] Generating embeddings for {len(texts)} chunks...")
-        embeddings = self.model.encode(texts, show_progress_bar = True)
-        print(f"[INFO] Embeddings shape : {embeddings.shape}")
-        return embeddings
+    def embed_chunks(self, chunks: List[Any], batch_size: int = 500) -> tuple[np.ndarray, List[Any]]:
+        """
+        Embeds chunks in batches for stability and better error reporting.
+        Returns a tuple of (embeddings_array, valid_chunks_list).
+        """
+        valid_chunks = []
+        all_embeddings = []
+        
+        print(f"[INFO] Generating embeddings for {len(chunks)} chunks in batches of {batch_size}...")
+        
+        for i in range(0, len(chunks), batch_size):
+            batch = chunks[i:i+batch_size]
+            batch_texts = []
+            current_batch_chunks = []
+            
+            # Additional validation per chunk
+            for chunk in batch:
+                content = getattr(chunk, 'page_content', None)
+                if content is not None and isinstance(content, str) and content.strip():
+                    batch_texts.append(content)
+                    current_batch_chunks.append(chunk)
+            
+            if not batch_texts:
+                continue
+                
+            try:
+                batch_embeddings = self.model.encode(batch_texts, show_progress_bar=False)
+                all_embeddings.append(batch_embeddings)
+                valid_chunks.extend(current_batch_chunks)
+            except Exception as e:
+                print(f"[ERROR] Failed to embed batch starting at index {i}. Error: {e}")
+                print("[INFO] Attempting to identify problematic chunk in batch...")
+                for j, text in enumerate(batch_texts):
+                    try:
+                        self.model.encode([text], show_progress_bar=False)
+                    except Exception as ex:
+                        print(f"[ERROR] Problematic chunk found at original index {i+j}!")
+                        # Safely encode for windows terminal printing to avoid crashes
+                        safe_text = text[:100].encode('ascii', 'replace').decode('ascii')
+                        print(f"[DEBUG] Content snippet: {safe_text}...")
+                        # We skip this specific chunk and continue
+                        continue
+        
+        if not all_embeddings:
+            print("[WARNING] No embeddings were generated.")
+            return np.array([]), []
+            
+        final_embeddings = np.vstack(all_embeddings)
+        print(f"[INFO] Total valid embeddings: {final_embeddings.shape[0]} / {len(chunks)}")
+        return final_embeddings, valid_chunks
     
 if __name__ == "__main__":
     docs = load_all_documents('Research/data/pdf')

src/vectorstore.py

@@ -32,8 +32,13 @@ class FaissVectorStore:
             chunk_overlap=self.chunk_overlap,
         )
         chunks = emb_pipe.chunk_documents(documents)
-        embeddings = emb_pipe.embed_chunks(chunks)
-        metadatas = [{"texts": chunk.page_content} for chunk in chunks]
+        embeddings, valid_chunks = emb_pipe.embed_chunks(chunks)
+        
+        if len(valid_chunks) == 0:
+            print("[WARNING] No valid chunks were embedded. Vector store will not be updated.")
+            return
+
+        metadatas = [{"texts": chunk.page_content} for chunk in valid_chunks]
         self.add_embeddings(np.array(embeddings).astype("float32"), metadatas)
         self.save()
         print(f"[INFO] Vector Store built and saved to {self.persist_dir}")