Update app.py

app.py

@@ -88,7 +88,7 @@ def initialize_embeddings():
     return embeddings
 
 def load_vector_store(embeddings):
-    """Load FAISS vector store with compatibility handling"""
+    """Load FAISS vector store with full Pydantic bypass"""
     logger.info("🔄 Loading FAISS vector store...")
     
     vector_store_path = CONFIG["vector_store_path"]
@@ -118,34 +118,87 @@ def load_vector_store(embeddings):
         logger.info(f"✅ FAISS vector store loaded successfully")
         return vectorstore
         
-    except (KeyError, AttributeError) as e:
-        logger.warning(f"⚠️ Pydantic version mismatch detected: {e}")
-        logger.info("🔄 Attempting compatibility fix...")
+    except (KeyError, AttributeError, Exception) as e:
+        logger.warning(f"⚠️ Pydantic version mismatch: {e}")
+        logger.info("🔄 Using custom pickle loader to bypass Pydantic...")
         
-        # Monkey-patch for Pydantic v1/v2 compatibility
-        import pickle
         import faiss
+        import pickle
+        from langchain_community.docstore.in_memory import InMemoryDocstore
+        
+        # Custom unpickler that bypasses Pydantic validation
+        class PydanticBypassUnpickler(pickle.Unpickler):
+            def find_class(self, module, name):
+                # Redirect Pydantic Document to LangChain Document
+                if 'pydantic' in module or name == 'Document':
+                    return Document
+                return super().find_class(module, name)
         
-        # Load FAISS index directly
+        # Load FAISS index
+        logger.info("   Loading FAISS index...")
         index = faiss.read_index(index_file)
         
-        # Load pickle with custom unpickler
+        # Load pickle with bypass
+        logger.info("   Loading documents with Pydantic bypass...")
         with open(pkl_file, "rb") as f:
+            unpickler = PydanticBypassUnpickler(f)
+            
+            # Manually parse pickle structure
             try:
-                data = pickle.load(f)
-                docstore = data[0]
-                index_to_docstore_id = data[1]
+                raw_data = unpickler.load()
+                
+                # Extract docstore and index mapping
+                if isinstance(raw_data, tuple) and len(raw_data) >= 2:
+                    docstore_data = raw_data[0]
+                    index_to_docstore_id = raw_data[1]
+                else:
+                    raise ValueError("Unexpected pickle structure")
+                
+                # Rebuild docstore with new Document objects
+                new_docstore_dict = {}
+                
+                if hasattr(docstore_data, '_dict'):
+                    old_docs = docstore_data._dict
+                elif isinstance(docstore_data, dict):
+                    old_docs = docstore_data
+                else:
+                    old_docs = {}
+                
+                logger.info(f"   Rebuilding {len(old_docs)} documents...")
+                
+                for doc_id, old_doc in old_docs.items():
+                    # Extract content and metadata safely
+                    if hasattr(old_doc, 'page_content'):
+                        content = old_doc.page_content
+                    elif isinstance(old_doc, dict):
+                        content = old_doc.get('page_content', '')
+                    else:
+                        content = str(old_doc)
+                    
+                    if hasattr(old_doc, 'metadata'):
+                        metadata = old_doc.metadata if isinstance(old_doc.metadata, dict) else {}
+                    elif isinstance(old_doc, dict):
+                        metadata = old_doc.get('metadata', {})
+                    else:
+                        metadata = {}
+                    
+                    # Create fresh Document without Pydantic issues
+                    new_doc = Document(
+                        page_content=content,
+                        metadata=metadata
+                    )
+                    new_docstore_dict[doc_id] = new_doc
+                
+                # Create new docstore
+                docstore = InMemoryDocstore(new_docstore_dict)
+                
+                logger.info(f"   ✅ Rebuilt {len(new_docstore_dict)} documents successfully")
+                
             except Exception as e2:
-                logger.error(f"❌ Failed to load pickle data: {e2}")
+                logger.error(f"❌ Custom unpickler failed: {e2}")
                 raise
         
-        # Create FAISS vectorstore manually
-        from langchain_community.docstore.in_memory import InMemoryDocstore
-        
-        # Ensure docstore is proper type
-        if not isinstance(docstore, InMemoryDocstore):
-            docstore = InMemoryDocstore(docstore._dict if hasattr(docstore, '_dict') else {})
-        
+        # Create FAISS vectorstore
         vectorstore = FAISS(
             embedding_function=embeddings,
             index=index,
@@ -153,7 +206,7 @@ def load_vector_store(embeddings):
             index_to_docstore_id=index_to_docstore_id
         )
         
-        logger.info(f"✅ FAISS vector store loaded with compatibility fix")
+        logger.info(f"✅ FAISS vector store loaded with custom loader")
         return vectorstore
 
 # ============================================================================