Update Ingest.py

Ingest.py

@@ -4,7 +4,7 @@ from langchain_community.document_loaders import DirectoryLoader
 from langchain_community.embeddings import HuggingFaceEmbeddings
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.vectorstores import FAISS
-from faiss import IndexFlatL2  # Assuming using L2 distance for simplicity
+from faiss import IndexFlatL2
 
 # Initialize Ray
 ray.init()
@@ -17,24 +17,29 @@ logging.info("Loading documents...")
 loader = DirectoryLoader('data', glob="./*.txt")
 documents = loader.load()
 
-# Extract text from documents and split into manageable texts with logging
+# Extract text from documents and split into manageable chunks with logging
 logging.info("Extracting and splitting texts from documents...")
 text_splitter = RecursiveCharacterTextSplitter(chunk_size=1024, chunk_overlap=200)
 texts = []
 for document in documents:
-    if hasattr(document, 'get_text'):
-        text_content = document.get_text()  # Adjust according to actual method
-    else:
-        text_content = ""  # Default to empty string if no text method is available
+    try:
+        if hasattr(document, 'get_text'):
+            text_content = document.get_text()  # Adjust according to actual method
+        else:
+            text_content = ""  # Default to empty string if no text method is available
 
-    texts.extend(text_splitter.split_text(text_content))
+        texts.extend(text_splitter.split_text(text_content))
+    except Exception as e:
+        logging.error(f"Error processing document {document}: {e}")
 
-# Define embedding function
+# Initialize embedding model once outside the loop
+embeddings_model = HuggingFaceEmbeddings(model_name="law-ai/InLegalBERT")
+
+# Define embedding function (optimized to use pre-initialized model)
 def embedding_function(text):
-    embeddings_model = HuggingFaceEmbeddings(model_name="law-ai/InLegalBERT")
     return embeddings_model.embed_query(text)
 
-# Create FAISS index for embeddings
+# Create FAISS index for embeddings (adjust dimension as needed)
 index = IndexFlatL2(768)  # Dimension of embeddings, adjust as needed
 
 # Assuming docstore as a simple dictionary to store document texts
@@ -47,12 +52,19 @@ faiss_db = FAISS(embedding_function, index, docstore, index_to_docstore_id)
 # Process and store embeddings
 logging.info("Storing embeddings in FAISS...")
 for i, text in enumerate(texts):
-    embedding = embedding_function(text)
-    faiss_db.add_documents([embedding])
+    try:
+        embedding = embedding_function(text)
+        faiss_db.add_documents([embedding])
+    except Exception as e:
+        logging.error(f"Error embedding document {i}: {e}")
 
 # Exporting the vector embeddings database with logging
 logging.info("Exporting the vector embeddings database...")
-faiss_db.save_local("ipc_embed_db")
+try:
+    faiss_db.save_local("ipc_embed_db")
+    logging.info("Export completed successfully.")
+except Exception as e:
+    logging.error(f"Error exporting FAISS database: {e}")
 
 # Log a message to indicate the completion of the process
 logging.info("Process completed successfully.")