Upload 7 files

embedding_utils.py

@@ -87,7 +87,7 @@ def parallel_generate_embeddings(
         
     processed = 0
     current_batch_size = batch_size
-    max_workers = 5  # Start with fewer workers
+    max_workers = 10  # Start with fewer workers
     
     logger.info(f"Starting embedding generation for {total_docs} documents")
     if callback:
@@ -122,7 +122,7 @@ def parallel_generate_embeddings(
                         logger.error(f"Error processing futures: {str(e)}")
                         # Reduce batch size and workers on error
                         current_batch_size = max(5, current_batch_size - 5)
-                        max_workers = max(3, max_workers - 2)
+                        max_workers = max(5, max_workers - 2)
                         logger.info(f"Reduced batch size to {current_batch_size}, workers to {max_workers}")
         
         # Process remaining batch