enhance performance

app.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Tuple
+from typing import Optional, Tuple
 from queue import Empty, Queue
 from threading import Thread
 from bot.web_scrapping.crawler_and_indexer import content_crawler_and_index
@@ -16,12 +16,10 @@ human_message_prompt_template = HumanMessagePromptTemplate.from_template("{text}
 
 
 def bot_learning(urls, file_formats, llm, prompt, chat_mode=False):
-    index = content_crawler_and_index(url=str(urls), llm=llm, prompt=prompt, file_format=file_formats)
     if chat_mode:
-        return index
+        return content_crawler_and_index(url=str(urls), llm=llm, prompt=prompt, file_format=file_formats)
     else:
-        fb = 'Training Completed'
-        return fb
+        return 'Training Completed'
 
 
 def chat_start(

bot/web_scrapping/searchable_index.py

@@ -13,6 +13,7 @@ import pandas as pd
 import threading
 import glob
 import os
+import asyncio
 import queue
 
 
@@ -72,41 +73,52 @@ class SearchableIndex:
         if os.path.exists(index_store):
             local_db = FAISS.load_local(index_store, embeddings)
             local_db.merge_from(faiss_db)
-            local_db.save_local(index_store)
-            logger.info("Merge index completed")
+            operation_info = "Merge"
         else:
-            faiss_db.save_local(folder_path=index_store)
-            logger.info("New store created and loaded...")
-            local_db = FAISS.load_local(index_store, embeddings)
+            local_db = faiss_db  # Use the provided faiss_db directly for a new store
+            operation_info = "New store creation"
+
+        local_db.save_local(index_store)
+        logger.info(f"{operation_info} index completed")
         return local_db
 
     @classmethod
-    def check_and_load_index(cls, index_files, embeddings, logger, result_queue):
+    def load_index(cls, index_files, embeddings, logger):
         if index_files:
-            local_db = FAISS.load_local(index_files[0], embeddings)
-        else:
-            raise logger.warning("Index store does not exist")
-        result_queue.put(local_db)  # Put the result in the queue
+            return FAISS.load_local(index_files[0], embeddings)
+        logger.warning("Index store does not exist")
+        return None
+
+    @classmethod
+    def check_and_load_index(cls, index_files, embeddings, logger, result_queue):
+        local_db = cls.load_index(index_files, embeddings, logger)
+        result_queue.put(local_db)
+
+    @classmethod
+    def load_index_asynchronously(cls, index_files, embeddings, logger):
+        result_queue = queue.Queue()
+        thread = threading.Thread(
+            target=cls.check_and_load_index,
+            args=(index_files, embeddings, logger, result_queue)
+        )
+        thread.start()
+        thread.join()  # Wait for the thread to finish
+        return result_queue.get()
 
     @classmethod
     def embed_index(cls, url, path, llm, prompt, target_col=None, sheet_name=None):
         embeddings = OpenAIEmbeddings()
 
-        if url != 'NO_URL' and path:
-            doc_list = cls.get_splits(path, target_col, sheet_name)
-            faiss_db = FAISS.from_texts(doc_list, embeddings)
-            index_store = os.path.splitext(path)[0] + "_index"
-            local_db = cls.merge_or_create_index(index_store, faiss_db, embeddings, logger)
-            return Query(prompt, llm, local_db)
-        elif url == 'NO_URL' and path:
-            index_files = glob.glob(os.path.join(path, '*_index'))
-
-            result_queue = queue.Queue()  # Create a queue to store the result
+        if path:
+            if url != 'NO_URL':
+                doc_list = cls.get_splits(path, target_col, sheet_name)
+                faiss_db = FAISS.from_texts(doc_list, embeddings)
+                index_store = os.path.splitext(path)[0] + "_index"
+                local_db = cls.merge_or_create_index(index_store, faiss_db, embeddings, logger)
+                return Query(prompt, llm, local_db)
 
-            thread = threading.Thread(target=cls.check_and_load_index,
-                                      args=(index_files, embeddings, logger, result_queue))
-            thread.start()
-            local_db = result_queue.get()  # Retrieve the result from the queue
+            index_files = glob.glob(os.path.join(path, '*_index'))
+            local_db = cls.load_index_asynchronously(index_files, embeddings, logger)
             return Query(prompt, llm, local_db)
 
 

learning_documents/combined_content_index/index.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bbc4c242c12ce0d77cdeb962c2854f6c0888ce56f6d098a40570e16caff75d52
-size 6059
+oid sha256:b0d19bc7afb8227f67225b52afd8c746bc67aceca43fb5e5c84a19e94cda0e9d
+size 3959

requirements.txt

@@ -154,6 +154,7 @@ typer==0.9.0
 typing-inspect==0.9.0
 typing_extensions==4.8.0
 tzdata==2023.3
+twine
 unstructured==0.10.29
 urllib3==1.26.18
 uvicorn==0.24.0.post1