resolving merge conflicts

app.py

@@ -7,14 +7,52 @@ from annotated_text import annotation
 from json import JSONDecodeError
 from markdown import markdown
 from utils.config import parser
-from utils.haystack import start_document_store, query, initialize_pipeline
+from utils.haystack import start_document_store, query, initialize_pipeline, start_preprocessor_node, start_retriever, start_reader
 from utils.ui import reset_results, set_initial_state
 import pandas as pd
 import haystack
 
+# Whether the file upload should be enabled or not
+DISABLE_FILE_UPLOAD = bool(os.getenv("DISABLE_FILE_UPLOAD"))
+# Define a function to handle file uploads
+def upload_files():
+    uploaded_files = st.sidebar.file_uploader(
+            "upload", type=["pdf", "txt", "docx"], accept_multiple_files=True, label_visibility="hidden"
+        )
+    return uploaded_files
+
+# Define a function to process a single file
+
+def process_file(data_file, preprocesor, document_store):
+    # read file and add content
+    file_contents = data_file.read().decode("utf-8")
+    docs = [{
+        'content': str(file_contents),
+        'meta': {'name': str(data_file.name)}
+    }]
+    try:
+        names = [item.meta.get('name') for item in document_store.get_all_documents()]
+        #if args.store == 'inmemory':
+        # doc = converter.convert(file_path=files, meta=None)
+        if data_file.name in names:
+            print(f"{data_file.name} already processed")
+        else:
+            print(f'preprocessing uploaded doc {data_file.name}.......')
+            #print(data_file.read().decode("utf-8"))
+            preprocessed_docs = preprocesor.process(docs)
+            print('writing to document store.......')
+            document_store.write_documents(preprocessed_docs)
+            print('updating emebdding.......')
+            document_store.update_embeddings(retriever)
+    except Exception as e:
+        print(e)
+
 try:
     args = parser.parse_args()
+    preprocesor = start_preprocessor_node()
     document_store = start_document_store(type=args.store)
+    retriever = start_retriever(document_store)
+    reader = start_reader()
     st.set_page_config(
         page_title="MLReplySearch",
         layout="centered",
@@ -42,19 +80,42 @@ try:
 
     # Check the task and initialize pipeline accordingly
     if task_selection == 'Extractive':
-        pipeline_extractive = initialize_pipeline("extractive", document_store)
+        pipeline_extractive = initialize_pipeline("extractive", document_store, retriever, reader)
     elif task_selection == 'Generative' and openai_key:  # Check for openai_key to ensure user has entered it
-        pipeline_rag = initialize_pipeline("rag", document_store, openai_key=openai_key)
+        pipeline_rag = initialize_pipeline("rag", document_store, retriever, reader, openai_key=openai_key)
+
 
     set_initial_state()
 
     st.write('# ' + args.name)
 
+
+    # File upload block
+    if not DISABLE_FILE_UPLOAD:
+        st.sidebar.write("## File Upload:")
+        #data_files = st.sidebar.file_uploader(
+        #    "upload", type=["pdf", "txt", "docx"], accept_multiple_files=True, label_visibility="hidden"
+        #)
+        data_files = upload_files()
+        if data_files is not None:
+            for data_file in data_files:
+                # Upload file
+                if data_file:
+                    try:
+                        #raw_json = upload_doc(data_file)
+                        # Call the process_file function for each uploaded file
+                        if args.store == 'inmemory':
+                            processed_data = process_file(data_file, preprocesor, document_store)
+                        st.sidebar.write(str(data_file.name) + "    ✅ ")
+                    except Exception as e:
+                        st.sidebar.write(str(data_file.name) + "    ❌ ")
+                        st.sidebar.write("_This file could not be parsed, see the logs for more information._")
+
     if "question" not in st.session_state:
         st.session_state.question = ""
     # Search bar
     question = st.text_input("", value=st.session_state.question, max_chars=100, on_change=reset_results)
-    
+
     run_pressed = st.button("Run")
 
     run_query = (
@@ -73,11 +134,11 @@ try:
                 except JSONDecodeError as je:
                     st.error(
                         "👓    An error occurred reading the results. Is the document store working?"
-                    )    
+                    )
                 except Exception as e:
                     logging.exception(e)
                     st.error("🐞    An error occurred during the request.")
-        
+
         elif task_selection == 'Generative':
             reset_results()
             st.session_state.question = question
@@ -88,7 +149,7 @@ try:
                 except JSONDecodeError as je:
                     st.error(
                         "👓    An error occurred reading the results. Is the document store working?"
-                    )    
+                    )
                 except Exception as e:
                     if "API key is invalid" in str(e):
                         logging.exception(e)
@@ -98,11 +159,11 @@ try:
                         st.error("🐞    An error occurred during the request.")
     # Display results
     if (st.session_state.results_extractive or st.session_state.results_generative) and run_query:
-        
+
         # Handle Extractive Answers
         if task_selection == 'Extractive':
             results = st.session_state.results_extractive
-            
+
             st.subheader("Extracted Answers:")
 
             if 'answers' in results:

utils/haystack.py

@@ -5,15 +5,34 @@ from haystack import Pipeline
 from haystack.schema import Answer
 from haystack.document_stores import BaseDocumentStore
 from haystack.document_stores import InMemoryDocumentStore, OpenSearchDocumentStore, WeaviateDocumentStore
-from haystack.nodes import EmbeddingRetriever, FARMReader, PromptNode
+from haystack.nodes import EmbeddingRetriever, FARMReader, PromptNode, PreProcessor
 from milvus_haystack import MilvusDocumentStore
 #Use this file to set up your Haystack pipeline and querying
 
+@st.cache_resource(show_spinner=False)
+def start_preprocessor_node():
+    print('initializing preprocessor node')
+    processor = PreProcessor(
+        clean_empty_lines= True,
+        clean_whitespace=True,
+        clean_header_footer=True,
+        #remove_substrings=None,
+        split_by="word",
+        split_length=100,
+        split_respect_sentence_boundary=True,
+        #split_overlap=0,
+        #max_chars_check= 10_000
+    )
+    return processor
+    #return docs
+
 @st.cache_resource(show_spinner=False)
 def start_document_store(type: str):
     #This function starts the documents store of your choice based on your command line preference
+    print('initializing document store')
     if type == 'inmemory':
         document_store = InMemoryDocumentStore(use_bm25=True, embedding_dim=384)
+        '''
         documents = [
             {
                 'content': "Pi is a super dog",
@@ -25,6 +44,7 @@ def start_document_store(type: str):
             },
         ]
         document_store.write_documents(documents)
+        '''
     elif type == 'opensearch':
         document_store = OpenSearchDocumentStore(scheme = document_store_configs['OPENSEARCH_SCHEME'], 
                                                  username = document_store_configs['OPENSEARCH_USERNAME'], 
@@ -47,32 +67,42 @@ def start_document_store(type: str):
 
 # cached to make index and models load only at start
 @st.cache_resource(show_spinner=False)
-def start_haystack_extractive(_document_store: BaseDocumentStore):
-    retriever = EmbeddingRetriever(document_store=_document_store, 
-                                   embedding_model=model_configs['EMBEDDING_MODEL'], 
+def start_retriever(_document_store: BaseDocumentStore):
+    print('initializing retriever')
+    retriever = EmbeddingRetriever(document_store=_document_store,
+                                   embedding_model=model_configs['EMBEDDING_MODEL'],
                                    top_k=5)
-    _document_store.update_embeddings(retriever)
+    #
+
+    #_document_store.update_embeddings(retriever)
+    return retriever
 
+
+@st.cache_resource(show_spinner=False)
+def start_reader():
+    print('initializing reader')
     reader = FARMReader(model_name_or_path=model_configs['EXTRACTIVE_MODEL'])
-    
-    pipe = Pipeline()
-    pipe.add_node(component=retriever, name="Retriever", inputs=["Query"])
-    pipe.add_node(component=reader, name="Reader", inputs=["Retriever"])
+    return reader
 
+
+
+# cached to make index and models load only at start
+@st.cache_resource(show_spinner=False)
+def start_haystack_extractive(_document_store: BaseDocumentStore, _retriever: EmbeddingRetriever, _reader: FARMReader):
+    print('initializing pipeline')
+    pipe = Pipeline()
+    pipe.add_node(component=_retriever, name="Retriever", inputs=["Query"])
+    pipe.add_node(component= _reader, name="Reader", inputs=["Retriever"])
     return pipe
 
 @st.cache_resource(show_spinner=False)
-def start_haystack_rag(_document_store: BaseDocumentStore, openai_key):
-    retriever = EmbeddingRetriever(document_store=_document_store, 
-                                   embedding_model=model_configs['EMBEDDING_MODEL'], 
-                                   top_k=5)
-    _document_store.update_embeddings(retriever)
+def start_haystack_rag(_document_store: BaseDocumentStore, _retriever: EmbeddingRetriever, openai_key):
     prompt_node = PromptNode(default_prompt_template="deepset/question-answering", 
                              model_name_or_path=model_configs['GENERATIVE_MODEL'],
                              api_key=openai_key)
     pipe = Pipeline()
 
-    pipe.add_node(component=retriever, name="Retriever", inputs=["Query"])
+    pipe.add_node(component=_retriever, name="Retriever", inputs=["Query"])
     pipe.add_node(component=prompt_node, name="PromptNode", inputs=["Retriever"])
 
     return pipe
@@ -83,8 +113,8 @@ def query(_pipeline, question):
     results = _pipeline.run(question, params=params)
     return results
 
-def initialize_pipeline(task, document_store, openai_key = ""):
+def initialize_pipeline(task, document_store, retriever, reader, openai_key = ""):
     if task == 'extractive':
-        return start_haystack_extractive(document_store)
+        return start_haystack_extractive(document_store, retriever, reader)
     elif task == 'rag':
-        return start_haystack_rag(document_store, openai_key)
+        return start_haystack_rag(document_store, retriever, openai_key)

utils/ui.py

@@ -12,5 +12,5 @@ def set_initial_state():
 
 def reset_results(*args):
     st.session_state.results_extractive = None
-    st.session_state.results_generative = None    
+    st.session_state.results_generative = None
     st.session_state.task = None