made corrections for preprocessing

app.py

@@ -8,7 +8,7 @@ from json import JSONDecodeError
 from markdown import markdown
 from utils.config import parser
 from utils.haystack import start_document_store, start_haystack_extractive, start_haystack_rag, query, start_preprocessor_node, start_retriever, start_reader
-from utils.ui import reset_results, set_initial_state, upload_doc
+from utils.ui import reset_results, set_initial_state
 
 # Sliders
 DEFAULT_DOCS_FROM_RETRIEVER = int(os.getenv("DEFAULT_DOCS_FROM_RETRIEVER", "3"))
@@ -34,7 +34,7 @@ def upload_files():
 
 def process_file(data_file, preprocesor, document_store):
     # read file and add content
-    file_contents = data_file.read()
+    file_contents = data_file.read().decode("utf-8")
     docs = [{
         'content': str(file_contents),
         'meta': {'name': str(data_file.name)}
@@ -47,6 +47,7 @@ def process_file(data_file, preprocesor, document_store):
             print(f"{data_file.name} already processed")
         else:
             print(f'preprocessing uploaded doc {data_file.name}.......')
+            #print(data_file.read().decode("utf-8"))
             preprocessed_docs = preprocesor.process(docs)
             print('writing to document store.......')
             document_store.write_documents(preprocessed_docs)

utils/haystack.py

@@ -13,7 +13,7 @@ from milvus_haystack import MilvusDocumentStore
 def start_preprocessor_node():
     print('initializing preprocessor node')
     processor = PreProcessor(
-        clean_empty_lines=True,
+        clean_empty_lines= True,
         clean_whitespace=True,
         clean_header_footer=True,
         #remove_substrings=None,