Update scripts/process.py

scripts/process.py

@@ -58,39 +58,29 @@ def load_document(
                               id_hash_keys=id_hash_keys))
     
     return documents
-    
- def preprocessing(document, 
-                  split_by: Literal["sentence", "word"] = 'sentence',
-                  split_length:int = 3):
-
+ 
+ def preprocessing(document):
     """
-    takes in haystack document object and splits it into synthetically generated paragraphs and applies simple cleaning.
+    takes in haystack document object and splits it into paragraphs and applies simple cleaning.
     Returns cleaned list of haystack document objects. One paragraph per object. Also returns pandas df and 
     list that contains all text joined together.
     """    
-    if split_by == 'sentence':
-      split_respect_sentence_boundary = False
-      split_overlap=0
-    else:
-      split_respect_sentence_boundary = True
-      split_overlap= 20
-    
+
     preprocessor = PreProcessor(
         clean_empty_lines=True,
         clean_whitespace=True,
         clean_header_footer=True,
-        split_by=split_by,
-        split_length=split_length,
-        split_respect_sentence_boundary= split_respect_sentence_boundary,
-        split_overlap=split_overlap
+        split_by="sentence",
+        split_length=3,
+        split_respect_sentence_boundary=False,
+        split_overlap=1
     )
     for i in document:
         docs_processed = preprocessor.process([i])
         for item in docs_processed:
             item.content = basic(item.content)
 
-    print("\n your document has been splitted to", len(docs_processed), "paragraphs")
-    # logger.info("document has been splitted to {}".format(len(docs_processed)))
+    st.write("your document has been splitted to", len(docs_processed), "paragraphs")
     
     # create dataframe of text and list of all text
     #df = pd.DataFrame(docs_processed)
@@ -98,5 +88,6 @@ def load_document(
     #par_list = df.content.to_list()
 
     return docs_processed #, df, all_text, par_list
+