Update document_processor.py

document_processor.py

@@ -7,7 +7,7 @@ from utils import clean_text, setup_logger
 
 logger = setup_logger('document_processor')
 
-def split_into_chunks(text, chunk_size=400, overlap=75):
+def split_into_chunks(text, chunk_size=300, overlap=100):
     """
     Split text into overlapping chunks
     
@@ -48,7 +48,7 @@ def split_into_chunks(text, chunk_size=400, overlap=75):
     
     return chunks
 
-def load_documents(folder_path, chunk_size=400, overlap=75):
+def load_documents(folder_path, chunk_size=300, overlap=100):
     """
     Load all .txt documents from folder and split them into chunks
     
@@ -102,7 +102,7 @@ def load_documents(folder_path, chunk_size=400, overlap=75):
     
     return df
 
-def load_single_document(file_path, chunk_size=400, overlap=75):
+def load_single_document(file_path, chunk_size=300, overlap=100):
     """
     Load a single document and split it into chunks