max chunk size= 4000 + max row = 5

documents_prep.py

@@ -7,7 +7,7 @@ from llama_index.core.text_splitter import SentenceSplitter
 from my_logging import log_message
 
 # Configuration
-CHUNK_SIZE = 512
+CHUNK_SIZE = 1024
 CHUNK_OVERLAP = 128
 
 def chunk_text_documents(documents):

table_prep.py

@@ -32,10 +32,6 @@ def create_table_content(table_data):
     
     return content
 
-from llama_index.core.text_splitter import SentenceSplitter
-from config import CHUNK_SIZE, CHUNK_OVERLAP
-
-
 def chunk_table_document(doc, max_chunk_size=MAX_CHUNK_SIZE, max_rows_per_chunk=MAX_ROWS_PER_CHUNK):
     lines = doc.text.strip().split('\n')