chunk size = 2048 + rows=15

documents_prep.py

@@ -7,7 +7,7 @@ from llama_index.core.text_splitter import SentenceSplitter
 from my_logging import log_message
 
 # Configuration
-CHUNK_SIZE = 1500
+CHUNK_SIZE = 2048
 CHUNK_OVERLAP = 128
 
 def chunk_text_documents(documents):
@@ -38,7 +38,7 @@ def chunk_text_documents(documents):
     return chunked
 
 
-def chunk_table_by_rows(table_data, doc_id, rows_per_chunk=20, max_chars=2000):
+def chunk_table_by_rows(table_data, doc_id, rows_per_chunk=15, max_chars=2000):
     """
     Chunk tables by rows with fallback to character limit.
     Keeps 3-4 rows together, but splits individual rows if they're too large.