chunk size = 2048 + rows=15

documents_prep.py

@@ -6,13 +6,14 @@ from llama_index.core import Document
 from llama_index.core.text_splitter import SentenceSplitter
 from my_logging import log_message
 
-CHUNK_SIZE = 2048
+# Configuration
+CHUNK_SIZE = 1500
 CHUNK_OVERLAP = 128
 
 def chunk_text_documents(documents):
     text_splitter = SentenceSplitter(
         chunk_size=CHUNK_SIZE,
-        chunk_overlap=CHUNK_OVERLAP,
+        chunk_overlap=CHUNK_OVERLAP
     )
     
     chunked = []
@@ -22,7 +23,7 @@ def chunk_text_documents(documents):
             chunk.metadata.update({
                 'chunk_id': i,
                 'total_chunks': len(chunks),
-                'chunk_size': len(chunk.text)
+                'chunk_size': len(chunk.text)  # Add chunk size
             })
             chunked.append(chunk)
     
@@ -37,7 +38,7 @@ def chunk_text_documents(documents):
     return chunked
 
 
-def chunk_table_by_rows(table_data, doc_id, rows_per_chunk=15, max_chars=3000):
+def chunk_table_by_rows(table_data, doc_id, rows_per_chunk=10, max_chars=2000):
     """
     Chunk tables by rows with fallback to character limit.
     Keeps 3-4 rows together, but splits individual rows if they're too large.