chunk size = 2048 + rows=15

documents_prep.py

@@ -6,14 +6,14 @@ from llama_index.core import Document
 from llama_index.core.text_splitter import SentenceSplitter
 from my_logging import log_message
 
-# Configuration
 CHUNK_SIZE = 2048
 CHUNK_OVERLAP = 128
 
 def chunk_text_documents(documents):
     text_splitter = SentenceSplitter(
         chunk_size=CHUNK_SIZE,
-        chunk_overlap=CHUNK_OVERLAP
+        chunk_overlap=CHUNK_OVERLAP,
+        metadata_chunk_size=2048  # ADD THIS LINE - match or exceed chunk_size
     )
     
     chunked = []
@@ -23,7 +23,7 @@ def chunk_text_documents(documents):
             chunk.metadata.update({
                 'chunk_id': i,
                 'total_chunks': len(chunks),
-                'chunk_size': len(chunk.text)  # Add chunk size
+                'chunk_size': len(chunk.text)
             })
             chunked.append(chunk)