chunk size = 8192

config.py

@@ -50,7 +50,7 @@ AVAILABLE_MODELS = {
 
 DEFAULT_MODEL = "Gemini 2.5 Flash"
 
-CHUNK_SIZE = 2048
+CHUNK_SIZE = 8192
 CHUNK_OVERLAP = 256
 
 CUSTOM_PROMPT = """

documents_prep.py

@@ -8,44 +8,15 @@ from llama_index.core.text_splitter import SentenceSplitter
 from config import CHUNK_SIZE, CHUNK_OVERLAP
 
 
-import tiktoken
-from transformers import AutoTokenizer
-
-def count_tokens(text, model="gpt-3.5-turbo"):
-    try:
-        tokenizer = AutoTokenizer.from_pretrained("microsoft/DialoGPT-medium")
-        max_chunk_size = 2048  
-        total_tokens = 0
-        
-        for i in range(0, len(text), max_chunk_size * 4):  # Approximate 4 chars per token
-            chunk = text[i:i + max_chunk_size * 4]
-            tokens = tokenizer.encode(chunk, add_special_tokens=False, truncation=True, max_length=1024)
-            total_tokens += len(tokens)
-        
-        return total_tokens
-    except:
-        try:
-            encoding = tiktoken.encoding_for_model(model)
-            return len(encoding.encode(text))
-        except:
-            # Final fallback: approximate 1 token = 4 characters
-            return len(text) // 4
-
 def chunk_document(doc, chunk_size=None, chunk_overlap=None):
-    """Chunk document based on tokens instead of characters"""
     if chunk_size is None:
         chunk_size = CHUNK_SIZE
     if chunk_overlap is None:
         chunk_overlap = CHUNK_OVERLAP
-    
-    from llama_index.core.text_splitter import TokenTextSplitter
-    
-    # Use TokenTextSplitter instead of SentenceSplitter
-    text_splitter = TokenTextSplitter(
+    text_splitter = SentenceSplitter(
         chunk_size=chunk_size,
         chunk_overlap=chunk_overlap,
-        separator=" ",
-        backup_separators=["\n", ".", "!", "?"]
+        separator=" "
     )
     
     text_chunks = text_splitter.split_text(doc.text)
@@ -53,12 +24,10 @@ def chunk_document(doc, chunk_size=None, chunk_overlap=None):
     chunked_docs = []
     for i, chunk_text in enumerate(text_chunks):
         chunk_metadata = doc.metadata.copy()
-        chunk_tokens = count_tokens(chunk_text)
         chunk_metadata.update({
             "chunk_id": i,
             "total_chunks": len(text_chunks),
-            "chunk_size_tokens": chunk_tokens,
-            "chunk_size_chars": len(chunk_text),
+            "chunk_size": len(chunk_text),
             "original_doc_id": doc.id_ if hasattr(doc, 'id_') else None
         })
         
@@ -70,6 +39,7 @@ def chunk_document(doc, chunk_size=None, chunk_overlap=None):
     
     return chunked_docs
 
+
 def process_documents_with_chunking(documents):
     all_chunked_docs = []
     chunk_info = []
@@ -81,27 +51,24 @@ def process_documents_with_chunking(documents):
     
     for doc in documents:
         doc_type = doc.metadata.get('type', 'text')
-        doc_tokens = count_tokens(doc.text)
-        doc_chars = len(doc.text)
         
         if doc_type == 'table':
             table_count += 1
-            if doc_tokens > CHUNK_SIZE:
+            doc_size = len(doc.text)
+            if doc_size > CHUNK_SIZE:
                 large_tables_count += 1
-                log_message(f"Large table found: {doc.metadata.get('table_number', 'unknown')} in document {doc.metadata.get('document_id', 'unknown')}, size: {doc_tokens} tokens")
+                log_message(f"Large table found: {doc.metadata.get('table_number', 'unknown')} in document {doc.metadata.get('document_id', 'unknown')}, size: {doc_size} characters")
                 
                 # Chunk large tables
                 chunked_docs = chunk_document(doc)
                 all_chunked_docs.extend(chunked_docs)
                 
                 for i, chunk_doc in enumerate(chunked_docs):
-                    chunk_tokens = chunk_doc.metadata.get('chunk_size_tokens', count_tokens(chunk_doc.text))
                     chunk_info.append({
                         'document_id': chunk_doc.metadata.get('document_id', 'unknown'),
                         'section_id': chunk_doc.metadata.get('section_id', 'unknown'),
                         'chunk_id': i,
-                        'chunk_size_tokens': chunk_tokens,
-                        'chunk_size_chars': len(chunk_doc.text),
+                        'chunk_size': len(chunk_doc.text),
                         'chunk_preview': chunk_doc.text[:200] + "..." if len(chunk_doc.text) > 200 else chunk_doc.text,
                         'type': 'table',
                         'table_number': chunk_doc.metadata.get('table_number', 'unknown')
@@ -112,8 +79,7 @@ def process_documents_with_chunking(documents):
                     'document_id': doc.metadata.get('document_id', 'unknown'),
                     'section_id': doc.metadata.get('section_id', 'unknown'),
                     'chunk_id': 0,
-                    'chunk_size_tokens': doc_tokens,
-                    'chunk_size_chars': doc_chars,
+                    'chunk_size': doc_size,
                     'chunk_preview': doc.text[:200] + "..." if len(doc.text) > 200 else doc.text,
                     'type': 'table',
                     'table_number': doc.metadata.get('table_number', 'unknown')
@@ -121,22 +87,21 @@ def process_documents_with_chunking(documents):
             
         elif doc_type == 'image':
             image_count += 1
-            if doc_tokens > CHUNK_SIZE:
+            doc_size = len(doc.text)
+            if doc_size > CHUNK_SIZE:
                 large_images_count += 1
-                log_message(f"Large image description found: {doc.metadata.get('image_number', 'unknown')} in document {doc.metadata.get('document_id', 'unknown')}, size: {doc_tokens} tokens")
+                log_message(f"Large image description found: {doc.metadata.get('image_number', 'unknown')} in document {doc.metadata.get('document_id', 'unknown')}, size: {doc_size} characters")
                 
                 # Chunk large images
                 chunked_docs = chunk_document(doc)
                 all_chunked_docs.extend(chunked_docs)
                 
                 for i, chunk_doc in enumerate(chunked_docs):
-                    chunk_tokens = chunk_doc.metadata.get('chunk_size_tokens', count_tokens(chunk_doc.text))
                     chunk_info.append({
                         'document_id': chunk_doc.metadata.get('document_id', 'unknown'),
                         'section_id': chunk_doc.metadata.get('section_id', 'unknown'),
                         'chunk_id': i,
-                        'chunk_size_tokens': chunk_tokens,
-                        'chunk_size_chars': len(chunk_doc.text),
+                        'chunk_size': len(chunk_doc.text),
                         'chunk_preview': chunk_doc.text[:200] + "..." if len(chunk_doc.text) > 200 else chunk_doc.text,
                         'type': 'image',
                         'image_number': chunk_doc.metadata.get('image_number', 'unknown')
@@ -147,27 +112,25 @@ def process_documents_with_chunking(documents):
                     'document_id': doc.metadata.get('document_id', 'unknown'),
                     'section_id': doc.metadata.get('section_id', 'unknown'),
                     'chunk_id': 0,
-                    'chunk_size_tokens': doc_tokens,
-                    'chunk_size_chars': doc_chars,
+                    'chunk_size': doc_size,
                     'chunk_preview': doc.text[:200] + "..." if len(doc.text) > 200 else doc.text,
                     'type': 'image',
                     'image_number': doc.metadata.get('image_number', 'unknown')
                 })
             
         else:  # text documents
-            if doc_tokens > CHUNK_SIZE:
+            doc_size = len(doc.text)
+            if doc_size > CHUNK_SIZE:
                 chunked_docs = chunk_document(doc)
                 all_chunked_docs.extend(chunked_docs)
                 text_chunks_count += len(chunked_docs)
                 
                 for i, chunk_doc in enumerate(chunked_docs):
-                    chunk_tokens = chunk_doc.metadata.get('chunk_size_tokens', count_tokens(chunk_doc.text))
                     chunk_info.append({
                         'document_id': chunk_doc.metadata.get('document_id', 'unknown'),
                         'section_id': chunk_doc.metadata.get('section_id', 'unknown'),
                         'chunk_id': i,
-                        'chunk_size_tokens': chunk_tokens,
-                        'chunk_size_chars': len(chunk_doc.text),
+                        'chunk_size': len(chunk_doc.text),
                         'chunk_preview': chunk_doc.text[:200] + "..." if len(chunk_doc.text) > 200 else chunk_doc.text,
                         'type': 'text'
                     })
@@ -177,24 +140,22 @@ def process_documents_with_chunking(documents):
                     'document_id': doc.metadata.get('document_id', 'unknown'),
                     'section_id': doc.metadata.get('section_id', 'unknown'),
                     'chunk_id': 0,
-                    'chunk_size_tokens': doc_tokens,
-                    'chunk_size_chars': doc_chars,
+                    'chunk_size': doc_size,
                     'chunk_preview': doc.text[:200] + "..." if len(doc.text) > 200 else doc.text,
                     'type': 'text'
                 })
     
     log_message(f"=== PROCESSING STATISTICS ===")
     log_message(f"Total tables processed: {table_count}")
-    log_message(f"Large tables (>{CHUNK_SIZE} tokens): {large_tables_count}")
+    log_message(f"Large tables (>{CHUNK_SIZE} chars): {large_tables_count}")
     log_message(f"Total images processed: {image_count}")
-    log_message(f"Large images (>{CHUNK_SIZE} tokens): {large_images_count}")
+    log_message(f"Large images (>{CHUNK_SIZE} chars): {large_images_count}")
     log_message(f"Total text chunks created: {text_chunks_count}")
     log_message(f"Total documents after processing: {len(all_chunked_docs)}")
     
     return all_chunked_docs, chunk_info
 
 
-
 def extract_text_from_json(data, document_id, document_name):
     documents = []