token based chunking

app.py

@@ -100,7 +100,7 @@ def initialize_system(repo_id, hf_token, download_dir, chunks_filename=None,
         log_message("Инициализация системы")
         os.makedirs(download_dir, exist_ok=True)
         from config import CHUNK_SIZE, CHUNK_OVERLAP
-        from llama_index.core.text_splitter import SentenceSplitter
+        from llama_index.core.text_splitter import TokenTextSplitter
         
         embed_model = get_embedding_model()
         llm = get_llm_model(DEFAULT_MODEL)
@@ -108,14 +108,15 @@ def initialize_system(repo_id, hf_token, download_dir, chunks_filename=None,
         
         Settings.embed_model = embed_model
         Settings.llm = llm
-        Settings.text_splitter = SentenceSplitter(
+        Settings.text_splitter = TokenTextSplitter(
             chunk_size=CHUNK_SIZE,
             chunk_overlap=CHUNK_OVERLAP,
-            separator=" "
+            separator=" ",
+            backup_separators=["\n", ".", "!", "?"]
         )
         
-        log_message(f"Configured chunk size: {CHUNK_SIZE}")
-        log_message(f"Configured chunk overlap: {CHUNK_OVERLAP}")
+        log_message(f"Configured chunk size: {CHUNK_SIZE} tokens")
+        log_message(f"Configured chunk overlap: {CHUNK_OVERLAP} tokens")
 
         all_documents = []
         chunks_df = None

documents_prep.py

@@ -8,15 +8,32 @@ from llama_index.core.text_splitter import SentenceSplitter
 from config import CHUNK_SIZE, CHUNK_OVERLAP
 
 
+import tiktoken
+
+def count_tokens(text, model="gpt-3.5-turbo"):
+    """Count tokens in text using tiktoken"""
+    try:
+        encoding = tiktoken.encoding_for_model(model)
+        return len(encoding.encode(text))
+    except:
+        # Fallback: approximate 1 token = 4 characters for Russian/English text
+        return len(text) // 4
+
 def chunk_document(doc, chunk_size=None, chunk_overlap=None):
+    """Chunk document based on tokens instead of characters"""
     if chunk_size is None:
         chunk_size = CHUNK_SIZE
     if chunk_overlap is None:
         chunk_overlap = CHUNK_OVERLAP
-    text_splitter = SentenceSplitter(
+    
+    from llama_index.core.text_splitter import TokenTextSplitter
+    
+    # Use TokenTextSplitter instead of SentenceSplitter
+    text_splitter = TokenTextSplitter(
         chunk_size=chunk_size,
         chunk_overlap=chunk_overlap,
-        separator=" "
+        separator=" ",
+        backup_separators=["\n", ".", "!", "?"]
     )
     
     text_chunks = text_splitter.split_text(doc.text)
@@ -24,10 +41,12 @@ def chunk_document(doc, chunk_size=None, chunk_overlap=None):
     chunked_docs = []
     for i, chunk_text in enumerate(text_chunks):
         chunk_metadata = doc.metadata.copy()
+        chunk_tokens = count_tokens(chunk_text)
         chunk_metadata.update({
             "chunk_id": i,
             "total_chunks": len(text_chunks),
-            "chunk_size": len(chunk_text),
+            "chunk_size_tokens": chunk_tokens,
+            "chunk_size_chars": len(chunk_text),
             "original_doc_id": doc.id_ if hasattr(doc, 'id_') else None
         })
         
@@ -39,7 +58,6 @@ def chunk_document(doc, chunk_size=None, chunk_overlap=None):
     
     return chunked_docs
 
-
 def process_documents_with_chunking(documents):
     all_chunked_docs = []
     chunk_info = []
@@ -51,24 +69,27 @@ def process_documents_with_chunking(documents):
     
     for doc in documents:
         doc_type = doc.metadata.get('type', 'text')
+        doc_tokens = count_tokens(doc.text)
+        doc_chars = len(doc.text)
         
         if doc_type == 'table':
             table_count += 1
-            doc_size = len(doc.text)
-            if doc_size > CHUNK_SIZE:
+            if doc_tokens > CHUNK_SIZE:
                 large_tables_count += 1
-                log_message(f"Large table found: {doc.metadata.get('table_number', 'unknown')} in document {doc.metadata.get('document_id', 'unknown')}, size: {doc_size} characters")
+                log_message(f"Large table found: {doc.metadata.get('table_number', 'unknown')} in document {doc.metadata.get('document_id', 'unknown')}, size: {doc_tokens} tokens ({doc_chars} characters)")
                 
                 # Chunk large tables
                 chunked_docs = chunk_document(doc)
                 all_chunked_docs.extend(chunked_docs)
                 
                 for i, chunk_doc in enumerate(chunked_docs):
+                    chunk_tokens = chunk_doc.metadata.get('chunk_size_tokens', count_tokens(chunk_doc.text))
                     chunk_info.append({
                         'document_id': chunk_doc.metadata.get('document_id', 'unknown'),
                         'section_id': chunk_doc.metadata.get('section_id', 'unknown'),
                         'chunk_id': i,
-                        'chunk_size': len(chunk_doc.text),
+                        'chunk_size_tokens': chunk_tokens,
+                        'chunk_size_chars': len(chunk_doc.text),
                         'chunk_preview': chunk_doc.text[:200] + "..." if len(chunk_doc.text) > 200 else chunk_doc.text,
                         'type': 'table',
                         'table_number': chunk_doc.metadata.get('table_number', 'unknown')
@@ -79,7 +100,8 @@ def process_documents_with_chunking(documents):
                     'document_id': doc.metadata.get('document_id', 'unknown'),
                     'section_id': doc.metadata.get('section_id', 'unknown'),
                     'chunk_id': 0,
-                    'chunk_size': doc_size,
+                    'chunk_size_tokens': doc_tokens,
+                    'chunk_size_chars': doc_chars,
                     'chunk_preview': doc.text[:200] + "..." if len(doc.text) > 200 else doc.text,
                     'type': 'table',
                     'table_number': doc.metadata.get('table_number', 'unknown')
@@ -87,21 +109,22 @@ def process_documents_with_chunking(documents):
             
         elif doc_type == 'image':
             image_count += 1
-            doc_size = len(doc.text)
-            if doc_size > CHUNK_SIZE:
+            if doc_tokens > CHUNK_SIZE:
                 large_images_count += 1
-                log_message(f"Large image description found: {doc.metadata.get('image_number', 'unknown')} in document {doc.metadata.get('document_id', 'unknown')}, size: {doc_size} characters")
+                log_message(f"Large image description found: {doc.metadata.get('image_number', 'unknown')} in document {doc.metadata.get('document_id', 'unknown')}, size: {doc_tokens} tokens ({doc_chars} characters)")
                 
                 # Chunk large images
                 chunked_docs = chunk_document(doc)
                 all_chunked_docs.extend(chunked_docs)
                 
                 for i, chunk_doc in enumerate(chunked_docs):
+                    chunk_tokens = chunk_doc.metadata.get('chunk_size_tokens', count_tokens(chunk_doc.text))
                     chunk_info.append({
                         'document_id': chunk_doc.metadata.get('document_id', 'unknown'),
                         'section_id': chunk_doc.metadata.get('section_id', 'unknown'),
                         'chunk_id': i,
-                        'chunk_size': len(chunk_doc.text),
+                        'chunk_size_tokens': chunk_tokens,
+                        'chunk_size_chars': len(chunk_doc.text),
                         'chunk_preview': chunk_doc.text[:200] + "..." if len(chunk_doc.text) > 200 else chunk_doc.text,
                         'type': 'image',
                         'image_number': chunk_doc.metadata.get('image_number', 'unknown')
@@ -112,25 +135,27 @@ def process_documents_with_chunking(documents):
                     'document_id': doc.metadata.get('document_id', 'unknown'),
                     'section_id': doc.metadata.get('section_id', 'unknown'),
                     'chunk_id': 0,
-                    'chunk_size': doc_size,
+                    'chunk_size_tokens': doc_tokens,
+                    'chunk_size_chars': doc_chars,
                     'chunk_preview': doc.text[:200] + "..." if len(doc.text) > 200 else doc.text,
                     'type': 'image',
                     'image_number': doc.metadata.get('image_number', 'unknown')
                 })
             
         else:  # text documents
-            doc_size = len(doc.text)
-            if doc_size > CHUNK_SIZE:
+            if doc_tokens > CHUNK_SIZE:
                 chunked_docs = chunk_document(doc)
                 all_chunked_docs.extend(chunked_docs)
                 text_chunks_count += len(chunked_docs)
                 
                 for i, chunk_doc in enumerate(chunked_docs):
+                    chunk_tokens = chunk_doc.metadata.get('chunk_size_tokens', count_tokens(chunk_doc.text))
                     chunk_info.append({
                         'document_id': chunk_doc.metadata.get('document_id', 'unknown'),
                         'section_id': chunk_doc.metadata.get('section_id', 'unknown'),
                         'chunk_id': i,
-                        'chunk_size': len(chunk_doc.text),
+                        'chunk_size_tokens': chunk_tokens,
+                        'chunk_size_chars': len(chunk_doc.text),
                         'chunk_preview': chunk_doc.text[:200] + "..." if len(chunk_doc.text) > 200 else chunk_doc.text,
                         'type': 'text'
                     })
@@ -140,22 +165,24 @@ def process_documents_with_chunking(documents):
                     'document_id': doc.metadata.get('document_id', 'unknown'),
                     'section_id': doc.metadata.get('section_id', 'unknown'),
                     'chunk_id': 0,
-                    'chunk_size': doc_size,
+                    'chunk_size_tokens': doc_tokens,
+                    'chunk_size_chars': doc_chars,
                     'chunk_preview': doc.text[:200] + "..." if len(doc.text) > 200 else doc.text,
                     'type': 'text'
                 })
     
     log_message(f"=== PROCESSING STATISTICS ===")
     log_message(f"Total tables processed: {table_count}")
-    log_message(f"Large tables (>{CHUNK_SIZE} chars): {large_tables_count}")
+    log_message(f"Large tables (>{CHUNK_SIZE} tokens): {large_tables_count}")
     log_message(f"Total images processed: {image_count}")
-    log_message(f"Large images (>{CHUNK_SIZE} chars): {large_images_count}")
+    log_message(f"Large images (>{CHUNK_SIZE} tokens): {large_images_count}")
     log_message(f"Total text chunks created: {text_chunks_count}")
     log_message(f"Total documents after processing: {len(all_chunked_docs)}")
     
     return all_chunked_docs, chunk_info
 
 
+
 def extract_text_from_json(data, document_id, document_name):
     documents = []
     

requirements.txt

@@ -14,4 +14,5 @@ python-docx
 openpyxl
 llama-index-llms-openai
 llama-index-vector-stores-faiss
-llama-index-retrievers-bm25
+llama-index-retrievers-bm25
+tiktoken