chunk size = 1024 + max chars = 1200 + deduplication variant

documents_prep.py

@@ -7,7 +7,7 @@ from llama_index.core.text_splitter import SentenceSplitter
 from my_logging import log_message
 
 # Configuration
-CHUNK_SIZE = 512
+CHUNK_SIZE = 1024
 CHUNK_OVERLAP = 128
 
 def chunk_text_documents(documents):
@@ -65,15 +65,28 @@ def chunk_table_by_content(table_data, doc_id, max_chars=1200):
     
     log_message(f"  📊 Processing: {doc_id} - {table_identifier} ({len(rows)} rows)")
     
-    # Calculate base metadata size (everything except row data)
-    base_content = format_table_header(doc_id, table_identifier, table_num, table_title, section, headers)
+    # SIMPLIFIED base content - remove redundant search keywords
+    base_content = f"ДОКУМЕНТ: {doc_id}\n"
+    base_content += f"ТАБЛИЦА: {table_identifier}\n"
+    if table_title:
+        base_content += f"НАЗВАНИЕ: {table_title}\n"
+    if section:
+        base_content += f"РАЗДЕЛ: {section}\n"
+    base_content += f"{'='*70}\n\n"
+    
+    if headers:
+        header_str = ' | '.join(str(h) for h in headers)
+        base_content += f"ЗАГОЛОВКИ: {header_str}\n\n"
+    
+    base_content += "ДАННЫЕ:\n"
+    
     base_size = len(base_content)
-    available_space = max_chars - base_size - 200 
+    available_space = max_chars - base_size - 100  # Reduced footer overhead
     
-    # If entire table fits, return as one chunk
+    # Rest of the function stays the same...
     full_rows_content = format_table_rows(rows)
     if base_size + len(full_rows_content) <= max_chars:
-        content = base_content + full_rows_content + format_table_footer(table_identifier, doc_id)
+        content = base_content + full_rows_content
         
         metadata = {
             'type': 'table',
@@ -84,13 +97,15 @@ def chunk_table_by_content(table_data, doc_id, max_chars=1200):
             'section': section,
             'total_rows': len(rows),
             'chunk_size': len(content),
-            'is_complete_table': True
+            'is_complete_table': True,
+            'row_start': 0,
+            'row_end': len(rows)
         }
         
         log_message(f"    Single chunk: {len(content)} chars, {len(rows)} rows")
         return [Document(text=content, metadata=metadata)]
     
-    # Otherwise, chunk by content size
+    # Chunking logic with row indices...
     chunks = []
     current_rows = []
     current_size = 0
@@ -100,11 +115,9 @@ def chunk_table_by_content(table_data, doc_id, max_chars=1200):
         row_text = format_single_row(row, i + 1)
         row_size = len(row_text)
         
-        # If adding this row exceeds limit, save current chunk
         if current_size + row_size > available_space and current_rows:
             content = base_content + format_table_rows(current_rows)
-            content += f"\n\nСтроки {current_rows[0]['_idx']}-{current_rows[-1]['_idx']} из {len(rows)}\n"
-            content += format_table_footer(table_identifier, doc_id)
+            content += f"\n[Строки {current_rows[0]['_idx']}-{current_rows[-1]['_idx']} из {len(rows)}]\n"
             
             metadata = {
                 'type': 'table',
@@ -122,23 +135,20 @@ def chunk_table_by_content(table_data, doc_id, max_chars=1200):
             }
             
             chunks.append(Document(text=content, metadata=metadata))
-            log_message(f"    Chunk {chunk_num + 1}: {len(content)} chars, {len(current_rows)} rows")
+            log_message(f"    Chunk {chunk_num + 1}: {len(content)} chars, rows {current_rows[0]['_idx']}-{current_rows[-1]['_idx']}")
             
             chunk_num += 1
             current_rows = []
             current_size = 0
         
-        # Add row index for tracking
         row_copy = row.copy() if isinstance(row, dict) else {'data': row}
         row_copy['_idx'] = i + 1
         current_rows.append(row_copy)
         current_size += row_size
     
-    # Add final chunk if rows remain
     if current_rows:
         content = base_content + format_table_rows(current_rows)
-        content += f"\n\nСтроки {current_rows[0]['_idx']}-{current_rows[-1]['_idx']} из {len(rows)}\n"
-        content += format_table_footer(table_identifier, doc_id)
+        content += f"\n[Строки {current_rows[0]['_idx']}-{current_rows[-1]['_idx']} из {len(rows)}]\n"
         
         metadata = {
             'type': 'table',
@@ -156,45 +166,11 @@ def chunk_table_by_content(table_data, doc_id, max_chars=1200):
         }
         
         chunks.append(Document(text=content, metadata=metadata))
-        log_message(f"    Chunk {chunk_num + 1}: {len(content)} chars, {len(current_rows)} rows")
+        log_message(f"    Chunk {chunk_num + 1}: {len(content)} chars, rows {current_rows[0]['_idx']}-{current_rows[-1]['_idx']}")
     
     return chunks
 
 
-def format_table_header(doc_id, table_identifier, table_num, table_title, section, headers):
-    """Format consistent table header"""
-    content = f"ДОКУМЕНТ: {doc_id}\n"
-    content += f"ТАБЛИЦА: {table_identifier}\n"
-    content += f"ПОЛНОЕ НАЗВАНИЕ: {table_identifier}\n"
-    content += f"НОМЕР ТАБЛИЦЫ: {table_num}\n"
-    if table_title:
-        content += f"НАЗВАНИЕ: {table_title}\n"
-    if section:
-        content += f"РАЗДЕЛ: {section}\n"
-    content += f"{'='*70}\n\n"
-    
-    # Enhanced search keywords
-    content += f"Это таблица {table_identifier} из документа {doc_id}. "
-    content += f"Идентификатор: {table_identifier}. Номер: {table_num}. Документ: {doc_id}. "
-    
-    if section:
-        content += f"Раздел: {section}. "
-        if 'приложени' in section.lower():
-            content += f"Таблица из приложения. "
-    
-    if table_title:
-        content += f"Название: {table_title}. "
-    
-    content += f"\n\nСОДЕРЖИМОЕ ТАБЛИЦЫ {table_identifier}:\n{'='*70}\n\n"
-    
-    if headers:
-        header_str = ' | '.join(str(h) for h in headers)
-        content += f"ЗАГОЛОВКИ: {header_str}\n\n"
-    
-    content += "ДАННЫЕ:\n"
-    return content
-
-
 def format_single_row(row, idx):
     """Format a single row"""
     if isinstance(row, dict):

index_retriever.py

@@ -39,43 +39,48 @@ def keyword_filter_nodes(query, nodes, min_keyword_matches=1):
     return filtered
 
 def create_query_engine(vector_index):
-    """Create hybrid retrieval engine with keyword boost"""
+    """Create hybrid retrieval engine with deduplication"""
     log_message("Creating query engine...")
 
     vector_retriever = VectorIndexRetriever(
         index=vector_index,
-        similarity_top_k=50
+        similarity_top_k=40  # Reduced from 50
     )
     bm25_retriever = BM25Retriever.from_defaults(
         docstore=vector_index.docstore,
-        similarity_top_k=50
+        similarity_top_k=40  # Reduced from 50
     )
     hybrid_retriever = QueryFusionRetriever(
         [vector_retriever, bm25_retriever],
-        similarity_top_k=60,
+        similarity_top_k=50,  # Reduced from 60
         num_queries=1
     )
-    response_synthesizer = get_response_synthesizer()
 
-    class KeywordBoostQueryEngine(RetrieverQueryEngine):
+    class DeduplicatedQueryEngine(RetrieverQueryEngine):
         def retrieve(self, query):
-            # Hybrid results
-            hybrid_nodes = hybrid_retriever.retrieve(query)
-            # Keyword filter from all indexed nodes
-            all_nodes = list(vector_index.docstore.values())
-            keyword_nodes = keyword_filter_nodes(query, all_nodes)
-            # Combine and deduplicate
-            all_candidates = {id(n): n for n in hybrid_nodes + keyword_nodes}
-            log_message(f"Hybrid: {len(hybrid_nodes)}, Keyword: {len(keyword_nodes)}, Total: {len(all_candidates)}")
-            return list(all_candidates.values())[:60]
-        def query(self, prompt):
-            nodes = self.retrieve(prompt)
-            return response_synthesizer.synthesize(prompt, nodes)
+            nodes = hybrid_retriever.retrieve(query)
+            
+            # CRITICAL: Deduplicate by text content hash
+            seen_hashes = set()
+            unique_nodes = []
+            
+            for node in nodes:
+                # Create hash from first 200 chars to detect duplicates
+                text_hash = hash(node.text[:200])
+                
+                if text_hash not in seen_hashes:
+                    seen_hashes.add(text_hash)
+                    unique_nodes.append(node)
+            
+            log_message(f"Retrieved: {len(nodes)} → Unique: {len(unique_nodes)}")
+            return unique_nodes[:50]  # Return top 50 unique
 
-    query_engine = KeywordBoostQueryEngine(
+    response_synthesizer = get_response_synthesizer()
+    
+    query_engine = DeduplicatedQueryEngine(
         retriever=hybrid_retriever,
         response_synthesizer=response_synthesizer
     )
 
-    log_message("✓ Query engine created (with keyword boost)")
+    log_message("✓ Query engine created (with deduplication)")
     return query_engine

utils.py

@@ -42,18 +42,21 @@ def answer_question(question, query_engine, reranker):
         log_message(f"\n{'='*70}")
         log_message(f"QUERY: {question}")
 
-        # Retrieve and rerank nodes
-        retrieved = query_engine.retriever.retrieve(question)
-        log_message(f"\nRETRIEVED: {len(retrieved)} nodes")
-        reranked = rerank_nodes(question, retrieved, reranker, top_k=25, min_score=0.3)
-        log_message(f"\nRERANKED: {len(reranked)} nodes")
+        # Retrieve nodes (already deduplicated)
+        retrieved = query_engine.retrieve(question)
+        log_message(f"RETRIEVED: {len(retrieved)} unique nodes")
+        
+        # Rerank
+        reranked = rerank_nodes(question, retrieved, reranker, top_k=15, min_score=0.25)  # Reduced top_k
+        log_message(f"RERANKED: {len(reranked)} nodes")
 
-        # Build context for prompt
+        # Build context - NO TRUNCATION
         context_parts = []
         for n in reranked:
             meta = n.metadata
             doc_id = meta.get('document_id', 'unknown')
             doc_type = meta.get('type', 'text')
+            
             if doc_type == 'table':
                 table_id = meta.get('table_identifier', meta.get('table_number', 'unknown'))
                 title = meta.get('table_title', '')
@@ -62,31 +65,32 @@ def answer_question(question, query_engine, reranker):
                     source_label += f" {title}"
             else:
                 source_label = f"[{doc_id}]"
-            context_parts.append(f"{source_label}\n{n.text}")  # Use FULL text, not [:500]
+            
+            context_parts.append(f"{source_label}\n{n.text}")  # Full text
 
         context = "\n\n" + ("="*50 + "\n\n").join(context_parts)
 
-        # Use CUSTOM_PROMPT from config
         from config import CUSTOM_PROMPT
         prompt = CUSTOM_PROMPT.format(context_str=context, query_str=question)
-        log_message(f"\nPROMPT LENGTH: {len(prompt)} chars\n")
+        log_message(f"PROMPT LENGTH: {len(prompt)} chars")
         
-        # CRITICAL FIX: Call LLM directly instead of query_engine.query()
         from llama_index.core import Settings
         response = Settings.llm.complete(prompt)
 
         sources = format_sources(reranked)
         
-        # Log retrieved chunks
+        # Log retrieved chunks WITHOUT duplicates
         log_message(f"\n{'='*70}")
         log_message("RETRIEVED CHUNKS:")
         for i, node in enumerate(reranked, 1):
             log_message(f"\n--- Chunk {i} ---")
-            log_message(f"Document: {node.metadata.get('document_id', 'unknown')}")
-            log_message(f"Type: {node.metadata.get('type', 'unknown')}")
+            log_message(f"Document: {node.metadata.get('document_id')}")
+            log_message(f"Type: {node.metadata.get('type')}")
             if node.metadata.get('type') == 'table':
-                log_message(f"Table: {node.metadata.get('table_identifier', 'unknown')}")
-            log_message(f"Text preview: {node.text[:500]}...")
+                table_id = node.metadata.get('table_identifier')
+                rows = f"{node.metadata.get('row_start', 0)}-{node.metadata.get('row_end', 0)}"
+                log_message(f"Table: {table_id} (rows {rows})")
+            log_message(f"Text: {node.text[:300]}...")
         
         return response.text, sources