max_chars = 1500 + doc id retriever

documents_prep.py

@@ -38,7 +38,7 @@ def chunk_text_documents(documents):
     return chunked
 
 
-def chunk_table_by_content(table_data, doc_id, max_chars=1200):
+def chunk_table_by_content(table_data, doc_id, max_chars=1500):
     """Chunk tables by content size instead of rows"""
     headers = table_data.get('headers', [])
     rows = table_data.get('data', [])
@@ -222,7 +222,7 @@ def load_table_documents(repo_id, hf_token, table_dir):
             for sheet in data.get('sheets', []):
                 sheet_doc_id = sheet.get('document_id', sheet.get('document', file_doc_id))
                 
-                chunks = chunk_table_by_content(sheet, sheet_doc_id, max_chars=1000)
+                chunks = chunk_table_by_content(sheet, sheet_doc_id, max_chars=1500)
                 all_chunks.extend(chunks)
                 
         except Exception as e:

index_retriever.py

@@ -44,15 +44,15 @@ def create_query_engine(vector_index):
 
     vector_retriever = VectorIndexRetriever(
         index=vector_index,
-        similarity_top_k=50  # Reduced from 50
+        similarity_top_k=60  # Reduced from 50
     )
     bm25_retriever = BM25Retriever.from_defaults(
         docstore=vector_index.docstore,
-        similarity_top_k=50  # Reduced from 50
+        similarity_top_k=60  # Reduced from 50
     )
     hybrid_retriever = QueryFusionRetriever(
         [vector_retriever, bm25_retriever],
-        similarity_top_k=60,  # Reduced from 60
+        similarity_top_k=80,  # Reduced from 60
         num_queries=1
     )
 

utils.py

@@ -37,20 +37,60 @@ def format_sources(nodes):
     
     return "\n".join(set(sources))
 
+import re
+
+def extract_document_id(query):
+    """Extract explicit document IDs from query"""
+    # Patterns for common document formats
+    patterns = [
+        r'ГОСТ\s*Р?\s*[\d.-]+',  # ГОСТ 59023.4, ГОСТ Р 59023.5-2020
+        r'НП-\d+-\d+',            # НП-105-18
+        r'МУ[_\s][\d.]+',         # МУ 1.1.4.01.1422-2019
+    ]
+    
+    for pattern in patterns:
+        match = re.search(pattern, query, re.IGNORECASE)
+        if match:
+            return match.group(0).strip()
+    return None
+
 def answer_question(question, query_engine, reranker):
     try:
         log_message(f"\n{'='*70}")
         log_message(f"QUERY: {question}")
 
-        # Retrieve nodes (already deduplicated)
+        # Check for explicit document reference
+        target_doc_id = extract_document_id(question)
+        if target_doc_id:
+            log_message(f"TARGET DOCUMENT: {target_doc_id}")
+
+        # Retrieve nodes
         retrieved = query_engine.retrieve(question)
         log_message(f"RETRIEVED: {len(retrieved)} unique nodes")
         
+        # Filter by document if explicitly mentioned
+        if target_doc_id:
+            # Normalize for comparison (remove spaces, case-insensitive)
+            target_normalized = target_doc_id.replace(' ', '').lower()
+            
+            filtered = [
+                node for node in retrieved 
+                if target_normalized in node.metadata.get('document_id', '').replace(' ', '').lower()
+            ]
+            
+            log_message(f"FILTERED TO TARGET DOC: {len(filtered)} nodes")
+            
+            if not filtered:
+                log_message(f"WARNING: No nodes found for {target_doc_id}")
+                return f"В базе данных не найдены таблицы из документа {target_doc_id}.", ""
+            
+            retrieved = filtered
+        
         # Rerank
         reranked = rerank_nodes(question, retrieved, reranker, top_k=25, min_score=0.25)  
         log_message(f"RERANKED: {len(reranked)} nodes")
 
-        # Build context - NO TRUNCATION
+        # Rest of your existing code...
         context_parts = []
         for n in reranked:
             meta = n.metadata
@@ -66,7 +106,7 @@ def answer_question(question, query_engine, reranker):
             else:
                 source_label = f"[{doc_id}]"
             
-            context_parts.append(f"{source_label}\n{n.text}")  # Full text
+            context_parts.append(f"{source_label}\n{n.text}")
 
         context = "\n\n" + ("="*50 + "\n\n").join(context_parts)
 
@@ -79,7 +119,7 @@ def answer_question(question, query_engine, reranker):
 
         sources = format_sources(reranked)
         
-        # Log retrieved chunks WITHOUT duplicates
+        # Log retrieved chunks
         log_message(f"\n{'='*70}")
         log_message("RETRIEVED CHUNKS:")
         for i, node in enumerate(reranked, 1):
@@ -99,7 +139,6 @@ def answer_question(question, query_engine, reranker):
         import traceback
         log_message(traceback.format_exc())
         return f"Ошибка: {e}", ""
-    
 
 def rerank_nodes(query, nodes, reranker, top_k=25, min_score=0.3):
     """Simple and effective reranking: sort by score and filter by threshold."""