index retriever = 100 + 100

index_retriever.py

@@ -44,15 +44,15 @@ def create_query_engine(vector_index):
 
     vector_retriever = VectorIndexRetriever(
         index=vector_index,
-        similarity_top_k=60  # Reduced from 50
+        similarity_top_k=80  # Reduced from 50
     )
     bm25_retriever = BM25Retriever.from_defaults(
         docstore=vector_index.docstore,
-        similarity_top_k=60  # Reduced from 50
+        similarity_top_k=80  # Reduced from 50
     )
     hybrid_retriever = QueryFusionRetriever(
         [vector_retriever, bm25_retriever],
-        similarity_top_k=80,  # Reduced from 60
+        similarity_top_k=100,  # Reduced from 60
         num_queries=1
     )
 
@@ -72,7 +72,7 @@ def create_query_engine(vector_index):
                     unique_nodes.append(node)
             
             log_message(f"Retrieved: {len(nodes)} → Unique: {len(unique_nodes)}")
-            return unique_nodes[:50]  # Return top 50 unique
+            return unique_nodes[:60]  # Return top 50 unique
 
     response_synthesizer = get_response_synthesizer()
     

utils.py

@@ -41,11 +41,10 @@ import re
 
 def extract_document_id(query):
     """Extract explicit document IDs from query"""
-    # Patterns for common document formats
     patterns = [
-        r'ГОСТ\s*Р?\s*[\d.-]+',  # ГОСТ 59023.4, ГОСТ Р 59023.5-2020
-        r'НП-\d+-\d+',            # НП-105-18
-        r'МУ[_\s][\d.]+',         # МУ 1.1.4.01.1422-2019
+        r'ГОСТ\s*Р?\s*[\d.-]+(?:-\d{4})?',  # ГОСТ 59023.4 or ГОСТ Р 59023.5-2020
+        r'НП-\d+-\d+',
+        r'МУ[_\s][\d.]+',
     ]
     
     for pattern in patterns:
@@ -54,43 +53,51 @@ def extract_document_id(query):
             return match.group(0).strip()
     return None
 
+def normalize_doc_id(doc_id):
+    """Normalize document ID for flexible matching"""
+    normalized = doc_id.replace(' ', '').replace('Р', '').replace('р', '').lower()
+    # Remove year suffix for comparison (e.g., -2020)
+    normalized = re.sub(r'-\d{4}$', '', normalized)
+    return normalized
+
 def answer_question(question, query_engine, reranker):
     try:
         log_message(f"\n{'='*70}")
         log_message(f"QUERY: {question}")
 
-        # Check for explicit document reference
         target_doc_id = extract_document_id(question)
         if target_doc_id:
             log_message(f"TARGET DOCUMENT: {target_doc_id}")
 
-        # Retrieve nodes
         retrieved = query_engine.retrieve(question)
         log_message(f"RETRIEVED: {len(retrieved)} unique nodes")
         
-        # Filter by document if explicitly mentioned
         if target_doc_id:
-            # Normalize for comparison (remove spaces, case-insensitive)
-            target_normalized = target_doc_id.replace(' ', '').lower()
+            target_normalized = normalize_doc_id(target_doc_id)
+            log_message(f"NORMALIZED TARGET: {target_normalized}")
             
             filtered = [
                 node for node in retrieved 
-                if target_normalized in node.metadata.get('document_id', '').replace(' ', '').lower()
+                if target_normalized in normalize_doc_id(node.metadata.get('document_id', ''))
             ]
             
             log_message(f"FILTERED TO TARGET DOC: {len(filtered)} nodes")
             
+            # Debug: show what document IDs were found
+            if not filtered and len(retrieved) > 0:
+                found_docs = set(node.metadata.get('document_id', 'unknown') for node in retrieved[:10])
+                log_message(f"AVAILABLE DOCS (sample): {', '.join(list(found_docs)[:5])}")
+            
             if not filtered:
                 log_message(f"WARNING: No nodes found for {target_doc_id}")
                 return f"В базе данных не найдены таблицы из документа {target_doc_id}.", ""
             
             retrieved = filtered
         
-        # Rerank
+        # Rest stays the same...
         reranked = rerank_nodes(question, retrieved, reranker, top_k=25, min_score=0.25)  
         log_message(f"RERANKED: {len(reranked)} nodes")
 
-        # Rest of your existing code...
         context_parts = []
         for n in reranked:
             meta = n.metadata
@@ -119,7 +126,6 @@ def answer_question(question, query_engine, reranker):
 
         sources = format_sources(reranked)
         
-        # Log retrieved chunks
         log_message(f"\n{'='*70}")
         log_message("RETRIEVED CHUNKS:")
         for i, node in enumerate(reranked, 1):