new doc id filter + 100 + 100 retrieval

index_retriever.py

@@ -26,12 +26,11 @@ def keyword_filter_nodes(query, nodes, min_keyword_matches=1):
 import re
 
 def extract_doc_id_from_query(query):
-    """Extract document IDs from query text"""
-    # Match patterns like: ГОСТ 59023.2, НП-104, ГОСТ Р 50.04.07-2022
+    """Extract document IDs from query text with better pattern matching"""
     patterns = [
-        r'(?:ГОСТ\s*Р?\s*)[\d\.]+(?:-\d{4})?',  # ГОСТ patterns
-        r'НП-\d+(?:-\d+)?',                       # НП patterns
-        r'МУ[_\s][\d\.]+',                        # МУ patterns
+        r'ГОСТ\s*Р?\s*\d+(?:\.\d+)*(?:-\d{4})?',  # ГОСТ 59023.4, ГОСТ Р 50.05.01-2018
+        r'НП-\d+(?:-\d+)?',                         # НП-104-18
+        r'МУ[_\s]\d+(?:\.\d+)+(?:\.\d+)*(?:-\d{4})?',  # МУ 1.2.3.07.0057-2018
     ]
     
     found_ids = []
@@ -39,33 +38,35 @@ def extract_doc_id_from_query(query):
         matches = re.findall(pattern, query, re.IGNORECASE)
         found_ids.extend(matches)
     
-    # Normalize spacing
-    normalized = [re.sub(r'\s+', ' ', id.strip()) for id in found_ids]
+    # Normalize spacing and preserve dots
+    normalized = [re.sub(r'\s+', ' ', id.strip().upper()) for id in found_ids]
     return normalized
 
-
-def filter_nodes_by_doc_id(nodes, doc_ids, threshold=0.8):
-    """Keep nodes that match any of the document IDs"""
+def filter_nodes_by_doc_id(nodes, doc_ids, threshold=0.85):
+    """Keep nodes that match any of the document IDs with better matching"""
     if not doc_ids:
         return nodes
     
-    from difflib import SequenceMatcher
-    
     filtered = []
     for node in nodes:
-        node_doc_id = node.metadata.get('document_id', '').upper()
+        node_doc_id = node.metadata.get('document_id', '').upper().strip()
+        node_doc_id_normalized = re.sub(r'\s+', ' ', node_doc_id)
         
         for query_doc_id in doc_ids:
-            query_doc_id = query_doc_id.upper()
+            query_doc_id = query_doc_id.upper().strip()
             
-            # Exact substring match
-            if query_doc_id in node_doc_id or node_doc_id in query_doc_id:
-                filtered.append(node)
-                break
+            # Extract base number for comparison (e.g., "59023.4" from "ГОСТ Р 59023.4-2020")
+            node_base = re.search(r'(\d+(?:\.\d+)+)', node_doc_id_normalized)
+            query_base = re.search(r'(\d+(?:\.\d+)+)', query_doc_id)
+            
+            # Match if base numbers are identical
+            if node_base and query_base:
+                if node_base.group(1) == query_base.group(1):
+                    filtered.append(node)
+                    break
             
-            # Fuzzy match for close variants
-            similarity = SequenceMatcher(None, query_doc_id, node_doc_id).ratio()
-            if similarity >= threshold:
+            # Fallback: exact substring match
+            if query_doc_id in node_doc_id_normalized or node_doc_id_normalized in query_doc_id:
                 filtered.append(node)
                 break
     
@@ -92,11 +93,11 @@ def create_query_engine(vector_index):
 
     vector_retriever = VectorIndexRetriever(
         index=vector_index,
-        similarity_top_k=50
+        similarity_top_k=100
     )
     bm25_retriever = BM25Retriever.from_defaults(
         docstore=vector_index.docstore,
-        similarity_top_k=50,
+        similarity_top_k=100,
         tokenizer=russian_tokenizer  # Add custom tokenizer
 
     )