Much lower reranking threshold (-0.5 instead of 0.1) + detailed score logging

documents_prep.py

@@ -18,12 +18,17 @@ def chunk_text_documents(documents):
     
     chunked = []
     for doc in documents:
+        # Add document ID to text for better BM25 matching
+        doc_id = doc.metadata.get('document_id', '')
+        if doc_id and doc_id not in doc.text[:200]:
+            doc.text = f"[Документ: {doc_id}]\n\n{doc.text}"
+        
         chunks = text_splitter.get_nodes_from_documents([doc])
         for i, chunk in enumerate(chunks):
             chunk.metadata.update({
                 'chunk_id': i,
                 'total_chunks': len(chunks),
-                'chunk_size': len(chunk.text)  # Add chunk size
+                'chunk_size': len(chunk.text)
             })
             chunked.append(chunk)
     

index_retriever.py

@@ -23,21 +23,86 @@ def keyword_filter_nodes(query, nodes, min_keyword_matches=1):
             filtered.append(node)
     return filtered
 
+import re
+
+def extract_doc_id_from_query(query):
+    """Extract document IDs from query text"""
+    # Match patterns like: ГОСТ 59023.2, НП-104, ГОСТ Р 50.04.07-2022
+    patterns = [
+        r'(?:ГОСТ\s*Р?\s*)[\d\.]+(?:-\d{4})?',  # ГОСТ patterns
+        r'НП-\d+(?:-\d+)?',                       # НП patterns
+        r'МУ[_\s][\d\.]+',                        # МУ patterns
+    ]
+    
+    found_ids = []
+    for pattern in patterns:
+        matches = re.findall(pattern, query, re.IGNORECASE)
+        found_ids.extend(matches)
+    
+    # Normalize spacing
+    normalized = [re.sub(r'\s+', ' ', id.strip()) for id in found_ids]
+    return normalized
+
+
+def filter_nodes_by_doc_id(nodes, doc_ids, threshold=0.8):
+    """Keep nodes that match any of the document IDs"""
+    if not doc_ids:
+        return nodes
+    
+    from difflib import SequenceMatcher
+    
+    filtered = []
+    for node in nodes:
+        node_doc_id = node.metadata.get('document_id', '').upper()
+        
+        for query_doc_id in doc_ids:
+            query_doc_id = query_doc_id.upper()
+            
+            # Exact substring match
+            if query_doc_id in node_doc_id or node_doc_id in query_doc_id:
+                filtered.append(node)
+                break
+            
+            # Fuzzy match for close variants
+            similarity = SequenceMatcher(None, query_doc_id, node_doc_id).ratio()
+            if similarity >= threshold:
+                filtered.append(node)
+                break
+    
+    return filtered
+
+def russian_tokenizer(text):
+    """Better tokenizer for Russian document IDs and technical terms"""
+    import re
+    
+    # Keep document ID patterns intact
+    text = re.sub(r'(ГОСТ\s*Р?\s*[\d\.]+(?:-\d{4})?)', r' \1 ', text)
+    text = re.sub(r'(НП-\d+(?:-\d+)?)', r' \1 ', text)
+    text = re.sub(r'(МУ[_\s][\d\.]+)', r' \1 ', text)
+    
+    # Split on whitespace and punctuation, but keep numbers with decimals
+    tokens = re.findall(r'\d+\.\d+|\w+', text.lower())
+    
+    return tokens
+
+
 def create_query_engine(vector_index):
-    """Create hybrid retrieval engine with better deduplication"""
+    """Create hybrid retrieval engine with document ID filtering"""
     log_message("Creating query engine...")
 
     vector_retriever = VectorIndexRetriever(
         index=vector_index,
-        similarity_top_k=50  # Reduced to get more diverse results
+        similarity_top_k=50
     )
     bm25_retriever = BM25Retriever.from_defaults(
         docstore=vector_index.docstore,
         similarity_top_k=50,
+        tokenizer=russian_tokenizer  # Add custom tokenizer
+
     )
     hybrid_retriever = QueryFusionRetriever(
         [vector_retriever, bm25_retriever],
-        similarity_top_k=60,  # Reduced
+        similarity_top_k=60,
         num_queries=1
     )
 
@@ -46,20 +111,33 @@ def create_query_engine(vector_index):
             nodes = hybrid_retriever.retrieve(query)
             log_message(f"Hybrid retrieval returned: {len(nodes)} nodes")
             
-            # Better deduplication using longer text snippet
+            # Extract document IDs from query
+            doc_ids = extract_doc_id_from_query(query)
+            if doc_ids:
+                log_message(f"Detected document IDs in query: {doc_ids}")
+                
+                # Filter by document ID
+                doc_filtered = filter_nodes_by_doc_id(nodes, doc_ids, threshold=0.7)
+                log_message(f"After doc ID filter: {len(doc_filtered)} nodes")
+                
+                # If we found matching documents, use only those
+                if doc_filtered:
+                    nodes = doc_filtered
+                else:
+                    log_message("WARNING: No nodes matched document IDs, using all results")
+            
+            # Deduplication
             seen_hashes = set()
             unique_nodes = []
             doc_type_counts = {'text': 0, 'table': 0, 'image': 0}
             
             for node in nodes:
-                # Use first 500 chars for dedup hash
                 text_hash = hash(node.text[:500])
                 
                 if text_hash not in seen_hashes:
                     seen_hashes.add(text_hash)
                     unique_nodes.append(node)
                     
-                    # Count by type
                     node_type = node.metadata.get('type', 'text')
                     doc_type_counts[node_type] = doc_type_counts.get(node_type, 0) + 1
             
@@ -68,6 +146,10 @@ def create_query_engine(vector_index):
                        f"table={doc_type_counts.get('table', 0)}, "
                        f"image={doc_type_counts.get('image', 0)}")
             
+            # Log which documents we're returning
+            returned_docs = set(n.metadata.get('document_id', 'unknown') for n in unique_nodes[:50])
+            log_message(f"Returning nodes from: {sorted(returned_docs)}")
+            
             return unique_nodes[:50]
 
     response_synthesizer = get_response_synthesizer()
@@ -77,5 +159,5 @@ def create_query_engine(vector_index):
         response_synthesizer=response_synthesizer
     )
 
-    log_message("✓ Query engine created")
+    log_message("✓ Query engine created with doc ID filtering")
     return query_engine