top k reranker = 20, max rows = 10, max chars= 2000 + new deduplication

documents_prep.py

@@ -38,7 +38,7 @@ def chunk_text_documents(documents):
     return chunked
 
 
-def chunk_table_by_rows(table_data, doc_id, rows_per_chunk=10, max_chars=4000):
+def chunk_table_by_rows(table_data, doc_id, rows_per_chunk=10, max_chars=2000):
     """
     Chunk tables by rows with fallback to character limit.
     Keeps 3-4 rows together, but splits individual rows if they're too large.

index_retriever.py

@@ -6,6 +6,12 @@ from llama_index.core.retrievers import QueryFusionRetriever
 from llama_index.core.response_synthesizers import get_response_synthesizer
 from my_logging import log_message
 
+import re
+
+import re
+from difflib import SequenceMatcher
+
+
 def create_vector_index(documents):
     """Create vector index from documents"""
     log_message(f"Building vector index from {len(documents)} documents...")
@@ -23,21 +29,96 @@ def keyword_filter_nodes(query, nodes, min_keyword_matches=1):
             filtered.append(node)
     return filtered
 
+
+def normalize_doc_id(doc_id: str) -> str:
+    """Normalize document ID - KEEP dots for numeric parts"""
+    doc_id = doc_id.upper().strip()
+    doc_id = re.sub(r'\s+', '', doc_id)  # Remove spaces only
+    doc_id = doc_id.replace("ГОСТР", "ГОСТ")
+    doc_id = doc_id.replace("GOSTR", "ГОСТ")
+    return doc_id
+
+def base_number(doc_id: str) -> str:
+    """Extract full numeric pattern including all parts (e.g., '59023.6' from 'ГОСТ 59023.6')"""
+    # Match: 59023.6 or 59023.4 or 50.05.01 etc.
+    m = re.search(r'(\d+(?:\.\d+)*)', doc_id)
+    return m.group(1) if m else ""
+
+def filter_nodes_by_doc_id(nodes, doc_ids, threshold=0.85):
+    """Filter nodes by document ID with strict numeric matching"""
+    if not doc_ids:
+        return nodes
+
+    filtered = []
+    doc_ids_norm = [normalize_doc_id(d) for d in doc_ids]
+    doc_ids_base = [base_number(d) for d in doc_ids_norm]
+
+    for node in nodes:
+        node_doc_id = normalize_doc_id(node.metadata.get('document_id', ''))
+        node_base = base_number(node_doc_id)
+
+        for q_doc, q_base in zip(doc_ids_norm, doc_ids_base):
+            # STRICT: base number must match exactly
+            if q_base and node_base and q_base == node_base:
+                filtered.append(node)
+                break
+            
+            # STRICT: full normalized ID must match exactly or have very high similarity
+            elif SequenceMatcher(None, node_doc_id, q_doc).ratio() >= threshold:
+                filtered.append(node)
+                break
+
+    return filtered if filtered else nodes
+
+
+def extract_doc_id_from_query(query):
+    """Extract document IDs from query text with better pattern matching"""
+    patterns = [
+        r'ГОСТ\s*Р?\s*\d+(?:\.\d+)*(?:-\d{4})?',  # ГОСТ 59023.4, ГОСТ Р 50.05.01-2018
+        r'НП-\d+(?:-\d+)?',                         # НП-104-18
+        r'МУ[_\s]\d+(?:\.\d+)+(?:\.\d+)*(?:-\d{4})?',  # МУ 1.2.3.07.0057-2018
+    ]
+    
+    found_ids = []
+    for pattern in patterns:
+        matches = re.findall(pattern, query, re.IGNORECASE)
+        found_ids.extend(matches)
+    
+    # Normalize spacing and preserve dots
+    normalized = [re.sub(r'\s+', ' ', id.strip().upper()) for id in found_ids]
+    return normalized
+def russian_tokenizer(text):
+    """Better tokenizer for Russian document IDs and technical terms"""
+    import re
+    
+    # Keep document ID patterns intact
+    text = re.sub(r'(ГОСТ\s*Р?\s*[\d\.]+(?:-\d{4})?)', r' \1 ', text)
+    text = re.sub(r'(НП-\d+(?:-\d+)?)', r' \1 ', text)
+    text = re.sub(r'(МУ[_\s][\d\.]+)', r' \1 ', text)
+    
+    # Split on whitespace and punctuation, but keep numbers with decimals
+    tokens = re.findall(r'\d+\.\d+|\w+', text.lower())
+    
+    return tokens
+
+
 def create_query_engine(vector_index):
-    """Create hybrid retrieval engine with better deduplication"""
+    """Create hybrid retrieval engine with document ID filtering"""
     log_message("Creating query engine...")
 
     vector_retriever = VectorIndexRetriever(
         index=vector_index,
-        similarity_top_k=50  # Reduced to get more diverse results
+        similarity_top_k=100
     )
     bm25_retriever = BM25Retriever.from_defaults(
         docstore=vector_index.docstore,
-        similarity_top_k=50,
+        similarity_top_k=100,
+        tokenizer=russian_tokenizer  # Add custom tokenizer
+
     )
     hybrid_retriever = QueryFusionRetriever(
         [vector_retriever, bm25_retriever],
-        similarity_top_k=60,  # Reduced
+        similarity_top_k=60,
         num_queries=1
     )
 
@@ -46,20 +127,28 @@ def create_query_engine(vector_index):
             nodes = hybrid_retriever.retrieve(query)
             log_message(f"Hybrid retrieval returned: {len(nodes)} nodes")
             
-            # Better deduplication using longer text snippet
+            # Extract document IDs from query
+            doc_ids = extract_doc_id_from_query(query)
+            if doc_ids:
+                log_message(f"Detected document IDs in query: {doc_ids}")
+                before = len(nodes)
+                nodes = filter_nodes_by_doc_id(nodes, doc_ids)
+                after = len(nodes)
+                log_message(f"Filtered by doc ID: {after}/{before} nodes kept (fallback safe)")
+
+            
+            # Deduplication
             seen_hashes = set()
             unique_nodes = []
             doc_type_counts = {'text': 0, 'table': 0, 'image': 0}
             
             for node in nodes:
-                # Use first 500 chars for dedup hash
                 text_hash = hash(node.text[:500])
                 
                 if text_hash not in seen_hashes:
                     seen_hashes.add(text_hash)
                     unique_nodes.append(node)
                     
-                    # Count by type
                     node_type = node.metadata.get('type', 'text')
                     doc_type_counts[node_type] = doc_type_counts.get(node_type, 0) + 1
             
@@ -68,6 +157,10 @@ def create_query_engine(vector_index):
                        f"table={doc_type_counts.get('table', 0)}, "
                        f"image={doc_type_counts.get('image', 0)}")
             
+            # Log which documents we're returning
+            returned_docs = set(n.metadata.get('document_id', 'unknown') for n in unique_nodes[:50])
+            log_message(f"Returning nodes from: {sorted(returned_docs)}")
+            
             return unique_nodes[:50]
 
     response_synthesizer = get_response_synthesizer()
@@ -77,5 +170,5 @@ def create_query_engine(vector_index):
         response_synthesizer=response_synthesizer
     )
 
-    log_message("✓ Query engine created")
+    log_message("✓ Query engine created with doc ID filtering")
     return query_engine

utils.py

@@ -49,6 +49,7 @@ def answer_question(question, query_engine, reranker):
 
         reranked = rerank_nodes(question, retrieved, reranker, top_k=20, min_score=-0.5)
         log_message(f"RERANKED: {len(reranked)} nodes")
+        
 
         # Group by document and type
         doc_groups = {}
@@ -81,12 +82,14 @@ def answer_question(question, query_engine, reranker):
                     title = meta.get('table_title', '')
                     doc_section.append(f"\n[Таблица {table_id}] {title}")
                     doc_section.append(n.text[:1500])  # Limit length
+                    log_message(f"  Included table {table_id} from {doc_id}")
             
             # Then text
             if groups['text']:
                 doc_section.append("\n--- ТЕКСТ ---")
                 for n in groups['text'][:3]:  # Limit text chunks
                     doc_section.append(n.text[:800])
+                    log_message(f"  Included text section from {doc_id}")
             
             context_parts.append("\n".join(doc_section))
 
@@ -109,7 +112,7 @@ def answer_question(question, query_engine, reranker):
         log_message(traceback.format_exc())
         return f"Ошибка: {e}", ""
 
-def rerank_nodes(query, nodes, reranker, top_k=20, min_score=-0.5):  # Much lower threshold
+def rerank_nodes(query, nodes, reranker, top_k=20, min_score=0.1):  # Much lower threshold
     """Rerank with detailed score logging"""
     if not nodes or not reranker:
         log_message("WARNING: No nodes or reranker available")