new keyword score based index retriever + answer question

index_retriever.py

@@ -1,8 +1,25 @@
 from llama_index.core import VectorStoreIndex
+from llama_index.core.query_engine import RetrieverQueryEngine
 from llama_index.core.retrievers import VectorIndexRetriever
 from llama_index.retrievers.bm25 import BM25Retriever
+from llama_index.core.retrievers import QueryFusionRetriever
+from llama_index.core.response_synthesizers import get_response_synthesizer
 from my_logging import log_message
-import re
+
+SIMPLE_PROMPT = """Вы - эксперт по нормативной документации.
+
+Контекст:
+{context_str}
+
+Вопрос: {query_str}
+
+Инструкция:
+1. Отвечайте ТОЛЬКО на основе предоставленного контекста
+2. Цитируйте конкретные источники (документ, раздел, таблицу)
+3. Если информации недостаточно, четко укажите это
+4. Будьте точны и конкретны
+
+Ответ:"""
 
 def create_vector_index(documents):
     """Create vector index from documents"""
@@ -11,151 +28,58 @@ def create_vector_index(documents):
     log_message("✓ Index created")
     return index
 
-
-def extract_keywords(query):
-    stopwords = {
-        'какой', 'какие', 'каком', 'какая', 'где', 'когда', 'как', 'что', 'чем',
-        'для', 'при', 'или', 'это', 'есть', 'быть', 'мочь', 'должен', 'нужно',
-        'можно', 'требуется', 'необходимо', 'я', 'мы', 'вы', 'он', 'она', 'они',
-        'в', 'на', 'с', 'по', 'из', 'к', 'о', 'об', 'и', 'а', 'но', 'за', 'до', 'от'
-    }
-    
-    words = re.findall(r'\b[\wа-яА-Я0-9]+\b', query.lower())
-    
-    # Filter keywords
-    keywords = []
-    for word in words:
-        if (word not in stopwords and len(word) > 2) or any(c.isdigit() for c in word):
-            keywords.append(word)
-    
-    # Also extract exact phrases with hyphens/caps (e.g., "08Х18Н10Т", "С-25")
-    exact_matches = re.findall(r'\b[А-ЯA-Z0-9][а-яА-Яa-zA-Z0-9\-]*\b', query)
-    keywords.extend([m.lower() for m in exact_matches if len(m) > 2])
-    
-    log_message(f"Keywords extracted: {set(keywords)}")
-    return list(set(keywords))
-
-
-def calculate_keyword_score(text, keywords):
-    """Calculate keyword match score for a text chunk"""
-    text_lower = text.lower()
-    score = 0
-    
-    for keyword in keywords:
-        # Exact match (case-insensitive)
-        count = text_lower.count(keyword.lower())
-        if count > 0:
-            # Higher weight for longer keywords (likely more specific)
-            weight = len(keyword) / 5.0
-            score += count * weight
-    
-    return score
-
-
-def deduplicate_nodes(nodes):
-    """Remove duplicate nodes based on text content"""
-    seen_hashes = set()
-    unique_nodes = []
-    
+def keyword_filter_nodes(query, nodes, min_keyword_matches=1):
+    """Return nodes that contain at least one keyword from the query."""
+    keywords = [w.lower() for w in query.split() if len(w) > 2]
+    filtered = []
     for node in nodes:
-        # Use first 200 chars as fingerprint
-        text_hash = hash(node.text[:200])
-        
-        if text_hash not in seen_hashes:
-            seen_hashes.add(text_hash)
-            unique_nodes.append(node)
-    
-    return unique_nodes
+        text = node.text.lower()
+        if any(k in text for k in keywords):
+            filtered.append(node)
+    return filtered
 
+def create_query_engine(vector_index):
+    """Create hybrid retrieval engine with deduplication"""
+    log_message("Creating query engine...")
 
-def hybrid_retrieve(query, vector_index, top_k=50):
-    """Hybrid retrieval: vector + BM25 + keyword boosting"""
-    
-    # 1. Vector retrieval
     vector_retriever = VectorIndexRetriever(
         index=vector_index,
-        similarity_top_k=top_k
+        similarity_top_k=50  # Reduced from 50
     )
-    vector_nodes = vector_retriever.retrieve(query)
-    
-    # 2. BM25 retrieval
     bm25_retriever = BM25Retriever.from_defaults(
         docstore=vector_index.docstore,
-        similarity_top_k=top_k
+        similarity_top_k=50  # Reduced from 50
+    )
+    hybrid_retriever = QueryFusionRetriever(
+        [vector_retriever, bm25_retriever],
+        similarity_top_k=60,  # Reduced from 60
+        num_queries=1
     )
-    bm25_nodes = bm25_retriever.retrieve(query)
-    
-    # 3. Combine and deduplicate
-    all_nodes = vector_nodes + bm25_nodes
-    unique_nodes = deduplicate_nodes(all_nodes)
-    
-    log_message(f"Vector: {len(vector_nodes)}, BM25: {len(bm25_nodes)}, Unique: {len(unique_nodes)}")
-    
-    # 4. Extract keywords
-    keywords = extract_keywords(query)
-    
-    # 5. Add keyword scores
-    scored_nodes = []
-    for node in unique_nodes:
-        keyword_score = calculate_keyword_score(node.text, keywords)
-        
-        # Combine with original similarity score
-        original_score = node.score if hasattr(node, 'score') and node.score else 0.5
-        
-        # Boost formula: original score + keyword bonus (capped at 0.3)
-        keyword_boost = min(keyword_score * 0.1, 0.3)
-        combined_score = original_score + keyword_boost
-        
-        scored_nodes.append((node, combined_score, keyword_score))
-    
-    # 6. Sort by combined score
-    scored_nodes.sort(key=lambda x: x[1], reverse=True)
-    
-    # Log top scores
-    log_message("\nTop 10 scores after keyword boosting:")
-    for i, (node, combined, kw_score) in enumerate(scored_nodes[:10], 1):
-        doc_id = node.metadata.get('document_id', '?')
-        node_type = node.metadata.get('type', '?')
-        log_message(f"  {i}. [{doc_id}] {node_type} - Score: {combined:.3f} (kw: {kw_score:.2f})")
-    
-    # Return nodes only (without scores)
-    return [node for node, _, _ in scored_nodes[:top_k]]
 
+    class DeduplicatedQueryEngine(RetrieverQueryEngine):
+        def retrieve(self, query):
+            nodes = hybrid_retriever.retrieve(query)
+            
+            # CRITICAL: Deduplicate by text content hash
+            seen_hashes = set()
+            unique_nodes = []
+            
+            for node in nodes:
+                text_hash = hash(node.text[:200])
+                
+                if text_hash not in seen_hashes:
+                    seen_hashes.add(text_hash)
+                    unique_nodes.append(node)
+            
+            log_message(f"Retrieved: {len(nodes)} → Unique: {len(unique_nodes)}")
+            return unique_nodes[:50]  # Return top 50 unique
 
-def keyword_retrieve_fallback(query, vector_index, keywords, top_k=20):
-    """Fallback: direct keyword search in all documents"""
-    all_nodes = list(vector_index.docstore.docs.values())
-    
-    scored = []
-    for node in all_nodes:
-        score = calculate_keyword_score(node.text, keywords)
-        if score > 0:
-            scored.append((node, score))
-    
-    scored.sort(key=lambda x: x[1], reverse=True)
+    response_synthesizer = get_response_synthesizer()
     
-    if scored:
-        log_message(f"\nKeyword fallback found {len(scored)} matches")
-        log_message(f"Top scores: {[s for _, s in scored[:5]]}")
-    
-    return [node for node, _ in scored[:top_k]]
-
+    query_engine = DeduplicatedQueryEngine(
+        retriever=hybrid_retriever,
+        response_synthesizer=response_synthesizer
+    )
 
-def create_query_engine(vector_index):
-    
-    def retrieve(query):
-        nodes = hybrid_retrieve(query, vector_index, top_k=60)
-        
-        # Fallback: If too few results, add pure keyword matches
-        keywords = extract_keywords(query)
-        if len(nodes) < 20 and keywords:
-            log_message("\n⚠ Adding keyword fallback results...")
-            fallback_nodes = keyword_retrieve_fallback(query, vector_index, keywords, top_k=30)
-            nodes.extend(fallback_nodes)
-            nodes = deduplicate_nodes(nodes)
-        
-        log_message(f"\nFinal retrieval: {len(nodes)} nodes")
-        return nodes[:50]  # Cap at 50
-    
-    log_message("✓ Query engine created (hybrid + keyword boost)")
-    return retrieve
+    log_message("✓ Query engine created (with deduplication)")
+    return query_engine

utils.py

@@ -37,23 +37,20 @@ def format_sources(nodes):
     
     return "\n".join(set(sources))
 
-def answer_question(question, retrieve_func, reranker):
+def answer_question(question, query_engine, reranker):
     try:
         log_message(f"\n{'='*70}")
         log_message(f"QUERY: {question}")
 
-        # Retrieve with keyword boosting
-        retrieved = retrieve_func(question)
-        log_message(f"RETRIEVED: {len(retrieved)} nodes")
+        # Retrieve nodes (already deduplicated)
+        retrieved = query_engine.retrieve(question)
+        log_message(f"RETRIEVED: {len(retrieved)} unique nodes")
         
-        # Rerank (optional - уже есть keyword boost)
-        if reranker:
-            reranked = rerank_nodes(question, retrieved, reranker, top_k=25, min_score=0.2)
-            log_message(f"RERANKED: {len(reranked)} nodes")
-        else:
-            reranked = retrieved[:25]
-        
-        # Build context
+        # Rerank
+        reranked = rerank_nodes(question, retrieved, reranker, top_k=25, min_score=0.25)  
+        log_message(f"RERANKED: {len(reranked)} nodes")
+
+        # Build context - NO TRUNCATION
         context_parts = []
         for n in reranked:
             meta = n.metadata
@@ -69,7 +66,7 @@ def answer_question(question, retrieve_func, reranker):
             else:
                 source_label = f"[{doc_id}]"
             
-            context_parts.append(f"{source_label}\n{n.text}")
+            context_parts.append(f"{source_label}\n{n.text}")  # Full text
 
         context = "\n\n" + ("="*50 + "\n\n").join(context_parts)
 
@@ -82,17 +79,18 @@ def answer_question(question, retrieve_func, reranker):
 
         sources = format_sources(reranked)
         
-        # Detailed logging
+        # Log retrieved chunks WITHOUT duplicates
         log_message(f"\n{'='*70}")
-        log_message("FINAL CHUNKS:")
+        log_message("RETRIEVED CHUNKS:")
         for i, node in enumerate(reranked, 1):
             log_message(f"\n--- Chunk {i} ---")
-            log_message(f"Doc: {node.metadata.get('document_id')}")
+            log_message(f"Document: {node.metadata.get('document_id')}")
             log_message(f"Type: {node.metadata.get('type')}")
             if node.metadata.get('type') == 'table':
                 table_id = node.metadata.get('table_identifier')
-                log_message(f"Table: {table_id}")
-            log_message(f"Preview: {node.text[:400]}...")
+                rows = f"{node.metadata.get('row_start', 0)}-{node.metadata.get('row_end', 0)}"
+                log_message(f"Table: {table_id} (rows {rows})")
+            log_message(f"Text: {node.text[:300]}...")
         
         return response.text, sources