new keyword score based index retriever + answer question

index_retriever.py

@@ -1,25 +1,8 @@
 from llama_index.core import VectorStoreIndex
-from llama_index.core.query_engine import RetrieverQueryEngine
 from llama_index.core.retrievers import VectorIndexRetriever
 from llama_index.retrievers.bm25 import BM25Retriever
-from llama_index.core.retrievers import QueryFusionRetriever
-from llama_index.core.response_synthesizers import get_response_synthesizer
 from my_logging import log_message
-
-SIMPLE_PROMPT = """Вы - эксперт по нормативной документации.
-
-Контекст:
-{context_str}
-
-Вопрос: {query_str}
-
-Инструкция:
-1. Отвечайте ТОЛЬКО на основе предоставленного контекста
-2. Цитируйте конкретные источники (документ, раздел, таблицу)
-3. Если информации недостаточно, четко укажите это
-4. Будьте точны и конкретны
-
-Ответ:"""
+import re
 
 def create_vector_index(documents):
     """Create vector index from documents"""
@@ -28,59 +11,151 @@ def create_vector_index(documents):
     log_message("✓ Index created")
     return index
 
-def keyword_filter_nodes(query, nodes, min_keyword_matches=1):
-    """Return nodes that contain at least one keyword from the query."""
-    keywords = [w.lower() for w in query.split() if len(w) > 2]
-    filtered = []
+
+def extract_keywords(query):
+    stopwords = {
+        'какой', 'какие', 'каком', 'какая', 'где', 'когда', 'как', 'что', 'чем',
+        'для', 'при', 'или', 'это', 'есть', 'быть', 'мочь', 'должен', 'нужно',
+        'можно', 'требуется', 'необходимо', 'я', 'мы', 'вы', 'он', 'она', 'они',
+        'в', 'на', 'с', 'по', 'из', 'к', 'о', 'об', 'и', 'а', 'но', 'за', 'до', 'от'
+    }
+    
+    words = re.findall(r'\b[\wа-яА-Я0-9]+\b', query.lower())
+    
+    # Filter keywords
+    keywords = []
+    for word in words:
+        if (word not in stopwords and len(word) > 2) or any(c.isdigit() for c in word):
+            keywords.append(word)
+    
+    # Also extract exact phrases with hyphens/caps (e.g., "08Х18Н10Т", "С-25")
+    exact_matches = re.findall(r'\b[А-ЯA-Z0-9][а-яА-Яa-zA-Z0-9\-]*\b', query)
+    keywords.extend([m.lower() for m in exact_matches if len(m) > 2])
+    
+    log_message(f"Keywords extracted: {set(keywords)}")
+    return list(set(keywords))
+
+
+def calculate_keyword_score(text, keywords):
+    """Calculate keyword match score for a text chunk"""
+    text_lower = text.lower()
+    score = 0
+    
+    for keyword in keywords:
+        # Exact match (case-insensitive)
+        count = text_lower.count(keyword.lower())
+        if count > 0:
+            # Higher weight for longer keywords (likely more specific)
+            weight = len(keyword) / 5.0
+            score += count * weight
+    
+    return score
+
+
+def deduplicate_nodes(nodes):
+    """Remove duplicate nodes based on text content"""
+    seen_hashes = set()
+    unique_nodes = []
+    
     for node in nodes:
-        text = node.text.lower()
-        if any(k in text for k in keywords):
-            filtered.append(node)
-    return filtered
+        # Use first 200 chars as fingerprint
+        text_hash = hash(node.text[:200])
+        
+        if text_hash not in seen_hashes:
+            seen_hashes.add(text_hash)
+            unique_nodes.append(node)
+    
+    return unique_nodes
 
-def create_query_engine(vector_index):
-    """Create hybrid retrieval engine with deduplication"""
-    log_message("Creating query engine...")
 
+def hybrid_retrieve(query, vector_index, top_k=50):
+    """Hybrid retrieval: vector + BM25 + keyword boosting"""
+    
+    # 1. Vector retrieval
     vector_retriever = VectorIndexRetriever(
         index=vector_index,
-        similarity_top_k=40  # Reduced from 50
+        similarity_top_k=top_k
     )
+    vector_nodes = vector_retriever.retrieve(query)
+    
+    # 2. BM25 retrieval
     bm25_retriever = BM25Retriever.from_defaults(
         docstore=vector_index.docstore,
-        similarity_top_k=40  # Reduced from 50
-    )
-    hybrid_retriever = QueryFusionRetriever(
-        [vector_retriever, bm25_retriever],
-        similarity_top_k=50,  # Reduced from 60
-        num_queries=1
+        similarity_top_k=top_k
     )
+    bm25_nodes = bm25_retriever.retrieve(query)
+    
+    # 3. Combine and deduplicate
+    all_nodes = vector_nodes + bm25_nodes
+    unique_nodes = deduplicate_nodes(all_nodes)
+    
+    log_message(f"Vector: {len(vector_nodes)}, BM25: {len(bm25_nodes)}, Unique: {len(unique_nodes)}")
+    
+    # 4. Extract keywords
+    keywords = extract_keywords(query)
+    
+    # 5. Add keyword scores
+    scored_nodes = []
+    for node in unique_nodes:
+        keyword_score = calculate_keyword_score(node.text, keywords)
+        
+        # Combine with original similarity score
+        original_score = node.score if hasattr(node, 'score') and node.score else 0.5
+        
+        # Boost formula: original score + keyword bonus (capped at 0.3)
+        keyword_boost = min(keyword_score * 0.1, 0.3)
+        combined_score = original_score + keyword_boost
+        
+        scored_nodes.append((node, combined_score, keyword_score))
+    
+    # 6. Sort by combined score
+    scored_nodes.sort(key=lambda x: x[1], reverse=True)
+    
+    # Log top scores
+    log_message("\nTop 10 scores after keyword boosting:")
+    for i, (node, combined, kw_score) in enumerate(scored_nodes[:10], 1):
+        doc_id = node.metadata.get('document_id', '?')
+        node_type = node.metadata.get('type', '?')
+        log_message(f"  {i}. [{doc_id}] {node_type} - Score: {combined:.3f} (kw: {kw_score:.2f})")
+    
+    # Return nodes only (without scores)
+    return [node for node, _, _ in scored_nodes[:top_k]]
 
-    class DeduplicatedQueryEngine(RetrieverQueryEngine):
-        def retrieve(self, query):
-            nodes = hybrid_retriever.retrieve(query)
-            
-            # CRITICAL: Deduplicate by text content hash
-            seen_hashes = set()
-            unique_nodes = []
-            
-            for node in nodes:
-                # Create hash from first 200 chars to detect duplicates
-                text_hash = hash(node.text[:200])
-                
-                if text_hash not in seen_hashes:
-                    seen_hashes.add(text_hash)
-                    unique_nodes.append(node)
-            
-            log_message(f"Retrieved: {len(nodes)} → Unique: {len(unique_nodes)}")
-            return unique_nodes[:50]  # Return top 50 unique
 
-    response_synthesizer = get_response_synthesizer()
+def keyword_retrieve_fallback(query, vector_index, keywords, top_k=20):
+    """Fallback: direct keyword search in all documents"""
+    all_nodes = list(vector_index.docstore.docs.values())
     
-    query_engine = DeduplicatedQueryEngine(
-        retriever=hybrid_retriever,
-        response_synthesizer=response_synthesizer
-    )
+    scored = []
+    for node in all_nodes:
+        score = calculate_keyword_score(node.text, keywords)
+        if score > 0:
+            scored.append((node, score))
+    
+    scored.sort(key=lambda x: x[1], reverse=True)
+    
+    if scored:
+        log_message(f"\nKeyword fallback found {len(scored)} matches")
+        log_message(f"Top scores: {[s for _, s in scored[:5]]}")
+    
+    return [node for node, _ in scored[:top_k]]
 
-    log_message("✓ Query engine created (with deduplication)")
-    return query_engine
+
+def create_query_engine(vector_index):
+    
+    def retrieve(query):
+        nodes = hybrid_retrieve(query, vector_index, top_k=60)
+        
+        # Fallback: If too few results, add pure keyword matches
+        keywords = extract_keywords(query)
+        if len(nodes) < 20 and keywords:
+            log_message("\n⚠ Adding keyword fallback results...")
+            fallback_nodes = keyword_retrieve_fallback(query, vector_index, keywords, top_k=30)
+            nodes.extend(fallback_nodes)
+            nodes = deduplicate_nodes(nodes)
+        
+        log_message(f"\nFinal retrieval: {len(nodes)} nodes")
+        return nodes[:50]  # Cap at 50
+    
+    log_message("✓ Query engine created (hybrid + keyword boost)")
+    return retrieve

utils.py

@@ -37,20 +37,23 @@ def format_sources(nodes):
     
     return "\n".join(set(sources))
 
-def answer_question(question, query_engine, reranker):
+def answer_question(question, retrieve_func, reranker):
     try:
         log_message(f"\n{'='*70}")
         log_message(f"QUERY: {question}")
 
-        # Retrieve nodes (already deduplicated)
-        retrieved = query_engine.retrieve(question)
-        log_message(f"RETRIEVED: {len(retrieved)} unique nodes")
+        # Retrieve with keyword boosting
+        retrieved = retrieve_func(question)
+        log_message(f"RETRIEVED: {len(retrieved)} nodes")
         
-        # Rerank
-        reranked = rerank_nodes(question, retrieved, reranker, top_k=15, min_score=0.25)  # Reduced top_k
-        log_message(f"RERANKED: {len(reranked)} nodes")
-
-        # Build context - NO TRUNCATION
+        # Rerank (optional - уже есть keyword boost)
+        if reranker:
+            reranked = rerank_nodes(question, retrieved, reranker, top_k=25, min_score=0.2)
+            log_message(f"RERANKED: {len(reranked)} nodes")
+        else:
+            reranked = retrieved[:25]
+        
+        # Build context
         context_parts = []
         for n in reranked:
             meta = n.metadata
@@ -66,7 +69,7 @@ def answer_question(question, query_engine, reranker):
             else:
                 source_label = f"[{doc_id}]"
             
-            context_parts.append(f"{source_label}\n{n.text}")  # Full text
+            context_parts.append(f"{source_label}\n{n.text}")
 
         context = "\n\n" + ("="*50 + "\n\n").join(context_parts)
 
@@ -79,18 +82,17 @@ def answer_question(question, query_engine, reranker):
 
         sources = format_sources(reranked)
         
-        # Log retrieved chunks WITHOUT duplicates
+        # Detailed logging
         log_message(f"\n{'='*70}")
-        log_message("RETRIEVED CHUNKS:")
+        log_message("FINAL CHUNKS:")
         for i, node in enumerate(reranked, 1):
             log_message(f"\n--- Chunk {i} ---")
-            log_message(f"Document: {node.metadata.get('document_id')}")
+            log_message(f"Doc: {node.metadata.get('document_id')}")
             log_message(f"Type: {node.metadata.get('type')}")
             if node.metadata.get('type') == 'table':
                 table_id = node.metadata.get('table_identifier')
-                rows = f"{node.metadata.get('row_start', 0)}-{node.metadata.get('row_end', 0)}"
-                log_message(f"Table: {table_id} (rows {rows})")
-            log_message(f"Text: {node.text[:300]}...")
+                log_message(f"Table: {table_id}")
+            log_message(f"Preview: {node.text[:400]}...")
         
         return response.text, sources