bm = 25, semantic = 35. hybrid = 40

index_retriever.py

@@ -11,29 +11,29 @@ from config import CUSTOM_PROMPT, PROMPT_SIMPLE_POISK
 def create_vector_index(documents):
     log_message("Строю векторный индекс")
     return VectorStoreIndex.from_documents(documents)
+
 def create_query_engine(vector_index):
     try:
         bm25_retriever = BM25Retriever.from_defaults(
             docstore=vector_index.docstore, 
-            similarity_top_k=15  # Lower since we're combining with semantic
+            similarity_top_k=25  
         )
         
         vector_retriever = VectorIndexRetriever(
             index=vector_index, 
-            similarity_top_k=15,  # Lower since we're combining with BM25
-            similarity_cutoff=0.6  # Slightly lower threshold
+            similarity_top_k=35,  
+            similarity_cutoff=0.7
         )
         
-        # Hybrid retriever combines both approaches
         hybrid_retriever = QueryFusionRetriever(
             [vector_retriever, bm25_retriever],
-            similarity_top_k=30,  # Final top_k after fusion
+            similarity_top_k=40,  
             num_queries=1
         )
         
         custom_prompt_template = PromptTemplate(PROMPT_SIMPLE_POISK)
         response_synthesizer = get_response_synthesizer(
-            response_mode=ResponseMode.TREE_SUMMARIZE,
+            response_mode=ResponseMode.TREE_SUMMARIZE,  
             text_qa_template=custom_prompt_template
         )
         
@@ -42,9 +42,74 @@ def create_query_engine(vector_index):
             response_synthesizer=response_synthesizer
         )
         
-        log_message("Query engine создан (BM25 + Semantic, без reranking)")
+        log_message("Query engine успешно создан")
         return query_engine
         
     except Exception as e:
         log_message(f"Ошибка создания query engine: {str(e)}")
-        raise
+        raise
+
+def rerank_nodes(query, nodes, reranker, top_k=25, min_score_threshold=0.5, diversity_penalty=0.3):
+    if not nodes or not reranker:
+        return nodes[:top_k]
+    
+    try:
+        log_message(f"Переранжирую {len(nodes)} узлов")
+        
+        pairs = [[query, node.text] for node in nodes]
+        scores = reranker.predict(pairs)
+        scored_nodes = list(zip(nodes, scores))
+        
+        scored_nodes.sort(key=lambda x: x[1], reverse=True)
+        
+        if min_score_threshold is not None:
+            scored_nodes = [(node, score) for node, score in scored_nodes 
+                          if score >= min_score_threshold]
+            log_message(f"После фильтрации по порогу {min_score_threshold}: {len(scored_nodes)} узлов")
+        
+        if not scored_nodes:
+            log_message("Нет узлов после фильтрации, снижаю порог")
+            scored_nodes = list(zip(nodes, scores))
+            scored_nodes.sort(key=lambda x: x[1], reverse=True)
+            min_score_threshold = scored_nodes[0][1] * 0.6  
+            scored_nodes = [(node, score) for node, score in scored_nodes 
+                          if score >= min_score_threshold]
+        
+        selected_nodes = []
+        selected_docs = set()
+        selected_sections = set()
+        
+        for node, score in scored_nodes:
+            if len(selected_nodes) >= top_k:
+                break
+            
+            metadata = node.metadata if hasattr(node, 'metadata') else {}
+            doc_id = metadata.get('document_id', 'unknown')
+            section_key = f"{doc_id}_{metadata.get('section_path', metadata.get('section_id', ''))}"
+            
+            # Apply diversity penalty
+            penalty = 0
+            if doc_id in selected_docs:
+                penalty += diversity_penalty * 0.5
+            if section_key in selected_sections:
+                penalty += diversity_penalty
+            
+            adjusted_score = score * (1 - penalty)
+            
+            # Add if still competitive
+            if not selected_nodes or adjusted_score >= selected_nodes[0][1] * 0.6:
+                selected_nodes.append((node, score))
+                selected_docs.add(doc_id)
+                selected_sections.add(section_key)
+        
+        log_message(f"Выбрано {len(selected_nodes)} узлов с разнообразием")
+        log_message(f"Уникальных документов: {len(selected_docs)}, секций: {len(selected_sections)}")
+        
+        if selected_nodes:
+            log_message(f"Score range: {selected_nodes[0][1]:.3f} to {selected_nodes[-1][1]:.3f}")
+        
+        return [node for node, score in selected_nodes]
+        
+    except Exception as e:
+        log_message(f"Ошибка переранжировки: {str(e)}")
+        return nodes[:top_k]

utils.py

@@ -6,7 +6,7 @@ from llama_index.embeddings.huggingface import HuggingFaceEmbedding
 from sentence_transformers import CrossEncoder
 from config import AVAILABLE_MODELS, DEFAULT_MODEL, GOOGLE_API_KEY
 import time
-# from index_retriever import rerank_nodes
+from index_retriever import rerank_nodes
 from my_logging import log_message
 from config import PROMPT_SIMPLE_POISK
 
@@ -260,15 +260,31 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
         
         llm = get_llm_model(current_model)
         
-        # Simple retrieval without query expansion
-        retrieved_nodes = query_engine.retriever.retrieve(question)
+        query_variations = expand_query(question, llm)
         
-        log_message(f"Получено {len(retrieved_nodes)} узлов (BM25 + Semantic)")
+        all_nodes = []
+        seen_node_ids = set()
         
-        # Use nodes directly without reranking
-        final_nodes = retrieved_nodes[:30]  # Ensure we use top 30
+        for query_var in query_variations:
+            retrieved = query_engine.retriever.retrieve(query_var)
+            for node in retrieved:
+                node_id = f"{node.node_id if hasattr(node, 'node_id') else hash(node.text)}"
+                if node_id not in seen_node_ids:
+                    all_nodes.append(node)
+                    seen_node_ids.add(node_id)
         
-        formatted_context = format_context_for_llm(final_nodes)
+        log_message(f"Получено {len(all_nodes)} уникальных узлов из {len(query_variations)} запросов")
+        
+        reranked_nodes = rerank_nodes(
+            question,  
+            all_nodes, 
+            reranker, 
+            top_k=25,
+            min_score_threshold=0.5, 
+            diversity_penalty=0.3
+        )
+        
+        formatted_context = format_context_for_llm(reranked_nodes)
         
         enhanced_question = f"""Контекст из базы данных:
 {formatted_context}
@@ -285,18 +301,18 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
         
         log_message(f"Обработка завершена за {processing_time:.2f}с")
         
-        sources_html = generate_sources_html(final_nodes, chunks_df)
+        sources_html = generate_sources_html(reranked_nodes, chunks_df)
         
         answer_with_time = f"""<div style='background-color: #2d3748; color: white; padding: 20px; border-radius: 10px; margin-bottom: 10px;'>
         <h3 style='color: #63b3ed; margin-top: 0;'>Ответ (Модель: {current_model}):</h3>
         <div style='line-height: 1.6; font-size: 16px;'>{response.response}</div>
         <div style='margin-top: 15px; padding-top: 10px; border-top: 1px solid #4a5568; font-size: 14px; color: #a0aec0;'>
-        Время обработки: {processing_time:.2f} секунд | Метод: BM25 + Semantic (без reranking)
+        Время обработки: {processing_time:.2f} секунд
         </div>
         </div>"""
         
         chunk_info = []
-        for node in final_nodes:
+        for node in reranked_nodes:
             metadata = node.metadata if hasattr(node, 'metadata') else {}
             chunk_info.append({
                 'document_id': metadata.get('document_id', 'unknown'),