with bm and semantic

index_retriever.py

@@ -3,19 +3,32 @@ from llama_index.core.query_engine import RetrieverQueryEngine
 from llama_index.core.retrievers import VectorIndexRetriever
 from llama_index.core.response_synthesizers import get_response_synthesizer, ResponseMode
 from llama_index.core.prompts import PromptTemplate
+from llama_index.retrievers.bm25 import BM25Retriever
+from llama_index.core.retrievers import QueryFusionRetriever
 from my_logging import log_message
-from config import PROMPT_SIMPLE_POISK
+from config import CUSTOM_PROMPT, PROMPT_SIMPLE_POISK
 
 def create_vector_index(documents):
     log_message("Строю векторный индекс")
     return VectorStoreIndex.from_documents(documents)
-
 def create_query_engine(vector_index):
     try:
+        bm25_retriever = BM25Retriever.from_defaults(
+            docstore=vector_index.docstore, 
+            similarity_top_k=15  # Lower since we're combining with semantic
+        )
+        
         vector_retriever = VectorIndexRetriever(
             index=vector_index, 
-            similarity_top_k=30,
-            similarity_cutoff=0.7
+            similarity_top_k=15,  # Lower since we're combining with BM25
+            similarity_cutoff=0.6  # Slightly lower threshold
+        )
+        
+        # Hybrid retriever combines both approaches
+        hybrid_retriever = QueryFusionRetriever(
+            [vector_retriever, bm25_retriever],
+            similarity_top_k=30,  # Final top_k after fusion
+            num_queries=1
         )
         
         custom_prompt_template = PromptTemplate(PROMPT_SIMPLE_POISK)
@@ -25,11 +38,11 @@ def create_query_engine(vector_index):
         )
         
         query_engine = RetrieverQueryEngine(
-            retriever=vector_retriever,
+            retriever=hybrid_retriever,
             response_synthesizer=response_synthesizer
         )
         
-        log_message("Query engine успешно создан (только векторный поиск)")
+        log_message("Query engine создан (BM25 + Semantic, без reranking)")
         return query_engine
         
     except Exception as e:

utils.py

@@ -260,11 +260,15 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
         
         llm = get_llm_model(current_model)
         
+        # Simple retrieval without query expansion
         retrieved_nodes = query_engine.retriever.retrieve(question)
         
-        log_message(f"Получено {len(retrieved_nodes)} узлов")
+        log_message(f"Получено {len(retrieved_nodes)} узлов (BM25 + Semantic)")
         
-        formatted_context = format_context_for_llm(retrieved_nodes)
+        # Use nodes directly without reranking
+        final_nodes = retrieved_nodes[:30]  # Ensure we use top 30
+        
+        formatted_context = format_context_for_llm(final_nodes)
         
         enhanced_question = f"""Контекст из базы данных:
 {formatted_context}
@@ -281,18 +285,18 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
         
         log_message(f"Обработка завершена за {processing_time:.2f}с")
         
-        sources_html = generate_sources_html(retrieved_nodes, chunks_df)
+        sources_html = generate_sources_html(final_nodes, chunks_df)
         
         answer_with_time = f"""<div style='background-color: #2d3748; color: white; padding: 20px; border-radius: 10px; margin-bottom: 10px;'>
         <h3 style='color: #63b3ed; margin-top: 0;'>Ответ (Модель: {current_model}):</h3>
         <div style='line-height: 1.6; font-size: 16px;'>{response.response}</div>
         <div style='margin-top: 15px; padding-top: 10px; border-top: 1px solid #4a5568; font-size: 14px; color: #a0aec0;'>
-        Время обработки: {processing_time:.2f} секунд
+        Время обработки: {processing_time:.2f} секунд | Метод: BM25 + Semantic (без reranking)
         </div>
         </div>"""
         
         chunk_info = []
-        for node in retrieved_nodes :
+        for node in final_nodes:
             metadata = node.metadata if hasattr(node, 'metadata') else {}
             chunk_info.append({
                 'document_id': metadata.get('document_id', 'unknown'),