bm25 full corpuse + COMPACT + NodePostprocessor

index_retriever.py

@@ -7,44 +7,54 @@ from llama_index.retrievers.bm25 import BM25Retriever
 from llama_index.core.retrievers import QueryFusionRetriever
 from my_logging import log_message
 from config import CUSTOM_PROMPT, PROMPT_SIMPLE_POISK
+from llama_index.core.postprocessor import LlamaIndexNodePostprocessor
+
 
 def create_vector_index(documents):
     log_message("Строю векторный индекс")
     return VectorStoreIndex.from_documents(documents)
 
-def create_query_engine(vector_index):
+def create_query_engine(vector_index, reranker=None):
     try:
+        # Ensure BM25 sees the full text corpus, not just docstore
         bm25_retriever = BM25Retriever.from_defaults(
-            docstore=vector_index.docstore, 
+            docstore=vector_index.docstore,
+            nodes=vector_index.get_nodes(),  # <-- add this line
             similarity_top_k=15
         )
-        
+
         vector_retriever = VectorIndexRetriever(
-            index=vector_index, 
-            similarity_top_k=25,
+            index=vector_index,
+            similarity_top_k=30,
             similarity_cutoff=0.5
         )
-        
+
         hybrid_retriever = QueryFusionRetriever(
             [vector_retriever, bm25_retriever],
             similarity_top_k=40,
             num_queries=1
         )
-        
+
         custom_prompt_template = PromptTemplate(PROMPT_SIMPLE_POISK)
         response_synthesizer = get_response_synthesizer(
-            response_mode=ResponseMode.TREE_SUMMARIZE,
+            response_mode=ResponseMode.COMPACT,
             text_qa_template=custom_prompt_template
         )
-        
+
+        # Add reranker as a NodePostprocessor if provided
+        node_postprocessors = []
+        if reranker is not None:
+            node_postprocessors.append(LlamaIndexNodePostprocessor(reranker))
+
         query_engine = RetrieverQueryEngine(
             retriever=hybrid_retriever,
-            response_synthesizer=response_synthesizer
+            response_synthesizer=response_synthesizer,
+            node_postprocessors=node_postprocessors if node_postprocessors else None
         )
-        
+
         log_message("Query engine успешно создан")
         return query_engine
-        
+
     except Exception as e:
         log_message(f"Ошибка создания query engine: {str(e)}")
         raise