new ways

index_retriever.py

@@ -16,24 +16,24 @@ def create_query_engine(vector_index):
     try:
         bm25_retriever = BM25Retriever.from_defaults(
             docstore=vector_index.docstore, 
-            similarity_top_k=15  
+            similarity_top_k=20  # Increased for more candidates
         )
         
         vector_retriever = VectorIndexRetriever(
             index=vector_index, 
-            similarity_top_k=20, 
-            similarity_cutoff=0.7  
+            similarity_top_k=25,  # Increased
+            similarity_cutoff=0.65  # Slightly lower for recall
         )
         
         hybrid_retriever = QueryFusionRetriever(
             [vector_retriever, bm25_retriever],
-            similarity_top_k=30,  
+            similarity_top_k=40,  # More candidates for reranking
             num_queries=1
         )
         
         custom_prompt_template = PromptTemplate(PROMPT_SIMPLE_POISK)
         response_synthesizer = get_response_synthesizer(
-            response_mode=ResponseMode.TREE_SUMMARIZE,
+            response_mode=ResponseMode.TREE_SUMMARIZE,  
             text_qa_template=custom_prompt_template
         )
         
@@ -49,16 +49,16 @@ def create_query_engine(vector_index):
         log_message(f"Ошибка создания query engine: {str(e)}")
         raise
 
-def rerank_nodes(query, nodes, reranker, top_k=20, min_score_threshold=None):
+def rerank_nodes(query, nodes, reranker, top_k=20, min_score_threshold=0.5, diversity_penalty=0.3):
     """
-    Rerank nodes with adaptive top_k based on score distribution
+    Rerank nodes with diversity and adaptive scoring
     """
     if not nodes or not reranker:
         return nodes[:top_k]
-
+    
     try:
         log_message(f"Переранжирую {len(nodes)} узлов")
-
+        
         pairs = [[query, node.text] for node in nodes]
         scores = reranker.predict(pairs)
         scored_nodes = list(zip(nodes, scores))
@@ -66,30 +66,56 @@ def rerank_nodes(query, nodes, reranker, top_k=20, min_score_threshold=None):
         # Sort by score descending
         scored_nodes.sort(key=lambda x: x[1], reverse=True)
         
-        # Apply minimum score threshold if specified
+        # Filter by minimum threshold (more strict)
         if min_score_threshold is not None:
-            scored_nodes = [(node, score) for node, score in scored_nodes if score >= min_score_threshold]
+            scored_nodes = [(node, score) for node, score in scored_nodes 
+                          if score >= min_score_threshold]
             log_message(f"После фильтрации по порогу {min_score_threshold}: {len(scored_nodes)} узлов")
         
-        # Adaptive top_k: if we have many high-scoring results, keep more
-        if len(scored_nodes) > top_k:
-            top_score = scored_nodes[0][1] if scored_nodes else 0
-            # If 30th node still has >70% of top score, expand to 30
-            if len(scored_nodes) >= 30 and scored_nodes[29][1] / top_score > 0.7:
-                effective_top_k = 30
-                log_message(f"Расширяю top_k до {effective_top_k} из-за высоких скоров")
-            else:
-                effective_top_k = top_k
-        else:
-            effective_top_k = len(scored_nodes)
+        if not scored_nodes:
+            log_message("Нет узлов после фильтрации, снижаю порог")
+            scored_nodes = list(zip(nodes, scores))
+            scored_nodes.sort(key=lambda x: x[1], reverse=True)
+            min_score_threshold = scored_nodes[0][1] * 0.5  # 50% of top score
+            scored_nodes = [(node, score) for node, score in scored_nodes 
+                          if score >= min_score_threshold]
         
-        reranked_nodes = [node for node, score in scored_nodes[:effective_top_k]]
+        # MMR-like diversity selection
+        selected_nodes = []
+        selected_docs = set()
+        selected_sections = set()
         
-        log_message(f"Возвращаю топ-{effective_top_k} узлов после переранжировки")
-        log_message(f"Score range: {scored_nodes[0][1]:.3f} to {scored_nodes[min(effective_top_k-1, len(scored_nodes)-1)][1]:.3f}")
+        for node, score in scored_nodes:
+            if len(selected_nodes) >= top_k:
+                break
+            
+            metadata = node.metadata if hasattr(node, 'metadata') else {}
+            doc_id = metadata.get('document_id', 'unknown')
+            section_key = f"{doc_id}_{metadata.get('section_path', metadata.get('section_id', ''))}"
+            
+            # Apply diversity penalty
+            penalty = 0
+            if doc_id in selected_docs:
+                penalty += diversity_penalty * 0.5
+            if section_key in selected_sections:
+                penalty += diversity_penalty
+            
+            adjusted_score = score * (1 - penalty)
+            
+            # Add if still competitive
+            if not selected_nodes or adjusted_score >= selected_nodes[0][1] * 0.6:
+                selected_nodes.append((node, score))
+                selected_docs.add(doc_id)
+                selected_sections.add(section_key)
+        
+        log_message(f"Выбрано {len(selected_nodes)} узлов с разнообразием")
+        log_message(f"Уникальных документов: {len(selected_docs)}, секций: {len(selected_sections)}")
+        
+        if selected_nodes:
+            log_message(f"Score range: {selected_nodes[0][1]:.3f} to {selected_nodes[-1][1]:.3f}")
+        
+        return [node for node, score in selected_nodes]
         
-        return reranked_nodes
-
     except Exception as e:
         log_message(f"Ошибка переранжировки: {str(e)}")
         return nodes[:top_k]

utils.py

@@ -10,39 +10,6 @@ from index_retriever import rerank_nodes
 from my_logging import log_message
 from config import PROMPT_SIMPLE_POISK
 
-def get_llm_model(model_name):
-    try:
-        model_config = AVAILABLE_MODELS.get(model_name)
-        if not model_config:
-            log_message(f"Модель {model_name} не найдена, использую модель по умолчанию")
-            model_config = AVAILABLE_MODELS[DEFAULT_MODEL]
-        
-        if not model_config.get("api_key"):
-            raise Exception(f"API ключ не найден для модели {model_name}")
-        
-        if model_config["provider"] == "google":
-            return GoogleGenAI(
-                model=model_config["model_name"], 
-                api_key=model_config["api_key"]
-            )
-        elif model_config["provider"] == "openai":
-            return OpenAI(
-                model=model_config["model_name"],
-                api_key=model_config["api_key"]
-            )
-        else:
-            raise Exception(f"Неподдерживаемый провайдер: {model_config['provider']}")
-            
-    except Exception as e:
-        log_message(f"Ошибка создания модели {model_name}: {str(e)}")
-        return GoogleGenAI(model="gemini-2.0-flash", api_key=GOOGLE_API_KEY)
-
-def get_embedding_model(model_name="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"):
-    return HuggingFaceEmbedding(model_name=model_name)
-
-def get_reranker_model(model_name='cross-encoder/ms-marco-MiniLM-L-12-v2'):
-    return CrossEncoder(model_name)
-
 def get_llm_model(model_name):
     try:
         model_config = AVAILABLE_MODELS.get(model_name)
@@ -168,7 +135,7 @@ def format_context_for_llm(nodes):
     
     return "\n".join(context_parts)
 
-    
+
 def generate_sources_html(nodes, chunks_df=None):
     html = "<div style='background-color: #2d3748; color: white; padding: 20px; border-radius: 10px; max-height: 400px; overflow-y: auto;'>"
     html += "<h3 style='color: #63b3ed; margin-top: 0;'>Источники:</h3>"
@@ -259,6 +226,31 @@ def generate_sources_html(nodes, chunks_df=None):
     html += "</div>"
     return html
 
+def expand_query(question, llm_model):
+    """
+    Generate multiple query variations for better retrieval
+    """
+    expansion_prompt = f"""Дан вопрос: "{question}"
+
+Сгенерируй 2 альтернативные формулировки этого вопроса для поиска в базе данных.
+Используй синонимы и разные формулировки, сохраняя смысл.
+
+Формат ответа (только вопросы, по одному на строку):
+1. [первая формулировка]
+2. [вторая формулировка]"""
+    
+    try:
+        response = llm_model.complete(expansion_prompt)
+        expanded = [q.strip() for q in response.text.split('\n') if q.strip() and not q.strip().startswith('1.') and not q.strip().startswith('2.')]
+        # Clean up
+        expanded = [q.lstrip('12. ').strip() for q in expanded if len(q) > 10][:2]
+        log_message(f"Query expansion: {len(expanded)} вариантов")
+        return [question] + expanded
+    except Exception as e:
+        log_message(f"Ошибка расширения запроса: {str(e)}")
+        return [question]
+
+
 def answer_question(question, query_engine, reranker, current_model, chunks_df=None):
     if query_engine is None:
         return "<div style='background-color: #e53e3e; color: white; padding: 20px; border-radius: 10px;'>Система не инициализирована</div>", "", ""
@@ -266,25 +258,45 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
     try:
         start_time = time.time()
         
-        retrieved_nodes = query_engine.retriever.retrieve(question)
-        log_message(f"Получено узлов после гибридного поиска: {len(retrieved_nodes)}")
+        # Get LLM for query expansion
+        llm = get_llm_model(current_model)
         
-        # Use adaptive reranking with lower threshold for better recall
+        # Expand query
+        query_variations = expand_query(question, llm)
+        
+        # Retrieve with multiple queries and deduplicate
+        all_nodes = []
+        seen_node_ids = set()
+        
+        for query_var in query_variations:
+            retrieved = query_engine.retriever.retrieve(query_var)
+            for node in retrieved:
+                node_id = f"{node.node_id if hasattr(node, 'node_id') else hash(node.text)}"
+                if node_id not in seen_node_ids:
+                    all_nodes.append(node)
+                    seen_node_ids.add(node_id)
+        
+        log_message(f"Получено {len(all_nodes)} уникальных узлов из {len(query_variations)} запросов")
+        
+        # Rerank with stricter threshold and diversity
         reranked_nodes = rerank_nodes(
-            question, 
-            retrieved_nodes, 
+            question,  # Use original question for reranking
+            all_nodes, 
             reranker, 
-            top_k=25,  # Increased from 20
-            min_score_threshold=-5.0  # Add threshold to filter very low scores
+            top_k=20,
+            min_score_threshold=0.5,  # Much stricter threshold
+            diversity_penalty=0.3
         )
         
         formatted_context = format_context_for_llm(reranked_nodes)
         
-        enhanced_question = f"""
-Контекст из базы данных:
+        enhanced_question = f"""Контекст из базы данных:
 {formatted_context}
 
-Вопрос пользователя: {question}"""
+Вопрос пользователя: {question}
+
+Инструкция: Ответь на вопрос, используя ТОЛЬКО информацию из контекста выше. 
+Если информации недостаточно, четко укажи это. Цитируй конкретные источники."""
         
         response = query_engine.query(enhanced_question)
         
@@ -299,7 +311,7 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
         <h3 style='color: #63b3ed; margin-top: 0;'>Ответ (Модель: {current_model}):</h3>
         <div style='line-height: 1.6; font-size: 16px;'>{response.response}</div>
         <div style='margin-top: 15px; padding-top: 10px; border-top: 1px solid #4a5568; font-size: 14px; color: #a0aec0;'>
-        Время обработки: {processing_time:.2f} секунд | Источников: {len(reranked_nodes)}
+        Время обработки: {processing_time:.2f} секунд | Источников: {len(reranked_nodes)} | Запросов: {len(query_variations)}
         </div>
         </div>"""