added 35 semantic similarity

index_retriever.py

@@ -22,7 +22,7 @@ def create_query_engine(vector_index):
         vector_retriever = VectorIndexRetriever(
             index=vector_index, 
             similarity_top_k=30,  # Increased
-            similarity_cutoff=0.7  # Slightly lower for recall
+            similarity_cutoff=0.5  # Slightly lower for recall
         )
         
         hybrid_retriever = QueryFusionRetriever(
@@ -50,9 +50,6 @@ def create_query_engine(vector_index):
         raise
 
 def rerank_nodes(query, nodes, reranker, top_k=20, min_score_threshold=0.5, diversity_penalty=0.3):
-    """
-    Rerank nodes with diversity and adaptive scoring
-    """
     if not nodes or not reranker:
         return nodes[:top_k]
     
@@ -74,11 +71,10 @@ def rerank_nodes(query, nodes, reranker, top_k=20, min_score_threshold=0.5, dive
             log_message("Нет узлов после фильтрации, снижаю порог")
             scored_nodes = list(zip(nodes, scores))
             scored_nodes.sort(key=lambda x: x[1], reverse=True)
-            min_score_threshold = scored_nodes[0][1] * 0.6  # 60% of top score
+            min_score_threshold = scored_nodes[0][1] * 0.6  
             scored_nodes = [(node, score) for node, score in scored_nodes 
                           if score >= min_score_threshold]
         
-        # MMR-like diversity selection
         selected_nodes = []
         selected_docs = set()
         selected_sections = set()

utils.py

@@ -258,13 +258,10 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
     try:
         start_time = time.time()
         
-        # Get LLM for query expansion
         llm = get_llm_model(current_model)
         
-        # Expand query
         query_variations = expand_query(question, llm)
         
-        # Retrieve with multiple queries and deduplicate
         all_nodes = []
         seen_node_ids = set()
         
@@ -278,13 +275,12 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
         
         log_message(f"Получено {len(all_nodes)} уникальных узлов из {len(query_variations)} запросов")
         
-        # Rerank with stricter threshold and diversity
         reranked_nodes = rerank_nodes(
-            question,  # Use original question for reranking
+            question,  
             all_nodes, 
             reranker, 
             top_k=20,
-            min_score_threshold=0.5,  # Much stricter threshold
+            min_score_threshold=0.5, 
             diversity_penalty=0.3
         )
         
@@ -311,7 +307,7 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
         <h3 style='color: #63b3ed; margin-top: 0;'>Ответ (Модель: {current_model}):</h3>
         <div style='line-height: 1.6; font-size: 16px;'>{response.response}</div>
         <div style='margin-top: 15px; padding-top: 10px; border-top: 1px solid #4a5568; font-size: 14px; color: #a0aec0;'>
-        Время обработки: {processing_time:.2f} секунд | Источников: {len(reranked_nodes)} | Запросов: {len(query_variations)}
+        Время обработки: {processing_time:.2f} секунд
         </div>
         </div>"""