eski holat with utils

index_retriever.py

@@ -12,7 +12,70 @@ def create_vector_index(documents):
     log_message("Строю векторный индекс")
     return VectorStoreIndex.from_documents(documents)
 
-
+def rerank_nodes(query, nodes, reranker, top_k=25, min_score_threshold=0.5, diversity_penalty=0.3):
+    if not nodes or not reranker:
+        return nodes[:top_k]
+    
+    try:
+        log_message(f"Переранжирую {len(nodes)} узлов")
+        
+        pairs = [[query, node.text] for node in nodes]
+        scores = reranker.predict(pairs)
+        scored_nodes = list(zip(nodes, scores))
+        
+        scored_nodes.sort(key=lambda x: x[1], reverse=True)
+        
+        if min_score_threshold is not None:
+            scored_nodes = [(node, score) for node, score in scored_nodes 
+                          if score >= min_score_threshold]
+            log_message(f"После фильтрации по порогу {min_score_threshold}: {len(scored_nodes)} узлов")
+        
+        if not scored_nodes:
+            log_message("Нет узлов после фильтрации, снижаю порог")
+            scored_nodes = list(zip(nodes, scores))
+            scored_nodes.sort(key=lambda x: x[1], reverse=True)
+            min_score_threshold = scored_nodes[0][1] * 0.6  
+            scored_nodes = [(node, score) for node, score in scored_nodes 
+                          if score >= min_score_threshold]
+        
+        selected_nodes = []
+        selected_docs = set()
+        selected_sections = set()
+        
+        for node, score in scored_nodes:
+            if len(selected_nodes) >= top_k:
+                break
+            
+            metadata = node.metadata if hasattr(node, 'metadata') else {}
+            doc_id = metadata.get('document_id', 'unknown')
+            section_key = f"{doc_id}_{metadata.get('section_path', metadata.get('section_id', ''))}"
+            
+            # Apply diversity penalty
+            penalty = 0
+            if doc_id in selected_docs:
+                penalty += diversity_penalty * 0.5
+            if section_key in selected_sections:
+                penalty += diversity_penalty
+            
+            adjusted_score = score * (1 - penalty)
+            
+            # Add if still competitive
+            if not selected_nodes or adjusted_score >= selected_nodes[0][1] * 0.6:
+                selected_nodes.append((node, score))
+                selected_docs.add(doc_id)
+                selected_sections.add(section_key)
+        
+        log_message(f"Выбрано {len(selected_nodes)} узлов с разнообразием")
+        log_message(f"Уникальных документов: {len(selected_docs)}, секций: {len(selected_sections)}")
+        
+        if selected_nodes:
+            log_message(f"Score range: {selected_nodes[0][1]:.3f} to {selected_nodes[-1][1]:.3f}")
+        
+        return [node for node, score in selected_nodes]
+        
+    except Exception as e:
+        log_message(f"Ошибка переранжировки: {str(e)}")
+        return nodes[:top_k]
 
 def create_query_engine(vector_index):
     try:

utils.py

@@ -226,6 +226,33 @@ def generate_sources_html(nodes, chunks_df=None):
     html += "</div>"
     return html
 
+def deduplicate_nodes(nodes):
+    """Deduplicate retrieved nodes based on unique identifiers"""
+    seen = set()
+    unique_nodes = []
+    
+    for node in nodes:
+        # Create unique identifier from metadata
+        doc_id = node.metadata.get('document_id', '')
+        section_id = node.metadata.get('section_id', '')
+        chunk_id = node.metadata.get('chunk_id', 0)
+        node_type = node.metadata.get('type', 'text')
+        
+        if node_type == 'table':
+            table_num = node.metadata.get('table_number', '')
+            identifier = f"{doc_id}|table|{table_num}|{chunk_id}"
+        elif node_type == 'image':
+            img_num = node.metadata.get('image_number', '')
+            identifier = f"{doc_id}|image|{img_num}"
+        else:
+            identifier = f"{doc_id}|{section_id}|{chunk_id}"
+        
+        if identifier not in seen:
+            seen.add(identifier)
+            unique_nodes.append(node)
+    
+    return unique_nodes
+
 
 def answer_question(question, query_engine, reranker, current_model, chunks_df=None):
     if query_engine is None:
@@ -239,11 +266,19 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
         # Direct retrieval without query expansion
         retrieved_nodes = query_engine.retriever.retrieve(question)
         
-        log_message(f"Получено {len(retrieved_nodes)} узлов")
+        total_retrieved = len(retrieved_nodes)
+        log_message(f"RETRIEVED: {total_retrieved} nodes (before deduplication)")
+        
+        # Deduplicate
+        unique_retrieved = deduplicate_nodes(retrieved_nodes)
+        duplicates_removed = total_retrieved - len(unique_retrieved)
+        log_message(f"DEDUPLICATION: {duplicates_removed} duplicates removed")
+        log_message(f"UNIQUE NODES: {len(unique_retrieved)} nodes")
+
         
         reranked_nodes = rerank_nodes(
             question,  
-            retrieved_nodes, 
+            unique_retrieved, 
             reranker, 
             top_k=20,
             min_score_threshold=0.5,