hybrid_retrieve_with_keywords implemented

index_retriever.py

@@ -49,6 +49,119 @@ def create_query_engine(vector_index):
         log_message(f"Ошибка создания query engine: {str(e)}")
         raise
 
+import re
+from typing import List, Dict, Set
+from my_logging import log_message
+
+def extract_keywords_from_query(query: str) -> Dict[str, List[str]]:
+    """Extract technical keywords from query"""
+    keywords = {
+        'materials': [],
+        'gosts': [],
+        'classes': [],
+        'technical_terms': []
+    }
+    
+    # Material codes: 08Х18Н10Т, 12Х18Н10Т, etc.
+    material_pattern = r'\b\d{2}[ХНТМКВБА]+\d{1,2}[ХНТМКВБА]*\d*\b'
+    keywords['materials'] = re.findall(material_pattern, query, re.IGNORECASE)
+    
+    # GOST standards
+    gost_pattern = r'ГОСТ\s+[РЕН\s]*\d+[\.\-\d]*'
+    keywords['gosts'] = re.findall(gost_pattern, query, re.IGNORECASE)
+    
+    # Classification codes: 3СIIIa, 1А, 2BII, etc.
+    class_pattern = r'\b\d[АБВГСD]+[IV]+[a-z]?\b'
+    keywords['classes'] = re.findall(class_pattern, query, re.IGNORECASE)
+    
+    # Technical terms
+    terms = ['полуфабрикат', 'план качества', 'контроль', 'арматура', 
+             'ультразвуковой', 'сварка', 'испытание']
+    for term in terms:
+        if term.lower() in query.lower():
+            keywords['technical_terms'].append(term)
+    
+    return keywords
+
+def keyword_search_nodes(nodes: List, keywords: Dict[str, List[str]]) -> List:
+    """Filter nodes by exact keyword matches"""
+    if not any(keywords.values()):
+        return nodes
+    
+    matched_nodes = []
+    
+    for node in nodes:
+        text_lower = node.text.lower()
+        metadata = node.metadata if hasattr(node, 'metadata') else {}
+        
+        # Check materials
+        for material in keywords['materials']:
+            if material.lower() in text_lower:
+                matched_nodes.append(node)
+                break
+        else:
+            # Check GOSTs
+            for gost in keywords['gosts']:
+                if gost.lower() in text_lower:
+                    matched_nodes.append(node)
+                    break
+            else:
+                # Check classes
+                for cls in keywords['classes']:
+                    if cls.lower() in text_lower:
+                        matched_nodes.append(node)
+                        break
+                else:
+                    # Check technical terms (at least 2 matches)
+                    term_matches = sum(1 for term in keywords['technical_terms'] 
+                                      if term.lower() in text_lower)
+                    if term_matches >= 2:
+                        matched_nodes.append(node)
+    
+    return matched_nodes
+
+def hybrid_retrieve_with_keywords(question: str, query_engine, top_k: int = 40) -> List:
+    """Retrieve using both vector search and keyword matching"""
+    
+    # Extract keywords from query
+    keywords = extract_keywords_from_query(question)
+    log_message(f"Извлечены ключевые слова: {keywords}")
+    
+    # Get vector search results
+    vector_nodes = query_engine.retriever.retrieve(question)
+    log_message(f"Векторный поиск: {len(vector_nodes)} узлов")
+    
+    # Apply keyword filtering
+    if any(keywords.values()):
+        keyword_nodes = keyword_search_nodes(vector_nodes, keywords)
+        log_message(f"После фильтрации по ключевым словам: {len(keyword_nodes)} узлов")
+        
+        # If keyword search found results, prioritize them
+        if keyword_nodes:
+            # Deduplicate and combine
+            seen_ids = set()
+            combined_nodes = []
+            
+            # First add keyword matches
+            for node in keyword_nodes[:top_k]:
+                node_id = id(node)
+                if node_id not in seen_ids:
+                    combined_nodes.append(node)
+                    seen_ids.add(node_id)
+            
+            # Then fill with vector results
+            for node in vector_nodes:
+                if len(combined_nodes) >= top_k:
+                    break
+                node_id = id(node)
+                if node_id not in seen_ids:
+                    combined_nodes.append(node)
+                    seen_ids.add(node_id)
+            
+            return combined_nodes[:top_k]
+    
+    return vector_nodes[:top_k]
+
 def rerank_nodes(query, nodes, reranker, top_k=25, min_score_threshold=0.5, diversity_penalty=0.3):
     if not nodes or not reranker:
         return nodes[:top_k]
@@ -114,140 +227,3 @@ def rerank_nodes(query, nodes, reranker, top_k=25, min_score_threshold=0.5, dive
         log_message(f"Ошибка переранжировки: {str(e)}")
         return nodes[:top_k]
     
-
-from rank_bm25 import BM25Okapi
-import numpy as np
-
-class HybridRetriever:
-    def __init__(self, vector_retriever, documents):
-        self.vector_retriever = vector_retriever
-        self.documents = documents
-        
-        # Build BM25 index
-        tokenized_docs = [doc.text.lower().split() for doc in documents]
-        self.bm25 = BM25Okapi(tokenized_docs)
-        
-        # Build metadata index for exact matching
-        self.metadata_index = self._build_metadata_index(documents)
-    
-    def _build_metadata_index(self, documents):
-        """Index by materials, GOSTs, classes for exact matching"""
-        index = {
-            'materials': {},
-            'gosts': {},
-            'classes': {},
-            'key_terms': {}
-        }
-        
-        for i, doc in enumerate(documents):
-            metadata = doc.metadata
-            
-            # Index materials
-            for material in metadata.get('materials', []):
-                if material not in index['materials']:
-                    index['materials'][material] = []
-                index['materials'][material].append(i)
-            
-            # Index GOSTs
-            for gost in metadata.get('gosts', []):
-                if gost not in index['gosts']:
-                    index['gosts'][gost] = []
-                index['gosts'][gost].append(i)
-            
-            # Index classes
-            for cls in metadata.get('classes', []):
-                if cls not in index['classes']:
-                    index['classes'][cls] = []
-                index['classes'][cls].append(i)
-            
-            # Index key terms
-            for term in metadata.get('key_terms', []):
-                term_lower = term.lower()
-                if term_lower not in index['key_terms']:
-                    index['key_terms'][term_lower] = []
-                index['key_terms'][term_lower].append(i)
-        
-        return index
-    
-    def retrieve(self, query, top_k=20, vector_weight=0.5, bm25_weight=0.3, metadata_weight=0.2):
-        """Hybrid retrieval combining vector, BM25, and metadata matching"""
-        
-        # 1. Vector search
-        vector_results = self.vector_retriever.retrieve(query)
-        vector_scores = {node.node_id: node.score for node in vector_results}
-        
-        # 2. BM25 search
-        tokenized_query = query.lower().split()
-        bm25_scores = self.bm25.get_scores(tokenized_query)
-        
-        # 3. Metadata exact matching
-        metadata_scores = self._get_metadata_scores(query)
-        
-        # 4. Combine scores
-        all_node_ids = set(list(vector_scores.keys()) + 
-                          list(range(len(self.documents))))
-        
-        combined_scores = {}
-        for node_id in all_node_ids:
-            vec_score = vector_scores.get(node_id, 0.0)
-            bm25_score = bm25_scores[node_id] if isinstance(node_id, int) and node_id < len(bm25_scores) else 0.0
-            meta_score = metadata_scores.get(node_id, 0.0)
-            
-            # Normalize and combine
-            combined_scores[node_id] = (
-                vector_weight * vec_score +
-                bm25_weight * (bm25_score / (max(bm25_scores) + 1e-6)) +
-                metadata_weight * meta_score
-            )
-        
-        # 5. Get top-k
-        sorted_nodes = sorted(combined_scores.items(), key=lambda x: x[1], reverse=True)[:top_k]
-        
-        # Return as node objects
-        results = []
-        for node_id, score in sorted_nodes:
-            if isinstance(node_id, int) and node_id < len(self.documents):
-                doc = self.documents[node_id]
-                # Create node-like object
-                from types import SimpleNamespace
-                node = SimpleNamespace(
-                    text=doc.text,
-                    metadata=doc.metadata,
-                    score=score,
-                    node_id=node_id
-                )
-                results.append(node)
-        
-        return results
-    
-    def _get_metadata_scores(self, query):
-        """Score documents by exact metadata matches"""
-        scores = {}
-        query_lower = query.lower()
-        
-        # Check for material codes
-        import re
-        material_pattern = r'\b\d{2}[ХНТМКВБА]+\d{1,2}[ХНТМКВБА]*\d*\b'
-        materials_in_query = re.findall(material_pattern, query, re.IGNORECASE)
-        
-        for material in materials_in_query:
-            if material in self.metadata_index['materials']:
-                for doc_id in self.metadata_index['materials'][material]:
-                    scores[doc_id] = scores.get(doc_id, 0) + 1.0
-        
-        # Check for GOSTs
-        gost_pattern = r'ГОСТ\s+[РЕН\s]*\d+[\.\-\d]*'
-        gosts_in_query = re.findall(gost_pattern, query, re.IGNORECASE)
-        
-        for gost in gosts_in_query:
-            if gost in self.metadata_index['gosts']:
-                for doc_id in self.metadata_index['gosts'][gost]:
-                    scores[doc_id] = scores.get(doc_id, 0) + 0.8
-        
-        # Check for key terms
-        for term, doc_ids in self.metadata_index['key_terms'].items():
-            if term in query_lower:
-                for doc_id in doc_ids:
-                    scores[doc_id] = scores.get(doc_id, 0) + 0.5
-        
-        return scores

utils.py

@@ -232,6 +232,7 @@ def generate_sources_html(nodes, chunks_df=None):
     html += "</div>"
     return html
 def answer_question(question, query_engine, reranker, current_model, chunks_df=None, hybrid_retriever=None):
+    from index_retriever import hybrid_retrieve_with_keywords
     if query_engine is None:
         return "<div style='background-color: #e53e3e; color: white; padding: 20px; border-radius: 10px;'>Система не инициализирована</div>", "", ""
     
@@ -240,22 +241,18 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
         
         llm = get_llm_model(current_model)
         
-        # Use hybrid retriever if available
-        if hybrid_retriever:
-            retrieved_nodes = hybrid_retriever.retrieve(question, top_k=30)
-            log_message(f"Hybrid retrieval: получено {len(retrieved_nodes)} узлов")
-        else:
-            retrieved_nodes = query_engine.retriever.retrieve(question)
-            log_message(f"Vector retrieval: получено {len(retrieved_nodes)} узлов")
+        # Use keyword-enhanced retrieval
+        retrieved_nodes = hybrid_retrieve_with_keywords(question, query_engine, top_k=40)
+        log_message(f"Hybrid keyword retrieval: получено {len(retrieved_nodes)} узлов")
         
-        # Rerank with increased top_k
+        # Rerank
         reranked_nodes = rerank_nodes(
             question,  
             retrieved_nodes, 
             reranker, 
-            top_k=25,  # Increased from 20
-            min_score_threshold=0.3,  # Lowered from 0.5 to catch more results
-            diversity_penalty=0.2  # Reduced penalty
+            top_k=25,
+            min_score_threshold=0.3,
+            diversity_penalty=0.2
         )
         
         formatted_context = format_context_for_llm(reranked_nodes)