new normalizer C to Latin C

documents_prep.py

@@ -34,6 +34,25 @@ def chunk_text_documents(documents):
     
     return chunked
 
+def normalize_text(text):
+    """
+    Normalize text by converting Latin C to Cyrillic С for consistency
+    This ensures "C-25" and "С-25" are treated as the same in search
+    """
+    if not text:
+        return text
+    
+    # Replace Latin 'C' with Cyrillic 'С' (U+0421)
+    # This is for welding types like C-25 -> С-25
+    text = text.replace('C-', 'С-')
+    text = text.replace('C ', 'С ')
+    
+    # Also handle cases like "Type C" or variations
+    import re
+    # Match "C" followed by digit or space in context of welding types
+    text = re.sub(r'\bC(\d)', r'С\1', text)
+    
+    return text
 
 def chunk_table_by_content(table_data, doc_id, max_chars=MAX_CHARS_TABLE, max_rows=MAX_ROWS_TABLE):
     headers = table_data.get('headers', [])
@@ -43,6 +62,7 @@ def chunk_table_by_content(table_data, doc_id, max_chars=MAX_CHARS_TABLE, max_ro
     section = table_data.get('section', '')
    
     table_num_clean = str(table_num).strip()
+    table_title_normalized = normalize_text(str(table_title))  # NORMALIZE TITLE
     
     import re
     if 'приложени' in section.lower():
@@ -60,8 +80,8 @@ def chunk_table_by_content(table_data, doc_id, max_chars=MAX_CHARS_TABLE, max_ro
     
     log_message(f"  📊 Processing: {doc_id} - {table_identifier} ({len(rows)} rows)")
     
-    # Calculate base metadata size
-    base_content = format_table_header(doc_id, table_identifier, table_num, table_title, section, headers)
+    # Calculate base metadata size with NORMALIZED title
+    base_content = format_table_header(doc_id, table_identifier, table_num, table_title_normalized, section, headers)
     base_size = len(base_content)
     available_space = max_chars - base_size - 200 
     
@@ -74,8 +94,8 @@ def chunk_table_by_content(table_data, doc_id, max_chars=MAX_CHARS_TABLE, max_ro
             'type': 'table',
             'document_id': doc_id,
             'table_number': table_num_clean,
-            'table_identifier': table_identifier,
-            'table_title': table_title,
+            'table_identifier': normalize_text(table_identifier),  # NORMALIZE identifier
+            'table_title': table_title_normalized,  # NORMALIZED
             'section': section,
             'total_rows': len(rows),
             'chunk_size': len(content),
@@ -105,8 +125,8 @@ def chunk_table_by_content(table_data, doc_id, max_chars=MAX_CHARS_TABLE, max_ro
                 'type': 'table',
                 'document_id': doc_id,
                 'table_number': table_num_clean,
-                'table_identifier': table_identifier,
-                'table_title': table_title,
+                'table_identifier': normalize_text(table_identifier),  # NORMALIZE
+                'table_title': table_title_normalized,  # NORMALIZED
                 'section': section,
                 'chunk_id': chunk_num,
                 'row_start': current_rows[0]['_idx'] - 1,
@@ -139,8 +159,8 @@ def chunk_table_by_content(table_data, doc_id, max_chars=MAX_CHARS_TABLE, max_ro
             'type': 'table',
             'document_id': doc_id,
             'table_number': table_num_clean,
-            'table_identifier': table_identifier,
-            'table_title': table_title,
+            'table_identifier': normalize_text(table_identifier),  # NORMALIZE
+            'table_title': table_title_normalized,  # NORMALIZED
             'section': section,
             'chunk_id': chunk_num,
             'row_start': current_rows[0]['_idx'] - 1,
@@ -156,15 +176,16 @@ def chunk_table_by_content(table_data, doc_id, max_chars=MAX_CHARS_TABLE, max_ro
     return chunks
 
 
+# MODIFIED: Update format_table_header function
 def format_table_header(doc_id, table_identifier, table_num, table_title, section, headers):
-    content = f"ТАБЛИЦА {table_identifier} из документа {doc_id}\n"
+    content = f"ТАБЛИЦА {normalize_text(table_identifier)} из документа {doc_id}\n"
     
     # Add table type/number prominently for matching
     if table_num:
-        content += f"ТИП: {table_num}\n"
+        content += f"ТИП: {normalize_text(table_num)}\n"
     
     if table_title:
-        content += f"НАЗВАНИЕ: {table_title}\n"
+        content += f"НАЗВАНИЕ: {normalize_text(table_title)}\n"
     
     if section:
         content += f"РАЗДЕЛ: {section}\n"

index_retriever.py

@@ -65,96 +65,9 @@ def rerank_nodes(query, nodes, reranker, top_k=25, min_score_threshold=0.5):
         log_message(f"Ошибка переранжировки: {str(e)}")
         return nodes[:top_k]
 
-def extract_weld_type_from_query(query):
-    """Extract welded joint type (С-XX, У-XX, etc.) from query"""
-    import re
-    
-    # Pattern for Russian weld types: С-25, У-12, Т-5, etc.
-    patterns = [
-        r'[СУТ]-\d+',  # Matches С-25, У-12, Т-5
-        r'(?:тип|тип[а-я]*)\s+([СУТ]-\d+)',  # "тип С-25" or "тип: С-25"
-    ]
-    
-    for pattern in patterns:
-        match = re.search(pattern, query, re.IGNORECASE)
-        if match:
-            if '-' in match.group(0):
-                return match.group(0).upper()
-            elif len(match.groups()) > 0:
-                return match.group(1).upper()
-    
-    return None
-
-
-def retrieve_nodes_with_weld_type_priority(query, vector_index, hybrid_retriever, reranker, top_k=20):
-    """
-    Enhanced retrieval that prioritizes welded joint type matches
-    """
-    from utils import deduplicate_nodes
-    
-    log_message(f"Enhanced retrieval for query: {query}")
-    
-    # Step 1: Try to extract weld type from query
-    weld_type = extract_weld_type_from_query(query)
-    
-    if weld_type:
-        log_message(f"Detected weld type in query: {weld_type}")
-        
-        # Step 2: Direct lookup in docstore for this weld type
-        direct_matches = []
-        all_nodes = list(vector_index.docstore.docs.values())
-        
-        for node in all_nodes:
-            metadata = node.metadata if hasattr(node, 'metadata') else {}
-            
-            # Check if this is a table node with matching weld type
-            if metadata.get('type') == 'table':
-                table_num = metadata.get('table_number', '')
-                table_title = metadata.get('table_title', '')
-                
-                # Check multiple fields for the weld type
-                if (weld_type in str(table_num) or 
-                    weld_type in str(table_title) or 
-                    weld_type in str(metadata.get('section', ''))):
-                    direct_matches.append(node)
-                    log_message(f"  Direct match found: {metadata.get('document_id')} - {table_title}")
-        
-        if direct_matches:
-            # Remove duplicates
-            direct_matches = deduplicate_nodes(direct_matches)
-            log_message(f"Found {len(direct_matches)} direct matches for {weld_type}")
-            
-            # Add some context from hybrid retriever
-            hybrid_results = hybrid_retriever.retrieve(query)
-            
-            # Combine: prioritize direct matches, supplement with hybrid results
-            combined = direct_matches + hybrid_results
-            combined = deduplicate_nodes(combined)
-            
-            # Rerank combined results
-            reranked = rerank_nodes(query, combined, reranker, top_k=top_k)
-            log_message(f"Combined retrieval: {len(direct_matches)} direct + hybrid, returning {len(reranked)} reranked")
-            
-            return reranked
-    
-    # Step 3: Fall back to normal hybrid retrieval if no weld type found
-    log_message("No weld type detected, using standard hybrid retrieval")
-    retrieved_nodes = hybrid_retriever.retrieve(query)
-    retrieved_nodes = deduplicate_nodes(retrieved_nodes)
-    reranked_nodes = rerank_nodes(query, retrieved_nodes, reranker, top_k=top_k)
-    
-    return reranked_nodes
-
-
-# Update create_query_engine to use the enhanced retrieval
 def create_query_engine(vector_index):
     try:
         from config import CUSTOM_PROMPT
-        from llama_index.core.prompts import PromptTemplate
-        from llama_index.core.response_synthesizers import get_response_synthesizer, ResponseMode
-        from llama_index.core.query_engine import RetrieverQueryEngine
-        from llama_index.retrievers.bm25 import BM25Retriever
-        from llama_index.core.retrievers import QueryFusionRetriever, VectorIndexRetriever
         
         bm25_retriever = BM25Retriever.from_defaults(
             docstore=vector_index.docstore, 
@@ -179,29 +92,14 @@ def create_query_engine(vector_index):
             text_qa_template=custom_prompt_template
         )
         
-        # Create custom query engine with enhanced retrieval
-        class EnhancedRetrieverQueryEngine(RetrieverQueryEngine):
-            def __init__(self, retriever, response_synthesizer, vector_index, reranker):
-                super().__init__(retriever=retriever, response_synthesizer=response_synthesizer)
-                self.vector_index = vector_index
-                self.reranker = reranker
-            
-            def retrieve(self, query):
-                """Override retrieve to use enhanced weld-type-aware retrieval"""
-                return retrieve_nodes_with_weld_type_priority(
-                    query, self.vector_index, self.retriever, self.reranker, top_k=20
-                )
-        
-        query_engine = EnhancedRetrieverQueryEngine(
+        query_engine = RetrieverQueryEngine(
             retriever=hybrid_retriever,
-            response_synthesizer=response_synthesizer,
-            vector_index=vector_index,
-            reranker=None  # Will be passed in later
+            response_synthesizer=response_synthesizer
         )
         
-        log_message("Enhanced query engine created with weld-type prioritization")
+        log_message("Query engine успешно создан")
         return query_engine
         
     except Exception as e:
-        log_message(f"Error creating enhanced query engine: {str(e)}")
+        log_message(f"Ошибка создания query engine: {str(e)}")
         raise

utils.py

@@ -195,42 +195,43 @@ def debug_search_tables(vector_index, search_term="С-25"):
     
     return matching
 
+# Add this import at the top of utils.py
+from documents_prep import normalize_text
+
+# MODIFIED: Update answer_question function
 def answer_question(question, query_engine, reranker, current_model, chunks_df=None):
+    # NORMALIZE the question to convert C to С
+    normalized_question = normalize_text(question)
+    
     if query_engine is None:
         return "<div style='background-color: #e53e3e; color: white; padding: 20px; border-radius: 10px;'>Система не инициализирована</div>", "", ""
     
     try:
         start_time = time.time()
-        
-        # Use enhanced retrieval if available
-        if hasattr(query_engine, 'retrieve'):
-            # Use the enhanced retrieval that's aware of weld types
-            retrieved_nodes = query_engine.retrieve(question)
-        else:
-            # Fallback to standard retrieval
-            retrieved_nodes = query_engine.retriever.retrieve(question)
-        
+        # Use NORMALIZED question for retrieval
+        retrieved_nodes = query_engine.retriever.retrieve(normalized_question)
         log_message(f"user query: {question}")
+        log_message(f"normalized query: {normalized_question}")
+
+        
         log_message(f"RETRIEVED: {len(retrieved_nodes)} nodes")
         
         unique_retrieved = deduplicate_nodes(retrieved_nodes)
+
+        # DEBUG: Log what was retrieved
         log_message(f"RETRIEVED: unique {len(unique_retrieved)} nodes")
-        
-        for i, node in enumerate(unique_retrieved[:15]):  # Log first 15
+        for i, node in enumerate(unique_retrieved):  # All debug
             table_num = node.metadata.get('table_number', 'N/A')
             table_title = node.metadata.get('table_title', 'N/A')
             doc_id = node.metadata.get('document_id', 'N/A')
             log_message(f"  [{i+1}] {doc_id} - Table {table_num}: {table_title[:50]}")
+        log_message(f"UNIQUE NODES: {len(unique_retrieved)} nodes")
         
-        # Rerank only if we have nodes
-        if unique_retrieved:
-            reranked_nodes = rerank_nodes(question, unique_retrieved, reranker, top_k=20)
-        else:
-            reranked_nodes = []
-            log_message("WARNING: No nodes to rerank!")
+        # Simple reranking with NORMALIZED question
+        reranked_nodes = rerank_nodes(normalized_question, unique_retrieved, reranker, top_k=20)
         
-        # Direct query without formatting
-        response = query_engine.query(question)
+        # Direct query without formatting - use normalized question
+        response = query_engine.query(normalized_question)
         
         end_time = time.time()
         processing_time = end_time - start_time
@@ -269,7 +270,5 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
         
     except Exception as e:
         log_message(f"Ошибка: {str(e)}")
-        import traceback
-        log_message(traceback.format_exc())
         error_msg = f"<div style='background-color: #e53e3e; color: white; padding: 20px; border-radius: 10px;'>Ошибка: {str(e)}</div>"
         return error_msg, "", ""