added the new llm query expanding

config.py

@@ -54,6 +54,18 @@ CHUNK_OVERLAP = 128
 MAX_CHARS_TABLE = 2000
 MAX_ROWS_TABLE = 30
 
+
+QUERY_EXPANSION_PROMPT = """Ты помощник для расширения поисковых запросов. 
+
+Пользователь задал вопрос: "{original_query}"
+
+Сгенерируй 5 альтернативных формулировок этого же вопроса, которые помогут найти релевантную информацию в технической документации. Формулировки должны быть:
+- Близкими по смыслу к оригинальному вопросу
+- Использовать технические термины и синонимы
+- Быть краткими (5-10 слов)
+
+Верни ТОЛЬКО 5 вопросов, каждый с новой строки, без нумерации и объяснений."""
+
 CUSTOM_PROMPT = """
 Вы являетесь высокоспециализированным Ассистентом для анализа нормативных документов (AIEXP). Ваша цель - предоставлять точные, корректные и контекстно релевантные ответы исключительно на основе предоставленного контекста из нормативной документации.
 СТРОГО ОТВЕТИТЬ ТОЛЬКО НА РУССКОМ!

documents_prep.py

@@ -25,12 +25,6 @@ def normalize_text(text):
 import re
 
 def normalize_steel_designations(text):
-    """
-    Normalize steel designations by converting Cyrillic letters to Latin.
-    This improves search/retrieval since embedding models work better with Latin.
-    Handles patterns like 08Х18Н10Т → 08X18H10T
-    Returns: (normalized_text, changes_count, changes_list)
-    """
     if not text:
         return text, 0, []
 

utils.py

@@ -197,48 +197,71 @@ def debug_search_tables(vector_index, search_term="С-25"):
 
 from documents_prep import normalize_text, normalize_steel_designations
 
-def enhance_query_for_steel_grades(query):
-    """Expand query with related terms for better steel grade retrieval"""
-    import re
-    
-    # Detect if query contains steel grades
-    steel_pattern = r'\b\d{1,3}[XHТCВKMAPХНТСВКМАР]\d*[XHТCВKMAPХНТСВКМАР\d]*\b'
-    matches = re.findall(steel_pattern, query, re.IGNORECASE)
-    
-    if matches:
-        # Add contextual terms
-        enhanced = query + " стандарт материал марка стали применение"
-        log_message(f"Enhanced query with steel context: {enhanced}")
-        return enhanced
-    
-    return query
+def expand_query_with_llm(query, llm_model):
+    """Generate 5 alternative query formulations using LLM"""
+    try:
+        from config import QUERY_EXPANSION_PROMPT
+        
+        expansion_prompt = QUERY_EXPANSION_PROMPT.format(original_query=query)
+        
+        log_message(f"Generating query variations for: {query}")
+        response = llm_model.complete(expansion_prompt)
+        
+        # Parse response - split by newlines and filter empty
+        variations = [line.strip() for line in response.text.split('\n') if line.strip()]
+        variations = variations[:5]  # Take only first 5
+        
+        if variations:
+            log_message(f"Generated {len(variations)} query variations:")
+            for i, var in enumerate(variations, 1):
+                log_message(f"  {i}. {var}")
+            
+            # Combine original + variations
+            combined_query = query + " " + " ".join(variations)
+            return combined_query
+        else:
+            log_message("No variations generated, using original query")
+            return query
+            
+    except Exception as e:
+        log_message(f"Error generating query variations: {e}")
+        return query
 
 
 def answer_question(question, query_engine, reranker, current_model, chunks_df=None, rerank_top_k=20):
     
+    # Apply normalizations
     normalized_question = normalize_text(question)
-    normalized_question_2, query_changes, change_list = normalize_steel_designations(question)  # FIX: 3 values
-    normalized_question_2 = enhance_query_for_steel_grades(normalized_question_2)
+    normalized_question_2, query_changes, change_list = normalize_steel_designations(normalized_question)
+    
     if change_list:
-        log_message(f"Query changes: {', '.join(change_list)}")    
+        log_message(f"Query changes: {', '.join(change_list)}")
+    
     if query_engine is None:
         return "<div style='background-color: #e53e3e; color: white; padding: 20px; border-radius: 10px;'>Система не инициализирована</div>", "", ""
     
     try:
         start_time = time.time()
-        retrieved_nodes = query_engine.retriever.retrieve(normalized_question_2)
+        
+        # EXPAND QUERY USING LLM
+        from utils import get_llm_model
+        llm = get_llm_model(current_model)
+        expanded_query = expand_query_with_llm(normalized_question_2, llm)
+        
+        # Use expanded query for retrieval
+        retrieved_nodes = query_engine.retriever.retrieve(expanded_query)
+        
         log_message(f"user query: {question}")
         log_message(f"normalized query: {normalized_question}")
         log_message(f"after steel normalization: {normalized_question_2}")
+        log_message(f"expanded query length: {len(expanded_query)} chars")
         log_message(f"Steel grades normalized in query: {query_changes}")
-
         
         log_message(f"RETRIEVED: {len(retrieved_nodes)} nodes")
         
         unique_retrieved = deduplicate_nodes(retrieved_nodes)
-
-        # IMPROVED DEBUG: Log what was actually retrieved with FULL metadata
         log_message(f"RETRIEVED: unique {len(unique_retrieved)} nodes")
+        
         for i, node in enumerate(unique_retrieved):
             node_type = node.metadata.get('type', 'text')
             doc_id = node.metadata.get('document_id', 'N/A')
@@ -247,7 +270,6 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
                 table_num = node.metadata.get('table_number', 'N/A')
                 table_id = node.metadata.get('table_identifier', 'N/A')
                 table_title = node.metadata.get('table_title', 'N/A')
-                # Show first 200 chars of content to verify it's the right table
                 content_preview = node.text[:200].replace('\n', ' ')
                 log_message(f"  [{i+1}] {doc_id} - Table {table_num} | ID: {table_id}")
                 log_message(f"      Title: {table_title[:80]}")
@@ -258,12 +280,11 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
         
         log_message(f"UNIQUE NODES: {len(unique_retrieved)} nodes")
         
-        # Simple reranking with NORMALIZED question and PARAMETERIZED top_k
-        reranked_nodes = rerank_nodes(normalized_question, unique_retrieved, reranker, 
-                                     top_k=rerank_top_k)  # NOW PARAMETERIZED
+        reranked_nodes = rerank_nodes(normalized_question_2, unique_retrieved, reranker, 
+                                     top_k=rerank_top_k)
         
-        # Direct query without formatting - use normalized question
-        response = query_engine.query(normalized_question)
+        # Use ORIGINAL normalized question for final answer generation
+        response = query_engine.query(normalized_question_2)
         
         end_time = time.time()
         processing_time = end_time - start_time