Update utils.py

utils.py

@@ -10,28 +10,6 @@ from index_retriever import rerank_nodes
 from my_logging import log_message
 from config import PROMPT_SIMPLE_POISK
 
-STEEL_GRADE_CONTEXT = {
-    # Austenitic stainless steels
-    "08X18H10T": "08X18H10T аустенитная нержавеющая сталь стандарт ГОСТ технические условия марка материал сварка применение",
-    "12X18H10T": "12X18H10T аустенитная нержавеющая сталь стандарт ГОСТ технические условия марка материал сварка применение",
-    "08X18H10": "08X18H10 аустенитная нержавеющая сталь стандарт ГОСТ технические условия марка материал сварка применение",
-    "12X18H9": "12X18H9 аустенитная нержавеющая сталь стандарт ГОСТ технические условия марка материал сварка применение",
-    "10X17H13M2T": "10X17H13M2T аустенитная нержавеющая сталь стандарт ГОСТ технические условия марка материал сварка применение",
-    
-    # Welding wires
-    "CB-08X19H10": "CB-08X19H10 сварочная проволока стандарт ГОСТ технические условия марка материал сварка применение",
-    "CB-08X18H10T": "CB-08X18H10T сварочная проволока стандарт ГОСТ технические условия марка материал сварка применение",
-  
-}
-
-
-
-
-# Generic context for any steel grade pattern
-GENERIC_STEEL_CONTEXT = "стандарт ГОСТ технические условия марка материал применение сварка"
-
-
-
 def get_llm_model(model_name):
     try:
         model_config = AVAILABLE_MODELS.get(model_name)
@@ -219,57 +197,43 @@ def debug_search_tables(vector_index, search_term="С-25"):
 
 from documents_prep import normalize_text, normalize_steel_designations
 
-def enhance_query_for_steel_grades(query):
-    import re
-    
-    # Detect steel grades in query
-    steel_pattern = r'\b\d{1,3}[XHТCВKMAPХНТСВКМАР]\d*[XHТCВKMAPХНТСВКМАР\d]*\b'
-    matches = re.findall(steel_pattern, query, re.IGNORECASE)
-    
-    if not matches:
-        return query
-    
-    # Collect context expansions
-    added_context = []
-    grades_found = []
-    
-    for match in matches:
-        match_upper = match.upper()
-        grades_found.append(match_upper)
+def expand_query_with_llm(query, llm_model):
+    """Generate 5 alternative query formulations using LLM"""
+    try:
+        from config import QUERY_EXPANSION_PROMPT
         
-        # Check if we have specific context for this grade
-        if match_upper in STEEL_GRADE_CONTEXT:
-            context = STEEL_GRADE_CONTEXT[match_upper]
-            added_context.append(context)
-            log_message(f"  Found specific context for {match_upper}")
-        else:
-            # Use generic context for unknown grades
-            added_context.append(GENERIC_STEEL_CONTEXT)
-            log_message(f"  Using generic context for {match_upper}")
-    
-    # Build enhanced query
-    if added_context:
-        # Remove duplicates from context
-        unique_context = ' '.join(set(' '.join(added_context).split()))
-        enhanced = f"{query} {unique_context}"
+        expansion_prompt = QUERY_EXPANSION_PROMPT.format(original_query=query)
         
-        log_message(f"Enhanced query for steel grades: {', '.join(grades_found)}")
-        log_message(f"Added context: {unique_context[:100]}...")
+        log_message(f"Generating query variations for: {query}")
+        response = llm_model.complete(expansion_prompt)
         
-        return enhanced
-    
-    return query
+        # Parse response - split by newlines and filter empty
+        variations = [line.strip() for line in response.text.split('\n') if line.strip()]
+        variations = variations[:5]  # Take only first 5
+        
+        if variations:
+            log_message(f"Generated {len(variations)} query variations:")
+            for i, var in enumerate(variations, 1):
+                log_message(f"  {i}. {var}")
+            
+            # Combine original + variations
+            combined_query = query + " " + " ".join(variations)
+            return combined_query
+        else:
+            log_message("No variations generated, using original query")
+            return query
+            
+    except Exception as e:
+        log_message(f"Error generating query variations: {e}")
+        return query
 
 
 def answer_question(question, query_engine, reranker, current_model, chunks_df=None, rerank_top_k=20):
     
-    # FIXED: Apply all normalizations in correct order
+    # Apply normalizations
     normalized_question = normalize_text(question)
     normalized_question_2, query_changes, change_list = normalize_steel_designations(normalized_question)
     
-    # FIX: Actually call enhance_query_for_steel_grades!
-    enhanced_query = enhance_query_for_steel_grades(normalized_question_2)
-    
     if change_list:
         log_message(f"Query changes: {', '.join(change_list)}")
     
@@ -279,22 +243,25 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
     try:
         start_time = time.time()
         
-        # FIX: Use enhanced_query instead of normalized_question_2
-        retrieved_nodes = query_engine.retriever.retrieve(enhanced_query)
+        # EXPAND QUERY USING LLM
+        from utils import get_llm_model
+        llm = get_llm_model(current_model)
+        expanded_query = expand_query_with_llm(normalized_question_2, llm)
+        
+        # Use expanded query for retrieval
+        retrieved_nodes = query_engine.retriever.retrieve(expanded_query)
         
         log_message(f"user query: {question}")
         log_message(f"normalized query: {normalized_question}")
         log_message(f"after steel normalization: {normalized_question_2}")
-        log_message(f"enhanced query: {enhanced_query}")  # NEW LOG
+        log_message(f"expanded query length: {len(expanded_query)} chars")
         log_message(f"Steel grades normalized in query: {query_changes}")
-
         
         log_message(f"RETRIEVED: {len(retrieved_nodes)} nodes")
         
         unique_retrieved = deduplicate_nodes(retrieved_nodes)
-
-        # IMPROVED DEBUG: Log what was actually retrieved with FULL metadata
         log_message(f"RETRIEVED: unique {len(unique_retrieved)} nodes")
+        
         for i, node in enumerate(unique_retrieved):
             node_type = node.metadata.get('type', 'text')
             doc_id = node.metadata.get('document_id', 'N/A')
@@ -303,7 +270,6 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
                 table_num = node.metadata.get('table_number', 'N/A')
                 table_id = node.metadata.get('table_identifier', 'N/A')
                 table_title = node.metadata.get('table_title', 'N/A')
-                # Show first 200 chars of content to verify it's the right table
                 content_preview = node.text[:200].replace('\n', ' ')
                 log_message(f"  [{i+1}] {doc_id} - Table {table_num} | ID: {table_id}")
                 log_message(f"      Title: {table_title[:80]}")
@@ -314,12 +280,11 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
         
         log_message(f"UNIQUE NODES: {len(unique_retrieved)} nodes")
         
-        # Simple reranking with NORMALIZED question and PARAMETERIZED top_k
-        reranked_nodes = rerank_nodes(enhanced_query, unique_retrieved, reranker, 
-                                     top_k=rerank_top_k)  # NOW PARAMETERIZED
+        reranked_nodes = rerank_nodes(normalized_question_2, unique_retrieved, reranker, 
+                                     top_k=rerank_top_k)
         
-        # Direct query without formatting - use normalized question
-        response = query_engine.query(enhanced_query)
+        # Use ORIGINAL normalized question for final answer generation
+        response = query_engine.query(normalized_question_2)
         
         end_time = time.time()
         processing_time = end_time - start_time