Update utils.py

utils.py

@@ -9,7 +9,6 @@ import time
 from index_retriever import rerank_nodes
 from my_logging import log_message
 from config import PROMPT_SIMPLE_POISK
-import re
 
 def get_llm_model(model_name):
     try:
@@ -173,14 +172,28 @@ def deduplicate_nodes(nodes):
     
     return unique_nodes
 
-def normalize_query(query):
-    def repl(m):
-        cyr_to_lat = {'С': 'C', 'с': 'C', 'Т': 'T', 'т': 'T', 'У': 'U', 'у': 'U'}
-        letter = cyr_to_lat.get(m.group(1), m.group(1))
-        return f"{letter}{m.group(2)}"
-
-    return re.sub(r'\b([СсТтУуCTU])[-\s]?(\d+)\b', repl, query)
-
+def debug_search_tables(vector_index, search_term="С-25"):
+    """Debug function to find all tables containing a specific term"""
+    all_nodes = list(vector_index.docstore.docs.values())
+    
+    matching = []
+    for node in all_nodes:
+        if node.metadata.get('type') == 'table':
+            text = node.get_content()
+            if search_term in text or search_term in node.metadata.get('table_title', ''):
+                matching.append({
+                    'doc_id': node.metadata.get('document_id'),
+                    'table_num': node.metadata.get('table_number'),
+                    'title': node.metadata.get('table_title', '')[:100]
+                })
+    
+    log_message(f"\n{'='*60}")
+    log_message(f"DEBUG: Found {len(matching)} tables containing '{search_term}'")
+    for m in matching:
+        log_message(f"  • {m['doc_id']} - Table {m['table_num']}: {m['title']}")
+    log_message(f"{'='*60}\n")
+    
+    return matching
 
 def answer_question(question, query_engine, reranker, current_model, chunks_df=None):
     if query_engine is None:
@@ -188,58 +201,28 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
     
     try:
         start_time = time.time()
-        
-        # NORMALIZE QUERY: Convert Cyrillic to Latin and remove hyphens
-        normalized_question = normalize_query(question)
-        log_message(f"Original query: {question}")
-        log_message(f"Normalized query: {normalized_question}")
-        
-        # Use normalized query for retrieval
         retrieved_nodes = query_engine.retriever.retrieve(question)
         log_message(f"user query: {question}")
+
         
         log_message(f"RETRIEVED: {len(retrieved_nodes)} nodes")
         
         unique_retrieved = deduplicate_nodes(retrieved_nodes)
+
+        # DEBUG: Log what was retrieved
+        log_message(f"RETRIEVED: unique {len(unique_retrieved)} nodes")
+        for i, node in enumerate(unique_retrieved):  # All debug
+            table_num = node.metadata.get('table_number', 'N/A')
+            table_title = node.metadata.get('table_title', 'N/A')
+            doc_id = node.metadata.get('document_id', 'N/A')
+            log_message(f"  [{i+1}] {doc_id} - Table {table_num}: {table_title[:50]}")
         log_message(f"UNIQUE NODES: {len(unique_retrieved)} nodes")
         
-        # Check for connection types
-        conn_types_retrieved = {}
-        for node in unique_retrieved:
-            if node.metadata.get('type') == 'table':
-                conn_type = node.metadata.get('connection_type', '')
-                if conn_type:
-                    conn_types_retrieved[conn_type] = conn_types_retrieved.get(conn_type, 0) + 1
-        
-        if conn_types_retrieved:
-            log_message("CONNECTION TYPES IN RETRIEVED:")
-            for ct, cnt in sorted(conn_types_retrieved.items()):
-                log_message(f"  {ct}: {cnt} chunks")
-        
-        # Check if target type was retrieved
-        # Normalize the check as well
-        normalized_check = normalize_query('С-25')  # Will become C25
-        if normalized_check in question or 'С-25' in question or 'C-25' in question:
-            if 'C25' in conn_types_retrieved:
-                log_message(f"✓ C25 RETRIEVED: {conn_types_retrieved['C25']} chunks")
-            else:
-                log_message("✗ C25 NOT RETRIEVED despite being in query!")
-        
-        # Sample of retrieved tables
-        log_message("SAMPLE OF RETRIEVED TABLES:")
-        for i, node in enumerate(unique_retrieved[:10]):
-            if node.metadata.get('type') == 'table':
-                table_num = node.metadata.get('table_number', 'N/A')
-                table_title = node.metadata.get('table_title', 'N/A')
-                conn_type = node.metadata.get('connection_type', 'N/A')
-                doc_id = node.metadata.get('document_id', 'N/A')
-                log_message(f"  [{i+1}] {doc_id} - Table {table_num} - Type: {conn_type}")
-        
-        # Rerank - use normalized query for consistency
-        reranked_nodes = rerank_nodes(normalized_question, unique_retrieved, reranker, top_k=20)
+        # Simple reranking
+        reranked_nodes = rerank_nodes(question, unique_retrieved, reranker, top_k=20)
         
-        # CRITICAL FIX: Use normalized query for LLM as well
-        response = query_engine.query(normalized_question)
+        # Direct query without formatting
+        response = query_engine.query(question)
         
         end_time = time.time()
         processing_time = end_time - start_time