removed normalization doc id

utils.py

@@ -39,62 +39,15 @@ def format_sources(nodes):
 
 import re
 
-def extract_document_id(query):
-    """Extract explicit document IDs from query"""
-    patterns = [
-        r'ГОСТ\s*Р?\s*[\d.-]+(?:-\d{4})?',  # ГОСТ 59023.4 or ГОСТ Р 59023.5-2020
-        r'НП-\d+-\d+',
-        r'МУ[_\s][\d.]+',
-    ]
-    
-    for pattern in patterns:
-        match = re.search(pattern, query, re.IGNORECASE)
-        if match:
-            return match.group(0).strip()
-    return None
-
-def normalize_doc_id(doc_id):
-    normalized = doc_id.replace(' ', '').replace('р', '').replace('Р', '').lower()
-    normalized = re.sub(r'-\d{4}$', '', normalized)
-    normalized = normalized.replace('.', '')  # Remove dots for flexible matching
-    return normalized
-
 def answer_question(question, query_engine, reranker):
     try:
         log_message(f"\n{'='*70}")
         log_message(f"QUERY: {question}")
 
-        target_doc_id = extract_document_id(question)
-        found_docs = set(normalize_doc_id(node.metadata.get('document_id', 'unknown')) for node in query_engine.retrieve(question))
-        log_message(f"NORMALIZED DOCS IN RETRIEVED: {', '.join(list(found_docs))}")
-        if target_doc_id:
-            log_message(f"TARGET DOCUMENT: {target_doc_id}")
-
         retrieved = query_engine.retrieve(question)
         log_message(f"RETRIEVED: {len(retrieved)} unique nodes")
-        
-        if target_doc_id:
-            target_normalized = normalize_doc_id(target_doc_id)
-            filtered = [
-                node for node in retrieved 
-                if target_normalized in normalize_doc_id(node.metadata.get('document_id', ''))
-            ]
-            
-            log_message(f"FILTERED TO TARGET DOC: {len(filtered)} nodes")
-            
-            # Debug: show what document IDs were found
-            if not filtered and len(retrieved) > 0:
-                found_docs = set(node.metadata.get('document_id', 'unknown') for node in retrieved[:10])
-                log_message(f"AVAILABLE DOCS (sample): {', '.join(list(found_docs)[:5])}")
-            
-            if not filtered:
-                log_message(f"WARNING: No nodes found for {target_doc_id}")
-                return f"В базе данных не найдены таблицы из документа {target_doc_id}.", ""
-            
-            retrieved = filtered
-        
-        # Rest stays the same...
-        reranked = rerank_nodes(question, retrieved, reranker, top_k=20, min_score=0.25)  
+
+        reranked = rerank_nodes(question, retrieved, reranker, top_k=20, min_score=0.3)
         log_message(f"RERANKED: {len(reranked)} nodes")
 
         context_parts = []
@@ -102,7 +55,6 @@ def answer_question(question, query_engine, reranker):
             meta = n.metadata
             doc_id = meta.get('document_id', 'unknown')
             doc_type = meta.get('type', 'text')
-            
             if doc_type == 'table':
                 table_id = meta.get('table_identifier', meta.get('table_number', 'unknown'))
                 title = meta.get('table_title', '')
@@ -111,7 +63,6 @@ def answer_question(question, query_engine, reranker):
                     source_label += f" {title}"
             else:
                 source_label = f"[{doc_id}]"
-            
             context_parts.append(f"{source_label}\n{n.text}")
 
         context = "\n\n" + ("="*50 + "\n\n").join(context_parts)
@@ -119,24 +70,11 @@ def answer_question(question, query_engine, reranker):
         from config import CUSTOM_PROMPT
         prompt = CUSTOM_PROMPT.format(context_str=context, query_str=question)
         log_message(f"PROMPT LENGTH: {len(prompt)} chars")
-        
+
         from llama_index.core import Settings
         response = Settings.llm.complete(prompt)
 
         sources = format_sources(reranked)
-        
-        log_message(f"\n{'='*70}")
-        log_message("RETRIEVED CHUNKS:")
-        for i, node in enumerate(reranked, 1):
-            log_message(f"\n--- Chunk {i} ---")
-            log_message(f"Document: {node.metadata.get('document_id')}")
-            log_message(f"Type: {node.metadata.get('type')}")
-            if node.metadata.get('type') == 'table':
-                table_id = node.metadata.get('table_identifier')
-                rows = f"{node.metadata.get('row_start', 0)}-{node.metadata.get('row_end', 0)}"
-                log_message(f"Table: {table_id} (rows {rows})")
-            log_message(f"Text: {node.text[:300]}...")
-        
         return response.text, sources
 
     except Exception as e: