big debug change

documents_prep.py

@@ -491,8 +491,6 @@ def load_image_documents(repo_id, hf_token, image_dir):
         log_message(f"✓ Loaded {len(documents)} images (avg size: {avg_size:.0f} chars)")
     
     return documents
-
-
 def load_all_documents(repo_id, hf_token, json_dir, table_dir, image_dir):
     """Main loader - combines all document types"""
     log_message("="*60)
@@ -506,6 +504,19 @@ def load_all_documents(repo_id, hf_token, json_dir, table_dir, image_dir):
     # Load tables (already chunked)
     table_chunks = load_table_documents(repo_id, hf_token, table_dir)
     
+    # NEW: Analyze connection types in tables
+    connection_types = {}
+    for chunk in table_chunks:
+        conn_type = chunk.metadata.get('connection_type', '')
+        if conn_type:
+            connection_types[conn_type] = connection_types.get(conn_type, 0) + 1
+    
+    log_message("="*60)
+    log_message("CONNECTION TYPES FOUND IN TABLES:")
+    for conn_type, count in sorted(connection_types.items()):
+        log_message(f"  {conn_type}: {count} chunks")
+    log_message("="*60)
+    
     # Load images (no chunking needed)
     image_docs = load_image_documents(repo_id, hf_token, image_dir)
     

index_retriever.py

@@ -10,6 +10,30 @@ from config import CUSTOM_PROMPT, PROMPT_SIMPLE_POISK
 
 def create_vector_index(documents):
     log_message("Строю векторный индекс")
+    
+    # NEW: Analyze connection types before indexing
+    connection_types = {}
+    table_count = 0
+    for doc in documents:
+        if doc.metadata.get('type') == 'table':
+            table_count += 1
+            conn_type = doc.metadata.get('connection_type', '')
+            if conn_type:
+                connection_types[conn_type] = connection_types.get(conn_type, 0) + 1
+    
+    log_message("="*60)
+    log_message(f"INDEXING {table_count} TABLE CHUNKS")
+    log_message("CONNECTION TYPES IN INDEX:")
+    for conn_type, count in sorted(connection_types.items()):
+        log_message(f"  {conn_type}: {count} chunks")
+    
+    # Check for С-25 specifically
+    if 'С-25' in connection_types:
+        log_message(f"✓ С-25 FOUND: {connection_types['С-25']} chunks")
+    else:
+        log_message("✗ С-25 NOT FOUND IN INDEX!")
+    log_message("="*60)
+    
     return VectorStoreIndex.from_documents(documents)
 
 def rerank_nodes(query, nodes, reranker, top_k=25, min_score_threshold=0.5):
@@ -44,66 +68,20 @@ def create_query_engine(vector_index):
     try:
         from config import CUSTOM_PROMPT
         
-        # Preprocess query to expand table number patterns
-        class TableAwareRetriever:
-            def __init__(self, base_retriever):
-                self.base_retriever = base_retriever
-            
-            def retrieve(self, query_str):
-                import re
-                
-                # Expand queries with table numbers
-                queries = [query_str]
-                
-                # Extract table numbers like С-25, C-25, С25
-                table_patterns = re.findall(r'[СCс]-?\s*\d+', query_str)
-                if table_patterns:
-                    for pattern in table_patterns:
-                        # Normalize: "С-25" -> ["С-25", "C-25", "С25", "C25"]
-                        normalized = pattern.upper().replace(' ', '')
-                        variants = [
-                            normalized,
-                            normalized.replace('С', 'C'),
-                            normalized.replace('-', ''),
-                            normalized.replace('С', 'C').replace('-', '')
-                        ]
-                        for variant in variants:
-                            queries.append(f"тип соединения {variant}")
-                            queries.append(f"таблица {variant}")
-                
-                log_message(f"Searching with {len(queries)} query variants: {queries[:3]}...")
-                
-                # Retrieve with all variants
-                all_nodes = []
-                seen_ids = set()
-                
-                for q in queries:
-                    nodes = self.base_retriever.retrieve(q)
-                    for node in nodes:
-                        node_id = id(node)
-                        if node_id not in seen_ids:
-                            seen_ids.add(node_id)
-                            all_nodes.append(node)
-                
-                return all_nodes
-        
         bm25_retriever = BM25Retriever.from_defaults(
             docstore=vector_index.docstore, 
-            similarity_top_k=100
+            similarity_top_k=200 
         )
         
         vector_retriever = VectorIndexRetriever(
             index=vector_index, 
-            similarity_top_k=50,
-            similarity_cutoff=0.3  # Lower threshold
+            similarity_top_k=200,
+            similarity_cutoff=0.15
         )
         
-        # Wrap retrievers with table-aware logic
-        table_aware_bm25 = TableAwareRetriever(bm25_retriever)
-        
         hybrid_retriever = QueryFusionRetriever(
-            [vector_retriever, table_aware_bm25],
-            similarity_top_k=200,  # Increase to capture more candidates
+            [vector_retriever, bm25_retriever],
+            similarity_top_k=150,
             num_queries=1
         )
         
@@ -120,6 +98,7 @@ def create_query_engine(vector_index):
         
         log_message("Query engine успешно создан")
         return query_engine
+        
     except Exception as e:
         log_message(f"Ошибка создания query engine: {str(e)}")
         raise

utils.py

@@ -181,24 +181,45 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
         start_time = time.time()
         retrieved_nodes = query_engine.retriever.retrieve(question)
         log_message(f"user query: {question}")
-
-        
         log_message(f"RETRIEVED: {len(retrieved_nodes)} nodes")
         
         unique_retrieved = deduplicate_nodes(retrieved_nodes)
-
-        # DEBUG: Log what was retrieved
-        log_message(f"RETRIEVED: unique {len(unique_retrieved)} nodes")
-        for i, node in enumerate(unique_retrieved):  # All debug
-            table_num = node.metadata.get('table_number', 'N/A')
-            table_title = node.metadata.get('table_title', 'N/A')
-            doc_id = node.metadata.get('document_id', 'N/A')
-            log_message(f"  [{i+1}] {doc_id} - Table {table_num}: {table_title[:50]}")
         log_message(f"UNIQUE NODES: {len(unique_retrieved)} nodes")
         
-        # Simple reranking
+        # NEW: Check for connection types in retrieved nodes
+        conn_types_retrieved = {}
+        for node in unique_retrieved:
+            if node.metadata.get('type') == 'table':
+                conn_type = node.metadata.get('connection_type', '')
+                if conn_type:
+                    conn_types_retrieved[conn_type] = conn_types_retrieved.get(conn_type, 0) + 1
+        
+        if conn_types_retrieved:
+            log_message("CONNECTION TYPES IN RETRIEVED:")
+            for ct, cnt in sorted(conn_types_retrieved.items()):
+                log_message(f"  {ct}: {cnt} chunks")
+        
+        # Check if С-25 was retrieved
+        if 'С-25' in question:
+            if 'С-25' in conn_types_retrieved:
+                log_message(f"✓ С-25 RETRIEVED: {conn_types_retrieved['С-25']} chunks")
+            else:
+                log_message("✗ С-25 NOT RETRIEVED despite being in query!")
+        
+        # Log sample of retrieved tables
+        log_message("SAMPLE OF RETRIEVED TABLES:")
+        for i, node in enumerate(unique_retrieved[:10]):
+            if node.metadata.get('type') == 'table':
+                table_num = node.metadata.get('table_number', 'N/A')
+                table_title = node.metadata.get('table_title', 'N/A')
+                conn_type = node.metadata.get('connection_type', 'N/A')
+                doc_id = node.metadata.get('document_id', 'N/A')
+                log_message(f"  [{i+1}] {doc_id} - Table {table_num} - Type: {conn_type}")
+        
+        # Rerank
         reranked_nodes = rerank_nodes(question, unique_retrieved, reranker, top_k=20)
         
+        
         # Direct query without formatting
         response = query_engine.query(question)