added the load_table_data function

documents_prep.py

@@ -391,7 +391,6 @@ def load_image_data(repo_id, hf_token, image_data_dir):
         log_message(f"Ошибка загрузки данных изображений: {str(e)}")
         return []
 
-
 def load_table_data(repo_id, hf_token, table_data_dir):
     """Load and process table data from HuggingFace repo"""
     log_message("=" * 60)
@@ -431,7 +430,13 @@ def load_table_data(repo_id, hf_token, table_data_dir):
                     table_data = json.load(f)
                     
                     if isinstance(table_data, dict):
-                        document_id = table_data.get('document', 'unknown')
+                        # FIXED: Properly extract document_id from multiple possible sources
+                        document_id = (
+                            table_data.get('document_id') or 
+                            table_data.get('document') or 
+                            table_data.get('Обозначение документа') or
+                            'unknown'
+                        )
                         
                         # Handle multiple sheets
                         if 'sheets' in table_data:
@@ -441,8 +446,13 @@ def load_table_data(repo_id, hf_token, table_data_dir):
                             )
                             
                             for sheet in sorted_sheets:
-                                sheet['document'] = document_id
-                                docs_list = table_to_document(sheet, document_id)
+                                # FIXED: Ensure document_id is always set in sheet data
+                                if 'document' not in sheet and 'document_id' not in sheet:
+                                    sheet['document'] = document_id
+                                    sheet['document_id'] = document_id
+                                
+                                # FIXED: Pass document_id explicitly
+                                docs_list = table_to_document(sheet, document_id=document_id)
                                 table_documents.extend(docs_list)
                                 
                                 for doc in docs_list:
@@ -452,8 +462,13 @@ def load_table_data(repo_id, hf_token, table_data_dir):
                                     stats['by_document'][document_id]['count'] += 1
                                     stats['by_document'][document_id]['size'] += size
                         else:
-                            # Single table
-                            docs_list = table_to_document(table_data, document_id)
+                            # Single table - FIXED: Ensure document_id is in table_data
+                            if 'document_id' not in table_data:
+                                table_data['document_id'] = document_id
+                            if 'document' not in table_data:
+                                table_data['document'] = document_id
+                                
+                            docs_list = table_to_document(table_data, document_id=document_id)
                             table_documents.extend(docs_list)
                             
                             for doc in docs_list:
@@ -465,6 +480,8 @@ def load_table_data(repo_id, hf_token, table_data_dir):
                         
             except Exception as e:
                 log_message(f"❌ ОШИБКА файла {file_path}: {str(e)}")
+                import traceback
+                log_message(f"Traceback: {traceback.format_exc()}")
                 continue
         
         # Log summary
@@ -486,6 +503,8 @@ def load_table_data(repo_id, hf_token, table_data_dir):
         
     except Exception as e:
         log_message(f"❌ КРИТИЧЕСКАЯ ОШИБКА: {str(e)}")
+        import traceback
+        log_message(f"Traceback: {traceback.format_exc()}")
         return []
 
 def load_csv_chunks(repo_id, hf_token, chunks_filename, download_dir):

index_retriever.py

@@ -14,20 +14,21 @@ def create_vector_index(documents):
 
 def create_query_engine(vector_index):
     try:
+        # FIXED: Increase retrieval numbers for tables
         bm25_retriever = BM25Retriever.from_defaults(
             docstore=vector_index.docstore, 
-            similarity_top_k=30 
+            similarity_top_k=50  # Increased from 30
         )
         
         vector_retriever = VectorIndexRetriever(
             index=vector_index, 
-            similarity_top_k=30,  
-            similarity_cutoff=0.65
+            similarity_top_k=50,  # Increased from 30
+            similarity_cutoff=0.55  # FIXED: Lowered from 0.65 to catch more tables
         )
         
         hybrid_retriever = QueryFusionRetriever(
             [vector_retriever, bm25_retriever],
-            similarity_top_k=40,  
+            similarity_top_k=60,  # Increased from 40
             num_queries=1
         )
         
@@ -42,14 +43,15 @@ def create_query_engine(vector_index):
             response_synthesizer=response_synthesizer
         )
         
-        log_message("Query engine успешно создан")
+        log_message("Query engine успешно создан с улучшенными параметрами поиска таблиц")
         return query_engine
         
     except Exception as e:
         log_message(f"Ошибка создания query engine: {str(e)}")
         raise
 
-def rerank_nodes(query, nodes, reranker, top_k=25, min_score_threshold=0.5, diversity_penalty=0.3):
+    
+def rerank_nodes(query, nodes, reranker, top_k=30, min_score_threshold=0.45, diversity_penalty=0.2):  # FIXED: Adjusted defaults
     if not nodes or not reranker:
         return nodes[:top_k]
     
@@ -62,6 +64,7 @@ def rerank_nodes(query, nodes, reranker, top_k=25, min_score_threshold=0.5, dive
         
         scored_nodes.sort(key=lambda x: x[1], reverse=True)
         
+        # FIXED: Lower threshold and add special handling for tables
         if min_score_threshold is not None:
             scored_nodes = [(node, score) for node, score in scored_nodes 
                           if score >= min_score_threshold]
@@ -71,13 +74,14 @@ def rerank_nodes(query, nodes, reranker, top_k=25, min_score_threshold=0.5, dive
             log_message("Нет узлов после фильтрации, снижаю порог")
             scored_nodes = list(zip(nodes, scores))
             scored_nodes.sort(key=lambda x: x[1], reverse=True)
-            min_score_threshold = scored_nodes[0][1] * 0.6  
+            min_score_threshold = scored_nodes[0][1] * 0.5  # FIXED: Lower threshold
             scored_nodes = [(node, score) for node, score in scored_nodes 
                           if score >= min_score_threshold]
         
         selected_nodes = []
         selected_docs = set()
         selected_sections = set()
+        selected_tables = set()  # FIXED: Track tables separately
         
         for node, score in scored_nodes:
             if len(selected_nodes) >= top_k:
@@ -85,25 +89,36 @@ def rerank_nodes(query, nodes, reranker, top_k=25, min_score_threshold=0.5, dive
             
             metadata = node.metadata if hasattr(node, 'metadata') else {}
             doc_id = metadata.get('document_id', 'unknown')
+            node_type = metadata.get('type', 'text')
             section_key = f"{doc_id}_{metadata.get('section_path', metadata.get('section_id', ''))}"
+            table_key = f"{doc_id}_{metadata.get('table_number', '')}" if node_type == 'table' else None
             
-            # Apply diversity penalty
+            # FIXED: Lower diversity penalty for tables
             penalty = 0
-            if doc_id in selected_docs:
-                penalty += diversity_penalty * 0.5
-            if section_key in selected_sections:
-                penalty += diversity_penalty
+            if node_type == 'table':
+                # Tables get less penalty - we want multiple tables from same document
+                if table_key and table_key in selected_tables:
+                    penalty += diversity_penalty * 0.3
+                else:
+                    penalty += diversity_penalty * 0.1 if doc_id in selected_docs else 0
+            else:
+                if doc_id in selected_docs:
+                    penalty += diversity_penalty * 0.5
+                if section_key in selected_sections:
+                    penalty += diversity_penalty
             
             adjusted_score = score * (1 - penalty)
             
-            # Add if still competitive
-            if not selected_nodes or adjusted_score >= selected_nodes[0][1] * 0.6:
+            # FIXED: More lenient threshold for adding nodes
+            if not selected_nodes or adjusted_score >= selected_nodes[0][1] * 0.5:
                 selected_nodes.append((node, score))
                 selected_docs.add(doc_id)
                 selected_sections.add(section_key)
+                if table_key:
+                    selected_tables.add(table_key)
         
         log_message(f"Выбрано {len(selected_nodes)} узлов с разнообразием")
-        log_message(f"Уникальных документов: {len(selected_docs)}, секций: {len(selected_sections)}")
+        log_message(f"Уникальных документов: {len(selected_docs)}, секций: {len(selected_sections)}, таблиц: {len(selected_tables)}")
         
         if selected_nodes:
             log_message(f"Score range: {selected_nodes[0][1]:.3f} to {selected_nodes[-1][1]:.3f}")

table_prep.py

@@ -5,28 +5,42 @@ from my_logging import log_message
 
 def create_table_content(table_data):
     """Create formatted content from table data"""
-    doc_id = table_data.get('document_id', table_data.get('document', 'Неизвестно'))
+    # FIXED: More robust field extraction
+    doc_id = (
+        table_data.get('document_id') or 
+        table_data.get('document') or 
+        table_data.get('Обозначение документа') or
+        'Неизвестно'
+    )
     table_num = table_data.get('table_number', 'Неизвестно')
     table_title = table_data.get('table_title', 'Неизвестно')
-    section = table_data.get('section', 'Неизвестно')
+    section = (
+        table_data.get('section') or 
+        table_data.get('Раздел документа') or
+        'Неизвестно'
+    )
     
-    # Header section
-    content = f"Таблица: {table_num}\n"
-    content += f"Название: {table_title}\n"
-    content += f"Документ: {doc_id}\n"
-    content += f"Раздел: {section}\n"
+    # FIXED: Add more context in content for better semantic search
+    content = f"Документ: {doc_id}\n"
+    content += f"Таблица: {table_num}\n"
+    content += f"Название таблицы: {table_title}\n"
+    content += f"Раздел документа: {section}\n"
+    content += f"Стандарт/ГОСТ: {doc_id}\n"  # Explicitly mention GOST for queries
     
     headers = table_data.get('headers', [])
     if headers:
-        content += f"\nЗаголовки: {' | '.join(headers)}\n"
+        content += f"\nЗаголовки колонок: {' | '.join(str(h) for h in headers)}\n"
     
     # Data section
     if 'data' in table_data and isinstance(table_data['data'], list):
-        content += "\nДанные таблицы:\n"
+        content += "\nСодержимое таблицы:\n"
         for row_idx, row in enumerate(table_data['data'], start=1):
             if isinstance(row, dict):
                 row_text = " | ".join([f"{k}: {v}" for k, v in row.items() if v])
                 content += f"Строка {row_idx}: {row_text}\n"
+            elif isinstance(row, list):
+                row_text = " | ".join([str(v) for v in row if v])
+                content += f"Строка {row_idx}: {row_text}\n"
     
     return content
 
@@ -122,10 +136,25 @@ def table_to_document(table_data, document_id=None):
     if not isinstance(table_data, dict):
         return []
     
-    doc_id = document_id or table_data.get('document_id') or table_data.get('document', 'Неизвестно')
+    # FIXED: More robust document_id extraction with multiple fallbacks
+    doc_id = (
+        document_id or 
+        table_data.get('document_id') or 
+        table_data.get('document') or 
+        table_data.get('Обозначение документа') or
+        'Неизвестно'
+    )
+    
     table_num = table_data.get('table_number', 'Неизвестно')
     table_title = table_data.get('table_title', 'Неизвестно')
-    section = table_data.get('section', 'Неизвестно')
+    
+    # FIXED: More robust section extraction
+    section = (
+        table_data.get('section') or 
+        table_data.get('Раздел документа') or 
+        table_data.get('section_id') or
+        'Неизвестно'
+    )
     
     table_rows = table_data.get('data', [])
     if not table_rows:
@@ -135,17 +164,20 @@ def table_to_document(table_data, document_id=None):
     content = create_table_content(table_data)
     content_size = len(content)
     
+    # FIXED: Enhanced metadata with more searchable fields
     base_doc = Document(
         text=content,
         metadata={
             "type": "table",
-            "table_number": table_num,
-            "table_title": table_title,
-            "document_id": doc_id,
-            "section": section,
-            "section_id": section,
+            "table_number": str(table_num),
+            "table_title": str(table_title),
+            "document_id": str(doc_id),
+            "section": str(section),
+            "section_id": str(section),
             "total_rows": len(table_rows),
-            "content_size": content_size
+            "content_size": content_size,
+            # FIXED: Add searchable composite field for better retrieval
+            "search_key": f"{doc_id} {table_num} {table_title} {section}".lower()
         }
     )
     
@@ -154,5 +186,5 @@ def table_to_document(table_data, document_id=None):
         log_message(f"📊 CHUNKING: Таблица {table_num} | {content_size} > {CHUNK_SIZE}")
         return chunk_table_document(base_doc)
     else:
-        log_message(f"✓ Таблица {table_num} добавлена целиком ({content_size} символов)")
+        log_message(f"✓ Таблица {table_num} добавлена целиком ({content_size} символов, doc_id={doc_id})")
         return [base_doc]