added sheet_name

documents_prep.py

@@ -407,7 +407,7 @@ def load_image_data(repo_id, hf_token, image_data_dir):
         return []
 
 def load_table_data(repo_id, hf_token, table_data_dir):
-    """Load and process table data with sheet-level document_id extraction"""
+    """Load and process table data with complete metadata preservation"""
     log_message("=" * 60)
     log_message("НАЧАЛО ЗАГРУЗКИ ТАБЛИЧНЫХ ДАННЫХ")
     log_message("=" * 60)
@@ -426,7 +426,8 @@ def load_table_data(repo_id, hf_token, table_data_dir):
         stats = {
             'total_tables': 0,
             'total_size': 0,
-            'by_document': defaultdict(lambda: {'count': 0, 'size': 0})
+            'by_document': defaultdict(lambda: {'count': 0, 'size': 0}),
+            'by_sheet': defaultdict(int)
         }
         
         for file_path in table_files:
@@ -439,42 +440,43 @@ def load_table_data(repo_id, hf_token, table_data_dir):
                     token=hf_token
                 )
                 
-                log_message(f"\nОбработка файла: {file_path}")
+                log_message(f"\n📂 Обработка файла: {file_path}")
                 
                 with open(local_path, 'r', encoding='utf-8') as f:
                     table_data = json.load(f)
                     
                     if isinstance(table_data, dict):
-                        # Extract file-level document_id
                         file_level_doc_id = (
                             table_data.get('document_id') or 
                             table_data.get('document') or 
-                            table_data.get('Обозначение документа') or
                             'unknown'
                         )
                         
-                        # Handle multiple sheets
                         if 'sheets' in table_data:
                             sorted_sheets = sorted(
                                 table_data['sheets'],
                                 key=lambda sheet: sheet.get('table_number', '')
                             )
                             
+                            log_message(f"  Найдено листов: {len(sorted_sheets)}")
+                            
                             for sheet in sorted_sheets:
-                                # CRITICAL FIX: Use sheet-level document_id if available
-                                sheet_doc_id = (
-                                    sheet.get('document_id') or 
-                                    sheet.get('document') or 
-                                    sheet.get('Обозначение документа') or
-                                    file_level_doc_id
-                                )
+                                # CRITICAL: sheet_name MUST be present
+                                if 'sheet_name' not in sheet:
+                                    log_message(f"  ⚠️ Пропущен лист без sheet_name")
+                                    continue
+                                
+                                sheet_name = sheet['sheet_name']
+                                sheet_doc_id = sheet.get('document_id', file_level_doc_id)
                                 
-                                log_message(f"  Sheet doc_id: {sheet_doc_id} (file: {file_level_doc_id})")
+                                log_message(f"  → Лист: {sheet_name} | doc_id: {sheet_doc_id}")
                                 
-                                # Pass sheet's own document_id
+                                # Pass complete sheet data to table_to_document
                                 docs_list = table_to_document(sheet, document_id=sheet_doc_id)
                                 table_documents.extend(docs_list)
                                 
+                                stats['by_sheet'][sheet_name] += len(docs_list)
+                                
                                 for doc in docs_list:
                                     stats['total_tables'] += 1
                                     size = doc.metadata.get('content_size', 0)
@@ -482,7 +484,7 @@ def load_table_data(repo_id, hf_token, table_data_dir):
                                     stats['by_document'][sheet_doc_id]['count'] += 1
                                     stats['by_document'][sheet_doc_id]['size'] += size
                         else:
-                            # Single table
+                            # Single table (no sheets structure)
                             docs_list = table_to_document(table_data, document_id=file_level_doc_id)
                             table_documents.extend(docs_list)
                             
@@ -499,18 +501,23 @@ def load_table_data(repo_id, hf_token, table_data_dir):
                 log_message(f"Traceback: {traceback.format_exc()}")
                 continue
         
-        # Log summary
+        # Enhanced logging with sheet breakdown
         log_message("\n" + "=" * 60)
         log_message("СТАТИСТИКА ПО ТАБЛИЦАМ")
         log_message("=" * 60)
-        log_message(f"Всего таблиц: {stats['total_tables']}")
+        log_message(f"Всего таблиц/чанков: {stats['total_tables']}")
         log_message(f"Общий размер: {stats['total_size']:,} символов")
         if stats['total_tables'] > 0:
             log_message(f"Средний размер: {stats['total_size'] // stats['total_tables']:,} символов")
         
         log_message("\nПо документам:")
         for doc_id, doc_stats in sorted(stats['by_document'].items()):
-            log_message(f"  • {doc_id}: {doc_stats['count']} таблиц, {doc_stats['size']:,} символов")
+            log_message(f"  • {doc_id}: {doc_stats['count']} элементов, {doc_stats['size']:,} символов")
+        
+        log_message("\nПо листам (топ-20):")
+        top_sheets = sorted(stats['by_sheet'].items(), key=lambda x: x[1], reverse=True)[:20]
+        for sheet_name, count in top_sheets:
+            log_message(f"  • {sheet_name}: {count} чанков")
         
         log_message("=" * 60)