simple

table_prep.py

@@ -5,6 +5,7 @@ from llama_index.core import Document
 from my_logging import log_message
 
 def create_table_content(table_data):
+    """Create formatted content from table data"""
     doc_id = table_data.get('document_id', table_data.get('document', 'Неизвестно'))
     table_num = table_data.get('table_number', 'Неизвестно')
     table_title = table_data.get('table_title', 'Неизвестно')
@@ -32,36 +33,32 @@ def table_to_document(table_data, document_id=None):
     """Convert table data to a single Document"""
     if not isinstance(table_data, dict):
         return []
-
+    
     doc_id = document_id or table_data.get('document_id', table_data.get('document', 'Неизвестно'))
     table_num = table_data.get('table_number', 'Неизвестно')
     table_title = table_data.get('table_title', 'Неизвестно')
     section = table_data.get('section', 'Неизвестно')
-    sheet_name = table_data.get('sheet_name', None)  # <-- add this
-
+    
     content = create_table_content(table_data)
     content_size = len(content)
-
+    
+    # Log table addition
     row_count = len(table_data.get('data', [])) if 'data' in table_data else 0
     log_message(f"✓ ДОБАВЛЕНА: Таблица {table_num} из документа '{doc_id}' | "
                 f"Размер: {content_size} символов | Строк: {row_count}")
-
-    metadata = {
-        "type": "table",
-        "table_number": table_num,
-        "table_title": table_title,
-        "document_id": doc_id,
-        "section": section,
-        "section_id": section,
-        "total_rows": row_count,
-        "content_size": content_size
-    }
-    if sheet_name:
-        metadata["sheet_name"] = sheet_name  
-
+    
     return [Document(
         text=content,
-        metadata=metadata
+        metadata={
+            "type": "table",
+            "table_number": table_num,
+            "table_title": table_title,
+            "document_id": doc_id,
+            "section": section,
+            "section_id": section,
+            "total_rows": row_count,
+            "content_size": content_size
+        }
     )]
 
 def load_table_data(repo_id, hf_token, table_data_dir):
@@ -152,66 +149,3 @@ def load_table_data(repo_id, hf_token, table_data_dir):
     except Exception as e:
         log_message(f"❌ КРИТИЧЕСКАЯ ОШИБКА загрузки табличных данных: {str(e)}")
         return []
-
-CUSTOM_TABLE_CONFIGS = {
-    "НП-105-18": {
-        "tables": {
-            "№ 4.8": {"method": "group_entire_table"}
-        }
-    },
-    "ГОСТ Р 50.05.23-2020": {
-        "tables": {
-            "№8": {"method": "group_entire_table"}
-        }
-    },
-    "ГОСТ Р 50.03.01-2017": {
-        "tables": {
-            "А.8": {"method": "group_entire_table"}
-        }
-    }
-}
-
-def create_meta_info(document_name, section, table_number, table_title):
-    meta_info = f"Документ: {document_name}\n"
-    meta_info += f"Раздел: {section}\n"
-    meta_info += f"Таблица: {table_number}\n"
-    meta_info += f"Название таблицы: {table_title}\n"
-    return meta_info
-
-def create_chunk_text(meta_info, headers, rows):
-
-    header_line = ", ".join(headers)
-    row_lines = ["; ".join(map(str, row)) for row in rows]
-    chunk = f"Meta: {meta_info}\nHeaders: {header_line}\nRows:\n" + "\n".join(row_lines)
-    return chunk
-
-def group_entire_table_method(table_data, document_name):
-    """Group entire table as one chunk"""
-    headers = table_data.get("headers", [])
-    rows = table_data.get("data", [])
-    section = table_data.get("section", "")
-    table_number = table_data.get("table_number", "")
-    table_title = table_data.get("table_title", "")
-    sheet_name = table_data.get("sheet_name", None) 
-
-    meta_info = create_meta_info(document_name, section, table_number, table_title)
-    chunk_text = create_chunk_text(meta_info, headers, rows)
-    metadata = {
-        "type": "table",
-        "table_number": table_number,
-        "table_title": table_title,
-        "document_id": document_name,
-        "section": section,
-        "section_id": section,
-        "total_rows": len(rows),
-        "processing_method": "group_entire_table"
-    }
-    if sheet_name:
-        metadata["sheet_name"] = sheet_name  
-
-    doc = Document(
-        text=chunk_text,
-        metadata=metadata
-    )
-    log_message(f"Grouped entire table {table_number}, rows: {len(rows)}, length: {len(chunk_text)}")
-    return [doc]