new documents prep

app.py

@@ -1,7 +1,7 @@
 import gradio as gr
 import os
 from llama_index.core import Settings
-from documents_prep import load_json_documents, load_table_data, load_image_data, load_csv_chunks
+from documents_prep import load_json_documents, load_table_documents, load_image_documents
 from utils import get_llm_model, get_embedding_model, get_reranker_model, answer_question
 from my_logging import log_message
 from index_retriever import create_vector_index, create_query_engine
@@ -96,14 +96,12 @@ def initialize_system(repo_id, hf_token, download_dir, chunks_filename=None,
                      json_files_dir=None, table_data_dir=None, image_data_dir=None, 
                      use_json_instead_csv=False):
     try:
-        from documents_prep import process_documents_with_chunking
         log_message("Инициализация системы")
         os.makedirs(download_dir, exist_ok=True)
         from config import CHUNK_SIZE, CHUNK_OVERLAP
         from llama_index.core.text_splitter import TokenTextSplitter
         
         embed_model = get_embedding_model()
-
         llm = get_llm_model(DEFAULT_MODEL)
         reranker = get_reranker_model()
         
@@ -121,49 +119,64 @@ def initialize_system(repo_id, hf_token, download_dir, chunks_filename=None,
 
         all_documents = []
         chunks_df = None
-        chunk_info = []
         
         if use_json_instead_csv and json_files_dir:
             log_message("Используем JSON файлы вместо CSV")
-            json_documents, json_chunk_info = load_json_documents(repo_id, hf_token, json_files_dir, download_dir)
-            all_documents.extend(json_documents)
-            chunk_info.extend(json_chunk_info)
+            from documents_prep import load_json_documents, chunk_text_documents
+            
+            # Load JSON docs (returns list of Documents)
+            json_documents = load_json_documents(repo_id, hf_token, json_files_dir)
+            # Chunk them
+            json_chunks = chunk_text_documents(json_documents)
+            all_documents.extend(json_chunks)
         else:
             if chunks_filename:
                 log_message("Загружаем данные из CSV")
-                csv_documents, chunks_df = load_csv_chunks(repo_id, hf_token, chunks_filename, download_dir)
-                all_documents.extend(csv_documents)
+               
         
         if table_data_dir:
             log_message("Добавляю табличные данные")
-            table_documents = load_table_data(repo_id, hf_token, table_data_dir)
-            log_message(f"Загружено {len(table_documents)} табличных документов")
+            from documents_prep import load_table_documents
             
-            # Process table documents through chunking
-            chunked_table_docs, table_chunk_info = process_documents_with_chunking(table_documents)
-            all_documents.extend(chunked_table_docs)
-            chunk_info.extend(table_chunk_info)
+            # load_table_documents already returns chunked documents
+            table_chunks = load_table_documents(repo_id, hf_token, table_data_dir)
+            log_message(f"Загружено {len(table_chunks)} табличных чанков")
+            all_documents.extend(table_chunks)
         
         if image_data_dir:
             log_message("Добавляю данные изображений")
-            image_documents = load_image_data(repo_id, hf_token, image_data_dir)
-            log_message(f"Загружено {len(image_documents)} документов изображений")
+            from documents_prep import load_image_documents
             
-            # Process image documents through chunking
-            chunked_image_docs, image_chunk_info = process_documents_with_chunking(image_documents)
-            all_documents.extend(chunked_image_docs)
-            chunk_info.extend(image_chunk_info)
+            # load_image_documents returns documents (no chunking needed)
+            image_documents = load_image_documents(repo_id, hf_token, image_data_dir)
+            log_message(f"Загружено {len(image_documents)} документов изображений")
+            all_documents.extend(image_documents)
         
         log_message(f"Всего документов после всей обработки: {len(all_documents)}")
         
         vector_index = create_vector_index(all_documents)
         query_engine = create_query_engine(vector_index)
         
+        # Create chunk_info for display (extract from documents metadata)
+        chunk_info = []
+        for doc in all_documents:
+            chunk_info.append({
+                'document_id': doc.metadata.get('document_id', 'unknown'),
+                'section_id': doc.metadata.get('section_id', 'unknown'),
+                'type': doc.metadata.get('type', 'text'),
+                'chunk_text': doc.text[:200] + '...' if len(doc.text) > 200 else doc.text,
+                'table_number': doc.metadata.get('table_number', ''),
+                'image_number': doc.metadata.get('image_number', ''),
+                'section': doc.metadata.get('section', ''),
+            })
+        
         log_message(f"Система успешно инициализирована")
         return query_engine, chunks_df, reranker, vector_index, chunk_info
         
     except Exception as e:
         log_message(f"Ошибка инициализации: {str(e)}")
+        import traceback
+        log_message(traceback.format_exc())
         return None, None, None, None, []
 
 def switch_model(model_name, vector_index):