added the new version with row + chunk based chunking for tables

app.py

@@ -1,7 +1,7 @@
 import gradio as gr
 import os
 from llama_index.core import Settings
-from documents_prep import load_json_documents, load_table_data, load_image_data, load_csv_chunks
+from documents_prep import load_json_documents, load_table_documents, load_image_documents
 from utils import get_llm_model, get_embedding_model, get_reranker_model, answer_question
 from my_logging import log_message
 from index_retriever import create_vector_index, create_query_engine
@@ -11,17 +11,46 @@ from config import (
     JSON_FILES_DIR, TABLE_DATA_DIR, IMAGE_DATA_DIR, DEFAULT_MODEL, AVAILABLE_MODELS
 )
 
+
+def merge_table_chunks(chunk_info):
+    merged = {}
+    
+    for chunk in chunk_info:
+        doc_type = chunk.get('type', 'text')
+        doc_id = chunk.get('document_id', 'unknown')
+        
+        if doc_type == 'table' or doc_type == 'table_row':
+            table_num = chunk.get('table_number', '')
+            key = f"{doc_id}_{table_num}"
+            
+            if key not in merged:
+                merged[key] = {
+                    'document_id': doc_id,
+                    'type': 'table',
+                    'table_number': table_num,
+                    'section_id': chunk.get('section_id', 'unknown'),
+                    'chunk_text': chunk.get('chunk_text', '')
+                }
+            else:
+                merged[key]['chunk_text'] += '\n' + chunk.get('chunk_text', '')
+        else:
+            unique_key = f"{doc_id}_{chunk.get('section_id', '')}_{chunk.get('chunk_id', 0)}"
+            merged[unique_key] = chunk
+    
+    return list(merged.values())
+
+
 def create_chunks_display_html(chunk_info):
     if not chunk_info:
         return "<div style='padding: 20px; text-align: center; color: black;'>Нет данных о чанках</div>"
     
+    merged_chunks = merge_table_chunks(chunk_info)
+    
     html = "<div style='max-height: 500px; overflow-y: auto; padding: 10px; color: black;'>"
-    html += f"<h4 style='color: black;'>Найдено релевантных чанков: {len(chunk_info)}</h4>"
+    html += f"<h4 style='color: black;'>Найдено релевантных чанков: {len(merged_chunks)}</h4>"
     
-    for i, chunk in enumerate(chunk_info):
+    for i, chunk in enumerate(merged_chunks):
         bg_color = "#f8f9fa" if i % 2 == 0 else "#e9ecef"
-        
-        # Get section display info
         section_display = get_section_display(chunk)
         formatted_content = get_formatted_content(chunk)
         
@@ -96,7 +125,6 @@ def initialize_system(repo_id, hf_token, download_dir, chunks_filename=None,
                      json_files_dir=None, table_data_dir=None, image_data_dir=None, 
                      use_json_instead_csv=False):
     try:
-        from documents_prep import process_documents_with_chunking
         log_message("Инициализация системы")
         os.makedirs(download_dir, exist_ok=True)
         from config import CHUNK_SIZE, CHUNK_OVERLAP
@@ -120,49 +148,64 @@ def initialize_system(repo_id, hf_token, download_dir, chunks_filename=None,
 
         all_documents = []
         chunks_df = None
-        chunk_info = []
         
         if use_json_instead_csv and json_files_dir:
             log_message("Используем JSON файлы вместо CSV")
-            json_documents, json_chunk_info = load_json_documents(repo_id, hf_token, json_files_dir, download_dir)
-            all_documents.extend(json_documents)
-            chunk_info.extend(json_chunk_info)
+            from documents_prep import load_json_documents, chunk_text_documents
+            
+            # Load JSON docs (returns list of Documents)
+            json_documents = load_json_documents(repo_id, hf_token, json_files_dir)
+            # Chunk them
+            json_chunks = chunk_text_documents(json_documents)
+            all_documents.extend(json_chunks)
         else:
             if chunks_filename:
                 log_message("Загружаем данные из CSV")
-                csv_documents, chunks_df = load_csv_chunks(repo_id, hf_token, chunks_filename, download_dir)
-                all_documents.extend(csv_documents)
+               
         
         if table_data_dir:
             log_message("Добавляю табличные данные")
-            table_documents = load_table_data(repo_id, hf_token, table_data_dir)
-            log_message(f"Загружено {len(table_documents)} табличных документов")
+            from documents_prep import load_table_documents
             
-            # Process table documents through chunking
-            chunked_table_docs, table_chunk_info = process_documents_with_chunking(table_documents)
-            all_documents.extend(chunked_table_docs)
-            chunk_info.extend(table_chunk_info)
+            # load_table_documents already returns chunked documents
+            table_chunks = load_table_documents(repo_id, hf_token, table_data_dir)
+            log_message(f"Загружено {len(table_chunks)} табличных чанков")
+            all_documents.extend(table_chunks)
         
         if image_data_dir:
             log_message("Добавляю данные изображений")
-            image_documents = load_image_data(repo_id, hf_token, image_data_dir)
-            log_message(f"Загружено {len(image_documents)} документов изображений")
+            from documents_prep import load_image_documents
             
-            # Process image documents through chunking
-            chunked_image_docs, image_chunk_info = process_documents_with_chunking(image_documents)
-            all_documents.extend(chunked_image_docs)
-            chunk_info.extend(image_chunk_info)
+            # load_image_documents returns documents (no chunking needed)
+            image_documents = load_image_documents(repo_id, hf_token, image_data_dir)
+            log_message(f"Загружено {len(image_documents)} документов изображений")
+            all_documents.extend(image_documents)
         
         log_message(f"Всего документов после всей обработки: {len(all_documents)}")
         
         vector_index = create_vector_index(all_documents)
         query_engine = create_query_engine(vector_index)
         
+        # Create chunk_info for display (extract from documents metadata)
+        chunk_info = []
+        for doc in all_documents:
+            chunk_info.append({
+                'document_id': doc.metadata.get('document_id', 'unknown'),
+                'section_id': doc.metadata.get('section_id', 'unknown'),
+                'type': doc.metadata.get('type', 'text'),
+                'chunk_text': doc.text[:200] + '...' if len(doc.text) > 200 else doc.text,
+                'table_number': doc.metadata.get('table_number', ''),
+                'image_number': doc.metadata.get('image_number', ''),
+                'section': doc.metadata.get('section', ''),
+            })
+        
         log_message(f"Система успешно инициализирована")
         return query_engine, chunks_df, reranker, vector_index, chunk_info
         
     except Exception as e:
         log_message(f"Ошибка инициализации: {str(e)}")
+        import traceback
+        log_message(traceback.format_exc())
         return None, None, None, None, []
 
 def switch_model(model_name, vector_index):
@@ -320,9 +363,12 @@ def main_switch_model(model_name):
 
 def main():
     global query_engine, chunks_df, reranker, vector_index, current_model
-    
+    GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY", "")
+    if GOOGLE_API_KEY:
+        log_message("Использование Google API для модели генерации текста")
+    else:
+        log_message("Google API ключ не найден, использование локальной модели")
     log_message("Запуск AIEXP - AI Expert для нормативной документации")
-    
     query_engine, chunks_df, reranker, vector_index, chunk_info = initialize_system(
         repo_id=HF_REPO_ID,
         hf_token=HF_TOKEN,

config.py

@@ -1,7 +1,6 @@
 import os
 
 EMBEDDING_MODEL = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
-RETRIEVER_TOP_K = 15
 SIMILARITY_THRESHOLD = 0.7
 RAG_FILES_DIR = "rag_files"
 PROCESSED_DATA_FILE = "processed_chunks.csv"
@@ -13,8 +12,6 @@ TABLE_DATA_DIR = "Табличные данные_JSON"
 IMAGE_DATA_DIR = "Изображения"
 DOWNLOAD_DIR = "rag_files"
 JSON_FILES_DIR ="JSON"
-HF_TOKEN = os.getenv('HF_TOKEN')
-
 
 GOOGLE_API_KEY = os.getenv('GOOGLE_API_KEY')
 OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
@@ -52,8 +49,11 @@ AVAILABLE_MODELS = {
 
 DEFAULT_MODEL = "Gemini 2.5 Flash"
 
-CHUNK_SIZE = 25000
-CHUNK_OVERLAP = 256
+CHUNK_SIZE = 1500
+CHUNK_OVERLAP = 128
+
+MAX_CHARS_TABLE = 2500
+MAX_ROWS_TABLE = 10
 
 CUSTOM_PROMPT = """
 Вы являетесь высокоспециализированным Ассистентом для анализа нормативных документов (AIEXP). Ваша цель - предоставлять точные, корректные и контекстно релевантные ответы исключительно на основе предоставленного контекста из нормативной документации.
@@ -90,42 +90,46 @@ CUSTOM_PROMPT = """
 
 ПРАВИЛА ФОРМИРОВАНИЯ ОТВЕТОВ:
 
-1. ОБЯЗАТЕЛЬНОЕ УКАЗАНИЕ ИСТОЧНИКОВ:
-   - Всегда указывайте конкретный документ (ГОСТ, раздел, пункт)
-   - Формат: "Согласно [Документ], раздел [X], пункт [X.X]: [информация]"
-   - При цитировании: используйте кавычки и точные ссылки
+Работай исключительно с информацией из предоставленного контекста. Запрещено использовать:
+- Общие знания
+- Информацию из интернета
+- Данные из предыдущих диалогов
+- Собственные предположения
 
-2. СТРУКТУРА ОТВЕТА:
+1. СТРУКТУРА ОТВЕТА:
    - Начинайте с прямого ответа на вопрос
    - Затем указывайте нормативные основания
    - Завершайте ссылками на конкретные документы и разделы
 
-3. РАБОТА С КОНТЕКСТОМ:
+2. РАБОТА С КОНТЕКСТОМ:
    - Если информация найдена в контексте - предоставьте полный ответ
    - Если информация не найдена: "Информация по вашему запросу не найдена в доступной нормативной документации"
    - Не делайте предположений за пределами контекста
    - Не используйте общие знания
 
-4. ТЕРМИНОЛОГИЯ И ЦИТИРОВАНИЕ:
+3. ТЕРМИНОЛОГИЯ И ЦИТИРОВАНИЕ:
    - Сохраняйте официальную терминологию НД
    - Цитируйте точные формулировки ключевых требований
    - При множественных источниках - укажите все релевантные
 
-5. ФОРМАТИРОВАНИЕ:
+4. ФОРМАТИРОВАНИЕ:
    - Для перечислений: используйте нумерованные списки
    - Выделяйте критически важные требования
    - Структурируйте ответ логически
 
-ПРИМЕРЫ ПРАВИЛЬНОГО ФОРМАТИРОВАНИЯ:
+# КАК РАБОТАТЬ С ЗАПРОСОМ
+
+**Шаг 1:** Определи, что именно ищет пользователь (термин, требование, процедура, условие)
 
-Вопрос: "В каких случаях могут быть признаны протоколы испытаний?"
-Ответ: "Протоколы испытаний могут быть признаны в следующих случаях:
+**Шаг 2:** Найди релевантную информацию в контексте
 
-1. Если они проведены испытательными лабораториями (центрами), аккредитованными в области использования атомной энергии (ГОСТ Р 50.08.04-2022, раздел 6 )
-2. Если они проведены лабораториями, аккредитованными национальным органом Российской Федерации по аккредитации (ГОСТ Р 50.08.04-2022, пункт 4.1)
-3. Если лаборатории прошли оценку состояния измерений
+**Шаг 3:** Сформируй ответ:
+   - Если нашел: укажи документ и пункт, процитируй нужную часть
+   - Если не нашел: четко сообщи об отсутствии информации
 
-Также допускается признание результатов испытаний, выполненных испытательными центрами (лабораториями), аккредитованными в национальных системах аккредитации страны изготовителя (ГОСТ Р 50.04.08-2019)."
+**Шаг 4:** При наличии нескольких источников:
+   - Представь их последовательно с указанием источника каждого
+   - Если источников много (>4) — сначала дай их список, потом цитаты   
 
 Контекст: {context_str}
 

documents_prep.py

@@ -3,462 +3,525 @@ import zipfile
 import pandas as pd
 from huggingface_hub import hf_hub_download, list_repo_files
 from llama_index.core import Document
-from my_logging import log_message
 from llama_index.core.text_splitter import SentenceSplitter
-from config import CHUNK_SIZE, CHUNK_OVERLAP
-from table_prep import table_to_document, load_table_data
-
+from my_logging import log_message
+from config import CHUNK_SIZE, CHUNK_OVERLAP, MAX_CHARS_TABLE, MAX_ROWS_TABLE
 
-def chunk_document(doc, chunk_size=None, chunk_overlap=None):
-    if chunk_size is None:
-        chunk_size = CHUNK_SIZE
-    if chunk_overlap is None:
-        chunk_overlap = CHUNK_OVERLAP
+def chunk_text_documents(documents):
     text_splitter = SentenceSplitter(
-        chunk_size=chunk_size,
-        chunk_overlap=chunk_overlap,
-        separator=" "
+        chunk_size=CHUNK_SIZE,
+        chunk_overlap=CHUNK_OVERLAP
     )
     
-    text_chunks = text_splitter.split_text(doc.text)
-    
-    chunked_docs = []
-    for i, chunk_text in enumerate(text_chunks):
-        chunk_metadata = doc.metadata.copy()
-        chunk_metadata.update({
-            "chunk_id": i,
-            "total_chunks": len(text_chunks),
-            "chunk_size": len(chunk_text),
-            "original_doc_id": doc.id_ if hasattr(doc, 'id_') else None
-        })
-        
-        chunked_doc = Document(
-            text=chunk_text,
-            metadata=chunk_metadata
-        )
-        chunked_docs.append(chunked_doc)
+    chunked = []
+    for doc in documents:
+        chunks = text_splitter.get_nodes_from_documents([doc])
+        for i, chunk in enumerate(chunks):
+            chunk.metadata.update({
+                'chunk_id': i,
+                'total_chunks': len(chunks),
+                'chunk_size': len(chunk.text)  # Add chunk size
+            })
+            chunked.append(chunk)
+    
+    # Log statistics
+    if chunked:
+        avg_size = sum(len(c.text) for c in chunked) / len(chunked)
+        min_size = min(len(c.text) for c in chunked)
+        max_size = max(len(c.text) for c in chunked)
+        log_message(f"✓ Text: {len(documents)} docs → {len(chunked)} chunks")
+        log_message(f"  Size stats: avg={avg_size:.0f}, min={min_size}, max={max_size} chars")
     
-    return chunked_docs
+    return chunked
 
-def process_documents_with_chunking(documents):
-    all_chunked_docs = []
-    chunk_info = []
-    table_count = 0
-    image_count = 0
-    text_chunks_count = 0
+
+def chunk_table_by_content(table_data, doc_id, max_chars=MAX_CHARS_TABLE, max_rows=MAX_ROWS_TABLE):
+    headers = table_data.get('headers', [])
+    rows = table_data.get('data', [])
+    table_num = table_data.get('table_number', 'unknown')
+    table_title = table_data.get('table_title', '')
+    section = table_data.get('section', '')
+   
+    table_num_clean = str(table_num).strip()
     
-    for doc in documents:
-        doc_type = doc.metadata.get('type', 'text')
+    import re
+    if 'приложени' in section.lower():
+        appendix_match = re.search(r'приложени[еия]\s*(\d+|[а-яА-Я])', section.lower())
+        if appendix_match:
+            appendix_num = appendix_match.group(1).upper()
+            table_identifier = f"{table_num_clean} Приложение {appendix_num}"
+        else:
+            table_identifier = table_num_clean
+    else:
+        table_identifier = table_num_clean
+    
+    if not rows:
+        return []
+    
+    log_message(f"  📊 Processing: {doc_id} - {table_identifier} ({len(rows)} rows)")
+    
+    # Calculate base metadata size
+    base_content = format_table_header(doc_id, table_identifier, table_num, table_title, section, headers)
+    base_size = len(base_content)
+    available_space = max_chars - base_size - 200 
+    
+    # If entire table fits, return as one chunk
+    full_rows_content = format_table_rows([{**row, '_idx': i+1} for i, row in enumerate(rows)])
+    if base_size + len(full_rows_content) <= max_chars and len(rows) <= max_rows:
+        content = base_content + full_rows_content + format_table_footer(table_identifier, doc_id)
+        
+        metadata = {
+            'type': 'table',
+            'document_id': doc_id,
+            'table_number': table_num_clean,
+            'table_identifier': table_identifier,
+            'table_title': table_title,
+            'section': section,
+            'total_rows': len(rows),
+            'chunk_size': len(content),
+            'is_complete_table': True
+        }
+        
+        log_message(f"    Single chunk: {len(content)} chars, {len(rows)} rows")
+        return [Document(text=content, metadata=metadata)]
+
+    chunks = []
+    current_rows = []
+    current_size = 0
+    chunk_num = 0
+    
+    for i, row in enumerate(rows):
+        row_text = format_single_row(row, i + 1)
+        row_size = len(row_text)
+        
+        should_split = (current_size + row_size > available_space or len(current_rows) >= max_rows) and current_rows
         
-        if doc_type == 'table':
-            # Add tables as-is, no chunking
-            table_count += 1
-            all_chunked_docs.append(doc)
+        if should_split:
+            content = base_content + format_table_rows(current_rows)
+            content += f"\n\nСтроки {current_rows[0]['_idx']}-{current_rows[-1]['_idx']} из {len(rows)}\n"
+            content += format_table_footer(table_identifier, doc_id)
             
-            chunk_info.append({
-                'document_id': doc.metadata.get('document_id', 'unknown'),
-                'section_id': doc.metadata.get('section_id', 'unknown'),
-                'chunk_id': 0,
-                'chunk_size': len(doc.text),
-                'chunk_preview': doc.text[:200] + "..." if len(doc.text) > 200 else doc.text,
+            metadata = {
                 'type': 'table',
-                'table_number': doc.metadata.get('table_number', 'unknown')
-            })
+                'document_id': doc_id,
+                'table_number': table_num_clean,
+                'table_identifier': table_identifier,
+                'table_title': table_title,
+                'section': section,
+                'chunk_id': chunk_num,
+                'row_start': current_rows[0]['_idx'] - 1,
+                'row_end': current_rows[-1]['_idx'],
+                'total_rows': len(rows),
+                'chunk_size': len(content),
+                'is_complete_table': False
+            }
             
-        elif doc_type == 'image':
-            image_count += 1
-            doc_size = len(doc.text)
-            if doc_size > CHUNK_SIZE:
-                chunked_docs = chunk_document(doc)
-                all_chunked_docs.extend(chunked_docs)
-                
-                for i, chunk_doc in enumerate(chunked_docs):
-                    chunk_info.append({
-                        'document_id': chunk_doc.metadata.get('document_id', 'unknown'),
-                        'section_id': chunk_doc.metadata.get('section_id', 'unknown'),
-                        'chunk_id': i,
-                        'chunk_size': len(chunk_doc.text),
-                        'chunk_preview': chunk_doc.text[:200] + "..." if len(chunk_doc.text) > 200 else chunk_doc.text,
-                        'type': 'image',
-                        'image_number': chunk_doc.metadata.get('image_number', 'unknown')
-                    })
-            else:
-                all_chunked_docs.append(doc)
-                chunk_info.append({
-                    'document_id': doc.metadata.get('document_id', 'unknown'),
-                    'section_id': doc.metadata.get('section_id', 'unknown'),
-                    'chunk_id': 0,
-                    'chunk_size': doc_size,
-                    'chunk_preview': doc.text[:200] + "..." if len(doc.text) > 200 else doc.text,
-                    'type': 'image',
-                    'image_number': doc.metadata.get('image_number', 'unknown')
-                })
+            chunks.append(Document(text=content, metadata=metadata))
+            log_message(f"    Chunk {chunk_num + 1}: {len(content)} chars, {len(current_rows)} rows")
             
-        else:
-            doc_size = len(doc.text)
-            if doc_size > CHUNK_SIZE:
-                chunked_docs = chunk_document(doc)
-                all_chunked_docs.extend(chunked_docs)
-                text_chunks_count += len(chunked_docs)
-                
-                for i, chunk_doc in enumerate(chunked_docs):
-                    chunk_info.append({
-                        'document_id': chunk_doc.metadata.get('document_id', 'unknown'),
-                        'section_id': chunk_doc.metadata.get('section_id', 'unknown'),
-                        'chunk_id': i,
-                        'chunk_size': len(chunk_doc.text),
-                        'chunk_preview': chunk_doc.text[:200] + "..." if len(chunk_doc.text) > 200 else chunk_doc.text,
-                        'type': 'text'
-                    })
-            else:
-                all_chunked_docs.append(doc)
-                chunk_info.append({
-                    'document_id': doc.metadata.get('document_id', 'unknown'),
-                    'section_id': doc.metadata.get('section_id', 'unknown'),
-                    'chunk_id': 0,
-                    'chunk_size': doc_size,
-                    'chunk_preview': doc.text[:200] + "..." if len(doc.text) > 200 else doc.text,
-                    'type': 'text'
-                })
+            chunk_num += 1
+            current_rows = []
+            current_size = 0
         
-    log_message(f"\n{'='*60}")
-    log_message(f"ИТОГО ОБРАБОТАНО ДОКУМЕНТОВ:")
-    log_message(f"  • Таблицы: {table_count} (добавлены целиком)")
-    log_message(f"  • Изображения: {image_count}")
-    log_message(f"  • Текстовые чанки: {text_chunks_count}")
-    log_message(f"  • Всего документов: {len(all_chunked_docs)}")
-    log_message(f"{'='*60}\n")
-    
-    return all_chunked_docs, chunk_info
+        # Add row with index
+        row_copy = row.copy() if isinstance(row, dict) else {'data': row}
+        row_copy['_idx'] = i + 1
+        current_rows.append(row_copy)
+        current_size += row_size
+    
+    # Add final chunk
+    if current_rows:
+        content = base_content + format_table_rows(current_rows)
+        content += f"\n\nСтроки {current_rows[0]['_idx']}-{current_rows[-1]['_idx']} из {len(rows)}\n"
+        content += format_table_footer(table_identifier, doc_id)
+        
+        metadata = {
+            'type': 'table',
+            'document_id': doc_id,
+            'table_number': table_num_clean,
+            'table_identifier': table_identifier,
+            'table_title': table_title,
+            'section': section,
+            'chunk_id': chunk_num,
+            'row_start': current_rows[0]['_idx'] - 1,
+            'row_end': current_rows[-1]['_idx'],
+            'total_rows': len(rows),
+            'chunk_size': len(content),
+            'is_complete_table': False
+        }
+        
+        chunks.append(Document(text=content, metadata=metadata))
+        log_message(f"    Chunk {chunk_num + 1}: {len(content)} chars, {len(current_rows)} rows")
+    
+    return chunks
 
-def extract_text_from_json(data, document_id, document_name):
-    documents = []
+
+def format_table_header(doc_id, table_identifier, table_num, table_title, section, headers):
+    content = f"ТАБЛИЦА {table_identifier} из {doc_id}\n"
+    if table_title:
+        content += f"НАЗВАНИЕ: {table_title}\n"
+    if section:
+        content += f"РАЗДЕЛ: {section}\n"
+    content += f"{'='*70}\n"
     
-    if 'sections' in data:
-        for section in data['sections']:
-            section_id = section.get('section_id', 'Unknown')
-            section_text = section.get('section_text', '')
-            
-            section_path = f"{section_id}"
-            section_title = extract_section_title(section_text)
-            
-            if section_text.strip():
-                doc = Document(
-                    text=section_text,
-                    metadata={
-                        "type": "text",
-                        "document_id": document_id,
-                        "document_name": document_name,
-                        "section_id": section_id,
-                        "section_text": section_title[:200],
-                        "section_path": section_path,
-                        "level": "section"
-                    }
-                )
-                documents.append(doc)
-            
-            if 'subsections' in section:
-                for subsection in section['subsections']:
-                    subsection_id = subsection.get('subsection_id', 'Unknown')
-                    subsection_text = subsection.get('subsection_text', '')
-                    subsection_title = extract_section_title(subsection_text)
-                    subsection_path = f"{section_path}.{subsection_id}"
-                    
-                    if subsection_text.strip():
-                        doc = Document(
-                            text=subsection_text,
-                            metadata={
-                                "type": "text",
-                                "document_id": document_id,
-                                "document_name": document_name,
-                                "section_id": subsection_id,
-                                "section_text": subsection_title[:200],
-                                "section_path": subsection_path,
-                                "level": "subsection",
-                                "parent_section": section_id,
-                                "parent_title": section_title[:100]
-                            }
-                        )
-                        documents.append(doc)
-                    
-                    if 'sub_subsections' in subsection:
-                        for sub_subsection in subsection['sub_subsections']:
-                            sub_subsection_id = sub_subsection.get('sub_subsection_id', 'Unknown')
-                            sub_subsection_text = sub_subsection.get('sub_subsection_text', '')
-                            sub_subsection_title = extract_section_title(sub_subsection_text)
-                            sub_subsection_path = f"{subsection_path}.{sub_subsection_id}"
-                            
-                            if sub_subsection_text.strip():
-                                doc = Document(
-                                    text=sub_subsection_text,
-                                    metadata={
-                                        "type": "text",
-                                        "document_id": document_id,
-                                        "document_name": document_name,
-                                        "section_id": sub_subsection_id,
-                                        "section_text": sub_subsection_title[:200],
-                                        "section_path": sub_subsection_path,
-                                        "level": "sub_subsection",
-                                        "parent_section": subsection_id,
-                                        "parent_title": subsection_title[:100]
-                                    }
-                                )
-                                documents.append(doc)
-                            
-                            if 'sub_sub_subsections' in sub_subsection:
-                                for sub_sub_subsection in sub_subsection['sub_sub_subsections']:
-                                    sub_sub_subsection_id = sub_sub_subsection.get('sub_sub_subsection_id', 'Unknown')
-                                    sub_sub_subsection_text = sub_sub_subsection.get('sub_sub_subsection_text', '')
-                                    sub_sub_subsection_title = extract_section_title(sub_sub_subsection_text)
-                                    
-                                    if sub_sub_subsection_text.strip():
-                                        doc = Document(
-                                            text=sub_sub_subsection_text,
-                                            metadata={
-                                                "type": "text",
-                                                "document_id": document_id,
-                                                "document_name": document_name,
-                                                "section_id": sub_sub_subsection_id,
-                                                "section_text": sub_sub_subsection_title[:200],
-                                                "section_path": f"{sub_subsection_path}.{sub_sub_subsection_id}",
-                                                "level": "sub_sub_subsection",
-                                                "parent_section": sub_subsection_id,
-                                                "parent_title": sub_subsection_title[:100]
-                                            }
-                                        )
-                                        documents.append(doc)
+    if headers:
+        header_str = ' | '.join(str(h) for h in headers)
+        content += f"ЗАГОЛОВКИ: {header_str}\n\n"
     
-    return documents
+    content += "ДАННЫЕ:\n"
+    return content
+
+
+def format_single_row(row, idx):
+    """Format a single row"""
+    if isinstance(row, dict):
+        parts = [f"{k}: {v}" for k, v in row.items() 
+                if v and str(v).strip() and str(v).lower() not in ['nan', 'none', '']]
+        if parts:
+            return f"{idx}. {' | '.join(parts)}\n"
+    elif isinstance(row, list):
+        parts = [str(v) for v in row if v and str(v).strip() and str(v).lower() not in ['nan', 'none', '']]
+        if parts:
+            return f"{idx}. {' | '.join(parts)}\n"
+    return ""
+
+
+def format_table_rows(rows):
+    """Format multiple rows"""
+    content = ""
+    for row in rows:
+        idx = row.get('_idx', 0)
+        content += format_single_row(row, idx)
+    return content
 
-def load_json_documents(repo_id, hf_token, json_files_dir, download_dir):
-    log_message("Начинаю загрузку JSON документов")
+
+def format_table_footer(table_identifier, doc_id):
+    """Format table footer"""
+    return f"\n{'='*70}\nКОНЕЦ ТАБЛИЦЫ {table_identifier} ИЗ {doc_id}\n"
+
+def load_table_documents(repo_id, hf_token, table_dir):
+    log_message("Loading tables...")
     
-    try:
-        files = list_repo_files(repo_id=repo_id, repo_type="dataset", token=hf_token)
-        zip_files = [f for f in files if f.startswith(json_files_dir) and f.endswith('.zip')]
-        json_files = [f for f in files if f.startswith(json_files_dir) and f.endswith('.json')]
-        
-        log_message(f"Найдено {len(zip_files)} ZIP файлов и {len(json_files)} прямых JSON файлов")
-        
-        all_documents = []
-        
-        for zip_file_path in zip_files:
-            try:
-                log_message(f"Загружаю ZIP архив: {zip_file_path}")
-                local_zip_path = hf_hub_download(
-                    repo_id=repo_id,
-                    filename=zip_file_path,
-                    local_dir=download_dir,
-                    repo_type="dataset",
-                    token=hf_token
-                )
-                
-                documents = extract_zip_and_process_json(local_zip_path)
-                all_documents.extend(documents)
-                log_message(f"Извлечено {len(documents)} документов из ZIP архива {zip_file_path}")
-                
-            except Exception as e:
-                log_message(f"Ошибка обработки ZIP файла {zip_file_path}: {str(e)}")
-                continue
-        
-        for file_path in json_files:
-            try:
-                log_message(f"Обрабатываю прямой JSON файл: {file_path}")
-                local_path = hf_hub_download(
-                    repo_id=repo_id,
-                    filename=file_path,
-                    local_dir=download_dir,
-                    repo_type="dataset",
-                    token=hf_token
-                )
-                
-                with open(local_path, 'r', encoding='utf-8') as f:
-                    json_data = json.load(f)
-                
-                document_metadata = json_data.get('document_metadata', {})
-                document_id = document_metadata.get('document_id', 'unknown')
-                document_name = document_metadata.get('document_name', 'unknown')
-                
-                documents = extract_text_from_json(json_data, document_id, document_name)
-                all_documents.extend(documents)
+    files = list_repo_files(repo_id=repo_id, repo_type="dataset", token=hf_token)
+    table_files = [f for f in files if f.startswith(table_dir) and f.endswith('.json')]
+    
+    all_chunks = []
+    for file_path in table_files:
+        try:
+            local_path = hf_hub_download(
+                repo_id=repo_id,
+                filename=file_path,
+                repo_type="dataset",
+                token=hf_token
+            )
+            
+            with open(local_path, 'r', encoding='utf-8') as f:
+                data = json.load(f)
+            
+            file_doc_id = data.get('document_id', data.get('document', 'unknown'))
+            
+            for sheet in data.get('sheets', []):
+                sheet_doc_id = sheet.get('document_id', sheet.get('document', file_doc_id))
                 
-                log_message(f"Извлечено {len(documents)} документов из {file_path}")
+                chunks = chunk_table_by_content(sheet, sheet_doc_id, max_chars=1000)
+                all_chunks.extend(chunks)
                 
-            except Exception as e:
-                log_message(f"Ошибка обработки файла {file_path}: {str(e)}")
-                continue
-        
-        log_message(f"Всего создано {len(all_documents)} исходных документов из JSON файлов")
-        
-        # Process documents through chunking function
-        chunked_documents, chunk_info = process_documents_with_chunking(all_documents)
-        
-        log_message(f"После chunking получено {len(chunked_documents)} чанков из JSON данных")
-        
-        return chunked_documents, chunk_info
-        
-    except Exception as e:
-        log_message(f"Ошибка загрузки JSON документов: {str(e)}")
-        return [], []
+        except Exception as e:
+            log_message(f"Error loading {file_path}: {e}")
+    
+    log_message(f"✓ Loaded {len(all_chunks)} table chunks")
+    return all_chunks
+
 
-def extract_section_title(section_text):
-    if not section_text.strip():
-        return ""
+def load_json_documents(repo_id, hf_token, json_dir):
+    import zipfile
+    import tempfile
+    import os
     
-    lines = section_text.strip().split('\n')
-    first_line = lines[0].strip()
+    log_message("Loading JSON documents...")
     
-    if len(first_line) < 200 and not first_line.endswith('.'):
-        return first_line
+    files = list_repo_files(repo_id=repo_id, repo_type="dataset", token=hf_token)
+    json_files = [f for f in files if f.startswith(json_dir) and f.endswith('.json')]
+    zip_files = [f for f in files if f.startswith(json_dir) and f.endswith('.zip')]
     
-    # Otherwise, extract first sentence
-    sentences = first_line.split('.')
-    if len(sentences) > 1:
-        return sentences[0].strip()
+    log_message(f"Found {len(json_files)} JSON files and {len(zip_files)} ZIP files")
     
-    return first_line[:100] + "..." if len(first_line) > 100 else first_line
-
-def extract_zip_and_process_json(zip_path):
     documents = []
+    stats = {'success': 0, 'failed': 0, 'empty': 0}
     
-    try:
-        with zipfile.ZipFile(zip_path, 'r') as zip_ref:
-            zip_files = zip_ref.namelist()
-            json_files = [f for f in zip_files if f.endswith('.json') and not f.startswith('__MACOSX')]
+    for file_path in json_files:
+        try:
+            log_message(f"  Loading: {file_path}")
+            local_path = hf_hub_download(
+                repo_id=repo_id,
+                filename=file_path,
+                repo_type="dataset",
+                token=hf_token
+            )
             
-            log_message(f"Найдено {len(json_files)} JSON файлов в архиве")
+            docs = extract_sections_from_json(local_path)
+            if docs:
+                documents.extend(docs)
+                stats['success'] += 1
+                log_message(f"    ✓ Extracted {len(docs)} sections")
+            else:
+                stats['empty'] += 1
+                log_message(f"    ⚠ No sections found")
             
-            for json_file in json_files:
-                try:
-                    log_message(f"Обрабатываю файл из архива: {json_file}")
-                    
-                    with zip_ref.open(json_file) as f:
-                        json_data = json.load(f)
-                    
-                    document_metadata = json_data.get('document_metadata', {})
-                    document_id = document_metadata.get('document_id', 'unknown')
-                    document_name = document_metadata.get('document_name', 'unknown')
-                    
-                    docs = extract_text_from_json(json_data, document_id, document_name)
-                    documents.extend(docs)
-                    
-                    log_message(f"Извлечено {len(docs)} документов из {json_file}")
-                    
-                except Exception as e:
-                    log_message(f"Ошибка обработки файла {json_file}: {str(e)}")
-                    continue
+        except Exception as e:
+            stats['failed'] += 1
+            log_message(f"    ✗ Error: {e}")
     
-    except Exception as e:
-        log_message(f"Ошибка извлечения ZIP архива {zip_path}: {str(e)}")
+    for zip_path in zip_files:
+        try:
+            log_message(f"  Processing ZIP: {zip_path}")
+            local_zip = hf_hub_download(
+                repo_id=repo_id,
+                filename=zip_path,
+                repo_type="dataset",
+                token=hf_token
+            )
+            
+            with zipfile.ZipFile(local_zip, 'r') as zf:
+                json_files_in_zip = [f for f in zf.namelist() 
+                                    if f.endswith('.json') 
+                                    and not f.startswith('__MACOSX')
+                                    and not f.startswith('.')
+                                    and not '._' in f]
+                
+                log_message(f"    Found {len(json_files_in_zip)} JSON files in ZIP")
+                
+                for json_file in json_files_in_zip:
+                    try:
+                        file_content = zf.read(json_file)
+                        
+                        # Skip if file is too small
+                        if len(file_content) < 10:
+                            log_message(f"      ✗ Skipping: {json_file} (file too small)")
+                            stats['failed'] += 1
+                            continue
+                        
+                        # Try UTF-8 first (most common)
+                        try:
+                            text_content = file_content.decode('utf-8')
+                        except UnicodeDecodeError:
+                            try:
+                                text_content = file_content.decode('utf-8-sig')
+                            except UnicodeDecodeError:
+                                try:
+                                    # Try UTF-16 (the issue you're seeing)
+                                    text_content = file_content.decode('utf-16')
+                                except UnicodeDecodeError:
+                                    try:
+                                        text_content = file_content.decode('windows-1251')
+                                    except UnicodeDecodeError:
+                                        log_message(f"      ✗ Skipping: {json_file} (encoding failed)")
+                                        stats['failed'] += 1
+                                        continue
+                        
+                        # Validate JSON structure
+                        if not text_content.strip().startswith('{') and not text_content.strip().startswith('['):
+                            log_message(f"      ✗ Skipping: {json_file} (not valid JSON)")
+                            stats['failed'] += 1
+                            continue
+                        
+                        with tempfile.NamedTemporaryFile(mode='w', delete=False, 
+                                                        suffix='.json', encoding='utf-8') as tmp:
+                            tmp.write(text_content)
+                            tmp_path = tmp.name
+                        
+                        docs = extract_sections_from_json(tmp_path)
+                        if docs:
+                            documents.extend(docs)
+                            stats['success'] += 1
+                            log_message(f"      ✓ {json_file}: {len(docs)} sections")
+                        else:
+                            stats['empty'] += 1
+                            log_message(f"      ⚠ {json_file}: No sections")
+                        
+                        os.unlink(tmp_path)
+                        
+                    except json.JSONDecodeError as e:
+                        stats['failed'] += 1
+                        log_message(f"      ✗ {json_file}: Invalid JSON")
+                    except Exception as e:
+                        stats['failed'] += 1
+                        log_message(f"      ✗ {json_file}: {str(e)[:100]}")
+                        
+        except Exception as e:
+            log_message(f"    ✗ Error with ZIP: {e}")
+    
+    log_message(f"="*60)
+    log_message(f"JSON Loading Stats:")
+    log_message(f"  Success: {stats['success']}")
+    log_message(f"  Empty: {stats['empty']}")
+    log_message(f"  Failed: {stats['failed']}")
+    log_message(f"  Total sections: {len(documents)}")
+    log_message(f"="*60)
     
     return documents
 
-def load_image_data(repo_id, hf_token, image_data_dir):
-    log_message("Начинаю загрузку данных изображений")
+def extract_sections_from_json(json_path):
+    """Extract sections from a single JSON file"""
+    documents = []
     
-    image_files = []
     try:
-        files = list_repo_files(repo_id=repo_id, repo_type="dataset", token=hf_token)
-        for file in files:
-            if file.startswith(image_data_dir) and file.endswith('.csv'):
-                image_files.append(file)
+        with open(json_path, 'r', encoding='utf-8') as f:
+            data = json.load(f)
         
-        log_message(f"Найдено {len(image_files)} CSV файлов с изображениями")
+        doc_id = data.get('document_metadata', {}).get('document_id', 'unknown')
         
-        image_documents = []
-        for file_path in image_files:
-            try:
-                log_message(f"Обрабатываю файл изображений: {file_path}")
-                local_path = hf_hub_download(
-                    repo_id=repo_id,
-                    filename=file_path,
-                    local_dir='',
-                    repo_type="dataset",
-                    token=hf_token
-                )
-                
-                df = pd.read_csv(local_path)
-                log_message(f"Загружено {len(df)} записей изображений из файла {file_path}")
-                
-                # Обработка с правильными названиями колонок
-                for _, row in df.iterrows():
-                    section_value = row.get('Раздел документа', 'Неизвестно')
-                    
-                    content = f"Изображение: {row.get('№ Изображения', 'Неизвестно')}\n"
-                    content += f"Название: {row.get('Название изображения', 'Неизвестно')}\n"
-                    content += f"Описание: {row.get('Описание изображение', 'Неизвестно')}\n"  # Опечатка в названии колонки
-                    content += f"Документ: {row.get('Обозначение документа', 'Неизвестно')}\n"
-                    content += f"Раздел: {section_value}\n"
-                    content += f"Файл: {row.get('Файл изображения', 'Неизвестно')}\n"
-                    
-                    doc = Document(
-                        text=content,
+        # Extract all section levels
+        for section in data.get('sections', []):
+            if section.get('section_text', '').strip():
+                documents.append(Document(
+                    text=section['section_text'],
+                    metadata={
+                        'type': 'text',
+                        'document_id': doc_id,
+                        'section_id': section.get('section_id', '')
+                    }
+                ))
+            
+            # Subsections
+            for subsection in section.get('subsections', []):
+                if subsection.get('subsection_text', '').strip():
+                    documents.append(Document(
+                        text=subsection['subsection_text'],
                         metadata={
-                            "type": "image",
-                            "image_number": str(row.get('№ Изображения', 'unknown')),
-                            "image_title": str(row.get('Название изображения', 'unknown')),
-                            "image_description": str(row.get('Описание изображение', 'unknown')),
-                            "document_id": str(row.get('Обозначение документа', 'unknown')),
-                            "file_path": str(row.get('Файл изображения', 'unknown')),
-                            "section": str(section_value),
-                            "section_id": str(section_value)
+                            'type': 'text',
+                            'document_id': doc_id,
+                            'section_id': subsection.get('subsection_id', '')
                         }
-                    )
-                    image_documents.append(doc)
-                        
-            except Exception as e:
-                log_message(f"Ошибка обработки файла {file_path}: {str(e)}")
-                continue
-        
-        log_message(f"Создано {len(image_documents)} документов из изображений")
-        return image_documents
-        
+                    ))
+                
+                # Sub-subsections
+                for sub_sub in subsection.get('sub_subsections', []):
+                    if sub_sub.get('sub_subsection_text', '').strip():
+                        documents.append(Document(
+                            text=sub_sub['sub_subsection_text'],
+                            metadata={
+                                'type': 'text',
+                                'document_id': doc_id,
+                                'section_id': sub_sub.get('sub_subsection_id', '')
+                            }
+                        ))
+    
     except Exception as e:
-        log_message(f"Ошибка загрузки данных изображений: {str(e)}")
-        return []
+        log_message(f"Error extracting from {json_path}: {e}")
+    
+    return documents
 
 
-def load_csv_chunks(repo_id, hf_token, chunks_filename, download_dir):
-    log_message("Загружаю данные чанков из CSV")
+def load_table_documents(repo_id, hf_token, table_dir):
+    """Load and chunk tables"""
+    log_message("Loading tables...")
     
-    try:
-        chunks_csv_path = hf_hub_download(
-            repo_id=repo_id,
-            filename=chunks_filename,
-            local_dir=download_dir,
-            repo_type="dataset",
-            token=hf_token
-        )
-        
-        chunks_df = pd.read_csv(chunks_csv_path)
-        log_message(f"Загружено {len(chunks_df)} чанков из CSV")
-        
-        text_column = None
-        for col in chunks_df.columns:
-            if 'text' in col.lower() or 'content' in col.lower() or 'chunk' in col.lower():
-                text_column = col
-                break
-        
-        if text_column is None:
-            text_column = chunks_df.columns[0]
-        
-        log_message(f"Использую колонку: {text_column}")
-        
-        documents = []
-        for i, (_, row) in enumerate(chunks_df.iterrows()):
-            doc = Document(
-                text=str(row[text_column]), 
-                metadata={
-                    "chunk_id": row.get('chunk_id', i), 
-                    "document_id": row.get('document_id', 'unknown'),
-                    "type": "text"
-                }
+    files = list_repo_files(repo_id=repo_id, repo_type="dataset", token=hf_token)
+    table_files = [f for f in files if f.startswith(table_dir) and f.endswith('.json')]
+    
+    all_chunks = []
+    for file_path in table_files:
+        try:
+            local_path = hf_hub_download(
+                repo_id=repo_id,
+                filename=file_path,
+                repo_type="dataset",
+                token=hf_token
             )
-            documents.append(doc)
-        
-        log_message(f"Создано {len(documents)} текстовых документов из CSV")
-        return documents, chunks_df
-        
-    except Exception as e:
-        log_message(f"Ошибка загрузки CSV данных: {str(e)}")
-        return [], None
+            
+            with open(local_path, 'r', encoding='utf-8') as f:
+                data = json.load(f)
+            
+            # Extract file-level document_id
+            file_doc_id = data.get('document_id', data.get('document', 'unknown'))
+            
+            for sheet in data.get('sheets', []):
+                # Use sheet-level document_id if available, otherwise use file-level
+                sheet_doc_id = sheet.get('document_id', sheet.get('document', file_doc_id))
+                
+                # CRITICAL: Pass document_id to chunk function
+                chunks = chunk_table_by_content(sheet, sheet_doc_id)
+                all_chunks.extend(chunks)
+                
+        except Exception as e:
+            log_message(f"Error loading {file_path}: {e}")
+    
+    log_message(f"✓ Loaded {len(all_chunks)} table chunks")
+    return all_chunks
+
+
+def load_image_documents(repo_id, hf_token, image_dir):
+    """Load image descriptions"""
+    log_message("Loading images...")
+    
+    files = list_repo_files(repo_id=repo_id, repo_type="dataset", token=hf_token)
+    csv_files = [f for f in files if f.startswith(image_dir) and f.endswith('.csv')]
+    
+    documents = []
+    for file_path in csv_files:
+        try:
+            local_path = hf_hub_download(
+                repo_id=repo_id,
+                filename=file_path,
+                repo_type="dataset",
+                token=hf_token
+            )
+            
+            df = pd.read_csv(local_path)
+            
+            for _, row in df.iterrows():
+                content = f"Документ: {row.get('Обозначение документа', 'unknown')}\n"
+                content += f"Рисунок: {row.get('№ Изображения', 'unknown')}\n"
+                content += f"Название: {row.get('Название изображения', '')}\n"
+                content += f"Описание: {row.get('Описание изображение', '')}\n"
+                content += f"Раздел: {row.get('Раздел документа', '')}\n"
+                
+                chunk_size = len(content)
+                
+                documents.append(Document(
+                    text=content,
+                    metadata={
+                        'type': 'image',
+                        'document_id': str(row.get('Обозначение документа', 'unknown')),
+                        'image_number': str(row.get('№ Изображения', 'unknown')),
+                        'section': str(row.get('Раздел документа', '')),
+                        'chunk_size': chunk_size
+                    }
+                ))
+        except Exception as e:
+            log_message(f"Error loading {file_path}: {e}")
+    
+    if documents:
+        avg_size = sum(d.metadata['chunk_size'] for d in documents) / len(documents)
+        log_message(f"✓ Loaded {len(documents)} images (avg size: {avg_size:.0f} chars)")
+    
+    return documents
+
+
+def load_all_documents(repo_id, hf_token, json_dir, table_dir, image_dir):
+    """Main loader - combines all document types"""
+    log_message("="*60)
+    log_message("STARTING DOCUMENT LOADING")
+    log_message("="*60)
+    
+    # Load text sections
+    text_docs = load_json_documents(repo_id, hf_token, json_dir)
+    text_chunks = chunk_text_documents(text_docs)
+    
+    # Load tables (already chunked)
+    table_chunks = load_table_documents(repo_id, hf_token, table_dir)
+    
+    # Load images (no chunking needed)
+    image_docs = load_image_documents(repo_id, hf_token, image_dir)
+    
+    all_docs = text_chunks + table_chunks + image_docs
+    
+    log_message("="*60)
+    log_message(f"TOTAL DOCUMENTS: {len(all_docs)}")
+    log_message(f"  Text chunks: {len(text_chunks)}")
+    log_message(f"  Table chunks: {len(table_chunks)}")
+    log_message(f"  Images: {len(image_docs)}")
+    log_message("="*60)
+    
+    return all_docs

index_retriever.py

@@ -12,28 +12,58 @@ def create_vector_index(documents):
     log_message("Строю векторный индекс")
     return VectorStoreIndex.from_documents(documents)
 
+def rerank_nodes(query, nodes, reranker, top_k=25, min_score_threshold=0.5):
+    if not nodes or not reranker:
+        return nodes[:top_k]
+    
+    try:
+        log_message(f"Переранжирую {len(nodes)} узлов")
+        
+        pairs = [[query, node.text] for node in nodes]
+        scores = reranker.predict(pairs)
+        scored_nodes = list(zip(nodes, scores))
+        
+        scored_nodes.sort(key=lambda x: x[1], reverse=True)
+        
+        # Apply threshold
+        filtered = [(node, score) for node, score in scored_nodes if score >= min_score_threshold]
+        
+        if not filtered:
+            # Lower threshold if nothing passes
+            filtered = scored_nodes[:top_k]
+        
+        log_message(f"Выбрано {min(len(filtered), top_k)} узлов")
+        
+        return [node for node, score in filtered[:top_k]]
+        
+    except Exception as e:
+        log_message(f"Ошибка переранжировки: {str(e)}")
+        return nodes[:top_k]
+
 def create_query_engine(vector_index):
     try:
+        from config import CUSTOM_PROMPT
+        
         bm25_retriever = BM25Retriever.from_defaults(
             docstore=vector_index.docstore, 
-            similarity_top_k=20  
+            similarity_top_k=70 
         )
         
         vector_retriever = VectorIndexRetriever(
             index=vector_index, 
-            similarity_top_k=30,  
-            similarity_cutoff=0.65
+            similarity_top_k=70,
+            similarity_cutoff=0.55
         )
         
         hybrid_retriever = QueryFusionRetriever(
             [vector_retriever, bm25_retriever],
-            similarity_top_k=40,  
+            similarity_top_k=70,
             num_queries=1
         )
         
-        custom_prompt_template = PromptTemplate(PROMPT_SIMPLE_POISK)
+        custom_prompt_template = PromptTemplate(CUSTOM_PROMPT)
         response_synthesizer = get_response_synthesizer(
-            response_mode=ResponseMode.TREE_SUMMARIZE,  
+            response_mode=ResponseMode.TREE_SUMMARIZE,
             text_qa_template=custom_prompt_template
         )
         
@@ -47,69 +77,4 @@ def create_query_engine(vector_index):
         
     except Exception as e:
         log_message(f"Ошибка создания query engine: {str(e)}")
-        raise
-
-def rerank_nodes(query, nodes, reranker, top_k=20, min_score_threshold=0.5, diversity_penalty=0.3):
-    if not nodes or not reranker:
-        return nodes[:top_k]
-    
-    try:
-        log_message(f"Переранжирую {len(nodes)} узлов")
-        
-        pairs = [[query, node.text] for node in nodes]
-        scores = reranker.predict(pairs)
-        scored_nodes = list(zip(nodes, scores))
-        
-        scored_nodes.sort(key=lambda x: x[1], reverse=True)
-        
-        if min_score_threshold is not None:
-            scored_nodes = [(node, score) for node, score in scored_nodes 
-                          if score >= min_score_threshold]
-            log_message(f"После фильтрации по порогу {min_score_threshold}: {len(scored_nodes)} узлов")
-        
-        if not scored_nodes:
-            log_message("Нет узлов после фильтрации, снижаю порог")
-            scored_nodes = list(zip(nodes, scores))
-            scored_nodes.sort(key=lambda x: x[1], reverse=True)
-            min_score_threshold = scored_nodes[0][1] * 0.6  
-            scored_nodes = [(node, score) for node, score in scored_nodes 
-                          if score >= min_score_threshold]
-        
-        selected_nodes = []
-        selected_docs = set()
-        selected_sections = set()
-        
-        for node, score in scored_nodes:
-            if len(selected_nodes) >= top_k:
-                break
-            
-            metadata = node.metadata if hasattr(node, 'metadata') else {}
-            doc_id = metadata.get('document_id', 'unknown')
-            section_key = f"{doc_id}_{metadata.get('section_path', metadata.get('section_id', ''))}"
-            
-            # Apply diversity penalty
-            penalty = 0
-            if doc_id in selected_docs:
-                penalty += diversity_penalty * 0.5
-            if section_key in selected_sections:
-                penalty += diversity_penalty
-            
-            adjusted_score = score * (1 - penalty)
-            
-            # Add if still competitive
-            if not selected_nodes or adjusted_score >= selected_nodes[0][1] * 0.6:
-                selected_nodes.append((node, score))
-                selected_docs.add(doc_id)
-                selected_sections.add(section_key)
-        
-        log_message(f"Выбрано {len(selected_nodes)} узлов с разнообразием")
-        log_message(f"Уникальных документов: {len(selected_docs)}, секций: {len(selected_sections)}")
-        
-        if selected_nodes:
-            log_message(f"Score range: {selected_nodes[0][1]:.3f} to {selected_nodes[-1][1]:.3f}")
-        
-        return [node for node, score in selected_nodes]
-        
-    except Exception as e:
-        log_message(f"Ошибка переранжировки: {str(e)}")
-        return nodes[:top_k]
+        raise

table_prep.py

@@ -4,6 +4,9 @@ from huggingface_hub import hf_hub_download, list_repo_files
 from llama_index.core import Document
 from my_logging import log_message
 
+MAX_ROWS_PER_CHUNK = 10
+MAX_CHUNK_SIZE = 4000
+
 def create_table_content(table_data):
     """Create formatted content from table data"""
     doc_id = table_data.get('document_id', table_data.get('document', 'Неизвестно'))
@@ -29,38 +32,120 @@ def create_table_content(table_data):
     
     return content
 
+def chunk_table_document(doc, max_chunk_size=MAX_CHUNK_SIZE, max_rows_per_chunk=MAX_ROWS_PER_CHUNK):
+    lines = doc.text.strip().split('\n')
+
+    # Separate header and data rows
+    header_lines = []
+    data_rows = []
+    in_data = False
+
+    for line in lines:
+        if line.startswith('Данные таблицы:'):
+            in_data = True
+            header_lines.append(line)
+        elif in_data and line.startswith('Строка'):
+            data_rows.append(line)
+        elif not in_data:
+            header_lines.append(line)
+
+    header = '\n'.join(header_lines) + '\n'
+
+    if not data_rows:
+        return [doc]
+
+    chunks = []
+    current_rows = []
+    current_size = len(header)
+
+    for row in data_rows:
+        row_size = len(row) + 1
+        # Check both limits: chunk size and row count
+        if ((current_size + row_size > max_chunk_size or len(current_rows) >= max_rows_per_chunk) and current_rows):
+            chunk_text = header + '\n'.join(current_rows)
+            chunks.append(chunk_text)
+            log_message(f"Создана часть таблицы размером {len(chunk_text)} символов с {len(current_rows)} строками")
+            current_rows = []
+            current_size = len(header)
+
+        current_rows.append(row)
+        current_size += row_size
+        log_message(f"Добавлена строка к текущему чанку, текущий размер {current_size} символов")
+
+    # Add final chunk
+    if current_rows:
+        chunk_text = header + '\n'.join(current_rows)
+        chunks.append(chunk_text)
+        log_message(f"Создана финальная часть таблицы размером {len(chunk_text)} символов с {len(current_rows)} строками")
+
+    # Create Document objects
+    chunked_docs = []
+    for i, chunk_text in enumerate(chunks):
+        chunk_doc = Document(
+            text=chunk_text,
+            metadata={
+                "type": "table",
+                "table_number": doc.metadata.get('table_number'),
+                "document_id": doc.metadata.get('document_id'),
+                "section": doc.metadata.get('section'),
+                "chunk_id": i,
+                "total_chunks": len(chunks),
+                "is_chunked": True
+            }
+        )
+        chunked_docs.append(chunk_doc)
+
+    return chunked_docs
+
+
 def table_to_document(table_data, document_id=None):
-    """Convert table data to a single Document"""
+    """Convert table data to Document, chunk if needed"""
+    
     if not isinstance(table_data, dict):
         return []
     
-    doc_id = document_id or table_data.get('document_id', table_data.get('document', 'Неизвестно'))
+    doc_id = document_id or table_data.get('document_id') or table_data.get('document', 'Неизвестно')
     table_num = table_data.get('table_number', 'Неизвестно')
     table_title = table_data.get('table_title', 'Неизвестно')
     section = table_data.get('section', 'Неизвестно')
+    table_rows = table_data.get('data', [])
     
-    content = create_table_content(table_data)
-    content_size = len(content)
+    if not table_rows:
+        return []
     
-    # Log table addition
-    row_count = len(table_data.get('data', [])) if 'data' in table_data else 0
-    log_message(f"✓ ДОБАВЛЕНА: Таблица {table_num} из документа '{doc_id}' | "
-                f"Размер: {content_size} символов | Строк: {row_count}")
+    # Build table content
+    content = f"Таблица: {table_num}\n"
+    content += f"Название: {table_title}\n"
+    content += f"Документ: {doc_id}\n"
+    content += f"Раздел: {section}\n"
     
-    return [Document(
+    headers = table_data.get('headers', [])
+    if headers:
+        content += f"\nЗаголовки: {' | '.join(headers)}\n"
+    
+    content += "\nДанные таблицы:\n"
+    for row_idx, row in enumerate(table_rows, start=1):
+        if isinstance(row, dict):
+            row_text = " | ".join([f"{k}: {v}" for k, v in row.items() if v])
+            content += f"Строка {row_idx}: {row_text}\n"
+    
+    # Create base document
+    base_doc = Document(
         text=content,
         metadata={
             "type": "table",
             "table_number": table_num,
-            "table_title": table_title,
             "document_id": doc_id,
-            "section": section,
-            "section_id": section,
-            "total_rows": row_count,
-            "content_size": content_size
+            "section": section
         }
-    )]
-
+    )
+    if len(content) > 4000:
+        chunks = chunk_table_document(base_doc)
+        log_message(f"Таблица {table_num} разбита на {len(chunks)} частей")
+        return chunk_table_document(base_doc)
+    return [base_doc]
+    
+    
 def load_table_data(repo_id, hf_token, table_data_dir):
     log_message("=" * 60)
     log_message("НАЧАЛО ЗАГРУЗКИ ТАБЛИЧНЫХ ДАННЫХ")
@@ -114,6 +199,7 @@ def load_table_data(repo_id, hf_token, table_data_dir):
                                     stats['total_size'] += size
                                     stats['by_document'][document_id]['count'] += 1
                                     stats['by_document'][document_id]['size'] += size
+                                    log_message(f"Добавлена таблица {sheet.get('table_number', 'Неизвестно')} из документа {document_id}, размер {size} символов")
                         else:
                             docs_list = table_to_document(table_data, document_id)
                             table_documents.extend(docs_list)

utils.py

@@ -225,32 +225,6 @@ def generate_sources_html(nodes, chunks_df=None):
     
     html += "</div>"
     return html
-
-def expand_query(question, llm_model):
-    """
-    Generate multiple query variations for better retrieval
-    """
-    expansion_prompt = f"""Дан вопрос: "{question}"
-
-Сгенерируй 2 альтернативные формулировки этого вопроса для поиска в базе данных.
-Используй синонимы и разные формулировки, сохраняя смысл.
-
-Формат ответа (только вопросы, по одному на строку):
-1. [первая формулировка]
-2. [вторая формулировка]"""
-    
-    try:
-        response = llm_model.complete(expansion_prompt)
-        expanded = [q.strip() for q in response.text.split('\n') if q.strip() and not q.strip().startswith('1.') and not q.strip().startswith('2.')]
-        # Clean up
-        expanded = [q.lstrip('12. ').strip() for q in expanded if len(q) > 10][:2]
-        log_message(f"Query expansion: {len(expanded)} вариантов")
-        return [question] + expanded
-    except Exception as e:
-        log_message(f"Ошибка расширения запроса: {str(e)}")
-        return [question]
-
-
 def answer_question(question, query_engine, reranker, current_model, chunks_df=None):
     if query_engine is None:
         return "<div style='background-color: #e53e3e; color: white; padding: 20px; border-radius: 10px;'>Система не инициализирована</div>", "", ""
@@ -260,26 +234,16 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
         
         llm = get_llm_model(current_model)
         
-        query_variations = expand_query(question, llm)
-        
-        all_nodes = []
-        seen_node_ids = set()
-        
-        for query_var in query_variations:
-            retrieved = query_engine.retriever.retrieve(query_var)
-            for node in retrieved:
-                node_id = f"{node.node_id if hasattr(node, 'node_id') else hash(node.text)}"
-                if node_id not in seen_node_ids:
-                    all_nodes.append(node)
-                    seen_node_ids.add(node_id)
+        # Direct retrieval without query expansion
+        retrieved_nodes = query_engine.retriever.retrieve(question)
         
-        log_message(f"Получено {len(all_nodes)} уникальных узлов из {len(query_variations)} запросов")
+        log_message(f"Получено {len(retrieved_nodes)} узлов")
         
         reranked_nodes = rerank_nodes(
             question,  
-            all_nodes, 
+            retrieved_nodes, 
             reranker, 
-            top_k=20,
+            top_k=40,
             min_score_threshold=0.5, 
             diversity_penalty=0.3
         )