new final version of the rag

app.py

@@ -10,6 +10,7 @@ from config import (
     HF_REPO_ID, HF_TOKEN, DOWNLOAD_DIR, CHUNKS_FILENAME,
     JSON_FILES_DIR, TABLE_DATA_DIR, IMAGE_DATA_DIR, DEFAULT_MODEL, AVAILABLE_MODELS
 )
+from converters.converter import convert_single_excel_to_json, convert_single_excel_to_csv
 
 
 def merge_table_chunks(chunk_info):
@@ -39,7 +40,6 @@ def merge_table_chunks(chunk_info):
     
     return list(merged.values())
 
-
 def create_chunks_display_html(chunk_info):
     if not chunk_info:
         return "<div style='padding: 20px; text-align: center; color: black;'>Нет данных о чанках</div>"
@@ -142,19 +142,14 @@ def initialize_system(repo_id, hf_token, download_dir, chunks_filename=None,
             separator=" ",
             backup_separators=["\n", ".", "!", "?"]
         )
-        
-        log_message(f"Configured chunk size: {CHUNK_SIZE} tokens")
-        log_message(f"Configured chunk overlap: {CHUNK_OVERLAP} tokens")
 
         all_documents = []
         chunks_df = None
         
-        # CHANGED: Use load_all_documents instead of loading separately
         if use_json_instead_csv and json_files_dir:
             log_message("Используем JSON файлы вместо CSV")
             from documents_prep import load_all_documents
             
-            # This will handle text, tables, and images all together with proper logging
             all_documents = load_all_documents(
                 repo_id=repo_id,
                 hf_token=hf_token,
@@ -163,12 +158,10 @@ def initialize_system(repo_id, hf_token, download_dir, chunks_filename=None,
                 image_dir=image_data_dir if image_data_dir else ""
             )
         else:
-            # OLD PATH: Loading separately (fallback)
             if chunks_filename:
                 log_message("Загружаем данные из CSV")
             
             if table_data_dir:
-                log_message("Добавляю табличные данные")
                 from documents_prep import load_table_documents
                 
                 table_chunks = load_table_documents(repo_id, hf_token, table_data_dir)
@@ -176,7 +169,6 @@ def initialize_system(repo_id, hf_token, download_dir, chunks_filename=None,
                 all_documents.extend(table_chunks)
             
             if image_data_dir:
-                log_message("Добавляю данные изображений")
                 from documents_prep import load_image_documents
                 
                 image_documents = load_image_documents(repo_id, hf_token, image_data_dir)
@@ -188,7 +180,6 @@ def initialize_system(repo_id, hf_token, download_dir, chunks_filename=None,
         vector_index = create_vector_index(all_documents)
         query_engine = create_query_engine(vector_index)
         
-        # Create chunk_info for display (extract from documents metadata)
         chunk_info = []
         for doc in all_documents:
             chunk_info.append({
@@ -233,16 +224,48 @@ def switch_model(model_name, vector_index):
         log_message(error_msg)
         return None, f"❌ {error_msg}"
 
+retrieval_params = {
+    'vector_top_k': 50,
+    'bm25_top_k': 50,
+    'similarity_cutoff': 0.55,
+    'hybrid_top_k': 100,
+    'rerank_top_k': 20
+}
+
+def create_query_engine(vector_index, vector_top_k=50, bm25_top_k=50, 
+                       similarity_cutoff=0.55, hybrid_top_k=100):
+    try:
+        from config import CUSTOM_PROMPT
+        from index_retriever import create_query_engine as create_index_query_engine
+        
+        query_engine = create_index_query_engine(
+            vector_index=vector_index,
+            vector_top_k=vector_top_k,
+            bm25_top_k=bm25_top_k,
+            similarity_cutoff=similarity_cutoff,
+            hybrid_top_k=hybrid_top_k
+        )
+        
+        log_message(f"Query engine created with params: vector_top_k={vector_top_k}, "
+                   f"bm25_top_k={bm25_top_k}, cutoff={similarity_cutoff}, hybrid_top_k={hybrid_top_k}")
+        return query_engine
+        
+    except Exception as e:
+        log_message(f"Ошибка создания query engine: {str(e)}")
+        raise
+
 def main_answer_question(question):
-    global query_engine, reranker, current_model, chunks_df
+    global query_engine, reranker, current_model, chunks_df, retrieval_params
     if not question.strip():
         return ("<div style='color: black;'>Пожалуйста, введите вопрос</div>",
                 "<div style='color: black;'>Источники появятся после обработки запроса</div>",
                 "<div style='color: black;'>Чанки появятся после обработки запроса</div>")
 
     try:
-        # Call the answer_question function which returns 3 values
-        answer_html, sources_html, chunks_html = answer_question(question, query_engine, reranker, current_model, chunks_df)
+        answer_html, sources_html, chunks_html = answer_question(
+            question, query_engine, reranker, current_model, chunks_df,
+            rerank_top_k=retrieval_params['rerank_top_k']
+        )
         return answer_html, sources_html, chunks_html
 
     except Exception as e:
@@ -251,6 +274,36 @@ def main_answer_question(question):
                 "<div style='color: black;'>Источники недоступны из-за ошибки</div>",
                 "<div style='color: black;'>Чанки недоступны из-за ошибки</div>")
 
+def update_retrieval_params(vector_top_k, bm25_top_k, similarity_cutoff, hybrid_top_k, rerank_top_k):
+    global query_engine, vector_index, retrieval_params
+    
+    try:
+        retrieval_params['vector_top_k'] = vector_top_k
+        retrieval_params['bm25_top_k'] = bm25_top_k
+        retrieval_params['similarity_cutoff'] = similarity_cutoff
+        retrieval_params['hybrid_top_k'] = hybrid_top_k
+        retrieval_params['rerank_top_k'] = rerank_top_k
+        
+        # Recreate query engine with new parameters
+        if vector_index is not None:
+            query_engine = create_query_engine(
+                vector_index=vector_index,
+                vector_top_k=vector_top_k,
+                bm25_top_k=bm25_top_k,
+                similarity_cutoff=similarity_cutoff,
+                hybrid_top_k=hybrid_top_k
+            )
+            log_message(f"Параметры поиска обновлены: vector_top_k={vector_top_k}, "
+                       f"bm25_top_k={bm25_top_k}, cutoff={similarity_cutoff}, "
+                       f"hybrid_top_k={hybrid_top_k}, rerank_top_k={rerank_top_k}")
+            return f"✅ Параметры обновлены"
+        else:
+            return "❌ Система не инициализирована"
+    except Exception as e:
+        error_msg = f"Ошибка обновления параметров: {str(e)}"
+        log_message(error_msg)
+        return f"❌ {error_msg}"
+
 def retrieve_chunks(question: str, top_k: int = 20) -> list:
     from index_retriever import rerank_nodes
     global query_engine, reranker
@@ -362,24 +415,128 @@ def create_demo_interface(answer_question_func, switch_model_func, current_model
                         label="Релевантные чанки",
                         value="<div style='background-color: #2d3748; color: white; padding: 20px; border-radius: 10px; text-align: center;'>Здесь появятся релевантные чанки...</div>",
                     )
+        
+        with gr.Tab("⚙️ Параметры поиска"):
+            gr.Markdown("### Настройка параметров векторного поиска и переранжирования")
             
-            switch_btn.click(
-                fn=switch_model_func,
-                inputs=[model_dropdown],
-                outputs=[model_status]
+            with gr.Row():
+                with gr.Column():
+                    vector_top_k = gr.Slider(
+                        minimum=10,
+                        maximum=200,
+                        value=50,
+                        step=10,
+                        label="Vector Top K",
+                        info="Количество результатов из векторного поиска"
+                    )
+                
+                with gr.Column():
+                    bm25_top_k = gr.Slider(
+                        minimum=10,
+                        maximum=200,
+                        value=50,
+                        step=10,
+                        label="BM25 Top K",
+                        info="Количество результатов из BM25 поиска"
+                    )
+            
+            with gr.Row():
+                with gr.Column():
+                    similarity_cutoff = gr.Slider(
+                        minimum=0.0,
+                        maximum=1.0,
+                        value=0.55,
+                        step=0.05,
+                        label="Similarity Cutoff",
+                        info="Минимальный порог схожести для векторного поиска"
+                    )
+                
+                with gr.Column():
+                    hybrid_top_k = gr.Slider(
+                        minimum=10,
+                        maximum=300,
+                        value=100,
+                        step=10,
+                        label="Hybrid Top K",
+                        info="Количество результатов из гибридного поиска"
+                    )
+            
+            with gr.Row():
+                with gr.Column():
+                    rerank_top_k = gr.Slider(
+                        minimum=5,
+                        maximum=100,
+                        value=20,
+                        step=5,
+                        label="Rerank Top K",
+                        info="Количество результатов после переранжирования"
+                    )
+                
+                with gr.Column():
+                    update_btn = gr.Button("Применить параметры", variant="primary")
+                    update_status = gr.Textbox(
+                        value="Параметры готовы к применению",
+                        label="Статус",
+                        interactive=False
+                    )
+            
+            gr.Markdown("""
+            ### Рекомендации:
+            - **Vector Top K**: Увеличьте для более полного поиска по семантике (50-100)
+            - **BM25 Top K**: Увеличьте для лучшего поиска по ключевым словам (30-80)
+            - **Similarity Cutoff**: Снизьте для более мягких критериев (0.3-0.6), повысьте для строгих (0.7-0.9)
+            - **Hybrid Top K**: Объединённые результаты (100-150)
+            - **Rerank Top K**: Финальные результаты (10-30)
+            """)
+            
+            update_btn.click(
+                fn=update_retrieval_params,
+                inputs=[vector_top_k, bm25_top_k, similarity_cutoff, hybrid_top_k, rerank_top_k],
+                outputs=[update_status]
+            )
+            
+            gr.Markdown("### Текущие параметры:")
+            current_params_display = gr.Textbox(
+                value="Vector: 50 | BM25: 50 | Cutoff: 0.55 | Hybrid: 100 | Rerank: 20",
+                label="",
+                interactive=False,
+                lines=2
             )
             
-            ask_btn.click(
-                fn=answer_question_func,
-                inputs=[question_input],
-                outputs=[answer_output, sources_output, chunks_output]
+            def display_current_params():
+                return f"""Vector Top K: {retrieval_params['vector_top_k']}
+BM25 Top K: {retrieval_params['bm25_top_k']}
+Similarity Cutoff: {retrieval_params['similarity_cutoff']}
+Hybrid Top K: {retrieval_params['hybrid_top_k']}
+Rerank Top K: {retrieval_params['rerank_top_k']}"""
+            
+            demo.load(
+                fn=display_current_params,
+                outputs=[current_params_display]
             )
             
-            question_input.submit(
-                fn=answer_question_func,
-                inputs=[question_input],
-                outputs=[answer_output, sources_output, chunks_output]
+            update_btn.click(
+                fn=display_current_params,
+                outputs=[current_params_display]
             )
+        
+        switch_btn.click(
+            fn=switch_model_func,
+            inputs=[model_dropdown],
+            outputs=[model_status]
+        )
+        
+        ask_btn.click(
+            fn=answer_question_func,
+            inputs=[question_input],
+            outputs=[answer_output, sources_output, chunks_output]
+        )
+        
+        question_input.submit(
+            fn=answer_question_func,
+            inputs=[question_input],
+            outputs=[answer_output, sources_output, chunks_output]
+        )
     return demo
 
 
@@ -389,13 +546,6 @@ reranker = None
 vector_index = None
 current_model = DEFAULT_MODEL
 
-def main_answer_question(question):
-    global query_engine, reranker, current_model, chunks_df
-    answer_html, sources_html, chunks_html = answer_question(
-        question, query_engine, reranker, current_model, chunks_df
-    )
-    return answer_html, sources_html, chunks_html
-
 def main_switch_model(model_name):
     global query_engine, vector_index, current_model
     
@@ -406,9 +556,6 @@ def main_switch_model(model_name):
     
     return status_message
 
-
-
-
 def main():
     global query_engine, chunks_df, reranker, vector_index, current_model
     GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY", "")

config.py

@@ -5,7 +5,7 @@ SIMILARITY_THRESHOLD = 0.7
 RAG_FILES_DIR = "rag_files"
 PROCESSED_DATA_FILE = "processed_chunks.csv"
 
-REPO_ID = "MrSimple01/AIEXP_RAG_FILES"
+REPO_ID = "RAG-AIEXP/ragfiles"
 faiss_index_filename = "cleaned_faiss_index.index"
 CHUNKS_FILENAME = "processed_chunks.csv"
 TABLE_DATA_DIR = "Табличные данные_JSON"
@@ -18,7 +18,6 @@ OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
 HF_REPO_ID = "MrSimple01/AIEXP_RAG_FILES"
 HF_TOKEN = os.getenv('HF_TOKEN')
 
-# Available models configuration
 AVAILABLE_MODELS = {
     "Gemini 2.5 Flash": {
         "provider": "google",
@@ -52,8 +51,22 @@ DEFAULT_MODEL = "Gemini 2.5 Flash"
 CHUNK_SIZE = 1500
 CHUNK_OVERLAP = 128
 
-MAX_CHARS_TABLE = 4500
-MAX_ROWS_TABLE = 50
+MAX_CHARS_TABLE = 3000
+MAX_ROWS_TABLE = 30
+
+
+QUERY_EXPANSION_PROMPT = """Ты — интеллектуальный помощник для расширения поисковых запросов по стандартам ГОСТ, ТУ, ISO, EN и другой технической документации.
+Твоя цель — помочь системе найти все возможные формулировки вопроса, включая те, где встречаются редкие или неочевидные термины.
+Пользователь задал вопрос: "{original_query}"
+
+Сгенерируй 5 вариантов запроса, которые:
+Сохраняют смысл исходного вопроса
+Используют синонимы и технические термины (например: "сталь" → "сплав", "марка", "материал")
+Добавляют возможные контекстные уточнения (например: "ГОСТ", "ТУ", "марка", "лист", "труба", "прокат", "применение", "химический состав")
+Могут охватывать как частотные, так и редкие термины
+Краткие — не более 10 слов каждая
+
+Верни только 5 запросов, каждый с новой строки, без нумерации и пояснений."""
 
 CUSTOM_PROMPT = """
 Вы являетесь высокоспециализированным Ассистентом для анализа нормативных документов (AIEXP). Ваша цель - предоставлять точные, корректные и контекстно релевантные ответы исключительно на основе предоставленного контекста из нормативной документации.

converters/converter.py

@@ -0,0 +1,116 @@
+from config import *
+from utils import log_message
+import json
+import pandas as pd
+import os
+
+def process_uploaded_file(file, file_type):
+    """Обработка загруженного файла и добавление в систему"""
+    try:
+        if file is None:
+            return "❌ Файл не выбран"
+        
+        from huggingface_hub import HfApi
+        import tempfile
+        import shutil
+        
+        # Создаем временную директорию
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Сохраняем загруженный файл
+            file_path = os.path.join(temp_dir, file.name)
+            shutil.copy(file.name, file_path)
+            
+            # Определяем целевую директорию на HuggingFace
+            if file_type == "Таблица":
+                target_dir = TABLE_DATA_DIR
+                # Конвертируем Excel в JSON
+                if file.name.endswith(('.xlsx', '.xls')):
+                    json_path = convert_single_excel_to_json(file_path, temp_dir)
+                    upload_file = json_path
+                else:
+                    upload_file = file_path
+            elif file_type == "Изображение (метаданные)":
+                target_dir = IMAGE_DATA_DIR
+                # Конвертируем Excel в CSV
+                if file.name.endswith(('.xlsx', '.xls')):
+                    csv_path = convert_single_excel_to_csv(file_path, temp_dir)
+                    upload_file = csv_path
+                else:
+                    upload_file = file_path
+            else:  # JSON документ
+                target_dir = JSON_FILES_DIR
+                upload_file = file_path
+            
+            # Загружаем на HuggingFace
+            api = HfApi()
+            api.upload_file(
+                path_or_fileobj=upload_file,
+                path_in_repo=f"{target_dir}/{os.path.basename(upload_file)}",
+                repo_id=HF_REPO_ID,
+                token=HF_TOKEN,
+                repo_type="dataset"
+            )
+            
+            log_message(f"Файл {file.name} успешно загружен в {target_dir}")
+            return f"✅ Файл успешно загружен и обработан: {os.path.basename(upload_file)}\n⚠️ Перезапустите систему для применения изменений"
+            
+    except Exception as e:
+        error_msg = f"Ошибка обработки файла: {str(e)}"
+        log_message(error_msg)
+        return f"❌ {error_msg}"
+
+def convert_single_excel_to_json(excel_path, output_dir):
+    """Конвертация одного Excel файла в JSON для таблиц"""
+    df_dict = pd.read_excel(excel_path, sheet_name=None)
+    
+    result = {
+        "document": os.path.basename(excel_path),
+        "total_sheets": len(df_dict),
+        "sheets": []
+    }
+    
+    for sheet_name, df in df_dict.items():
+        if df.empty or "Номер таблицы" not in df.columns:
+            continue
+            
+        df = df.dropna(how='all').fillna("")
+        grouped = df.groupby("Номер таблицы")
+        
+        for table_number, group in grouped:
+            group = group.reset_index(drop=True)
+            
+            sheet_data = {
+                "sheet_name": sheet_name,
+                "document_id": str(group.iloc[0].get("Обозначение документа", "")),
+                "section": str(group.iloc[0].get("Раздел документа", "")),
+                "table_number": str(table_number),
+                "table_title": str(group.iloc[0].get("Название таблицы", "")),
+                "table_description": str(group.iloc[0].get("Примечание", "")),
+                "headers": [col for col in df.columns if col not in 
+                           ["Обозначение документа", "Раздел документа", "Номер таблицы", 
+                            "Название таблицы", "Примечание"]],
+                "data": []
+            }
+            
+            for _, row in group.iterrows():
+                row_dict = {col: str(row[col]) if pd.notna(row[col]) else "" 
+                           for col in sheet_data["headers"]}
+                sheet_data["data"].append(row_dict)
+            
+            result["sheets"].append(sheet_data)
+    
+    json_filename = os.path.basename(excel_path).replace('.xlsx', '.json').replace('.xls', '.json')
+    json_path = os.path.join(output_dir, json_filename)
+    
+    with open(json_path, 'w', encoding='utf-8') as f:
+        json.dump(result, f, ensure_ascii=False, indent=2)
+    
+    return json_path
+
+def convert_single_excel_to_csv(excel_path, output_dir):
+    """Конвертация одного Excel файла в CSV для изображений"""
+    df = pd.read_excel(excel_path)
+    csv_filename = os.path.basename(excel_path).replace('.xlsx', '.csv').replace('.xls', '.csv')
+    csv_path = os.path.join(output_dir, csv_filename)
+    df.to_csv(csv_path, index=False, encoding='utf-8')
+    return csv_path

documents_prep.py

@@ -6,21 +6,83 @@ from llama_index.core import Document
 from llama_index.core.text_splitter import SentenceSplitter
 from my_logging import log_message
 from config import CHUNK_SIZE, CHUNK_OVERLAP, MAX_CHARS_TABLE, MAX_ROWS_TABLE
+import re
+
+def normalize_text(text):
+    if not text:
+        return text
+    
+    # Replace Cyrillic 'C' with Latin 'С' (U+0421)
+    # This is for welding types like C-25 -> С-25
+    text = text.replace('С-', 'C')
+    text = re.sub(r'\bС(\d)', r'С\1', text)
+    return text
+
+def normalize_steel_designations(text):
+    if not text:
+        return text, 0, []
+
+    import re
+    
+    changes_count = 0
+    changes_list = []
+
+    # Mapping of Cyrillic to Latin for steel designations
+    replacements = {
+        'Х': 'X',  
+        'Н': 'H', 
+        'Т': 'T', 
+        'С': 'C', 
+        'В': 'B',  
+        'К': 'K',  
+        'М': 'M', 
+        'А': 'A',  
+        'Р': 'P', 
+    }
+
+    # Pattern: starts with digits, then letters+digits (steel grade pattern)
+    # Examples: 08Х18Н10Т, 12Х18Н9, 10Н17Н13М2Т, СВ-08Х19Н10
+    pattern = r'\b\d{1,3}(?:[A-ZА-ЯЁ]\d*)+\b'
+    
+    # Also match welding wire patterns like СВ-08Х19Н10
+    pattern_wire = r'\b[СC][ВB]-\d{1,3}(?:[A-ZА-ЯЁ]\d*)+\b'
+
+    def replace_in_steel_grade(match):
+        nonlocal changes_count, changes_list
+        original = match.group(0)
+        converted = ''.join(replacements.get(ch, ch) for ch in original)
+        if converted != original:
+            changes_count += 1
+            changes_list.append(f"{original} → {converted}")
+        return converted
+    normalized_text = re.sub(pattern, replace_in_steel_grade, text)
+    normalized_text = re.sub(pattern_wire, replace_in_steel_grade, normalized_text)
+
+    return normalized_text, changes_count, changes_list
 
 def chunk_text_documents(documents):
     text_splitter = SentenceSplitter(
         chunk_size=CHUNK_SIZE,
         chunk_overlap=CHUNK_OVERLAP
     )
+    total_normalizations = 0
+    chunks_with_changes = 0
     
     chunked = []
     for doc in documents:
         chunks = text_splitter.get_nodes_from_documents([doc])
         for i, chunk in enumerate(chunks):
+            original_text = chunk.text
+            chunk.text, changes, change_list = normalize_steel_designations(chunk.text)  
+            
+            if changes > 0:
+                chunks_with_changes += 1
+                total_normalizations += changes
+            
             chunk.metadata.update({
                 'chunk_id': i,
                 'total_chunks': len(chunks),
-                'chunk_size': len(chunk.text)  # Add chunk size
+                'chunk_size': len(chunk.text)
             })
             chunked.append(chunk)
     
@@ -31,23 +93,14 @@ def chunk_text_documents(documents):
         max_size = max(len(c.text) for c in chunked)
         log_message(f"✓ Text: {len(documents)} docs → {len(chunked)} chunks")
         log_message(f"  Size stats: avg={avg_size:.0f}, min={min_size}, max={max_size} chars")
+        log_message(f"  Steel designation normalization:")
+        log_message(f"    - Chunks with changes: {chunks_with_changes}/{len(chunked)}")
+        log_message(f"    - Total steel grades normalized: {total_normalizations}")
+        log_message(f"    - Avg per affected chunk: {total_normalizations/chunks_with_changes:.1f}" if chunks_with_changes > 0 else "    - No normalizations needed")
     
-    return chunked
-
-def normalize_text(text):
-    if not text:
-        return text
-    
-    # Replace Cyrillic 'C' with Latin 'С' (U+0421)
-    # This is for welding types like C-25 -> С-25
-    text = text.replace('С-', 'C')
-    
-    # Also handle cases like "Type C" or variations
-    import re
-    # Match "C" followed by digit or space in context of welding types
-    text = re.sub(r'\bС(\d)', r'С\1', text)
+    log_message("="*60)
     
-    return text
+    return chunked
 
 def chunk_table_by_content(table_data, doc_id, max_chars=MAX_CHARS_TABLE, max_rows=MAX_ROWS_TABLE):
     headers = table_data.get('headers', [])
@@ -55,49 +108,108 @@ def chunk_table_by_content(table_data, doc_id, max_chars=MAX_CHARS_TABLE, max_ro
     table_num = table_data.get('table_number', 'unknown')
     table_title = table_data.get('table_title', '')
     section = table_data.get('section', '')
+    sheet_name = table_data.get('sheet_name', '')
    
+    # Apply steel designation normalization to title and section
+    table_title, title_changes, title_list = normalize_steel_designations(str(table_title))
+    section, section_changes, section_list = normalize_steel_designations(section)
+    
     table_num_clean = str(table_num).strip()
-    table_title_normalized = normalize_text(str(table_title))  # NORMALIZE TITLE
     
     import re
-    if 'приложени' in section.lower():
-        appendix_match = re.search(r'приложени[еия]\s*(\d+|[а-яА-Я])', section.lower())
-        if appendix_match:
-            appendix_num = appendix_match.group(1).upper()
-            table_identifier = f"{table_num_clean} Приложение {appendix_num}"
+    
+    if table_num_clean in ['-', '', 'unknown', 'nan']:
+        if 'приложени' in sheet_name.lower() or 'приложени' in section.lower():
+            appendix_match = re.search(r'приложени[еия]\s*[№]?\s*(\d+)', 
+                                      (sheet_name + ' ' + section).lower())
+            if appendix_match:
+                appendix_num = appendix_match.group(1)
+                table_identifier = f"Приложение {appendix_num}"
+            else:
+                table_identifier = "Приложение"
         else:
-            table_identifier = table_num_clean
+            if table_title:
+                first_words = ' '.join(table_title.split()[:5])
+                table_identifier = f"{first_words}"
+            else:
+                table_identifier = section.split(',')[0] if section else "БезНомера"
     else:
-        table_identifier = table_num_clean
+        if 'приложени' in section.lower():
+            appendix_match = re.search(r'приложени[еия]\s*[№]?\s*(\d+)', section.lower())
+            if appendix_match:
+                appendix_num = appendix_match.group(1)
+                table_identifier = f"{table_num_clean} Приложение {appendix_num}"
+            else:
+                table_identifier = table_num_clean
+        else:
+            table_identifier = table_num_clean
     
     if not rows:
         return []
     
     log_message(f"  📊 Processing: {doc_id} - {table_identifier} ({len(rows)} rows)")
     
-    # Calculate base metadata size with NORMALIZED title
-    base_content = format_table_header(doc_id, table_identifier, table_num, table_title_normalized, section, headers)
+    # Normalize all row content (including steel designations)
+    normalized_rows = []
+    total_row_changes = 0
+    rows_with_changes = 0
+    all_row_changes = []  
+    
+    for row in rows:
+        if isinstance(row, dict):
+            normalized_row = {}
+            row_had_changes = False
+            for k, v in row.items():
+                normalized_val, changes, change_list = normalize_steel_designations(str(v))
+                normalized_row[k] = normalized_val
+                if changes > 0:
+                    total_row_changes += changes
+                    row_had_changes = True
+                    all_row_changes.extend(change_list)  # NEW
+            if row_had_changes:
+                rows_with_changes += 1
+            normalized_rows.append(normalized_row)
+        else:
+            normalized_rows.append(row)
+    
+    # Log normalization stats with examples
+    if total_row_changes > 0 or title_changes > 0 or section_changes > 0:
+        log_message(f"    Steel normalization: title={title_changes}, section={section_changes}, "
+                   f"rows={rows_with_changes}/{len(rows)} ({total_row_changes} total)")
+        if title_list:
+            log_message(f"      Title changes: {', '.join(title_list[:3])}")
+        if section_list:
+            log_message(f"      Section changes: {', '.join(section_list[:3])}")
+        if all_row_changes:
+            log_message(f"      Row examples: {', '.join(all_row_changes[:5])}")    
+    base_content = format_table_header(doc_id, table_identifier, table_num, 
+                                       table_title, section, headers, 
+                                       sheet_name)
     base_size = len(base_content)
     available_space = max_chars - base_size - 200 
     
     # If entire table fits, return as one chunk
-    full_rows_content = format_table_rows([{**row, '_idx': i+1} for i, row in enumerate(rows)])
-    if base_size + len(full_rows_content) <= max_chars and len(rows) <= max_rows:
+    full_rows_content = format_table_rows([{**row, '_idx': i+1} 
+                                           for i, row in enumerate(normalized_rows)])
+    
+    if base_size + len(full_rows_content) <= max_chars and len(normalized_rows) <= max_rows:
         content = base_content + full_rows_content + format_table_footer(table_identifier, doc_id)
         
         metadata = {
             'type': 'table',
             'document_id': doc_id,
-            'table_number': table_num_clean,
-            'table_identifier': normalize_text(table_identifier),  # NORMALIZE identifier
-            'table_title': table_title_normalized,  # NORMALIZED
+            'table_number': table_num_clean if table_num_clean not in ['-', 'unknown'] else table_identifier,
+            'table_identifier': table_identifier,
+            'table_title': table_title,
             'section': section,
-            'total_rows': len(rows),
+            'sheet_name': sheet_name,
+            'total_rows': len(normalized_rows),
             'chunk_size': len(content),
-            'is_complete_table': True
+            'is_complete_table': True,
+            'keywords': f"{doc_id} {table_identifier} {table_title} {section} сталь материал"
         }
         
-        log_message(f"    Single chunk: {len(content)} chars, {len(rows)} rows")
+        log_message(f"    Single chunk: {len(content)} chars, {len(normalized_rows)} rows")
         return [Document(text=content, metadata=metadata)]
 
     chunks = []
@@ -105,30 +217,33 @@ def chunk_table_by_content(table_data, doc_id, max_chars=MAX_CHARS_TABLE, max_ro
     current_size = 0
     chunk_num = 0
     
-    for i, row in enumerate(rows):
+    for i, row in enumerate(normalized_rows):
         row_text = format_single_row(row, i + 1)
         row_size = len(row_text)
         
-        should_split = (current_size + row_size > available_space or len(current_rows) >= max_rows) and current_rows
+        should_split = (current_size + row_size > available_space or 
+                       len(current_rows) >= max_rows) and current_rows
         
         if should_split:
             content = base_content + format_table_rows(current_rows)
-            content += f"\n\nСтроки {current_rows[0]['_idx']}-{current_rows[-1]['_idx']} из {len(rows)}\n"
+            content += f"\n\nСтроки {current_rows[0]['_idx']}-{current_rows[-1]['_idx']} из {len(normalized_rows)}\n"
             content += format_table_footer(table_identifier, doc_id)
             
             metadata = {
                 'type': 'table',
                 'document_id': doc_id,
-                'table_number': table_num_clean,
-                'table_identifier': normalize_text(table_identifier),  # NORMALIZE
-                'table_title': table_title_normalized,  # NORMALIZED
+                'table_number': table_num_clean if table_num_clean not in ['-', 'unknown'] else table_identifier,
+                'table_identifier': table_identifier,
+                'table_title': table_title,
                 'section': section,
+                'sheet_name': sheet_name,
                 'chunk_id': chunk_num,
                 'row_start': current_rows[0]['_idx'] - 1,
                 'row_end': current_rows[-1]['_idx'],
-                'total_rows': len(rows),
+                'total_rows': len(normalized_rows),
                 'chunk_size': len(content),
-                'is_complete_table': False
+                'is_complete_table': False,
+                'keywords': f"{doc_id} {table_identifier} {table_title} {section} сталь материал"
             }
             
             chunks.append(Document(text=content, metadata=metadata))
@@ -138,31 +253,31 @@ def chunk_table_by_content(table_data, doc_id, max_chars=MAX_CHARS_TABLE, max_ro
             current_rows = []
             current_size = 0
         
-        # Add row with index
         row_copy = row.copy() if isinstance(row, dict) else {'data': row}
         row_copy['_idx'] = i + 1
         current_rows.append(row_copy)
         current_size += row_size
-    
-    # Add final chunk
+
     if current_rows:
         content = base_content + format_table_rows(current_rows)
-        content += f"\n\nСтроки {current_rows[0]['_idx']}-{current_rows[-1]['_idx']} из {len(rows)}\n"
+        content += f"\n\nСтроки {current_rows[0]['_idx']}-{current_rows[-1]['_idx']} из {len(normalized_rows)}\n"
         content += format_table_footer(table_identifier, doc_id)
         
         metadata = {
             'type': 'table',
             'document_id': doc_id,
-            'table_number': table_num_clean,
-            'table_identifier': normalize_text(table_identifier),  # NORMALIZE
-            'table_title': table_title_normalized,  # NORMALIZED
+            'table_number': table_num_clean if table_num_clean not in ['-', 'unknown'] else table_identifier,
+            'table_identifier': table_identifier,
+            'table_title': table_title,
             'section': section,
+            'sheet_name': sheet_name,
             'chunk_id': chunk_num,
             'row_start': current_rows[0]['_idx'] - 1,
             'row_end': current_rows[-1]['_idx'],
-            'total_rows': len(rows),
+            'total_rows': len(normalized_rows),
             'chunk_size': len(content),
-            'is_complete_table': False
+            'is_complete_table': False,
+            'keywords': f"{doc_id} {table_identifier} {table_title} {section} сталь материал"
         }
         
         chunks.append(Document(text=content, metadata=metadata))
@@ -170,33 +285,36 @@ def chunk_table_by_content(table_data, doc_id, max_chars=MAX_CHARS_TABLE, max_ro
     
     return chunks
 
-
-# MODIFIED: Update format_table_header function
-def format_table_header(doc_id, table_identifier, table_num, table_title, section, headers):
+def format_table_header(doc_id, table_identifier, table_num, table_title, section, headers, sheet_name=''):
     content = f"ТАБЛИЦА {normalize_text(table_identifier)} из документа {doc_id}\n"
     
-    # Add table type/number prominently for matching
-    if table_num:
-        content += f"ТИП: {normalize_text(table_num)}\n"
+    # Add multiple searchable identifiers
+    if table_num and table_num not in ['-', 'unknown']:
+        content += f"НОМЕР ТАБЛИЦЫ: {normalize_text(table_num)}\n"
+    
+    if sheet_name:
+        content += f"ЛИСТ: {sheet_name}\n"
     
     if table_title:
         content += f"НАЗВАНИЕ: {normalize_text(table_title)}\n"
     
     if section:
         content += f"РАЗДЕЛ: {section}\n"
+
+    content += f"КЛЮЧЕВЫЕ СЛОВА: материалы стали марки стандарты {doc_id}\n"
     
     content += f"{'='*70}\n"
     
     if headers:
-        header_str = ' | '.join(str(h) for h in headers)
+        # Normalize headers too
+        normalized_headers = [normalize_text(str(h)) for h in headers]
+        header_str = ' | '.join(normalized_headers)
         content += f"ЗАГОЛОВКИ: {header_str}\n\n"
     
     content += "ДАННЫЕ:\n"
     return content
 
-
 def format_single_row(row, idx):
-    """Format a single row"""
     if isinstance(row, dict):
         parts = [f"{k}: {v}" for k, v in row.items() 
                 if v and str(v).strip() and str(v).lower() not in ['nan', 'none', '']]
@@ -208,18 +326,14 @@ def format_single_row(row, idx):
             return f"{idx}. {' | '.join(parts)}\n"
     return ""
 
-
 def format_table_rows(rows):
-    """Format multiple rows"""
     content = ""
     for row in rows:
         idx = row.get('_idx', 0)
         content += format_single_row(row, idx)
     return content
 
-
 def format_table_footer(table_identifier, doc_id):
-    """Format table footer"""
     return f"\n{'='*70}\nКОНЕЦ ТАБЛИЦЫ {table_identifier} ИЗ {doc_id}\n"
 
 def load_json_documents(repo_id, hf_token, json_dir):
@@ -290,7 +404,6 @@ def load_json_documents(repo_id, hf_token, json_dir):
                             stats['failed'] += 1
                             continue
                         
-                        # Try UTF-8 first (most common)
                         try:
                             text_content = file_content.decode('utf-8')
                         except UnicodeDecodeError:
@@ -298,7 +411,6 @@ def load_json_documents(repo_id, hf_token, json_dir):
                                 text_content = file_content.decode('utf-8-sig')
                             except UnicodeDecodeError:
                                 try:
-                                    # Try UTF-16 (the issue you're seeing)
                                     text_content = file_content.decode('utf-16')
                                 except UnicodeDecodeError:
                                     try:
@@ -345,13 +457,11 @@ def load_json_documents(repo_id, hf_token, json_dir):
     log_message(f"  Success: {stats['success']}")
     log_message(f"  Empty: {stats['empty']}")
     log_message(f"  Failed: {stats['failed']}")
-    log_message(f"  Total sections: {len(documents)}")
     log_message(f"="*60)
     
     return documents
 
 def extract_sections_from_json(json_path):
-    """Extract sections from a single JSON file"""
     documents = []
     
     try:
@@ -401,14 +511,15 @@ def extract_sections_from_json(json_path):
     
     return documents
 
-
 def load_table_documents(repo_id, hf_token, table_dir):
     log_message("Loading tables...")
-    
+    log_message("="*60)    
     files = list_repo_files(repo_id=repo_id, repo_type="dataset", token=hf_token)
     table_files = [f for f in files if f.startswith(table_dir) and f.endswith('.json')]
     
     all_chunks = []
+    tables_processed = 0
+    
     for file_path in table_files:
         try:
             local_path = hf_hub_download(
@@ -425,20 +536,22 @@ def load_table_documents(repo_id, hf_token, table_dir):
             
             for sheet in data.get('sheets', []):
                 sheet_doc_id = sheet.get('document_id', sheet.get('document', file_doc_id))
+                tables_processed += 1
                 
-                # Use the consistent MAX_CHARS_TABLE from config
-                chunks = chunk_table_by_content(sheet, sheet_doc_id, max_chars=MAX_CHARS_TABLE, max_rows=MAX_ROWS_TABLE)
+                chunks = chunk_table_by_content(sheet, sheet_doc_id, 
+                                               max_chars=MAX_CHARS_TABLE, 
+                                               max_rows=MAX_ROWS_TABLE)
                 all_chunks.extend(chunks)
                 
         except Exception as e:
             log_message(f"Error loading {file_path}: {e}")
     
-    log_message(f"✓ Loaded {len(all_chunks)} table chunks")
+    log_message(f"✓ Loaded {len(all_chunks)} table chunks from {tables_processed} tables")
+    log_message("="*60)
+    
     return all_chunks
 
-
 def load_image_documents(repo_id, hf_token, image_dir):
-    """Load image descriptions"""
     log_message("Loading images...")
     
     files = list_repo_files(repo_id=repo_id, repo_type="dataset", token=hf_token)
@@ -484,7 +597,6 @@ def load_image_documents(repo_id, hf_token, image_dir):
     
     return documents
 
-
 def load_all_documents(repo_id, hf_token, json_dir, table_dir, image_dir):
     """Main loader - combines all document types"""
     log_message("="*60)

index_retriever.py

@@ -10,7 +10,6 @@ from config import CUSTOM_PROMPT, PROMPT_SIMPLE_POISK
 
 def create_vector_index(documents):
     log_message("Строю векторный индекс")
-    
     connection_type_sources = {}
     table_count = 0
     
@@ -22,21 +21,9 @@ def create_vector_index(documents):
                 table_id = f"{doc.metadata.get('document_id', 'unknown')} Table {doc.metadata.get('table_number', 'N/A')}"
                 if conn_type not in connection_type_sources:
                     connection_type_sources[conn_type] = []
-                connection_type_sources[conn_type].append(table_id)
-    
-    log_message("="*60)
-    log_message(f"INDEXING {table_count} TABLE CHUNKS")
-    log_message("CONNECTION TYPES IN INDEX WITH SOURCES:")
-    for conn_type in sorted(connection_type_sources.keys()):
-        sources = list(set(connection_type_sources[conn_type]))  # Unique sources
-        log_message(f"  {conn_type}: {len(connection_type_sources[conn_type])} chunks from {len(sources)} tables")
-        for src in sources:
-            log_message(f"    - {src}")
-    log_message("="*60)
-    
+                connection_type_sources[conn_type].append(table_id)  
     return VectorStoreIndex.from_documents(documents)
 
-
 def rerank_nodes(query, nodes, reranker, top_k=25, min_score_threshold=0.5):
     if not nodes or not reranker:
         return nodes[:top_k]
@@ -48,13 +35,10 @@ def rerank_nodes(query, nodes, reranker, top_k=25, min_score_threshold=0.5):
         scores = reranker.predict(pairs)
         scored_nodes = list(zip(nodes, scores))
         
-        scored_nodes.sort(key=lambda x: x[1], reverse=True)
-        
-        # Apply threshold
+        scored_nodes.sort(key=lambda x: x[1], reverse=True)        
         filtered = [(node, score) for node, score in scored_nodes if score >= min_score_threshold]
         
         if not filtered:
-            # Lower threshold if nothing passes
             filtered = scored_nodes[:top_k]
         
         log_message(f"Выбрано {min(len(filtered), top_k)} узлов")
@@ -65,24 +49,25 @@ def rerank_nodes(query, nodes, reranker, top_k=25, min_score_threshold=0.5):
         log_message(f"Ошибка переранжировки: {str(e)}")
         return nodes[:top_k]
 
-def create_query_engine(vector_index):
+def create_query_engine(vector_index, vector_top_k=50, bm25_top_k=50, 
+                       similarity_cutoff=0.55, hybrid_top_k=100):
     try:
         from config import CUSTOM_PROMPT
         
         bm25_retriever = BM25Retriever.from_defaults(
             docstore=vector_index.docstore, 
-            similarity_top_k=70 
+            similarity_top_k=bm25_top_k 
         )
         
         vector_retriever = VectorIndexRetriever(
             index=vector_index, 
-            similarity_top_k=70,
-            similarity_cutoff=0.6
+            similarity_top_k=vector_top_k,  
+            similarity_cutoff=similarity_cutoff 
         )
         
         hybrid_retriever = QueryFusionRetriever(
             [vector_retriever, bm25_retriever],
-            similarity_top_k=140,
+            similarity_top_k=hybrid_top_k,  
             num_queries=1
         )
         
@@ -97,7 +82,9 @@ def create_query_engine(vector_index):
             response_synthesizer=response_synthesizer
         )
         
-        log_message("Query engine успешно создан")
+        log_message(f"Query engine created: vector_top_k={vector_top_k}, "
+                   f"bm25_top_k={bm25_top_k}, similarity_cutoff={similarity_cutoff}, "
+                   f"hybrid_top_k={hybrid_top_k}")
         return query_engine
         
     except Exception as e:

table_prep.py

@@ -1,229 +0,0 @@
-from collections import defaultdict
-import json
-from huggingface_hub import hf_hub_download, list_repo_files
-from llama_index.core import Document
-from my_logging import log_message
-from config import MAX_CHARS_TABLE, MAX_ROWS_TABLE
-
-
-def create_table_content(table_data):
-    doc_id = table_data.get('document_id', table_data.get('document', 'Неизвестно'))
-    table_num = table_data.get('table_number', 'Неизвестно')
-    table_title = table_data.get('table_title', 'Неизвестно')
-    section = table_data.get('section', 'Неизвестно')
-    
-    content = f"Таблица: {table_num}\n"
-    content += f"Название: {table_title}\n"
-    content += f"Документ: {doc_id}\n"
-    content += f"Раздел: {section}\n"
-    
-    headers = table_data.get('headers', [])
-    if headers:
-        content += f"\nЗаголовки: {' | '.join(headers)}\n"
-    
-    if 'data' in table_data and isinstance(table_data['data'], list):
-        content += "\nДанные таблицы:\n"
-        for row_idx, row in enumerate(table_data['data'], start=1):
-            if isinstance(row, dict):
-                row_text = " | ".join([f"{k}: {v}" for k, v in row.items() if v])
-                content += f"Строка {row_idx}: {row_text}\n"
-    
-    return content
-
-def chunk_table_document(doc, max_chunk_size=MAX_CHARS_TABLE, max_rows_per_chunk=MAX_ROWS_TABLE):
-    lines = doc.text.strip().split('\n')
-
-    header_lines = []
-    data_rows = []
-    in_data = False
-
-    for line in lines:
-        if line.startswith('Данные таблицы:'):
-            in_data = True
-            header_lines.append(line)
-        elif in_data and line.startswith('Строка'):
-            data_rows.append(line)
-        elif not in_data:
-            header_lines.append(line)
-
-    header = '\n'.join(header_lines) + '\n'
-
-    if not data_rows:
-        return [doc]
-
-    chunks = []
-    current_rows = []
-    current_size = len(header)
-
-    for row in data_rows:
-        row_size = len(row) + 1
-        # Check both limits: chunk size and row count
-        if ((current_size + row_size > max_chunk_size or len(current_rows) >= max_rows_per_chunk) and current_rows):
-            chunk_text = header + '\n'.join(current_rows)
-            chunks.append(chunk_text)
-            log_message(f"Создана часть таблицы размером {len(chunk_text)} символов с {len(current_rows)} строками")
-            current_rows = []
-            current_size = len(header)
-
-        current_rows.append(row)
-        current_size += row_size
-        log_message(f"Добавлена строка к текущему чанку, текущий размер {current_size} символов")
-
-    # Add final chunk
-    if current_rows:
-        chunk_text = header + '\n'.join(current_rows)
-        chunks.append(chunk_text)
-        log_message(f"Создана финальная часть таблицы размером {len(chunk_text)} символов с {len(current_rows)} строками")
-
-    # Create Document objects
-    chunked_docs = []
-    for i, chunk_text in enumerate(chunks):
-        chunk_doc = Document(
-            text=chunk_text,
-            metadata={
-                "type": "table",
-                "table_number": doc.metadata.get('table_number'),
-                "document_id": doc.metadata.get('document_id'),
-                "section": doc.metadata.get('section'),
-                "chunk_id": i,
-                "total_chunks": len(chunks),
-                "is_chunked": True
-            }
-        )
-        chunked_docs.append(chunk_doc)
-
-    return chunked_docs
-
-
-# def table_to_document(table_data, document_id=None):
-#     if not isinstance(table_data, dict):
-#         return []
-    
-#     doc_id = document_id or table_data.get('document_id') or table_data.get('document', 'Неизвестно')
-#     table_num = table_data.get('table_number', 'Неизвестно')
-#     table_title = table_data.get('table_title', 'Неизвестно')
-#     section = table_data.get('section', 'Неизвестно')
-#     table_rows = table_data.get('data', [])
-    
-#     if not table_rows:
-#         return []
-    
-#     # Build table content
-#     content = f"Таблица: {table_num}\n"
-#     content += f"Название: {table_title}\n"
-#     content += f"Документ: {doc_id}\n"
-#     content += f"Раздел: {section}\n"
-    
-#     headers = table_data.get('headers', [])
-#     if headers:
-#         content += f"\nЗаголовки: {' | '.join(headers)}\n"
-    
-#     content += "\nДанные таблицы:\n"
-#     for row_idx, row in enumerate(table_rows, start=1):
-#         if isinstance(row, dict):
-#             row_text = " | ".join([f"{k}: {v}" for k, v in row.items() if v])
-#             content += f"Строка {row_idx}: {row_text}\n"
-    
-#     # Create base document
-#     base_doc = Document(
-#         text=content,
-#         metadata={
-#             "type": "table",
-#             "table_number": table_num,
-#             "document_id": doc_id,
-#             "section": section
-#         }
-#     )
-#     if len(content) > 4000:
-#         chunks = chunk_table_document(base_doc)
-#         log_message(f"Таблица {table_num} разбита на {len(chunks)} частей")
-#         return chunk_table_document(base_doc)
-#     return [base_doc]
-    
-    
-# def load_table_data(repo_id, hf_token, table_data_dir):    
-#     try:
-#         files = list_repo_files(repo_id=repo_id, repo_type="dataset", token=hf_token)
-#         table_files = [f for f in files if f.startswith(table_data_dir) and f.endswith('.json')]
-        
-#         log_message(f"Найдено {len(table_files)} JSON файлов с таблицами")
-        
-#         table_documents = []
-#         stats = {
-#             'total_tables': 0,
-#             'total_size': 0,
-#             'by_document': defaultdict(lambda: {'count': 0, 'size': 0})
-#         }
-        
-#         for file_path in table_files:
-#             try:
-#                 local_path = hf_hub_download(
-#                     repo_id=repo_id,
-#                     filename=file_path,
-#                     local_dir='',
-#                     repo_type="dataset",
-#                     token=hf_token
-#                 )
-                
-#                 log_message(f"\nОбработка файла: {file_path}")
-                
-#                 with open(local_path, 'r', encoding='utf-8') as f:
-#                     table_data = json.load(f)
-                    
-#                     if isinstance(table_data, dict):
-#                         document_id = table_data.get('document', 'unknown')
-                        
-#                         if 'sheets' in table_data:
-#                             sorted_sheets = sorted(
-#                                 table_data['sheets'],
-#                                 key=lambda sheet: sheet.get('table_number', '')  # or use 'table_number'
-#                             )
-                            
-#                             for sheet in sorted_sheets:
-#                                 sheet['document'] = document_id
-#                                 docs_list = table_to_document(sheet, document_id)
-#                                 table_documents.extend(docs_list)
-                                
-#                                 for doc in docs_list:
-#                                     stats['total_tables'] += 1
-#                                     size = doc.metadata.get('content_size', 0)
-#                                     stats['total_size'] += size
-#                                     stats['by_document'][document_id]['count'] += 1
-#                                     stats['by_document'][document_id]['size'] += size
-#                                     log_message(f"Добавлена таблица {sheet.get('table_number', 'Неизвестно')} из документа {document_id}, размер {size} символов")
-#                         else:
-#                             docs_list = table_to_document(table_data, document_id)
-#                             table_documents.extend(docs_list)
-                            
-#                             for doc in docs_list:
-#                                 stats['total_tables'] += 1
-#                                 size = doc.metadata.get('content_size', 0)
-#                                 stats['total_size'] += size
-#                                 stats['by_document'][document_id]['count'] += 1
-#                                 stats['by_document'][document_id]['size'] += size
-                                
-                        
-#             except Exception as e:
-#                 log_message(f"❌ ОШИБКА файла {file_path}: {str(e)}")
-#                 continue
-        
-#         # Log summary statistics
-#         log_message("\n" + "=" * 60)
-#         log_message("СТАТИСТИКА ПО ТАБЛИЦАМ")
-#         log_message("=" * 60)
-#         log_message(f"Всего таблиц добавлено: {stats['total_tables']}")
-#         log_message(f"Общий размер: {stats['total_size']:,} символов")
-#         log_message(f"Средний размер таблицы: {stats['total_size'] // stats['total_tables'] if stats['total_tables'] > 0 else 0:,} символов")
-        
-#         log_message("\nПо документам:")
-#         for doc_id, doc_stats in sorted(stats['by_document'].items()):
-#             log_message(f"  • {doc_id}: {doc_stats['count']} таблиц, "
-#                        f"{doc_stats['size']:,} символов")
-        
-#         log_message("=" * 60)
-        
-#         return table_documents
-        
-#     except Exception as e:
-#         log_message(f"❌ КРИТИЧЕСКАЯ ОШИБКА загрузки табличных данных: {str(e)}")
-#         return []

utils.py

@@ -9,6 +9,18 @@ import time
 from index_retriever import rerank_nodes
 from my_logging import log_message
 from config import PROMPT_SIMPLE_POISK
+from config import QUERY_EXPANSION_PROMPT
+from documents_prep import normalize_text, normalize_steel_designations
+
+
+KEYWORD_EXPANSIONS = {
+    "08X18H10T": ["Листы", "Трубы", "Поковки", "Крепежные изделия", "Сортовой прокат", "Отливки"],
+    "12X18H10T": ["Листы", "Поковки", "Сортовой прокат"],
+    "10X17H13M2T": ["Трубы", "Арматура", "Поковки", "Фланцы"],
+    "20X23H18": ["Листы", "Сортовой прокат", "Поковки"],
+    "03X17H14M3": ["Трубы", "Листы", "Проволока"],
+    "СВ-08X19H10": ["Сварочная проволока", "Сварка", "Сварочные материалы"],
+}
 
 def get_llm_model(model_name):
     try:
@@ -172,65 +184,83 @@ def deduplicate_nodes(nodes):
     
     return unique_nodes
 
-def debug_search_tables(vector_index, search_term="С-25"):
-    """Debug function to find all tables containing a specific term"""
-    all_nodes = list(vector_index.docstore.docs.values())
+def enhance_query_with_keywords(query):
+    query_upper = query.upper()
     
-    matching = []
-    for node in all_nodes:
-        if node.metadata.get('type') == 'table':
-            text = node.get_content()
-            if search_term in text or search_term in node.metadata.get('table_title', ''):
-                matching.append({
-                    'doc_id': node.metadata.get('document_id'),
-                    'table_num': node.metadata.get('table_number'),
-                    'title': node.metadata.get('table_title', '')[:100]
-                })
+    added_context = []
+    keywords_found = []
     
-    log_message(f"\n{'='*60}")
-    log_message(f"DEBUG: Found {len(matching)} tables containing '{search_term}'")
-    for m in matching:
-        log_message(f"  • {m['doc_id']} - Table {m['table_num']}: {m['title']}")
-    log_message(f"{'='*60}\n")
+    for keyword, expansions in KEYWORD_EXPANSIONS.items():
+        keyword_upper = keyword.upper()
+
+        if keyword_upper in query_upper:
+            context = ' '.join(expansions)
+            added_context.append(context)
+            keywords_found.append(keyword)
+            log_message(f"  Found keyword '{keyword}': added context '{context}'")
     
-    return matching
+    if added_context:
+        unique_context = ' '.join(set(' '.join(added_context).split()))
+        enhanced = f"{query} {unique_context}"
+        
+        log_message(f"Enhanced query with keywords: {', '.join(keywords_found)}")
+        log_message(f"Added context: {unique_context[:100]}...")
+        
+        return enhanced
+    return f"{query}"
 
-from documents_prep import normalize_text
 
-# MODIFIED: Update answer_question function
-def answer_question(question, query_engine, reranker, current_model, chunks_df=None):
-    # NORMALIZE the question to convert C to С
+def answer_question(question, query_engine, reranker, current_model, chunks_df=None, rerank_top_k=20):
     normalized_question = normalize_text(question)
+    normalized_question_2, query_changes, change_list = normalize_steel_designations(question) 
+    enhanced_question = enhance_query_with_keywords(normalized_question_2)
+    
+    try:
+        llm = get_llm_model(current_model)
+        expansion_prompt = QUERY_EXPANSION_PROMPT.format(original_query=enhanced_question)
+        expanded_queries = llm.complete(expansion_prompt).text.strip()
+        enhanced_question = f"{enhanced_question} {expanded_queries}"
+        log_message(f"LLM expanded query: {expanded_queries[:200]}...")
+    except Exception as e:
+        log_message(f"Query expansion failed: {e}, using keyword-only enhancement")
     
+    if change_list:
+        log_message(f"Query changes: {', '.join(change_list)}")
+    if change_list:
+        log_message(f"Query changes: {', '.join(change_list)}")    
     if query_engine is None:
         return "<div style='background-color: #e53e3e; color: white; padding: 20px; border-radius: 10px;'>Система не инициализирована</div>", "", ""
     
     try:
         start_time = time.time()
-        # Use NORMALIZED question for retrieval
-        retrieved_nodes = query_engine.retriever.retrieve(normalized_question)
+        retrieved_nodes = query_engine.retriever.retrieve(enhanced_question)
         log_message(f"user query: {question}")
-        log_message(f"normalized query: {normalized_question}")
-
-        
-        log_message(f"RETRIEVED: {len(retrieved_nodes)} nodes")
-        
+        log_message(f"after steel normalization: {normalized_question_2}")
+        log_message(f"enhanced query: {enhanced_question}")                 
         unique_retrieved = deduplicate_nodes(retrieved_nodes)
-
-        # DEBUG: Log what was retrieved
         log_message(f"RETRIEVED: unique {len(unique_retrieved)} nodes")
-        for i, node in enumerate(unique_retrieved):  # All debug
-            table_num = node.metadata.get('table_number', 'N/A')
-            table_title = node.metadata.get('table_title', 'N/A')
+        for i, node in enumerate(unique_retrieved):
+            node_type = node.metadata.get('type', 'text')
             doc_id = node.metadata.get('document_id', 'N/A')
-            log_message(f"  [{i+1}] {doc_id} - Table {table_num}: {table_title[:50]}")
+            
+            if node_type == 'table':
+                table_num = node.metadata.get('table_number', 'N/A')
+                table_id = node.metadata.get('table_identifier', 'N/A')
+                table_title = node.metadata.get('table_title', 'N/A')
+                content_preview = node.text[:200].replace('\n', ' ')
+                log_message(f"  [{i+1}] {doc_id} - Table {table_num} | ID: {table_id}")
+                log_message(f"      Title: {table_title[:80]}")
+                log_message(f"      Content: {content_preview}...")
+            else:
+                section = node.metadata.get('section_id', 'N/A')
+                log_message(f"  [{i+1}] {doc_id} - Text section {section}")
+        
         log_message(f"UNIQUE NODES: {len(unique_retrieved)} nodes")
         
-        # Simple reranking with NORMALIZED question
-        reranked_nodes = rerank_nodes(normalized_question, unique_retrieved, reranker, top_k=20)
+        reranked_nodes = rerank_nodes(enhanced_question, unique_retrieved, reranker, 
+                                     top_k=rerank_top_k)  
         
-        # Direct query without formatting - use normalized question
-        response = query_engine.query(normalized_question)
+        response = query_engine.query(enhanced_question)
         
         end_time = time.time()
         processing_time = end_time - start_time
@@ -243,7 +273,7 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
         <h3 style='color: #63b3ed; margin-top: 0;'>Ответ (Модель: {current_model}):</h3>
         <div style='line-height: 1.6; font-size: 16px;'>{response.response}</div>
         <div style='margin-top: 15px; padding-top: 10px; border-top: 1px solid #4a5568; font-size: 14px; color: #a0aec0;'>
-        Время обработки: {processing_time:.2f} секунд
+        Время обработки: {processing_time:.2f} секунд 
         </div>
         </div>"""
         log_message(f"Model Answer: {response.response}")