a new version with the normalization = 3rd release

app.py

@@ -149,37 +149,39 @@ def initialize_system(repo_id, hf_token, download_dir, chunks_filename=None,
         all_documents = []
         chunks_df = None
         
+        # CHANGED: Use load_all_documents instead of loading separately
         if use_json_instead_csv and json_files_dir:
             log_message("Используем JSON файлы вместо CSV")
-            from documents_prep import load_json_documents, chunk_text_documents
+            from documents_prep import load_all_documents
             
-            # Load JSON docs (returns list of Documents)
-            json_documents = load_json_documents(repo_id, hf_token, json_files_dir)
-            # Chunk them
-            json_chunks = chunk_text_documents(json_documents)
-            all_documents.extend(json_chunks)
+            # This will handle text, tables, and images all together with proper logging
+            all_documents = load_all_documents(
+                repo_id=repo_id,
+                hf_token=hf_token,
+                json_dir=json_files_dir,
+                table_dir=table_data_dir if table_data_dir else "",
+                image_dir=image_data_dir if image_data_dir else ""
+            )
         else:
+            # OLD PATH: Loading separately (fallback)
             if chunks_filename:
                 log_message("Загружаем данные из CSV")
-               
-        
-        if table_data_dir:
-            log_message("Добавляю табличные данные")
-            from documents_prep import load_table_documents
             
-            # load_table_documents already returns chunked documents
-            table_chunks = load_table_documents(repo_id, hf_token, table_data_dir)
-            log_message(f"Загружено {len(table_chunks)} табличных чанков")
-            all_documents.extend(table_chunks)
-        
-        if image_data_dir:
-            log_message("Добавляю данные изображений")
-            from documents_prep import load_image_documents
+            if table_data_dir:
+                log_message("Добавляю табличные данные")
+                from documents_prep import load_table_documents
+                
+                table_chunks = load_table_documents(repo_id, hf_token, table_data_dir)
+                log_message(f"Загружено {len(table_chunks)} табличных чанков")
+                all_documents.extend(table_chunks)
             
-            # load_image_documents returns documents (no chunking needed)
-            image_documents = load_image_documents(repo_id, hf_token, image_data_dir)
-            log_message(f"Загружено {len(image_documents)} документов изображений")
-            all_documents.extend(image_documents)
+            if image_data_dir:
+                log_message("Добавляю данные изображений")
+                from documents_prep import load_image_documents
+                
+                image_documents = load_image_documents(repo_id, hf_token, image_data_dir)
+                log_message(f"Загружено {len(image_documents)} документов изображений")
+                all_documents.extend(image_documents)
         
         log_message(f"Всего документов после всей обработки: {len(all_documents)}")
         
@@ -197,6 +199,7 @@ def initialize_system(repo_id, hf_token, download_dir, chunks_filename=None,
                 'table_number': doc.metadata.get('table_number', ''),
                 'image_number': doc.metadata.get('image_number', ''),
                 'section': doc.metadata.get('section', ''),
+                'connection_type': doc.metadata.get('connection_type', '')  # ADD THIS
             })
         
         log_message(f"Система успешно инициализирована")

config.py

@@ -53,10 +53,11 @@ CHUNK_SIZE = 1500
 CHUNK_OVERLAP = 128
 
 MAX_CHARS_TABLE = 2500
-MAX_ROWS_TABLE = 10
+MAX_ROWS_TABLE = 15
 
 CUSTOM_PROMPT = """
 Вы являетесь высокоспециализированным Ассистентом для анализа нормативных документов (AIEXP). Ваша цель - предоставлять точные, корректные и контекстно релевантные ответы исключительно на основе предоставленного контекста из нормативной документации.
+СТРОГО ОТВЕТИТЬ ТОЛЬКО НА РУССКОМ!
 
 ПРАВИЛА АНАЛИЗА ЗАПРОСА:
 

documents_prep.py

@@ -34,6 +34,26 @@ def chunk_text_documents(documents):
     
     return chunked
 
+def normalize_connection_type(s):
+    # Replace Cyrillic with Latin
+    s = s.replace('С', 'C').replace('с', 'c')
+    s = s.replace('У', 'U').replace('у', 'u')
+    s = s.replace('Т', 'T').replace('т', 't')
+    s= s.replace('С-', 'C-').replace('с-', 'c-')
+    s = s.replace('У-', 'U-').replace('у-', 'u-')
+    s = s.replace('Т-', 'T-').replace('т-', 't-')
+    # REMOVE ALL HYPHENS for consistent tokenization
+    s = s.replace('-', '')
+    return s
+
+def extract_connection_type(text):
+    import re
+    # Match pattern with or without hyphens: C-25, C-25-1, С25, etc.
+    match = re.search(r'[СCс]-?\d+(?:-\d+)*', text)
+    if match:
+        normalized = normalize_connection_type(match.group(0))
+        return normalized
+    return ''
 
 def chunk_table_by_content(table_data, doc_id, max_chars=MAX_CHARS_TABLE, max_rows=MAX_ROWS_TABLE):
     headers = table_data.get('headers', [])
@@ -41,6 +61,7 @@ def chunk_table_by_content(table_data, doc_id, max_chars=MAX_CHARS_TABLE, max_ro
     table_num = table_data.get('table_number', 'unknown')
     table_title = table_data.get('table_title', '')
     section = table_data.get('section', '')
+    table_description = table_data.get('table_description', '')  
    
     table_num_clean = str(table_num).strip()
     
@@ -60,8 +81,13 @@ def chunk_table_by_content(table_data, doc_id, max_chars=MAX_CHARS_TABLE, max_ro
     
     log_message(f"  📊 Processing: {doc_id} - {table_identifier} ({len(rows)} rows)")
     
-    # Calculate base metadata size
+    # Calculate base metadata size - NOW INCLUDING DESCRIPTION
     base_content = format_table_header(doc_id, table_identifier, table_num, table_title, section, headers)
+    
+    # ADD DESCRIPTION HERE if it exists
+    if table_description:
+        base_content += f"ОПИСАНИЕ: {table_description}\n\n"
+    
     base_size = len(base_content)
     available_space = max_chars - base_size - 200 
     
@@ -79,7 +105,9 @@ def chunk_table_by_content(table_data, doc_id, max_chars=MAX_CHARS_TABLE, max_ro
             'section': section,
             'total_rows': len(rows),
             'chunk_size': len(content),
-            'is_complete_table': True
+            'is_complete_table': True,
+            'connection_type': extract_connection_type(table_title) if table_title else ''  # NEW
+
         }
         
         log_message(f"    Single chunk: {len(content)} chars, {len(rows)} rows")
@@ -113,7 +141,8 @@ def chunk_table_by_content(table_data, doc_id, max_chars=MAX_CHARS_TABLE, max_ro
                 'row_end': current_rows[-1]['_idx'],
                 'total_rows': len(rows),
                 'chunk_size': len(content),
-                'is_complete_table': False
+                'is_complete_table': False,
+                'connection_type': extract_connection_type(table_title) if table_title else ''  # NEW
             }
             
             chunks.append(Document(text=content, metadata=metadata))
@@ -155,37 +184,62 @@ def chunk_table_by_content(table_data, doc_id, max_chars=MAX_CHARS_TABLE, max_ro
     
     return chunks
 
-
 def format_table_header(doc_id, table_identifier, table_num, table_title, section, headers):
-    content = f"ТАБЛИЦА {table_identifier} из {doc_id}\n"
+    content = f"ДОКУМЕНТ: {doc_id}\n"
+    content += f"ТАБЛИЦА: {table_identifier}\n"
+    
     if table_title:
-        content += f"НАЗВАНИЕ: {table_title}\n"
+        # Normalize the title text itself for better searchability
+        normalized_title = normalize_connection_type(table_title)
+        content += f"НАЗВАНИЕ ТАБЛИЦЫ: {normalized_title}\n"
+        
+        # Extract and store the normalized connection type
+        connection_type = extract_connection_type(table_title)
+        if connection_type:
+            content += f"ТИП СОЕДИНЕНИЯ: {connection_type}\n"
+    
+    if table_num and table_num != table_identifier:
+        content += f"НОМЕР ТАБЛИЦЫ: {table_num}\n"
+    
     if section:
-        content += f"РАЗДЕЛ: {section}\n"
-    content += f"{'='*70}\n"
+        content += f"РАЗДЕЛ ДОКУМЕНТА: {section}\n"
     
-    if headers:
-        header_str = ' | '.join(str(h) for h in headers)
-        content += f"ЗАГОЛОВКИ: {header_str}\n\n"
+    content += f"\n{'='*70}\n"
     
-    content += "ДАННЫЕ:\n"
+    if headers:
+        content += "СТОЛБЦЫ ТАБЛИЦЫ:\n"
+        for i, h in enumerate(headers, 1):
+            # NORMALIZE HEADERS TOO
+            normalized_header = normalize_connection_type(h)
+            content += f"  {i}. {normalized_header}\n"
+        content += "\n"
+    
+    content += "ДАННЫЕ ТАБЛИЦЫ:\n"
     return content
 
 
 def format_single_row(row, idx):
-    """Format a single row"""
+    """Format a single row with normalization"""
     if isinstance(row, dict):
-        parts = [f"{k}: {v}" for k, v in row.items() 
-                if v and str(v).strip() and str(v).lower() not in ['nan', 'none', '']]
+        # NORMALIZE VALUES IN ROWS
+        parts = []
+        for k, v in row.items():
+            if v and str(v).strip() and str(v).lower() not in ['nan', 'none', '']:
+                normalized_v = normalize_connection_type(str(v))
+                parts.append(f"{k}: {normalized_v}")
         if parts:
             return f"{idx}. {' | '.join(parts)}\n"
     elif isinstance(row, list):
-        parts = [str(v) for v in row if v and str(v).strip() and str(v).lower() not in ['nan', 'none', '']]
+        # NORMALIZE LIST VALUES
+        parts = []
+        for v in row:
+            if v and str(v).strip() and str(v).lower() not in ['nan', 'none', '']:
+                normalized_v = normalize_connection_type(str(v))
+                parts.append(normalized_v)
         if parts:
             return f"{idx}. {' | '.join(parts)}\n"
     return ""
 
-
 def format_table_rows(rows):
     """Format multiple rows"""
     content = ""
@@ -199,40 +253,6 @@ def format_table_footer(table_identifier, doc_id):
     """Format table footer"""
     return f"\n{'='*70}\nКОНЕЦ ТАБЛИЦЫ {table_identifier} ИЗ {doc_id}\n"
 
-def load_table_documents(repo_id, hf_token, table_dir):
-    log_message("Loading tables...")
-    
-    files = list_repo_files(repo_id=repo_id, repo_type="dataset", token=hf_token)
-    table_files = [f for f in files if f.startswith(table_dir) and f.endswith('.json')]
-    
-    all_chunks = []
-    for file_path in table_files:
-        try:
-            local_path = hf_hub_download(
-                repo_id=repo_id,
-                filename=file_path,
-                repo_type="dataset",
-                token=hf_token
-            )
-            
-            with open(local_path, 'r', encoding='utf-8') as f:
-                data = json.load(f)
-            
-            file_doc_id = data.get('document_id', data.get('document', 'unknown'))
-            
-            for sheet in data.get('sheets', []):
-                sheet_doc_id = sheet.get('document_id', sheet.get('document', file_doc_id))
-                
-                chunks = chunk_table_by_content(sheet, sheet_doc_id, max_chars=1000)
-                all_chunks.extend(chunks)
-                
-        except Exception as e:
-            log_message(f"Error loading {file_path}: {e}")
-    
-    log_message(f"✓ Loaded {len(all_chunks)} table chunks")
-    return all_chunks
-
-
 def load_json_documents(repo_id, hf_token, json_dir):
     import zipfile
     import tempfile
@@ -414,13 +434,14 @@ def extract_sections_from_json(json_path):
 
 
 def load_table_documents(repo_id, hf_token, table_dir):
-    """Load and chunk tables"""
     log_message("Loading tables...")
     
     files = list_repo_files(repo_id=repo_id, repo_type="dataset", token=hf_token)
     table_files = [f for f in files if f.startswith(table_dir) and f.endswith('.json')]
     
     all_chunks = []
+    connection_type_sources = {}  # Track which table each type comes from
+    
     for file_path in table_files:
         try:
             local_path = hf_hub_download(
@@ -433,23 +454,38 @@ def load_table_documents(repo_id, hf_token, table_dir):
             with open(local_path, 'r', encoding='utf-8') as f:
                 data = json.load(f)
             
-            # Extract file-level document_id
             file_doc_id = data.get('document_id', data.get('document', 'unknown'))
             
             for sheet in data.get('sheets', []):
-                # Use sheet-level document_id if available, otherwise use file-level
                 sheet_doc_id = sheet.get('document_id', sheet.get('document', file_doc_id))
+                table_num = sheet.get('table_number', 'unknown')
+                table_title = sheet.get('table_title', '')
                 
-                # CRITICAL: Pass document_id to chunk function
-                chunks = chunk_table_by_content(sheet, sheet_doc_id)
+                chunks = chunk_table_by_content(sheet, sheet_doc_id, max_chars=MAX_CHARS_TABLE, max_rows=MAX_ROWS_TABLE)
                 all_chunks.extend(chunks)
                 
+                # Track connection type source
+                conn_type = extract_connection_type(table_title)
+                if conn_type:
+                    if conn_type not in connection_type_sources:
+                        connection_type_sources[conn_type] = []
+                    connection_type_sources[conn_type].append(f"{sheet_doc_id} Table {table_num}")
+                
         except Exception as e:
             log_message(f"Error loading {file_path}: {e}")
     
     log_message(f"✓ Loaded {len(all_chunks)} table chunks")
-    return all_chunks
 
+    log_message("="*60)
+    log_message("CONNECTION TYPES AND THEIR SOURCES:")
+    for conn_type in sorted(connection_type_sources.keys()):
+        sources = connection_type_sources[conn_type]
+        log_message(f"  {conn_type}: {len(sources)} tables")
+        for src in sources:
+            log_message(f"    - {src}")
+    log_message("="*60)
+    
+    return all_chunks
 
 def load_image_documents(repo_id, hf_token, image_dir):
     """Load image descriptions"""
@@ -498,9 +534,7 @@ def load_image_documents(repo_id, hf_token, image_dir):
     
     return documents
 
-
 def load_all_documents(repo_id, hf_token, json_dir, table_dir, image_dir):
-    """Main loader - combines all document types"""
     log_message("="*60)
     log_message("STARTING DOCUMENT LOADING")
     log_message("="*60)
@@ -512,6 +546,19 @@ def load_all_documents(repo_id, hf_token, json_dir, table_dir, image_dir):
     # Load tables (already chunked)
     table_chunks = load_table_documents(repo_id, hf_token, table_dir)
     
+    # NEW: Analyze connection types in tables
+    connection_types = {}
+    for chunk in table_chunks:
+        conn_type = chunk.metadata.get('connection_type', '')
+        if conn_type:
+            connection_types[conn_type] = connection_types.get(conn_type, 0) + 1
+    
+    log_message("="*60)
+    log_message("CONNECTION TYPES FOUND IN TABLES:")
+    for conn_type, count in sorted(connection_types.items()):
+        log_message(f"  {conn_type}: {count} chunks")
+    log_message("="*60)
+    
     # Load images (no chunking needed)
     image_docs = load_image_documents(repo_id, hf_token, image_dir)
     

index_retriever.py

@@ -10,8 +10,33 @@ from config import CUSTOM_PROMPT, PROMPT_SIMPLE_POISK
 
 def create_vector_index(documents):
     log_message("Строю векторный индекс")
+    
+    connection_type_sources = {}
+    table_count = 0
+    
+    for doc in documents:
+        if doc.metadata.get('type') == 'table':
+            table_count += 1
+            conn_type = doc.metadata.get('connection_type', '')
+            if conn_type:
+                table_id = f"{doc.metadata.get('document_id', 'unknown')} Table {doc.metadata.get('table_number', 'N/A')}"
+                if conn_type not in connection_type_sources:
+                    connection_type_sources[conn_type] = []
+                connection_type_sources[conn_type].append(table_id)
+    
+    log_message("="*60)
+    log_message(f"INDEXING {table_count} TABLE CHUNKS")
+    log_message("CONNECTION TYPES IN INDEX WITH SOURCES:")
+    for conn_type in sorted(connection_type_sources.keys()):
+        sources = list(set(connection_type_sources[conn_type]))  # Unique sources
+        log_message(f"  {conn_type}: {len(connection_type_sources[conn_type])} chunks from {len(sources)} tables")
+        for src in sources:
+            log_message(f"    - {src}")
+    log_message("="*60)
+    
     return VectorStoreIndex.from_documents(documents)
 
+
 def rerank_nodes(query, nodes, reranker, top_k=25, min_score_threshold=0.5):
     if not nodes or not reranker:
         return nodes[:top_k]
@@ -46,18 +71,18 @@ def create_query_engine(vector_index):
         
         bm25_retriever = BM25Retriever.from_defaults(
             docstore=vector_index.docstore, 
-            similarity_top_k=70 
+            similarity_top_k=100 
         )
         
         vector_retriever = VectorIndexRetriever(
             index=vector_index, 
-            similarity_top_k=70,
+            similarity_top_k=100,
             similarity_cutoff=0.55
         )
         
         hybrid_retriever = QueryFusionRetriever(
             [vector_retriever, bm25_retriever],
-            similarity_top_k=70,
+            similarity_top_k=100,
             num_queries=1
         )
         

table_prep.py

@@ -95,135 +95,135 @@ def chunk_table_document(doc, max_chunk_size=MAX_CHARS_TABLE, max_rows_per_chunk
     return chunked_docs
 
 
-def table_to_document(table_data, document_id=None):
-    if not isinstance(table_data, dict):
-        return []
+# def table_to_document(table_data, document_id=None):
+#     if not isinstance(table_data, dict):
+#         return []
     
-    doc_id = document_id or table_data.get('document_id') or table_data.get('document', 'Неизвестно')
-    table_num = table_data.get('table_number', 'Неизвестно')
-    table_title = table_data.get('table_title', 'Неизвестно')
-    section = table_data.get('section', 'Неизвестно')
-    table_rows = table_data.get('data', [])
+#     doc_id = document_id or table_data.get('document_id') or table_data.get('document', 'Неизвестно')
+#     table_num = table_data.get('table_number', 'Неизвестно')
+#     table_title = table_data.get('table_title', 'Неизвестно')
+#     section = table_data.get('section', 'Неизвестно')
+#     table_rows = table_data.get('data', [])
     
-    if not table_rows:
-        return []
+#     if not table_rows:
+#         return []
     
-    # Build table content
-    content = f"Таблица: {table_num}\n"
-    content += f"Название: {table_title}\n"
-    content += f"Документ: {doc_id}\n"
-    content += f"Раздел: {section}\n"
+#     # Build table content
+#     content = f"Таблица: {table_num}\n"
+#     content += f"Название: {table_title}\n"
+#     content += f"Документ: {doc_id}\n"
+#     content += f"Раздел: {section}\n"
     
-    headers = table_data.get('headers', [])
-    if headers:
-        content += f"\nЗаголовки: {' | '.join(headers)}\n"
+#     headers = table_data.get('headers', [])
+#     if headers:
+#         content += f"\nЗаголовки: {' | '.join(headers)}\n"
     
-    content += "\nДанные таблицы:\n"
-    for row_idx, row in enumerate(table_rows, start=1):
-        if isinstance(row, dict):
-            row_text = " | ".join([f"{k}: {v}" for k, v in row.items() if v])
-            content += f"Строка {row_idx}: {row_text}\n"
+#     content += "\nДанные таблицы:\n"
+#     for row_idx, row in enumerate(table_rows, start=1):
+#         if isinstance(row, dict):
+#             row_text = " | ".join([f"{k}: {v}" for k, v in row.items() if v])
+#             content += f"Строка {row_idx}: {row_text}\n"
     
-    # Create base document
-    base_doc = Document(
-        text=content,
-        metadata={
-            "type": "table",
-            "table_number": table_num,
-            "document_id": doc_id,
-            "section": section
-        }
-    )
-    if len(content) > 4000:
-        chunks = chunk_table_document(base_doc)
-        log_message(f"Таблица {table_num} разбита на {len(chunks)} частей")
-        return chunk_table_document(base_doc)
-    return [base_doc]
+#     # Create base document
+#     base_doc = Document(
+#         text=content,
+#         metadata={
+#             "type": "table",
+#             "table_number": table_num,
+#             "document_id": doc_id,
+#             "section": section
+#         }
+#     )
+#     if len(content) > 4000:
+#         chunks = chunk_table_document(base_doc)
+#         log_message(f"Таблица {table_num} разбита на {len(chunks)} частей")
+#         return chunk_table_document(base_doc)
+#     return [base_doc]
     
     
-def load_table_data(repo_id, hf_token, table_data_dir):    
-    try:
-        files = list_repo_files(repo_id=repo_id, repo_type="dataset", token=hf_token)
-        table_files = [f for f in files if f.startswith(table_data_dir) and f.endswith('.json')]
+# def load_table_data(repo_id, hf_token, table_data_dir):    
+#     try:
+#         files = list_repo_files(repo_id=repo_id, repo_type="dataset", token=hf_token)
+#         table_files = [f for f in files if f.startswith(table_data_dir) and f.endswith('.json')]
         
-        log_message(f"Найдено {len(table_files)} JSON файлов с таблицами")
+#         log_message(f"Найдено {len(table_files)} JSON файлов с таблицами")
         
-        table_documents = []
-        stats = {
-            'total_tables': 0,
-            'total_size': 0,
-            'by_document': defaultdict(lambda: {'count': 0, 'size': 0})
-        }
+#         table_documents = []
+#         stats = {
+#             'total_tables': 0,
+#             'total_size': 0,
+#             'by_document': defaultdict(lambda: {'count': 0, 'size': 0})
+#         }
         
-        for file_path in table_files:
-            try:
-                local_path = hf_hub_download(
-                    repo_id=repo_id,
-                    filename=file_path,
-                    local_dir='',
-                    repo_type="dataset",
-                    token=hf_token
-                )
+#         for file_path in table_files:
+#             try:
+#                 local_path = hf_hub_download(
+#                     repo_id=repo_id,
+#                     filename=file_path,
+#                     local_dir='',
+#                     repo_type="dataset",
+#                     token=hf_token
+#                 )
                 
-                log_message(f"\nОбработка файла: {file_path}")
+#                 log_message(f"\nОбработка файла: {file_path}")
                 
-                with open(local_path, 'r', encoding='utf-8') as f:
-                    table_data = json.load(f)
+#                 with open(local_path, 'r', encoding='utf-8') as f:
+#                     table_data = json.load(f)
                     
-                    if isinstance(table_data, dict):
-                        document_id = table_data.get('document', 'unknown')
+#                     if isinstance(table_data, dict):
+#                         document_id = table_data.get('document', 'unknown')
                         
-                        if 'sheets' in table_data:
-                            sorted_sheets = sorted(
-                                table_data['sheets'],
-                                key=lambda sheet: sheet.get('table_number', '')  # or use 'table_number'
-                            )
+#                         if 'sheets' in table_data:
+#                             sorted_sheets = sorted(
+#                                 table_data['sheets'],
+#                                 key=lambda sheet: sheet.get('table_number', '')  # or use 'table_number'
+#                             )
                             
-                            for sheet in sorted_sheets:
-                                sheet['document'] = document_id
-                                docs_list = table_to_document(sheet, document_id)
-                                table_documents.extend(docs_list)
+#                             for sheet in sorted_sheets:
+#                                 sheet['document'] = document_id
+#                                 docs_list = table_to_document(sheet, document_id)
+#                                 table_documents.extend(docs_list)
                                 
-                                for doc in docs_list:
-                                    stats['total_tables'] += 1
-                                    size = doc.metadata.get('content_size', 0)
-                                    stats['total_size'] += size
-                                    stats['by_document'][document_id]['count'] += 1
-                                    stats['by_document'][document_id]['size'] += size
-                                    log_message(f"Добавлена таблица {sheet.get('table_number', 'Неизвестно')} из документа {document_id}, размер {size} символов")
-                        else:
-                            docs_list = table_to_document(table_data, document_id)
-                            table_documents.extend(docs_list)
+#                                 for doc in docs_list:
+#                                     stats['total_tables'] += 1
+#                                     size = doc.metadata.get('content_size', 0)
+#                                     stats['total_size'] += size
+#                                     stats['by_document'][document_id]['count'] += 1
+#                                     stats['by_document'][document_id]['size'] += size
+#                                     log_message(f"Добавлена таблица {sheet.get('table_number', 'Неизвестно')} из документа {document_id}, размер {size} символов")
+#                         else:
+#                             docs_list = table_to_document(table_data, document_id)
+#                             table_documents.extend(docs_list)
                             
-                            for doc in docs_list:
-                                stats['total_tables'] += 1
-                                size = doc.metadata.get('content_size', 0)
-                                stats['total_size'] += size
-                                stats['by_document'][document_id]['count'] += 1
-                                stats['by_document'][document_id]['size'] += size
+#                             for doc in docs_list:
+#                                 stats['total_tables'] += 1
+#                                 size = doc.metadata.get('content_size', 0)
+#                                 stats['total_size'] += size
+#                                 stats['by_document'][document_id]['count'] += 1
+#                                 stats['by_document'][document_id]['size'] += size
                                 
                         
-            except Exception as e:
-                log_message(f"❌ ОШИБКА файла {file_path}: {str(e)}")
-                continue
+#             except Exception as e:
+#                 log_message(f"❌ ОШИБКА файла {file_path}: {str(e)}")
+#                 continue
         
-        # Log summary statistics
-        log_message("\n" + "=" * 60)
-        log_message("СТАТИСТИКА ПО ТАБЛИЦАМ")
-        log_message("=" * 60)
-        log_message(f"Всего таблиц добавлено: {stats['total_tables']}")
-        log_message(f"Общий размер: {stats['total_size']:,} символов")
-        log_message(f"Средний размер таблицы: {stats['total_size'] // stats['total_tables'] if stats['total_tables'] > 0 else 0:,} символов")
+#         # Log summary statistics
+#         log_message("\n" + "=" * 60)
+#         log_message("СТАТИСТИКА ПО ТАБЛИЦАМ")
+#         log_message("=" * 60)
+#         log_message(f"Всего таблиц добавлено: {stats['total_tables']}")
+#         log_message(f"Общий размер: {stats['total_size']:,} символов")
+#         log_message(f"Средний размер таблицы: {stats['total_size'] // stats['total_tables'] if stats['total_tables'] > 0 else 0:,} символов")
         
-        log_message("\nПо документам:")
-        for doc_id, doc_stats in sorted(stats['by_document'].items()):
-            log_message(f"  • {doc_id}: {doc_stats['count']} таблиц, "
-                       f"{doc_stats['size']:,} символов")
+#         log_message("\nПо документам:")
+#         for doc_id, doc_stats in sorted(stats['by_document'].items()):
+#             log_message(f"  • {doc_id}: {doc_stats['count']} таблиц, "
+#                        f"{doc_stats['size']:,} символов")
         
-        log_message("=" * 60)
+#         log_message("=" * 60)
         
-        return table_documents
+#         return table_documents
         
-    except Exception as e:
-        log_message(f"❌ КРИТИЧЕСКАЯ ОШИБКА загрузки табличных данных: {str(e)}")
-        return []
+#     except Exception as e:
+#         log_message(f"❌ КРИТИЧЕСКАЯ ОШИБКА загрузки табличных данных: {str(e)}")
+#         return []

utils.py

@@ -9,6 +9,7 @@ import time
 from index_retriever import rerank_nodes
 from my_logging import log_message
 from config import PROMPT_SIMPLE_POISK
+import re
 
 def get_llm_model(model_name):
     try:
@@ -172,6 +173,14 @@ def deduplicate_nodes(nodes):
     
     return unique_nodes
 
+def normalize_query(query):
+    def repl(m):
+        cyr_to_lat = {'С': 'C', 'с': 'C', 'Т': 'T', 'т': 'T', 'У': 'U', 'у': 'U'}
+        letter = cyr_to_lat.get(m.group(1), m.group(1))
+        return f"{letter}{m.group(2)}"
+
+    return re.sub(r'\b([СсТтУуCTU])[-\s]?(\d+)\b', repl, query)
+
 
 def answer_question(question, query_engine, reranker, current_model, chunks_df=None):
     if query_engine is None:
@@ -179,18 +188,58 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
     
     try:
         start_time = time.time()
-        retrieved_nodes = query_engine.retriever.retrieve(question)
+        
+        # NORMALIZE QUERY: Convert Cyrillic to Latin and remove hyphens
+        normalized_question = normalize_query(question)
+        log_message(f"Original query: {question}")
+        log_message(f"Normalized query: {normalized_question}")
+        
+        # Use normalized query for retrieval
+        retrieved_nodes = query_engine.retriever.retrieve(normalized_question)
+        log_message(f"user query: {question}")
         
         log_message(f"RETRIEVED: {len(retrieved_nodes)} nodes")
         
         unique_retrieved = deduplicate_nodes(retrieved_nodes)
         log_message(f"UNIQUE NODES: {len(unique_retrieved)} nodes")
         
-        # Simple reranking
-        reranked_nodes = rerank_nodes(question, unique_retrieved, reranker, top_k=20)
+        # Check for connection types
+        conn_types_retrieved = {}
+        for node in unique_retrieved:
+            if node.metadata.get('type') == 'table':
+                conn_type = node.metadata.get('connection_type', '')
+                if conn_type:
+                    conn_types_retrieved[conn_type] = conn_types_retrieved.get(conn_type, 0) + 1
+        
+        if conn_types_retrieved:
+            log_message("CONNECTION TYPES IN RETRIEVED:")
+            for ct, cnt in sorted(conn_types_retrieved.items()):
+                log_message(f"  {ct}: {cnt} chunks")
+        
+        # Check if target type was retrieved
+        # Normalize the check as well
+        normalized_check = normalize_query('С-25')  # Will become C25
+        if normalized_check in question or 'С-25' in question or 'C-25' in question:
+            if 'C25' in conn_types_retrieved:
+                log_message(f"✓ C25 RETRIEVED: {conn_types_retrieved['C25']} chunks")
+            else:
+                log_message("✗ C25 NOT RETRIEVED despite being in query!")
+        
+        # Sample of retrieved tables
+        log_message("SAMPLE OF RETRIEVED TABLES:")
+        for i, node in enumerate(unique_retrieved[:10]):
+            if node.metadata.get('type') == 'table':
+                table_num = node.metadata.get('table_number', 'N/A')
+                table_title = node.metadata.get('table_title', 'N/A')
+                conn_type = node.metadata.get('connection_type', 'N/A')
+                doc_id = node.metadata.get('document_id', 'N/A')
+                log_message(f"  [{i+1}] {doc_id} - Table {table_num} - Type: {conn_type}")
+        
+        # Rerank - use normalized query for consistency
+        reranked_nodes = rerank_nodes(normalized_question, unique_retrieved, reranker, top_k=20)
         
-        # Direct query without formatting
-        response = query_engine.query(question)
+        # CRITICAL FIX: Use normalized query for LLM as well
+        response = query_engine.query(normalized_question)
         
         end_time = time.time()
         processing_time = end_time - start_time