a new way with keywords

documents_prep.py

@@ -14,206 +14,147 @@ def chunk_document(doc, chunk_size=None, chunk_overlap=None):
         chunk_size = CHUNK_SIZE
     if chunk_overlap is None:
         chunk_overlap = CHUNK_OVERLAP
+    text_splitter = SentenceSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        separator=" "
+    )
     
-    text = doc.text
+    text_chunks = text_splitter.split_text(doc.text)
     
-    # Try to split by double newlines (paragraphs) first
-    paragraphs = text.split('\n\n')
-    
-    chunks = []
-    current_chunk = ""
-    
-    for para in paragraphs:
-        para = para.strip()
-        if not para:
-            continue
-        
-        # If adding this paragraph exceeds limit, save current chunk
-        if len(current_chunk) + len(para) + 2 > chunk_size and current_chunk:
-            chunks.append(current_chunk.strip())
-            # Add overlap from end of previous chunk
-            overlap_text = current_chunk[-chunk_overlap:] if len(current_chunk) > chunk_overlap else current_chunk
-            current_chunk = overlap_text + "\n\n" + para
-        else:
-            if current_chunk:
-                current_chunk += "\n\n" + para
-            else:
-                current_chunk = para
-    
-    # Add last chunk
-    if current_chunk:
-        chunks.append(current_chunk.strip())
-    
-    # If single paragraph is too large, fall back to sentence splitting
-    final_chunks = []
-    for chunk_text in chunks:
-        if len(chunk_text) > chunk_size:
-            splitter = SentenceSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
-            final_chunks.extend(splitter.split_text(chunk_text))
-        else:
-            final_chunks.append(chunk_text)
-    
-    log_message(f"  ✂️ Текст разбит на {len(final_chunks)} семантических чанков")
-    
-    # Create documents
     chunked_docs = []
-    for i, chunk_text in enumerate(final_chunks):
+    for i, chunk_text in enumerate(text_chunks):
         chunk_metadata = doc.metadata.copy()
         chunk_metadata.update({
             "chunk_id": i,
-            "total_chunks": len(final_chunks),
+            "total_chunks": len(text_chunks),
             "chunk_size": len(chunk_text),
-            "is_chunked": True
+            "original_doc_id": doc.id_ if hasattr(doc, 'id_') else None
         })
-        chunked_docs.append(Document(text=chunk_text, metadata=chunk_metadata))
+        
+        chunked_doc = Document(
+            text=chunk_text,
+            metadata=chunk_metadata
+        )
+        chunked_docs.append(chunked_doc)
     
     return chunked_docs
 
 def process_documents_with_chunking(documents):
-    log_message("\n" + "="*60)
-    log_message("🔄 НАЧАЛО ПРОЦЕССА ЧАНКИНГА")
-    log_message("="*60)
-    
     all_chunked_docs = []
     chunk_info = []
+    table_count = 0
+    table_chunks_count = 0
+    image_count = 0
+    image_chunks_count = 0
+    text_chunks_count = 0
     
-    # Counters
-    table_whole_count = 0      # Целые таблицы (не нуждаются в чанкинге)
-    table_chunked_count = 0    # Таблицы, которые УЖЕ разбиты
-    image_whole_count = 0      # Целые изображения
-    image_chunked_count = 0    # Изображения, разбитые на чанки
-    text_whole_count = 0       # Целые текстовые документы
-    text_chunked_count = 0     # Текстовые документы, разбитые на чанки
-    
-    for idx, doc in enumerate(documents):
+    for doc in documents:
         doc_type = doc.metadata.get('type', 'text')
         is_already_chunked = doc.metadata.get('is_chunked', False)
-        doc_size = len(doc.text)
-        
-        log_message(f"\n📄 Документ {idx+1}/{len(documents)} | "
-                   f"Тип: {doc_type} | "
-                   f"Размер: {doc_size} | "
-                   f"Уже разбит: {is_already_chunked}")
         
         if doc_type == 'table':
             if is_already_chunked:
-                # Таблица уже разбита на чанки в table_prep.py
-                table_chunked_count += 1
+                table_chunks_count += 1
                 all_chunked_docs.append(doc)
-                log_message(f"  ✓ Таблица (чанк {doc.metadata.get('chunk_id', 0) + 1}/"
-                           f"{doc.metadata.get('total_chunks', 1)}) добавлена без изменений")
+                chunk_info.append({
+                    'document_id': doc.metadata.get('document_id', 'unknown'),
+                    'section_id': doc.metadata.get('section_id', 'unknown'),
+                    'chunk_id': doc.metadata.get('chunk_id', 0),
+                    'total_chunks': doc.metadata.get('total_chunks', 1),
+                    'chunk_size': len(doc.text),
+                    'chunk_preview': doc.text[:200] + "..." if len(doc.text) > 200 else doc.text,
+                    'type': 'table',
+                    'table_number': doc.metadata.get('table_number', 'unknown')
+                })
             else:
-                # Целая таблица
-                table_whole_count += 1
+                table_count += 1
                 all_chunked_docs.append(doc)
-                log_message(f"  ✓ Целая таблица добавлена | "
-                           f"Номер: {doc.metadata.get('table_number', 'unknown')}")
-            
-            chunk_info.append({
-                'document_id': doc.metadata.get('document_id', 'unknown'),
-                'section_id': doc.metadata.get('section_id', 'unknown'),
-                'chunk_id': doc.metadata.get('chunk_id', 0),
-                'total_chunks': doc.metadata.get('total_chunks', 1),
-                'chunk_size': doc_size,
-                'chunk_preview': doc.text[:200] + "..." if len(doc.text) > 200 else doc.text,
-                'type': 'table',
-                'table_number': doc.metadata.get('table_number', 'unknown'),
-                'is_chunked': is_already_chunked
-            })
+                chunk_info.append({
+                    'document_id': doc.metadata.get('document_id', 'unknown'),
+                    'section_id': doc.metadata.get('section_id', 'unknown'),
+                    'chunk_id': 0,
+                    'chunk_size': len(doc.text),
+                    'chunk_preview': doc.text[:200] + "..." if len(doc.text) > 200 else doc.text,
+                    'type': 'table',
+                    'table_number': doc.metadata.get('table_number', 'unknown')
+                })
             
         elif doc_type == 'image':
+            image_count += 1
+            doc_size = len(doc.text)
             if doc_size > CHUNK_SIZE:
-                log_message(f"  📷 Изображение требует чанкинга | Размер: {doc_size} > {CHUNK_SIZE}")
+                log_message(f"📷 CHUNKING: Изображение {doc.metadata.get('image_number', 'unknown')} | "
+                           f"Размер: {doc_size} > {CHUNK_SIZE}")
                 chunked_docs = chunk_document(doc)
-                image_chunked_count += len(chunked_docs)
+                image_chunks_count += len(chunked_docs)
                 all_chunked_docs.extend(chunked_docs)
+                log_message(f"  ✂️ Разделено на {len(chunked_docs)} чанков")
                 
-                for chunk_doc in chunked_docs:
+                for i, chunk_doc in enumerate(chunked_docs):
                     chunk_info.append({
                         'document_id': chunk_doc.metadata.get('document_id', 'unknown'),
                         'section_id': chunk_doc.metadata.get('section_id', 'unknown'),
-                        'chunk_id': chunk_doc.metadata.get('chunk_id', 0),
-                        'total_chunks': chunk_doc.metadata.get('total_chunks', 1),
+                        'chunk_id': i,
                         'chunk_size': len(chunk_doc.text),
                         'chunk_preview': chunk_doc.text[:200] + "..." if len(chunk_doc.text) > 200 else chunk_doc.text,
                         'type': 'image',
-                        'image_number': chunk_doc.metadata.get('image_number', 'unknown'),
-                        'is_chunked': True
+                        'image_number': chunk_doc.metadata.get('image_number', 'unknown')
                     })
             else:
-                image_whole_count += 1
                 all_chunked_docs.append(doc)
-                log_message(f"  ✓ Целое изображение добавлено | Размер: {doc_size}")
-                
                 chunk_info.append({
                     'document_id': doc.metadata.get('document_id', 'unknown'),
                     'section_id': doc.metadata.get('section_id', 'unknown'),
                     'chunk_id': 0,
-                    'total_chunks': 1,
                     'chunk_size': doc_size,
                     'chunk_preview': doc.text[:200] + "..." if len(doc.text) > 200 else doc.text,
                     'type': 'image',
-                    'image_number': doc.metadata.get('image_number', 'unknown'),
-                    'is_chunked': False
+                    'image_number': doc.metadata.get('image_number', 'unknown')
                 })
             
-        else:  # text
+        else:
+            doc_size = len(doc.text)
             if doc_size > CHUNK_SIZE:
-                log_message(f"  📝 Текст требует чанкинга | "
-                           f"Документ: {doc.metadata.get('document_id', 'unknown')} | "
-                           f"Раздел: {doc.metadata.get('section_id', 'unknown')} | "
+                log_message(f"📝 CHUNKING: Текст из '{doc.metadata.get('document_id', 'unknown')}' | "
                            f"Размер: {doc_size} > {CHUNK_SIZE}")
-                
                 chunked_docs = chunk_document(doc)
-                text_chunked_count += len(chunked_docs)
+                text_chunks_count += len(chunked_docs)
                 all_chunked_docs.extend(chunked_docs)
+                log_message(f"  ✂️ Разделен на {len(chunked_docs)} чанков")
                 
-                for chunk_doc in chunked_docs:
+                for i, chunk_doc in enumerate(chunked_docs):
                     chunk_info.append({
                         'document_id': chunk_doc.metadata.get('document_id', 'unknown'),
                         'section_id': chunk_doc.metadata.get('section_id', 'unknown'),
-                        'chunk_id': chunk_doc.metadata.get('chunk_id', 0),
-                        'total_chunks': chunk_doc.metadata.get('total_chunks', 1),
+                        'chunk_id': i,
                         'chunk_size': len(chunk_doc.text),
                         'chunk_preview': chunk_doc.text[:200] + "..." if len(chunk_doc.text) > 200 else chunk_doc.text,
-                        'type': 'text',
-                        'is_chunked': True
+                        'type': 'text'
                     })
             else:
-                text_whole_count += 1
                 all_chunked_docs.append(doc)
-                log_message(f"  ✓ Целый текстовый документ добавлен | Размер: {doc_size}")
-                
                 chunk_info.append({
                     'document_id': doc.metadata.get('document_id', 'unknown'),
                     'section_id': doc.metadata.get('section_id', 'unknown'),
                     'chunk_id': 0,
-                    'total_chunks': 1,
                     'chunk_size': doc_size,
                     'chunk_preview': doc.text[:200] + "..." if len(doc.text) > 200 else doc.text,
-                    'type': 'text',
-                    'is_chunked': False
+                    'type': 'text'
                 })
         
     log_message(f"\n{'='*60}")
-    log_message(f"📊 ИТОГОВАЯ СТАТИСТИКА ЧАНКИНГА:")
-    log_message(f"{'='*60}")
-    log_message(f"  ТАБЛИЦЫ:")
-    log_message(f"    • Целые (не нуждались в чанкинге): {table_whole_count}")
-    log_message(f"    • Чанки (разбиты в table_prep.py): {table_chunked_count}")
-    log_message(f"  ИЗОБРАЖЕНИЯ:")
-    log_message(f"    • Целые: {image_whole_count}")
-    log_message(f"    • Чанки: {image_chunked_count}")
-    log_message(f"  ТЕКСТ:")
-    log_message(f"    • Целые документы: {text_whole_count}")
-    log_message(f"    • Чанки: {text_chunked_count}")
-    log_message(f"  {'─'*58}")
-    log_message(f"  ВСЕГО ДОКУМЕНТОВ В ИНДЕКСЕ: {len(all_chunked_docs)}")
+    log_message(f"ИТОГО ОБРАБОТАНО ДОКУМЕНТОВ:")
+    log_message(f"  • Таблицы (целые): {table_count}")
+    log_message(f"  • Таблицы (чанки): {table_chunks_count}")
+    log_message(f"  • Изображения (целые): {image_count - (image_chunks_count > 0)}")
+    log_message(f"  • Изображения (чанки): {image_chunks_count}")
+    log_message(f"  • Текстовые чанки: {text_chunks_count}")
+    log_message(f"  • Всего документов: {len(all_chunked_docs)}")
     log_message(f"{'='*60}\n")
     
     return all_chunked_docs, chunk_info
 
-
 def extract_text_from_json(data, document_id, document_name):
     documents = []
     

index_retriever.py

@@ -112,4 +112,142 @@ def rerank_nodes(query, nodes, reranker, top_k=25, min_score_threshold=0.5, dive
         
     except Exception as e:
         log_message(f"Ошибка переранжировки: {str(e)}")
-        return nodes[:top_k]
+        return nodes[:top_k]
+    
+
+from rank_bm25 import BM25Okapi
+import numpy as np
+
+class HybridRetriever:
+    def __init__(self, vector_retriever, documents):
+        self.vector_retriever = vector_retriever
+        self.documents = documents
+        
+        # Build BM25 index
+        tokenized_docs = [doc.text.lower().split() for doc in documents]
+        self.bm25 = BM25Okapi(tokenized_docs)
+        
+        # Build metadata index for exact matching
+        self.metadata_index = self._build_metadata_index(documents)
+    
+    def _build_metadata_index(self, documents):
+        """Index by materials, GOSTs, classes for exact matching"""
+        index = {
+            'materials': {},
+            'gosts': {},
+            'classes': {},
+            'key_terms': {}
+        }
+        
+        for i, doc in enumerate(documents):
+            metadata = doc.metadata
+            
+            # Index materials
+            for material in metadata.get('materials', []):
+                if material not in index['materials']:
+                    index['materials'][material] = []
+                index['materials'][material].append(i)
+            
+            # Index GOSTs
+            for gost in metadata.get('gosts', []):
+                if gost not in index['gosts']:
+                    index['gosts'][gost] = []
+                index['gosts'][gost].append(i)
+            
+            # Index classes
+            for cls in metadata.get('classes', []):
+                if cls not in index['classes']:
+                    index['classes'][cls] = []
+                index['classes'][cls].append(i)
+            
+            # Index key terms
+            for term in metadata.get('key_terms', []):
+                term_lower = term.lower()
+                if term_lower not in index['key_terms']:
+                    index['key_terms'][term_lower] = []
+                index['key_terms'][term_lower].append(i)
+        
+        return index
+    
+    def retrieve(self, query, top_k=20, vector_weight=0.5, bm25_weight=0.3, metadata_weight=0.2):
+        """Hybrid retrieval combining vector, BM25, and metadata matching"""
+        
+        # 1. Vector search
+        vector_results = self.vector_retriever.retrieve(query)
+        vector_scores = {node.node_id: node.score for node in vector_results}
+        
+        # 2. BM25 search
+        tokenized_query = query.lower().split()
+        bm25_scores = self.bm25.get_scores(tokenized_query)
+        
+        # 3. Metadata exact matching
+        metadata_scores = self._get_metadata_scores(query)
+        
+        # 4. Combine scores
+        all_node_ids = set(list(vector_scores.keys()) + 
+                          list(range(len(self.documents))))
+        
+        combined_scores = {}
+        for node_id in all_node_ids:
+            vec_score = vector_scores.get(node_id, 0.0)
+            bm25_score = bm25_scores[node_id] if isinstance(node_id, int) and node_id < len(bm25_scores) else 0.0
+            meta_score = metadata_scores.get(node_id, 0.0)
+            
+            # Normalize and combine
+            combined_scores[node_id] = (
+                vector_weight * vec_score +
+                bm25_weight * (bm25_score / (max(bm25_scores) + 1e-6)) +
+                metadata_weight * meta_score
+            )
+        
+        # 5. Get top-k
+        sorted_nodes = sorted(combined_scores.items(), key=lambda x: x[1], reverse=True)[:top_k]
+        
+        # Return as node objects
+        results = []
+        for node_id, score in sorted_nodes:
+            if isinstance(node_id, int) and node_id < len(self.documents):
+                doc = self.documents[node_id]
+                # Create node-like object
+                from types import SimpleNamespace
+                node = SimpleNamespace(
+                    text=doc.text,
+                    metadata=doc.metadata,
+                    score=score,
+                    node_id=node_id
+                )
+                results.append(node)
+        
+        return results
+    
+    def _get_metadata_scores(self, query):
+        """Score documents by exact metadata matches"""
+        scores = {}
+        query_lower = query.lower()
+        
+        # Check for material codes
+        import re
+        material_pattern = r'\b\d{2}[ХНТМКВБА]+\d{1,2}[ХНТМКВБА]*\d*\b'
+        materials_in_query = re.findall(material_pattern, query, re.IGNORECASE)
+        
+        for material in materials_in_query:
+            if material in self.metadata_index['materials']:
+                for doc_id in self.metadata_index['materials'][material]:
+                    scores[doc_id] = scores.get(doc_id, 0) + 1.0
+        
+        # Check for GOSTs
+        gost_pattern = r'ГОСТ\s+[РЕ��\s]*\d+[\.\-\d]*'
+        gosts_in_query = re.findall(gost_pattern, query, re.IGNORECASE)
+        
+        for gost in gosts_in_query:
+            if gost in self.metadata_index['gosts']:
+                for doc_id in self.metadata_index['gosts'][gost]:
+                    scores[doc_id] = scores.get(doc_id, 0) + 0.8
+        
+        # Check for key terms
+        for term, doc_ids in self.metadata_index['key_terms'].items():
+            if term in query_lower:
+                for doc_id in doc_ids:
+                    scores[doc_id] = scores.get(doc_id, 0) + 0.5
+        
+        return scores

requirements.txt

@@ -15,4 +15,5 @@ openpyxl
 llama-index-llms-openai
 llama-index-vector-stores-faiss
 llama-index-retrievers-bm25
-tiktoken
+tiktoken
+rank-bm25

table_prep.py

@@ -32,80 +32,93 @@ def create_table_content(table_data):
 from llama_index.core.text_splitter import SentenceSplitter
 from config import CHUNK_SIZE, CHUNK_OVERLAP
 
-def create_table_chunks_with_headers(table_data, rows_per_chunk=10):
-    """
-    Intelligently chunk tables by preserving headers and grouping rows
-    """
-    doc_id = table_data.get('document_id') or table_data.get('document', 'Неизвестно')
-    table_num = table_data.get('table_number', 'Неизвестно')
-    table_title = table_data.get('table_title', 'Неизвестно')
-    section = table_data.get('section', 'Неизвестно')
-    headers = table_data.get('headers', [])
-    table_rows = table_data.get('data', [])
-    
-    if not table_rows:
-        return []
-    
-    # Create header string that will be included in EVERY chunk
-    header_context = f"Таблица {table_num}: {table_title}\n"
-    header_context += f"Документ: {doc_id}\n"
-    header_context += f"Раздел: {section}\n"
-    if headers:
-        header_context += f"Заголовки: {' | '.join(headers)}\n"
-    header_context += f"Всего строк в таблице: {len(table_rows)}\n\n"
+# In table_prep.py - replace chunk_table_document function
+
+def chunk_table_document(doc, chunk_size=None, chunk_overlap=None):
+    if chunk_size is None:
+        chunk_size = CHUNK_SIZE
+    if chunk_overlap is None:
+        chunk_overlap = CHUNK_OVERLAP
     
-    # Calculate optimal rows per chunk based on content size
-    avg_row_size = sum(len(str(row)) for row in table_rows[:5]) / min(5, len(table_rows))
-    max_chunk_size = CHUNK_SIZE - len(header_context) - 500  # Safety margin
-    optimal_rows = max(5, int(max_chunk_size / avg_row_size))
+    # Extract critical metadata from table before chunking
+    table_metadata = extract_table_metadata(doc.text)
     
-    log_message(f"  📐 Средний размер строки: {avg_row_size:.0f} символов")
-    log_message(f"  📊 Оптимальное кол-во строк на чанк: {optimal_rows}")
+    text_splitter = SentenceSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        separator="\n"
+    )
     
-    chunks = []
-    total_rows = len(table_rows)
+    text_chunks = text_splitter.split_text(doc.text)
     
-    for i in range(0, total_rows, optimal_rows):
-        chunk_rows = table_rows[i:i + optimal_rows]
-        
-        # Build chunk content
-        chunk_content = header_context
-        chunk_content += f"[Строки {i+1}-{min(i+optimal_rows, total_rows)} из {total_rows}]\n"
-        chunk_content += "Данные:\n"
+    chunked_docs = []
+    for i, chunk_text in enumerate(text_chunks):
+        chunk_metadata = doc.metadata.copy()
         
-        for row_idx, row in enumerate(chunk_rows, start=i+1):
-            if isinstance(row, dict):
-                row_text = " | ".join([f"{k}: {v}" for k, v in row.items() if v])
-                chunk_content += f"Строка {row_idx}: {row_text}\n"
-        
-        chunk_metadata = {
-            "type": "table",
-            "table_number": table_num,
-            "table_title": table_title,
-            "document_id": doc_id,
-            "section": section,
-            "section_id": section,
-            "headers": headers,
-            "chunk_id": i // optimal_rows,
-            "total_chunks": (total_rows + optimal_rows - 1) // optimal_rows,
-            "row_range": f"{i+1}-{min(i+optimal_rows, total_rows)}",
-            "total_table_rows": total_rows,
-            "is_chunked": True
-        }
+        # Add extracted keywords/materials to each chunk
+        chunk_metadata.update({
+            "chunk_id": i,
+            "total_chunks": len(text_chunks),
+            "chunk_size": len(chunk_text),
+            "is_chunked": True,
+            "materials": table_metadata.get("materials", []),  # All materials from table
+            "key_terms": table_metadata.get("key_terms", []),  # Technical terms
+            "table_summary": table_metadata.get("summary", "")  # Brief table description
+        })
         
-        doc = Document(text=chunk_content, metadata=chunk_metadata)
-        chunks.append(doc)
+        # Enrich chunk text with context from full table
+        enriched_text = f"""[Таблица {doc.metadata.get('table_number')}: {doc.metadata.get('table_title')}]
+[Материалы в таблице: {', '.join(table_metadata.get('materials', [])[:10])}]
+[Ключевые термины: {', '.join(table_metadata.get('key_terms', [])[:10])}]
+
+{chunk_text}"""
         
-        log_message(f"    Чанк {len(chunks)}: строки {i+1}-{min(i+optimal_rows, total_rows)} | "
-                   f"{len(chunk_content)} символов")
+        chunked_doc = Document(
+            text=enriched_text,
+            metadata=chunk_metadata
+        )
+        chunked_docs.append(chunked_doc)
     
-    return chunks
+    return chunked_docs
+
 
+def extract_table_metadata(table_text):
+    """Extract searchable metadata from table content"""
+    import re
+    
+    # Extract material codes (e.g., 08Х18Н10Т)
+    material_pattern = r'\b\d{2}[ХНТМКВБА]+\d{1,2}[ХНТМКВБА]*\d*\b'
+    materials = list(set(re.findall(material_pattern, table_text, re.IGNORECASE)))
+    
+    # Extract GOST standards
+    gost_pattern = r'ГОСТ\s+[РЕН\s]*\d+[\.\-\d]*'
+    gosts = list(set(re.findall(gost_pattern, table_text, re.IGNORECASE)))
+    
+    # Extract class/category codes
+    class_pattern = r'\b\d[АБВСI]+[IVX]+[a-z]*\b'
+    classes = list(set(re.findall(class_pattern, table_text, re.IGNORECASE)))
+    
+    # Extract common technical terms
+    tech_terms = []
+    keywords = ['контроль', 'испытание', 'сертификат', 'качество', 'план', 
+                'полуфабрикат', 'оборудование', 'арматура', 'деталь']
+    for keyword in keywords:
+        if keyword.lower() in table_text.lower():
+            tech_terms.append(keyword)
+    
+    # Create brief summary
+    lines = table_text.split('\n')[:5]
+    summary = ' '.join([l.strip() for l in lines if l.strip()])[:200]
+    
+    return {
+        "materials": materials,
+        "gosts": gosts,
+        "classes": classes,
+        "key_terms": tech_terms + gosts,
+        "summary": summary
+    }
 
 def table_to_document(table_data, document_id=None):
-    """
-    Convert table to Document(s) with intelligent chunking
-    """
     if not isinstance(table_data, dict):
         log_message(f"⚠️ ПРОПУЩЕНА: table_data не является словарем")
         return []
@@ -116,46 +129,41 @@ def table_to_document(table_data, document_id=None):
     section = table_data.get('section', 'Неизвестно')
     
     table_rows = table_data.get('data', [])
-    if not table_rows:
-        log_message(f"⚠️ ПРОПУЩЕНА: Таблица {table_num} из '{doc_id}' - нет данных")
+    if not table_rows or len(table_rows) == 0:
+        log_message(f"⚠️ ПРОПУЩЕНА: Таблица {table_num} из '{doc_id}' - нет данных в 'data'")
         return []
     
-    log_message(f"\n📊 Обработка таблицы {table_num} из документа '{doc_id}'")
-    log_message(f"  Название: {table_title}")
-    log_message(f"  Раздел: {section}")
-    log_message(f"  Строк данных: {len(table_rows)}")
-    
-    # Estimate if table needs chunking
-    sample_content = create_table_content(table_data)
-    estimated_size = len(sample_content)
-    
-    log_message(f"  Оценочный размер: {estimated_size} символов")
-    
-    # Threshold: if table is small enough, keep it whole
-    if estimated_size <= CHUNK_SIZE * 0.8:  # 80% of limit for safety
-        log_message(f"  ✅ Таблица достаточно мала, хранится целиком")
-        doc = Document(
-            text=sample_content,
-            metadata={
-                "type": "table",
-                "table_number": table_num,
-                "table_title": table_title,
-                "document_id": doc_id,
-                "section": section,
-                "section_id": section,
-                "headers": table_data.get('headers', []),
-                "total_rows": len(table_rows),
-                "content_size": estimated_size,
-                "is_chunked": False
-            }
-        )
-        return [doc]
+    content = create_table_content(table_data)
+    content_size = len(content)
+    row_count = len(table_rows)
+    
+    base_doc = Document(
+        text=content,
+        metadata={
+            "type": "table",
+            "table_number": table_num,
+            "table_title": table_title,
+            "document_id": doc_id,
+            "section": section,
+            "section_id": section,
+            "total_rows": row_count,
+            "content_size": content_size
+        }
+    )
+    
+    if content_size > CHUNK_SIZE:
+        log_message(f"📊 CHUNKING: Таблица {table_num} из '{doc_id}' | "
+                   f"Размер: {content_size} > {CHUNK_SIZE} | Строк: {row_count}")
+        chunked_docs = chunk_table_document(base_doc)
+        log_message(f"  ✂️ Разделена на {len(chunked_docs)} чанков")
+        for i, chunk_doc in enumerate(chunked_docs):
+            log_message(f"    Чанк {i+1}: {chunk_doc.metadata['chunk_size']} символов")
+        return chunked_docs
     else:
-        log_message(f"  ⚠️ Таблица слишком большая ({estimated_size} > {CHUNK_SIZE})")
-        log_message(f"  🔄 Применяется умный чанкинг с сохранением заголовков...")
-        chunks = create_table_chunks_with_headers(table_data)
-        log_message(f"  ✅ Таблица разбита на {len(chunks)} чанков с сохранением структуры")
-        return chunks
+        log_message(f"✓ ДОБАВЛЕНА: Таблица {table_num} из документа '{doc_id}' | "
+                   f"Размер: {content_size} символов | Строк: {row_count}")
+        return [base_doc]
+    
     
 def load_table_data(repo_id, hf_token, table_data_dir):
     log_message("=" * 60)

utils.py

@@ -21,9 +21,11 @@ def get_llm_model(model_name):
             raise Exception(f"API ключ не найден для модели {model_name}")
         
         if model_config["provider"] == "google":
+            # Fix: Remove image_config parameter or set it properly
             return GoogleGenAI(
                 model=model_config["model_name"], 
-                api_key=model_config["api_key"]
+                api_key=model_config["api_key"],
+                # Don't pass image_config=None
             )
         elif model_config["provider"] == "openai":
             return OpenAI(
@@ -35,7 +37,11 @@ def get_llm_model(model_name):
             
     except Exception as e:
         log_message(f"Ошибка создания модели {model_name}: {str(e)}")
-        return GoogleGenAI(model="gemini-2.0-flash", api_key=GOOGLE_API_KEY)
+        # Fix: Also apply to fallback model
+        return GoogleGenAI(
+            model="gemini-2.0-flash", 
+            api_key=GOOGLE_API_KEY
+        )
 
 def get_embedding_model(model_name="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"):
     return HuggingFaceEmbedding(model_name=model_name)
@@ -225,7 +231,7 @@ def generate_sources_html(nodes, chunks_df=None):
     
     html += "</div>"
     return html
-def answer_question(question, query_engine, reranker, current_model, chunks_df=None):
+def answer_question(question, query_engine, reranker, current_model, chunks_df=None, hybrid_retriever=None):
     if query_engine is None:
         return "<div style='background-color: #e53e3e; color: white; padding: 20px; border-radius: 10px;'>Система не инициализирована</div>", "", ""
     
@@ -234,18 +240,22 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
         
         llm = get_llm_model(current_model)
         
-        # Direct retrieval without query expansion
-        retrieved_nodes = query_engine.retriever.retrieve(question)
-        
-        log_message(f"Получено {len(retrieved_nodes)} узлов")
+        # Use hybrid retriever if available
+        if hybrid_retriever:
+            retrieved_nodes = hybrid_retriever.retrieve(question, top_k=30)
+            log_message(f"Hybrid retrieval: получено {len(retrieved_nodes)} узлов")
+        else:
+            retrieved_nodes = query_engine.retriever.retrieve(question)
+            log_message(f"Vector retrieval: получено {len(retrieved_nodes)} узлов")
         
+        # Rerank with increased top_k
         reranked_nodes = rerank_nodes(
             question,  
             retrieved_nodes, 
             reranker, 
-            top_k=20,
-            min_score_threshold=0.5, 
-            diversity_penalty=0.3
+            top_k=25,  # Increased from 20
+            min_score_threshold=0.3,  # Lowered from 0.5 to catch more results
+            diversity_penalty=0.2  # Reduced penalty
         )
         
         formatted_context = format_context_for_llm(reranked_nodes)