added sheet_name

index_retriever.py

@@ -13,77 +13,111 @@ def create_vector_index(documents):
     return VectorStoreIndex.from_documents(documents)
 
 def create_query_engine(vector_index):
-    bm25_retriever = BM25Retriever.from_defaults(
-        docstore=vector_index.docstore, 
-        similarity_top_k=80
-    )
-    
-    vector_retriever = VectorIndexRetriever(
-        index=vector_index, 
-        similarity_top_k=80,
-        similarity_cutoff=0.45
-    )
-    
-    hybrid_retriever = QueryFusionRetriever(
-        [vector_retriever, bm25_retriever],
-        similarity_top_k=100,
-        num_queries=1
-    )
-    
-    custom_prompt_template = PromptTemplate(PROMPT_SIMPLE_POISK)
-    response_synthesizer = get_response_synthesizer(
-        response_mode=ResponseMode.TREE_SUMMARIZE,  
-        text_qa_template=custom_prompt_template
-    )
-    
-    query_engine = RetrieverQueryEngine(
-        retriever=hybrid_retriever,
-        response_synthesizer=response_synthesizer
-    )
-    
-    return query_engine
+    try:
+        bm25_retriever = BM25Retriever.from_defaults(
+            docstore=vector_index.docstore, 
+            similarity_top_k=30 
+        )
+        
+        vector_retriever = VectorIndexRetriever(
+            index=vector_index, 
+            similarity_top_k=30,  
+            similarity_cutoff=0.65
+        )
+        
+        hybrid_retriever = QueryFusionRetriever(
+            [vector_retriever, bm25_retriever],
+            similarity_top_k=40,  
+            num_queries=1
+        )
+        
+        custom_prompt_template = PromptTemplate(PROMPT_SIMPLE_POISK)
+        response_synthesizer = get_response_synthesizer(
+            response_mode=ResponseMode.TREE_SUMMARIZE,  
+            text_qa_template=custom_prompt_template
+        )
+        
+        query_engine = RetrieverQueryEngine(
+            retriever=hybrid_retriever,
+            response_synthesizer=response_synthesizer
+        )
+        
+        log_message("Query engine успешно создан")
+        return query_engine
+        
+    except Exception as e:
+        log_message(f"Ошибка создания query engine: {str(e)}")
+        raise
 
-    
-def rerank_nodes(query, nodes, reranker, top_k=40, min_score_threshold=0.35, diversity_penalty=0.15):
+def rerank_nodes(query, nodes, reranker, top_k=25, min_score_threshold=0.45, diversity_penalty=0.2):
+    """Rerank with better handling of specific technical queries"""
     if not nodes or not reranker:
         return nodes[:top_k]
     
-    pairs = [[query, node.text] for node in nodes]
-    scores = reranker.predict(pairs)
-    scored_nodes = list(zip(nodes, scores))
-    scored_nodes.sort(key=lambda x: x[1], reverse=True)
-    
-    if min_score_threshold:
-        scored_nodes = [(node, score) for node, score in scored_nodes 
-                      if score >= min_score_threshold]
-    
-    if not scored_nodes:
+    try:
+        log_message(f"Переранжирую {len(nodes)} узлов для запроса: {query[:50]}...")
+        
+        pairs = [[query, node.text] for node in nodes]
+        scores = reranker.predict(pairs)
         scored_nodes = list(zip(nodes, scores))
+        
         scored_nodes.sort(key=lambda x: x[1], reverse=True)
-        scored_nodes = scored_nodes[:top_k]
-    
-    selected = []
-    seen_docs = {}
-    
-    for node, score in scored_nodes:
-        if len(selected) >= top_k:
-            break
         
-        meta = node.metadata if hasattr(node, 'metadata') else {}
-        doc_id = meta.get('document_id', 'unknown')
-        node_type = meta.get('type', 'text')
-        table_num = meta.get('table_number', '')
+        # Lower threshold for technical queries
+        if min_score_threshold is not None:
+            scored_nodes = [(node, score) for node, score in scored_nodes 
+                          if score >= min_score_threshold]
+            log_message(f"После фильтрации (порог {min_score_threshold}): {len(scored_nodes)} узлов")
         
-        key = f"{doc_id}_{table_num}" if node_type == 'table' else f"{doc_id}_{meta.get('section_id', '')}"
+        if not scored_nodes:
+            log_message("⚠️ Нет узлов после фильтрации, снижаю порог до 0.3")
+            scored_nodes = list(zip(nodes, scores))
+            scored_nodes.sort(key=lambda x: x[1], reverse=True)
+            min_score_threshold = max(0.3, scored_nodes[0][1] * 0.5)
+            scored_nodes = [(node, score) for node, score in scored_nodes 
+                          if score >= min_score_threshold]
         
-        if key in seen_docs:
-            penalty = diversity_penalty * 0.2 if node_type == 'table' else diversity_penalty
+        selected_nodes = []
+        selected_docs = {}  # Track count per document
+        selected_tables = set()
+        
+        for node, score in scored_nodes:
+            if len(selected_nodes) >= top_k:
+                break
+            
+            metadata = node.metadata if hasattr(node, 'metadata') else {}
+            doc_id = metadata.get('document_id', 'unknown')
+            node_type = metadata.get('type', 'text')
+            
+            # Track table uniqueness
+            if node_type == 'table':
+                table_id = metadata.get('full_table_id', '')
+                if table_id in selected_tables:
+                    continue  # Skip duplicate table chunks
+                selected_tables.add(table_id)
+            
+            # Apply lighter diversity penalty
+            penalty = 0
+            doc_count = selected_docs.get(doc_id, 0)
+            if doc_count > 0:
+                penalty = min(diversity_penalty * doc_count, 0.5)
+            
             adjusted_score = score * (1 - penalty)
-        else:
-            adjusted_score = score
-            seen_docs[key] = 1
+            
+            # Accept if competitive
+            if not selected_nodes or adjusted_score >= selected_nodes[0][1] * 0.5:
+                selected_nodes.append((node, score))
+                selected_docs[doc_id] = doc_count + 1
         
-        if not selected or adjusted_score >= selected[0][1] * 0.4:
-            selected.append((node, score))
-    
-    return [node for node, score in selected]
+        log_message(f"✓ Выбрано {len(selected_nodes)} узлов")
+        log_message(f"  Уникальных документов: {len(selected_docs)}")
+        log_message(f"  Уникальных таблиц: {len(selected_tables)}")
+        
+        if selected_nodes:
+            log_message(f"  Score: {selected_nodes[0][1]:.3f} → {selected_nodes[-1][1]:.3f}")
+        
+        return [node for node, score in selected_nodes]
+        
+    except Exception as e:
+        log_message(f"❌ Ошибка переранжировки: {str(e)}")
+        return nodes[:top_k]

table_prep.py

@@ -17,9 +17,8 @@ def normalize_table_number(table_num, section):
     
     return f"№{tn}"
 
-
 def create_table_content(table_data):
-    """Create formatted content with strong contextual anchors"""
+    """Create formatted content optimized for semantic search"""
     doc_id = (
         table_data.get('document_id') or 
         table_data.get('document') or 
@@ -33,43 +32,48 @@ def create_table_content(table_data):
         table_data.get('Раздел документа') or
         'Неизвестно'
     )
+    sheet_name = table_data.get('sheet_name', '')
     
-    # Normalize table number
+    # Enhanced table number with appendix context
     normalized_num = normalize_table_number(table_num, section)
+    if 'Приложени' in str(section):
+        # Extract appendix number
+        import re
+        appendix_match = re.search(r'Приложени[ея]\s*(\d+)', str(section))
+        if appendix_match:
+            appendix_num = appendix_match.group(1)
+            normalized_num = f"{normalized_num} Приложения {appendix_num}"
     
-    # STRONG ANCHOR: Unique identification for semantic search
-    content = f"=== ИСТОЧНИК ДАННЫХ ===\n"
-    content += f"Документ: {doc_id}\n"
-    content += f"Стандарт: {doc_id}\n"
+    # Build searchable header
+    content = f"Документ: {doc_id}\n"
     content += f"Раздел: {section}\n"
     content += f"Таблица: {normalized_num}\n"
-    content += f"Полное название: {table_title}\n"
-    content += f"Уникальный ID: {doc_id} | {section} | {normalized_num}\n"
-    content += f"======================\n\n"
+    content += f"Название: {table_title}\n"
+    if sheet_name:
+        content += f"Лист: {sheet_name}\n"
+    content += f"\n"
     
     headers = table_data.get('headers', [])
     if headers:
-        content += f"Заголовки колонок: {' | '.join(str(h) for h in headers)}\n\n"
+        header_str = ' | '.join(str(h) for h in headers)
+        content += f"Колонки: {header_str}\n\n"
     
-    # Structured row data with JSON-like clarity
+    # CRITICAL: Preserve searchable row identifiers
     if 'data' in table_data and isinstance(table_data['data'], list):
-        content += "Содержимое таблицы:\n"
         for row_idx, row in enumerate(table_data['data'], start=1):
             if isinstance(row, dict):
-                # Add row identifier if available
-                row_id = row.get('Условное обозначение сварного соединения', 
-                                row.get('Обозначение', ''))
-                if row_id:
-                    content += f"Строка {row_idx} ({row_id}): "
-                else:
-                    content += f"Строка {row_idx}: "
+                # Extract ALL key-value pairs naturally
+                row_parts = []
+                for k, v in row.items():
+                    if v and str(v).strip() and str(v) != 'nan':
+                        row_parts.append(f"{k}: {v}")
                 
-                # Structured key-value pairs for better semantic understanding
-                row_parts = [f"{k}={v}" for k, v in row.items() if v and str(v).strip()]
-                content += " | ".join(row_parts) + "\n"
+                if row_parts:
+                    content += ' | '.join(row_parts) + "\n"
             elif isinstance(row, list):
-                content += f"Строка {row_idx}: "
-                content += " | ".join([str(v) for v in row if v and str(v).strip()]) + "\n"
+                row_str = ' | '.join([str(v) for v in row if v and str(v).strip() and str(v) != 'nan'])
+                if row_str:
+                    content += row_str + "\n"
     
     return content, normalized_num
 
@@ -83,28 +87,20 @@ def chunk_table_document(doc, chunk_size=None, chunk_overlap=None):
     table_num = doc.metadata.get('table_number', 'unknown')
     doc_id = doc.metadata.get('document_id', 'unknown')
     section = doc.metadata.get('section', 'Неизвестно')
-    table_title = doc.metadata.get('table_title', 'Неизвестно')
     
-    # Create unique anchor for this table
     full_table_id = f"{doc_id} | {section} | {table_num}"
     
     lines = doc.text.strip().split('\n')
     
-    # Extract header (everything before data rows)
-    table_header_lines = []
-    data_rows = []
-    in_data = False
-    
-    for line in lines:
-        if line.startswith('Содержимое таблицы:'):
-            in_data = True
-            table_header_lines.append(line)
-        elif in_data and line.startswith('Строка'):
-            data_rows.append(line)
-        elif not in_data:
-            table_header_lines.append(line)
+    # Find where data rows start
+    data_start_idx = 0
+    for i, line in enumerate(lines):
+        if line.startswith('Колонки:'):
+            data_start_idx = i + 2  # Skip header and blank line
+            break
     
-    table_header = '\n'.join(table_header_lines) + '\n'
+    table_header = '\n'.join(lines[:data_start_idx])
+    data_rows = lines[data_start_idx:]
     
     if not data_rows or len(doc.text) < chunk_size * 1.5:
         log_message(f"  📊 {full_table_id}: малая таблица, без разбиения")
@@ -113,7 +109,7 @@ def chunk_table_document(doc, chunk_size=None, chunk_overlap=None):
     log_message(f"  📋 {full_table_id}: {len(data_rows)} строк → chunking")
     
     header_size = len(table_header)
-    available_size = chunk_size - header_size - 200  # More reserve for anchor
+    available_size = chunk_size - header_size - 100
     
     text_chunks = []
     current_chunk_rows = []
@@ -123,11 +119,11 @@ def chunk_table_document(doc, chunk_size=None, chunk_overlap=None):
         row_size = len(row) + 1
         
         if current_size + row_size > available_size and current_chunk_rows:
-            chunk_text = table_header + '\n'.join(current_chunk_rows)
+            chunk_text = table_header + '\n' + '\n'.join(current_chunk_rows)
             text_chunks.append(chunk_text)
             
-            # Overlap: keep last 3 rows for better context
-            overlap_count = min(3, len(current_chunk_rows))
+            # Keep last 2 rows for overlap
+            overlap_count = min(2, len(current_chunk_rows))
             current_chunk_rows = current_chunk_rows[-overlap_count:]
             current_size = sum(len(r) + 1 for r in current_chunk_rows)
         
@@ -135,12 +131,11 @@ def chunk_table_document(doc, chunk_size=None, chunk_overlap=None):
         current_size += row_size
     
     if current_chunk_rows:
-        chunk_text = table_header + '\n'.join(current_chunk_rows)
+        chunk_text = table_header + '\n' + '\n'.join(current_chunk_rows)
         text_chunks.append(chunk_text)
     
     log_message(f"  ✂️ {full_table_id} → {len(text_chunks)} чанков")
     
-    # Create chunks with strong anchors
     chunked_docs = []
     for i, chunk_text in enumerate(text_chunks):
         chunk_metadata = doc.metadata.copy()
@@ -149,11 +144,8 @@ def chunk_table_document(doc, chunk_size=None, chunk_overlap=None):
             "total_chunks": len(text_chunks),
             "chunk_size": len(chunk_text),
             "is_chunked": True,
-            # CRITICAL: Add unique identifiers
             "full_table_id": full_table_id,
-            "chunk_anchor": f"{full_table_id} | chunk_{i+1}/{len(text_chunks)}",
-            "document_section": section,
-            "table_number_normalized": table_num
+            "table_number_normalized": doc.metadata.get('table_number_normalized')
         })
         
         chunked_doc = Document(
@@ -165,6 +157,57 @@ def chunk_table_document(doc, chunk_size=None, chunk_overlap=None):
     return chunked_docs
 
 
+def table_to_document(table_data, document_id=None):
+    """Convert table data to Document with complete metadata"""
+    if not isinstance(table_data, dict):
+        return []
+    
+    sheet_doc_id = (
+        table_data.get('document_id') or 
+        table_data.get('document') or 
+        table_data.get('Обозначение документа')
+    )
+    
+    doc_id = sheet_doc_id or document_id or 'Неизвестно'
+    
+    table_num = table_data.get('table_number', 'Неизвестно')
+    table_title = table_data.get('table_title', 'Неизвестно')
+    section = table_data.get('section', table_data.get('Раздел документа', 'Неизвестно'))
+    sheet_name = table_data.get('sheet_name', '')
+    
+    table_rows = table_data.get('data', [])
+    if not table_rows:
+        log_message(f"⚠️ Таблица {table_num} ({doc_id}) пропущена: нет данных")
+        return []
+    
+    content, normalized_num = create_table_content(table_data)
+    content_size = len(content)
+    
+    base_doc = Document(
+        text=content,
+        metadata={
+            "type": "table",
+            "table_number": table_num,
+            "table_number_normalized": normalized_num,
+            "table_title": table_title,
+            "document_id": doc_id,
+            "section": section,
+            "section_id": section,
+            "sheet_name": sheet_name,
+            "total_rows": len(table_rows),
+            "content_size": content_size,
+            "full_table_id": f"{doc_id} | {section} | {normalized_num}"
+        }
+    )
+    
+    if content_size > CHUNK_SIZE:
+        log_message(f"📊 CHUNKING: {doc_id} | {normalized_num} | {content_size} > {CHUNK_SIZE}")
+        return chunk_table_document(base_doc)
+    else:
+        log_message(f"✓ {doc_id} | {normalized_num} ({content_size} символов)")
+        return [base_doc]
+    
+    
 def table_to_document(table_data, document_id=None):
     """Convert table data to Document with proper metadata"""
     if not isinstance(table_data, dict):