table prep changed

index_retriever.py

@@ -16,7 +16,7 @@ def create_query_engine(vector_index):
     try:
         bm25_retriever = BM25Retriever.from_defaults(
             docstore=vector_index.docstore, 
-            similarity_top_k=20 
+            similarity_top_k=30 
         )
         
         vector_retriever = VectorIndexRetriever(
@@ -49,119 +49,6 @@ def create_query_engine(vector_index):
         log_message(f"Ошибка создания query engine: {str(e)}")
         raise
 
-import re
-from typing import List, Dict, Set
-from my_logging import log_message
-
-def extract_keywords_from_query(query: str) -> Dict[str, List[str]]:
-    """Extract technical keywords from query"""
-    keywords = {
-        'materials': [],
-        'gosts': [],
-        'classes': [],
-        'technical_terms': []
-    }
-    
-    # Material codes: 08Х18Н10Т, 12Х18Н10Т, etc.
-    material_pattern = r'\b\d{2}[ХНТМКВБА]+\d{1,2}[ХНТМКВБА]*\d*\b'
-    keywords['materials'] = re.findall(material_pattern, query, re.IGNORECASE)
-    
-    # GOST standards
-    gost_pattern = r'ГОСТ\s+[РЕН\s]*\d+[\.\-\d]*'
-    keywords['gosts'] = re.findall(gost_pattern, query, re.IGNORECASE)
-    
-    # Classification codes: 3СIIIa, 1А, 2BII, etc.
-    class_pattern = r'\b\d[АБВГСD]+[IV]+[a-z]?\b'
-    keywords['classes'] = re.findall(class_pattern, query, re.IGNORECASE)
-    
-    # Technical terms
-    terms = ['полуфабрикат', 'план качества', 'контроль', 'арматура', 
-             'ультразвуковой', 'сварка', 'испытание']
-    for term in terms:
-        if term.lower() in query.lower():
-            keywords['technical_terms'].append(term)
-    
-    return keywords
-
-def keyword_search_nodes(nodes: List, keywords: Dict[str, List[str]]) -> List:
-    """Filter nodes by exact keyword matches"""
-    if not any(keywords.values()):
-        return nodes
-    
-    matched_nodes = []
-    
-    for node in nodes:
-        text_lower = node.text.lower()
-        metadata = node.metadata if hasattr(node, 'metadata') else {}
-        
-        # Check materials
-        for material in keywords['materials']:
-            if material.lower() in text_lower:
-                matched_nodes.append(node)
-                break
-        else:
-            # Check GOSTs
-            for gost in keywords['gosts']:
-                if gost.lower() in text_lower:
-                    matched_nodes.append(node)
-                    break
-            else:
-                # Check classes
-                for cls in keywords['classes']:
-                    if cls.lower() in text_lower:
-                        matched_nodes.append(node)
-                        break
-                else:
-                    # Check technical terms (at least 2 matches)
-                    term_matches = sum(1 for term in keywords['technical_terms'] 
-                                      if term.lower() in text_lower)
-                    if term_matches >= 2:
-                        matched_nodes.append(node)
-    
-    return matched_nodes
-
-def hybrid_retrieve_with_keywords(question: str, query_engine, top_k: int = 40) -> List:
-    """Retrieve using both vector search and keyword matching"""
-    
-    # Extract keywords from query
-    keywords = extract_keywords_from_query(question)
-    log_message(f"Извлечены ключевые слова: {keywords}")
-    
-    # Get vector search results
-    vector_nodes = query_engine.retriever.retrieve(question)
-    log_message(f"Векторный поиск: {len(vector_nodes)} узлов")
-    
-    # Apply keyword filtering
-    if any(keywords.values()):
-        keyword_nodes = keyword_search_nodes(vector_nodes, keywords)
-        log_message(f"После фильтрации по ключевым словам: {len(keyword_nodes)} узлов")
-        
-        # If keyword search found results, prioritize them
-        if keyword_nodes:
-            # Deduplicate and combine
-            seen_ids = set()
-            combined_nodes = []
-            
-            # First add keyword matches
-            for node in keyword_nodes[:top_k]:
-                node_id = id(node)
-                if node_id not in seen_ids:
-                    combined_nodes.append(node)
-                    seen_ids.add(node_id)
-            
-            # Then fill with vector results
-            for node in vector_nodes:
-                if len(combined_nodes) >= top_k:
-                    break
-                node_id = id(node)
-                if node_id not in seen_ids:
-                    combined_nodes.append(node)
-                    seen_ids.add(node_id)
-            
-            return combined_nodes[:top_k]
-    
-    return vector_nodes[:top_k]
-
 def rerank_nodes(query, nodes, reranker, top_k=25, min_score_threshold=0.5, diversity_penalty=0.3):
     if not nodes or not reranker:
         return nodes[:top_k]
@@ -225,5 +112,4 @@ def rerank_nodes(query, nodes, reranker, top_k=25, min_score_threshold=0.5, dive
         
     except Exception as e:
         log_message(f"Ошибка переранжировки: {str(e)}")
-        return nodes[:top_k]
-    
+        return nodes[:top_k]

table_prep.py

@@ -32,7 +32,21 @@ def create_table_content(table_data):
 from llama_index.core.text_splitter import SentenceSplitter
 from config import CHUNK_SIZE, CHUNK_OVERLAP
 
-# In table_prep.py - replace chunk_table_document function
+def extract_table_metadata(table_text: str) -> dict:
+    words = table_text.split()
+    unique_words = set(words)
+
+    from collections import Counter
+    stopwords = {"и", "в", "на", "по", "с", "для", "из", "при", "а", "как", "или", "но", "к", "от"}
+    filtered = [w for w in words if len(w) > 3 and w.lower() not in stopwords]
+    common = Counter(filtered).most_common(15)
+    key_terms = [w for w, _ in common]
+
+    return {
+        "summary": f"Таблица содержит около {len(words)} слов и {len(unique_words)} уникальных терминов.",
+        "materials": [],   # if you want to extract material names, hook in regex or LLM here
+        "key_terms": key_terms
+    }
 
 def chunk_table_document(doc, chunk_size=None, chunk_overlap=None):
     if chunk_size is None:
@@ -42,37 +56,109 @@ def chunk_table_document(doc, chunk_size=None, chunk_overlap=None):
     
     # Extract critical metadata from table before chunking
     table_metadata = extract_table_metadata(doc.text)
+    table_num = doc.metadata.get('table_number', 'unknown')
+    table_title = doc.metadata.get('table_title', 'unknown')
+    doc_id = doc.metadata.get('document_id', 'unknown')
+    section = doc.metadata.get('section', 'unknown')
     
-    text_splitter = SentenceSplitter(
-        chunk_size=chunk_size,
-        chunk_overlap=chunk_overlap,
-        separator="\n"
-    )
+    # Parse table structure from your create_table_content format
+    lines = doc.text.strip().split('\n')
+    
+    # Find where data rows start
+    table_header_lines = []
+    data_rows = []
+    in_data = False
+    
+    for line in lines:
+        if line.startswith('Данные таблицы:'):
+            in_data = True
+            table_header_lines.append(line)
+        elif in_data and line.startswith('Строка'):
+            data_rows.append(line)
+        elif not in_data:
+            table_header_lines.append(line)
+    
+    table_header = '\n'.join(table_header_lines) + '\n'
+    
+    if not data_rows:
+        log_message(f"  ⚠️ Таблица {table_num}: нет строк данных, использую стандартное разбиение")
+        text_splitter = SentenceSplitter(
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap,
+            separator="\n"
+        )
+        text_chunks = text_splitter.split_text(doc.text)
+        log_message(f"  📊 Стандартное разбиение: {len(text_chunks)} чанков")
+    else:
+        # Row-based chunking
+        log_message(f"  📋 Таблица {table_num}: найдено {len(data_rows)} строк данных")
+        
+        header_size = len(table_header)
+        # Reserve space for enrichment prefix
+        available_size = chunk_size - header_size - 300
+        
+        text_chunks = []
+        current_chunk_rows = []
+        current_size = 0
+        
+        for row in data_rows:
+            row_size = len(row) + 1
+            
+            # Check if adding this row exceeds limit
+            if current_size + row_size > available_size and current_chunk_rows:
+                # Create chunk
+                chunk_text = table_header + '\n'.join(current_chunk_rows)
+                text_chunks.append(chunk_text)
+                log_message(f"    ✂️ Чанк создан: {len(current_chunk_rows)} строк, {len(chunk_text)} символов")
+                
+                # Overlap: keep last 2 rows
+                overlap_count = min(2, len(current_chunk_rows))
+                current_chunk_rows = current_chunk_rows[-overlap_count:]
+                current_size = sum(len(r) + 1 for r in current_chunk_rows)
+            
+            current_chunk_rows.append(row)
+            current_size += row_size
+        
+        # Final chunk
+        if current_chunk_rows:
+            chunk_text = table_header + '\n'.join(current_chunk_rows)
+            text_chunks.append(chunk_text)
+            log_message(f"    ✂️ Последний чанк: {len(current_chunk_rows)} строк, {len(chunk_text)} символов")
     
-    text_chunks = text_splitter.split_text(doc.text)
+    log_message(f"  📊 Таблица {table_num} разделена на {len(text_chunks)} чанков")
     
+    # Create enriched chunks
     chunked_docs = []
+    materials = table_metadata.get("materials", [])
+    key_terms = table_metadata.get("key_terms", [])
+    
     for i, chunk_text in enumerate(text_chunks):
         chunk_metadata = doc.metadata.copy()
-        
-        # Add extracted keywords/materials to each chunk
         chunk_metadata.update({
             "chunk_id": i,
             "total_chunks": len(text_chunks),
             "chunk_size": len(chunk_text),
             "is_chunked": True,
-            "materials": table_metadata.get("materials", []),  # All materials from table
-            "key_terms": table_metadata.get("key_terms", []),  # Technical terms
-            "table_summary": table_metadata.get("summary", "")  # Brief table description
+            "materials": materials,
+            "key_terms": key_terms,
+            "table_summary": table_metadata.get("summary", "")
         })
         
-        # Enrich chunk text with context from full table
-        enriched_text = f"""[Таблица {doc.metadata.get('table_number')}: {doc.metadata.get('table_title')}]
-[Материалы в таблице: {', '.join(table_metadata.get('materials', [])[:10])}]
-[Ключевые термины: {', '.join(table_metadata.get('key_terms', [])[:10])}]
+        # Enrichment prefix
+        materials_str = ', '.join(materials[:10]) if materials else 'нет'
+        terms_str = ', '.join(key_terms[:10]) if key_terms else 'нет'
+        
+        enriched_text = f"""[Таблица {table_num}: {table_title}]
+[Материалы в таблице: {materials_str}]
+[Ключевые термины: {terms_str}]
 
 {chunk_text}"""
         
+        log_message(f"    ✓ Чанк {i+1}/{len(text_chunks)}: "
+                   f"размер={len(enriched_text)}, "
+                   f"материалов={len(materials)}, "
+                   f"терминов={len(key_terms)}")
+        
         chunked_doc = Document(
             text=enriched_text,
             metadata=chunk_metadata
@@ -81,43 +167,6 @@ def chunk_table_document(doc, chunk_size=None, chunk_overlap=None):
     
     return chunked_docs
 
-
-def extract_table_metadata(table_text):
-    """Extract searchable metadata from table content"""
-    import re
-    
-    # Extract material codes (e.g., 08Х18Н10Т)
-    material_pattern = r'\b\d{2}[ХНТМКВБА]+\d{1,2}[ХНТМКВБА]*\d*\b'
-    materials = list(set(re.findall(material_pattern, table_text, re.IGNORECASE)))
-    
-    # Extract GOST standards
-    gost_pattern = r'ГОСТ\s+[РЕН\s]*\d+[\.\-\d]*'
-    gosts = list(set(re.findall(gost_pattern, table_text, re.IGNORECASE)))
-    
-    # Extract class/category codes
-    class_pattern = r'\b\d[АБВСI]+[IVX]+[a-z]*\b'
-    classes = list(set(re.findall(class_pattern, table_text, re.IGNORECASE)))
-    
-    # Extract common technical terms
-    tech_terms = []
-    keywords = ['контроль', 'испытание', 'сертификат', 'качество', 'план', 
-                'полуфабрикат', 'оборудование', 'арматура', 'деталь']
-    for keyword in keywords:
-        if keyword.lower() in table_text.lower():
-            tech_terms.append(keyword)
-    
-    # Create brief summary
-    lines = table_text.split('\n')[:5]
-    summary = ' '.join([l.strip() for l in lines if l.strip()])[:200]
-    
-    return {
-        "materials": materials,
-        "gosts": gosts,
-        "classes": classes,
-        "key_terms": tech_terms + gosts,
-        "summary": summary
-    }
-
 def table_to_document(table_data, document_id=None):
     if not isinstance(table_data, dict):
         log_message(f"⚠️ ПРОПУЩЕНА: table_data не является словарем")

utils.py

@@ -231,8 +231,7 @@ def generate_sources_html(nodes, chunks_df=None):
     
     html += "</div>"
     return html
-def answer_question(question, query_engine, reranker, current_model, chunks_df=None, hybrid_retriever=None):
-    from index_retriever import hybrid_retrieve_with_keywords
+def answer_question(question, query_engine, reranker, current_model, chunks_df=None):
     if query_engine is None:
         return "<div style='background-color: #e53e3e; color: white; padding: 20px; border-radius: 10px;'>Система не инициализирована</div>", "", ""
     
@@ -241,18 +240,18 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
         
         llm = get_llm_model(current_model)
         
-        # Use keyword-enhanced retrieval
-        retrieved_nodes = hybrid_retrieve_with_keywords(question, query_engine, top_k=40)
-        log_message(f"Hybrid keyword retrieval: получено {len(retrieved_nodes)} узлов")
+        # Direct retrieval without query expansion
+        retrieved_nodes = query_engine.retriever.retrieve(question)
+        
+        log_message(f"Получено {len(retrieved_nodes)} узлов")
         
-        # Rerank
         reranked_nodes = rerank_nodes(
             question,  
             retrieved_nodes, 
             reranker, 
             top_k=25,
-            min_score_threshold=0.3,
-            diversity_penalty=0.2
+            min_score_threshold=0.5, 
+            diversity_penalty=0.3
         )
         
         formatted_context = format_context_for_llm(reranked_nodes)