chunk size = 1024 + max chars = 1200 + keyword based

documents_prep.py

@@ -38,7 +38,7 @@ def chunk_text_documents(documents):
     return chunked
 
 
-def chunk_table_by_content(table_data, doc_id, max_chars=1000):
+def chunk_table_by_content(table_data, doc_id, max_chars=1200):
     """Chunk tables by content size instead of rows"""
     headers = table_data.get('headers', [])
     rows = table_data.get('data', [])

index_retriever.py

@@ -27,89 +27,55 @@ def create_vector_index(documents):
     index = VectorStoreIndex.from_documents(documents)
     log_message("✓ Index created")
     return index
-from llama_index.core.vector_stores import MetadataFilters, ExactMatchFilter
-import re
-
-def extract_document_id(query):
-    """Extract GOST document ID from query"""
-    patterns = [
-        r'ГОСТ\s*Р?\s*([\d\.]+(?:-\d{4})?)',
-        r'НП-[\d\-]+',
-        r'ПН\s+АЭ\s+Г-[\d\-]+'
-    ]
-    
-    for pattern in patterns:
-        match = re.search(pattern, query, re.IGNORECASE)
-        if match:
-            doc_id = match.group(0)
-            # Normalize
-            doc_id = re.sub(r'ГОСТ\s*Р', 'ГОСТ Р', doc_id, flags=re.IGNORECASE)
-            if 'ГОСТ' in doc_id and '-' not in doc_id:
-                doc_id += '-2020'
-            return doc_id
-    return None
 
+def keyword_filter_nodes(query, nodes, min_keyword_matches=1):
+    """Return nodes that contain at least one keyword from the query."""
+    keywords = [w.lower() for w in query.split() if len(w) > 2]
+    filtered = []
+    for node in nodes:
+        text = node.text.lower()
+        if any(k in text for k in keywords):
+            filtered.append(node)
+    return filtered
 
 def create_query_engine(vector_index):
-    """Create hybrid retrieval engine with document filtering"""
+    """Create hybrid retrieval engine with keyword boost"""
     log_message("Creating query engine...")
-    
-    def retrieve_with_filter(query_str):
-        """Custom retrieval with optional document filtering"""
-        doc_id = extract_document_id(query_str)
-        
-        if doc_id:
-            log_message(f"Detected document filter: {doc_id}")
-            
-            # Try filtered retrieval first
-            filters = MetadataFilters(
-                filters=[ExactMatchFilter(key="document_id", value=doc_id)]
-            )
-            
-            filtered_retriever = VectorIndexRetriever(
-                index=vector_index,
-                similarity_top_k=30,
-                filters=filters
-            )
-            
-            filtered_results = filtered_retriever.retrieve(query_str)
-            log_message(f"Filtered retrieval: {len(filtered_results)} results from {doc_id}")
-            
-            if len(filtered_results) >= 10:
-                # Good enough, use filtered results
-                return filtered_results
-            else:
-                log_message("Not enough filtered results, falling back to hybrid")
-        
-        # Fallback to hybrid retrieval
-        vector_retriever = VectorIndexRetriever(
-            index=vector_index,
-            similarity_top_k=50
-        )
-        
-        bm25_retriever = BM25Retriever.from_defaults(
-            docstore=vector_index.docstore,
-            similarity_top_k=50
-        )
-        
-        hybrid_retriever = QueryFusionRetriever(
-            [vector_retriever, bm25_retriever],
-            similarity_top_k=60,
-            num_queries=1
-        )
-        
-        return hybrid_retriever.retrieve(query_str)
-    
-    # Create custom query engine
-    class CustomRetriever:
-        def retrieve(self, query_str):
-            return retrieve_with_filter(query_str)
-    
+
+    vector_retriever = VectorIndexRetriever(
+        index=vector_index,
+        similarity_top_k=50
+    )
+    bm25_retriever = BM25Retriever.from_defaults(
+        docstore=vector_index.docstore,
+        similarity_top_k=50
+    )
+    hybrid_retriever = QueryFusionRetriever(
+        [vector_retriever, bm25_retriever],
+        similarity_top_k=60,
+        num_queries=1
+    )
     response_synthesizer = get_response_synthesizer()
-    query_engine = RetrieverQueryEngine(
-        retriever=CustomRetriever(),
+
+    class KeywordBoostQueryEngine(RetrieverQueryEngine):
+        def retrieve(self, query):
+            # Hybrid results
+            hybrid_nodes = hybrid_retriever.retrieve(query)
+            # Keyword filter from all indexed nodes
+            all_nodes = list(vector_index.docstore.values())
+            keyword_nodes = keyword_filter_nodes(query, all_nodes)
+            # Combine and deduplicate
+            all_candidates = {id(n): n for n in hybrid_nodes + keyword_nodes}
+            log_message(f"Hybrid: {len(hybrid_nodes)}, Keyword: {len(keyword_nodes)}, Total: {len(all_candidates)}")
+            return list(all_candidates.values())[:60]
+        def query(self, prompt):
+            nodes = self.retrieve(prompt)
+            return response_synthesizer.synthesize(prompt, nodes)
+
+    query_engine = KeywordBoostQueryEngine(
+        retriever=hybrid_retriever,
         response_synthesizer=response_synthesizer
     )
-    
-    log_message("✓ Query engine created with document filtering")
+
+    log_message("✓ Query engine created (with keyword boost)")
     return query_engine