removed normalization doc id

documents_prep.py

@@ -38,7 +38,7 @@ def chunk_text_documents(documents):
     return chunked
 
 
-def chunk_table_by_content(table_data, doc_id, max_chars=2000):
+def chunk_table_by_content(table_data, doc_id, max_chars=1024):
     """Chunk tables by content size instead of rows"""
     headers = table_data.get('headers', [])
     rows = table_data.get('data', [])
@@ -222,7 +222,7 @@ def load_table_documents(repo_id, hf_token, table_dir):
             for sheet in data.get('sheets', []):
                 sheet_doc_id = sheet.get('document_id', sheet.get('document', file_doc_id))
                 
-                chunks = chunk_table_by_content(sheet, sheet_doc_id, max_chars=2000)
+                chunks = chunk_table_by_content(sheet, sheet_doc_id, max_chars=1024)
                 all_chunks.extend(chunks)
                 
         except Exception as e:

index_retriever.py

@@ -6,21 +6,6 @@ from llama_index.core.retrievers import QueryFusionRetriever
 from llama_index.core.response_synthesizers import get_response_synthesizer
 from my_logging import log_message
 
-SIMPLE_PROMPT = """Вы - эксперт по нормативной документации.
-
-Контекст:
-{context_str}
-
-Вопрос: {query_str}
-
-Инструкция:
-1. Отвечайте ТОЛЬКО на основе предоставленного контекста
-2. Цитируйте конкретные источники (документ, раздел, таблицу)
-3. Если информации недостаточно, четко укажите это
-4. Будьте точны и конкретны
-
-Ответ:"""
-
 def create_vector_index(documents):
     """Create vector index from documents"""
     log_message(f"Building vector index from {len(documents)} documents...")
@@ -44,15 +29,15 @@ def create_query_engine(vector_index):
 
     vector_retriever = VectorIndexRetriever(
         index=vector_index,
-        similarity_top_k=80  # Reduced from 50
+        similarity_top_k=80
     )
     bm25_retriever = BM25Retriever.from_defaults(
         docstore=vector_index.docstore,
-        similarity_top_k=80  # Reduced from 50
+        similarity_top_k=80,
     )
     hybrid_retriever = QueryFusionRetriever(
         [vector_retriever, bm25_retriever],
-        similarity_top_k=100,  # Reduced from 60
+        similarity_top_k=100,
         num_queries=1
     )
 
@@ -73,11 +58,20 @@ def create_query_engine(vector_index):
             
             log_message(f"Retrieved: {len(nodes)} → Unique: {len(unique_nodes)}")
             return unique_nodes[:50]  # Return top 50 unique
+        
+        # FIX: Override query method to use our retrieve
+        def query(self, query_bundle):
+            nodes = self.retrieve(query_bundle.query_str)
+            response = self._response_synthesizer.synthesize(
+                query=query_bundle,
+                nodes=nodes
+            )
+            return response
 
     response_synthesizer = get_response_synthesizer()
     
     query_engine = DeduplicatedQueryEngine(
-        retriever=hybrid_retriever,
+        retriever=hybrid_retriever,  # Still pass it but we override retrieve()
         response_synthesizer=response_synthesizer
     )
 

utils.py

@@ -47,7 +47,7 @@ def answer_question(question, query_engine, reranker):
         retrieved = query_engine.retrieve(question)
         log_message(f"RETRIEVED: {len(retrieved)} unique nodes")
 
-        reranked = rerank_nodes(question, retrieved, reranker, top_k=20, min_score=0.3)
+        reranked = rerank_nodes(question, retrieved, reranker, top_k=25, min_score=0.1)
         log_message(f"RERANKED: {len(reranked)} nodes")
 
         context_parts = []
@@ -83,7 +83,7 @@ def answer_question(question, query_engine, reranker):
         log_message(traceback.format_exc())
         return f"Ошибка: {e}", ""
 
-def rerank_nodes(query, nodes, reranker, top_k=20, min_score=0.3):
+def rerank_nodes(query, nodes, reranker, top_k=20, min_score=0.1):
     """Simple and effective reranking: sort by score and filter by threshold."""
     if not nodes or not reranker:
         return nodes[:top_k]