Much lower reranking threshold (-0.5 instead of 0.1) + detailed score logging

documents_prep.py

@@ -412,44 +412,6 @@ def extract_sections_from_json(json_path):
     return documents
 
 
-def load_table_documents(repo_id, hf_token, table_dir):
-    """Load and chunk tables"""
-    log_message("Loading tables...")
-    
-    files = list_repo_files(repo_id=repo_id, repo_type="dataset", token=hf_token)
-    table_files = [f for f in files if f.startswith(table_dir) and f.endswith('.json')]
-    
-    all_chunks = []
-    for file_path in table_files:
-        try:
-            local_path = hf_hub_download(
-                repo_id=repo_id,
-                filename=file_path,
-                repo_type="dataset",
-                token=hf_token
-            )
-            
-            with open(local_path, 'r', encoding='utf-8') as f:
-                data = json.load(f)
-            
-            # Extract file-level document_id
-            file_doc_id = data.get('document_id', data.get('document', 'unknown'))
-            
-            for sheet in data.get('sheets', []):
-                # Use sheet-level document_id if available, otherwise use file-level
-                sheet_doc_id = sheet.get('document_id', sheet.get('document', file_doc_id))
-                
-                # CRITICAL: Pass document_id to chunk function
-                chunks = chunk_table_by_content(sheet, sheet_doc_id)
-                all_chunks.extend(chunks)
-                
-        except Exception as e:
-            log_message(f"Error loading {file_path}: {e}")
-    
-    log_message(f"✓ Loaded {len(all_chunks)} table chunks")
-    return all_chunks
-
-
 def load_image_documents(repo_id, hf_token, image_dir):
     """Load image descriptions"""
     log_message("Loading images...")

index_retriever.py

@@ -24,56 +24,58 @@ def keyword_filter_nodes(query, nodes, min_keyword_matches=1):
     return filtered
 
 def create_query_engine(vector_index):
-    """Create hybrid retrieval engine with deduplication"""
+    """Create hybrid retrieval engine with better deduplication"""
     log_message("Creating query engine...")
 
     vector_retriever = VectorIndexRetriever(
         index=vector_index,
-        similarity_top_k=80
+        similarity_top_k=50  # Reduced to get more diverse results
     )
     bm25_retriever = BM25Retriever.from_defaults(
         docstore=vector_index.docstore,
-        similarity_top_k=80,
+        similarity_top_k=50,
     )
     hybrid_retriever = QueryFusionRetriever(
         [vector_retriever, bm25_retriever],
-        similarity_top_k=100,
+        similarity_top_k=60,  # Reduced
         num_queries=1
     )
 
     class DeduplicatedQueryEngine(RetrieverQueryEngine):
         def retrieve(self, query):
             nodes = hybrid_retriever.retrieve(query)
+            log_message(f"Hybrid retrieval returned: {len(nodes)} nodes")
             
-            # CRITICAL: Deduplicate by text content hash
+            # Better deduplication using longer text snippet
             seen_hashes = set()
             unique_nodes = []
+            doc_type_counts = {'text': 0, 'table': 0, 'image': 0}
             
             for node in nodes:
-                text_hash = hash(node.text[:200])
+                # Use first 500 chars for dedup hash
+                text_hash = hash(node.text[:500])
                 
                 if text_hash not in seen_hashes:
                     seen_hashes.add(text_hash)
                     unique_nodes.append(node)
+                    
+                    # Count by type
+                    node_type = node.metadata.get('type', 'text')
+                    doc_type_counts[node_type] = doc_type_counts.get(node_type, 0) + 1
             
-            log_message(f"Retrieved: {len(nodes)} → Unique: {len(unique_nodes)}")
-            return unique_nodes[:50]  # Return top 50 unique
-        
-        # FIX: Override query method to use our retrieve
-        def query(self, query_bundle):
-            nodes = self.retrieve(query_bundle.query_str)
-            response = self._response_synthesizer.synthesize(
-                query=query_bundle,
-                nodes=nodes
-            )
-            return response
+            log_message(f"After dedup: {len(unique_nodes)} unique nodes")
+            log_message(f"Types: text={doc_type_counts.get('text', 0)}, "
+                       f"table={doc_type_counts.get('table', 0)}, "
+                       f"image={doc_type_counts.get('image', 0)}")
+            
+            return unique_nodes[:50]
 
     response_synthesizer = get_response_synthesizer()
     
     query_engine = DeduplicatedQueryEngine(
-        retriever=hybrid_retriever,  # Still pass it but we override retrieve()
+        retriever=hybrid_retriever,
         response_synthesizer=response_synthesizer
     )
 
-    log_message("✓ Query engine created (with deduplication)")
+    log_message("✓ Query engine created")
     return query_engine

utils.py

@@ -47,29 +47,55 @@ def answer_question(question, query_engine, reranker):
         retrieved = query_engine.retrieve(question)
         log_message(f"RETRIEVED: {len(retrieved)} unique nodes")
 
-        reranked = rerank_nodes(question, retrieved, reranker, top_k=25, min_score=0.1)
+        reranked = rerank_nodes(question, retrieved, reranker, top_k=15, min_score=-0.5)
         log_message(f"RERANKED: {len(reranked)} nodes")
 
-        context_parts = []
+        # Group by document and type
+        doc_groups = {}
         for n in reranked:
-            meta = n.metadata
-            doc_id = meta.get('document_id', 'unknown')
-            doc_type = meta.get('type', 'text')
-            if doc_type == 'table':
-                table_id = meta.get('table_identifier', meta.get('table_number', 'unknown'))
-                title = meta.get('table_title', '')
-                source_label = f"[{doc_id} - Таблица {table_id}]"
-                if title:
-                    source_label += f" {title}"
+            doc_id = n.metadata.get('document_id', 'unknown')
+            if doc_id not in doc_groups:
+                doc_groups[doc_id] = {'tables': [], 'text': [], 'images': []}
+            
+            node_type = n.metadata.get('type', 'text')
+            if node_type == 'table':
+                doc_groups[doc_id]['tables'].append(n)
+            elif node_type == 'image':
+                doc_groups[doc_id]['images'].append(n)
             else:
-                source_label = f"[{doc_id}]"
-            context_parts.append(f"{source_label}\n{n.text}")
+                doc_groups[doc_id]['text'].append(n)
+        
+        log_message(f"Documents found: {list(doc_groups.keys())}")
 
-        context = "\n\n" + ("="*50 + "\n\n").join(context_parts)
+        # Format context by document
+        context_parts = []
+        for doc_id, groups in doc_groups.items():
+            doc_section = [f"=== ДОКУМЕНТ: {doc_id} ==="]
+            
+            # Tables first (most important for your queries)
+            if groups['tables']:
+                doc_section.append("\n--- ТАБЛИЦЫ ---")
+                for n in groups['tables']:
+                    meta = n.metadata
+                    table_id = meta.get('table_identifier', meta.get('table_number', 'unknown'))
+                    title = meta.get('table_title', '')
+                    doc_section.append(f"\n[Таблица {table_id}] {title}")
+                    doc_section.append(n.text[:1500])  # Limit length
+            
+            # Then text
+            if groups['text']:
+                doc_section.append("\n--- ТЕКСТ ---")
+                for n in groups['text'][:3]:  # Limit text chunks
+                    doc_section.append(n.text[:800])
+            
+            context_parts.append("\n".join(doc_section))
+
+        context = "\n\n" + ("="*70 + "\n\n").join(context_parts)
+        
+        log_message(f"Context length: {len(context)} chars")
 
         from config import CUSTOM_PROMPT
         prompt = CUSTOM_PROMPT.format(context_str=context, query_str=question)
-        log_message(f"PROMPT LENGTH: {len(prompt)} chars")
 
         from llama_index.core import Settings
         response = Settings.llm.complete(prompt)
@@ -83,15 +109,30 @@ def answer_question(question, query_engine, reranker):
         log_message(traceback.format_exc())
         return f"Ошибка: {e}", ""
 
-def rerank_nodes(query, nodes, reranker, top_k=20, min_score=0.1):
-    """Simple and effective reranking: sort by score and filter by threshold."""
+def rerank_nodes(query, nodes, reranker, top_k=20, min_score=-0.5):  # Much lower threshold
+    """Rerank with detailed score logging"""
     if not nodes or not reranker:
+        log_message("WARNING: No nodes or reranker available")
         return nodes[:top_k]
 
-    pairs = [[query, n.text] for n in nodes]
+    pairs = [[query, n.text[:500]] for n in nodes]  # Limit text length for reranker
     scores = reranker.predict(pairs)
     scored = sorted(zip(nodes, scores), key=lambda x: x[1], reverse=True)
+    
+    # Detailed logging
+    if scored:
+        top_5_scores = [s for _, s in scored[:5]]
+        bottom_5_scores = [s for _, s in scored[-5:]]
+        log_message(f"Score range: {min(scores):.3f} to {max(scores):.3f}")
+        log_message(f"Top 5 scores: {top_5_scores}")
+        log_message(f"Bottom 5 scores: {bottom_5_scores}")
+    
+    # Count how many pass threshold
+    above_threshold = sum(1 for _, s in scored if s >= min_score)
+    log_message(f"Nodes above threshold ({min_score}): {above_threshold}/{len(scored)}")
+    
     filtered = [n for n, s in scored if s >= min_score]
-
-    # Return top_k filtered nodes, or fallback to top_k overall
-    return filtered[:top_k] if filtered else [n for n, _ in scored[:top_k]]
+    result = filtered[:top_k] if filtered else [n for n, _ in scored[:top_k]]
+    
+    log_message(f"Returning {len(result)} nodes after reranking")
+    return result