removed the part removing hyperh + top 80, cutoff = 0.55

documents_prep.py

@@ -36,11 +36,14 @@ def chunk_text_documents(documents):
 
 def normalize_connection_type(s):
     # Replace Cyrillic with Latin
-    s = s.replace('С', 'C').replace('с', 'c')
-    s = s.replace('У', 'U').replace('у', 'u')
-    s = s.replace('Т', 'T').replace('т', 't')
+    # s = s.replace('С', 'C').replace('с', 'c')
+    # s = s.replace('У', 'U').replace('у', 'u')
+    # s = s.replace('Т', 'T').replace('т', 't')
+    s= s.replace('С-', 'C-').replace('с-', 'c-')
+    s = s.replace('У-', 'U-').replace('у-', 'u-')
+    s = s.replace('Т-', 'T-').replace('т-', 't-')
     # REMOVE ALL HYPHENS for consistent tokenization
-    s = s.replace('-', '')
+    # s = s.replace('-', '')
     return s
 
 def extract_connection_type(text):
@@ -77,8 +80,6 @@ def chunk_table_by_content(table_data, doc_id, max_chars=MAX_CHARS_TABLE, max_ro
         return []
     
     log_message(f"  📊 Processing: {doc_id} - {table_identifier} ({len(rows)} rows)")
-    
-    # Calculate base metadata size - NOW INCLUDING DESCRIPTION
     base_content = format_table_header(doc_id, table_identifier, table_num, table_title, section, headers)
     
     # ADD DESCRIPTION HERE if it exists

index_retriever.py

@@ -71,18 +71,18 @@ def create_query_engine(vector_index):
         
         bm25_retriever = BM25Retriever.from_defaults(
             docstore=vector_index.docstore, 
-            similarity_top_k=100 
+            similarity_top_k=80 
         )
         
         vector_retriever = VectorIndexRetriever(
             index=vector_index, 
-            similarity_top_k=100,
-            similarity_cutoff=0.45
+            similarity_top_k=80,
+            similarity_cutoff=0.55
         )
         
         hybrid_retriever = QueryFusionRetriever(
             [vector_retriever, bm25_retriever],
-            similarity_top_k=100,
+            similarity_top_k=80,
             num_queries=1
         )
         

utils.py

@@ -179,7 +179,10 @@ def normalize_query(query):
     query = query.replace('С-', 'C-').replace('с-', 'c-')
     query = query.replace('У-', 'U-').replace('у-', 'u-')
     query = query.replace('Т-', 'T-').replace('т-', 't-')
-    query = query.replace('-', '')
+    # query = query.replace('С', 'C').replace('с', 'C')
+    # query = query.replace('У', 'U').replace('у', 'U')
+    # query = query.replace('Т', 'T').replace('т', 'T')
+    # query = query.replace('-', '')
     
     return query
 
@@ -191,7 +194,7 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
     try:
         start_time = time.time()
         
-        # NORMALIZE QUERY: Convert Cyrillic to Latin
+        # NORMALIZE QUERY: Convert Cyrillic to Latin and remove hyphens
         normalized_question = normalize_query(question)
         log_message(f"Original query: {question}")
         log_message(f"Normalized query: {normalized_question}")
@@ -218,12 +221,14 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
             for ct, cnt in sorted(conn_types_retrieved.items()):
                 log_message(f"  {ct}: {cnt} chunks")
         
-        # Check if target type was retrieved (keep original Cyrillic)
-        if 'С-25' in question:  # Use Cyrillic
-            if 'С-25' in conn_types_retrieved:
-                log_message(f"✓ С-25 RETRIEVED: {conn_types_retrieved['С-25']} chunks")
+        # Check if target type was retrieved
+        # Normalize the check as well
+        normalized_check = normalize_query('С-25')  # Will become C25
+        if normalized_check in question or 'С-25' in question or 'C-25' in question:
+            if 'C25' in conn_types_retrieved:
+                log_message(f"✓ C25 RETRIEVED: {conn_types_retrieved['C25']} chunks")
             else:
-                log_message("✗ С-25 NOT RETRIEVED despite being in query!")
+                log_message("✗ C25 NOT RETRIEVED despite being in query!")
         
         # Sample of retrieved tables
         log_message("SAMPLE OF RETRIEVED TABLES:")
@@ -235,11 +240,11 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
                 doc_id = node.metadata.get('document_id', 'N/A')
                 log_message(f"  [{i+1}] {doc_id} - Table {table_num} - Type: {conn_type}")
         
-        # Rerank
-        reranked_nodes = rerank_nodes(question, unique_retrieved, reranker, top_k=20)
+        # Rerank - use normalized query for consistency
+        reranked_nodes = rerank_nodes(normalized_question, unique_retrieved, reranker, top_k=20)
         
-        # Direct query without formatting
-        response = query_engine.query(question)
+        # CRITICAL FIX: Use normalized query for LLM as well
+        response = query_engine.query(normalized_question)
         
         end_time = time.time()
         processing_time = end_time - start_time