added the 100 topk

documents_prep.py

@@ -36,14 +36,14 @@ def chunk_text_documents(documents):
 
 def normalize_connection_type(s):
     # Replace Cyrillic with Latin
-    # s = s.replace('С', 'C').replace('с', 'c')
-    # s = s.replace('У', 'U').replace('у', 'u')
-    # s = s.replace('Т', 'T').replace('т', 't')
+    s = s.replace('С', 'C').replace('с', 'c')
+    s = s.replace('У', 'U').replace('у', 'u')
+    s = s.replace('Т', 'T').replace('т', 't')
     s= s.replace('С-', 'C-').replace('с-', 'c-')
     s = s.replace('У-', 'U-').replace('у-', 'u-')
     s = s.replace('Т-', 'T-').replace('т-', 't-')
     # REMOVE ALL HYPHENS for consistent tokenization
-    # s = s.replace('-', '')
+    s = s.replace('-', '')
     return s
 
 def extract_connection_type(text):
@@ -80,6 +80,8 @@ def chunk_table_by_content(table_data, doc_id, max_chars=MAX_CHARS_TABLE, max_ro
         return []
     
     log_message(f"  📊 Processing: {doc_id} - {table_identifier} ({len(rows)} rows)")
+    
+    # Calculate base metadata size - NOW INCLUDING DESCRIPTION
     base_content = format_table_header(doc_id, table_identifier, table_num, table_title, section, headers)
     
     # ADD DESCRIPTION HERE if it exists

index_retriever.py

@@ -71,18 +71,18 @@ def create_query_engine(vector_index):
         
         bm25_retriever = BM25Retriever.from_defaults(
             docstore=vector_index.docstore, 
-            similarity_top_k=80 
+            similarity_top_k=100 
         )
         
         vector_retriever = VectorIndexRetriever(
             index=vector_index, 
-            similarity_top_k=80,
+            similarity_top_k=100,
             similarity_cutoff=0.55
         )
         
         hybrid_retriever = QueryFusionRetriever(
             [vector_retriever, bm25_retriever],
-            similarity_top_k=80,
+            similarity_top_k=100,
             num_queries=1
         )
         

utils.py

@@ -179,10 +179,10 @@ def normalize_query(query):
     query = query.replace('С-', 'C-').replace('с-', 'c-')
     query = query.replace('У-', 'U-').replace('у-', 'u-')
     query = query.replace('Т-', 'T-').replace('т-', 't-')
-    # query = query.replace('С', 'C').replace('с', 'C')
-    # query = query.replace('У', 'U').replace('у', 'U')
-    # query = query.replace('Т', 'T').replace('т', 'T')
-    # query = query.replace('-', '')
+    query = query.replace('С', 'C').replace('с', 'C')
+    query = query.replace('У', 'U').replace('у', 'U')
+    query = query.replace('Т', 'T').replace('т', 'T')
+    query = query.replace('-', '')
     
     return query