update the seaction tools

prepare_env.py

@@ -128,9 +128,20 @@ def create_vectorstore_from_json(json_path: str, collection_name: str, embedding
     print(f"✅ Vector store created with collection: {collection_name}")
     return vectorstore, documents
 
-def create_retriever(vectorstore, docs, llm):
-    """Create ensemble retriever with vector and BM25 search"""
+def create_retriever(vectorstore, docs, llm, get_all: bool = False):
+    """Create ensemble retriever with vector and BM25 search
+    
+    Args:
+        vectorstore: The vector store for similarity search
+        docs: Documents for BM25 retriever
+        llm: Language model for multi-query generation
+        get_all: If True, configure retriever to return all documents
+    
+    Returns:
+        Configured retriever (MultiQueryRetriever or EnsembleRetriever)
+    """
     print("🔍 Creating ensemble retriever...")
+    
     # PromptTemplate for Vaccine Assistant MultiQuery Retriever
     VACCINE_MULTIQUERY_PROMPT = PromptTemplate(
         input_variables=["question"],
@@ -150,17 +161,28 @@ def create_retriever(vectorstore, docs, llm):
 
     Provide only the alternative questions, one per line."""
     )
+    
+    # Determine k values based on get_all parameter
+    if get_all:
+        vector_k = len(docs)  # Get all documents
+        bm25_k = len(docs)    # Get all documents
+        print(f"📄 GET_ALL mode: Setting k={len(docs)} (total documents)")
+    else:
+        vector_k = 6
+        bm25_k = 3
+        print(f"🎯 FILTERED mode: Vector k={vector_k}, BM25 k={bm25_k}")
+    
     # Vector retriever
     vector_retriever = vectorstore.as_retriever(
         search_type="similarity",
-        search_kwargs={"k": 6}
+        search_kwargs={"k": vector_k}
     )
-    print("✅ Vector retriever created (k=6)")
+    print(f"✅ Vector retriever created (k={vector_k})")
     
     # BM25 retriever
     bm25_retriever = BM25Retriever.from_documents(docs)
-    bm25_retriever.k = 3
-    print("✅ BM25 retriever created (k=2)")
+    bm25_retriever.k = bm25_k
+    print(f"✅ BM25 retriever created (k={bm25_k})")
     
     # Ensemble retriever
     ensemble_retriever = EnsembleRetriever(
@@ -169,7 +191,12 @@ def create_retriever(vectorstore, docs, llm):
     )
     print("✅ Ensemble retriever created (weights: 0.5, 0.5)")
     
-    # Multi-query expanding retriever
+    # If get_all is True, return ensemble retriever directly to avoid query processing overhead
+    if get_all:
+        print("📋 Returning ensemble retriever (bypassing MultiQuery for get_all mode)")
+        return ensemble_retriever
+    
+    # Multi-query expanding retriever (only for filtered mode)
     expanding_retriever = MultiQueryRetriever.from_llm(
         retriever=ensemble_retriever,
         llm=llm,
@@ -266,7 +293,7 @@ def create_section_tools(embedding_function, llm):
             if os.path.exists(path):
                 print(f"📁 Creating retriever for section {section} from {path}")
                 vstore, docs = create_vectorstore_from_json(path, f"Guide_2023_{section}", embedding_function)
-                section_retrievers[section] = create_retriever(vstore, docs, llm)
+                section_retrievers[section] = create_retriever(vstore, docs, llm, get_all=True)
                 print(f"✅ Successfully created retriever for section {section}")
             else:
                 print(f"⚠️ Warning: File not found for section {section}: {path}")