update the retrever and the system prompt

prepare_env.py

@@ -17,7 +17,11 @@ from langchain.retrievers.multi_query import MultiQueryRetriever
 from langchain_google_genai import ChatGoogleGenerativeAI
 from llama_index.core.tools import FunctionTool
 from llama_index.core.schema import TextNode
+from langchain.prompts import PromptTemplate
+import logging
 
+logging.basicConfig()
+logging.getLogger("langchain.retrievers.multi_query").setLevel(logging.INFO)
 
 def extract_source_ids(response_text):
     """
@@ -127,7 +131,25 @@ def create_vectorstore_from_json(json_path: str, collection_name: str, embedding
 def create_retriever(vectorstore, docs, llm):
     """Create ensemble retriever with vector and BM25 search"""
     print("🔍 Creating ensemble retriever...")
-    
+    # PromptTemplate for Vaccine Assistant MultiQuery Retriever
+    VACCINE_MULTIQUERY_PROMPT = PromptTemplate(
+        input_variables=["question"],
+        template="""You are an AI assistant specialized in vaccine-related medical information retrieval.
+    Your task is to generate multiple search queries based on the original question to find relevant information from official vaccine medical documents.
+
+    IMPORTANT GUIDELINES:
+    - Keep all vaccine-specific terminology and medical terms intact
+    - Maintain the clinical and medical context
+    - Focus on evidence-based vaccine information
+    - Preserve any specific vaccine names, diseases, or medical conditions mentioned
+    - Generate queries that would help retrieve information about vaccine schedules, dosing, contraindications, adverse events, and disease prevention
+
+    Original question: {question}
+
+    Generate 4 different search queries that rephrase the original question while maintaining vaccine terminology and medical accuracy. Each query should approach the topic from a slightly different angle to maximize retrieval from vaccine medical documents.
+
+    Provide only the alternative questions, one per line."""
+    )
     # Vector retriever
     vector_retriever = vectorstore.as_retriever(
         search_type="similarity",
@@ -150,7 +172,8 @@ def create_retriever(vectorstore, docs, llm):
     # Multi-query expanding retriever
     expanding_retriever = MultiQueryRetriever.from_llm(
         retriever=ensemble_retriever,
-        llm=llm
+        llm=llm,
+        prompt=VACCINE_MULTIQUERY_PROMPT,
     )
     print("✅ Multi-query expanding retriever created")
     

rag_pipeline.py

@@ -143,26 +143,6 @@ Answer the doctor's question accurately and concisely using only the provided in
    - Include the citation in the table caption, e.g., 'Table: Vaccination Schedule [Source ID]'.
 2. For lists, maintain the original bullet points/numbering and include citations.
 3. Present information concisely but ensure clinical accuracy is never compromised.
-
-### CRITICAL: Efficient Fallback Strategy
-1. **MANDATORY SEARCH**: Use each relevant tool at least once to search for information, even if you suspect the information might not be available.
-2. **BREAK DOWN COMPLEX QUERIES**: For comparative or multi-part questions (e.g., comparing Algerian and WHO guidelines), break the query into sub-queries and use the appropriate tool for each part:
-   - Use Guide_vector_tool for Algerian-specific information (e.g., national schedules, coverage targets).
-   - Use Immunization_in_Practice_tool for WHO-specific information (e.g., global recommendations, coverage targets).
-3. **DO NOT STOP PREMATURELY**: Do not conclude "no information is available" without using the relevant tool(s) to search for the answer.
-4. **BE DECISIVE**: Once you find relevant information for each sub-query, formulate your response immediately.
-5. **ANSWER FULLY**: Address all parts of the question, using multiple tools if required by the query.
-
-### Response Guidelines
-- **MANDATORY TOOL SELECTION**:
-  - For queries mentioning "WHO," "World Health Organization," "international," "global guidance," or WHO documents (e.g., page numbers), use Immunization_in_Practice_tool first.
-  - For queries mentioning "Algerian," "national guide," or Algerian-specific terms (e.g., page numbers), use Guide_vector_tool first.
-  - For comparative queries (e.g., Algerian vs. WHO), use both Guide_vector_tool and Immunization_in_Practice_tool, addressing each part systematically.
-- **EXPLICIT REASONING**: Before answering, log your reasoning steps, including which tools you will use and why, based on the query’s content.
-- **Query Decomposition**: Break comparative or multi-part queries into sub-queries (e.g., one for Algerian information, one for WHO information) and use the appropriate tool for each.
-- Provide all found information with proper citations using Source IDs only.
-- If information is limited, clearly state: "Based on the available documents, I can provide the following information..." and indicate what is not available.
-
 ---
 """
     else:
@@ -190,32 +170,6 @@ Answer the doctor's question accurately and concisely using only the provided in
 2. For lists, maintain the original bullet points/numbering and include citations.
 3. Present information concisely but ensure clinical accuracy is never compromised.
 
-### CRITICAL: Efficient Response Strategy
-1. **MANDATORY SEARCH**: Always use the relevant tool(s) to search for information before answering, even if you initially think no information is available.
-2. **MANDATORY TOOL SELECTION**:
-   - For queries mentioning "WHO," "World Health Organization," "international," "global guidance," or WHO documents (e.g., page numbers), use Immunization_in_Practice_tool first.
-   - For queries mentioning "Algerian," "national guide," or Algerian-specific terms (e.g., page numbers), use Guide_vector_tool first.
-   - For comparative queries (e.g., Algerian vs. WHO), use both Guide_vector_tool and Immunization_in_Practice_tool, addressing each part systematically.
-3. **Query Decomposition**: Break comparative or multi-part queries into sub-queries (e.g., one for Algerian information, one for WHO information) and use the appropriate tool for each.
-4. **DO NOT STOP PREMATURELY**: Do not conclude "no information is available" without using the relevant tool(s) to search for the answer.
-5. **EXPLICIT REASONING**: Before answering, log your reasoning steps, including which tools you will use and why, based on the query’s content.
-6. **BE DECISIVE**: Once you find relevant information for each sub-query, formulate your response immediately.
-7. **ANSWER FULLY**: Address all parts of the question, using multiple tools if required by the query.
-8. **STOP WHEN SUFFICIENT**: If you have found adequate information to answer all parts of the question, provide the response and stop.
-
-### Response Guidelines for Complex Questions
-- For comparative questions: Break the query into sub-queries (e.g., Algerian vs. WHO), use Guide_vector_tool for Algerian specifics and Immunization_in_Practice_tool for WHO specifics, then provide the comparison.
-- For multi-part questions: Address each part systematically, using the appropriate tool for each sub-query.
-- If information is not found after using the relevant tool(s): State clearly: "Based on the available documents, I can provide the following information..." and specify what is not available.
-- Do not repeatedly search for the same terms or rephrase searches excessively.
-
-### When Information is Limited
-If you cannot find complete information to fully answer a question:
-1. Provide whatever relevant information you did find with proper citations using Source IDs only.
-2. Clearly state: "Based on the available documents, I can provide the following information..."
-3. Indicate what specific information is not available: "However, information about [specific topic] was not found in the provided documents after searching with the relevant tool(s)."
-4. Do not conclude "no information is available" without attempting a search with the appropriate tool(s).
-
 ---
 """