add log to system

prepare_env.py

@@ -70,10 +70,13 @@ def extract_source_ids(response_text):
 
 def setup_models():
     """Initialize embedding model and LLM"""
+    print("🔧 Setting up embedding model and LLM...")
+    
     # Initialize embedding model
     embedding_function = HuggingFaceEmbeddings(
         model_name="intfloat/multilingual-e5-base"
     )
+    print("✅ Embedding model initialized: intfloat/multilingual-e5-base")
     
     # Initialize LLM
     genai_api_key = os.getenv('GOOGLE_API_KEY')
@@ -81,15 +84,20 @@ def setup_models():
         model="gemini-2.0-flash",
         google_api_key=genai_api_key
     )
+    print("✅ LLM initialized: gemini-2.0-flash")
     
     return embedding_function, llm
 
 def create_vectorstore_from_json(json_path: str, collection_name: str, embedding_function):
     """Create vector store from JSON chunks"""
+    print(f"📚 Creating vector store from: {json_path}")
+    
     # Load the chunks.json
     with open(json_path, "r", encoding="utf-8") as f:
         chunks_data = json.load(f)
     
+    print(f"📊 Loaded {len(chunks_data)} chunks from JSON")
+    
     documents = []
     for element in chunks_data:
         text = element["text"]
@@ -113,31 +121,38 @@ def create_vectorstore_from_json(json_path: str, collection_name: str, embedding
         collection_name=collection_name,
         persist_directory="chroma_db_multilingual"
     )
+    print(f"✅ Vector store created with collection: {collection_name}")
     return vectorstore, documents
 
 def create_retriever(vectorstore, docs, llm):
     """Create ensemble retriever with vector and BM25 search"""
+    print("🔍 Creating ensemble retriever...")
+    
     # Vector retriever
     vector_retriever = vectorstore.as_retriever(
         search_type="similarity",
         search_kwargs={"k": 6}
     )
+    print("✅ Vector retriever created (k=6)")
     
     # BM25 retriever
     bm25_retriever = BM25Retriever.from_documents(docs)
     bm25_retriever.k = 2
+    print("✅ BM25 retriever created (k=2)")
     
     # Ensemble retriever
     ensemble_retriever = EnsembleRetriever(
         retrievers=[vector_retriever, bm25_retriever],
         weights=[0.5, 0.5]
     )
+    print("✅ Ensemble retriever created (weights: 0.5, 0.5)")
     
     # Multi-query expanding retriever
     expanding_retriever = MultiQueryRetriever.from_llm(
         retriever=ensemble_retriever,
         llm=llm
     )
+    print("✅ Multi-query expanding retriever created")
     
     return expanding_retriever
 
@@ -164,18 +179,27 @@ def convert_chromadb_to_llamaindex_nodes(chromadb_documents: List) -> List[TextN
 
 def section_tool_wrapper(retriever, section_path_chunks, query):
     """Generic section tool wrapper"""
+    print(f"🔍 TOOL CALL: Searching for query: '{query[:100]}...' in {section_path_chunks}")
+    
     try:
         retrieved_docs = retriever.get_relevant_documents(query)
+        print(f"📄 Retrieved {len(retrieved_docs)} documents")
+        
         nodes_from_retrieved_docs = convert_chromadb_to_llamaindex_nodes(retrieved_docs)
 
         if not nodes_from_retrieved_docs:
+            print("❌ No relevant documents found for the query")
             return "No relevant documents found for the query."
 
         chunk_ids = [node.metadata['element_id'] for node in retrieved_docs]
+        print(f"🆔 Found chunk IDs: {chunk_ids}")
+        
         with open(section_path_chunks, "r", encoding="utf-8") as f:
             chunks_data = json.load(f)
 
         chunks_unique = [node for node in chunks_data if node.get('element_id', 'Unknown') in chunk_ids]
+        print(f"✅ Matched {len(chunks_unique)} unique chunks")
+        
         combined_text = []
         
         for chu in chunks_unique:
@@ -188,14 +212,15 @@ def section_tool_wrapper(retriever, section_path_chunks, query):
                     combined_text.append(text)
 
         result = "\n---\n".join(combined_text)
-        print(f"Retrieved {len(nodes_from_retrieved_docs)} documents for query: {query[:50]}...")
+        print(f"✅ TOOL RESPONSE: Generated response with {len(combined_text)} text sections")
         return result
     except Exception as e:
-        print(f"Error in section tool: {e}")
+        print(f"❌ TOOL ERROR: {e}")
         return f"Error retrieving documents: {str(e)}"
 
 def create_section_tools(embedding_function, llm):
     """Create all section-specific retrieval tools"""
+    print("🛠️ Creating section-specific retrieval tools...")
     
     # Define section paths - Fixed path structure
     section_paths = {
@@ -216,15 +241,15 @@ def create_section_tools(embedding_function, llm):
     for section, path in section_paths.items():
         try:
             if os.path.exists(path):
-                print(f"Creating retriever for section {section} from {path}")
+                print(f"📁 Creating retriever for section {section} from {path}")
                 vstore, docs = create_vectorstore_from_json(path, f"Guide_2023_{section}", embedding_function)
                 section_retrievers[section] = create_retriever(vstore, docs, llm)
-                print(f"Successfully created retriever for section {section}")
+                print(f"✅ Successfully created retriever for section {section}")
             else:
-                print(f"Warning: File not found for section {section}: {path}")
+                print(f"⚠️ Warning: File not found for section {section}: {path}")
                 section_retrievers[section] = None
         except Exception as e:
-            print(f"Error creating retriever for section {section}: {e}")
+            print(f"❌ Error creating retriever for section {section}: {e}")
             section_retrievers[section] = None
     
     # Create main guide retriever
@@ -232,32 +257,32 @@ def create_section_tools(embedding_function, llm):
     guide_retriever = None
     try:
         if os.path.exists(guide_path):
-            print("Creating main guide retriever...")
+            print("📚 Creating main guide retriever...")
             guide_vstore, guide_docs = create_vectorstore_from_json(guide_path, "Guide_2023_multilingual", embedding_function)
             guide_retriever = create_retriever(guide_vstore, guide_docs, llm)
-            print("Successfully created main guide retriever")
+            print("✅ Successfully created main guide retriever")
         else:
-            print(f"Warning: Main guide file not found: {guide_path}")
+            print(f"⚠️ Warning: Main guide file not found: {guide_path}")
     except Exception as e:
-        print(f"Error creating main guide retriever: {e}")
+        print(f"❌ Error creating main guide retriever: {e}")
 
     # WHO Immunization in Practice Tool
     immunization_path = './data/Immunization in Practice_WHO_eng_2015.json'
     immunization_retriever = None
     try:
         if os.path.exists(immunization_path):
-            print("Creating immunization retriever...")
+            print("🌍 Creating immunization retriever...")
             immunization_vstore, immunization_docs = create_vectorstore_from_json(
                 immunization_path,
                 "Immunization_in_Practice_WHO_eng_2015",
                 embedding_function
             )
             immunization_retriever = create_retriever(immunization_vstore, immunization_docs, llm)
-            print("Successfully created immunization retriever")
+            print("✅ Successfully created immunization retriever")
         else:
-            print(f"Warning: Immunization file not found: {immunization_path}")
+            print(f"⚠️ Warning: Immunization file not found: {immunization_path}")
     except Exception as e:
-        print(f"Error creating immunization retriever: {e}")
+        print(f"❌ Error creating immunization retriever: {e}")
 
     # General-purpose tool (entire Algerian guide)
     def guide_retrieval_tool(query: str) -> str:
@@ -279,11 +304,14 @@ def create_section_tools(embedding_function, llm):
         Returns:
             str: Synthesized answer from the entire national guide.
         """
+        print(f"🏥 GUIDE TOOL CALLED: {query[:50]}...")
         if not guide_retriever:
+            print("❌ Guide retriever not available - main guide file may be missing")
             return "Guide retriever not available - main guide file may be missing"
         try:
             return section_tool_wrapper(guide_retriever, guide_path, query)
         except Exception as e:
+            print(f"❌ Error accessing guide retriever: {str(e)}")
             return f"Error accessing guide retriever: {str(e)}"
 
     def immunization_tool(query: str) -> str:
@@ -302,11 +330,14 @@ def create_section_tools(embedding_function, llm):
         Returns:
             str: Content from the WHO Immunization in Practice guide.
         """
+        print(f"🌍 WHO TOOL CALLED: {query[:50]}...")
         if not immunization_retriever:
+            print("❌ Immunization in Practice retriever not available - WHO guide file may be missing")
             return "Immunization in Practice retriever not available - WHO guide file may be missing"
         try:
             return section_tool_wrapper(immunization_retriever, immunization_path, query)
         except Exception as e:
+            print(f"❌ Error accessing immunization retriever: {str(e)}")
             return f"Error accessing immunization retriever: {str(e)}"
 
     # Section-Specific Tools - Fixed implementation
@@ -325,11 +356,14 @@ def create_section_tools(embedding_function, llm):
         Returns:
             str: Response from Section 1.
         """
+        print(f"📋 SECTION 1 TOOL CALLED: {query[:50]}...")
         if not section_retrievers.get('one'):
+            print("❌ Section 1 retriever not available - file may be missing")
             return "Section 1 retriever not available - file may be missing"
         try:
             return section_tool_wrapper(section_retrievers['one'], section_paths['one'], query)
         except Exception as e:
+            print(f"❌ Error accessing section 1: {str(e)}")
             return f"Error accessing section 1: {str(e)}"
 
     def section_two_tool(query: str) -> str:
@@ -347,11 +381,14 @@ def create_section_tools(embedding_function, llm):
         Returns:
             str: Disease-specific content from Section 2.
         """
+        print(f"🦠 SECTION 2 TOOL CALLED: {query[:50]}...")
         if not section_retrievers.get('two'):
+            print("❌ Section 2 retriever not available - file may be missing")
             return "Section 2 retriever not available - file may be missing"
         try:
             return section_tool_wrapper(section_retrievers['two'], section_paths['two'], query)
         except Exception as e:
+            print(f"❌ Error accessing section 2: {str(e)}")
             return f"Error accessing section 2: {str(e)}"
 
     def section_three_tool(query: str) -> str:
@@ -369,11 +406,14 @@ def create_section_tools(embedding_function, llm):
         Returns:
             str: Vaccine info from Section 3.
         """
+        print(f"💉 SECTION 3 TOOL CALLED: {query[:50]}...")
         if not section_retrievers.get('three'):
+            print("❌ Section 3 retriever not available - file may be missing")
             return "Section 3 retriever not available - file may be missing"
         try:
             return section_tool_wrapper(section_retrievers['three'], section_paths['three'], query)
         except Exception as e:
+            print(f"❌ Error accessing section 3: {str(e)}")
             return f"Error accessing section 3: {str(e)}"
 
     def section_four_tool(query: str) -> str:
@@ -391,11 +431,14 @@ def create_section_tools(embedding_function, llm):
         Returns:
             str: Catch-up guidance from Section 4.
         """
+        print(f"🔄 SECTION 4 TOOL CALLED: {query[:50]}...")
         if not section_retrievers.get('four'):
+            print("❌ Section 4 retriever not available - file may be missing")
             return "Section 4 retriever not available - file may be missing"
         try:
             return section_tool_wrapper(section_retrievers['four'], section_paths['four'], query)
         except Exception as e:
+            print(f"❌ Error accessing section 4: {str(e)}")
             return f"Error accessing section 4: {str(e)}"
 
     def section_five_tool(query: str) -> str:
@@ -413,11 +456,14 @@ def create_section_tools(embedding_function, llm):
         Returns:
             str: Custom recommendations from Section 5.
         """
+        print(f"👥 SECTION 5 TOOL CALLED: {query[:50]}...")
         if not section_retrievers.get('five'):
+            print("❌ Section 5 retriever not available - file may be missing")
             return "Section 5 retriever not available - file may be missing"
         try:
             return section_tool_wrapper(section_retrievers['five'], section_paths['five'], query)
         except Exception as e:
+            print(f"❌ Error accessing section 5: {str(e)}")
             return f"Error accessing section 5: {str(e)}"
 
     def section_six_tool(query: str) -> str:
@@ -435,11 +481,14 @@ def create_section_tools(embedding_function, llm):
         Returns:
             str: Cold chain instructions from Section 6.
         """
+        print(f"❄️ SECTION 6 TOOL CALLED: {query[:50]}...")
         if not section_retrievers.get('six'):
+            print("❌ Section 6 retriever not available - file may be missing")
             return "Section 6 retriever not available - file may be missing"
         try:
             return section_tool_wrapper(section_retrievers['six'], section_paths['six'], query)
         except Exception as e:
+            print(f"❌ Error accessing section 6: {str(e)}")
             return f"Error accessing section 6: {str(e)}"
 
     def section_seven_tool(query: str) -> str:
@@ -457,11 +506,14 @@ def create_section_tools(embedding_function, llm):
         Returns:
             str: Best practices from Section 7.
         """
+        print(f"🛡️ SECTION 7 TOOL CALLED: {query[:50]}...")
         if not section_retrievers.get('seven'):
+            print("❌ Section 7 retriever not available - file may be missing")
             return "Section 7 retriever not available - file may be missing"
         try:
             return section_tool_wrapper(section_retrievers['seven'], section_paths['seven'], query)
         except Exception as e:
+            print(f"❌ Error accessing section 7: {str(e)}")
             return f"Error accessing section 7: {str(e)}"
 
     def section_eight_tool(query: str) -> str:
@@ -479,11 +531,14 @@ def create_section_tools(embedding_function, llm):
         Returns:
             str: Workflow and safety monitoring details from Section 8.
         """
+        print(f"📊 SECTION 8 TOOL CALLED: {query[:50]}...")
         if not section_retrievers.get('eight'):
+            print("❌ Section 8 retriever not available - file may be missing")
             return "Section 8 retriever not available - file may be missing"
         try:
             return section_tool_wrapper(section_retrievers['eight'], section_paths['eight'], query)
         except Exception as e:
+            print(f"❌ Error accessing section 8: {str(e)}")
             return f"Error accessing section 8: {str(e)}"
 
     def section_nine_tool(query: str) -> str:
@@ -501,11 +556,14 @@ def create_section_tools(embedding_function, llm):
         Returns:
             str: Planning and stock guidance from Section 9.
         """
+        print(f"📅 SECTION 9 TOOL CALLED: {query[:50]}...")
         if not section_retrievers.get('nine'):
+            print("❌ Section 9 retriever not available - file may be missing")
             return "Section 9 retriever not available - file may be missing"
         try:
             return section_tool_wrapper(section_retrievers['nine'], section_paths['nine'], query)
         except Exception as e:
+            print(f"❌ Error accessing section 9: {str(e)}")
             return f"Error accessing section 9: {str(e)}"
 
     def section_ten_tool(query: str) -> str:
@@ -523,11 +581,14 @@ def create_section_tools(embedding_function, llm):
         Returns:
             str: Public mobilization strategies from Section 10.
         """
+        print(f"📢 SECTION 10 TOOL CALLED: {query[:50]}...")
         if not section_retrievers.get('ten'):
+            print("❌ Section 10 retriever not available - file may be missing")
             return "Section 10 retriever not available - file may be missing"
         try:
             return section_tool_wrapper(section_retrievers['ten'], section_paths['ten'], query)
         except Exception as e:
+            print(f"❌ Error accessing section 10: {str(e)}")
             return f"Error accessing section 10: {str(e)}"
 
     # Create FunctionTool objects
@@ -547,17 +608,19 @@ def create_section_tools(embedding_function, llm):
         FunctionTool.from_defaults(name="section_ten_vector_query_tool", fn=section_ten_tool),
     ]
     
+    print(f"✅ Created {len(tools)} section tools")
     return tools
 
 def prepare_environment():
     """Main function to prepare the environment and return tools"""
-    print("Setting up models...")
+    print("🚀 Starting environment preparation...")
+    print("🔧 Setting up models...")
     embedding_function, llm = setup_models()
     
-    print("Creating section tools...")
+    print("🛠️ Creating section tools...")
     tools = create_section_tools(embedding_function, llm)
     
-    print("Environment prepared successfully!")
-    print(f"Created {len(tools)} tools")
+    print("✅ Environment prepared successfully!")
+    print(f"📋 Created {len(tools)} tools")
     
     return tools, llm

rag_pipeline.py

@@ -11,6 +11,7 @@ from llama_index.core.agent import ReActAgent
 from llama_index.llms.google_genai import GoogleGenAI
 from langdetect import detect
 import os
+import time
 
 
 def extract_source_ids(response_text):
@@ -28,6 +29,8 @@ def extract_source_ids(response_text):
     """
     import re
 
+    print(f"[LOG] Extracting source IDs from response text (length: {len(response_text)} chars)")
+    
     # First, extract all source IDs from inline citations with adjacent brackets [ID1][ID2]
     # Replace them with single brackets with comma separation to standardize format
     consolidated_text = re.sub(r'\][\s]*\[', '][', response_text)
@@ -35,6 +38,7 @@ def extract_source_ids(response_text):
     
     # Now extract all source IDs from any format (single ID or comma-separated IDs)
     inline_citations = re.findall(r'\[([^\[\]]+)\]', consolidated_text)
+    print(f"[LOG] Found {len(inline_citations)} inline citations")
     
     if not inline_citations:
         print("Warning: No source IDs found in the response text.")
@@ -55,6 +59,8 @@ def extract_source_ids(response_text):
             seen.add(id_str)
             source_ids.append(id_str)
     
+    print(f"[LOG] Extracted {len(source_ids)} unique source IDs: {source_ids[:3]}{'...' if len(source_ids) > 3 else ''}")
+    
     if not source_ids:
         print("Warning: No valid source IDs found after filtering.")
         return []
@@ -73,6 +79,8 @@ def convert_citations_to_sequential(response_text, source_id_to_number_map):
     Returns:
         str: Response text with sequential number citations
     """
+    print(f"[LOG] Converting {len(source_id_to_number_map)} source IDs to sequential numbers")
+    
     def replace_citation(match):
         citation_content = match.group(1)
         # Handle multiple IDs in one citation (comma-separated)
@@ -94,6 +102,7 @@ def convert_citations_to_sequential(response_text, source_id_to_number_map):
     
     # Replace all citations in the text
     sequential_response = re.sub(r'\[([^\[\]]+)\]', replace_citation, response_text)
+    print("[LOG] Successfully converted citations to sequential format")
     return sequential_response
 
 
@@ -101,6 +110,8 @@ def convert_citations_to_sequential(response_text, source_id_to_number_map):
 def create_safe_custom_prompt(tools, llm):
     """Create a safe version that won't have formatting conflicts"""
     
+    print(f"[LOG] Creating custom prompt with {len(tools)} tools")
+    
     custom_instructions = """
 ## MEDICAL ASSISTANT ROLE
 You are a helpful and knowledgeable AI-powered vaccine assistant designed to support doctors in clinical decision-making.
@@ -165,19 +176,24 @@ If you cannot find complete information to fully answer a question:
             template_vars=original_prompt.template_vars,
             metadata=original_prompt.metadata if hasattr(original_prompt, 'metadata') else None
         )
+        print("[LOG] ✅ Successfully created safe custom prompt")
         return new_prompt
     except:
         # Even safer fallback
+        print("[LOG] ⚠️ Using fallback prompt template")
         return PromptTemplate(template=safe_template)
 
 def create_agent(tools, llm):
     """Create the ReAct agent with custom prompt"""
     
+    print(f"[LOG] Creating ReAct agent with {len(tools)} tools and max_iterations=8")
+    
     # Create agent with increased max iterations and better handling
+    # Force verbose=True to see the Thought/Action/Observation cycle
     agent = ReActAgent.from_tools(
         tools,
         llm=llm,
-        verbose=True,
+        verbose=True,  # This should show the ReAct reasoning steps
         max_iterations=8,  # Reduced from default to prevent excessive looping
     )
 
@@ -190,12 +206,17 @@ def create_agent(tools, llm):
         print(f"❌ Safe prompt update failed: {e}")
         print("⚠️  Using original agent without modifications")
 
+    print("[LOG] Agent creation completed")
     return agent
 
 def initialize_rag_pipeline(tools):
     """Initialize the RAG pipeline with tools"""
     
+    print("[LOG] Initializing RAG pipeline...")
+    print(f"[LOG] Available tools: {[tool.metadata.name if hasattr(tool, 'metadata') else str(tool) for tool in tools]}")
+    
     # Initialize LlamaIndex LLM
+    print("[LOG] Initializing Google GenAI LLM (gemini-2.0-flash)")
     llama_index_llm = GoogleGenAI(
         model="models/gemini-2.0-flash",
         api_key=os.getenv('GOOGLE_API_KEY'),
@@ -204,15 +225,33 @@ def initialize_rag_pipeline(tools):
     # Create agent
     agent = create_agent(tools, llama_index_llm)
     
+    print("[LOG] ✅ RAG pipeline initialization completed")
     return agent
 
 def process_question(agent, question: str) -> str:
     """Process a question through the RAG pipeline"""
+    print(f"[LOG] Processing question: '{question[:100]}{'...' if len(question) > 100 else ''}'")
+    print("="*50)
+    print("AGENT REASONING PROCESS:")
+    print("="*50)
+    start_time = time.time()
+    
     try:
+        # The agent.chat() call should now show the full ReAct process
         response = agent.chat(question)
+        
+        print("="*50)
+        print("END OF AGENT REASONING")
+        print("="*50)
+        
+        elapsed_time = time.time() - start_time
+        print(f"[LOG] ✅ Agent response received in {elapsed_time:.2f} seconds")
+        print(f"[LOG] Response length: {len(response.response)} characters")
+        
         return response.response
     except Exception as e:
-        print(f"Error processing question: {e}")
+        elapsed_time = time.time() - start_time
+        print(f"[LOG] ❌ Error processing question after {elapsed_time:.2f} seconds: {e}")
         return f"Error processing your question: {str(e)}"
 
 def aswer_language_detection(response_text: str) -> str:
@@ -225,15 +264,19 @@ def aswer_language_detection(response_text: str) -> str:
     Returns:
         str: Detected language code (e.g., 'en', 'fr', etc.)
     """
+    print("[LOG] Detecting response language...")
 
     try:
             # Detect the language of the first 5 words of the response
             first_line = " ".join(response_text.split()[:5])
             first_line = re.sub(r'\[.*?\]', '', first_line)  # Remove citations
             answer_language = detect(first_line)
+            print(f"[LOG] Detected language: {answer_language}")
             if answer_language not in ['en', 'ar', 'fr']:
+                print(f"[LOG] Language {answer_language} not in supported list, defaulting to 'en'")
                 answer_language ='en'
     except:
+            print("[LOG] Language detection failed, defaulting to 'en'")
             answer_language ='en'
     
     finally:
@@ -257,17 +300,35 @@ def process_question_with_sequential_citations(agent, question: str, chunks_dire
             "citation_mapping": dict  # Mapping from source ID to citation number
         }
     """
+    print(f"\n[LOG] === STARTING QUESTION PROCESSING ===")
+    print(f"[LOG] Question: '{question[:150]}{'...' if len(question) > 150 else ''}'")
+    print(f"[LOG] Chunks directory: {chunks_directory}")
+    start_time = time.time()
+    
     try:
         # Get the response from the agent
+        print("\n" + "="*60)
+        print("🤖 AGENT REASONING PROCESS STARTING...")
+        print("="*60)
+        
         response = agent.chat(question)
+        
+        print("="*60)
+        print("🤖 AGENT REASONING PROCESS COMPLETED")
+        print("="*60)
         response_text = response.response
         
+        agent_time = time.time() - start_time
+        print(f"[LOG] Agent processing completed in {agent_time:.2f} seconds")
+        print(f"[LOG] Raw response length: {len(response_text)} characters")
+        
         # Enhanced handling for max iterations error
         if ("max iterations" in response_text.lower() or 
             "reached max iterations" in response_text.lower() or 
             len(response_text.strip()) == 0 or
             "agent stopped due to max iterations" in response_text.lower()):
             
+            print("[LOG] ⚠️ Detected max iterations error, providing fallback response")
             # Provide a more helpful fallback response
             response_text = ("I apologize, but I encountered difficulties processing your question within the available search iterations. "
                            "This may be due to the complexity of your query or limitations in finding specific information in the available documents. "
@@ -278,55 +339,77 @@ def process_question_with_sequential_citations(agent, question: str, chunks_dire
         
         # Create mapping from source ID to sequential number
         source_id_to_number = {source_id: i + 1 for i, source_id in enumerate(unique_ids)}
+        print(f"[LOG] Created citation mapping for {len(source_id_to_number)} sources")
         
         # Convert citations to sequential numbers
         sequential_response = convert_citations_to_sequential(response_text, source_id_to_number)
         
         # Load all chunks data to find cited elements
+        print("[LOG] Loading chunks data for citation lookup...")
         all_chunks_data = []
         min_chunks_files = ["Guide-pratique-de-mise-en-oeuvre-du-calendrier-national-de-vaccination-2023.json", 
                            "Immunization in Practice_WHO_eng_2015.json"]
         
         for json_file in min_chunks_files:
             json_path = os.path.join(chunks_directory, json_file)
+            print(f"[LOG] Loading {json_file}...")
             try:
                 with open(json_path, "r", encoding="utf-8") as f:
                     chunks_data = json.load(f)
                     all_chunks_data.extend(chunks_data)
+                    print(f"[LOG] ✅ Loaded {len(chunks_data)} chunks from {json_file}")
             except Exception as e:
-                print(f"Warning: Could not load {json_file}: {e}")
+                print(f"[LOG] ❌ Warning: Could not load {json_file}: {e}")
+        
+        print(f"[LOG] Total chunks loaded: {len(all_chunks_data)}")
         
         # Get cited elements in the same order as the sequential citations
+        print("[LOG] Finding cited elements...")
         cited_elements_ordered = []
-        for source_id in unique_ids:  # This preserves the order
+        for i, source_id in enumerate(unique_ids):  # This preserves the order
+            print(f"[LOG] Looking for source ID {i+1}/{len(unique_ids)}: {source_id}")
+            found = False
             for element in all_chunks_data:
                 if element.get("type") == 'TableElement':
                     if element.get("elements",{}).get("element_id") == source_id:
                         cited_elements_ordered.append(element.get("elements",{}))
+                        found = True
                         break
                 else:
                     if "elements" in element:
                         for nested_element in element["elements"]:
                             if nested_element.get("element_id") == source_id:
                                 cited_elements_ordered.append(nested_element)
+                                found = True
                                 break
                         else:
                             continue
                         break
+            if not found:
+                print(f"[LOG] ⚠️ Source ID {source_id} not found in chunks data")
+        
+        print(f"[LOG] Found {len(cited_elements_ordered)} cited elements")
         
         # Convert to JSON
         cited_elements_json = json.dumps(cited_elements_ordered, ensure_ascii=False, indent=2)
-        aswer_language= aswer_language_detection(response_text)
+        aswer_language = aswer_language_detection(response_text)
+        
+        total_time = time.time() - start_time
+        print(f"[LOG] ✅ Processing completed in {total_time:.2f} seconds total")
+        print(f"[LOG] Final response length: {len(sequential_response)} characters")
+        print(f"[LOG] === QUESTION PROCESSING COMPLETED ===\n")
+        
         return {
             "response": sequential_response,
             "cited_elements_json": cited_elements_json,
             "unique_ids": unique_ids,
             "citation_mapping": source_id_to_number,
-            "answer_language":aswer_language
+            "answer_language": aswer_language
         }
         
     except Exception as e:
-        print(f"Error processing question: {e}")
+        elapsed_time = time.time() - start_time
+        print(f"[LOG] ❌ Error processing question after {elapsed_time:.2f} seconds: {e}")
         error_response = "I apologize, but I encountered an error while processing your question. Please try rephrasing your question or asking about a more specific topic."
         
         return {
@@ -342,4 +425,5 @@ def process_question_with_citations(agent, question: str, chunks_directory="./da
     Legacy function - maintained for backward compatibility.
     Now calls the new sequential citation function.
     """
+    print("[LOG] Using legacy function wrapper - redirecting to sequential citations")
     return process_question_with_sequential_citations(agent, question, chunks_directory)