Update app.py

app.py

@@ -1,66 +1,58 @@
-""" Enhanced LangGraph Agent Evaluation Runner - Final Version"""
+""" Enhanced Multi-LLM Agent Evaluation Runner with Vector Database Integration"""
 import os
 import gradio as gr
 import requests
 import pandas as pd
 from langchain_core.messages import HumanMessage
-from veryfinal import build_graph
+from veryfinal import build_graph, HybridLangGraphMultiLLMSystem
 
 # --- Constants ---
 DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 
 # --- Enhanced Agent Definition ---
-class EnhancedLangGraphAgent:
-    """Enhanced LangGraph agent with proper response handling."""
+class EnhancedMultiLLMAgent:
+    """A multi-provider LangGraph agent with vector database integration."""
     def __init__(self):
-        print("Enhanced LangGraph Agent initialized.")
+        print("Enhanced Multi-LLM Agent with Vector Database initialized.")
         try:
-            self.graph = build_graph(provider="groq")
-            print("LangGraph built successfully.")
+            self.system = HybridLangGraphMultiLLMSystem(provider="groq")
+            self.graph = self.system.graph
+            
+            # Load metadata if available
+            if os.path.exists("metadata.jsonl"):
+                print("Loading question metadata...")
+                count = self.system.load_metadata_from_jsonl("metadata.jsonl")
+                print(f"Loaded {count} questions into vector database")
+            
+            print("Enhanced Multi-LLM Graph built successfully.")
         except Exception as e:
             print(f"Error building graph: {e}")
             self.graph = None
+            self.system = None
 
     def __call__(self, question: str) -> str:
-        print(f"Processing: {question[:100]}...")
+        print(f"Agent received question: {question[:100]}...")
         
-        if self.graph is None:
+        if self.graph is None or self.system is None:
             return "Error: Agent not properly initialized"
         
         try:
-            # Create messages and config
-            messages = [HumanMessage(content=question)]
-            config = {"configurable": {"thread_id": f"eval_{hash(question)}"}}
+            # Use the enhanced system's process_query method
+            answer = self.system.process_query(question)
             
-            # Invoke the graph
-            result = self.graph.invoke({"messages": messages}, config)
-            
-            # Extract the final answer
-            if result and "messages" in result and result["messages"]:
-                final_message = result["messages"][-1]
-                if hasattr(final_message, 'content'):
-                    answer = final_message.content
-                else:
-                    answer = str(final_message)
-                
-                # Clean up the answer
-                if "FINAL ANSWER:" in answer:
-                    answer = answer.split("FINAL ANSWER:")[-1].strip()
-                
-                # Validate the answer
-                if not answer or answer == question or len(answer.strip()) == 0:
-                    return "Information not available"
-                
-                return answer.strip()
-            else:
+            # Additional validation
+            if not answer or answer == question or len(answer.strip()) == 0:
                 return "Information not available"
                 
+            return answer.strip()
+                
         except Exception as e:
-            print(f"Error processing question: {e}")
-            return f"Error: {str(e)}"
+            error_msg = f"Error: {str(e)}"
+            print(error_msg)
+            return error_msg
 
 def run_and_submit_all(profile: gr.OAuthProfile | None):
-    """Fetch questions, run agent, and submit answers."""
+    """Fetch questions, run enhanced agent, and submit answers."""
     space_id = os.getenv("SPACE_ID")
     
     if profile:
@@ -76,7 +68,7 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
 
     # 1. Instantiate Agent
     try:
-        agent = EnhancedLangGraphAgent()
+        agent = EnhancedMultiLLMAgent()
         if agent.graph is None:
             return "Error: Failed to initialize agent properly", None
     except Exception as e:
@@ -84,6 +76,7 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
         return f"Error initializing agent: {e}", None
     
     agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main" if space_id else "No space ID available"
+    print(f"Agent code URL: {agent_code}")
 
     # 2. Fetch Questions
     print(f"Fetching questions from: {questions_url}")
@@ -92,27 +85,35 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
         response.raise_for_status()
         questions_data = response.json()
         if not questions_data:
+            print("Fetched questions list is empty.")
             return "Fetched questions list is empty or invalid format.", None
         print(f"Fetched {len(questions_data)} questions.")
     except Exception as e:
+        print(f"Error fetching questions: {e}")
         return f"Error fetching questions: {e}", None
 
-    # 3. Run Agent
+    # 3. Run Enhanced Agent
     results_log = []
     answers_payload = []
-    print(f"Running Enhanced LangGraph agent on {len(questions_data)} questions...")
+    print(f"Running Enhanced Multi-LLM agent with vector database on {len(questions_data)} questions...")
     
     for i, item in enumerate(questions_data):
         task_id = item.get("task_id")
         question_text = item.get("question")
         
         if not task_id or question_text is None:
+            print(f"Skipping item with missing task_id or question: {item}")
             continue
             
         print(f"Processing question {i+1}/{len(questions_data)}: {task_id}")
         
         try:
             submitted_answer = agent(question_text)
+            
+            # Additional validation to prevent question repetition
+            if submitted_answer == question_text or submitted_answer.startswith(question_text):
+                submitted_answer = "Information not available"
+            
             answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
             results_log.append({
                 "Task ID": task_id, 
@@ -121,6 +122,7 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
             })
         except Exception as e:
             error_msg = f"AGENT ERROR: {e}"
+            print(f"Error running agent on task {task_id}: {e}")
             answers_payload.append({"task_id": task_id, "submitted_answer": error_msg})
             results_log.append({
                 "Task ID": task_id, 
@@ -129,12 +131,16 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
             })
 
     if not answers_payload:
+        print("Agent did not produce any answers to submit.")
         return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
 
-    # 4. Submit
+    # 4. Prepare Submission 
     submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
-    print(f"Submitting {len(answers_payload)} answers...")
-    
+    status_update = f"Enhanced Multi-LLM Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
+    print(status_update)
+
+    # 5. Submit
+    print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
     try:
         response = requests.post(submit_url, json=submission_data, timeout=60)
         response.raise_for_status()
@@ -146,35 +152,51 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
             f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
             f"Message: {result_data.get('message', 'No message received.')}"
         )
-        return final_status, pd.DataFrame(results_log)
+        print("Submission successful.")
+        results_df = pd.DataFrame(results_log)
+        return final_status, results_df
     except Exception as e:
-        return f"Submission Failed: {e}", pd.DataFrame(results_log)
+        status_message = f"Submission Failed: {e}"
+        print(status_message)
+        results_df = pd.DataFrame(results_log)
+        return status_message, results_df
 
-# --- Gradio Interface ---
+# --- Build Gradio Interface ---
 with gr.Blocks() as demo:
-    gr.Markdown("# Enhanced LangGraph Agent - Final Version")
+    gr.Markdown("# Enhanced Multi-LLM Agent with Vector Database Integration")
     gr.Markdown(
         """
-        **Features:**
-        - ✅ Proper LangGraph structure with tool integration
-        - ✅ Multi-LLM support (Groq, Google, HuggingFace)
-        - ✅ Enhanced search capabilities (Wikipedia, Tavily, ArXiv)
-        - ✅ Mathematical tools for calculations
-        - ✅ Vector store integration for similar questions
-        - ✅ Proper response formatting and validation
-        - ✅ Error handling and fallback mechanisms
+        **Instructions:**
+        1. Log in to your Hugging Face account using the button below.
+        2. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.
+        
+        **Enhanced Agent Features:**
+        - **Multi-LLM Support**: Groq (Llama-3 8B/70B, DeepSeek)
+        - **Vector Database Integration**: FAISS + Supabase for similar question retrieval
+        - **Intelligent Routing**: Automatically selects best provider based on query complexity
+        - **Enhanced Tools**: Mathematical operations, web search, Wikipedia integration
+        - **Question-Answering**: Optimized for evaluation tasks with proper formatting
+        - **Similar Questions Context**: Uses vector similarity to provide relevant context
+        - **Error Handling**: Robust fallback mechanisms and comprehensive logging
         
-        **Tools Available:**
-        - Mathematical operations (add, subtract, multiply, divide, modulus)
-        - Wikipedia search for encyclopedic information
-        - Web search via Tavily for current information
-        - ArXiv search for academic papers
-        - Vector similarity search for related questions
+        **Routing Examples:**
+        - Math: "What is 25 multiplied by 17?" → Llama-3 70B
+        - Search: "Find information about Mercedes Sosa" → Search-Enhanced
+        - Complex: "Analyze quantum computing principles" → DeepSeek
+        - Simple: "What is the capital of France?" → Llama-3 8B
+        
+        **Vector Database Features:**
+        - Automatic loading of metadata.jsonl if present
+        - Similar question retrieval for enhanced context
+        - Supabase integration for persistent storage
+        - FAISS for fast vector similarity search
         """
     )
 
     gr.LoginButton()
+
     run_button = gr.Button("Run Evaluation & Submit All Answers", variant="primary")
+
     status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
     results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
 
@@ -184,5 +206,5 @@ with gr.Blocks() as demo:
     )
 
 if __name__ == "__main__":
-    print("\n" + "-"*30 + " Enhanced LangGraph Agent Starting " + "-"*30)
+    print("\n" + "-"*30 + " Enhanced Multi-LLM Agent with Vector DB Starting " + "-"*30)
     demo.launch(debug=True, share=False)