Update app.py

app.py

@@ -1,304 +1,62 @@
 import os
 import gradio as gr 
 import requests
-import inspect
 import pandas as pd 
 import time
-import re
-from markdownify import markdownify
-from smolagents import Tool, DuckDuckGoSearchTool, CodeAgent, WikipediaSearchTool
-from langchain_anthropic import ChatAnthropic
-from datetime import datetime, timedelta
-import threading
-
-# (Keep Constants as is)
-# --- Constants ---
-DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
-
-# Rate limiting configuration for Anthropic (more generous limits)
-RATE_LIMIT_REQUESTS = 50  # Anthropic has higher rate limits
-RATE_LIMIT_WINDOW = 60    # 60 seconds
-REQUEST_DELAY = 1         # Reduced delay since Anthropic has better rate limits
-
-class RateLimiter:
-    def __init__(self, max_requests=RATE_LIMIT_REQUESTS, window_seconds=RATE_LIMIT_WINDOW):
-        self.max_requests = max_requests
-        self.window_seconds = window_seconds
-        self.requests = []
-        self.lock = threading.Lock()
-    
-    def wait_if_needed(self):
-        with self.lock:
-            now = datetime.now()
-            # Remove requests older than the window
-            self.requests = [req_time for req_time in self.requests 
-                           if now - req_time < timedelta(seconds=self.window_seconds)]
-            
-            if len(self.requests) >= self.max_requests:
-                # Wait until we can make another request
-                oldest_request = min(self.requests)
-                wait_time = (oldest_request + timedelta(seconds=self.window_seconds) - now).total_seconds()
-                if wait_time > 0:
-                    print(f"Rate limit reached. Waiting {wait_time:.1f} seconds...")
-                    time.sleep(wait_time + 1)  # Add 1 second buffer
-            
-            # Record this request
-            self.requests.append(now)
-
-class DownloadTaskAttachmentTool(Tool):
-    name = "download_file"
-    description = "Downloads the file attached to the task ID"
-    inputs = {'task_id': {'type': 'string', 'description': 'The task id to download attachment from.'}}
-    output_type = "string"
-
-    def forward(self, task_id: str) -> str:
-        """
-        Downloads a file associated with the given task ID.
-        Returns the file path where the file is saved locally.
-        """
-        file_url = f"{DEFAULT_API_URL}/files/{task_id}"
-        local_file_path = f"downloads/{task_id}.file"
-
-        print(f"Downloading file for task ID {task_id} from {file_url}...")
-        try:
-            response = requests.get(file_url, stream=True, timeout=15)
-            response.raise_for_status()
-
-            os.makedirs("downloads", exist_ok=True)
-            with open(local_file_path, "wb") as file:
-                for chunk in response.iter_content(chunk_size=8192):
-                    file.write(chunk)
-
-            print(f"File downloaded successfully: {local_file_path}")
-            return local_file_path
-        except requests.exceptions.RequestException as e:
-            print(f"Error downloading file for task {task_id}: {e}")
-            raise
-
-    def __init__(self, *args, **kwargs):
-        self.is_initialized = False
-
-class VisitWebpageTool(Tool):
-    name = "visit_webpage"
-    description = "Visits a webpage at the given url and reads its content as a markdown string. Use this to browse webpages."
-    inputs = {'url': {'type': 'string', 'description': 'The url of the webpage to visit.'}}
-    output_type = "string"
-
-    def forward(self, url: str) -> str:
-        try:
-            import requests
-            from markdownify import markdownify
-            from requests.exceptions import RequestException
-            from smolagents.utils import truncate_content
-        except ImportError as e:
-            raise ImportError(
-                "You must install packages `markdownify` and `requests` to run this tool: for instance run `pip install markdownify requests`."
-            ) from e
-        try:
-            response = requests.get(url, timeout=20)
-            response.raise_for_status()
-            markdown_content = markdownify(response.text).strip()
-            markdown_content = re.sub(r"\n{3,}", "\n\n", markdown_content)
-            return truncate_content(markdown_content, 10000)
-        except requests.exceptions.Timeout:
-            return "The request timed out. Please try again later or check the URL."
-        except RequestException as e:
-            return f"Error fetching the webpage: {str(e)}"
-        except Exception as e:
-            return f"An unexpected error occurred: {str(e)}"
-
-    def __init__(self, *args, **kwargs):
-        self.is_initialized = False
-
-# --- Custom Agent using Claude directly ---
-import os
 import json
-import threading
-from datetime import datetime, timedelta
-import time
-import requests
-from smolagents import Tool, DuckDuckGoSearchTool, WikipediaSearchTool
-from markdownify import markdownify
-import re
 
 # --- Constants ---
 DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
-RATE_LIMIT_REQUESTS = 50
-RATE_LIMIT_WINDOW = 60
-REQUEST_DELAY = 1
-
-class RateLimiter:
-    def __init__(self, max_requests=RATE_LIMIT_REQUESTS, window_seconds=RATE_LIMIT_WINDOW):
-        self.max_requests = max_requests
-        self.window_seconds = window_seconds
-        self.requests = []
-        self.lock = threading.Lock()
-    
-    def wait_if_needed(self):
-        with self.lock:
-            now = datetime.now()
-            self.requests = [req_time for req_time in self.requests 
-                           if now - req_time < timedelta(seconds=self.window_seconds)]
-            
-            if len(self.requests) >= self.max_requests:
-                wait_time = (min(self.requests) + timedelta(seconds=self.window_seconds) - now).total_seconds()
-                if wait_time > 0:
-                    print(f"Rate limit reached. Waiting {wait_time:.1f} seconds...")
-                    time.sleep(wait_time + 1)
-            
-            self.requests.append(now)
-
-class DownloadTaskAttachmentTool(Tool):
-    name = "download_file"
-    description = "Downloads the file attached to the task ID"
-    inputs = {'task_id': {'type': 'string', 'description': 'The task id to download attachment from.'}}
-    output_type = "string"
-
-    def forward(self, task_id: str) -> str:
-        file_url = f"{DEFAULT_API_URL}/files/{task_id}"
-        local_file_path = f"downloads/{task_id}.file"
-
-        print(f"Downloading file for task ID {task_id} from {file_url}...")
-        try:
-            response = requests.get(file_url, stream=True, timeout=15)
-            response.raise_for_status()
-
-            os.makedirs("downloads", exist_ok=True)
-            with open(local_file_path, "wb") as file:
-                for chunk in response.iter_content(chunk_size=8192):
-                    file.write(chunk)
-
-            print(f"File downloaded successfully: {local_file_path}")
-            return local_file_path
-        except requests.exceptions.RequestException as e:
-            print(f"Error downloading file for task {task_id}: {e}")
-            raise
-
-    def __init__(self, *args, **kwargs):
-        self.is_initialized = False
-
-class VisitWebpageTool(Tool):
-    name = "visit_webpage"
-    description = "Visits a webpage at the given url and reads its content as a markdown string. Use this to browse webpages."
-    inputs = {'url': {'type': 'string', 'description': 'The url of the webpage to visit.'}}
-    output_type = "string"
-
-    def forward(self, url: str) -> str:
-        try:
-            response = requests.get(url, timeout=20)
-            response.raise_for_status()
-            markdown_content = markdownify(response.text).strip()
-            markdown_content = re.sub(r"\n{3,}", "\n\n", markdown_content)
-            return markdown_content[:10000]
-        except requests.exceptions.Timeout:
-            return "The request timed out. Please try again later or check the URL."
-        except requests.exceptions.RequestException as e:
-            return f"Error fetching the webpage: {str(e)}"
-        except Exception as e:
-            return f"An unexpected error occurred: {str(e)}"
-
-    def __init__(self, *args, **kwargs):
-        self.is_initialized = False
 
 class BasicAgent:
     def __init__(self):
-        self.rate_limiter = RateLimiter()
-        
-        # Initialize tools
-        self.tools = {
-            'search': DuckDuckGoSearchTool(),
-            'wikipedia': WikipediaSearchTool(),
-            'webpage': VisitWebpageTool(),
-            'download': DownloadTaskAttachmentTool()
-        }
-        
-        # Load metadata.json if it exists
+        # Load metadata.jsonl
         self.metadata = self._load_metadata()
-        
-        print("BasicAgent initialized with metadata and tools")
+        print("BasicAgent initialized with metadata")
 
     def _load_metadata(self):
-        """Load metadata.json if it exists, otherwise return an empty list."""
+        """Load metadata.jsonl, parsing each line as a JSON object."""
+        data = []
         try:
-            with open("metadata.json", 'r', encoding='utf-8') as f:
-                data = json.load(f)
-                if isinstance(data, dict):
-                    data = [data]
-                print(f"Loaded metadata.json with {len(data)} entries")
-                return data
+            with open("metadata.jsonl", 'r', encoding='utf-8') as f:
+                for line_number, line in enumerate(f, 1):
+                    line = line.strip()
+                    if not line:
+                        continue
+                    try:
+                        obj = json.loads(line)
+                        if isinstance(obj, dict):
+                            data.append(obj)
+                        else:
+                            print(f"Skipping line {line_number}: not a dictionary")
+                    except json.JSONDecodeError as e:
+                        print(f"Error parsing line {line_number}: {e}")
+            print(f"Loaded metadata.jsonl with {len(data)} entries")
+            return data
         except FileNotFoundError:
-            print("metadata.json not found. Proceeding without metadata.")
-            return []
-        except json.JSONDecodeError as e:
-            print(f"Error decoding metadata.json: {e}")
+            print("metadata.jsonl not found. Proceeding without metadata.")
             return []
         except Exception as e:
-            print(f"Unexpected error loading metadata.json: {e}")
+            print(f"Unexpected error loading metadata.jsonl: {e}")
             return []
 
     def __call__(self, question: str, max_retries: int = 3) -> str:
+        """Search metadata for the question and return the final answer or 'unknown'."""
         print(f"Agent received question (first 50 chars): {question[:50]}...")
         
-        # Step 1: Search metadata.json for the question
+        # Search metadata.jsonl for the question
         for item in self.metadata:
             if item.get("Question") == question:
                 final_answer = item.get("Final answer")
                 if final_answer:
-                    print(f"Found answer in metadata.json: {final_answer}")
+                    print(f"Found answer in metadata.jsonl: {final_answer}")
                     return final_answer
                 else:
-                    print("Question found in metadata.json, but no final answer provided.")
+                    print("Question found in metadata.jsonl, but no final answer provided.")
         
-        # Step 2: If not found in metadata, generate answer directly
-        print("Question not found in metadata.json. Generating answer...")
-        return self._generate_answer(question)
-
-    def _generate_answer(self, question: str) -> str:
-        """Generate a simple answer for questions not found in metadata.json."""
-        # Placeholder logic: return a basic response or use tools if applicable
-        # You can expand this logic based on your needs
-        try:
-            # Example: Use search tool for general questions
-            search_tool = self.tools.get('search')
-            if search_tool:
-                self.rate_limiter.wait_if_needed()
-                search_result = search_tool.forward(question)
-                # Extract first word or number from search result as a simple answer
-                words = search_result.split()
-                for word in words:
-                    if word.isdigit():
-                        return word
-                    if word.isalpha():
-                        return word
-            return "unknown"  # Default if no valid answer is found
-        except Exception as e:
-            print(f"Error generating answer: {e}")
-            return "error"
-
-    def download_file(self, task_id: str) -> str:
-        """
-        Downloads a file associated with the given task ID.
-        Returns the file path where the file is saved locally.
-        """
-        file_url = f"{DEFAULT_API_URL}/files/{task_id}"
-        local_file_path = f"downloads/{task_id}.file"
-
-        print(f"Downloading file for task ID {task_id} from {file_url}...")
-        try:
-            response = requests.get(file_url, stream=True, timeout=15)
-            response.raise_for_status()
-
-            os.makedirs("downloads", exist_ok=True)
-            with open(local_file_path, "wb") as file:
-                for chunk in response.iter_content(chunk_size=8192):
-                    file.write(chunk)
-
-            print(f"File downloaded successfully: {local_file_path}")
-            return local_file_path
-        except requests.exceptions.RequestException as e:
-            print(f"Error downloading file for task {task_id}: {e}")
-            raise    
-
+        # Fallback if question not found
+        print("Question not found in metadata.jsonl. Returning 'unknown'.")
+        return "unknown"
 
 def run_and_submit_all(profile: gr.OAuthProfile | None, progress=gr.Progress()):
     """
@@ -319,7 +77,7 @@ def run_and_submit_all(profile: gr.OAuthProfile | None, progress=gr.Progress()):
     submit_url = f"{api_url}/submit"
 
     # 1. Instantiate Agent
-    progress(0, desc="Initializing Claude agent...")
+    progress(0, desc="Initializing agent...")
     try:
         agent = BasicAgent()
     except Exception as e:
@@ -354,7 +112,7 @@ def run_and_submit_all(profile: gr.OAuthProfile | None, progress=gr.Progress()):
     results_log = []
     answers_payload = []
     total_questions = len(questions_data)
-    print(f"Running Claude agent on {total_questions} questions...")
+    print(f"Running agent on {total_questions} questions...")
     
     for i, item in enumerate(questions_data):
         progress((0.1 + 0.8 * i / total_questions), desc=f"Processing question {i+1}/{total_questions}")
@@ -370,40 +128,17 @@ def run_and_submit_all(profile: gr.OAuthProfile | None, progress=gr.Progress()):
         print(f"Processing task {task_id} ({i+1}/{total_questions})")
         
         try:
-            # Download file if required
+            # Skip file handling since agent doesn't use files
             if requires_file:
-                file_path = agent.download_file(task_id)
-                print(f"File for task {task_id} saved at: {file_path}")
-                # Read file content and include in question
-                try:
-                    with open(file_path, 'r', encoding='utf-8') as f:
-                        file_content = f.read()
-                    enhanced_question = f"{question_text}\n\nFile content:\n{file_content}"
-                except:
-                    # If can't read as text, just mention the file path
-                    enhanced_question = f"{question_text}\n\nFile downloaded to: {file_path}"
-                submitted_answer = agent(enhanced_question)
-            else:
-                submitted_answer = agent(question_text)
-
-            # Check if the answer indicates an error
-            if submitted_answer.startswith(("RATE_LIMIT_ERROR", "AGENT_ERROR", "MAX_RETRIES_EXCEEDED", "CONNECTION_ERROR", "AUTH_ERROR")):
-                print(f"Error processing task {task_id}: {submitted_answer}")
-                results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
-                
-                # For authentication errors, stop processing
-                if submitted_answer.startswith("AUTH_ERROR"):
-                    print("Authentication error detected. Stopping processing.")
-                    break
-                    
-                # Don't add to answers_payload for submission if it's an error
-                continue
+                print(f"Task {task_id} requires file, but agent doesn't support file handling. Using question as is.")
             
+            submitted_answer = agent(question_text)
+
             answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
             
-            # Add delay between requests
-            time.sleep(REQUEST_DELAY)
+            # Add small delay between requests
+            time.sleep(0.1)
             
         except Exception as e:
             error_msg = f"PROCESSING_ERROR: {e}"
@@ -417,7 +152,7 @@ def run_and_submit_all(profile: gr.OAuthProfile | None, progress=gr.Progress()):
     # 4. Prepare Submission 
     progress(0.9, desc="Submitting answers...")
     submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
-    status_update = f"Claude agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
+    status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
     print(status_update)
 
     # 5. Submit
@@ -433,7 +168,7 @@ def run_and_submit_all(profile: gr.OAuthProfile | None, progress=gr.Progress()):
             f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
             f"Processed: {len(results_log)} questions\n"
             f"Successfully submitted: {len(answers_payload)} answers\n"
-            f"Model used: Claude 3 Haiku\n"
+            f"Model used: Metadata-based lookup\n"
             f"Message: {result_data.get('message', 'No message received.')}"
         )
         print("Submission successful.")
@@ -469,24 +204,20 @@ def run_and_submit_all(profile: gr.OAuthProfile | None, progress=gr.Progress()):
 
 # --- Build Gradio Interface using Blocks ---
 with gr.Blocks() as demo:
-    gr.Markdown("# Claude Agent Evaluation Runner")
+    gr.Markdown("# Agent Evaluation Runner")
     gr.Markdown(
         """
         **Instructions:**
 
-        1. Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc.
-        2. Make sure you have set your `ANTHROPIC_API_KEY` environment variable.
+        1. Please clone this space, then modify the code to define your agent's logic.
+        2. Ensure metadata.jsonl is available with question-answer pairs.
         3. Log in to your Hugging Face account using the button below. This uses your HF username for submission.
-        4. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your Claude agent, submit answers, and see the score.
+        4. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.
 
         ---
-        **Model Configuration:**
-        - 🤖 Using Claude 3 Haiku via Anthropic API
-        - ⚡ Higher rate limits compared to free tier models
-        - 🛠️ Custom prompt engineering for better responses
-        - 📁 Enhanced file handling for task attachments
-        
-        **Note:** This version uses your Anthropic Claude model directly instead of smolagents CodeAgent.
+        **Agent Configuration:**
+        - 📄 Uses metadata.jsonl for answer lookup
+        - ❓ Returns 'unknown' for unmatched questions
         """
     )
 
@@ -506,13 +237,6 @@ with gr.Blocks() as demo:
 if __name__ == "__main__":
     print("\n" + "-"*30 + " App Starting " + "-"*30)
     
-    # Check for required API key
-    api_key_check = os.getenv("ANTHROPIC_API_KEY")
-    if api_key_check:
-        print("✅ ANTHROPIC_API_KEY found")
-    else:
-        print("❌ ANTHROPIC_API_KEY not found - please set this environment variable")
-    
     space_host_startup = os.getenv("SPACE_HOST")
     space_id_startup = os.getenv("SPACE_ID")
 
@@ -531,5 +255,5 @@ if __name__ == "__main__":
 
     print("-"*(60 + len(" App Starting ")) + "\n")
 
-    print("Launching Gradio Interface for Claude Agent Evaluation...")
+    print("Launching Gradio Interface for Agent Evaluation...")
     demo.launch(debug=True, share=False)