app.py

import os
import gradio as gr
import requests
import inspect
import pandas as pd
import asyncio
import aiohttp
from smolagents import FinalAnswerTool, Tool, tool, OpenAIServerModel, DuckDuckGoSearchTool, CodeAgent, VisitWebpageTool


from dotenv import load_dotenv

load_dotenv()
# (Keep Constants as is)
# --- Constants ---
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"


OPENAI_TOKEN = os.getenv("OPENAI_API_KEY")

# --- Basic Agent Definition ---
# ----- THIS IS WERE YOU CAN BUILD WHAT YOU WANT ------
class SlpMultiAgent:
    def __init__(self):
        print("BasicAgent initialized.")
        
    async def __call__(self, question: str) -> str:
        print(f"Agent received question (first 50 chars): {question[:50]}...")
        fixed_answer = "This is a default answer."
        print(f"Agent returning fixed answer: {fixed_answer}")
        
        # Truncate question to avoid exceeding model context length
        MAX_QUESTION_LENGTH = 1000
        short_question = question  # [:MAX_QUESTION_LENGTH]
        
        # Use GPT-4o model with larger context window
        model = OpenAIServerModel(
            model_id="gpt-4o", 
            temperature=0.0, 
            max_tokens=1500
        )
        
        # Here you can implement your agent logic, tools, and model calls
        web_agent = CodeAgent(
            tools=[DuckDuckGoSearchTool(), VisitWebpageTool()],
            model=model,
            additional_authorized_imports=["pandas"],
            max_steps=10,
            name="WebAgent",
            verbosity_level=0,
            description="An agent that can search the web, visit webpages, and calculate cargo travel times between locations."
        )

        manager_agent = CodeAgent(
            model=OpenAIServerModel("gpt-4o"),
            tools=[],
            managed_agents=[web_agent],
            name="ManagerAgent",
            description="A manager agent that can delegate tasks to other agents and manage their execution.",
            additional_authorized_imports=[
                "pandas",
            ],
            planning_interval=5,
            verbosity_level=2,
            max_steps=15,
            final_answer_checks=[check_reasoning]
        )
        
        # Create a task for the agent run to avoid blocking
        loop = asyncio.get_event_loop()
        result = await loop.run_in_executor(
            None,
            lambda: manager_agent.run(f"""
            You are a question answering agent. That specializes in complex questions that require multiple steps to answer.
            Take a few steps and think about the question before answering.
            You can use the tools available to you, but you should not use them unless necessary.
            You should always try to answer the question using your own knowledge and reasoning.
            If you need to use a tool, you should explain why you are using it and what you expect to find.
            If you are not sure about something, you should say so and explain why you are not sure.
            You should always try to provide a complete and accurate answer to the question.
            If you are not able to answer the question, you should say so and explain why

            Never try to process strings using code: when you have a string to read, just print it and you'll see it.
                            
            Here is the question: {short_question}
            Thoughts: [your reasoning about how to solve the problem]
            Code:
            ```py
            # Your Python code here
            ```<end_code>

            The code block MUST start with ```py on its own line and end with ```<end_code> on its own line.
            """)
        )
        
        # Return the result from the agent
        return result
    
def check_reasoning(final_answer, agent_memory):
    multimodal_model = OpenAIServerModel("gpt-4o", 
    max_tokens=1500)
    prompt = (
        f"Here is a user-given task and the agent steps: {agent_memory.get_succinct_steps()}. Now here is the plot that was made."
        "Please check that the reasoning process and plot are correct: do they correctly answer the given task?"
        "First list reasons why yes/no, then write your final decision: PASS in caps lock if it is satisfactory, FAIL if it is not."
        "Don't be harsh: if the plot mostly solves the task, it should pass."
        "To pass the question should be answered correctly and the reasoning should be sound."
        "The final answer is: {final_answer}. "
    )
    messages = [
        {
            "role": "user",
            "content": [
                {
                    "type": "text",
                    "text": prompt,
                }
            ],
        }
    ]
    output = multimodal_model(messages).content
    print("Reasoning and plot check output:", output)
    if "fail" in output.lower():
        print("Reasoning check failed. Please review the agent's reasoning.")


async def run_and_submit_all(profile):
    """
    Fetches all questions, runs the BasicAgent on them, submits all answers,
    and displays the results asynchronously.
    """
    # --- Determine HF Space Runtime URL and Repo URL ---
    space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code

    # Handle different profile types
    if profile:
        if hasattr(profile, 'username'):
            # It's an OAuthProfile object
            username = profile.username
        else:
            # It's a string or other type
            username = str(profile)
        print(f"User logged in: {username}")
    else:
        print("User not logged in.")
        return "Please Login to Hugging Face with the button.", None

    api_url = DEFAULT_API_URL
    questions_url = f"{api_url}/questions"
    submit_url = f"{api_url}/submit"

    # 1. Instantiate Agent ( modify this part to create your agent)
    try:
        agent = SlpMultiAgent()
    except Exception as e:
        print(f"Error instantiating agent: {e}")
        return f"Error initializing agent: {e}", None
    # In the case of an app running as a hugging Face space, this link points toward your codebase ( usefull for others so please keep it public)
    agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
    print(agent_code)

    # 2. Fetch Questions
    print(f"Fetching questions from: {questions_url}")
    try:
        async with aiohttp.ClientSession() as session:
            async with session.get(questions_url, timeout=15) as response:
                response.raise_for_status()
                questions_data = await response.json()
                if not questions_data:
                    print("Fetched questions list is empty.")
                    return "Fetched questions list is empty or invalid format.", None
                print(f"Fetched {len(questions_data)} questions.")
    except aiohttp.ClientError as e:
        print(f"Error fetching questions: {e}")
        return f"Error fetching questions: {e}", None
    except ValueError as e:  # JSON decode error
        print(f"Error decoding JSON response from questions endpoint: {e}")
        return f"Error decoding server response for questions: {e}", None
    except Exception as e:
        print(f"An unexpected error occurred fetching questions: {e}")
        return f"An unexpected error occurred fetching questions: {e}", None

    # 3. Run your Agent
    results_log = []
    answers_payload = []
    print(f"Running agent on {len(questions_data)} questions...")
    
    # Process questions concurrently with a semaphore to limit concurrency
    semaphore = asyncio.Semaphore(3)  # Limit to 3 concurrent requests
    
    async def process_question(item):
        task_id = item.get("task_id")
        question_text = item.get("question")
        if not task_id or question_text is None:
            print(f"Skipping item with missing task_id or question: {item}")
            return None
        
        async with semaphore:
            try:
                submitted_answer = await agent(question_text)
                return {"task_id": task_id, "submitted_answer": submitted_answer, 
                        "log": {"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer}}
            except Exception as e:
                print(f"Error running agent on task {task_id}: {e}")
                return {"task_id": task_id, "submitted_answer": f"AGENT ERROR: {e}", 
                        "log": {"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"}}
    
    # Create tasks for all questions
    tasks = [process_question(item) for item in questions_data]
    results = await asyncio.gather(*tasks)
    
    # Process results
    for result in results:
        if result is not None:
            answers_payload.append({"task_id": result["task_id"], "submitted_answer": result["submitted_answer"]})
            results_log.append(result["log"])

    if not answers_payload:
        print("Agent did not produce any answers to submit.")
        return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)

    # 4. Prepare Submission 
    submission_data = {"username": str(username).strip(), "agent_code": agent_code, "answers": answers_payload}
    status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
    print(status_update)

    # 5. Submit
    print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
    try:
        async with aiohttp.ClientSession() as session:
            async with session.post(submit_url, json=submission_data, timeout=60) as response:
                response.raise_for_status()
                result_data = await response.json()
                final_status = (
                    f"Submission Successful!\n"
                    f"User: {result_data.get('username')}\n"
                    f"Overall Score: {result_data.get('score', 'N/A')}% "
                    f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
                    f"Message: {result_data.get('message', 'No message received.')}"
                )
                print("Submission successful.")
                results_df = pd.DataFrame(results_log)
                return final_status, results_df
    except aiohttp.ClientResponseError as e:
        error_detail = f"Server responded with status {e.status}."
        try:
            error_text = await e.response.text()
            try:
                error_json = await e.response.json()
                error_detail += f" Detail: {error_json.get('detail', error_text)}"
            except ValueError:
                error_detail += f" Response: {error_text[:500]}"
        except:
            pass
        status_message = f"Submission Failed: {error_detail}"
        print(status_message)
        results_df = pd.DataFrame(results_log)
        return status_message, results_df
    except asyncio.TimeoutError:
        status_message = "Submission Failed: The request timed out."
        print(status_message)
        results_df = pd.DataFrame(results_log)
        return status_message, results_df
    except aiohttp.ClientError as e:
        status_message = f"Submission Failed: Network error - {e}"
        print(status_message)
        results_df = pd.DataFrame(results_log)
        return status_message, results_df
    except Exception as e:
        status_message = f"An unexpected error occurred during submission: {e}"
        print(status_message)
        results_df = pd.DataFrame(results_log)
        return status_message, results_df


# --- Build Gradio Interface using Blocks ---
with gr.Blocks() as demo:
    gr.Markdown("# Basic Agent Evaluation Runner")
    gr.Markdown(
        """
        **Instructions:**

        1.  Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ...
        2.  Log in to your Hugging Face account using the button below. This uses your HF username for submission.
        3.  Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.

        ---
        **Disclaimers:**
        Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions).
        This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async.
        """
    )

    login_button = gr.LoginButton()

    run_button = gr.Button("Run Evaluation & Submit All Answers")

    status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
    # Removed max_rows=10 from DataFrame constructor
    results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
    
    def sync_wrapper(profile):
        # This wrapper ensures we have access to the profile
        if not profile:
            print("No profile available in sync_wrapper")
            return "Please Login to Hugging Face with the button.", None
        print(f"Profile type in wrapper: {type(profile)}")
        try:
            return asyncio.run(run_and_submit_all(profile))
        except Exception as e:
            print(f"Error in sync_wrapper: {e}")
            return f"Error processing request: {e}", None
    
    run_button.click(
        fn=sync_wrapper,
        inputs=login_button,
        outputs=[status_output, results_table]
    )

if __name__ == "__main__":
    print("\n" + "-"*30 + " App Starting " + "-"*30)
    # Check for SPACE_HOST and SPACE_ID at startup for information
    space_host_startup = os.getenv("SPACE_HOST")
    space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup

    if space_host_startup:
        print(f"✅ SPACE_HOST found: {space_host_startup}")
        print(f"   Runtime URL should be: https://{space_host_startup}.hf.space")
    else:
        print("ℹ️  SPACE_HOST environment variable not found (running locally?).")

    if space_id_startup: # Print repo URLs if SPACE_ID is found
        print(f"✅ SPACE_ID found: {space_id_startup}")
        print(f"   Repo URL: https://huggingface.co/spaces/{space_id_startup}")
        print(f"   Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")
    else:
        print("ℹ️  SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")

    print("-"*(60 + len(" App Starting ")) + "\n")

    print("Launching Gradio Interface for Basic Agent Evaluation...")
    demo.launch(debug=True, share=False)