import os import gradio as gr import requests import pandas as pd from my_agent import GeminiAgentContainer from markdownify import markdownify as to_markdown import time import json # (Keep Constants as is) # --- Constants --- DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space" # --- Global Variables --- questions = None results_log = [] answers_by_task = {} def load_questions(questions_url): print(f"Fetching questions from: {questions_url}") try: response = requests.get(questions_url, timeout=15) response.raise_for_status() questions_data = response.json() if not questions_data: print("Fetched questions list is empty or invalid.") return None print(f"Fetched {len(questions_data)} questions.") except requests.exceptions.RequestException as e: print(f"Error fetching questions: {e}") return None except requests.exceptions.JSONDecodeError as e: print(f"Error decoding JSON response from questions endpoint: {e}") print(f"Response text: {response.text[:500]}") return None except Exception as e: print(f"An unexpected error occurred fetching questions: {e}") return None return questions_data def answer_one(agent, question_data): """ Runs the agent on a single question and returns the result. """ task_id = question_data.get("task_id") question_text = question_data.get("question") filename = question_data.get("file_name") payload = None submitted_answer = None agent_error = None try: if not task_id or question_text is None: raise ValueError(f"Missing task_id or question in item: {question_data}") if filename: file_prompt = f"\nThere is an attached file with task id `{task_id}` available.\n" question_text = file_prompt + question_text submitted_answer = agent(question_text) payload = {"task_id": task_id, "submitted_answer": submitted_answer} except Exception as e: print(agent) print(f"Error running agent on task {task_id}: {e}") agent_error = f"AGENT ERROR: {e}" finally: log_entry = { "Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer or agent_error, } return payload, log_entry def _submit_all(username, agent_code, answers_payload, submit_url): # Prepare Submission submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload} status_update = f"Submitting {len(answers_payload)} answers for user '{username}'..." print(status_update) # Submit Answers print(f"Submitting {len(answers_payload)} answers to: {submit_url}") try: response = requests.post(submit_url, json=submission_data, timeout=60) response.raise_for_status() result_data = response.json() final_status = ( f"Submission Successful!\n" f"User: {result_data.get('username')}\n" f"Overall Score: {result_data.get('score', 'N/A')}% " f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n" f"Message: {result_data.get('message', 'No message received.')}" ) print(final_status) return final_status except requests.exceptions.HTTPError as e: error_detail = f"Server responded with status {e.response.status_code}." try: error_json = e.response.json() error_detail += f" Detail: {error_json.get('detail', e.response.text)}" except requests.exceptions.JSONDecodeError: error_detail += f" Response: {e.response.text[:500]}" status_message = f"Submission Failed: {error_detail}" print(status_message) return status_message except requests.exceptions.Timeout: status_message = "Submission Failed: The request timed out." print(status_message) return status_message except requests.exceptions.RequestException as e: status_message = f"Submission Failed: Network error - {e}" print(status_message) return status_message except Exception as e: status_message = f"An unexpected error occurred during submission: {e}" print(status_message) return status_message def prepare_agent(api_key=None): # 1. Instantiate Agent ( modify this part to create your agent) try: agent = GeminiAgentContainer(api_key=api_key) print(agent.system_prompt) except Exception as e: print(f"Error instantiating agent: {e}") return None return agent def save_answers_to_file(): """ Submits the answers to a local file named with the current epoch time. """ if not answers_by_task: return ("Nothing to save, no answers found.") answers_payload = list(answers_by_task.values()) file_path = f"answers-{int(time.time())}.json" print(f"Saving answers to file: {file_path}") try: with open(file_path, "w") as file: json.dump(answers_payload, file, indent=4) submit_status = (f"Answers successfully written to {file_path}") except Exception as e: submit_status = (f"Error writing answers to file: {e}") print(submit_status) return submit_status def run_all(api_key: str | None = None): """ Fetches all questions, runs the BasicAgent on them, """ questions_url = f"{DEFAULT_API_URL}/questions" agent = prepare_agent(api_key) questions_data = load_questions(questions_url) # 3. Run your Agent print(f"Running agent on {len(questions_data)} questions...") for item in questions_data: payload_data, log_entry = answer_one(agent, item) if payload_data: task_id = payload_data.get("task_id") answers_by_task[task_id] = payload_data results_log.append(log_entry) time.sleep(3) if not answers_by_task: final_status = "Agent did not produce any answers to submit." else: final_status = f"Agent finished, {len(answers_by_task)} answers produced." print(final_status) return final_status, pd.DataFrame(results_log) def submit_all( profile: gr.OAuthProfile | None): """ Submits all answers and displays the results. """ submit_url = f"{DEFAULT_API_URL}/submit" if profile: username= f"{profile.username}" print(f"User logged in: {username}") else: print("User not logged in.") return "Please Login to Hugging Face with the button." # --- Determine HF Space Runtime URL and Repo URL --- space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code if not answers_by_task: submit_status = "No answers to submit." else: # 4. Submit all answers # In the case of an app running as a hugging Face space, this link points toward your codebase ( usefull for others so please keep it public) agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main" submit_status = _submit_all(username, agent_code, list(answers_by_task.values()), submit_url) return submit_status # --- Build Gradio Interface using Blocks --- with gr.Blocks() as demo: gr.Markdown("# Basic Agent Evaluation Runner") gr.Markdown( """ **Instructions:** 1. Please use your own Gemini API key to run the agent. You can find your API key in your [Gemini account settings](https://gemini.com/account/settings). 2. Click 'Run Evaluation' to fetch questions, run the agent, and see the answers. 3. Click 'Submit All Answers' to submit the answers to the server. """ ) gr.LoginButton() api_key_input = gr.Textbox( label="Gemini API Key", placeholder="Enter your Gemini API key here", type="password", lines=1, visible=True ) run_button = gr.Button("Run Evaluation") save_button = gr.Button("Save Answers to File") submit_button = gr.Button("Submit All Answers") status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False) results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True) run_button.click( fn=run_all, inputs=[api_key_input], outputs=[status_output, results_table] ) save_button.click( fn=save_answers_to_file, outputs=[status_output] ) submit_button.click( fn=submit_all, outputs=[status_output] ) if __name__ == "__main__": print("\n" + "-"*30 + " App Starting " + "-"*30) # Check for SPACE_HOST and SPACE_ID at startup for information space_host_startup = os.getenv("SPACE_HOST") space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup if space_host_startup: print(f"✅ SPACE_HOST found: {space_host_startup}") print(f" Runtime URL should be: https://{space_host_startup}.hf.space") else: print("ℹ️ SPACE_HOST environment variable not found (running locally?).") if space_id_startup: # Print repo URLs if SPACE_ID is found print(f"✅ SPACE_ID found: {space_id_startup}") print(f" Repo URL: https://huggingface.co/spaces/{space_id_startup}") print(f" Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main") else: print("ℹ️ SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.") print(f"API KEY: {os.getenv('GOOGLE_API_KEY')}") print("-"*(60 + len(" App Starting ")) + "\n") print("Launching Gradio Interface for Basic Agent Evaluation...") demo.launch(debug=True, share=False)