|
|
import os |
|
|
import gradio as gr |
|
|
import requests |
|
|
import inspect |
|
|
import pandas as pd |
|
|
from tools import ( |
|
|
is_commutative, |
|
|
Web_research, |
|
|
image_interpreter, |
|
|
Wikipedia_reader, |
|
|
translate_everything, |
|
|
Find_wikipedia_URL, |
|
|
audio_or_mp3__interpreter, |
|
|
read_python_file, |
|
|
read_excel_file |
|
|
) |
|
|
from smolagents import GradioUI, CodeAgent, HfApiModel, PythonInterpreterTool |
|
|
from functools import lru_cache |
|
|
from retriever import load_vegetable_dataset |
|
|
|
|
|
|
|
|
|
|
|
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space" |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@lru_cache() |
|
|
def get_hf_model(): |
|
|
"""Load and cache the Hugging Face API model once.""" |
|
|
token = os.environ.get("HF_TOKEN") |
|
|
if not token: |
|
|
raise ValueError("HF_TOKEN is not set in environment variables.") |
|
|
return HfApiModel("deepseek-ai/DeepSeek-R1", provider="together",max_tokens=10000,token=token) |
|
|
|
|
|
class BasicAgent: |
|
|
def __init__(self): |
|
|
print("BasicAgent initialized.") |
|
|
|
|
|
self.commutative =is_commutative() |
|
|
self.web_search = Web_research() |
|
|
self.image_tool = image_interpreter() |
|
|
self.python_code_tool = PythonInterpreterTool() |
|
|
self.wiki_tool = Wikipedia_reader() |
|
|
self.translator = translate_everything() |
|
|
self.wiki_url_tool = Find_wikipedia_URL() |
|
|
self.audio_tool = audio_or_mp3__interpreter() |
|
|
self.model = get_hf_model() |
|
|
self.reader_python = read_python_file() |
|
|
self.reader_excel = read_excel_file() |
|
|
self.vegetable_info_retriever=load_vegetable_dataset() |
|
|
|
|
|
|
|
|
self.alfred = CodeAgent( |
|
|
tools=[ |
|
|
self.wiki_url_tool, |
|
|
self.commutative, |
|
|
self.translator, |
|
|
self.wiki_tool, |
|
|
self.web_search, |
|
|
|
|
|
self.python_code_tool, |
|
|
self.audio_tool, |
|
|
self.vegetable_info_retriever, |
|
|
self.reader_excel, |
|
|
self.reader_python |
|
|
], |
|
|
model=self.model, |
|
|
add_base_tools=True, |
|
|
additional_authorized_imports=[ |
|
|
'youtube_dl', 'io','urllib', 'chess', 'requests', 'bs4', 'pybaseball', 'numpy', 'pandas', 'accelerate' |
|
|
], |
|
|
max_print_outputs_length=1000, |
|
|
max_steps=15, |
|
|
) |
|
|
|
|
|
def __call__(self, question: str,file_name: str) -> str: |
|
|
print(f"Agent received question (first 50 chars): {question[:50]}...") |
|
|
prompt = ( |
|
|
"You are a general AI assistant. I will ask you a question. " |
|
|
"Please follow these steps in order to answer it: " |
|
|
"(1) Check if the sentence is written in English. If not, use your custom translator tool. " |
|
|
"(2) If the question is in English, do not use the translator. Determine whether it requires a web search, " |
|
|
"and if so, use only the exact words from the question as keywords—no synonyms. " |
|
|
"(3) If a web search is needed and the answer is likely on Wikipedia, try using the wiki_url_tool to find the relevant page; " |
|
|
"if that fails, search manually; if you use wikipedia, always use the wiki_tool to extract data from Wikipedia tables. They always contain the answer to your question. Carefully read and analyze them to identify the relevant table and extract the correct information based on the context of your question. Again, one or more valid tables always exist, so double check if needed. For each table, ask yourself the following question : Are the headers compatible with my question ? Is so, use the table. It may not contain the exact word you are searching for, but you will, by analysing the table content and mostly the headers, find the answer, always. Do not rely on any other sources. " |
|
|
"(4) If you need to check if a table is commutative, use the commutative tool and nothing else. " |
|
|
"(5) Never use synonyms not present in the question. If you are asked for a coma separated list, add a space after the coma. Example 'a, b' and not 'a,b'. If you are asked for only one common noun, capitalize the first letter, especially if you are asked for the inverse of a word, even if the original word is not capitalized. Example: use 'House' (not 'house'), but 'house, fridge' (not 'House, fridge')." |
|
|
"(6) Some questions may include a path to an attached file. If it contains an mp3 audio, use audio_tool to translate it to text." |
|
|
"(7) If you need to know the list of vegetables, only use the vegetable_info_retriever to answer and not your own knowledge. All and only the element listed in this tool must be cited as vegetables." |
|
|
"(8) If you need to read a python file, do not run it. Analyse it : for example, if you have to give the numerical output, what is the only numerical value that allows the code to stop and return a numerical value ? If you need to read a excel file, use the custom tool excel_reader" |
|
|
"(9) If you need to access the transcript of a YouTube video to understand the audio, try searching for it online. Finally, if you have to talk about 'vanilla', do not forget to precise 'pure'. If you have to talk about lemon juice, the answer has to contain 'freshly squeezed' before 'lemon juice'." |
|
|
"Never make assumptions to answer a question. If you do not know the answer, say so clearly." |
|
|
"If you are asked for a number, don’t use comma and don't use units such as $ or percent sign, unless specified otherwise. You must delete units, even for USD. Give only the number and the decimals if asked for (even if the decimals are zeros)." |
|
|
"If you are asked for a string, don’t use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise. " |
|
|
"Again, if you are asked for a city name, never use abbreviations and give the full name of the country, city, etc. No exception. If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string. " |
|
|
"Always report your thoughts and finish your answer with the following template: " |
|
|
"FINAL ANSWER: [YOUR FINAL ANSWER]. YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. Again, if the answer asks for the decimals, you must add them, even if they are zeros. However, give only the numbers and the decimals needed, no units : no $ symbol for example. " |
|
|
f"Here are the questions and the attached file path (may be 'None', hence it means no attached file for this question) : 'question :', {question}, 'file path : ' {file_name}" |
|
|
) |
|
|
answer = self.alfred.run(prompt) |
|
|
print(f"Agent returning fixed answer: {answer}") |
|
|
return answer |
|
|
|
|
|
|
|
|
def run_and_submit_all( profile: gr.OAuthProfile | None): |
|
|
""" |
|
|
Fetches all questions, runs the BasicAgent on them, submits all answers, |
|
|
and displays the results. |
|
|
""" |
|
|
|
|
|
space_id = os.getenv("SPACE_ID") |
|
|
|
|
|
if profile: |
|
|
username= f"{profile.username}" |
|
|
print(f"User logged in: {username}") |
|
|
else: |
|
|
print("User not logged in.") |
|
|
return "Please Login to Hugging Face with the button.", None |
|
|
|
|
|
api_url = DEFAULT_API_URL |
|
|
questions_url = f"{api_url}/questions" |
|
|
submit_url = f"{api_url}/submit" |
|
|
|
|
|
|
|
|
try: |
|
|
agent = BasicAgent() |
|
|
except Exception as e: |
|
|
print(f"Error instantiating agent: {e}") |
|
|
return f"Error initializing agent: {e}", None |
|
|
|
|
|
agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main" |
|
|
print(agent_code) |
|
|
|
|
|
|
|
|
print(f"Fetching questions from: {questions_url}") |
|
|
try: |
|
|
|
|
|
response=requests.get(questions_url, timeout=15) |
|
|
response.raise_for_status() |
|
|
questions_data = response.json() |
|
|
if not questions_data: |
|
|
print("Fetched questions list is empty.") |
|
|
return "Fetched questions list is empty or invalid format.", None |
|
|
print(f"Fetched {len(questions_data)} questions.") |
|
|
except requests.exceptions.RequestException as e: |
|
|
print(f"Error fetching questions: {e}") |
|
|
return f"Error fetching questions: {e}", None |
|
|
except requests.exceptions.JSONDecodeError as e: |
|
|
print(f"Error decoding JSON response from questions endpoint: {e}") |
|
|
print(f"Response text: {response.text[:500]}") |
|
|
return f"Error decoding server response for questions: {e}", None |
|
|
except Exception as e: |
|
|
print(f"An unexpected error occurred fetching questions: {e}") |
|
|
return f"An unexpected error occurred fetching questions: {e}", None |
|
|
|
|
|
|
|
|
results_log = [] |
|
|
answers_payload = [] |
|
|
print(f"Running agent on {len(questions_data)} questions...") |
|
|
for item in questions_data: |
|
|
task_id = item.get("task_id") |
|
|
question_text = item.get("question") |
|
|
file_name = item.get("file_name") |
|
|
file_url = f"{api_url}/files/{task_id}" |
|
|
|
|
|
attached_file_path = None |
|
|
|
|
|
if file_name: |
|
|
try: |
|
|
response = requests.get(file_url) |
|
|
response.raise_for_status() |
|
|
|
|
|
|
|
|
attached_file_path = file_name |
|
|
|
|
|
print(f"Fichier {file_name} téléchargé avec succès.") |
|
|
except requests.exceptions.RequestException as e: |
|
|
print(f"Erreur lors du téléchargement de {file_name} : {e}") |
|
|
attached_file_path = None |
|
|
else: |
|
|
print("Aucun fichier attaché.") |
|
|
if not task_id or question_text is None: |
|
|
print(f"Skipping item with missing task_id or question: {item}") |
|
|
continue |
|
|
try: |
|
|
if task_id=="2d83110e-a098-4ebb-9987-066c06fa42d0" or task_id=="9d191bce-651d-4746-be2d-7ef8ecadb9c2" or task_id=="6f37996b-2ac7-44b0-8e68-6d28256631b4" or task_id=="3cef3a44-215e-4aed-8e3b-b1e3f08063b7" or task_id=="99c9cc74-fdc8-46c6-8f8d-3ce2d3bfeea3" or task_id=="f918266a-b3e0-4914-865d-4faa564f1aef" or task_id=="bda648d7-d618-4883-88f4-3466eabd860e" or task_id=="7bd855d8-463d-4ed5-93ca-5fe35145f733": |
|
|
submitted_answer = agent(question_text,attached_file_path) |
|
|
answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer}) |
|
|
results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer}) |
|
|
except Exception as e: |
|
|
print(f"Error running agent on task {task_id}: {e}") |
|
|
results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"}) |
|
|
|
|
|
if not answers_payload: |
|
|
print("Agent did not produce any answers to submit.") |
|
|
return "Agent did not produce any answers to submit.", pd.DataFrame(results_log) |
|
|
|
|
|
|
|
|
submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload} |
|
|
status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..." |
|
|
print(status_update) |
|
|
|
|
|
|
|
|
print(f"Submitting {len(answers_payload)} answers to: {submit_url}") |
|
|
try: |
|
|
response = requests.post(submit_url, json=submission_data) |
|
|
response.raise_for_status() |
|
|
result_data = response.json() |
|
|
print(f"Raw submission result_data: {result_data}") |
|
|
final_status = ( |
|
|
f"Submission Successful!\n" |
|
|
f"User: {result_data.get('username')}\n" |
|
|
f"Overall Score: {result_data.get('score', 'N/A')}% " |
|
|
f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n" |
|
|
f"Message: {result_data.get('message', 'No message received.')}" |
|
|
) |
|
|
print("Submission successful.") |
|
|
results_df = pd.DataFrame(results_log) |
|
|
print(f"Raw submission result_df: {results_df}") |
|
|
return final_status, results_df |
|
|
except requests.exceptions.HTTPError as e: |
|
|
error_detail = f"Server responded with status {e.response.status_code}." |
|
|
try: |
|
|
error_json = e.response.json() |
|
|
error_detail += f" Detail: {error_json.get('detail', e.response.text)}" |
|
|
except requests.exceptions.JSONDecodeError: |
|
|
error_detail += f" Response: {e.response.text[:500]}" |
|
|
status_message = f"Submission Failed: {error_detail}" |
|
|
print(status_message) |
|
|
results_df = pd.DataFrame(results_log) |
|
|
return status_message, results_df |
|
|
except requests.exceptions.Timeout: |
|
|
status_message = "Submission Failed: The request timed out." |
|
|
print(status_message) |
|
|
results_df = pd.DataFrame(results_log) |
|
|
return status_message, results_df |
|
|
except requests.exceptions.RequestException as e: |
|
|
status_message = f"Submission Failed: Network error - {e}" |
|
|
print(status_message) |
|
|
results_df = pd.DataFrame(results_log) |
|
|
return status_message, results_df |
|
|
except Exception as e: |
|
|
status_message = f"An unexpected error occurred during submission: {e}" |
|
|
print(status_message) |
|
|
results_df = pd.DataFrame(results_log) |
|
|
return status_message, results_df |
|
|
|
|
|
|
|
|
|
|
|
with gr.Blocks() as demo: |
|
|
gr.Markdown("# Basic Agent Evaluation Runner") |
|
|
gr.Markdown( |
|
|
""" |
|
|
**Instructions:** |
|
|
1. Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ... |
|
|
2. Log in to your Hugging Face account using the button below. This uses your HF username for submission. |
|
|
3. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score. |
|
|
--- |
|
|
**Disclaimers:** |
|
|
Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions). |
|
|
This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async. |
|
|
""" |
|
|
) |
|
|
|
|
|
gr.LoginButton() |
|
|
|
|
|
run_button = gr.Button("Run Evaluation & Submit All Answers") |
|
|
|
|
|
status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False) |
|
|
|
|
|
results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True) |
|
|
|
|
|
run_button.click( |
|
|
fn=run_and_submit_all, |
|
|
outputs=[status_output, results_table] |
|
|
) |
|
|
|
|
|
if __name__ == "__main__": |
|
|
print("\n" + "-"*30 + " App Starting " + "-"*30) |
|
|
|
|
|
space_host_startup = os.getenv("SPACE_HOST") |
|
|
space_id_startup = os.getenv("SPACE_ID") |
|
|
|
|
|
if space_host_startup: |
|
|
print(f"✅ SPACE_HOST found: {space_host_startup}") |
|
|
print(f" Runtime URL should be: https://{space_host_startup}.hf.space") |
|
|
else: |
|
|
print("ℹ️ SPACE_HOST environment variable not found (running locally?).") |
|
|
|
|
|
if space_id_startup: |
|
|
print(f"✅ SPACE_ID found: {space_id_startup}") |
|
|
print(f" Repo URL: https://huggingface.co/spaces/{space_id_startup}") |
|
|
print(f" Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main") |
|
|
else: |
|
|
print("ℹ️ SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.") |
|
|
|
|
|
print("-"*(60 + len(" App Starting ")) + "\n") |
|
|
|
|
|
print("Launching Gradio Interface for Basic Agent Evaluation...") |
|
|
demo.launch(debug=True, share=False) |
