Files changed (1) hide show
  1. app.py +183 -171
app.py CHANGED
@@ -1,196 +1,208 @@
 
 
 
 
 
 
 
1
  import os
 
 
 
2
  import gradio as gr
3
- import requests
4
- import inspect
5
  import pandas as pd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6
 
7
- # (Keep Constants as is)
8
- # --- Constants ---
9
- DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
10
 
11
- # --- Basic Agent Definition ---
12
- # ----- THIS IS WERE YOU CAN BUILD WHAT YOU WANT ------
13
  class BasicAgent:
14
  def __init__(self):
15
- print("BasicAgent initialized.")
16
- def __call__(self, question: str) -> str:
17
- print(f"Agent received question (first 50 chars): {question[:50]}...")
18
- fixed_answer = "This is a default answer."
19
- print(f"Agent returning fixed answer: {fixed_answer}")
20
- return fixed_answer
21
-
22
- def run_and_submit_all( profile: gr.OAuthProfile | None):
23
- """
24
- Fetches all questions, runs the BasicAgent on them, submits all answers,
25
- and displays the results.
26
- """
27
- # --- Determine HF Space Runtime URL and Repo URL ---
28
- space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code
29
-
30
- if profile:
31
- username= f"{profile.username}"
32
- print(f"User logged in: {username}")
33
- else:
34
- print("User not logged in.")
35
- return "Please Login to Hugging Face with the button.", None
36
-
37
- api_url = DEFAULT_API_URL
38
- questions_url = f"{api_url}/questions"
39
- submit_url = f"{api_url}/submit"
40
-
41
- # 1. Instantiate Agent ( modify this part to create your agent)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
42
  try:
 
 
 
 
 
43
  agent = BasicAgent()
44
- except Exception as e:
45
- print(f"Error instantiating agent: {e}")
46
- return f"Error initializing agent: {e}", None
47
- # In the case of an app running as a hugging Face space, this link points toward your codebase ( usefull for others so please keep it public)
48
- agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
49
- print(agent_code)
50
-
51
- # 2. Fetch Questions
52
- print(f"Fetching questions from: {questions_url}")
 
 
 
 
 
 
53
  try:
54
- response = requests.get(questions_url, timeout=15)
 
55
  response.raise_for_status()
56
- questions_data = response.json()
57
- if not questions_data:
58
- print("Fetched questions list is empty.")
59
- return "Fetched questions list is empty or invalid format.", None
60
- print(f"Fetched {len(questions_data)} questions.")
61
- except requests.exceptions.RequestException as e:
62
- print(f"Error fetching questions: {e}")
63
- return f"Error fetching questions: {e}", None
64
- except requests.exceptions.JSONDecodeError as e:
65
- print(f"Error decoding JSON response from questions endpoint: {e}")
66
- print(f"Response text: {response.text[:500]}")
67
- return f"Error decoding server response for questions: {e}", None
68
- except Exception as e:
69
- print(f"An unexpected error occurred fetching questions: {e}")
70
- return f"An unexpected error occurred fetching questions: {e}", None
71
-
72
- # 3. Run your Agent
73
- results_log = []
74
- answers_payload = []
75
- print(f"Running agent on {len(questions_data)} questions...")
76
- for item in questions_data:
77
  task_id = item.get("task_id")
78
- question_text = item.get("question")
79
- if not task_id or question_text is None:
80
- print(f"Skipping item with missing task_id or question: {item}")
81
  continue
82
  try:
83
- submitted_answer = agent(question_text)
84
- answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
85
- results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
86
- except Exception as e:
87
- print(f"Error running agent on task {task_id}: {e}")
88
- results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
89
-
90
- if not answers_payload:
91
- print("Agent did not produce any answers to submit.")
92
- return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
93
-
94
- # 4. Prepare Submission
95
- submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
96
- status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
97
- print(status_update)
98
-
99
- # 5. Submit
100
- print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
 
 
101
  try:
102
- response = requests.post(submit_url, json=submission_data, timeout=60)
103
  response.raise_for_status()
104
- result_data = response.json()
105
- final_status = (
106
- f"Submission Successful!\n"
107
- f"User: {result_data.get('username')}\n"
108
- f"Overall Score: {result_data.get('score', 'N/A')}% "
109
- f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
110
- f"Message: {result_data.get('message', 'No message received.')}"
 
111
  )
112
- print("Submission successful.")
113
- results_df = pd.DataFrame(results_log)
114
- return final_status, results_df
115
- except requests.exceptions.HTTPError as e:
116
- error_detail = f"Server responded with status {e.response.status_code}."
117
- try:
118
- error_json = e.response.json()
119
- error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
120
- except requests.exceptions.JSONDecodeError:
121
- error_detail += f" Response: {e.response.text[:500]}"
122
- status_message = f"Submission Failed: {error_detail}"
123
- print(status_message)
124
- results_df = pd.DataFrame(results_log)
125
- return status_message, results_df
126
- except requests.exceptions.Timeout:
127
- status_message = "Submission Failed: The request timed out."
128
- print(status_message)
129
- results_df = pd.DataFrame(results_log)
130
- return status_message, results_df
131
- except requests.exceptions.RequestException as e:
132
- status_message = f"Submission Failed: Network error - {e}"
133
- print(status_message)
134
- results_df = pd.DataFrame(results_log)
135
- return status_message, results_df
136
- except Exception as e:
137
- status_message = f"An unexpected error occurred during submission: {e}"
138
- print(status_message)
139
- results_df = pd.DataFrame(results_log)
140
- return status_message, results_df
141
-
142
-
143
- # --- Build Gradio Interface using Blocks ---
144
  with gr.Blocks() as demo:
145
- gr.Markdown("# Basic Agent Evaluation Runner")
146
  gr.Markdown(
147
- """
148
- **Instructions:**
149
-
150
- 1. Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ...
151
- 2. Log in to your Hugging Face account using the button below. This uses your HF username for submission.
152
- 3. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.
153
-
154
- ---
155
- **Disclaimers:**
156
- Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions).
157
- This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async.
158
- """
159
  )
160
-
161
  gr.LoginButton()
 
 
 
162
 
163
- run_button = gr.Button("Run Evaluation & Submit All Answers")
164
-
165
- status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
166
- # Removed max_rows=10 from DataFrame constructor
167
  results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
168
-
169
- run_button.click(
170
- fn=run_and_submit_all,
171
- outputs=[status_output, results_table]
172
- )
173
 
174
  if __name__ == "__main__":
175
- print("\n" + "-"*30 + " App Starting " + "-"*30)
176
- # Check for SPACE_HOST and SPACE_ID at startup for information
177
- space_host_startup = os.getenv("SPACE_HOST")
178
- space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup
179
-
180
- if space_host_startup:
181
- print(f"✅ SPACE_HOST found: {space_host_startup}")
182
- print(f" Runtime URL should be: https://{space_host_startup}.hf.space")
183
- else:
184
- print("ℹ️ SPACE_HOST environment variable not found (running locally?).")
185
-
186
- if space_id_startup: # Print repo URLs if SPACE_ID is found
187
- print(f"✅ SPACE_ID found: {space_id_startup}")
188
- print(f" Repo URL: https://huggingface.co/spaces/{space_id_startup}")
189
- print(f" Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")
190
- else:
191
- print("ℹ️ SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")
192
-
193
- print("-"*(60 + len(" App Starting ")) + "\n")
194
-
195
- print("Launching Gradio Interface for Basic Agent Evaluation...")
196
- demo.launch(debug=True, share=False)
 
1
+ """Hugging Face Agents Course Unit 4 - GAIA evaluation Space.
2
+
3
+ Based on the public Final_Assignment_Template API contract. It is a starter
4
+ agent, not a guaranteed passing solution. Keep your Space public for scoring.
5
+ """
6
+
7
+ import io
8
  import os
9
+ import re
10
+ from pathlib import Path
11
+
12
  import gradio as gr
 
 
13
  import pandas as pd
14
+ import requests
15
+ from smolagents import (
16
+ CodeAgent,
17
+ DuckDuckGoSearchTool,
18
+ InferenceClientModel,
19
+ VisitWebpageTool,
20
+ )
21
+
22
+ API_URL = "https://agents-course-unit4-scoring.hf.space"
23
+ MODEL_ID = os.getenv("MODEL_ID", "Qwen/Qwen3-Next-80B-A3B-Instruct")
24
+
25
+
26
+ def read_attachment(task_id: str, file_name: str) -> str:
27
+ """Download an official task attachment and extract readable text."""
28
+ if not file_name:
29
+ return ""
30
+ try:
31
+ response = requests.get(f"{API_URL}/files/{task_id}", timeout=30)
32
+ response.raise_for_status()
33
+ if len(response.content) > 15_000_000:
34
+ return "Attachment is too large to read with this starter."
35
+ suffix = Path(file_name).suffix.lower()
36
+ data = io.BytesIO(response.content)
37
+
38
+ if suffix in {".txt", ".md", ".py", ".json", ".html", ".xml"}:
39
+ contents = response.content.decode("utf-8", errors="replace")
40
+ elif suffix == ".pdf":
41
+ from pypdf import PdfReader
42
+ pdf = PdfReader(data)
43
+ contents = "\n".join(page.extract_text() or "" for page in pdf.pages[:40])
44
+ elif suffix == ".csv":
45
+ contents = pd.read_csv(data).head(100).to_string(index=False)
46
+ elif suffix in {".xlsx", ".xls"}:
47
+ sheets = pd.read_excel(data, sheet_name=None)
48
+ contents = "\n".join(
49
+ f"SHEET {name}:\n{frame.head(60).to_string(index=False)}"
50
+ for name, frame in sheets.items()
51
+ )
52
+ elif suffix == ".docx":
53
+ from docx import Document
54
+ doc = Document(data)
55
+ contents = "\n".join(p.text for p in doc.paragraphs)
56
+ else:
57
+ return (
58
+ f"Attachment '{file_name}' is present, but this starter cannot "
59
+ "interpret its media format. Do not guess its contents."
60
+ )
61
+ return f"Attachment ({file_name}):\n{contents[:14000]}"
62
+ except Exception as exc:
63
+ return f"Could not read attachment {file_name}: {type(exc).__name__}: {exc}"
64
 
 
 
 
65
 
 
 
66
  class BasicAgent:
67
  def __init__(self):
68
+ token = os.getenv("HF_TOKEN")
69
+ if not token:
70
+ raise RuntimeError(
71
+ "HF_TOKEN Secret is missing. Add it in Space Settings "
72
+ "-> Variables and secrets -> New secret."
73
+ )
74
+ model = InferenceClientModel(
75
+ model_id=MODEL_ID,
76
+ token=token,
77
+ timeout=120,
78
+ max_tokens=1800,
79
+ )
80
+ self.agent = CodeAgent(
81
+ tools=[DuckDuckGoSearchTool(max_results=5), VisitWebpageTool()],
82
+ model=model,
83
+ max_steps=6,
84
+ additional_authorized_imports=[
85
+ "math", "statistics", "datetime", "re", "json", "itertools"
86
+ ],
87
+ )
88
+
89
+ def __call__(self, question: str, attachment: str = "") -> str:
90
+ prompt = (
91
+ "You are solving a GAIA benchmark question. Use web search and "
92
+ "calculations if needed. Check evidence and do not invent facts. "
93
+ "Your final_answer must contain ONLY the shortest precise answer "
94
+ "requested by the question. No explanation, citations, markdown, "
95
+ "or the words FINAL ANSWER. Preserve requested units and format.\n\n"
96
+ f"QUESTION:\n{question}\n\n"
97
+ f"{attachment}"
98
+ )
99
+ answer = str(self.agent.run(prompt, reset=True)).strip()
100
+ answer = re.sub(r"^\s*(?:final answer|answer)\s*:\s*", "", answer, flags=re.I)
101
+ answer = re.sub(r"<think>.*?</think>", "", answer, flags=re.I | re.S).strip()
102
+ return answer.strip("` \n")
103
+
104
+
105
+ def answer_task(agent: BasicAgent, item: dict) -> str:
106
+ file_context = read_attachment(item["task_id"], item.get("file_name") or "")
107
+ return agent(item["question"], file_context)
108
+
109
+
110
+ def test_one_question():
111
+ """Try one official sample before running all 20 questions."""
112
  try:
113
+ response = requests.get(f"{API_URL}/random-question", timeout=20)
114
+ response.raise_for_status()
115
+ item = response.json()
116
+ if not isinstance(item, dict) or not item.get("task_id"):
117
+ return "The sample API returned an unexpected response."
118
  agent = BasicAgent()
119
+ answer = answer_task(agent, item)
120
+ return f"Question: {item['question']}\n\nAgent answer: {answer}"
121
+ except Exception as exc:
122
+ return f"Test failed: {type(exc).__name__}: {exc}"
123
+
124
+
125
+ def run_and_submit_all(profile: gr.OAuthProfile | None):
126
+ """Run the official GAIA question set and submit the generated answers."""
127
+ if not profile:
128
+ return "Please log in with the Hugging Face button first.", None
129
+ space_id = os.getenv("SPACE_ID")
130
+ if not space_id:
131
+ return "SPACE_ID is missing. Run this code inside your own Hugging Face Space.", None
132
+ username = profile.username
133
+
134
  try:
135
+ agent = BasicAgent()
136
+ response = requests.get(f"{API_URL}/questions", timeout=30)
137
  response.raise_for_status()
138
+ questions = response.json()
139
+ if not isinstance(questions, list) or not questions:
140
+ return "The question API did not return a question list.", None
141
+ except Exception as exc:
142
+ return f"Initialization failed: {type(exc).__name__}: {exc}", None
143
+
144
+ payload = []
145
+ rows = []
146
+ for index, item in enumerate(questions, 1):
 
 
 
 
 
 
 
 
 
 
 
 
147
  task_id = item.get("task_id")
148
+ question = item.get("question")
149
+ if not task_id or not question:
 
150
  continue
151
  try:
152
+ answer = answer_task(agent, item)
153
+ if not answer:
154
+ raise ValueError("The agent returned an empty answer")
155
+ payload.append({"task_id": task_id, "submitted_answer": answer})
156
+ rows.append({"Task ID": task_id, "Question": question, "Submitted Answer": answer})
157
+ print(f"Task {index}/{len(questions)} completed")
158
+ except Exception as exc:
159
+ error = f"ERROR: {type(exc).__name__}: {exc}"
160
+ print(f"Task {index}/{len(questions)} failed: {error}")
161
+ rows.append({"Task ID": task_id, "Question": question, "Submitted Answer": error})
162
+
163
+ table = pd.DataFrame(rows)
164
+ if not payload:
165
+ return "No answers generated. Check the Space logs and HF_TOKEN / model quota.", table
166
+
167
+ request_data = {
168
+ "username": username,
169
+ "agent_code": f"https://huggingface.co/spaces/{space_id}/tree/main",
170
+ "answers": payload,
171
+ }
172
  try:
173
+ response = requests.post(f"{API_URL}/submit", json=request_data, timeout=90)
174
  response.raise_for_status()
175
+ result = response.json()
176
+ status = (
177
+ "Submission Successful!\n"
178
+ f"User: {result.get('username', username)}\n"
179
+ f"Overall Score: {result.get('score', '?')}% "
180
+ f"({result.get('correct_count', '?')}/{result.get('total_attempted', '?')} correct)\n"
181
+ f"Submitted answers: {len(payload)}/{len(questions)}\n"
182
+ f"Message: {result.get('message', '')}"
183
  )
184
+ return status, table
185
+ except Exception as exc:
186
+ return f"Submission failed: {type(exc).__name__}: {exc}", table
187
+
188
+
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
189
  with gr.Blocks() as demo:
190
+ gr.Markdown("# Hugging Face Agents Course - Unit 4")
191
  gr.Markdown(
192
+ "First configure the **HF_TOKEN** Secret in your Space Settings. "
193
+ "Log in below for the leaderboard. Test one question before submitting all 20. "
194
+ "This starter handles web search and text-like attachments; image/audio "
195
+ "tasks need additional tools. Model calls may consume paid credits."
 
 
 
 
 
 
 
 
196
  )
 
197
  gr.LoginButton()
198
+ test_button = gr.Button("1. Test one question")
199
+ sample_output = gr.Textbox(label="Sample test result", lines=8)
200
+ test_button.click(fn=test_one_question, outputs=sample_output)
201
 
202
+ run_button = gr.Button("2. Run Evaluation & Submit All Answers", variant="primary")
203
+ status_output = gr.Textbox(label="Submission result", lines=6)
 
 
204
  results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
205
+ run_button.click(fn=run_and_submit_all, outputs=[status_output, results_table])
 
 
 
 
206
 
207
  if __name__ == "__main__":
208
+ demo.launch()