Files changed (1) hide show
  1. app.py +1201 -134
app.py CHANGED
@@ -1,196 +1,1263 @@
1
  import os
 
 
 
2
  import gradio as gr
3
  import requests
4
- import inspect
5
  import pandas as pd
6
 
7
- # (Keep Constants as is)
8
- # --- Constants ---
 
 
 
9
  DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
10
 
11
- # --- Basic Agent Definition ---
12
- # ----- THIS IS WERE YOU CAN BUILD WHAT YOU WANT ------
13
- class BasicAgent:
14
- def __init__(self):
15
- print("BasicAgent initialized.")
16
- def __call__(self, question: str) -> str:
17
- print(f"Agent received question (first 50 chars): {question[:50]}...")
18
- fixed_answer = "This is a default answer."
19
- print(f"Agent returning fixed answer: {fixed_answer}")
20
- return fixed_answer
21
-
22
- def run_and_submit_all( profile: gr.OAuthProfile | None):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
23
  """
24
- Fetches all questions, runs the BasicAgent on them, submits all answers,
25
- and displays the results.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
26
  """
27
- # --- Determine HF Space Runtime URL and Repo URL ---
28
- space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
29
 
30
  if profile:
31
- username= f"{profile.username}"
32
- print(f"User logged in: {username}")
 
 
 
 
 
 
 
33
  else:
34
- print("User not logged in.")
35
- return "Please Login to Hugging Face with the button.", None
36
 
37
- api_url = DEFAULT_API_URL
38
- questions_url = f"{api_url}/questions"
39
- submit_url = f"{api_url}/submit"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
40
 
41
- # 1. Instantiate Agent ( modify this part to create your agent)
42
  try:
 
43
  agent = BasicAgent()
 
44
  except Exception as e:
45
- print(f"Error instantiating agent: {e}")
46
- return f"Error initializing agent: {e}", None
47
- # In the case of an app running as a hugging Face space, this link points toward your codebase ( usefull for others so please keep it public)
48
- agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
49
- print(agent_code)
50
-
51
- # 2. Fetch Questions
52
- print(f"Fetching questions from: {questions_url}")
 
 
 
 
 
 
53
  try:
54
- response = requests.get(questions_url, timeout=15)
55
- response.raise_for_status()
56
- questions_data = response.json()
57
- if not questions_data:
58
- print("Fetched questions list is empty.")
59
- return "Fetched questions list is empty or invalid format.", None
60
- print(f"Fetched {len(questions_data)} questions.")
61
- except requests.exceptions.RequestException as e:
62
- print(f"Error fetching questions: {e}")
63
- return f"Error fetching questions: {e}", None
64
- except requests.exceptions.JSONDecodeError as e:
65
- print(f"Error decoding JSON response from questions endpoint: {e}")
66
- print(f"Response text: {response.text[:500]}")
67
- return f"Error decoding server response for questions: {e}", None
68
  except Exception as e:
69
- print(f"An unexpected error occurred fetching questions: {e}")
70
- return f"An unexpected error occurred fetching questions: {e}", None
71
 
72
- # 3. Run your Agent
 
 
 
 
 
 
 
 
73
  results_log = []
 
74
  answers_payload = []
75
- print(f"Running agent on {len(questions_data)} questions...")
76
- for item in questions_data:
77
- task_id = item.get("task_id")
78
- question_text = item.get("question")
79
- if not task_id or question_text is None:
80
- print(f"Skipping item with missing task_id or question: {item}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
81
  continue
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
82
  try:
83
- submitted_answer = agent(question_text)
84
- answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
85
- results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
86
  except Exception as e:
87
- print(f"Error running agent on task {task_id}: {e}")
88
- results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
89
 
90
  if not answers_payload:
91
- print("Agent did not produce any answers to submit.")
92
- return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
93
 
94
- # 4. Prepare Submission
95
- submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
96
- status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
97
- print(status_update)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
98
 
99
- # 5. Submit
100
- print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
101
  try:
102
- response = requests.post(submit_url, json=submission_data, timeout=60)
 
 
 
 
 
 
103
  response.raise_for_status()
 
104
  result_data = response.json()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
105
  final_status = (
106
- f"Submission Successful!\n"
107
- f"User: {result_data.get('username')}\n"
108
- f"Overall Score: {result_data.get('score', 'N/A')}% "
109
- f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
110
- f"Message: {result_data.get('message', 'No message received.')}"
111
- )
112
- print("Submission successful.")
113
- results_df = pd.DataFrame(results_log)
114
- return final_status, results_df
 
 
 
 
 
 
 
 
 
 
 
115
  except requests.exceptions.HTTPError as e:
116
- error_detail = f"Server responded with status {e.response.status_code}."
 
 
 
 
117
  try:
118
- error_json = e.response.json()
119
- error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
120
- except requests.exceptions.JSONDecodeError:
121
- error_detail += f" Response: {e.response.text[:500]}"
122
- status_message = f"Submission Failed: {error_detail}"
123
- print(status_message)
124
- results_df = pd.DataFrame(results_log)
125
- return status_message, results_df
 
 
 
 
 
 
 
 
 
 
 
 
 
 
126
  except requests.exceptions.Timeout:
127
- status_message = "Submission Failed: The request timed out."
128
- print(status_message)
129
- results_df = pd.DataFrame(results_log)
130
- return status_message, results_df
 
 
 
 
 
 
131
  except requests.exceptions.RequestException as e:
132
- status_message = f"Submission Failed: Network error - {e}"
133
- print(status_message)
134
- results_df = pd.DataFrame(results_log)
135
- return status_message, results_df
 
 
 
 
136
  except Exception as e:
137
- status_message = f"An unexpected error occurred during submission: {e}"
138
- print(status_message)
139
- results_df = pd.DataFrame(results_log)
140
- return status_message, results_df
141
 
 
 
 
 
 
 
 
 
 
 
 
142
 
143
- # --- Build Gradio Interface using Blocks ---
144
  with gr.Blocks() as demo:
145
- gr.Markdown("# Basic Agent Evaluation Runner")
 
 
 
 
146
  gr.Markdown(
147
  """
148
- **Instructions:**
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
149
 
150
- 1. Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ...
151
- 2. Log in to your Hugging Face account using the button below. This uses your HF username for submission.
152
- 3. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.
153
 
154
- ---
155
- **Disclaimers:**
156
- Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions).
157
- This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async.
 
 
 
158
  """
159
  )
160
 
 
 
 
 
161
  gr.LoginButton()
162
 
163
- run_button = gr.Button("Run Evaluation & Submit All Answers")
 
 
 
164
 
165
- status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
166
- # Removed max_rows=10 from DataFrame constructor
167
- results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
 
 
 
 
 
 
 
168
 
169
  run_button.click(
170
  fn=run_and_submit_all,
171
- outputs=[status_output, results_table]
 
 
 
172
  )
173
 
 
 
 
 
 
174
  if __name__ == "__main__":
175
- print("\n" + "-"*30 + " App Starting " + "-"*30)
176
- # Check for SPACE_HOST and SPACE_ID at startup for information
177
- space_host_startup = os.getenv("SPACE_HOST")
178
- space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup
179
-
180
- if space_host_startup:
181
- print(f"βœ… SPACE_HOST found: {space_host_startup}")
182
- print(f" Runtime URL should be: https://{space_host_startup}.hf.space")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
183
  else:
184
- print("ℹ️ SPACE_HOST environment variable not found (running locally?).")
185
 
186
- if space_id_startup: # Print repo URLs if SPACE_ID is found
187
- print(f"βœ… SPACE_ID found: {space_id_startup}")
188
- print(f" Repo URL: https://huggingface.co/spaces/{space_id_startup}")
189
- print(f" Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
190
  else:
191
- print("ℹ️ SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")
192
 
193
- print("-"*(60 + len(" App Starting ")) + "\n")
 
 
 
 
 
 
194
 
195
- print("Launching Gradio Interface for Basic Agent Evaluation...")
196
- demo.launch(debug=True, share=False)
 
 
 
1
  import os
2
+ import tempfile
3
+ from pathlib import Path
4
+
5
  import gradio as gr
6
  import requests
 
7
  import pandas as pd
8
 
9
+
10
+ # ============================================================
11
+ # CONFIGURATION
12
+ # ============================================================
13
+
14
  DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
15
 
16
+ HF_TOKEN = os.getenv("HF_TOKEN")
17
+
18
+ # Model used by the agent.
19
+ # Hugging Face's current Agents Course examples use
20
+ # InferenceClientModel with CodeAgent.
21
+ MODEL_ID = os.getenv(
22
+ "MODEL_ID",
23
+ "Qwen/Qwen2.5-Coder-32B-Instruct"
24
+ )
25
+
26
+
27
+ # ============================================================
28
+ # SMOLAGENTS IMPORTS
29
+ # ============================================================
30
+
31
+ from smolagents import (
32
+ CodeAgent,
33
+ InferenceClientModel,
34
+ DuckDuckGoSearchTool,
35
+ PythonInterpreterTool,
36
+ VisitWebpageTool,
37
+ SpeechToTextTool,
38
+ Tool,
39
+ )
40
+
41
+
42
+ # ============================================================
43
+ # CUSTOM TOOL: READ LOCAL FILE
44
+ # ============================================================
45
+
46
+ class ReadLocalFileTool(Tool):
47
+
48
+ name = "read_local_file"
49
+
50
+ description = """
51
+ Read the contents of a local text, Python, CSV, JSON,
52
+ Markdown, or other text-based file.
53
+
54
+ Use this tool when a GAIA question provides a local file.
55
+ """
56
+
57
+ inputs = {
58
+ "file_path": {
59
+ "type": "string",
60
+ "description": "Path of the file to read."
61
+ }
62
+ }
63
+
64
+ output_type = "string"
65
+
66
+ def forward(self, file_path):
67
+
68
+ try:
69
+
70
+ path = Path(file_path)
71
+
72
+ if not path.exists():
73
+ return f"File does not exist: {file_path}"
74
+
75
+ # Limit extremely large files
76
+ text = path.read_text(
77
+ encoding="utf-8",
78
+ errors="ignore"
79
+ )
80
+
81
+ if len(text) > 50000:
82
+ text = text[:50000]
83
+
84
+ return text
85
+
86
+ except Exception as e:
87
+
88
+ return f"Could not read file: {e}"
89
+
90
+
91
+ # ============================================================
92
+ # CUSTOM TOOL: READ EXCEL
93
+ # ============================================================
94
+
95
+ class ReadExcelTool(Tool):
96
+
97
+ name = "read_excel_file"
98
+
99
+ description = """
100
+ Read an Excel XLSX file using pandas.
101
+
102
+ Returns sheet names and the contents of each sheet.
103
+ Use this for GAIA questions involving spreadsheets,
104
+ sales tables, numbers, or Excel data.
105
+ """
106
+
107
+ inputs = {
108
+ "file_path": {
109
+ "type": "string",
110
+ "description": "Path to an XLSX file."
111
+ }
112
+ }
113
+
114
+ output_type = "string"
115
+
116
+ def forward(self, file_path):
117
+
118
+ try:
119
+
120
+ path = Path(file_path)
121
+
122
+ if not path.exists():
123
+ return f"Excel file does not exist: {file_path}"
124
+
125
+ excel = pd.ExcelFile(path)
126
+
127
+ output = []
128
+
129
+ for sheet in excel.sheet_names:
130
+
131
+ df = pd.read_excel(
132
+ path,
133
+ sheet_name=sheet
134
+ )
135
+
136
+ output.append(
137
+ f"\n--- SHEET: {sheet} ---\n"
138
+ )
139
+
140
+ output.append(
141
+ df.to_string(index=False)
142
+ )
143
+
144
+ result = "\n".join(output)
145
+
146
+ if len(result) > 50000:
147
+ result = result[:50000]
148
+
149
+ return result
150
+
151
+ except Exception as e:
152
+
153
+ return f"Could not read Excel file: {e}"
154
+
155
+
156
+ # ============================================================
157
+ # CUSTOM TOOL: FILE INFORMATION
158
+ # ============================================================
159
+
160
+ class FileInfoTool(Tool):
161
+
162
+ name = "file_information"
163
+
164
+ description = """
165
+ Inspect a local file and return its name, extension,
166
+ size, and basic information.
167
  """
168
+
169
+ inputs = {
170
+ "file_path": {
171
+ "type": "string",
172
+ "description": "Path to the local file."
173
+ }
174
+ }
175
+
176
+ output_type = "string"
177
+
178
+ def forward(self, file_path):
179
+
180
+ try:
181
+
182
+ path = Path(file_path)
183
+
184
+ if not path.exists():
185
+ return "File does not exist."
186
+
187
+ size = path.stat().st_size
188
+
189
+ return (
190
+ f"File name: {path.name}\n"
191
+ f"Extension: {path.suffix}\n"
192
+ f"Size: {size} bytes\n"
193
+ f"Path: {path}"
194
+ )
195
+
196
+ except Exception as e:
197
+
198
+ return f"Error inspecting file: {e}"
199
+
200
+
201
+ # ============================================================
202
+ # CUSTOM TOOL: YOUTUBE TRANSCRIPT
203
+ # ============================================================
204
+
205
+ class YouTubeTranscriptTool(Tool):
206
+
207
+ name = "youtube_transcript"
208
+
209
+ description = """
210
+ Retrieve a transcript from a YouTube video when captions
211
+ are available.
212
+
213
+ Use this for questions asking what someone said in a
214
+ YouTube video or asking about spoken dialogue.
215
  """
216
+
217
+ inputs = {
218
+ "video_url": {
219
+ "type": "string",
220
+ "description": "Full YouTube video URL."
221
+ }
222
+ }
223
+
224
+ output_type = "string"
225
+
226
+ def forward(self, video_url):
227
+
228
+ try:
229
+
230
+ from youtube_transcript_api import (
231
+ YouTubeTranscriptApi
232
+ )
233
+
234
+ # Extract video ID
235
+ video_id = None
236
+
237
+ if "v=" in video_url:
238
+ video_id = video_url.split("v=")[1].split("&")[0]
239
+
240
+ elif "youtu.be/" in video_url:
241
+ video_id = video_url.split("youtu.be/")[1].split("?")[0]
242
+
243
+ elif "youtube.com/shorts/" in video_url:
244
+ video_id = video_url.split("youtube.com/shorts/")[1].split("?")[0]
245
+
246
+ if not video_id:
247
+ return "Could not extract YouTube video ID."
248
+
249
+ api = YouTubeTranscriptApi()
250
+
251
+ transcript = api.fetch(video_id)
252
+
253
+ text_parts = []
254
+
255
+ for item in transcript:
256
+
257
+ try:
258
+ text_parts.append(item.text)
259
+ except Exception:
260
+ text_parts.append(str(item))
261
+
262
+ result = " ".join(text_parts)
263
+
264
+ if len(result) > 60000:
265
+ result = result[:60000]
266
+
267
+ return result
268
+
269
+ except Exception as e:
270
+
271
+ return (
272
+ "Could not retrieve YouTube transcript. "
273
+ f"Reason: {e}"
274
+ )
275
+
276
+
277
+ # ============================================================
278
+ # GAIA AGENT
279
+ # ============================================================
280
+
281
+ class BasicAgent:
282
+
283
+ def __init__(self):
284
+
285
+ print("=" * 60)
286
+ print("Initializing GAIA Agent")
287
+ print("=" * 60)
288
+
289
+ # ----------------------------------------------------
290
+ # MODEL
291
+ # ----------------------------------------------------
292
+
293
+ model_kwargs = {
294
+ "model_id": MODEL_ID,
295
+ "temperature": 0.1,
296
+ "max_tokens": 4096,
297
+ }
298
+
299
+ if HF_TOKEN:
300
+ model_kwargs["token"] = HF_TOKEN
301
+
302
+ self.model = InferenceClientModel(
303
+ **model_kwargs
304
+ )
305
+
306
+ # ----------------------------------------------------
307
+ # TOOLS
308
+ # ----------------------------------------------------
309
+
310
+ tools = []
311
+
312
+ # Web search
313
+ try:
314
+
315
+ tools.append(
316
+ DuckDuckGoSearchTool(
317
+ max_results=8,
318
+ rate_limit=1.0
319
+ )
320
+ )
321
+
322
+ print("Web search tool enabled.")
323
+
324
+ except Exception as e:
325
+
326
+ print(
327
+ f"Could not initialize web search: {e}"
328
+ )
329
+
330
+ # Webpage reader
331
+ try:
332
+
333
+ tools.append(
334
+ VisitWebpageTool(
335
+ max_output_length=30000
336
+ )
337
+ )
338
+
339
+ print("Webpage tool enabled.")
340
+
341
+ except Exception as e:
342
+
343
+ print(
344
+ f"Could not initialize webpage tool: {e}"
345
+ )
346
+
347
+ # Python
348
+ try:
349
+
350
+ tools.append(
351
+ PythonInterpreterTool(
352
+ timeout_seconds=30
353
+ )
354
+ )
355
+
356
+ print("Python tool enabled.")
357
+
358
+ except Exception as e:
359
+
360
+ print(
361
+ f"Could not initialize Python tool: {e}"
362
+ )
363
+
364
+ # File reader
365
+ tools.append(
366
+ ReadLocalFileTool()
367
+ )
368
+
369
+ # Excel
370
+ tools.append(
371
+ ReadExcelTool()
372
+ )
373
+
374
+ # File information
375
+ tools.append(
376
+ FileInfoTool()
377
+ )
378
+
379
+ # YouTube transcript
380
+ tools.append(
381
+ YouTubeTranscriptTool()
382
+ )
383
+
384
+ # Speech-to-text
385
+ try:
386
+
387
+ tools.append(
388
+ SpeechToTextTool()
389
+ )
390
+
391
+ print(
392
+ "Speech-to-text tool enabled."
393
+ )
394
+
395
+ except Exception as e:
396
+
397
+ print(
398
+ f"Speech-to-text unavailable: {e}"
399
+ )
400
+
401
+ # ----------------------------------------------------
402
+ # CREATE CODE AGENT
403
+ # ----------------------------------------------------
404
+
405
+ self.agent = CodeAgent(
406
+ model=self.model,
407
+ tools=tools,
408
+ max_steps=12,
409
+ verbosity_level=1,
410
+ )
411
+
412
+ print(
413
+ f"GAIA Agent ready with {len(tools)} tools."
414
+ )
415
+
416
+ print("=" * 60)
417
+
418
+
419
+ # ========================================================
420
+ # RUN AGENT
421
+ # ========================================================
422
+
423
+ def __call__(
424
+ self,
425
+ question: str,
426
+ file_path: str | None = None
427
+ ) -> str:
428
+
429
+ print("\n")
430
+ print("=" * 60)
431
+ print("NEW GAIA QUESTION")
432
+ print("=" * 60)
433
+
434
+ print(question)
435
+
436
+ # ----------------------------------------------------
437
+ # Build task
438
+ # ----------------------------------------------------
439
+
440
+ task = f"""
441
+ You are an expert autonomous agent solving a GAIA Level 1
442
+ benchmark question.
443
+
444
+ Your goal is to produce the EXACT answer required by the
445
+ question.
446
+
447
+ QUESTION:
448
+
449
+ {question}
450
+
451
+ ------------------------------------------------------------
452
+
453
+ IMPORTANT RULES
454
+
455
+ 1. Carefully read the entire question.
456
+
457
+ 2. Determine exactly what the question is asking.
458
+
459
+ 3. If external information is required:
460
+ use web search.
461
+
462
+ 4. If a webpage must be inspected:
463
+ use the webpage tool.
464
+
465
+ 5. If calculations are required:
466
+ use Python.
467
+
468
+ 6. If a spreadsheet is provided:
469
+ inspect it with the Excel tool and Python/pandas.
470
+
471
+ 7. If a Python file is provided:
472
+ read and execute/analyze it with Python.
473
+
474
+ 8. If an audio file is provided:
475
+ use speech-to-text if necessary.
476
+
477
+ 9. If the question contains a YouTube URL:
478
+ try the YouTube transcript tool when the question
479
+ concerns spoken dialogue.
480
+
481
+ 10. Never guess when the information can be obtained
482
+ from a tool.
483
+
484
+ 11. Verify important calculations.
485
+
486
+ 12. Follow the requested output format EXACTLY.
487
+
488
+ 13. Pay attention to:
489
+ - capitalization
490
+ - commas
491
+ - ordering
492
+ - decimal places
493
+ - units
494
+ - first name vs surname
495
+ - city vs country
496
+ - IOC codes
497
+ - algebraic chess notation
498
+ - requested number of words
499
+
500
+ 14. Do not add explanations to the final answer.
501
+
502
+ 15. Do not write:
503
+ "The answer is..."
504
+
505
+ 16. Do not write:
506
+ "FINAL ANSWER"
507
+
508
+ 17. Return ONLY the answer requested by the question.
509
+
510
+ ------------------------------------------------------------
511
+ """
512
+
513
+ # ----------------------------------------------------
514
+ # Add file information
515
+ # ----------------------------------------------------
516
+
517
+ if file_path:
518
+
519
+ file_extension = Path(
520
+ file_path
521
+ ).suffix.lower()
522
+
523
+ task += f"""
524
+
525
+ A FILE IS ATTACHED TO THIS QUESTION.
526
+
527
+ Local file path:
528
+
529
+ {file_path}
530
+
531
+ File extension:
532
+
533
+ {file_extension}
534
+
535
+ You MUST inspect the file when it is relevant.
536
+
537
+ Available file-related tools include:
538
+
539
+ - file_information
540
+ - read_local_file
541
+ - read_excel_file
542
+ - Python
543
+
544
+ If the file is XLSX:
545
+ use read_excel_file and/or pandas.
546
+
547
+ If the file is Python:
548
+ read the code and execute/analyze it.
549
+
550
+ If the file is text:
551
+ read it.
552
+
553
+ If the file is audio:
554
+ use speech-to-text.
555
+
556
+ Do not ignore the attached file.
557
+ """
558
+
559
+ # ----------------------------------------------------
560
+ # Run
561
+ # ----------------------------------------------------
562
+
563
+ try:
564
+
565
+ result = self.agent.run(task)
566
+
567
+ answer = str(result).strip()
568
+
569
+ # ------------------------------------------------
570
+ # Clean accidental formatting
571
+ # ------------------------------------------------
572
+
573
+ answer = clean_final_answer(
574
+ answer
575
+ )
576
+
577
+ print(
578
+ "FINAL SUBMITTED ANSWER:"
579
+ )
580
+
581
+ print(answer)
582
+
583
+ print("=" * 60)
584
+
585
+ return answer
586
+
587
+ except Exception as e:
588
+
589
+ print(
590
+ f"Agent execution error: {e}"
591
+ )
592
+
593
+ return (
594
+ f"ERROR: {e}"
595
+ )
596
+
597
+
598
+ # ============================================================
599
+ # CLEAN FINAL ANSWER
600
+ # ============================================================
601
+
602
+ def clean_final_answer(answer: str) -> str:
603
+
604
+ answer = answer.strip()
605
+
606
+ # Remove common accidental prefixes
607
+
608
+ prefixes = [
609
+ "FINAL ANSWER:",
610
+ "FINAL ANSWER",
611
+ "Answer:",
612
+ "ANSWER:",
613
+ "The answer is:",
614
+ "The answer is"
615
+ ]
616
+
617
+ for prefix in prefixes:
618
+
619
+ if answer.lower().startswith(
620
+ prefix.lower()
621
+ ):
622
+
623
+ answer = answer[
624
+ len(prefix):
625
+ ].strip()
626
+
627
+ # Remove markdown code fences
628
+
629
+ if answer.startswith("```"):
630
+
631
+ lines = answer.splitlines()
632
+
633
+ if len(lines) >= 3:
634
+
635
+ lines = lines[1:-1]
636
+
637
+ answer = "\n".join(
638
+ lines
639
+ ).strip()
640
+
641
+ return answer
642
+
643
+
644
+ # ============================================================
645
+ # DOWNLOAD GAIA FILE
646
+ # ============================================================
647
+
648
+ def download_task_file(
649
+ task_id: str,
650
+ file_name: str
651
+ ):
652
+
653
+ if not file_name:
654
+
655
+ return None
656
+
657
+ try:
658
+
659
+ file_url = (
660
+ f"{DEFAULT_API_URL}/files/{task_id}"
661
+ )
662
+
663
+ print(
664
+ f"Downloading attachment from: {file_url}"
665
+ )
666
+
667
+ response = requests.get(
668
+ file_url,
669
+ timeout=60
670
+ )
671
+
672
+ response.raise_for_status()
673
+
674
+ # Temporary directory
675
+ temp_dir = (
676
+ Path(tempfile.gettempdir())
677
+ / "gaia_files"
678
+ )
679
+
680
+ temp_dir.mkdir(
681
+ parents=True,
682
+ exist_ok=True
683
+ )
684
+
685
+ safe_name = Path(
686
+ file_name
687
+ ).name
688
+
689
+ file_path = (
690
+ temp_dir / safe_name
691
+ )
692
+
693
+ file_path.write_bytes(
694
+ response.content
695
+ )
696
+
697
+ print(
698
+ f"Downloaded: {file_path}"
699
+ )
700
+
701
+ return str(file_path)
702
+
703
+ except Exception as e:
704
+
705
+ print(
706
+ f"File download failed: {e}"
707
+ )
708
+
709
+ return None
710
+
711
+
712
+ # ============================================================
713
+ # FETCH QUESTIONS
714
+ # ============================================================
715
+
716
+ def fetch_questions():
717
+
718
+ questions_url = (
719
+ f"{DEFAULT_API_URL}/questions"
720
+ )
721
+
722
+ print(
723
+ f"Fetching questions from: {questions_url}"
724
+ )
725
+
726
+ response = requests.get(
727
+ questions_url,
728
+ timeout=30
729
+ )
730
+
731
+ response.raise_for_status()
732
+
733
+ data = response.json()
734
+
735
+ if not isinstance(data, list):
736
+
737
+ raise ValueError(
738
+ "Questions API did not return a list."
739
+ )
740
+
741
+ print(
742
+ f"Fetched {len(data)} questions."
743
+ )
744
+
745
+ return data
746
+
747
+
748
+ # ============================================================
749
+ # RUN EVERYTHING AND SUBMIT
750
+ # ============================================================
751
+
752
+ def run_and_submit_all(
753
+ profile: gr.OAuthProfile | None
754
+ ):
755
+
756
+ # --------------------------------------------------------
757
+ # Check login
758
+ # --------------------------------------------------------
759
 
760
  if profile:
761
+
762
+ username = str(
763
+ profile.username
764
+ )
765
+
766
+ print(
767
+ f"Logged in user: {username}"
768
+ )
769
+
770
  else:
 
 
771
 
772
+ print(
773
+ "User is not logged in."
774
+ )
775
+
776
+ return (
777
+ "Please login to Hugging Face first.",
778
+ None
779
+ )
780
+
781
+ # --------------------------------------------------------
782
+ # Get Space ID
783
+ # --------------------------------------------------------
784
+
785
+ space_id = os.getenv(
786
+ "SPACE_ID"
787
+ )
788
+
789
+ if not space_id:
790
+
791
+ return (
792
+ "SPACE_ID environment variable was not found. "
793
+ "Run this inside your Hugging Face Space.",
794
+ None
795
+ )
796
+
797
+ agent_code = (
798
+ f"https://huggingface.co/spaces/"
799
+ f"{space_id}/tree/main"
800
+ )
801
+
802
+ print(
803
+ f"Agent code: {agent_code}"
804
+ )
805
+
806
+ # --------------------------------------------------------
807
+ # Initialize agent
808
+ # --------------------------------------------------------
809
 
 
810
  try:
811
+
812
  agent = BasicAgent()
813
+
814
  except Exception as e:
815
+
816
+ print(
817
+ f"Agent initialization failed: {e}"
818
+ )
819
+
820
+ return (
821
+ f"Agent initialization failed: {e}",
822
+ None
823
+ )
824
+
825
+ # --------------------------------------------------------
826
+ # Fetch questions
827
+ # --------------------------------------------------------
828
+
829
  try:
830
+
831
+ questions_data = (
832
+ fetch_questions()
833
+ )
834
+
 
 
 
 
 
 
 
 
 
835
  except Exception as e:
 
 
836
 
837
+ return (
838
+ f"Could not fetch questions: {e}",
839
+ None
840
+ )
841
+
842
+ # --------------------------------------------------------
843
+ # Process questions
844
+ # --------------------------------------------------------
845
+
846
  results_log = []
847
+
848
  answers_payload = []
849
+
850
+ total_questions = len(
851
+ questions_data
852
+ )
853
+
854
+ print(
855
+ f"Processing {total_questions} questions..."
856
+ )
857
+
858
+ for index, item in enumerate(
859
+ questions_data,
860
+ start=1
861
+ ):
862
+
863
+ print("\n")
864
+ print(
865
+ f"QUESTION {index}/{total_questions}"
866
+ )
867
+
868
+ task_id = item.get(
869
+ "task_id"
870
+ )
871
+
872
+ question_text = item.get(
873
+ "question"
874
+ )
875
+
876
+ file_name = item.get(
877
+ "file_name",
878
+ ""
879
+ )
880
+
881
+ # ----------------------------------------------------
882
+ # Validate
883
+ # ----------------------------------------------------
884
+
885
+ if not task_id:
886
+
887
+ print(
888
+ "Skipping: missing task_id"
889
+ )
890
+
891
  continue
892
+
893
+ if question_text is None:
894
+
895
+ print(
896
+ "Skipping: missing question"
897
+ )
898
+
899
+ continue
900
+
901
+ # ----------------------------------------------------
902
+ # Download attachment
903
+ # ----------------------------------------------------
904
+
905
+ file_path = None
906
+
907
+ if file_name:
908
+
909
+ file_path = download_task_file(
910
+ task_id,
911
+ file_name
912
+ )
913
+
914
+ # ----------------------------------------------------
915
+ # Run agent
916
+ # ----------------------------------------------------
917
+
918
  try:
919
+
920
+ submitted_answer = agent(
921
+ question_text,
922
+ file_path
923
+ )
924
+
925
+ # Safety conversion
926
+ submitted_answer = str(
927
+ submitted_answer
928
+ ).strip()
929
+
930
+ answers_payload.append(
931
+ {
932
+ "task_id": task_id,
933
+ "submitted_answer": submitted_answer
934
+ }
935
+ )
936
+
937
+ results_log.append(
938
+ {
939
+ "Task ID": task_id,
940
+ "Question": question_text,
941
+ "Submitted Answer": submitted_answer
942
+ }
943
+ )
944
+
945
+ print(
946
+ f"Question {index} completed."
947
+ )
948
+
949
  except Exception as e:
950
+
951
+ print(
952
+ f"Question failed: {e}"
953
+ )
954
+
955
+ results_log.append(
956
+ {
957
+ "Task ID": task_id,
958
+ "Question": question_text,
959
+ "Submitted Answer":
960
+ f"AGENT ERROR: {e}"
961
+ }
962
+ )
963
+
964
+ # --------------------------------------------------------
965
+ # Check answers
966
+ # --------------------------------------------------------
967
 
968
  if not answers_payload:
 
 
969
 
970
+ return (
971
+ "Agent produced no answers.",
972
+ pd.DataFrame(
973
+ results_log
974
+ )
975
+ )
976
+
977
+ # --------------------------------------------------------
978
+ # Submission
979
+ # --------------------------------------------------------
980
+
981
+ submission_data = {
982
+ "username": username.strip(),
983
+ "agent_code": agent_code,
984
+ "answers": answers_payload
985
+ }
986
+
987
+ submit_url = (
988
+ f"{DEFAULT_API_URL}/submit"
989
+ )
990
+
991
+ print("\n")
992
+ print("=" * 60)
993
+ print(
994
+ f"Submitting {len(answers_payload)} answers"
995
+ )
996
+ print("=" * 60)
997
 
 
 
998
  try:
999
+
1000
+ response = requests.post(
1001
+ submit_url,
1002
+ json=submission_data,
1003
+ timeout=300
1004
+ )
1005
+
1006
  response.raise_for_status()
1007
+
1008
  result_data = response.json()
1009
+
1010
+ score = result_data.get(
1011
+ "score",
1012
+ "N/A"
1013
+ )
1014
+
1015
+ correct_count = result_data.get(
1016
+ "correct_count",
1017
+ "?"
1018
+ )
1019
+
1020
+ total_attempted = result_data.get(
1021
+ "total_attempted",
1022
+ "?"
1023
+ )
1024
+
1025
+ message = result_data.get(
1026
+ "message",
1027
+ "No message."
1028
+ )
1029
+
1030
  final_status = (
1031
+ "Submission Successful!\n\n"
1032
+ f"User: {result_data.get('username', username)}\n"
1033
+ f"Score: {score}%\n"
1034
+ f"Correct: "
1035
+ f"{correct_count}/"
1036
+ f"{total_attempted}\n\n"
1037
+ f"Message: {message}"
1038
+ )
1039
+
1040
+ print(
1041
+ final_status
1042
+ )
1043
+
1044
+ return (
1045
+ final_status,
1046
+ pd.DataFrame(
1047
+ results_log
1048
+ )
1049
+ )
1050
+
1051
  except requests.exceptions.HTTPError as e:
1052
+
1053
+ detail = (
1054
+ f"HTTP error {e.response.status_code}"
1055
+ )
1056
+
1057
  try:
1058
+
1059
+ error_json = (
1060
+ e.response.json()
1061
+ )
1062
+
1063
+ detail += (
1064
+ f": {error_json}"
1065
+ )
1066
+
1067
+ except Exception:
1068
+
1069
+ detail += (
1070
+ f": {e.response.text[:1000]}"
1071
+ )
1072
+
1073
+ return (
1074
+ f"Submission failed: {detail}",
1075
+ pd.DataFrame(
1076
+ results_log
1077
+ )
1078
+ )
1079
+
1080
  except requests.exceptions.Timeout:
1081
+
1082
+ return (
1083
+ "Submission failed: "
1084
+ "request timed out. "
1085
+ "The agent may have taken too long.",
1086
+ pd.DataFrame(
1087
+ results_log
1088
+ )
1089
+ )
1090
+
1091
  except requests.exceptions.RequestException as e:
1092
+
1093
+ return (
1094
+ f"Submission network error: {e}",
1095
+ pd.DataFrame(
1096
+ results_log
1097
+ )
1098
+ )
1099
+
1100
  except Exception as e:
 
 
 
 
1101
 
1102
+ return (
1103
+ f"Unexpected submission error: {e}",
1104
+ pd.DataFrame(
1105
+ results_log
1106
+ )
1107
+ )
1108
+
1109
+
1110
+ # ============================================================
1111
+ # GRADIO INTERFACE
1112
+ # ============================================================
1113
 
 
1114
  with gr.Blocks() as demo:
1115
+
1116
+ gr.Markdown(
1117
+ "# πŸ€– GAIA Agent Evaluation"
1118
+ )
1119
+
1120
  gr.Markdown(
1121
  """
1122
+ ### Final Agent Assignment
1123
+
1124
+ This Space runs an autonomous agent against the
1125
+ GAIA evaluation questions.
1126
+
1127
+ **Capabilities**
1128
+
1129
+ - 🌐 Web search
1130
+ - πŸ”— Webpage reading
1131
+ - 🐍 Python calculations
1132
+ - πŸ“Š Excel analysis
1133
+ - πŸ“ File analysis
1134
+ - 🎡 Speech-to-text
1135
+ - ▢️ YouTube transcript extraction
1136
+ - πŸ€– Multi-step reasoning
1137
+
1138
+ **Important:** The final agent response is submitted
1139
+ as the answer, so it must follow the exact format
1140
+ requested by each question.
1141
+ """
1142
+ )
1143
 
1144
+ gr.Markdown(
1145
+ """
1146
+ ### Instructions
1147
 
1148
+ 1. Log in to Hugging Face.
1149
+ 2. Click **Run Evaluation & Submit All Answers**.
1150
+ 3. The agent will retrieve the GAIA questions.
1151
+ 4. The agent will solve each question.
1152
+ 5. Attachments will be downloaded automatically.
1153
+ 6. Answers will be submitted to the evaluation API.
1154
+ 7. Your score will appear below.
1155
  """
1156
  )
1157
 
1158
+ gr.Markdown(
1159
+ "---"
1160
+ )
1161
+
1162
  gr.LoginButton()
1163
 
1164
+ run_button = gr.Button(
1165
+ "πŸš€ Run Evaluation & Submit All Answers",
1166
+ variant="primary"
1167
+ )
1168
 
1169
+ status_output = gr.Textbox(
1170
+ label="Run Status / Submission Result",
1171
+ lines=8,
1172
+ interactive=False
1173
+ )
1174
+
1175
+ results_table = gr.DataFrame(
1176
+ label="Questions and Agent Answers",
1177
+ wrap=True
1178
+ )
1179
 
1180
  run_button.click(
1181
  fn=run_and_submit_all,
1182
+ outputs=[
1183
+ status_output,
1184
+ results_table
1185
+ ]
1186
  )
1187
 
1188
+
1189
+ # ============================================================
1190
+ # START APPLICATION
1191
+ # ============================================================
1192
+
1193
  if __name__ == "__main__":
1194
+
1195
+ print(
1196
+ "\n"
1197
+ + "-" * 30
1198
+ + " GAIA Agent Starting "
1199
+ + "-" * 30
1200
+ )
1201
+
1202
+ space_host = os.getenv(
1203
+ "SPACE_HOST"
1204
+ )
1205
+
1206
+ space_id = os.getenv(
1207
+ "SPACE_ID"
1208
+ )
1209
+
1210
+ if space_host:
1211
+
1212
+ print(
1213
+ f"SPACE_HOST: {space_host}"
1214
+ )
1215
+
1216
+ print(
1217
+ "Runtime URL: "
1218
+ f"https://{space_host}.hf.space"
1219
+ )
1220
+
1221
  else:
 
1222
 
1223
+ print(
1224
+ "SPACE_HOST not found."
1225
+ )
1226
+
1227
+ if space_id:
1228
+
1229
+ print(
1230
+ f"SPACE_ID: {space_id}"
1231
+ )
1232
+
1233
+ print(
1234
+ "Repository:"
1235
+ )
1236
+
1237
+ print(
1238
+ f"https://huggingface.co/spaces/{space_id}"
1239
+ )
1240
+
1241
+ print(
1242
+ "Code:"
1243
+ )
1244
+
1245
+ print(
1246
+ f"https://huggingface.co/spaces/"
1247
+ f"{space_id}/tree/main"
1248
+ )
1249
+
1250
  else:
 
1251
 
1252
+ print(
1253
+ "SPACE_ID not found."
1254
+ )
1255
+
1256
+ print(
1257
+ "-" * 70
1258
+ )
1259
 
1260
+ demo.launch(
1261
+ debug=True,
1262
+ share=False
1263
+ )