AlanRocha commited on
Commit
aa7dde1
·
verified ·
1 Parent(s): 9950dc7

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +220 -259
app.py CHANGED
@@ -1,36 +1,53 @@
1
  import os
 
 
2
  import gradio as gr
3
  import requests
4
- import inspect
5
  import pandas as pd
6
  import tempfile
7
  import threading
8
  import queue
9
 
10
- from smolagents import CodeAgent, InferenceClientModel, LiteLLMModel, WebSearchTool, tool
11
 
12
- # (Keep Constants as is)
13
  # --- Constants ---
14
  DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
15
 
16
 
17
  # --- Custom tools for reading task attachments ---
18
 
19
  def _download_task_file(task_id: str) -> str:
20
- """Internal helper: downloads the file attached to a task_id and saves
21
- it to a temp folder, returning the local file path (or '' if none)."""
22
  url = f"{DEFAULT_API_URL}/files/{task_id}"
23
  try:
24
  response = requests.get(url, timeout=8)
25
  if response.status_code != 200:
26
  return ""
27
- # Try to get a filename from the Content-Disposition header
28
  cd = response.headers.get("content-disposition", "")
29
  filename = task_id
30
  if "filename=" in cd:
31
  filename = cd.split("filename=")[-1].strip('"; ')
32
  else:
33
- # Guess an extension from content-type
34
  ctype = response.headers.get("content-type", "")
35
  if "spreadsheet" in ctype or "excel" in ctype:
36
  filename = f"{task_id}.xlsx"
@@ -63,7 +80,7 @@ def download_task_file(task_id: str) -> str:
63
 
64
  Returns:
65
  The local file path where the file was saved, or 'NO_FILE_AVAILABLE' if there
66
- is no file for this task_id (in that case, do not retry - use web_search instead).
67
  """
68
  result = _download_task_file(task_id)
69
  return result if result else "NO_FILE_AVAILABLE"
@@ -71,8 +88,7 @@ def download_task_file(task_id: str) -> str:
71
 
72
  @tool
73
  def read_excel_file(file_path: str) -> str:
74
- """Reads an Excel (.xlsx/.xls) file and returns its content as readable text
75
- (one table per sheet). Use this after downloading the file with download_task_file.
76
 
77
  Args:
78
  file_path: Local path to the Excel file.
@@ -143,94 +159,108 @@ def transcribe_audio_file(file_path: str) -> str:
143
  return f"Error transcribing audio file: {e}"
144
 
145
 
146
- # --- Basic Agent Definition ---
147
- # ----- THIS IS WERE YOU CAN BUILD WHAT YOU WANT ------
148
  class BasicAgent:
149
  """
150
- A real agent built with smolagents.
151
- Uses a free Hugging Face hosted model, web search, and a set of file
152
- tools (Excel/CSV/text/audio) to handle GAIA-style benchmark questions
153
- that come with an attachment.
 
 
 
 
154
  """
 
155
  def __init__(self):
156
  print("BasicAgent initializing...")
157
 
158
- gemini_key = os.getenv("GEMINI_API_KEY")
159
- groq_key = os.getenv("GROQ_API_KEY")
160
- cerebras_key = os.getenv("CEREBRAS_API_KEY")
 
161
 
162
- # Build a router that tries multiple free providers in order and
163
- # automatically fails over to the next one if a call errors out
164
- # (rate limit, quota exceeded, timeout, etc). This means a single
165
- # exhausted free tier no longer kills the whole 20-question run -
166
- # the router just moves to the next provider for the NEXT call.
167
  model_list = []
168
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
169
  if cerebras_key:
170
- # Cerebras free tier: up to 1M tokens/day, fast, no waitlist -
171
- # the most generous free option, so it goes first.
172
  model_list.append({
173
  "model_name": "agent-model",
174
  "litellm_params": {
175
- "model": "cerebras/gpt-oss-120b",
176
  "api_key": cerebras_key,
177
  "num_retries": 0,
178
- "timeout": 8,
179
  },
180
  })
181
- if groq_key:
182
- # Groq: generous free tier (100k tokens/day), very fast LPUs.
 
183
  model_list.append({
184
  "model_name": "agent-model",
185
  "litellm_params": {
186
- "model": "groq/llama-3.3-70b-versatile",
187
- "api_key": groq_key,
188
  "num_retries": 0,
189
- "timeout": 8,
190
  },
191
  })
 
192
  if gemini_key:
193
- # Gemini: good quality, but this account's free tier is capped
194
- # at only 20 requests/day, so it's last among the paid-key options.
195
  model_list.append({
196
  "model_name": "agent-model",
197
  "litellm_params": {
198
  "model": "gemini/gemini-2.5-flash",
199
  "api_key": gemini_key,
200
  "num_retries": 0,
201
- "timeout": 8,
202
  },
203
  })
204
 
205
  if model_list:
206
  from smolagents import LiteLLMRouterModel
207
- provider_names = []
208
- if cerebras_key:
209
- provider_names.append("Cerebras")
210
- if groq_key:
211
- provider_names.append("Groq")
212
- if gemini_key:
213
- provider_names.append("Gemini")
214
- print(f"Using router with fallback chain: {' -> '.join(provider_names)}")
215
  self.model = LiteLLMRouterModel(
216
  model_id="agent-model",
217
  model_list=model_list,
218
  client_kwargs={
219
- "num_retries": 0, # router-level: also fail fast
220
- "timeout": 20,
221
  "routing_strategy": "simple-shuffle",
222
  },
223
  )
224
  else:
225
- # Final fallback: free model hosted by Hugging Face Inference Providers.
226
- print("No CEREBRAS/GROQ/GEMINI API key set - falling back to HF Inference Providers.")
227
  self.model = InferenceClientModel(
228
- model_id="Qwen/Qwen2.5-Coder-32B-Instruct",
229
  )
230
 
231
- from smolagents import PythonInterpreterTool
232
-
233
- self.agent = CodeAgent(
234
  tools=[
235
  WebSearchTool(),
236
  download_task_file,
@@ -238,291 +268,222 @@ class BasicAgent:
238
  read_csv_file,
239
  read_text_file,
240
  transcribe_audio_file,
241
- PythonInterpreterTool(),
242
  ],
243
  model=self.model,
244
- add_base_tools=False, # we add tools manually below to EXCLUDE visit_webpage,
245
- # which has caused 150-800 second hangs in testing
246
- additional_authorized_imports=[
247
- "pandas", "numpy", "json", "re", "math", "datetime",
248
- "openpyxl", "io", "csv",
249
- ],
250
- max_steps=3, # keep this LOW for speed - we only need 30%, not perfection
251
  )
252
 
253
- # Extra safety net: if visit_webpage somehow still ended up in the
254
- # toolbox (e.g. via a future smolagents default change), remove it.
255
- if "visit_webpage" in self.agent.tools:
256
- del self.agent.tools["visit_webpage"]
257
-
258
  print("BasicAgent initialized.")
259
 
260
  def __call__(self, question: str, task_id: str = "") -> str:
261
- print(f"Agent received question (first 50 chars): {question[:50]}...")
262
-
263
- # Some GAIA questions are written backwards as a "riddle" test.
264
- # Detect this and flip it back before sending to the model.
265
- reversed_hint = ""
266
- if question.strip().endswith(".") and question.strip()[:1].islower():
267
- # crude heuristic: try reversing and see if it reads like English
268
- flipped = question.strip()[::-1]
269
- if flipped[:1].isupper() or flipped.split(" ")[0].isalpha():
270
- reversed_hint = (
271
- f"\n\nNote: this question may be written backwards. "
272
- f"Reversed, it reads: {flipped}"
273
- )
274
-
275
- # Strong instruction to keep answers in the exact-match format
276
- # the GAIA benchmark expects: no "FINAL ANSWER" prefix, no extra
277
- # explanation, just the bare answer.
278
- instructions = (
279
- "You are a general AI assistant answering a benchmark question. "
280
- "You have a STRICT step budget (max 3 steps) - be fast and efficient, "
281
- "do not waste steps retrying things that already failed.\n\n"
282
- "RULES TO SAVE STEPS AND TIME:\n"
283
- "- PREFER web_search over visit_webpage in almost all cases - it is faster "
284
- "and more reliable. Only use visit_webpage if web_search snippets are not "
285
- "enough AND the URL is not youtube.com/youtu.be.\n"
286
- "- NEVER call visit_webpage on a youtube.com/youtu.be URL - it always "
287
- "fails with a connection error. For video questions, only use web_search "
288
- "to find what others have already said about the video content.\n"
289
- "- If download_task_file returns 'NO_FILE_AVAILABLE', do NOT call it "
290
- "again - immediately move on to web_search instead.\n"
291
- "- If visit_webpage returns a 403 or connection error, do NOT retry the "
292
- "same URL - immediately try web_search instead.\n"
293
- "- Answer in as few steps as possible - ideally in just 1 step if you "
294
- "already know the answer or can compute it directly. Do not over-verify.\n\n"
295
- f"The task_id for this question is '{task_id}'. If the question "
296
- "mentions an attached file (Excel, CSV, audio, image, code, etc.), "
297
- "call download_task_file('" + task_id + "') ONCE first to get its local "
298
- "path, then use the matching reading tool (read_excel_file, "
299
- "read_csv_file, read_text_file, or transcribe_audio_file) on that path.\n\n"
300
- "Report your thoughts, then finish with the answer. "
301
- "Your final output must be ONLY the answer itself: "
302
- "no explanations, no extra words, no 'FINAL ANSWER' prefix. "
303
- "If the answer is a number, write only the number (no units unless "
304
- "explicitly requested). If it's a string, give the minimal exact phrase "
305
- "requested, avoiding articles and abbreviations unless asked otherwise. "
306
- "If it's a list, give a comma separated list following the same rules."
307
- f"{reversed_hint}\n\n"
308
  f"Question: {question}"
309
  )
310
 
311
- # HARD TIMEOUT: run the agent in a background thread and give up after
312
- # a fixed number of seconds no matter what is happening internally
313
- # (rate limit waits, hanging network calls, retries the library does
314
- # on its own, etc). This guarantees the whole 20-question run can
315
- # never stall for minutes/hours on a single question.
316
- PER_QUESTION_TIMEOUT = 300 # seconds
317
 
318
  result_queue: "queue.Queue" = queue.Queue()
319
 
320
- def _run_agent():
321
  try:
322
- r = self.agent.run(instructions)
323
  result_queue.put(("ok", r))
324
  except Exception as exc:
325
  result_queue.put(("error", exc))
326
 
327
- worker = threading.Thread(target=_run_agent, daemon=True)
328
  worker.start()
329
  worker.join(timeout=PER_QUESTION_TIMEOUT)
330
 
331
  if worker.is_alive():
332
- # Still running after the deadline - give up on this question and
333
- # move on. The thread is daemonized so it won't block process exit.
334
- print(
335
- f"Agent exceeded {PER_QUESTION_TIMEOUT}s hard timeout - "
336
- "abandoning this question and moving to the next one."
337
- )
338
- answer = "I don't know."
 
 
 
339
  else:
340
- try:
341
- status, payload = result_queue.get_nowait()
342
- except Exception:
343
- status, payload = "error", "no result produced"
344
- if status == "ok":
345
- answer = str(payload).strip()
346
- else:
347
- print(f"Agent error while answering: {payload}")
348
- answer = "I don't know."
349
-
350
- print(f"Agent returning answer: {answer}")
351
  return answer
352
 
353
 
354
- def run_and_submit_all( profile: gr.OAuthProfile | None):
355
- """
356
- Fetches all questions, runs the BasicAgent on them, submits all answers,
357
- and displays the results.
358
- """
359
- # --- Determine HF Space Runtime URL and Repo URL ---
360
- space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code
361
 
362
- if profile:
363
- username= f"{profile.username}"
364
- print(f"User logged in: {username}")
365
- else:
366
- print("User not logged in.")
367
- return "Please Login to Hugging Face with the button.", None
 
 
368
 
369
- api_url = DEFAULT_API_URL
370
  questions_url = f"{api_url}/questions"
371
- submit_url = f"{api_url}/submit"
372
 
373
- # 1. Instantiate Agent ( modify this part to create your agent)
374
  try:
375
  agent = BasicAgent()
376
  except Exception as e:
377
- print(f"Error instantiating agent: {e}")
378
  return f"Error initializing agent: {e}", None
379
- # In the case of an app running as a hugging Face space, this link points toward your codebase ( usefull for others so please keep it public)
380
- agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
381
- print(agent_code)
382
 
383
- # 2. Fetch Questions
 
 
 
384
  print(f"Fetching questions from: {questions_url}")
385
  try:
386
- response = requests.get(questions_url, timeout=15)
387
- response.raise_for_status()
388
- questions_data = response.json()
389
  if not questions_data:
390
- print("Fetched questions list is empty.")
391
- return "Fetched questions list is empty or invalid format.", None
392
  print(f"Fetched {len(questions_data)} questions.")
393
- except requests.exceptions.RequestException as e:
394
- print(f"Error fetching questions: {e}")
395
- return f"Error fetching questions: {e}", None
396
- except requests.exceptions.JSONDecodeError as e:
397
- print(f"Error decoding JSON response from questions endpoint: {e}")
398
- print(f"Response text: {response.text[:500]}")
399
- return f"Error decoding server response for questions: {e}", None
400
  except Exception as e:
401
- print(f"An unexpected error occurred fetching questions: {e}")
402
- return f"An unexpected error occurred fetching questions: {e}", None
403
 
404
- # 3. Run your Agent
405
- results_log = []
 
 
 
 
406
  answers_payload = []
407
- print(f"Running agent on {len(questions_data)} questions...")
408
  for item in questions_data:
409
- task_id = item.get("task_id")
410
  question_text = item.get("question")
411
  if not task_id or question_text is None:
412
- print(f"Skipping item with missing task_id or question: {item}")
413
  continue
414
- try:
415
- submitted_answer = agent(question_text, task_id=task_id)
416
- answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
417
- results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
418
- except Exception as e:
419
- print(f"Error running agent on task {task_id}: {e}")
420
- results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
421
 
422
  if not answers_payload:
423
- print("Agent did not produce any answers to submit.")
424
- return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
425
 
426
- # 4. Prepare Submission
427
- submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
428
- status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
429
- print(status_update)
 
 
 
430
 
431
- # 5. Submit
432
- print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
433
  try:
434
- response = requests.post(submit_url, json=submission_data, timeout=60)
435
- response.raise_for_status()
436
- result_data = response.json()
437
  final_status = (
438
  f"Submission Successful!\n"
439
  f"User: {result_data.get('username')}\n"
440
- f"Overall Score: {result_data.get('score', 'N/A')}% "
441
  f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
442
  f"Message: {result_data.get('message', 'No message received.')}"
443
  )
444
  print("Submission successful.")
445
- results_df = pd.DataFrame(results_log)
446
- return final_status, results_df
447
  except requests.exceptions.HTTPError as e:
448
- error_detail = f"Server responded with status {e.response.status_code}."
449
  try:
450
- error_json = e.response.json()
451
- error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
452
- except requests.exceptions.JSONDecodeError:
453
- error_detail += f" Response: {e.response.text[:500]}"
454
- status_message = f"Submission Failed: {error_detail}"
455
- print(status_message)
456
- results_df = pd.DataFrame(results_log)
457
- return status_message, results_df
458
- except requests.exceptions.Timeout:
459
- status_message = "Submission Failed: The request timed out."
460
- print(status_message)
461
- results_df = pd.DataFrame(results_log)
462
- return status_message, results_df
463
- except requests.exceptions.RequestException as e:
464
- status_message = f"Submission Failed: Network error - {e}"
465
- print(status_message)
466
- results_df = pd.DataFrame(results_log)
467
- return status_message, results_df
468
  except Exception as e:
469
- status_message = f"An unexpected error occurred during submission: {e}"
470
- print(status_message)
471
- results_df = pd.DataFrame(results_log)
472
- return status_message, results_df
473
 
 
474
 
475
- # --- Build Gradio Interface using Blocks ---
476
  with gr.Blocks() as demo:
477
- gr.Markdown("# Basic Agent Evaluation Runner")
478
  gr.Markdown(
479
  """
480
  **Instructions:**
481
-
482
- 1. Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ...
483
- 2. Log in to your Hugging Face account using the button below. This uses your HF username for submission.
484
- 3. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.
485
-
486
- ---
487
- **Disclaimers:**
488
- Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions).
489
- This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async.
 
 
 
 
490
  """
491
  )
492
 
493
  gr.LoginButton()
494
 
495
- run_button = gr.Button("Run Evaluation & Submit All Answers")
 
 
496
 
497
- status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
498
- # Removed max_rows=10 from DataFrame constructor
499
- results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
500
 
501
- run_button.click(
502
- fn=run_and_submit_all,
503
- outputs=[status_output, results_table]
504
- )
505
 
506
  if __name__ == "__main__":
507
- print("\n" + "-"*30 + " App Starting " + "-"*30)
508
- # Check for SPACE_HOST and SPACE_ID at startup for information
509
- space_host_startup = os.getenv("SPACE_HOST")
510
- space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup
511
-
512
- if space_host_startup:
513
- print(f"✅ SPACE_HOST found: {space_host_startup}")
514
- print(f" Runtime URL should be: https://{space_host_startup}.hf.space")
515
- else:
516
- print("ℹ️ SPACE_HOST environment variable not found (running locally?).")
517
 
518
- if space_id_startup: # Print repo URLs if SPACE_ID is found
519
- print(f"SPACE_ID found: {space_id_startup}")
520
- print(f" Repo URL: https://huggingface.co/spaces/{space_id_startup}")
521
- print(f" Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")
 
522
  else:
523
- print("ℹ️ SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")
524
 
525
- print("-"*(60 + len(" App Starting ")) + "\n")
 
 
 
 
526
 
527
- print("Launching Gradio Interface for Basic Agent Evaluation...")
 
528
  demo.launch(debug=True, share=False, ssr_mode=False)
 
1
  import os
2
+ import json
3
+ import time
4
  import gradio as gr
5
  import requests
 
6
  import pandas as pd
7
  import tempfile
8
  import threading
9
  import queue
10
 
11
+ from smolagents import ToolCallingAgent, InferenceClientModel, LiteLLMModel, WebSearchTool, tool
12
 
 
13
  # --- Constants ---
14
  DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
15
+ CACHE_FILE = "/tmp/gaia_answers_cache.json"
16
+
17
+
18
+ # --- Cache helpers ---
19
+
20
+ def load_cache() -> dict:
21
+ if os.path.exists(CACHE_FILE):
22
+ try:
23
+ with open(CACHE_FILE, "r") as f:
24
+ return json.load(f)
25
+ except Exception:
26
+ return {}
27
+ return {}
28
+
29
+
30
+ def save_cache(cache: dict):
31
+ try:
32
+ with open(CACHE_FILE, "w") as f:
33
+ json.dump(cache, f)
34
+ except Exception as e:
35
+ print(f"Warning: could not save cache: {e}")
36
 
37
 
38
  # --- Custom tools for reading task attachments ---
39
 
40
  def _download_task_file(task_id: str) -> str:
 
 
41
  url = f"{DEFAULT_API_URL}/files/{task_id}"
42
  try:
43
  response = requests.get(url, timeout=8)
44
  if response.status_code != 200:
45
  return ""
 
46
  cd = response.headers.get("content-disposition", "")
47
  filename = task_id
48
  if "filename=" in cd:
49
  filename = cd.split("filename=")[-1].strip('"; ')
50
  else:
 
51
  ctype = response.headers.get("content-type", "")
52
  if "spreadsheet" in ctype or "excel" in ctype:
53
  filename = f"{task_id}.xlsx"
 
80
 
81
  Returns:
82
  The local file path where the file was saved, or 'NO_FILE_AVAILABLE' if there
83
+ is no file for this task_id.
84
  """
85
  result = _download_task_file(task_id)
86
  return result if result else "NO_FILE_AVAILABLE"
 
88
 
89
  @tool
90
  def read_excel_file(file_path: str) -> str:
91
+ """Reads an Excel (.xlsx/.xls) file and returns its content as readable text.
 
92
 
93
  Args:
94
  file_path: Local path to the Excel file.
 
159
  return f"Error transcribing audio file: {e}"
160
 
161
 
162
+ # --- Agent ---
163
+
164
  class BasicAgent:
165
  """
166
+ Token-efficient agent for the GAIA benchmark.
167
+
168
+ Key optimizations vs the original:
169
+ - ToolCallingAgent instead of CodeAgent → ~40% fewer tokens per step
170
+ - Small/fast model first (Groq llama-3.1-8b-instant, free tier)
171
+ - Lean prompt (~80 tokens instead of ~400)
172
+ - Per-run answer cache so re-runs never re-spend tokens on answered questions
173
+ - Hard 120 s timeout per question (down from 300 s)
174
  """
175
+
176
  def __init__(self):
177
  print("BasicAgent initializing...")
178
 
179
+ groq_key = os.getenv("GROQ_API_KEY")
180
+ cerebras_key = os.getenv("CEREBRAS_API_KEY")
181
+ gemini_key = os.getenv("GEMINI_API_KEY")
182
+ anthropic_key = os.getenv("ANTHROPIC_API_KEY")
183
 
184
+ # Build priority list: cheapest/fastest first.
 
 
 
 
185
  model_list = []
186
 
187
+ if groq_key:
188
+ # Groq free tier — 70B first for quality, 8B as fallback when rate limited
189
+ model_list.append({
190
+ "model_name": "agent-model",
191
+ "litellm_params": {
192
+ "model": "groq/llama-3.3-70b-versatile",
193
+ "api_key": groq_key,
194
+ "num_retries": 0,
195
+ "timeout": 20,
196
+ },
197
+ })
198
+ model_list.append({
199
+ "model_name": "agent-model",
200
+ "litellm_params": {
201
+ "model": "groq/llama-3.1-8b-instant",
202
+ "api_key": groq_key,
203
+ "num_retries": 0,
204
+ "timeout": 15,
205
+ },
206
+ })
207
+
208
  if cerebras_key:
 
 
209
  model_list.append({
210
  "model_name": "agent-model",
211
  "litellm_params": {
212
+ "model": "cerebras/llama3.1-8b", # free, very fast
213
  "api_key": cerebras_key,
214
  "num_retries": 0,
215
+ "timeout": 15,
216
  },
217
  })
218
+
219
+ if anthropic_key:
220
+ # Haiku is Anthropic's cheapest model — ~25x cheaper than Sonnet.
221
  model_list.append({
222
  "model_name": "agent-model",
223
  "litellm_params": {
224
+ "model": "anthropic/claude-haiku-4-5-20251001",
225
+ "api_key": anthropic_key,
226
  "num_retries": 0,
227
+ "timeout": 20,
228
  },
229
  })
230
+
231
  if gemini_key:
232
+ # gemini-2.5-flash: free tier, 15 RPM, 1500 RPD gemini-2.0-flash was deprecated Jun 2026
 
233
  model_list.append({
234
  "model_name": "agent-model",
235
  "litellm_params": {
236
  "model": "gemini/gemini-2.5-flash",
237
  "api_key": gemini_key,
238
  "num_retries": 0,
239
+ "timeout": 20,
240
  },
241
  })
242
 
243
  if model_list:
244
  from smolagents import LiteLLMRouterModel
245
+ print(f"Router with {len(model_list)} model slots configured.")
 
 
 
 
 
 
 
246
  self.model = LiteLLMRouterModel(
247
  model_id="agent-model",
248
  model_list=model_list,
249
  client_kwargs={
250
+ "num_retries": 0,
251
+ "timeout": 30,
252
  "routing_strategy": "simple-shuffle",
253
  },
254
  )
255
  else:
256
+ print("No API keys found falling back to HF Inference Providers (Qwen 7B).")
 
257
  self.model = InferenceClientModel(
258
+ model_id="Qwen/Qwen2.5-7B-Instruct", # smaller than 32B
259
  )
260
 
261
+ # ToolCallingAgent: generates only tool-call JSON, not full Python code.
262
+ # This alone cuts token usage by ~40% compared to CodeAgent.
263
+ self.agent = ToolCallingAgent(
264
  tools=[
265
  WebSearchTool(),
266
  download_task_file,
 
268
  read_csv_file,
269
  read_text_file,
270
  transcribe_audio_file,
 
271
  ],
272
  model=self.model,
273
+ max_steps=4,
 
 
 
 
 
 
274
  )
275
 
 
 
 
 
 
276
  print("BasicAgent initialized.")
277
 
278
  def __call__(self, question: str, task_id: str = "") -> str:
279
+ print(f"Agent received question (first 60 chars): {question[:60]}...")
280
+
281
+ # Lean prompt every token here is multiplied by max_steps calls.
282
+ prompt = (
283
+ "You are a precise AI assistant. Answer the question below with ONLY "
284
+ "the bare answer (no explanation, no preamble, no 'FINAL ANSWER' prefix). "
285
+ "Numbers: digits only unless units explicitly requested. "
286
+ "Strings: minimal exact phrase. Lists: comma-separated.\n"
287
+ f"task_id='{task_id}' — if the question mentions an attached file, "
288
+ "call download_task_file ONCE first, then the matching read tool.\n"
289
+ "Use web_search for factual lookups; never visit youtube URLs.\n\n"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
290
  f"Question: {question}"
291
  )
292
 
293
+ PER_QUESTION_TIMEOUT = 120 # seconds tighter than original 300 s
 
 
 
 
 
294
 
295
  result_queue: "queue.Queue" = queue.Queue()
296
 
297
+ def _run():
298
  try:
299
+ r = self.agent.run(prompt)
300
  result_queue.put(("ok", r))
301
  except Exception as exc:
302
  result_queue.put(("error", exc))
303
 
304
+ worker = threading.Thread(target=_run, daemon=True)
305
  worker.start()
306
  worker.join(timeout=PER_QUESTION_TIMEOUT)
307
 
308
  if worker.is_alive():
309
+ print(f"Timeout after {PER_QUESTION_TIMEOUT}s skipping question.")
310
+ return "I don't know."
311
+
312
+ try:
313
+ status, payload = result_queue.get_nowait()
314
+ except Exception:
315
+ return "I don't know."
316
+
317
+ if status == "ok":
318
+ answer = str(payload).strip()
319
  else:
320
+ print(f"Agent error: {payload}")
321
+ answer = "I don't know."
322
+
323
+ print(f"Agent answer: {answer}")
 
 
 
 
 
 
 
324
  return answer
325
 
326
 
327
+ # --- Main evaluation runner ---
 
 
 
 
 
 
328
 
329
+ def run_and_submit_all(profile: gr.OAuthProfile | None):
330
+ space_id = os.getenv("SPACE_ID")
331
+
332
+ if not profile:
333
+ return "Please log in to Hugging Face first.", None
334
+
335
+ username = profile.username
336
+ print(f"Logged in as: {username}")
337
 
338
+ api_url = DEFAULT_API_URL
339
  questions_url = f"{api_url}/questions"
340
+ submit_url = f"{api_url}/submit"
341
 
342
+ # --- Instantiate agent ---
343
  try:
344
  agent = BasicAgent()
345
  except Exception as e:
 
346
  return f"Error initializing agent: {e}", None
 
 
 
347
 
348
+ agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main" if space_id else "unknown"
349
+ print(f"Agent code URL: {agent_code}")
350
+
351
+ # --- Fetch questions ---
352
  print(f"Fetching questions from: {questions_url}")
353
  try:
354
+ resp = requests.get(questions_url, timeout=15)
355
+ resp.raise_for_status()
356
+ questions_data = resp.json()
357
  if not questions_data:
358
+ return "Fetched questions list is empty.", None
 
359
  print(f"Fetched {len(questions_data)} questions.")
 
 
 
 
 
 
 
360
  except Exception as e:
361
+ return f"Error fetching questions: {e}", None
 
362
 
363
+ # --- Load cache (avoids re-spending tokens on already-answered questions) ---
364
+ cache = load_cache()
365
+ print(f"Cache loaded: {len(cache)} previously answered questions.")
366
+
367
+ # --- Run agent ---
368
+ results_log = []
369
  answers_payload = []
370
+
371
  for item in questions_data:
372
+ task_id = item.get("task_id")
373
  question_text = item.get("question")
374
  if not task_id or question_text is None:
375
+ print(f"Skipping malformed item: {item}")
376
  continue
377
+
378
+ if task_id in cache:
379
+ submitted_answer = cache[task_id]
380
+ print(f"[CACHE HIT] task_id={task_id} {submitted_answer}")
381
+ else:
382
+ try:
383
+ submitted_answer = agent(question_text, task_id=task_id)
384
+ except Exception as e:
385
+ print(f"Error on task {task_id}: {e}")
386
+ submitted_answer = "I don't know."
387
+ cache[task_id] = submitted_answer
388
+ save_cache(cache) # persist after every answer so crashes don't lose progress
389
+
390
+ # Rate limit guard: 5s between questions keeps us under 12 req/min,
391
+ # safely below Gemini free tier's 15 RPM and Groq's burst limits.
392
+ print("Waiting 5s before next question (rate limit guard)...")
393
+ time.sleep(5)
394
+
395
+ answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
396
+ results_log.append({
397
+ "Task ID": task_id,
398
+ "Question": question_text,
399
+ "Submitted Answer": submitted_answer,
400
+ })
401
 
402
  if not answers_payload:
403
+ return "Agent produced no answers.", pd.DataFrame(results_log)
 
404
 
405
+ # --- Submit ---
406
+ submission_data = {
407
+ "username": username.strip(),
408
+ "agent_code": agent_code,
409
+ "answers": answers_payload,
410
+ }
411
+ print(f"Submitting {len(answers_payload)} answers...")
412
 
 
 
413
  try:
414
+ resp = requests.post(submit_url, json=submission_data, timeout=60)
415
+ resp.raise_for_status()
416
+ result_data = resp.json()
417
  final_status = (
418
  f"Submission Successful!\n"
419
  f"User: {result_data.get('username')}\n"
420
+ f"Score: {result_data.get('score', 'N/A')}% "
421
  f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
422
  f"Message: {result_data.get('message', 'No message received.')}"
423
  )
424
  print("Submission successful.")
425
+ return final_status, pd.DataFrame(results_log)
426
+
427
  except requests.exceptions.HTTPError as e:
428
+ detail = f"HTTP {e.response.status_code}"
429
  try:
430
+ detail += f" — {e.response.json().get('detail', e.response.text)}"
431
+ except Exception:
432
+ detail += f" — {e.response.text[:300]}"
433
+ return f"Submission failed: {detail}", pd.DataFrame(results_log)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
434
  except Exception as e:
435
+ return f"Submission failed: {e}", pd.DataFrame(results_log)
436
+
 
 
437
 
438
+ # --- Gradio UI ---
439
 
 
440
  with gr.Blocks() as demo:
441
+ gr.Markdown("# GAIA Agent Evaluation Runner")
442
  gr.Markdown(
443
  """
444
  **Instructions:**
445
+ 1. Clone this Space and add your API keys as Secrets (`GROQ_API_KEY`, `CEREBRAS_API_KEY`, `ANTHROPIC_API_KEY`, or `GEMINI_API_KEY`).
446
+ 2. Log in with your Hugging Face account below.
447
+ 3. Click **Run Evaluation & Submit All Answers**.
448
+
449
+ **Token-saving features in this version:**
450
+ - `ToolCallingAgent` instead of `CodeAgent` (~40% fewer tokens/step)
451
+ - Groq `llama-3.3-70b-versatile` as primary (free tier, 100k tokens/day)
452
+ - Groq `llama-3.1-8b-instant` as first fallback (free, very fast)
453
+ - Gemini `gemini-2.5-flash` as second fallback (free, 1500 req/day)
454
+ - Lean prompt (~80 tokens vs ~400 in the original)
455
+ - Answer cache: re-runs never re-spend tokens on already-answered questions
456
+ - 5s sleep between questions to avoid 429 rate limit errors
457
+ - 120s hard timeout per question
458
  """
459
  )
460
 
461
  gr.LoginButton()
462
 
463
+ run_button = gr.Button("Run Evaluation & Submit All Answers")
464
+ status_output = gr.Textbox(label="Status / Result", lines=6, interactive=False)
465
+ results_table = gr.DataFrame(label="Questions and Answers", wrap=True)
466
 
467
+ run_button.click(fn=run_and_submit_all, outputs=[status_output, results_table])
 
 
468
 
 
 
 
 
469
 
470
  if __name__ == "__main__":
471
+ print("\n" + "-" * 30 + " App Starting " + "-" * 30)
 
 
 
 
 
 
 
 
 
472
 
473
+ space_host = os.getenv("SPACE_HOST")
474
+ space_id = os.getenv("SPACE_ID")
475
+
476
+ if space_host:
477
+ print(f"✅ SPACE_HOST: {space_host}")
478
  else:
479
+ print("ℹ️ SPACE_HOST not set (running locally?).")
480
 
481
+ if space_id:
482
+ print(f"✅ SPACE_ID: {space_id}")
483
+ print(f" Repo: https://huggingface.co/spaces/{space_id}/tree/main")
484
+ else:
485
+ print("ℹ️ SPACE_ID not set (running locally?).")
486
 
487
+ print("-" * (60 + len(" App Starting ")) + "\n")
488
+ print("Launching Gradio interface...")
489
  demo.launch(debug=True, share=False, ssr_mode=False)