Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -1,36 +1,53 @@
|
|
| 1 |
import os
|
|
|
|
|
|
|
| 2 |
import gradio as gr
|
| 3 |
import requests
|
| 4 |
-
import inspect
|
| 5 |
import pandas as pd
|
| 6 |
import tempfile
|
| 7 |
import threading
|
| 8 |
import queue
|
| 9 |
|
| 10 |
-
from smolagents import
|
| 11 |
|
| 12 |
-
# (Keep Constants as is)
|
| 13 |
# --- Constants ---
|
| 14 |
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
|
| 17 |
# --- Custom tools for reading task attachments ---
|
| 18 |
|
| 19 |
def _download_task_file(task_id: str) -> str:
|
| 20 |
-
"""Internal helper: downloads the file attached to a task_id and saves
|
| 21 |
-
it to a temp folder, returning the local file path (or '' if none)."""
|
| 22 |
url = f"{DEFAULT_API_URL}/files/{task_id}"
|
| 23 |
try:
|
| 24 |
response = requests.get(url, timeout=8)
|
| 25 |
if response.status_code != 200:
|
| 26 |
return ""
|
| 27 |
-
# Try to get a filename from the Content-Disposition header
|
| 28 |
cd = response.headers.get("content-disposition", "")
|
| 29 |
filename = task_id
|
| 30 |
if "filename=" in cd:
|
| 31 |
filename = cd.split("filename=")[-1].strip('"; ')
|
| 32 |
else:
|
| 33 |
-
# Guess an extension from content-type
|
| 34 |
ctype = response.headers.get("content-type", "")
|
| 35 |
if "spreadsheet" in ctype or "excel" in ctype:
|
| 36 |
filename = f"{task_id}.xlsx"
|
|
@@ -63,7 +80,7 @@ def download_task_file(task_id: str) -> str:
|
|
| 63 |
|
| 64 |
Returns:
|
| 65 |
The local file path where the file was saved, or 'NO_FILE_AVAILABLE' if there
|
| 66 |
-
is no file for this task_id
|
| 67 |
"""
|
| 68 |
result = _download_task_file(task_id)
|
| 69 |
return result if result else "NO_FILE_AVAILABLE"
|
|
@@ -71,8 +88,7 @@ def download_task_file(task_id: str) -> str:
|
|
| 71 |
|
| 72 |
@tool
|
| 73 |
def read_excel_file(file_path: str) -> str:
|
| 74 |
-
"""Reads an Excel (.xlsx/.xls) file and returns its content as readable text
|
| 75 |
-
(one table per sheet). Use this after downloading the file with download_task_file.
|
| 76 |
|
| 77 |
Args:
|
| 78 |
file_path: Local path to the Excel file.
|
|
@@ -143,94 +159,108 @@ def transcribe_audio_file(file_path: str) -> str:
|
|
| 143 |
return f"Error transcribing audio file: {e}"
|
| 144 |
|
| 145 |
|
| 146 |
-
# ---
|
| 147 |
-
|
| 148 |
class BasicAgent:
|
| 149 |
"""
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 154 |
"""
|
|
|
|
| 155 |
def __init__(self):
|
| 156 |
print("BasicAgent initializing...")
|
| 157 |
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
|
|
|
| 161 |
|
| 162 |
-
# Build
|
| 163 |
-
# automatically fails over to the next one if a call errors out
|
| 164 |
-
# (rate limit, quota exceeded, timeout, etc). This means a single
|
| 165 |
-
# exhausted free tier no longer kills the whole 20-question run -
|
| 166 |
-
# the router just moves to the next provider for the NEXT call.
|
| 167 |
model_list = []
|
| 168 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 169 |
if cerebras_key:
|
| 170 |
-
# Cerebras free tier: up to 1M tokens/day, fast, no waitlist -
|
| 171 |
-
# the most generous free option, so it goes first.
|
| 172 |
model_list.append({
|
| 173 |
"model_name": "agent-model",
|
| 174 |
"litellm_params": {
|
| 175 |
-
"model": "cerebras/
|
| 176 |
"api_key": cerebras_key,
|
| 177 |
"num_retries": 0,
|
| 178 |
-
"timeout":
|
| 179 |
},
|
| 180 |
})
|
| 181 |
-
|
| 182 |
-
|
|
|
|
| 183 |
model_list.append({
|
| 184 |
"model_name": "agent-model",
|
| 185 |
"litellm_params": {
|
| 186 |
-
"model": "
|
| 187 |
-
"api_key":
|
| 188 |
"num_retries": 0,
|
| 189 |
-
"timeout":
|
| 190 |
},
|
| 191 |
})
|
|
|
|
| 192 |
if gemini_key:
|
| 193 |
-
#
|
| 194 |
-
# at only 20 requests/day, so it's last among the paid-key options.
|
| 195 |
model_list.append({
|
| 196 |
"model_name": "agent-model",
|
| 197 |
"litellm_params": {
|
| 198 |
"model": "gemini/gemini-2.5-flash",
|
| 199 |
"api_key": gemini_key,
|
| 200 |
"num_retries": 0,
|
| 201 |
-
"timeout":
|
| 202 |
},
|
| 203 |
})
|
| 204 |
|
| 205 |
if model_list:
|
| 206 |
from smolagents import LiteLLMRouterModel
|
| 207 |
-
|
| 208 |
-
if cerebras_key:
|
| 209 |
-
provider_names.append("Cerebras")
|
| 210 |
-
if groq_key:
|
| 211 |
-
provider_names.append("Groq")
|
| 212 |
-
if gemini_key:
|
| 213 |
-
provider_names.append("Gemini")
|
| 214 |
-
print(f"Using router with fallback chain: {' -> '.join(provider_names)}")
|
| 215 |
self.model = LiteLLMRouterModel(
|
| 216 |
model_id="agent-model",
|
| 217 |
model_list=model_list,
|
| 218 |
client_kwargs={
|
| 219 |
-
"num_retries": 0,
|
| 220 |
-
"timeout":
|
| 221 |
"routing_strategy": "simple-shuffle",
|
| 222 |
},
|
| 223 |
)
|
| 224 |
else:
|
| 225 |
-
|
| 226 |
-
print("No CEREBRAS/GROQ/GEMINI API key set - falling back to HF Inference Providers.")
|
| 227 |
self.model = InferenceClientModel(
|
| 228 |
-
model_id="Qwen/Qwen2.5-
|
| 229 |
)
|
| 230 |
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
self.agent =
|
| 234 |
tools=[
|
| 235 |
WebSearchTool(),
|
| 236 |
download_task_file,
|
|
@@ -238,291 +268,222 @@ class BasicAgent:
|
|
| 238 |
read_csv_file,
|
| 239 |
read_text_file,
|
| 240 |
transcribe_audio_file,
|
| 241 |
-
PythonInterpreterTool(),
|
| 242 |
],
|
| 243 |
model=self.model,
|
| 244 |
-
|
| 245 |
-
# which has caused 150-800 second hangs in testing
|
| 246 |
-
additional_authorized_imports=[
|
| 247 |
-
"pandas", "numpy", "json", "re", "math", "datetime",
|
| 248 |
-
"openpyxl", "io", "csv",
|
| 249 |
-
],
|
| 250 |
-
max_steps=3, # keep this LOW for speed - we only need 30%, not perfection
|
| 251 |
)
|
| 252 |
|
| 253 |
-
# Extra safety net: if visit_webpage somehow still ended up in the
|
| 254 |
-
# toolbox (e.g. via a future smolagents default change), remove it.
|
| 255 |
-
if "visit_webpage" in self.agent.tools:
|
| 256 |
-
del self.agent.tools["visit_webpage"]
|
| 257 |
-
|
| 258 |
print("BasicAgent initialized.")
|
| 259 |
|
| 260 |
def __call__(self, question: str, task_id: str = "") -> str:
|
| 261 |
-
print(f"Agent received question (first
|
| 262 |
-
|
| 263 |
-
#
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
if
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
f"Reversed, it reads: {flipped}"
|
| 273 |
-
)
|
| 274 |
-
|
| 275 |
-
# Strong instruction to keep answers in the exact-match format
|
| 276 |
-
# the GAIA benchmark expects: no "FINAL ANSWER" prefix, no extra
|
| 277 |
-
# explanation, just the bare answer.
|
| 278 |
-
instructions = (
|
| 279 |
-
"You are a general AI assistant answering a benchmark question. "
|
| 280 |
-
"You have a STRICT step budget (max 3 steps) - be fast and efficient, "
|
| 281 |
-
"do not waste steps retrying things that already failed.\n\n"
|
| 282 |
-
"RULES TO SAVE STEPS AND TIME:\n"
|
| 283 |
-
"- PREFER web_search over visit_webpage in almost all cases - it is faster "
|
| 284 |
-
"and more reliable. Only use visit_webpage if web_search snippets are not "
|
| 285 |
-
"enough AND the URL is not youtube.com/youtu.be.\n"
|
| 286 |
-
"- NEVER call visit_webpage on a youtube.com/youtu.be URL - it always "
|
| 287 |
-
"fails with a connection error. For video questions, only use web_search "
|
| 288 |
-
"to find what others have already said about the video content.\n"
|
| 289 |
-
"- If download_task_file returns 'NO_FILE_AVAILABLE', do NOT call it "
|
| 290 |
-
"again - immediately move on to web_search instead.\n"
|
| 291 |
-
"- If visit_webpage returns a 403 or connection error, do NOT retry the "
|
| 292 |
-
"same URL - immediately try web_search instead.\n"
|
| 293 |
-
"- Answer in as few steps as possible - ideally in just 1 step if you "
|
| 294 |
-
"already know the answer or can compute it directly. Do not over-verify.\n\n"
|
| 295 |
-
f"The task_id for this question is '{task_id}'. If the question "
|
| 296 |
-
"mentions an attached file (Excel, CSV, audio, image, code, etc.), "
|
| 297 |
-
"call download_task_file('" + task_id + "') ONCE first to get its local "
|
| 298 |
-
"path, then use the matching reading tool (read_excel_file, "
|
| 299 |
-
"read_csv_file, read_text_file, or transcribe_audio_file) on that path.\n\n"
|
| 300 |
-
"Report your thoughts, then finish with the answer. "
|
| 301 |
-
"Your final output must be ONLY the answer itself: "
|
| 302 |
-
"no explanations, no extra words, no 'FINAL ANSWER' prefix. "
|
| 303 |
-
"If the answer is a number, write only the number (no units unless "
|
| 304 |
-
"explicitly requested). If it's a string, give the minimal exact phrase "
|
| 305 |
-
"requested, avoiding articles and abbreviations unless asked otherwise. "
|
| 306 |
-
"If it's a list, give a comma separated list following the same rules."
|
| 307 |
-
f"{reversed_hint}\n\n"
|
| 308 |
f"Question: {question}"
|
| 309 |
)
|
| 310 |
|
| 311 |
-
|
| 312 |
-
# a fixed number of seconds no matter what is happening internally
|
| 313 |
-
# (rate limit waits, hanging network calls, retries the library does
|
| 314 |
-
# on its own, etc). This guarantees the whole 20-question run can
|
| 315 |
-
# never stall for minutes/hours on a single question.
|
| 316 |
-
PER_QUESTION_TIMEOUT = 300 # seconds
|
| 317 |
|
| 318 |
result_queue: "queue.Queue" = queue.Queue()
|
| 319 |
|
| 320 |
-
def
|
| 321 |
try:
|
| 322 |
-
r = self.agent.run(
|
| 323 |
result_queue.put(("ok", r))
|
| 324 |
except Exception as exc:
|
| 325 |
result_queue.put(("error", exc))
|
| 326 |
|
| 327 |
-
worker = threading.Thread(target=
|
| 328 |
worker.start()
|
| 329 |
worker.join(timeout=PER_QUESTION_TIMEOUT)
|
| 330 |
|
| 331 |
if worker.is_alive():
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
|
|
|
|
|
|
|
|
|
| 339 |
else:
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-
|
| 344 |
-
if status == "ok":
|
| 345 |
-
answer = str(payload).strip()
|
| 346 |
-
else:
|
| 347 |
-
print(f"Agent error while answering: {payload}")
|
| 348 |
-
answer = "I don't know."
|
| 349 |
-
|
| 350 |
-
print(f"Agent returning answer: {answer}")
|
| 351 |
return answer
|
| 352 |
|
| 353 |
|
| 354 |
-
|
| 355 |
-
"""
|
| 356 |
-
Fetches all questions, runs the BasicAgent on them, submits all answers,
|
| 357 |
-
and displays the results.
|
| 358 |
-
"""
|
| 359 |
-
# --- Determine HF Space Runtime URL and Repo URL ---
|
| 360 |
-
space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code
|
| 361 |
|
| 362 |
-
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
|
| 367 |
-
|
|
|
|
|
|
|
| 368 |
|
| 369 |
-
api_url
|
| 370 |
questions_url = f"{api_url}/questions"
|
| 371 |
-
submit_url
|
| 372 |
|
| 373 |
-
#
|
| 374 |
try:
|
| 375 |
agent = BasicAgent()
|
| 376 |
except Exception as e:
|
| 377 |
-
print(f"Error instantiating agent: {e}")
|
| 378 |
return f"Error initializing agent: {e}", None
|
| 379 |
-
# In the case of an app running as a hugging Face space, this link points toward your codebase ( usefull for others so please keep it public)
|
| 380 |
-
agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
|
| 381 |
-
print(agent_code)
|
| 382 |
|
| 383 |
-
|
|
|
|
|
|
|
|
|
|
| 384 |
print(f"Fetching questions from: {questions_url}")
|
| 385 |
try:
|
| 386 |
-
|
| 387 |
-
|
| 388 |
-
questions_data =
|
| 389 |
if not questions_data:
|
| 390 |
-
|
| 391 |
-
return "Fetched questions list is empty or invalid format.", None
|
| 392 |
print(f"Fetched {len(questions_data)} questions.")
|
| 393 |
-
except requests.exceptions.RequestException as e:
|
| 394 |
-
print(f"Error fetching questions: {e}")
|
| 395 |
-
return f"Error fetching questions: {e}", None
|
| 396 |
-
except requests.exceptions.JSONDecodeError as e:
|
| 397 |
-
print(f"Error decoding JSON response from questions endpoint: {e}")
|
| 398 |
-
print(f"Response text: {response.text[:500]}")
|
| 399 |
-
return f"Error decoding server response for questions: {e}", None
|
| 400 |
except Exception as e:
|
| 401 |
-
|
| 402 |
-
return f"An unexpected error occurred fetching questions: {e}", None
|
| 403 |
|
| 404 |
-
#
|
| 405 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 406 |
answers_payload = []
|
| 407 |
-
|
| 408 |
for item in questions_data:
|
| 409 |
-
task_id
|
| 410 |
question_text = item.get("question")
|
| 411 |
if not task_id or question_text is None:
|
| 412 |
-
print(f"Skipping
|
| 413 |
continue
|
| 414 |
-
|
| 415 |
-
|
| 416 |
-
|
| 417 |
-
|
| 418 |
-
|
| 419 |
-
|
| 420 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 421 |
|
| 422 |
if not answers_payload:
|
| 423 |
-
|
| 424 |
-
return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
|
| 425 |
|
| 426 |
-
#
|
| 427 |
-
submission_data = {
|
| 428 |
-
|
| 429 |
-
|
|
|
|
|
|
|
|
|
|
| 430 |
|
| 431 |
-
# 5. Submit
|
| 432 |
-
print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
|
| 433 |
try:
|
| 434 |
-
|
| 435 |
-
|
| 436 |
-
result_data =
|
| 437 |
final_status = (
|
| 438 |
f"Submission Successful!\n"
|
| 439 |
f"User: {result_data.get('username')}\n"
|
| 440 |
-
f"
|
| 441 |
f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
|
| 442 |
f"Message: {result_data.get('message', 'No message received.')}"
|
| 443 |
)
|
| 444 |
print("Submission successful.")
|
| 445 |
-
|
| 446 |
-
|
| 447 |
except requests.exceptions.HTTPError as e:
|
| 448 |
-
|
| 449 |
try:
|
| 450 |
-
|
| 451 |
-
|
| 452 |
-
|
| 453 |
-
|
| 454 |
-
status_message = f"Submission Failed: {error_detail}"
|
| 455 |
-
print(status_message)
|
| 456 |
-
results_df = pd.DataFrame(results_log)
|
| 457 |
-
return status_message, results_df
|
| 458 |
-
except requests.exceptions.Timeout:
|
| 459 |
-
status_message = "Submission Failed: The request timed out."
|
| 460 |
-
print(status_message)
|
| 461 |
-
results_df = pd.DataFrame(results_log)
|
| 462 |
-
return status_message, results_df
|
| 463 |
-
except requests.exceptions.RequestException as e:
|
| 464 |
-
status_message = f"Submission Failed: Network error - {e}"
|
| 465 |
-
print(status_message)
|
| 466 |
-
results_df = pd.DataFrame(results_log)
|
| 467 |
-
return status_message, results_df
|
| 468 |
except Exception as e:
|
| 469 |
-
|
| 470 |
-
|
| 471 |
-
results_df = pd.DataFrame(results_log)
|
| 472 |
-
return status_message, results_df
|
| 473 |
|
|
|
|
| 474 |
|
| 475 |
-
# --- Build Gradio Interface using Blocks ---
|
| 476 |
with gr.Blocks() as demo:
|
| 477 |
-
gr.Markdown("#
|
| 478 |
gr.Markdown(
|
| 479 |
"""
|
| 480 |
**Instructions:**
|
| 481 |
-
|
| 482 |
-
|
| 483 |
-
|
| 484 |
-
|
| 485 |
-
|
| 486 |
-
-
|
| 487 |
-
|
| 488 |
-
|
| 489 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 490 |
"""
|
| 491 |
)
|
| 492 |
|
| 493 |
gr.LoginButton()
|
| 494 |
|
| 495 |
-
run_button
|
|
|
|
|
|
|
| 496 |
|
| 497 |
-
|
| 498 |
-
# Removed max_rows=10 from DataFrame constructor
|
| 499 |
-
results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
|
| 500 |
|
| 501 |
-
run_button.click(
|
| 502 |
-
fn=run_and_submit_all,
|
| 503 |
-
outputs=[status_output, results_table]
|
| 504 |
-
)
|
| 505 |
|
| 506 |
if __name__ == "__main__":
|
| 507 |
-
print("\n" + "-"*30 + " App Starting " + "-"*30)
|
| 508 |
-
# Check for SPACE_HOST and SPACE_ID at startup for information
|
| 509 |
-
space_host_startup = os.getenv("SPACE_HOST")
|
| 510 |
-
space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup
|
| 511 |
-
|
| 512 |
-
if space_host_startup:
|
| 513 |
-
print(f"✅ SPACE_HOST found: {space_host_startup}")
|
| 514 |
-
print(f" Runtime URL should be: https://{space_host_startup}.hf.space")
|
| 515 |
-
else:
|
| 516 |
-
print("ℹ️ SPACE_HOST environment variable not found (running locally?).")
|
| 517 |
|
| 518 |
-
|
| 519 |
-
|
| 520 |
-
|
| 521 |
-
|
|
|
|
| 522 |
else:
|
| 523 |
-
print("ℹ️
|
| 524 |
|
| 525 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 526 |
|
| 527 |
-
print("
|
|
|
|
| 528 |
demo.launch(debug=True, share=False, ssr_mode=False)
|
|
|
|
| 1 |
import os
|
| 2 |
+
import json
|
| 3 |
+
import time
|
| 4 |
import gradio as gr
|
| 5 |
import requests
|
|
|
|
| 6 |
import pandas as pd
|
| 7 |
import tempfile
|
| 8 |
import threading
|
| 9 |
import queue
|
| 10 |
|
| 11 |
+
from smolagents import ToolCallingAgent, InferenceClientModel, LiteLLMModel, WebSearchTool, tool
|
| 12 |
|
|
|
|
| 13 |
# --- Constants ---
|
| 14 |
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
|
| 15 |
+
CACHE_FILE = "/tmp/gaia_answers_cache.json"
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
# --- Cache helpers ---
|
| 19 |
+
|
| 20 |
+
def load_cache() -> dict:
|
| 21 |
+
if os.path.exists(CACHE_FILE):
|
| 22 |
+
try:
|
| 23 |
+
with open(CACHE_FILE, "r") as f:
|
| 24 |
+
return json.load(f)
|
| 25 |
+
except Exception:
|
| 26 |
+
return {}
|
| 27 |
+
return {}
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def save_cache(cache: dict):
|
| 31 |
+
try:
|
| 32 |
+
with open(CACHE_FILE, "w") as f:
|
| 33 |
+
json.dump(cache, f)
|
| 34 |
+
except Exception as e:
|
| 35 |
+
print(f"Warning: could not save cache: {e}")
|
| 36 |
|
| 37 |
|
| 38 |
# --- Custom tools for reading task attachments ---
|
| 39 |
|
| 40 |
def _download_task_file(task_id: str) -> str:
|
|
|
|
|
|
|
| 41 |
url = f"{DEFAULT_API_URL}/files/{task_id}"
|
| 42 |
try:
|
| 43 |
response = requests.get(url, timeout=8)
|
| 44 |
if response.status_code != 200:
|
| 45 |
return ""
|
|
|
|
| 46 |
cd = response.headers.get("content-disposition", "")
|
| 47 |
filename = task_id
|
| 48 |
if "filename=" in cd:
|
| 49 |
filename = cd.split("filename=")[-1].strip('"; ')
|
| 50 |
else:
|
|
|
|
| 51 |
ctype = response.headers.get("content-type", "")
|
| 52 |
if "spreadsheet" in ctype or "excel" in ctype:
|
| 53 |
filename = f"{task_id}.xlsx"
|
|
|
|
| 80 |
|
| 81 |
Returns:
|
| 82 |
The local file path where the file was saved, or 'NO_FILE_AVAILABLE' if there
|
| 83 |
+
is no file for this task_id.
|
| 84 |
"""
|
| 85 |
result = _download_task_file(task_id)
|
| 86 |
return result if result else "NO_FILE_AVAILABLE"
|
|
|
|
| 88 |
|
| 89 |
@tool
|
| 90 |
def read_excel_file(file_path: str) -> str:
|
| 91 |
+
"""Reads an Excel (.xlsx/.xls) file and returns its content as readable text.
|
|
|
|
| 92 |
|
| 93 |
Args:
|
| 94 |
file_path: Local path to the Excel file.
|
|
|
|
| 159 |
return f"Error transcribing audio file: {e}"
|
| 160 |
|
| 161 |
|
| 162 |
+
# --- Agent ---
|
| 163 |
+
|
| 164 |
class BasicAgent:
|
| 165 |
"""
|
| 166 |
+
Token-efficient agent for the GAIA benchmark.
|
| 167 |
+
|
| 168 |
+
Key optimizations vs the original:
|
| 169 |
+
- ToolCallingAgent instead of CodeAgent → ~40% fewer tokens per step
|
| 170 |
+
- Small/fast model first (Groq llama-3.1-8b-instant, free tier)
|
| 171 |
+
- Lean prompt (~80 tokens instead of ~400)
|
| 172 |
+
- Per-run answer cache so re-runs never re-spend tokens on answered questions
|
| 173 |
+
- Hard 120 s timeout per question (down from 300 s)
|
| 174 |
"""
|
| 175 |
+
|
| 176 |
def __init__(self):
|
| 177 |
print("BasicAgent initializing...")
|
| 178 |
|
| 179 |
+
groq_key = os.getenv("GROQ_API_KEY")
|
| 180 |
+
cerebras_key = os.getenv("CEREBRAS_API_KEY")
|
| 181 |
+
gemini_key = os.getenv("GEMINI_API_KEY")
|
| 182 |
+
anthropic_key = os.getenv("ANTHROPIC_API_KEY")
|
| 183 |
|
| 184 |
+
# Build priority list: cheapest/fastest first.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 185 |
model_list = []
|
| 186 |
|
| 187 |
+
if groq_key:
|
| 188 |
+
# Groq free tier — 70B first for quality, 8B as fallback when rate limited
|
| 189 |
+
model_list.append({
|
| 190 |
+
"model_name": "agent-model",
|
| 191 |
+
"litellm_params": {
|
| 192 |
+
"model": "groq/llama-3.3-70b-versatile",
|
| 193 |
+
"api_key": groq_key,
|
| 194 |
+
"num_retries": 0,
|
| 195 |
+
"timeout": 20,
|
| 196 |
+
},
|
| 197 |
+
})
|
| 198 |
+
model_list.append({
|
| 199 |
+
"model_name": "agent-model",
|
| 200 |
+
"litellm_params": {
|
| 201 |
+
"model": "groq/llama-3.1-8b-instant",
|
| 202 |
+
"api_key": groq_key,
|
| 203 |
+
"num_retries": 0,
|
| 204 |
+
"timeout": 15,
|
| 205 |
+
},
|
| 206 |
+
})
|
| 207 |
+
|
| 208 |
if cerebras_key:
|
|
|
|
|
|
|
| 209 |
model_list.append({
|
| 210 |
"model_name": "agent-model",
|
| 211 |
"litellm_params": {
|
| 212 |
+
"model": "cerebras/llama3.1-8b", # free, very fast
|
| 213 |
"api_key": cerebras_key,
|
| 214 |
"num_retries": 0,
|
| 215 |
+
"timeout": 15,
|
| 216 |
},
|
| 217 |
})
|
| 218 |
+
|
| 219 |
+
if anthropic_key:
|
| 220 |
+
# Haiku is Anthropic's cheapest model — ~25x cheaper than Sonnet.
|
| 221 |
model_list.append({
|
| 222 |
"model_name": "agent-model",
|
| 223 |
"litellm_params": {
|
| 224 |
+
"model": "anthropic/claude-haiku-4-5-20251001",
|
| 225 |
+
"api_key": anthropic_key,
|
| 226 |
"num_retries": 0,
|
| 227 |
+
"timeout": 20,
|
| 228 |
},
|
| 229 |
})
|
| 230 |
+
|
| 231 |
if gemini_key:
|
| 232 |
+
# gemini-2.5-flash: free tier, 15 RPM, 1500 RPD — gemini-2.0-flash was deprecated Jun 2026
|
|
|
|
| 233 |
model_list.append({
|
| 234 |
"model_name": "agent-model",
|
| 235 |
"litellm_params": {
|
| 236 |
"model": "gemini/gemini-2.5-flash",
|
| 237 |
"api_key": gemini_key,
|
| 238 |
"num_retries": 0,
|
| 239 |
+
"timeout": 20,
|
| 240 |
},
|
| 241 |
})
|
| 242 |
|
| 243 |
if model_list:
|
| 244 |
from smolagents import LiteLLMRouterModel
|
| 245 |
+
print(f"Router with {len(model_list)} model slots configured.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 246 |
self.model = LiteLLMRouterModel(
|
| 247 |
model_id="agent-model",
|
| 248 |
model_list=model_list,
|
| 249 |
client_kwargs={
|
| 250 |
+
"num_retries": 0,
|
| 251 |
+
"timeout": 30,
|
| 252 |
"routing_strategy": "simple-shuffle",
|
| 253 |
},
|
| 254 |
)
|
| 255 |
else:
|
| 256 |
+
print("No API keys found — falling back to HF Inference Providers (Qwen 7B).")
|
|
|
|
| 257 |
self.model = InferenceClientModel(
|
| 258 |
+
model_id="Qwen/Qwen2.5-7B-Instruct", # smaller than 32B
|
| 259 |
)
|
| 260 |
|
| 261 |
+
# ToolCallingAgent: generates only tool-call JSON, not full Python code.
|
| 262 |
+
# This alone cuts token usage by ~40% compared to CodeAgent.
|
| 263 |
+
self.agent = ToolCallingAgent(
|
| 264 |
tools=[
|
| 265 |
WebSearchTool(),
|
| 266 |
download_task_file,
|
|
|
|
| 268 |
read_csv_file,
|
| 269 |
read_text_file,
|
| 270 |
transcribe_audio_file,
|
|
|
|
| 271 |
],
|
| 272 |
model=self.model,
|
| 273 |
+
max_steps=4,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 274 |
)
|
| 275 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 276 |
print("BasicAgent initialized.")
|
| 277 |
|
| 278 |
def __call__(self, question: str, task_id: str = "") -> str:
|
| 279 |
+
print(f"Agent received question (first 60 chars): {question[:60]}...")
|
| 280 |
+
|
| 281 |
+
# Lean prompt — every token here is multiplied by max_steps calls.
|
| 282 |
+
prompt = (
|
| 283 |
+
"You are a precise AI assistant. Answer the question below with ONLY "
|
| 284 |
+
"the bare answer (no explanation, no preamble, no 'FINAL ANSWER' prefix). "
|
| 285 |
+
"Numbers: digits only unless units explicitly requested. "
|
| 286 |
+
"Strings: minimal exact phrase. Lists: comma-separated.\n"
|
| 287 |
+
f"task_id='{task_id}' — if the question mentions an attached file, "
|
| 288 |
+
"call download_task_file ONCE first, then the matching read tool.\n"
|
| 289 |
+
"Use web_search for factual lookups; never visit youtube URLs.\n\n"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 290 |
f"Question: {question}"
|
| 291 |
)
|
| 292 |
|
| 293 |
+
PER_QUESTION_TIMEOUT = 120 # seconds — tighter than original 300 s
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 294 |
|
| 295 |
result_queue: "queue.Queue" = queue.Queue()
|
| 296 |
|
| 297 |
+
def _run():
|
| 298 |
try:
|
| 299 |
+
r = self.agent.run(prompt)
|
| 300 |
result_queue.put(("ok", r))
|
| 301 |
except Exception as exc:
|
| 302 |
result_queue.put(("error", exc))
|
| 303 |
|
| 304 |
+
worker = threading.Thread(target=_run, daemon=True)
|
| 305 |
worker.start()
|
| 306 |
worker.join(timeout=PER_QUESTION_TIMEOUT)
|
| 307 |
|
| 308 |
if worker.is_alive():
|
| 309 |
+
print(f"Timeout after {PER_QUESTION_TIMEOUT}s — skipping question.")
|
| 310 |
+
return "I don't know."
|
| 311 |
+
|
| 312 |
+
try:
|
| 313 |
+
status, payload = result_queue.get_nowait()
|
| 314 |
+
except Exception:
|
| 315 |
+
return "I don't know."
|
| 316 |
+
|
| 317 |
+
if status == "ok":
|
| 318 |
+
answer = str(payload).strip()
|
| 319 |
else:
|
| 320 |
+
print(f"Agent error: {payload}")
|
| 321 |
+
answer = "I don't know."
|
| 322 |
+
|
| 323 |
+
print(f"Agent answer: {answer}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 324 |
return answer
|
| 325 |
|
| 326 |
|
| 327 |
+
# --- Main evaluation runner ---
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 328 |
|
| 329 |
+
def run_and_submit_all(profile: gr.OAuthProfile | None):
|
| 330 |
+
space_id = os.getenv("SPACE_ID")
|
| 331 |
+
|
| 332 |
+
if not profile:
|
| 333 |
+
return "Please log in to Hugging Face first.", None
|
| 334 |
+
|
| 335 |
+
username = profile.username
|
| 336 |
+
print(f"Logged in as: {username}")
|
| 337 |
|
| 338 |
+
api_url = DEFAULT_API_URL
|
| 339 |
questions_url = f"{api_url}/questions"
|
| 340 |
+
submit_url = f"{api_url}/submit"
|
| 341 |
|
| 342 |
+
# --- Instantiate agent ---
|
| 343 |
try:
|
| 344 |
agent = BasicAgent()
|
| 345 |
except Exception as e:
|
|
|
|
| 346 |
return f"Error initializing agent: {e}", None
|
|
|
|
|
|
|
|
|
|
| 347 |
|
| 348 |
+
agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main" if space_id else "unknown"
|
| 349 |
+
print(f"Agent code URL: {agent_code}")
|
| 350 |
+
|
| 351 |
+
# --- Fetch questions ---
|
| 352 |
print(f"Fetching questions from: {questions_url}")
|
| 353 |
try:
|
| 354 |
+
resp = requests.get(questions_url, timeout=15)
|
| 355 |
+
resp.raise_for_status()
|
| 356 |
+
questions_data = resp.json()
|
| 357 |
if not questions_data:
|
| 358 |
+
return "Fetched questions list is empty.", None
|
|
|
|
| 359 |
print(f"Fetched {len(questions_data)} questions.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 360 |
except Exception as e:
|
| 361 |
+
return f"Error fetching questions: {e}", None
|
|
|
|
| 362 |
|
| 363 |
+
# --- Load cache (avoids re-spending tokens on already-answered questions) ---
|
| 364 |
+
cache = load_cache()
|
| 365 |
+
print(f"Cache loaded: {len(cache)} previously answered questions.")
|
| 366 |
+
|
| 367 |
+
# --- Run agent ---
|
| 368 |
+
results_log = []
|
| 369 |
answers_payload = []
|
| 370 |
+
|
| 371 |
for item in questions_data:
|
| 372 |
+
task_id = item.get("task_id")
|
| 373 |
question_text = item.get("question")
|
| 374 |
if not task_id or question_text is None:
|
| 375 |
+
print(f"Skipping malformed item: {item}")
|
| 376 |
continue
|
| 377 |
+
|
| 378 |
+
if task_id in cache:
|
| 379 |
+
submitted_answer = cache[task_id]
|
| 380 |
+
print(f"[CACHE HIT] task_id={task_id} → {submitted_answer}")
|
| 381 |
+
else:
|
| 382 |
+
try:
|
| 383 |
+
submitted_answer = agent(question_text, task_id=task_id)
|
| 384 |
+
except Exception as e:
|
| 385 |
+
print(f"Error on task {task_id}: {e}")
|
| 386 |
+
submitted_answer = "I don't know."
|
| 387 |
+
cache[task_id] = submitted_answer
|
| 388 |
+
save_cache(cache) # persist after every answer so crashes don't lose progress
|
| 389 |
+
|
| 390 |
+
# Rate limit guard: 5s between questions keeps us under 12 req/min,
|
| 391 |
+
# safely below Gemini free tier's 15 RPM and Groq's burst limits.
|
| 392 |
+
print("Waiting 5s before next question (rate limit guard)...")
|
| 393 |
+
time.sleep(5)
|
| 394 |
+
|
| 395 |
+
answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
|
| 396 |
+
results_log.append({
|
| 397 |
+
"Task ID": task_id,
|
| 398 |
+
"Question": question_text,
|
| 399 |
+
"Submitted Answer": submitted_answer,
|
| 400 |
+
})
|
| 401 |
|
| 402 |
if not answers_payload:
|
| 403 |
+
return "Agent produced no answers.", pd.DataFrame(results_log)
|
|
|
|
| 404 |
|
| 405 |
+
# --- Submit ---
|
| 406 |
+
submission_data = {
|
| 407 |
+
"username": username.strip(),
|
| 408 |
+
"agent_code": agent_code,
|
| 409 |
+
"answers": answers_payload,
|
| 410 |
+
}
|
| 411 |
+
print(f"Submitting {len(answers_payload)} answers...")
|
| 412 |
|
|
|
|
|
|
|
| 413 |
try:
|
| 414 |
+
resp = requests.post(submit_url, json=submission_data, timeout=60)
|
| 415 |
+
resp.raise_for_status()
|
| 416 |
+
result_data = resp.json()
|
| 417 |
final_status = (
|
| 418 |
f"Submission Successful!\n"
|
| 419 |
f"User: {result_data.get('username')}\n"
|
| 420 |
+
f"Score: {result_data.get('score', 'N/A')}% "
|
| 421 |
f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
|
| 422 |
f"Message: {result_data.get('message', 'No message received.')}"
|
| 423 |
)
|
| 424 |
print("Submission successful.")
|
| 425 |
+
return final_status, pd.DataFrame(results_log)
|
| 426 |
+
|
| 427 |
except requests.exceptions.HTTPError as e:
|
| 428 |
+
detail = f"HTTP {e.response.status_code}"
|
| 429 |
try:
|
| 430 |
+
detail += f" — {e.response.json().get('detail', e.response.text)}"
|
| 431 |
+
except Exception:
|
| 432 |
+
detail += f" — {e.response.text[:300]}"
|
| 433 |
+
return f"Submission failed: {detail}", pd.DataFrame(results_log)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 434 |
except Exception as e:
|
| 435 |
+
return f"Submission failed: {e}", pd.DataFrame(results_log)
|
| 436 |
+
|
|
|
|
|
|
|
| 437 |
|
| 438 |
+
# --- Gradio UI ---
|
| 439 |
|
|
|
|
| 440 |
with gr.Blocks() as demo:
|
| 441 |
+
gr.Markdown("# GAIA Agent Evaluation Runner")
|
| 442 |
gr.Markdown(
|
| 443 |
"""
|
| 444 |
**Instructions:**
|
| 445 |
+
1. Clone this Space and add your API keys as Secrets (`GROQ_API_KEY`, `CEREBRAS_API_KEY`, `ANTHROPIC_API_KEY`, or `GEMINI_API_KEY`).
|
| 446 |
+
2. Log in with your Hugging Face account below.
|
| 447 |
+
3. Click **Run Evaluation & Submit All Answers**.
|
| 448 |
+
|
| 449 |
+
**Token-saving features in this version:**
|
| 450 |
+
- `ToolCallingAgent` instead of `CodeAgent` (~40% fewer tokens/step)
|
| 451 |
+
- Groq `llama-3.3-70b-versatile` as primary (free tier, 100k tokens/day)
|
| 452 |
+
- Groq `llama-3.1-8b-instant` as first fallback (free, very fast)
|
| 453 |
+
- Gemini `gemini-2.5-flash` as second fallback (free, 1500 req/day)
|
| 454 |
+
- Lean prompt (~80 tokens vs ~400 in the original)
|
| 455 |
+
- Answer cache: re-runs never re-spend tokens on already-answered questions
|
| 456 |
+
- 5s sleep between questions to avoid 429 rate limit errors
|
| 457 |
+
- 120s hard timeout per question
|
| 458 |
"""
|
| 459 |
)
|
| 460 |
|
| 461 |
gr.LoginButton()
|
| 462 |
|
| 463 |
+
run_button = gr.Button("Run Evaluation & Submit All Answers")
|
| 464 |
+
status_output = gr.Textbox(label="Status / Result", lines=6, interactive=False)
|
| 465 |
+
results_table = gr.DataFrame(label="Questions and Answers", wrap=True)
|
| 466 |
|
| 467 |
+
run_button.click(fn=run_and_submit_all, outputs=[status_output, results_table])
|
|
|
|
|
|
|
| 468 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 469 |
|
| 470 |
if __name__ == "__main__":
|
| 471 |
+
print("\n" + "-" * 30 + " App Starting " + "-" * 30)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 472 |
|
| 473 |
+
space_host = os.getenv("SPACE_HOST")
|
| 474 |
+
space_id = os.getenv("SPACE_ID")
|
| 475 |
+
|
| 476 |
+
if space_host:
|
| 477 |
+
print(f"✅ SPACE_HOST: {space_host}")
|
| 478 |
else:
|
| 479 |
+
print("ℹ️ SPACE_HOST not set (running locally?).")
|
| 480 |
|
| 481 |
+
if space_id:
|
| 482 |
+
print(f"✅ SPACE_ID: {space_id}")
|
| 483 |
+
print(f" Repo: https://huggingface.co/spaces/{space_id}/tree/main")
|
| 484 |
+
else:
|
| 485 |
+
print("ℹ️ SPACE_ID not set (running locally?).")
|
| 486 |
|
| 487 |
+
print("-" * (60 + len(" App Starting ")) + "\n")
|
| 488 |
+
print("Launching Gradio interface...")
|
| 489 |
demo.launch(debug=True, share=False, ssr_mode=False)
|