"""Run the agent over all GAIA questions locally WITHOUT submitting. Fetches the question set, answers each, prints a per-question log, and saves results to results.json so you can inspect them before any real submission. """ from __future__ import annotations import json import sys import time # Windows consoles default to cp1252; GAIA questions/answers contain Unicode # (e.g. macrons, accents). Force UTF-8 so printing never crashes the run. try: sys.stdout.reconfigure(encoding="utf-8", errors="replace") sys.stderr.reconfigure(encoding="utf-8", errors="replace") except Exception: # noqa: BLE001 pass from dotenv import load_dotenv load_dotenv() import requests from agent import GaiaAgent from tools import DEFAULT_API_URL QUESTIONS_URL = f"{DEFAULT_API_URL}/questions" def main() -> None: print(f"Fetching questions from {QUESTIONS_URL} ...") resp = requests.get(QUESTIONS_URL, timeout=30) resp.raise_for_status() questions = resp.json() print(f"Got {len(questions)} questions.\n") agent = GaiaAgent() results = [] t_start = time.time() for i, item in enumerate(questions, 1): task_id = item.get("task_id") question = item.get("question", "") has_file = bool(item.get("file_name")) print(f"[{i}/{len(questions)}] task {task_id} (file: {has_file})") print(f" Q: {question[:160]}{'...' if len(question) > 160 else ''}") t0 = time.time() try: answer = agent(question, task_id, item.get("file_name")) except Exception as exc: # noqa: BLE001 answer = f"AGENT ERROR: {exc}" dt = time.time() - t0 print(f" A: {answer!r} ({dt:.1f}s)\n") results.append( { "task_id": task_id, "question": question, "file_name": item.get("file_name", ""), "submitted_answer": answer, "seconds": round(dt, 1), } ) with open("results.json", "w", encoding="utf-8") as f: json.dump(results, f, indent=2, ensure_ascii=False) errors = sum(1 for r in results if r["submitted_answer"].startswith("AGENT ERROR")) blanks = sum(1 for r in results if not r["submitted_answer"].strip()) print("=" * 60) print(f"Done in {time.time() - t_start:.0f}s. {len(results)} answers.") print(f"Agent errors: {errors} Blank answers: {blanks}") print("Saved -> results.json (nothing was submitted)") if __name__ == "__main__": main()