run_batch.py

"""Run the agent over all GAIA questions locally WITHOUT submitting.

Fetches the question set, answers each, prints a per-question log, and saves
results to results.json so you can inspect them before any real submission.
"""

from __future__ import annotations

import json
import sys
import time

# Windows consoles default to cp1252; GAIA questions/answers contain Unicode
# (e.g. macrons, accents). Force UTF-8 so printing never crashes the run.
try:
    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
    sys.stderr.reconfigure(encoding="utf-8", errors="replace")
except Exception:  # noqa: BLE001
    pass

from dotenv import load_dotenv

load_dotenv()

import requests

from agent import GaiaAgent
from tools import DEFAULT_API_URL

QUESTIONS_URL = f"{DEFAULT_API_URL}/questions"


def main() -> None:
    print(f"Fetching questions from {QUESTIONS_URL} ...")
    resp = requests.get(QUESTIONS_URL, timeout=30)
    resp.raise_for_status()
    questions = resp.json()
    print(f"Got {len(questions)} questions.\n")

    agent = GaiaAgent()
    results = []
    t_start = time.time()

    for i, item in enumerate(questions, 1):
        task_id = item.get("task_id")
        question = item.get("question", "")
        has_file = bool(item.get("file_name"))

        print(f"[{i}/{len(questions)}] task {task_id}  (file: {has_file})")
        print(f"  Q: {question[:160]}{'...' if len(question) > 160 else ''}")

        t0 = time.time()
        try:
            answer = agent(question, task_id, item.get("file_name"))
        except Exception as exc:  # noqa: BLE001
            answer = f"AGENT ERROR: {exc}"
        dt = time.time() - t0

        print(f"  A: {answer!r}   ({dt:.1f}s)\n")
        results.append(
            {
                "task_id": task_id,
                "question": question,
                "file_name": item.get("file_name", ""),
                "submitted_answer": answer,
                "seconds": round(dt, 1),
            }
        )

    with open("results.json", "w", encoding="utf-8") as f:
        json.dump(results, f, indent=2, ensure_ascii=False)

    errors = sum(1 for r in results if r["submitted_answer"].startswith("AGENT ERROR"))
    blanks = sum(1 for r in results if not r["submitted_answer"].strip())
    print("=" * 60)
    print(f"Done in {time.time() - t_start:.0f}s. {len(results)} answers.")
    print(f"Agent errors: {errors}   Blank answers: {blanks}")
    print("Saved -> results.json  (nothing was submitted)")


if __name__ == "__main__":
    main()