run_local_eval.py

"""Checkpointed concurrent CLI dry-run; contains no submission call."""

from __future__ import annotations

import argparse
import json
import os
import sys
from pathlib import Path

from agent import GaiaAgent
from cache import AnswerCache, ResultStore
from config import Settings
from evaluation import run_evaluation
from gaia_client import GaiaClient


def main() -> int:
    # Windows commonly defaults redirected stdout to cp1252, while GAIA
    # questions and evidence contain arbitrary Unicode.
    if hasattr(sys.stdout, "reconfigure"):
        sys.stdout.reconfigure(encoding="utf-8", errors="replace")
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--no-cache", action="store_true")
    parser.add_argument("--force", action="store_true")
    parser.add_argument("--task-id", action="append", default=[])
    args = parser.parse_args()
    settings = Settings.from_env()
    answer_cache = AnswerCache(
        settings.cache_dir / "answers.json",
        enabled=settings.use_cache and not args.no_cache,
    )
    store = ResultStore(Path(os.getenv("GAIA_RESULTS_PATH", "results/results.json")))
    results = run_evaluation(
        GaiaClient(settings),
        GaiaAgent(settings, answer_cache),
        store,
        progress=lambda current, total, task_id: print(
            f"[{current}/{total}] {task_id}", flush=True
        ),
        force=args.force,
        task_ids=set(args.task_id) or None,
    )
    print(json.dumps(results, ensure_ascii=False, indent=2))
    failures = sum(row["status"] != "ok" for row in results)
    print(f"Dry run finished with {failures} failure(s). Nothing was submitted.")
    return 1 if failures else 0


if __name__ == "__main__":
    raise SystemExit(main())