File size: 1,759 Bytes
c641d5f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | """Checkpointed concurrent CLI dry-run; contains no submission call."""
from __future__ import annotations
import argparse
import json
import os
import sys
from pathlib import Path
from agent import GaiaAgent
from cache import AnswerCache, ResultStore
from config import Settings
from evaluation import run_evaluation
from gaia_client import GaiaClient
def main() -> int:
# Windows commonly defaults redirected stdout to cp1252, while GAIA
# questions and evidence contain arbitrary Unicode.
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--no-cache", action="store_true")
parser.add_argument("--force", action="store_true")
parser.add_argument("--task-id", action="append", default=[])
args = parser.parse_args()
settings = Settings.from_env()
answer_cache = AnswerCache(
settings.cache_dir / "answers.json",
enabled=settings.use_cache and not args.no_cache,
)
store = ResultStore(Path(os.getenv("GAIA_RESULTS_PATH", "results/results.json")))
results = run_evaluation(
GaiaClient(settings),
GaiaAgent(settings, answer_cache),
store,
progress=lambda current, total, task_id: print(
f"[{current}/{total}] {task_id}", flush=True
),
force=args.force,
task_ids=set(args.task_id) or None,
)
print(json.dumps(results, ensure_ascii=False, indent=2))
failures = sum(row["status"] != "ok" for row in results)
print(f"Dry run finished with {failures} failure(s). Nothing was submitted.")
return 1 if failures else 0
if __name__ == "__main__":
raise SystemExit(main())
|