submit_official.py

"""Run the agent over all questions and submit OFFICIALLY to the leaderboard.

Username is derived from the HF token (whoami). The agent_code link defaults to
the user's existing public Space but can be overridden with AGENT_CODE_URL.
"""

from __future__ import annotations

import json
import os
import sys
import time

try:
    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
    sys.stderr.reconfigure(encoding="utf-8", errors="replace")
except Exception:  # noqa: BLE001
    pass

from dotenv import load_dotenv

load_dotenv()

import requests
from huggingface_hub import HfApi

from agent import GaiaAgent
from tools import DEFAULT_API_URL

QUESTIONS_URL = f"{DEFAULT_API_URL}/questions"
SUBMIT_URL = f"{DEFAULT_API_URL}/submit"

DEFAULT_AGENT_CODE = os.getenv(
    "AGENT_CODE_URL",
    "https://huggingface.co/spaces/SeppoR/First_agent_template/tree/main",
)


def main() -> None:
    username = HfApi(token=os.getenv("HF_TOKEN")).whoami().get("name")
    print(f"Submitting as: {username}")
    print(f"Agent code:    {DEFAULT_AGENT_CODE}\n")

    questions = requests.get(QUESTIONS_URL, timeout=30).json()
    print(f"Fetched {len(questions)} questions.\n")

    agent = GaiaAgent()
    answers = []
    log = []
    for i, item in enumerate(questions, 1):
        tid = item.get("task_id")
        q = item.get("question", "")
        print(f"[{i}/{len(questions)}] {tid}")
        try:
            ans = agent(q, tid, item.get("file_name"))
        except Exception as exc:  # noqa: BLE001
            ans = f"AGENT ERROR: {exc}"
        print(f"  -> {ans!r}")
        answers.append({"task_id": tid, "submitted_answer": ans})
        log.append({"task_id": tid, "question": q, "submitted_answer": ans})

    with open("results.json", "w", encoding="utf-8") as f:
        json.dump(log, f, indent=2, ensure_ascii=False)

    errs = sum(1 for a in answers if a["submitted_answer"].startswith("AGENT ERROR"))
    print(f"\nPrepared {len(answers)} answers ({errs} agent errors).")

    payload = {
        "username": username.strip(),
        "agent_code": DEFAULT_AGENT_CODE,
        "answers": answers,
    }
    print("Submitting to leaderboard...")
    resp = requests.post(SUBMIT_URL, json=payload, timeout=120)
    try:
        resp.raise_for_status()
    except requests.HTTPError:
        print("Submission FAILED:", resp.status_code, resp.text[:500])
        return
    data = resp.json()
    print("=" * 60)
    print("SUBMISSION SUCCESSFUL")
    print(f"  User:           {data.get('username')}")
    print(
        f"  Score:          {data.get('score')}% "
        f"({data.get('correct_count')}/{data.get('total_attempted')})"
    )
    print(f"  Message:        {data.get('message', '')}")


if __name__ == "__main__":
    t = time.time()
    main()
    print(f"\nElapsed {time.time() - t:.0f}s")