tests/test_agent_architecture.py

import requests
from smolagents import CodeAgent, ToolCallingAgent

from agent import GaiaAgent
from config import Settings


def settings(tmp_path, **overrides):
    values = {
        "api_url": "https://example.test",
        "hf_token": "fake-token",
        "model_id": "primary",
        "vision_model_id": "vision",
        "inference_provider": "provider-a",
        "asr_model_id": "asr",
        "cache_dir": tmp_path,
        "request_timeout": 10,
        "retries": 1,
        "backoff_seconds": 0,
        "max_steps": 2,
        "use_cache": False,
        "stockfish_path": None,
        "model_requests_per_minute": 60_000,
        "search_requests_per_minute": 60_000,
    }
    values.update(overrides)
    return Settings(**values)


def test_managed_web_research_architecture_and_model_reuse(tmp_path):
    agent = GaiaAgent(settings(tmp_path))
    spec = ("primary", "provider-a")
    manager = agent._manager(spec)
    assert isinstance(manager, CodeAgent)
    assert isinstance(manager.managed_agents["web_researcher"], ToolCallingAgent)
    assert manager.managed_agents["web_researcher"].max_steps == 10
    assert agent._model(spec) is agent._model(spec)


def test_transient_primary_failure_uses_configured_fallback(tmp_path, monkeypatch):
    agent = GaiaAgent(
        settings(
            tmp_path,
            fallback_model_id="secondary",
            fallback_provider="provider-b",
        )
    )

    class Runner:
        def __init__(self, value):
            self.value = value

        def run(self, prompt, images=None):
            if isinstance(self.value, Exception):
                raise self.value
            return self.value

    runners = {
        ("primary", "provider-a"): Runner(requests.ConnectionError("DNS failure")),
        ("secondary", "provider-b"): Runner(
            '{"candidate_answer":"42","evidence":[],"confidence":1}'
        ),
    }
    monkeypatch.setattr(agent, "_manager", lambda spec: runners[spec])
    assert "42" in str(agent._run_model("question"))


def test_nontransient_agent_failure_also_tries_fallback(tmp_path, monkeypatch):
    agent = GaiaAgent(settings(tmp_path, fallback_model_id="secondary"))

    class Runner:
        def run(self, prompt, images=None):
            raise ValueError("bad tool arguments")

    monkeypatch.setattr(agent, "_manager", lambda spec: Runner())
    import pytest

    with pytest.raises(
        RuntimeError, match="All inference model/provider attempts failed"
    ) as exc:
        agent._run_model("question")
    assert str(exc.value).count("bad tool arguments") == 2