tests/test_llm.py

"""tests/test_llm.py — provider-agnostic LLM factory (agent/llm.py)."""
from __future__ import annotations

import pytest

from agent.llm import (
    ANTHROPIC_DEFAULT_MODEL,
    RunConfig,
    build_system_message,
    classify_llm_error,
    console_url,
    default_config,
    key_looks_valid,
    make_chat_model,
    resolve_api_key,
)


# ── default_config ────────────────────────────────────────────────────────────

def test_default_config_is_anthropic_haiku_with_no_key():
    cfg = default_config()
    assert cfg.provider == "anthropic"
    assert cfg.model == ANTHROPIC_DEFAULT_MODEL
    assert cfg.api_key is None


# ── resolve_api_key precedence ───────────────────────────────────────────────

def test_resolve_api_key_prefers_user_key_over_env(monkeypatch):
    monkeypatch.setenv("ANTHROPIC_API_KEY", "env-key")
    key, source = resolve_api_key("anthropic", "user-pasted-key")
    assert key == "user-pasted-key"
    assert source == "user"


def test_resolve_api_key_falls_back_to_env(monkeypatch):
    monkeypatch.setenv("ANTHROPIC_API_KEY", "env-key")
    key, source = resolve_api_key("anthropic", "")
    assert key == "env-key"
    assert source == "env"


def test_resolve_api_key_missing_when_neither_present(monkeypatch):
    monkeypatch.delenv("OPENAI_API_KEY", raising=False)
    key, source = resolve_api_key("openai", None)
    assert key is None
    assert source == "missing"


def test_resolve_api_key_strips_whitespace(monkeypatch):
    monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
    key, source = resolve_api_key("anthropic", "   ")
    assert key is None
    assert source == "missing"


# ── make_chat_model ───────────────────────────────────────────────────────────

def test_make_chat_model_anthropic_returns_chat_anthropic():
    cfg = RunConfig(provider="anthropic", model="claude-haiku-4-5-20251001", api_key="sk-ant-test")
    model = make_chat_model(cfg)
    assert type(model).__name__ == "ChatAnthropic"
    assert model.model == "claude-haiku-4-5-20251001"


def test_make_chat_model_openai_returns_chat_openai():
    cfg = RunConfig(provider="openai", model="gpt-5-mini", api_key="sk-test")
    model = make_chat_model(cfg)
    assert type(model).__name__ == "ChatOpenAI"


def test_make_chat_model_passes_max_tokens_only_when_given():
    cfg = RunConfig(provider="anthropic", api_key="sk-ant-test")
    model_default = make_chat_model(cfg)
    model_explicit = make_chat_model(cfg, max_tokens=8192)
    assert getattr(model_explicit, "max_tokens", None) == 8192
    # Default path should not silently pick up 8192 from a shared default.
    assert getattr(model_default, "max_tokens", None) != 8192


def test_make_chat_model_no_key_does_not_raise(monkeypatch):
    # No key anywhere — construction still succeeds (the class reads its own
    # env var, which may be unset); failure happens on network call, not here.
    monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
    cfg = RunConfig(provider="anthropic", api_key=None)
    make_chat_model(cfg)  # must not raise


# ── build_system_message ─────────────────────────────────────────────────────

def test_build_system_message_anthropic_has_cache_control_on_primary_block():
    cfg = RunConfig(provider="anthropic")
    msg = build_system_message(cfg, "primary text")
    assert isinstance(msg.content, list)
    assert msg.content[0]["text"] == "primary text"
    assert msg.content[0]["cache_control"] == {"type": "ephemeral"}


def test_build_system_message_anthropic_extra_block_is_uncached():
    cfg = RunConfig(provider="anthropic")
    msg = build_system_message(cfg, "primary", extra_texts=["language directive"])
    assert len(msg.content) == 2
    assert msg.content[1]["text"] == "language directive"
    assert "cache_control" not in msg.content[1]


def test_build_system_message_openai_is_plain_string():
    cfg = RunConfig(provider="openai")
    msg = build_system_message(cfg, "primary", extra_texts=["extra"])
    assert isinstance(msg.content, str)
    assert "primary" in msg.content
    assert "extra" in msg.content


# ── classify_llm_error ────────────────────────────────────────────────────────

@pytest.mark.parametrize("text", [
    "authentication_error: invalid x-api-key",
    "Error code: 401 - Incorrect API key provided",
])
def test_classify_llm_error_recognizes_auth_failures(text):
    result = classify_llm_error(text, "anthropic")
    assert result is not None
    assert "API key was rejected" in result


def test_classify_llm_error_recognizes_overload():
    result = classify_llm_error("overloaded_error: please retry", "anthropic")
    assert result is not None
    assert "overloaded" in result.lower()


def test_classify_llm_error_recognizes_rate_limit():
    result = classify_llm_error("429 rate_limit exceeded", "openai")
    assert result is not None
    assert "rate limit" in result.lower()


def test_classify_llm_error_returns_none_for_unknown_errors():
    assert classify_llm_error("connection reset by peer", "anthropic") is None


def test_classify_llm_error_never_echoes_the_key():
    exc = Exception("authentication_error with key sk-ant-super-secret-123")
    result = classify_llm_error(exc, "anthropic")
    assert "sk-ant-super-secret-123" not in result


# ── misc ───────────────────────────────────────────────────────────────────────

def test_key_looks_valid_catches_wrong_provider_prefix():
    assert key_looks_valid("anthropic", "sk-ant-abc123") is True
    assert key_looks_valid("anthropic", "sk-proj-abc123") is False
    assert key_looks_valid("openai", "sk-proj-abc123") is True


def test_console_url_known_providers():
    assert "anthropic.com" in console_url("anthropic")
    assert "openai.com" in console_url("openai")