File size: 6,457 Bytes
7880373 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 | """tests/test_llm.py — provider-agnostic LLM factory (agent/llm.py)."""
from __future__ import annotations
import pytest
from agent.llm import (
ANTHROPIC_DEFAULT_MODEL,
RunConfig,
build_system_message,
classify_llm_error,
console_url,
default_config,
key_looks_valid,
make_chat_model,
resolve_api_key,
)
# ── default_config ────────────────────────────────────────────────────────────
def test_default_config_is_anthropic_haiku_with_no_key():
cfg = default_config()
assert cfg.provider == "anthropic"
assert cfg.model == ANTHROPIC_DEFAULT_MODEL
assert cfg.api_key is None
# ── resolve_api_key precedence ───────────────────────────────────────────────
def test_resolve_api_key_prefers_user_key_over_env(monkeypatch):
monkeypatch.setenv("ANTHROPIC_API_KEY", "env-key")
key, source = resolve_api_key("anthropic", "user-pasted-key")
assert key == "user-pasted-key"
assert source == "user"
def test_resolve_api_key_falls_back_to_env(monkeypatch):
monkeypatch.setenv("ANTHROPIC_API_KEY", "env-key")
key, source = resolve_api_key("anthropic", "")
assert key == "env-key"
assert source == "env"
def test_resolve_api_key_missing_when_neither_present(monkeypatch):
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
key, source = resolve_api_key("openai", None)
assert key is None
assert source == "missing"
def test_resolve_api_key_strips_whitespace(monkeypatch):
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
key, source = resolve_api_key("anthropic", " ")
assert key is None
assert source == "missing"
# ── make_chat_model ───────────────────────────────────────────────────────────
def test_make_chat_model_anthropic_returns_chat_anthropic():
cfg = RunConfig(provider="anthropic", model="claude-haiku-4-5-20251001", api_key="sk-ant-test")
model = make_chat_model(cfg)
assert type(model).__name__ == "ChatAnthropic"
assert model.model == "claude-haiku-4-5-20251001"
def test_make_chat_model_openai_returns_chat_openai():
cfg = RunConfig(provider="openai", model="gpt-5-mini", api_key="sk-test")
model = make_chat_model(cfg)
assert type(model).__name__ == "ChatOpenAI"
def test_make_chat_model_passes_max_tokens_only_when_given():
cfg = RunConfig(provider="anthropic", api_key="sk-ant-test")
model_default = make_chat_model(cfg)
model_explicit = make_chat_model(cfg, max_tokens=8192)
assert getattr(model_explicit, "max_tokens", None) == 8192
# Default path should not silently pick up 8192 from a shared default.
assert getattr(model_default, "max_tokens", None) != 8192
def test_make_chat_model_no_key_does_not_raise(monkeypatch):
# No key anywhere — construction still succeeds (the class reads its own
# env var, which may be unset); failure happens on network call, not here.
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
cfg = RunConfig(provider="anthropic", api_key=None)
make_chat_model(cfg) # must not raise
# ── build_system_message ─────────────────────────────────────────────────────
def test_build_system_message_anthropic_has_cache_control_on_primary_block():
cfg = RunConfig(provider="anthropic")
msg = build_system_message(cfg, "primary text")
assert isinstance(msg.content, list)
assert msg.content[0]["text"] == "primary text"
assert msg.content[0]["cache_control"] == {"type": "ephemeral"}
def test_build_system_message_anthropic_extra_block_is_uncached():
cfg = RunConfig(provider="anthropic")
msg = build_system_message(cfg, "primary", extra_texts=["language directive"])
assert len(msg.content) == 2
assert msg.content[1]["text"] == "language directive"
assert "cache_control" not in msg.content[1]
def test_build_system_message_openai_is_plain_string():
cfg = RunConfig(provider="openai")
msg = build_system_message(cfg, "primary", extra_texts=["extra"])
assert isinstance(msg.content, str)
assert "primary" in msg.content
assert "extra" in msg.content
# ── classify_llm_error ────────────────────────────────────────────────────────
@pytest.mark.parametrize("text", [
"authentication_error: invalid x-api-key",
"Error code: 401 - Incorrect API key provided",
])
def test_classify_llm_error_recognizes_auth_failures(text):
result = classify_llm_error(text, "anthropic")
assert result is not None
assert "API key was rejected" in result
def test_classify_llm_error_recognizes_overload():
result = classify_llm_error("overloaded_error: please retry", "anthropic")
assert result is not None
assert "overloaded" in result.lower()
def test_classify_llm_error_recognizes_rate_limit():
result = classify_llm_error("429 rate_limit exceeded", "openai")
assert result is not None
assert "rate limit" in result.lower()
def test_classify_llm_error_returns_none_for_unknown_errors():
assert classify_llm_error("connection reset by peer", "anthropic") is None
def test_classify_llm_error_never_echoes_the_key():
exc = Exception("authentication_error with key sk-ant-super-secret-123")
result = classify_llm_error(exc, "anthropic")
assert "sk-ant-super-secret-123" not in result
# ── misc ───────────────────────────────────────────────────────────────────────
def test_key_looks_valid_catches_wrong_provider_prefix():
assert key_looks_valid("anthropic", "sk-ant-abc123") is True
assert key_looks_valid("anthropic", "sk-proj-abc123") is False
assert key_looks_valid("openai", "sk-proj-abc123") is True
def test_console_url_known_providers():
assert "anthropic.com" in console_url("anthropic")
assert "openai.com" in console_url("openai")
|