File size: 3,985 Bytes
10aced5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 | """
Integration tests — require a running API server.
Run with: make test-integration
Server URL: VALIDATOR_URL env var (default: http://localhost:8000)
Design notes:
- All tests are read-only: the API has no mutable state, so cleanup is N/A.
- Tests are order-independent: no shared mutable fixtures.
- Each test is self-contained: uses only the api_client session fixture.
"""
import pytest
from client.client import ValidatorClient
from client.exceptions import APIError
from client.models import ConfigResponse, QueryResponse
pytestmark = pytest.mark.integration
@pytest.fixture(scope="module")
def config(api_client: ValidatorClient) -> ConfigResponse:
return api_client.get_config()
class TestHealth:
def test_api_is_reachable(self, api_client: ValidatorClient) -> None:
assert api_client.health() is True
class TestConfig:
def test_returns_retail_and_pharma_domains(self, config: ConfigResponse) -> None:
assert "retail" in config.domains
assert "pharma" in config.domains
def test_each_domain_has_two_clients(self, config: ConfigResponse) -> None:
for domain, clients in config.domains.items():
assert len(clients) == 2, f"{domain} should have 2 clients"
def test_client_ids_are_strings(self, config: ConfigResponse) -> None:
for clients in config.domains.values():
for c in clients:
assert isinstance(c.id, str)
assert isinstance(c.display, str)
class TestQuery:
def test_valid_query_returns_answer(self, api_client: ValidatorClient) -> None:
response = api_client.query(
"What happens when a product runs out of stock?",
"novamart",
)
assert isinstance(response, QueryResponse)
assert len(response.answer) > 0
def test_response_includes_all_five_metrics(self, api_client: ValidatorClient) -> None:
response = api_client.query("How do I add a new supplier?", "shelfwise")
metrics = response.evaluation.metrics
expected = {"pii_leakage", "token_budget", "answer_relevancy", "faithfulness", "chain_terminology"}
assert set(metrics.keys()) == expected
def test_metric_scores_are_in_valid_range(self, api_client: ValidatorClient) -> None:
response = api_client.query("What is prior authorization?", "clinixone")
for name, metric in response.evaluation.metrics.items():
assert 0.0 <= metric.score <= 1.0, f"{name} score out of range: {metric.score}"
def test_sources_are_returned(self, api_client: ValidatorClient) -> None:
response = api_client.query("How do compliance reports work?", "shelfwise")
assert len(response.sources) > 0
for source in response.sources:
assert source.title
assert 0.0 <= source.score <= 1.0
def test_client_display_name_matches_client_id(self, api_client: ValidatorClient) -> None:
response = api_client.query("What is formulary pre-approval?", "pharmalink")
assert response.client == "pharmalink"
assert response.client_display == "PharmaLink"
def test_unknown_client_raises_api_error(self, api_client: ValidatorClient) -> None:
with pytest.raises(APIError) as exc_info:
api_client.query("Any question", "nonexistent_client")
assert exc_info.value.status_code == 400
def test_empty_query_raises_api_error(self, api_client: ValidatorClient) -> None:
with pytest.raises(APIError) as exc_info:
api_client.query(" ", "novamart")
assert exc_info.value.status_code == 400
def test_pharma_query_uses_client_terminology(self, api_client: ValidatorClient) -> None:
response = api_client.query(
"How do I get approval before dispensing a drug?",
"pharmalink",
)
assert "formulary pre-approval" in response.answer.lower() or \
response.evaluation.metrics["chain_terminology"].passed
|