""" Integration tests — require a running API server. Run with: make test-integration Server URL: VALIDATOR_URL env var (default: http://localhost:8000) Design notes: - All tests are read-only: the API has no mutable state, so cleanup is N/A. - Tests are order-independent: no shared mutable fixtures. - Each test is self-contained: uses only the api_client session fixture. """ import pytest from client.client import ValidatorClient from client.exceptions import APIError from client.models import ConfigResponse, QueryResponse pytestmark = pytest.mark.integration @pytest.fixture(scope="module") def config(api_client: ValidatorClient) -> ConfigResponse: return api_client.get_config() class TestHealth: def test_api_is_reachable(self, api_client: ValidatorClient) -> None: assert api_client.health() is True class TestConfig: def test_returns_retail_and_pharma_domains(self, config: ConfigResponse) -> None: assert "retail" in config.domains assert "pharma" in config.domains def test_each_domain_has_two_clients(self, config: ConfigResponse) -> None: for domain, clients in config.domains.items(): assert len(clients) == 2, f"{domain} should have 2 clients" def test_client_ids_are_strings(self, config: ConfigResponse) -> None: for clients in config.domains.values(): for c in clients: assert isinstance(c.id, str) assert isinstance(c.display, str) class TestQuery: def test_valid_query_returns_answer(self, api_client: ValidatorClient) -> None: response = api_client.query( "What happens when a product runs out of stock?", "novamart", ) assert isinstance(response, QueryResponse) assert len(response.answer) > 0 def test_response_includes_all_five_metrics(self, api_client: ValidatorClient) -> None: response = api_client.query("How do I add a new supplier?", "shelfwise") metrics = response.evaluation.metrics expected = {"pii_leakage", "token_budget", "answer_relevancy", "faithfulness", "chain_terminology"} assert set(metrics.keys()) == expected def test_metric_scores_are_in_valid_range(self, api_client: ValidatorClient) -> None: response = api_client.query("What is prior authorization?", "clinixone") for name, metric in response.evaluation.metrics.items(): assert 0.0 <= metric.score <= 1.0, f"{name} score out of range: {metric.score}" def test_sources_are_returned(self, api_client: ValidatorClient) -> None: response = api_client.query("How do compliance reports work?", "shelfwise") assert len(response.sources) > 0 for source in response.sources: assert source.title assert 0.0 <= source.score <= 1.0 def test_client_display_name_matches_client_id(self, api_client: ValidatorClient) -> None: response = api_client.query("What is formulary pre-approval?", "pharmalink") assert response.client == "pharmalink" assert response.client_display == "PharmaLink" def test_unknown_client_raises_api_error(self, api_client: ValidatorClient) -> None: with pytest.raises(APIError) as exc_info: api_client.query("Any question", "nonexistent_client") assert exc_info.value.status_code == 400 def test_empty_query_raises_api_error(self, api_client: ValidatorClient) -> None: with pytest.raises(APIError) as exc_info: api_client.query(" ", "novamart") assert exc_info.value.status_code == 400 def test_pharma_query_uses_client_terminology(self, api_client: ValidatorClient) -> None: response = api_client.query( "How do I get approval before dispensing a drug?", "pharmalink", ) assert "formulary pre-approval" in response.answer.lower() or \ response.evaluation.metrics["chain_terminology"].passed