Spaces:
Sleeping
Sleeping
File size: 16,763 Bytes
d66d64c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 | """Wire-level contract of the shared Claude client: what each request carries and how replies are read."""
from __future__ import annotations
import json
import os
import sys
from types import SimpleNamespace
from typing import Callable
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import anthropic
import httpx2
import pytest
from services import ai_client
USER_HI = [{"role": "user", "content": "hi"}]
CODE_FENCED_JSON = '```json\n{"score": 3}\n```'
REFUSAL_DETAILS = {"type": "refusal", "category": "cyber", "explanation": None}
SCORE_SCHEMA = {
"type": "object",
"properties": {"score": {"type": "integer"}},
"required": ["score"],
"additionalProperties": False,
}
def _text_block(text: str) -> dict:
return {"type": "text", "text": text}
def _message_response(*content: dict, stop_reason: str = "end_turn", stop_details: dict | None = None) -> httpx2.Response:
return httpx2.Response(
200,
json={
"id": "msg_test",
"type": "message",
"role": "assistant",
"model": ai_client.CHAT_MODEL,
"content": list(content),
"stop_reason": stop_reason,
"stop_sequence": None,
"stop_details": stop_details,
"usage": {"input_tokens": 11, "output_tokens": 7},
},
)
def _stream_response(*, thinking: str = "", stop_reason: str = "end_turn", stop_details: dict | None = None) -> httpx2.Response:
"""A server-sent-event reply that streams an optional thinking block, then the text "Hel" + "lo"."""
events = [
{
"type": "message_start",
"message": {
"id": "msg_test",
"type": "message",
"role": "assistant",
"model": ai_client.REASONER_MODEL,
"content": [],
"stop_reason": None,
"stop_sequence": None,
"usage": {"input_tokens": 9, "output_tokens": 1},
},
}
]
index = 0
if thinking:
events += [
{"type": "content_block_start", "index": index, "content_block": {"type": "thinking", "thinking": "", "signature": ""}},
{"type": "content_block_delta", "index": index, "delta": {"type": "thinking_delta", "thinking": thinking}},
{"type": "content_block_delta", "index": index, "delta": {"type": "signature_delta", "signature": "sig"}},
{"type": "content_block_stop", "index": index},
]
index += 1
events += [
{"type": "content_block_start", "index": index, "content_block": {"type": "text", "text": ""}},
{"type": "content_block_delta", "index": index, "delta": {"type": "text_delta", "text": "Hel"}},
{"type": "content_block_delta", "index": index, "delta": {"type": "text_delta", "text": "lo"}},
{"type": "content_block_stop", "index": index},
{
"type": "message_delta",
"delta": {"stop_reason": stop_reason, "stop_sequence": None, "stop_details": stop_details},
"usage": {"output_tokens": 5},
},
{"type": "message_stop"},
]
body = "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events)
return httpx2.Response(200, headers={"content-type": "text/event-stream"}, content=body.encode())
class MockApi:
"""Stands in for api.anthropic.com: records each request and answers with `reply()`."""
def __init__(self) -> None:
self.sent: list[SimpleNamespace] = []
self.reply: Callable[[], httpx2.Response] = lambda: _message_response(_text_block("ok"))
def handle(self, request: httpx2.Request) -> httpx2.Response:
self.sent.append(SimpleNamespace(headers=request.headers, body=json.loads(request.content)))
return self.reply()
@pytest.fixture
def api(monkeypatch: pytest.MonkeyPatch):
"""Build the shared client for real, but over a mock transport so nothing reaches the network."""
mock_api = MockApi()
sdk_client = anthropic.Anthropic
monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-test-key")
monkeypatch.delenv("ANTHROPIC_WORKSPACE_ID", raising=False)
monkeypatch.setattr(
ai_client,
"Anthropic",
lambda **options: sdk_client(**options, http_client=httpx2.Client(transport=httpx2.MockTransport(mock_api.handle))),
)
ai_client.get_anthropic_client.cache_clear()
yield mock_api
ai_client.get_anthropic_client.cache_clear()
def test_default_model_ids_are_current_api_ids() -> None:
assert ai_client.CHAT_MODEL == "claude-haiku-5-5"
assert ai_client.REASONER_MODEL == "claude-opus-5-5"
def test_current_model_id_keeps_claude_ids_and_blanks_the_rest() -> None:
assert ai_client.current_model_id(" claude-opus-5-5 ") == "claude-opus-5-5"
assert ai_client.current_model_id("retired-chat-model") == ""
assert ai_client.current_model_id("") == ""
def test_client_requires_an_api_key(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
ai_client.get_anthropic_client.cache_clear()
with pytest.raises(ValueError, match="ANTHROPIC_API_KEY"):
ai_client.get_anthropic_client()
def test_chat_model_request_disables_thinking(api: MockApi) -> None:
ai_client.create_message(USER_HI, model=ai_client.CHAT_MODEL, max_tokens=300)
request = api.sent[0]
assert request.body["model"] == ai_client.CHAT_MODEL
assert request.body["thinking"] == {"type": "disabled"}
assert request.body["max_tokens"] == 300
assert "output_config" not in request.body
assert "fallbacks" not in request.body
assert "anthropic-beta" not in request.headers
def test_reasoner_request_sets_effort_token_floor_and_refusal_fallback(api: MockApi) -> None:
ai_client.create_message(USER_HI, model=ai_client.REASONER_MODEL, max_tokens=500)
request = api.sent[0]
assert request.body["model"] == ai_client.REASONER_MODEL
assert request.body.get("thinking", {}).get("type") != "disabled"
assert request.body["output_config"] == {"effort": "medium"}
assert request.body["max_tokens"] >= 16384
assert request.headers["anthropic-beta"] == "server-side-fallback-2026-07-01"
assert request.body["fallbacks"] == "default"
def test_effort_argument_reaches_the_request(api: MockApi) -> None:
ai_client.create_message(USER_HI, model=ai_client.REASONER_MODEL, effort="high")
ai_client.create_message(USER_HI, model=ai_client.CHAT_MODEL, effort="low")
assert [request.body["output_config"] for request in api.sent] == [{"effort": "high"}, {"effort": "low"}]
assert api.sent[1].body["thinking"] == {"type": "disabled"}
def test_sampling_parameters_are_not_accepted(api: MockApi) -> None:
with pytest.raises(TypeError):
ai_client.create_message(USER_HI, temperature=0.2) # type: ignore[call-arg]
for model in (ai_client.CHAT_MODEL, ai_client.REASONER_MODEL):
ai_client.create_message(USER_HI, model=model)
for request in api.sent:
assert not {"temperature", "top_p", "top_k"} & request.body.keys()
def test_workspace_id_is_sent_as_a_header_when_configured(api: MockApi, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("ANTHROPIC_WORKSPACE_ID", "wrkspc_test")
ai_client.get_anthropic_client.cache_clear()
ai_client.create_message(USER_HI)
assert api.sent[0].headers["anthropic-workspace-id"] == "wrkspc_test"
def test_workspace_header_is_omitted_by_default(api: MockApi) -> None:
ai_client.create_message(USER_HI)
assert "anthropic-workspace-id" not in api.sent[0].headers
def test_system_messages_are_hoisted_to_the_system_parameter(api: MockApi) -> None:
ai_client.create_message(
[
{"role": "system", "content": "Be brief."},
{"role": "user", "content": "Hi"},
{"role": "system", "content": "Use plain words."},
{"role": "assistant", "content": "Hello"},
{"role": "user", "content": "Add 2 and 2"},
]
)
body = api.sent[0].body
assert body["system"] == "Be brief.\n\nUse plain words."
assert body["messages"] == [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hello"},
{"role": "user", "content": "Add 2 and 2"},
]
def test_leading_assistant_and_blank_turns_are_dropped(api: MockApi) -> None:
ai_client.create_message(
[
{"role": "assistant", "content": "Welcome back"},
{"role": "user", "content": " "},
{"role": "user", "content": "Hi"},
]
)
body = api.sent[0].body
assert body["messages"] == [{"role": "user", "content": "Hi"}]
assert "system" not in body
def test_json_output_adds_an_instruction_and_strips_the_code_fence(api: MockApi) -> None:
api.reply = lambda: _message_response(_text_block(CODE_FENCED_JSON))
reply = ai_client.complete_text(
[{"role": "system", "content": "Grade it."}, {"role": "user", "content": "2 + 2"}], json_output=True
)
assert json.loads(reply) == {"score": 3}
system = api.sent[0].body["system"]
assert system.startswith("Grade it.")
assert "JSON" in system
def test_json_output_recovers_json_wrapped_in_prose(api: MockApi) -> None:
api.reply = lambda: _message_response(
_text_block('Here is the classification:\n{"risk_label": "high risk of failing", "confidence": 0.91}\nHope it helps.')
)
reply = ai_client.complete_text(USER_HI, json_output=True)
assert json.loads(reply) == {"risk_label": "high risk of failing", "confidence": 0.91}
def test_json_output_returns_the_text_when_no_json_can_be_recovered(api: MockApi) -> None:
api.reply = lambda: _message_response(_text_block("I cannot classify this."))
assert ai_client.complete_text(USER_HI, json_output=True) == "I cannot classify this."
def test_json_output_keeps_a_truncated_reply_whole_for_the_callers_salvage_parser(api: MockApi) -> None:
truncated = '[{"question": "Q1", "answer": "A"}, {"question": "Q2", "answer": "B"}, {"question": "Q3", "ans'
api.reply = lambda: _message_response(_text_block(truncated), stop_reason="max_tokens")
assert ai_client.complete_text(USER_HI, json_output=True) == truncated
def test_json_output_never_returns_a_nested_fragment_of_malformed_json(api: MockApi) -> None:
malformed = '{"summary": "Review the "hole" rule", "practice": {"problem": "Find the domain", "solution": "x != 2"}}'
api.reply = lambda: _message_response(_text_block(malformed))
assert ai_client.complete_text(USER_HI, json_output=True) == malformed
def test_reply_keeps_the_code_fence_unless_json_output_is_requested(api: MockApi) -> None:
api.reply = lambda: _message_response(_text_block(CODE_FENCED_JSON))
assert ai_client.complete_text(USER_HI) == CODE_FENCED_JSON
assert "system" not in api.sent[0].body
def test_schema_turns_on_structured_outputs_without_a_json_instruction(api: MockApi) -> None:
ai_client.create_message(
[{"role": "system", "content": "Grade it."}, {"role": "user", "content": "2 + 2"}], schema=SCORE_SCHEMA
)
body = api.sent[0].body
assert body["output_config"] == {"format": {"type": "json_schema", "schema": SCORE_SCHEMA}}
assert body["system"] == "Grade it."
def test_structured_outputs_keep_the_effort_setting(api: MockApi) -> None:
ai_client.create_message(USER_HI, model=ai_client.REASONER_MODEL, schema=SCORE_SCHEMA)
assert api.sent[0].body["output_config"] == {
"effort": "medium",
"format": {"type": "json_schema", "schema": SCORE_SCHEMA},
}
def test_complete_json_returns_the_parsed_object(api: MockApi) -> None:
api.reply = lambda: _message_response(_text_block('{"score": 3}'))
assert ai_client.complete_json(USER_HI, schema=SCORE_SCHEMA) == {"score": 3}
assert api.sent[0].body["output_config"]["format"]["schema"] == SCORE_SCHEMA
def test_complete_json_raises_when_claude_declines(api: MockApi) -> None:
api.reply = lambda: _message_response(stop_reason="refusal", stop_details=REFUSAL_DETAILS)
with pytest.raises(ai_client.AIRefusalError):
ai_client.complete_json(USER_HI, schema=SCORE_SCHEMA)
@pytest.mark.parametrize("stale_id", ["retired-chat-model", "custom/model", ""])
def test_non_claude_model_id_is_replaced_by_the_chat_model(api: MockApi, stale_id: str) -> None:
ai_client.create_message(USER_HI, model=stale_id)
assert api.sent[0].body["model"] == ai_client.CHAT_MODEL
def test_reply_text_joins_text_blocks_and_skips_thinking(api: MockApi) -> None:
api.reply = lambda: _message_response(
{"type": "thinking", "thinking": "Considering.", "signature": "sig"},
_text_block("Hello "),
_text_block("world"),
)
assert ai_client.complete_text(USER_HI, model=ai_client.REASONER_MODEL) == "Hello world"
def test_refusal_raises_ai_refusal_error(api: MockApi) -> None:
api.reply = lambda: _message_response(stop_reason="refusal", stop_details=REFUSAL_DETAILS)
message = ai_client.create_message(USER_HI)
assert message.stop_reason == "refusal"
with pytest.raises(ai_client.AIRefusalError, match="cyber"):
ai_client.reply_text(message)
with pytest.raises(ai_client.AIRefusalError):
ai_client.complete_text(USER_HI)
def test_rate_limit_surfaces_without_sdk_retries(api: MockApi) -> None:
api.reply = lambda: httpx2.Response(
429, json={"type": "error", "error": {"type": "rate_limit_error", "message": "slow down"}}
)
with pytest.raises(ai_client.RateLimitError):
ai_client.create_message(USER_HI)
assert len(api.sent) == 1, "retries belong to InferenceClient; SDK retries would stack under them"
def test_stream_text_yields_text_deltas_and_streams_summarized_thinking(api: MockApi) -> None:
api.reply = lambda: _stream_response(thinking="Weighing options.")
assert list(ai_client.stream_text(USER_HI, model=ai_client.REASONER_MODEL)) == ["Hel", "lo"]
body = api.sent[0].body
assert body["stream"] is True
assert body["thinking"] == {"type": "adaptive", "display": "summarized"}
def test_chat_model_stream_keeps_thinking_disabled(api: MockApi) -> None:
api.reply = lambda: _stream_response()
assert "".join(ai_client.stream_text(USER_HI, model=ai_client.CHAT_MODEL)) == "Hello"
assert api.sent[0].body["thinking"] == {"type": "disabled"}
def _drain_open_stream(model: str) -> None:
with ai_client.open_stream(USER_HI, model=model) as stream:
stream.get_final_message()
def _drain_stream_text(model: str) -> None:
list(ai_client.stream_text(USER_HI, model=model))
STREAM_ENTRY_POINTS = pytest.mark.parametrize(
"drain", [_drain_open_stream, _drain_stream_text], ids=["open_stream", "stream_text"]
)
@STREAM_ENTRY_POINTS
def test_reasoner_stream_carries_the_refusal_fallback(api: MockApi, drain: Callable[[str], None]) -> None:
api.reply = lambda: _stream_response(thinking="Weighing options.")
drain(ai_client.REASONER_MODEL)
request = api.sent[0]
assert request.body["stream"] is True
assert request.headers["anthropic-beta"] == "server-side-fallback-2026-07-01"
assert request.body["fallbacks"] == "default"
@STREAM_ENTRY_POINTS
def test_chat_model_stream_carries_no_refusal_fallback(api: MockApi, drain: Callable[[str], None]) -> None:
api.reply = lambda: _stream_response()
drain(ai_client.CHAT_MODEL)
request = api.sent[0]
assert request.body["stream"] is True
assert "anthropic-beta" not in request.headers
assert "fallbacks" not in request.body
def test_open_stream_exposes_thinking_deltas_and_final_usage(api: MockApi) -> None:
api.reply = lambda: _stream_response(thinking="Weighing options.")
with ai_client.open_stream(USER_HI, model=ai_client.REASONER_MODEL) as stream:
deltas = [event.delta for event in stream if event.type == "content_block_delta"]
final = stream.get_final_message()
assert "".join(delta.thinking for delta in deltas if delta.type == "thinking_delta") == "Weighing options."
assert "".join(delta.text for delta in deltas if delta.type == "text_delta") == "Hello"
assert final.stop_reason == "end_turn"
assert (final.usage.input_tokens, final.usage.output_tokens) == (9, 5)
def test_stream_text_raises_after_the_stream_when_claude_declines(api: MockApi) -> None:
api.reply = lambda: _stream_response(stop_reason="refusal", stop_details=REFUSAL_DETAILS)
received: list[str] = []
with pytest.raises(ai_client.AIRefusalError):
for chunk in ai_client.stream_text(USER_HI, model=ai_client.CHAT_MODEL):
received.append(chunk)
assert received == ["Hel", "lo"]
|