"""Provider abstraction for the eval agent.
One small interface, `ChatProvider.complete(messages, tools) -> ChatReply`.
Adapters:
* `OpenAICompatibleProvider` — OpenAI Chat Completions wire format. Covers
local **vLLM** (matches Training's rollout path) and **OpenRouter**
(the Phase-0 test target) by base_url alone.
* `BedrockProvider` — AWS Bedrock Converse. Translates the agent's
OpenAI-shape messages + tool schemas to Bedrock Converse and back to
the same `ChatReply` the OpenAI path returns, so the agent stays
provider-agnostic. Auth comes from the AWS credential chain (env /
shared config / role) — never hardcoded.
Selection is pure config (`ProviderConfig`); no provider-specific code
leaks into the agent.
"""
from __future__ import annotations
import json
import os
from dataclasses import dataclass, field
from typing import Any, Literal
import httpx
ProviderName = Literal["openai", "vllm", "openrouter", "bedrock", "together"]
# Convenience presets; base_url/api_key_env still overridable in config.
_PRESETS: dict[str, dict[str, str]] = {
"openrouter": {
"base_url": "https://openrouter.ai/api/v1",
"api_key_env": "OPENROUTER_API_KEY",
},
"vllm": {
"base_url": "http://localhost:8100/v1",
"api_key_env": "VLLM_API_KEY", # vLLM ignores the value
},
"openai": {
"base_url": "https://api.openai.com/v1",
"api_key_env": "OPENAI_API_KEY",
},
# together.ai — OpenAI-compatible API. Models are namespaced like
# `Qwen/Qwen3.6-Plus`, `meta-llama/Llama-3.3-70B-Instruct-Turbo`,
# etc. Native tool-calling supported on most chat models; check
# https://docs.together.ai/docs/function-calling for per-model
# gating before enabling tool use on a new model.
"together": {
"base_url": "https://api.together.xyz/v1",
"api_key_env": "TOGETHER_API_KEY",
},
# AWS Bedrock — auth via the boto3 credential chain (env, shared
# config, instance/role). `base_url` is unused (the SDK derives the
# endpoint from the region). `api_key_env` is unused (left for
# interface parity); `bedrock_region` on ProviderConfig wins.
"bedrock": {
"base_url": "",
"api_key_env": "",
},
}
@dataclass
class ProviderConfig:
provider: ProviderName = "openrouter"
model: str = "anthropic/claude-3.5-sonnet"
base_url: str | None = None
api_key_env: str | None = None
temperature: float = 0.7
max_tokens: int = 1024
timeout_s: float = 120.0
vision: bool = True
# Spatial channel: "vision" = PNG minimap; "structured" = NO image,
# a text "Unexplored regions" block instead (text-vs-vision A/B;
# pair structured runs with the easy/medium level of the setup).
fog_mode: str = "vision"
# Minimap unit colours: "auto" = per-type palette on hard, constant
# own/enemy colours on easy/medium; or force "per_type"/"constant".
minimap_color_mode: str = "auto"
extra_headers: dict[str, str] = field(default_factory=dict)
# Merged into the request JSON body — e.g. OpenRouter provider
# routing to avoid the rate-limited free pool:
# extra_body={"provider": {"sort": "throughput",
# "allow_fallbacks": True}}
# (premium/paid routing also needs account credits).
extra_body: dict = field(default_factory=dict)
# Streaming: some models (notably together.ai's Qwen3.6-Plus and
# other newer ones) refuse non-streaming requests with
# `streaming_required` 400. Set True to send `"stream": true` and
# accumulate the SSE chunks into a single ChatReply. The non-
# streaming path is the default for backward compatibility.
stream: bool = False
# Resilience (real OpenRouter runs): bounded retry, throttle, price.
max_retries: int = 5
retry_base_s: float = 1.0
retry_cap_s: float = 30.0
qps: float = 0.0 # 0 = unthrottled; shared limiter set by evaluate
max_history_turns: int = 16 # sliding wire-history window (0=unbounded)
price_in_per_m: float = 0.0 # USD / 1M prompt tokens
price_out_per_m: float = 0.0 # USD / 1M completion tokens
# AWS Bedrock: inference region. Sonnet 4.6 is exposed via the
# `us.anthropic.claude-sonnet-4-6` cross-region inference profile,
# which routes from `us-west-2` (the on-demand model id returns
# ValidationException — only the inference profile is callable).
bedrock_region: str = "us-west-2"
def resolved_base_url(self) -> str:
if self.base_url:
return self.base_url
preset = _PRESETS.get(self.provider)
if not preset:
raise ValueError(
f"no base_url and no preset for provider {self.provider!r}"
)
return preset["base_url"]
def resolved_api_key(self) -> str:
env = self.api_key_env or _PRESETS.get(self.provider, {}).get("api_key_env")
if not env:
raise ValueError(f"no api_key_env for provider {self.provider!r}")
key = os.environ.get(env, "")
if not key and self.provider != "vllm":
raise RuntimeError(
f"{env} not set — required for provider {self.provider!r}"
)
return key or "not-needed"
@dataclass
class ChatReply:
"""Normalized model reply."""
text: str
tool_calls: list[dict] # [{"name": str, "arguments": dict}]
reasoning: str = "" # chain-of-thought, when the model/provider emits it
usage: dict = field(default_factory=dict) # prompt/completion tokens
raw: dict = field(default_factory=dict)
class ChatProvider:
def complete(self, messages: list[dict], tools: list[dict]) -> ChatReply:
raise NotImplementedError
class OpenAICompatibleProvider(ChatProvider):
"""OpenAI /chat/completions with `tools`. vLLM + OpenRouter + OpenAI."""
def __init__(self, cfg: ProviderConfig, *, rate_limiter=None,
cost_meter=None):
self.cfg = cfg
self._client = httpx.Client(timeout=cfg.timeout_s)
from .resilience import CostMeter, RateLimiter, RetryPolicy
self._rl = rate_limiter or RateLimiter(cfg.qps)
self._cost = cost_meter or CostMeter(
cfg.price_in_per_m, cfg.price_out_per_m
)
self._policy = RetryPolicy(
max_attempts=max(1, cfg.max_retries),
base=cfg.retry_base_s,
cap=cfg.retry_cap_s,
)
# Audit hook: when set (a list), every successful complete()
# appends a dict {"request":
, "response": } so the
# FullPlayback recorder can capture the literal wire payloads.
# Drained by the caller after each turn. None disables capture.
self.request_log: list[dict] | None = None
@property
def cost_meter(self):
return self._cost
def _post_once(self, url, headers, body):
from .resilience import FatalProviderError
try:
resp = self._client.post(url, headers=headers, json=body)
except httpx.TimeoutException as e:
e.transient = True # type: ignore[attr-defined]
e.retry_after = None # type: ignore[attr-defined]
raise
except httpx.TransportError as e:
e.transient = True # type: ignore[attr-defined]
e.retry_after = None # type: ignore[attr-defined]
raise
if resp.status_code >= 400:
ra = resp.headers.get("retry-after")
try:
retry_after = float(ra) if ra is not None else None
except ValueError:
retry_after = None
transient = self._policy.is_transient_status(resp.status_code)
cls = RuntimeError if transient else FatalProviderError
exc = cls(
f"{resp.status_code} from provider: {resp.text[:800]}"
)
exc.transient = transient # type: ignore[attr-defined]
exc.retry_after = retry_after # type: ignore[attr-defined]
raise exc
return resp
def complete(self, messages: list[dict], tools: list[dict]) -> ChatReply:
from .resilience import retry_call
cfg = self.cfg
headers = {
"Authorization": f"Bearer {cfg.resolved_api_key()}",
"Content-Type": "application/json",
**cfg.extra_headers,
}
body: dict[str, Any] = {
"model": cfg.model,
"messages": self._wire_messages(messages),
"temperature": cfg.temperature,
"max_tokens": cfg.max_tokens,
}
if tools:
body["tools"] = tools
body["tool_choice"] = "auto"
if cfg.extra_body:
# e.g. OpenRouter {"provider": {...}} routing — premium/
# paid endpoints instead of the rate-limited free pool.
body.update(cfg.extra_body)
url = f"{cfg.resolved_base_url()}/chat/completions"
self._rl.acquire()
if cfg.stream:
body["stream"] = True
# together.ai gates usage emission on this flag (otherwise
# usage is null in streaming mode → cost-meter sees zero).
body.setdefault(
"stream_options", {"include_usage": True}
)
reply = retry_call(
lambda: self._stream_once(url, headers, body), self._policy
)
else:
resp = retry_call(
lambda: self._post_once(url, headers, body), self._policy
)
reply = self._reply_from_data(resp.json())
u = reply.usage or {}
self._cost.add(u.get("prompt_tokens", 0), u.get("completion_tokens", 0))
self._cost.check() # raises BudgetExceeded → evaluate finalizes
if self.request_log is not None:
# Audit capture: redact the bearer header (the body is the
# interesting part) and store the literal request + raw
# response side-by-side. FullPlayback drains after each turn.
try:
self.request_log.append(
{
"request": {
"url": url,
"body": body,
},
"response": {
"raw": reply.raw,
"text": reply.text,
"tool_calls": reply.tool_calls,
"reasoning": reply.reasoning,
"usage": dict(reply.usage or {}),
"finish_reason": (
(reply.raw.get("choices") or [{}])[0]
.get("finish_reason")
if isinstance(reply.raw, dict)
else None
),
},
}
)
except Exception: # noqa: BLE001 — audit must never break a run
pass
return reply
def _stream_once(self, url, headers, body) -> ChatReply:
"""Streaming POST: accumulate SSE chunks into a single ChatReply.
OpenAI's stream format yields chunks with `choices[0].delta`
containing partial `content`, partial `tool_calls`, and
eventually `finish_reason`. Tool-call `function.arguments`
arrives as a stream of JSON-string fragments that must be
concatenated per `index`. The final chunk (with
`stream_options: include_usage`) carries the usage dict.
"""
from .resilience import FatalProviderError
content_parts: list[str] = []
reasoning_parts: list[str] = []
# tool_calls accumulator keyed by delta.tool_calls[i].index
# — providers stream calls in any interleaving; we re-assemble
# by index, then materialise to a list in index order.
tcs_acc: dict[int, dict[str, Any]] = {}
usage: dict[str, int] | None = None
finish_reason: str | None = None
try:
with self._client.stream(
"POST", url, headers=headers, json=body
) as resp:
if resp.status_code >= 400:
body_text = resp.read().decode("utf-8", errors="replace")
ra = resp.headers.get("retry-after")
try:
retry_after = float(ra) if ra is not None else None
except ValueError:
retry_after = None
transient = self._policy.is_transient_status(
resp.status_code
)
cls = RuntimeError if transient else FatalProviderError
exc = cls(
f"{resp.status_code} from provider: "
f"{body_text[:800]}"
)
exc.transient = transient # type: ignore[attr-defined]
exc.retry_after = retry_after # type: ignore[attr-defined]
raise exc
for line in resp.iter_lines():
if not line or not line.startswith("data:"):
continue
payload = line[5:].lstrip()
if payload == "[DONE]":
break
try:
chunk = json.loads(payload)
except json.JSONDecodeError:
continue
if chunk.get("usage"):
usage = chunk["usage"]
for ch in chunk.get("choices") or []:
d = ch.get("delta") or {}
if d.get("content"):
content_parts.append(d["content"])
# vLLM / DeepSeek-style reasoning channel
rc = d.get("reasoning_content") or d.get("reasoning")
if rc:
reasoning_parts.append(
rc if isinstance(rc, str) else str(rc)
)
for tc in d.get("tool_calls") or []:
idx = tc.get("index", 0)
slot = tcs_acc.setdefault(
idx, {"id": "", "type": "function",
"function": {"name": "",
"arguments": ""}}
)
if tc.get("id"):
slot["id"] = tc["id"]
fn = tc.get("function") or {}
if fn.get("name"):
slot["function"]["name"] = fn["name"]
if fn.get("arguments") is not None:
slot["function"]["arguments"] += fn[
"arguments"
]
if ch.get("finish_reason"):
finish_reason = ch["finish_reason"]
except httpx.TimeoutException as e:
e.transient = True # type: ignore[attr-defined]
e.retry_after = None # type: ignore[attr-defined]
raise
except httpx.TransportError as e:
e.transient = True # type: ignore[attr-defined]
e.retry_after = None # type: ignore[attr-defined]
raise
# Re-pack into the non-streaming response shape and re-use
# the existing _reply_from_data parser (which already handles
# tool_call argument JSON-decoding).
tool_calls_list = [tcs_acc[i] for i in sorted(tcs_acc)]
message: dict[str, Any] = {"role": "assistant"}
if content_parts:
message["content"] = "".join(content_parts)
if tool_calls_list:
message["tool_calls"] = tool_calls_list
if reasoning_parts:
message["reasoning"] = "".join(reasoning_parts)
data = {
"choices": [{"message": message, "finish_reason": finish_reason}],
"usage": usage,
}
return self._reply_from_data(data)
# Keys the OpenAI Chat Completions wire format accepts per message.
# `history` carries extra playback-only keys (notably "reasoning");
# those must never be posted back or strict servers (vLLM) 400.
_WIRE_KEYS = frozenset(
{"role", "content", "name", "tool_calls", "tool_call_id"}
)
@staticmethod
def _wire_messages(messages: list[dict]) -> list[dict]:
"""Pure: project each message onto OpenAI-legal keys only, and
coerce `tool_calls[].function.arguments` to a JSON **string**
(the wire spec requires a string; history keeps the dict for
readable playback). Pure — inputs are not mutated."""
out: list[dict] = []
for m in messages:
wm = {
k: v for k, v in m.items()
if k in OpenAICompatibleProvider._WIRE_KEYS
}
tcs = wm.get("tool_calls")
if tcs:
fixed = []
for tc in tcs:
fn = dict(tc.get("function", {}))
args = fn.get("arguments", {})
if not isinstance(args, str):
fn["arguments"] = json.dumps(args)
fixed.append({**tc, "function": fn})
wm["tool_calls"] = fixed
elif "tool_calls" in wm:
# Strict endpoints (Together's Qwen3.6-Plus, some vLLM
# builds) reject an assistant message that carries
# `tool_calls: []` — "Empty tool_calls is not supported
# in message." A plain-text assistant turn must omit
# the key entirely.
wm.pop("tool_calls", None)
out.append(wm)
return out
@staticmethod
def _reply_from_data(data: dict) -> ChatReply:
"""Pure parse of a Chat Completions response, including the
provider-specific reasoning channel (vLLM/DeepSeek emit
`reasoning_content`; OpenRouter/others a flat `reasoning`)."""
msg = data["choices"][0]["message"]
calls: list[dict] = []
for tc in msg.get("tool_calls") or []:
fn = tc.get("function", {})
args = fn.get("arguments", {})
if isinstance(args, str):
try:
args = json.loads(args or "{}")
except json.JSONDecodeError:
args = {}
calls.append({"name": fn.get("name", ""), "arguments": args})
rc = msg.get("reasoning_content") or msg.get("reasoning") or ""
if isinstance(rc, list): # some providers chunk it
rc = "".join(
p.get("text", "") if isinstance(p, dict) else str(p) for p in rc
)
usage = data.get("usage") or {}
return ChatReply(
text=msg.get("content") or "",
tool_calls=calls,
reasoning=str(rc),
usage={
"prompt_tokens": usage.get("prompt_tokens", 0),
"completion_tokens": usage.get("completion_tokens", 0),
},
raw=data,
)
def close(self) -> None:
self._client.close()
class BedrockProvider(ChatProvider):
"""AWS Bedrock Converse adapter.
Translates between the agent's OpenAI-shape messages + tool
schemas and the Bedrock Converse wire format, and translates the
response back to a `ChatReply` so the agent and FullPlayback see
the SAME shape they get from the OpenAI-compatible path. Auth
flows through boto3's standard credential chain — env vars, the
shared config file, IAM role, etc. The model id is the inference
profile id (`us.anthropic.claude-sonnet-4-6`), not the on-demand
model id (which returns ValidationException).
Wire-shape mapping:
* OpenAI `system` messages → top-level `system: [{text}]`
* OpenAI text user/assistant → `content: [{text}]`
* OpenAI multimodal user content → `content: [{text}, {image}]`
* OpenAI assistant `tool_calls` → `content: [{toolUse}]`
* OpenAI `tool` reply → user `[{toolResult}]`
* OpenAI `tools` (JSON-Schema) → `toolConfig: {tools: [{toolSpec}]}`
* Bedrock `output.message.content` → ChatReply.text + tool_calls
* Bedrock `usage.{input,output}Tokens` → usage.{prompt,completion}_tokens
Tool-call ids: Bedrock requires a `toolUseId` on every assistant
`toolUse` and the matching user `toolResult`. The bench agent
canonicalises these as `c0/c1/...` per turn, so the translation
passes them straight through.
"""
def __init__(self, cfg: ProviderConfig, *, rate_limiter=None,
cost_meter=None, client=None):
self.cfg = cfg
self.model_id = cfg.model
from .resilience import CostMeter, RateLimiter, RetryPolicy
self._rl = rate_limiter or RateLimiter(cfg.qps)
self._cost = cost_meter or CostMeter(
cfg.price_in_per_m, cfg.price_out_per_m
)
self._policy = RetryPolicy(
max_attempts=max(1, cfg.max_retries),
base=cfg.retry_base_s,
cap=cfg.retry_cap_s,
)
# Lazy import: keep boto3 a soft dep — only providers='bedrock'
# forces the dependency, never the OpenRouter / vLLM paths.
if client is not None:
self._client = client
else:
try:
import boto3
except ImportError as e: # pragma: no cover — env-dep
raise RuntimeError(
"BedrockProvider needs boto3. Install with "
"`pip install boto3`."
) from e
self._client = boto3.client(
"bedrock-runtime", region_name=cfg.bedrock_region
)
# Audit hook (parallels OpenAICompatibleProvider): when set to
# a list, every successful complete() appends a record so
# FullPlayback can capture literal request + raw response.
self.request_log: list[dict] | None = None
@property
def cost_meter(self):
return self._cost
# ── Wire translation: OpenAI → Bedrock ──────────────────────────
@staticmethod
def _to_bedrock_messages(messages: list[dict]) -> tuple[list[dict], list[dict]]:
"""Pure: split OpenAI messages into (system, conversation).
System messages are concatenated into a list of `{text}` blocks
for Bedrock's top-level `system` parameter. Tool replies
(`role=tool`) become user-role `toolResult` content blocks; an
assistant message with `tool_calls` becomes Bedrock `toolUse`
content blocks (text content, if any, is preserved alongside).
Adjacent same-role messages are merged because Bedrock REQUIRES
strictly alternating user/assistant turns — a `tool` reply
followed by another user briefing must collapse into ONE
Bedrock user message with multiple content blocks.
"""
sys_blocks: list[dict] = []
out: list[dict] = []
for m in messages:
role = m.get("role")
if role == "system":
txt = m.get("content")
if isinstance(txt, list):
txt = "\n".join(
p.get("text", "") for p in txt
if isinstance(p, dict) and p.get("type") == "text"
)
if txt:
sys_blocks.append({"text": str(txt)})
continue
blocks = BedrockProvider._content_to_blocks(m)
if not blocks:
continue
br_role = "user" if role in ("user", "tool") else "assistant"
if out and out[-1]["role"] == br_role:
out[-1]["content"].extend(blocks)
else:
out.append({"role": br_role, "content": blocks})
return sys_blocks, out
@staticmethod
def _content_to_blocks(msg: dict) -> list[dict]:
"""Pure: OpenAI message → list of Bedrock content blocks."""
role = msg.get("role")
# Tool-result reply → toolResult block.
if role == "tool":
tcid = msg.get("tool_call_id") or ""
content = msg.get("content")
if isinstance(content, list):
content = " ".join(
p.get("text", "") for p in content
if isinstance(p, dict) and p.get("type") == "text"
)
return [{
"toolResult": {
"toolUseId": str(tcid),
"content": [{"text": str(content) if content else "ok"}],
}
}]
blocks: list[dict] = []
c = msg.get("content")
if isinstance(c, str):
if c:
blocks.append({"text": c})
elif isinstance(c, list):
for part in c:
if not isinstance(part, dict):
continue
t = part.get("type")
if t == "text":
txt = part.get("text", "")
if txt:
blocks.append({"text": txt})
elif t == "image_url":
iu = part.get("image_url") or {}
url = iu.get("url", "") if isinstance(iu, dict) else ""
img = BedrockProvider._image_block_from_data_url(url)
if img is not None:
blocks.append(img)
# Assistant tool_calls → toolUse blocks (after any text).
for tc in msg.get("tool_calls") or []:
fn = tc.get("function") or {}
args = fn.get("arguments", {})
if isinstance(args, str):
try:
args = json.loads(args or "{}")
except json.JSONDecodeError:
args = {}
if not isinstance(args, dict):
args = {}
blocks.append({
"toolUse": {
"toolUseId": str(tc.get("id") or ""),
"name": fn.get("name", ""),
"input": args,
}
})
return blocks
@staticmethod
def _image_block_from_data_url(url: str) -> dict | None:
"""Pure: turn a `data:image/png;base64,...` URL into a Bedrock
`{image: {format, source: {bytes}}}` block. Bedrock accepts
png / jpeg / gif / webp; the bench only emits png minimaps."""
import base64
if not url.startswith("data:"):
return None
try:
header, b64 = url.split(",", 1)
except ValueError:
return None
fmt = "png"
if "image/" in header:
mt = header.split("image/", 1)[1].split(";", 1)[0].lower()
if mt in ("png", "jpeg", "jpg", "gif", "webp"):
fmt = "jpeg" if mt == "jpg" else mt
try:
raw = base64.b64decode(b64)
except (ValueError, TypeError):
return None
return {"image": {"format": fmt, "source": {"bytes": raw}}}
@staticmethod
def _to_bedrock_tools(tools: list[dict]) -> dict | None:
"""Pure: OpenAI tool list → Bedrock `toolConfig`. The OpenAI
schema is `{type: "function", function: {name, description,
parameters}}`; Bedrock wants `{toolSpec: {name, description,
inputSchema: {json: }}}`. Bedrock additionally
requires `inputSchema.json.type` (some agents emit empty
params) — we backfill an empty object schema."""
if not tools:
return None
specs = []
for t in tools:
fn = t.get("function") or {}
params = fn.get("parameters") or {"type": "object", "properties": {}}
if "type" not in params:
params = {"type": "object", **params}
specs.append({
"toolSpec": {
"name": fn.get("name", ""),
"description": fn.get("description", ""),
"inputSchema": {"json": params},
}
})
return {"tools": specs}
# ── Wire translation: Bedrock → ChatReply ───────────────────────
@staticmethod
def _reply_from_bedrock(resp: dict) -> ChatReply:
"""Pure: parse a Bedrock Converse response into a ChatReply.
Bedrock emits one assistant message; its content blocks are
either `{text}` (plain reply) or `{toolUse}` (a function call).
We concatenate text blocks and lift toolUse blocks into the
same `[{name, arguments}]` list the OpenAI parser produces."""
msg = (resp.get("output") or {}).get("message") or {}
content_blocks = msg.get("content") or []
text_parts: list[str] = []
calls: list[dict] = []
reasoning_parts: list[str] = []
for blk in content_blocks:
if not isinstance(blk, dict):
continue
if "text" in blk:
text_parts.append(blk["text"])
elif "toolUse" in blk:
tu = blk["toolUse"]
calls.append({
"name": tu.get("name", ""),
"arguments": tu.get("input") or {},
})
elif "reasoningContent" in blk:
# Bedrock surfaces extended thinking under
# reasoningContent.{reasoningText: {text}} — preserve
# it on the reply for FullPlayback.
rc = blk["reasoningContent"] or {}
rt = rc.get("reasoningText") or {}
t = rt.get("text") if isinstance(rt, dict) else None
if t:
reasoning_parts.append(str(t))
usage = resp.get("usage") or {}
return ChatReply(
text="".join(text_parts),
tool_calls=calls,
reasoning="".join(reasoning_parts),
usage={
"prompt_tokens": int(usage.get("inputTokens", 0) or 0),
"completion_tokens": int(usage.get("outputTokens", 0) or 0),
},
raw=resp,
)
# ── Public API ──────────────────────────────────────────────────
def _converse_once(self, system_blocks, br_messages, tool_config,
inference_cfg) -> dict:
from .resilience import FatalProviderError
try:
kwargs = {
"modelId": self.model_id,
"messages": br_messages,
"inferenceConfig": inference_cfg,
}
if system_blocks:
kwargs["system"] = system_blocks
if tool_config:
kwargs["toolConfig"] = tool_config
return self._client.converse(**kwargs)
except Exception as e: # noqa: BLE001
# Boto raises ClientError with a `response[Error][Code]`.
code = ""
status = 0
try:
err = getattr(e, "response", {}) or {}
meta = err.get("ResponseMetadata") or {}
status = int(meta.get("HTTPStatusCode", 0) or 0)
code = (err.get("Error") or {}).get("Code", "")
except Exception: # noqa: BLE001
pass
transient = status in (408, 425, 429, 500, 502, 503, 504) or code in (
"ThrottlingException",
"ServiceUnavailableException",
"ModelTimeoutException",
"InternalServerException",
"ModelStreamErrorException",
)
cls = RuntimeError if transient else FatalProviderError
new = cls(f"bedrock {code or status or 'error'}: {e}")
new.transient = transient # type: ignore[attr-defined]
new.retry_after = None # type: ignore[attr-defined]
raise new from e
def complete(self, messages: list[dict], tools: list[dict]) -> ChatReply:
from .resilience import retry_call
cfg = self.cfg
sys_blocks, br_messages = self._to_bedrock_messages(messages)
tool_config = self._to_bedrock_tools(tools)
inference_cfg = {
"temperature": cfg.temperature,
"maxTokens": cfg.max_tokens,
}
self._rl.acquire()
resp = retry_call(
lambda: self._converse_once(
sys_blocks, br_messages, tool_config, inference_cfg,
),
self._policy,
)
reply = self._reply_from_bedrock(resp)
u = reply.usage or {}
self._cost.add(u.get("prompt_tokens", 0), u.get("completion_tokens", 0))
self._cost.check()
if self.request_log is not None:
try:
# Redact image bytes from the request log (they're
# huge, and duplicated per turn). Replace with a
# short placeholder; the rest of the body is small.
def _redact(b):
if isinstance(b, dict):
return {k: _redact(v) for k, v in b.items()}
if isinstance(b, list):
return [_redact(x) for x in b]
if isinstance(b, (bytes, bytearray)):
return f""
return b
self.request_log.append({
"request": {
"model": self.model_id,
"system": _redact(sys_blocks),
"messages": _redact(br_messages),
"toolConfig": tool_config,
"inferenceConfig": inference_cfg,
},
"response": {
"raw": _redact(reply.raw),
"text": reply.text,
"tool_calls": reply.tool_calls,
"reasoning": reply.reasoning,
"usage": dict(reply.usage or {}),
"finish_reason": resp.get("stopReason"),
},
})
except Exception: # noqa: BLE001 — audit must never break a run
pass
return reply
def close(self) -> None: # noqa: D401 — interface parity
# boto3 clients don't need explicit close; provided for
# symmetry with OpenAICompatibleProvider.
pass
def make_provider(cfg: ProviderConfig, *, rate_limiter=None,
cost_meter=None) -> ChatProvider:
if cfg.provider == "bedrock":
return BedrockProvider(
cfg, rate_limiter=rate_limiter, cost_meter=cost_meter,
)
if cfg.provider in ("openai", "vllm", "openrouter", "together"):
# together.ai's newer Qwen3.x and Llama-3.x families gate on
# streaming (`streaming_required` 400 in non-stream mode); flip
# the default on for that provider so the SSE-accumulating path
# in OpenAICompatibleProvider takes over. Users can still
# force-disable via cfg.stream=False at construction.
if cfg.provider == "together" and cfg.stream is False:
cfg.stream = True
return OpenAICompatibleProvider(
cfg, rate_limiter=rate_limiter, cost_meter=cost_meter
)
raise ValueError(f"unknown provider {cfg.provider!r}")