Spaces:
Running
Running
Retry transient non-JSON model responses
Browse files- app/llm.py +9 -0
- tests/test_llm.py +31 -0
app/llm.py
CHANGED
|
@@ -267,6 +267,15 @@ class LLMClient:
|
|
| 267 |
try:
|
| 268 |
payload = response.json()
|
| 269 |
except json.JSONDecodeError as exc:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 270 |
raise LLMError("The model endpoint returned non-JSON.",
|
| 271 |
code="llm_bad_response") from exc
|
| 272 |
parsed = _parse_anthropic(payload) if style == "anthropic" else _parse_openai(payload)
|
|
|
|
| 267 |
try:
|
| 268 |
payload = response.json()
|
| 269 |
except json.JSONDecodeError as exc:
|
| 270 |
+
# Some upstream gateways occasionally return an HTML or
|
| 271 |
+
# plain-text error body with HTTP 200 after a long model
|
| 272 |
+
# wait. Treat that exactly like the transient transport
|
| 273 |
+
# failures above while the configured retry budget
|
| 274 |
+
# remains; never expose or log the provider body.
|
| 275 |
+
if attempt + 1 < attempts:
|
| 276 |
+
await asyncio.sleep(delay + random.uniform(0, delay))
|
| 277 |
+
delay = min(delay * 4, 32.0)
|
| 278 |
+
continue
|
| 279 |
raise LLMError("The model endpoint returned non-JSON.",
|
| 280 |
code="llm_bad_response") from exc
|
| 281 |
parsed = _parse_anthropic(payload) if style == "anthropic" else _parse_openai(payload)
|
tests/test_llm.py
CHANGED
|
@@ -124,6 +124,37 @@ class TestAnthropicCall:
|
|
| 124 |
assert reply.text == "done"
|
| 125 |
assert len(calls) == 3
|
| 126 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 127 |
@pytest.mark.asyncio
|
| 128 |
async def test_truncation_raises(self, make_settings):
|
| 129 |
def handler(request):
|
|
|
|
| 124 |
assert reply.text == "done"
|
| 125 |
assert len(calls) == 3
|
| 126 |
|
| 127 |
+
@pytest.mark.asyncio
|
| 128 |
+
async def test_retry_on_non_json_success_then_success(
|
| 129 |
+
self, make_settings, monkeypatch
|
| 130 |
+
):
|
| 131 |
+
calls = []
|
| 132 |
+
|
| 133 |
+
async def no_sleep(_):
|
| 134 |
+
return None
|
| 135 |
+
|
| 136 |
+
monkeypatch.setattr("app.llm.asyncio.sleep", no_sleep)
|
| 137 |
+
|
| 138 |
+
def handler(request):
|
| 139 |
+
calls.append(request)
|
| 140 |
+
if len(calls) < 3:
|
| 141 |
+
return httpx.Response(
|
| 142 |
+
200,
|
| 143 |
+
text="<html>temporary upstream gateway response</html>",
|
| 144 |
+
headers={"content-type": "text/html"},
|
| 145 |
+
)
|
| 146 |
+
return anthropic_ok("done")
|
| 147 |
+
|
| 148 |
+
settings = make_settings(LLM_MAX_RETRIES=2)
|
| 149 |
+
transport = httpx.MockTransport(handler)
|
| 150 |
+
async with httpx.AsyncClient(transport=transport) as http:
|
| 151 |
+
client = LLMClient(settings, http=http)
|
| 152 |
+
reply = await client.complete_vision(
|
| 153 |
+
system="s", messages=[user_turn("hi")]
|
| 154 |
+
)
|
| 155 |
+
assert reply.text == "done"
|
| 156 |
+
assert len(calls) == 3
|
| 157 |
+
|
| 158 |
@pytest.mark.asyncio
|
| 159 |
async def test_truncation_raises(self, make_settings):
|
| 160 |
def handler(request):
|