File size: 4,595 Bytes
53b2ac9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 | # SPDX-License-Identifier: MIT
"""llm_request must surface server errors instead of hiding them.
Before: an HTTP error status (e.g. llama-server's 400 "request exceeds the
available context size") was reported as "Cannot reach llama-server" and
retried twice; an error event inside the SSE stream, or a plain JSON reply
from a server that ignored "stream": true, became a silent empty answer.
"""
import json
import os
import sys
import threading
from http.server import BaseHTTPRequestHandler, HTTPServer
import pytest
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
import trashclaw
@pytest.fixture
def fake_server(monkeypatch):
state = {"status": 200, "ctype": "text/event-stream", "body": b"", "posts": 0}
class Handler(BaseHTTPRequestHandler):
def log_message(self, *args):
pass
def do_POST(self):
self.rfile.read(int(self.headers.get("Content-Length", 0)))
state["posts"] += 1
body = state["body"]
self.send_response(state["status"])
self.send_header("Content-Type", state["ctype"])
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
server = HTTPServer(("127.0.0.1", 0), Handler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
monkeypatch.setattr(trashclaw, "LLAMA_URL", "http://127.0.0.1:%d" % server.server_address[1])
monkeypatch.setattr(trashclaw, "LLM_RETRY_DELAY", 0)
monkeypatch.setattr(trashclaw, "_INTERRUPTED", False)
yield state
server.shutdown()
server.server_close()
MSGS = [{"role": "user", "content": "hi"}]
def test_http_400_reports_server_message_and_is_not_retried(fake_server):
fake_server.update(status=400, ctype="application/json", body=json.dumps(
{"error": {"code": 400, "message": "the request exceeds the available context size"}}
).encode())
result = trashclaw.llm_request_with_retry(MSGS)
assert "error" in result
assert "Cannot reach" not in result["error"]
assert "HTTP 400" in result["error"]
assert "exceeds the available context size" in result["error"]
assert fake_server["posts"] == 1
def test_http_503_is_retried(fake_server):
fake_server.update(status=503, ctype="application/json",
body=b'{"error": {"message": "Loading model"}}')
result = trashclaw.llm_request_with_retry(MSGS)
assert "HTTP 503" in result["error"]
assert "Loading model" in result["error"]
assert fake_server["posts"] == trashclaw.LLM_RETRY_ATTEMPTS + 1
def test_error_event_in_stream_is_reported(fake_server):
fake_server.update(body=b'data: {"error": {"message": "model crashed"}}\n\n')
result = trashclaw.llm_request(MSGS)
assert "error" in result
assert "model crashed" in result["error"]
def test_plain_json_reply_is_used(fake_server):
fake_server.update(ctype="application/json", body=json.dumps({
"choices": [{"message": {"content": "hello there"}, "finish_reason": "stop"}]
}).encode())
result = trashclaw.llm_request(MSGS)
assert result["choices"][0]["message"]["content"] == "hello there"
def test_plain_json_error_is_reported(fake_server):
fake_server.update(ctype="application/json", body=b'{"error": "bad model name"}')
result = trashclaw.llm_request(MSGS)
assert "bad model name" in result.get("error", "")
def test_normal_stream_still_works(fake_server):
chunks = [
{"choices": [{"delta": {"content": "Hel"}}]},
{"choices": [{"delta": {"content": "lo"}, "finish_reason": "stop"}]},
]
body = ": keep-alive\n\n" + "".join("data: %s\n\n" % json.dumps(c) for c in chunks) + "data: [DONE]\n\n"
fake_server.update(body=body.encode())
result = trashclaw.llm_request(MSGS)
assert result["choices"][0]["message"]["content"] == "Hello"
assert result["choices"][0]["finish_reason"] == "stop"
def test_plain_json_reply_updates_stats(fake_server, monkeypatch):
monkeypatch.setattr(trashclaw, "LAST_GENERATION_STATS", {})
monkeypatch.setattr(trashclaw, "SESSION_STATS", dict(trashclaw.SESSION_STATS, turns=0,
total_tokens=0, total_seconds=0.0))
fake_server.update(ctype="application/json", body=json.dumps({
"choices": [{"message": {"content": "x" * 40}, "finish_reason": "stop"}]
}).encode())
trashclaw.llm_request(MSGS)
assert trashclaw.LAST_GENERATION_STATS["tokens"] == 10
assert trashclaw.SESSION_STATS["turns"] == 1
|