Spaces:
Running
Running
Wimmboo commited on
Commit ·
dbc3826
1
Parent(s): 21088e6
fix(google): buffer non-streaming JSON, convert to fake SSE chunks
Browse filesGoogle streaming through HF Spaces HTTP/2 edge causes
ERR_HTTP2_PROTOCOL_ERROR regardless of content. Workaround:
force stream=false upstream, buffer full JSON response, convert
to OpenAI SSE format (role + content + final + [DONE]),
return as plain Response with text/event-stream Content-Type.
No StreamingResponse involved = no HTTP/2 streaming edge case.
- proxy/client.py +52 -6
proxy/client.py
CHANGED
|
@@ -5,7 +5,7 @@ from typing import Any, AsyncIterator
|
|
| 5 |
|
| 6 |
import httpx
|
| 7 |
from fastapi import HTTPException
|
| 8 |
-
from fastapi.responses import StreamingResponse
|
| 9 |
from starlette.requests import Request
|
| 10 |
|
| 11 |
from config import POLL_INTERVAL, POLL_MAX_ATTEMPTS
|
|
@@ -111,6 +111,47 @@ def _strip_extra_content_json(content: bytes) -> bytes:
|
|
| 111 |
return content
|
| 112 |
|
| 113 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 114 |
async def _stream_response(provider: Provider, real_key: str, body: dict[str, Any]) -> AsyncIterator[bytes]:
|
| 115 |
"""Async generator that manages its own httpx client for streaming."""
|
| 116 |
target_url = f"{provider.base_url}/v1/chat/completions"
|
|
@@ -140,12 +181,12 @@ async def forward_request(provider: Provider, real_key: str, request: Request, b
|
|
| 140 |
body = _sanitize_body(provider, body)
|
| 141 |
target_url = f"{provider.base_url}/v1/chat/completions"
|
| 142 |
headers = _build_headers(provider, real_key, dict(request.headers))
|
|
|
|
| 143 |
|
| 144 |
-
if
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
return StreamingResponse(_stream_response(provider, real_key, body), media_type="text/event-stream")
|
| 149 |
|
| 150 |
async with httpx.AsyncClient(timeout=_TIMEOUT, follow_redirects=True) as client:
|
| 151 |
try:
|
|
@@ -172,6 +213,11 @@ async def forward_request(provider: Provider, real_key: str, request: Request, b
|
|
| 172 |
content = upstream.content
|
| 173 |
if provider.name == "google":
|
| 174 |
content = _strip_extra_content_json(content)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 175 |
|
| 176 |
return {
|
| 177 |
"status_code": upstream.status_code,
|
|
|
|
| 5 |
|
| 6 |
import httpx
|
| 7 |
from fastapi import HTTPException
|
| 8 |
+
from fastapi.responses import Response, StreamingResponse
|
| 9 |
from starlette.requests import Request
|
| 10 |
|
| 11 |
from config import POLL_INTERVAL, POLL_MAX_ATTEMPTS
|
|
|
|
| 111 |
return content
|
| 112 |
|
| 113 |
|
| 114 |
+
def _json_to_sse(content: bytes) -> bytes:
|
| 115 |
+
"""Convert a non-streaming OpenAI JSON response into SSE chunks."""
|
| 116 |
+
try:
|
| 117 |
+
obj = json.loads(content)
|
| 118 |
+
base = {
|
| 119 |
+
"id": obj.get("id", "chatcmpl-proxy"),
|
| 120 |
+
"object": "chat.completion.chunk",
|
| 121 |
+
"created": obj.get("created", 0),
|
| 122 |
+
"model": obj.get("model", ""),
|
| 123 |
+
}
|
| 124 |
+
events = []
|
| 125 |
+
for choice in obj.get("choices", []):
|
| 126 |
+
idx = choice.get("index", 0)
|
| 127 |
+
message = choice.get("message", {})
|
| 128 |
+
content_text = message.get("content", "")
|
| 129 |
+
finish_reason = choice.get("finish_reason", None)
|
| 130 |
+
|
| 131 |
+
role_chunk = dict(base)
|
| 132 |
+
role_chunk["choices"] = [{"index": idx, "delta": {"role": message.get("role", "assistant")}}]
|
| 133 |
+
if finish_reason:
|
| 134 |
+
role_chunk["choices"][0]["finish_reason"] = finish_reason
|
| 135 |
+
events.append(f"data: {json.dumps(role_chunk, ensure_ascii=False)}\n\n")
|
| 136 |
+
|
| 137 |
+
if content_text:
|
| 138 |
+
content_chunk = dict(base)
|
| 139 |
+
content_chunk["choices"] = [{"index": idx, "delta": {"content": content_text}}]
|
| 140 |
+
if finish_reason:
|
| 141 |
+
content_chunk["choices"][0]["finish_reason"] = finish_reason
|
| 142 |
+
events.append(f"data: {json.dumps(content_chunk, ensure_ascii=False)}\n\n")
|
| 143 |
+
|
| 144 |
+
if finish_reason:
|
| 145 |
+
final_chunk = dict(base)
|
| 146 |
+
final_chunk["choices"] = [{"index": idx, "delta": {}, "finish_reason": finish_reason}]
|
| 147 |
+
events.append(f"data: {json.dumps(final_chunk, ensure_ascii=False)}\n\n")
|
| 148 |
+
|
| 149 |
+
events.append("data: [DONE]\n\n")
|
| 150 |
+
return "".join(events).encode("utf-8")
|
| 151 |
+
except (json.JSONDecodeError, KeyError, TypeError, UnicodeDecodeError):
|
| 152 |
+
return content
|
| 153 |
+
|
| 154 |
+
|
| 155 |
async def _stream_response(provider: Provider, real_key: str, body: dict[str, Any]) -> AsyncIterator[bytes]:
|
| 156 |
"""Async generator that manages its own httpx client for streaming."""
|
| 157 |
target_url = f"{provider.base_url}/v1/chat/completions"
|
|
|
|
| 181 |
body = _sanitize_body(provider, body)
|
| 182 |
target_url = f"{provider.base_url}/v1/chat/completions"
|
| 183 |
headers = _build_headers(provider, real_key, dict(request.headers))
|
| 184 |
+
original_stream = bool(body.get("stream"))
|
| 185 |
|
| 186 |
+
if original_stream and provider.name == "google":
|
| 187 |
+
body["stream"] = False
|
| 188 |
+
elif original_stream:
|
| 189 |
+
return StreamingResponse(_stream_response(provider, real_key, body), media_type="text/event-stream")
|
|
|
|
| 190 |
|
| 191 |
async with httpx.AsyncClient(timeout=_TIMEOUT, follow_redirects=True) as client:
|
| 192 |
try:
|
|
|
|
| 213 |
content = upstream.content
|
| 214 |
if provider.name == "google":
|
| 215 |
content = _strip_extra_content_json(content)
|
| 216 |
+
if original_stream:
|
| 217 |
+
content = _json_to_sse(content)
|
| 218 |
+
|
| 219 |
+
if original_stream and provider.name == "google":
|
| 220 |
+
return Response(content=content, media_type="text/event-stream")
|
| 221 |
|
| 222 |
return {
|
| 223 |
"status_code": upstream.status_code,
|