Wimmboo commited on
Commit
dbc3826
·
1 Parent(s): 21088e6

fix(google): buffer non-streaming JSON, convert to fake SSE chunks

Browse files

Google streaming through HF Spaces HTTP/2 edge causes
ERR_HTTP2_PROTOCOL_ERROR regardless of content. Workaround:
force stream=false upstream, buffer full JSON response, convert
to OpenAI SSE format (role + content + final + [DONE]),
return as plain Response with text/event-stream Content-Type.
No StreamingResponse involved = no HTTP/2 streaming edge case.

Files changed (1) hide show
  1. proxy/client.py +52 -6
proxy/client.py CHANGED
@@ -5,7 +5,7 @@ from typing import Any, AsyncIterator
5
 
6
  import httpx
7
  from fastapi import HTTPException
8
- from fastapi.responses import StreamingResponse
9
  from starlette.requests import Request
10
 
11
  from config import POLL_INTERVAL, POLL_MAX_ATTEMPTS
@@ -111,6 +111,47 @@ def _strip_extra_content_json(content: bytes) -> bytes:
111
  return content
112
 
113
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
114
  async def _stream_response(provider: Provider, real_key: str, body: dict[str, Any]) -> AsyncIterator[bytes]:
115
  """Async generator that manages its own httpx client for streaming."""
116
  target_url = f"{provider.base_url}/v1/chat/completions"
@@ -140,12 +181,12 @@ async def forward_request(provider: Provider, real_key: str, request: Request, b
140
  body = _sanitize_body(provider, body)
141
  target_url = f"{provider.base_url}/v1/chat/completions"
142
  headers = _build_headers(provider, real_key, dict(request.headers))
 
143
 
144
- if bool(body.get("stream")):
145
- if provider.name == "google":
146
- body["stream"] = False
147
- else:
148
- return StreamingResponse(_stream_response(provider, real_key, body), media_type="text/event-stream")
149
 
150
  async with httpx.AsyncClient(timeout=_TIMEOUT, follow_redirects=True) as client:
151
  try:
@@ -172,6 +213,11 @@ async def forward_request(provider: Provider, real_key: str, request: Request, b
172
  content = upstream.content
173
  if provider.name == "google":
174
  content = _strip_extra_content_json(content)
 
 
 
 
 
175
 
176
  return {
177
  "status_code": upstream.status_code,
 
5
 
6
  import httpx
7
  from fastapi import HTTPException
8
+ from fastapi.responses import Response, StreamingResponse
9
  from starlette.requests import Request
10
 
11
  from config import POLL_INTERVAL, POLL_MAX_ATTEMPTS
 
111
  return content
112
 
113
 
114
+ def _json_to_sse(content: bytes) -> bytes:
115
+ """Convert a non-streaming OpenAI JSON response into SSE chunks."""
116
+ try:
117
+ obj = json.loads(content)
118
+ base = {
119
+ "id": obj.get("id", "chatcmpl-proxy"),
120
+ "object": "chat.completion.chunk",
121
+ "created": obj.get("created", 0),
122
+ "model": obj.get("model", ""),
123
+ }
124
+ events = []
125
+ for choice in obj.get("choices", []):
126
+ idx = choice.get("index", 0)
127
+ message = choice.get("message", {})
128
+ content_text = message.get("content", "")
129
+ finish_reason = choice.get("finish_reason", None)
130
+
131
+ role_chunk = dict(base)
132
+ role_chunk["choices"] = [{"index": idx, "delta": {"role": message.get("role", "assistant")}}]
133
+ if finish_reason:
134
+ role_chunk["choices"][0]["finish_reason"] = finish_reason
135
+ events.append(f"data: {json.dumps(role_chunk, ensure_ascii=False)}\n\n")
136
+
137
+ if content_text:
138
+ content_chunk = dict(base)
139
+ content_chunk["choices"] = [{"index": idx, "delta": {"content": content_text}}]
140
+ if finish_reason:
141
+ content_chunk["choices"][0]["finish_reason"] = finish_reason
142
+ events.append(f"data: {json.dumps(content_chunk, ensure_ascii=False)}\n\n")
143
+
144
+ if finish_reason:
145
+ final_chunk = dict(base)
146
+ final_chunk["choices"] = [{"index": idx, "delta": {}, "finish_reason": finish_reason}]
147
+ events.append(f"data: {json.dumps(final_chunk, ensure_ascii=False)}\n\n")
148
+
149
+ events.append("data: [DONE]\n\n")
150
+ return "".join(events).encode("utf-8")
151
+ except (json.JSONDecodeError, KeyError, TypeError, UnicodeDecodeError):
152
+ return content
153
+
154
+
155
  async def _stream_response(provider: Provider, real_key: str, body: dict[str, Any]) -> AsyncIterator[bytes]:
156
  """Async generator that manages its own httpx client for streaming."""
157
  target_url = f"{provider.base_url}/v1/chat/completions"
 
181
  body = _sanitize_body(provider, body)
182
  target_url = f"{provider.base_url}/v1/chat/completions"
183
  headers = _build_headers(provider, real_key, dict(request.headers))
184
+ original_stream = bool(body.get("stream"))
185
 
186
+ if original_stream and provider.name == "google":
187
+ body["stream"] = False
188
+ elif original_stream:
189
+ return StreamingResponse(_stream_response(provider, real_key, body), media_type="text/event-stream")
 
190
 
191
  async with httpx.AsyncClient(timeout=_TIMEOUT, follow_redirects=True) as client:
192
  try:
 
213
  content = upstream.content
214
  if provider.name == "google":
215
  content = _strip_extra_content_json(content)
216
+ if original_stream:
217
+ content = _json_to_sse(content)
218
+
219
+ if original_stream and provider.name == "google":
220
+ return Response(content=content, media_type="text/event-stream")
221
 
222
  return {
223
  "status_code": upstream.status_code,