fix(stream): add proxy_buffering off, X-Accel-Buffering no headers, and sitecustomize monkeypatch to eliminate blank responses and sentence-transformers errors

entrypoint.sh

@@ -565,10 +565,13 @@ os.environ["OPENAI_API_KEY"] = "sk-2e556e0437ee2958-7baf2d-b4133935"
 os.environ["HERMES_API_BASE_URL"] = "http://127.0.0.1:20128/api/v1"
 os.environ["HERMES_API_KEY"] = "sk-2e556e0437ee2958-7baf2d-b4133935"
 try:
-    import litellm
-    litellm.api_base = "http://127.0.0.1:20128/api/v1"
-    litellm.api_key = "sk-2e556e0437ee2958-7baf2d-b4133935"
-    litellm.suppress_debug_info = True
+    import open_webui.routers.retrieval as rrouter
+    rrouter.get_ef = lambda engine, model: None
+except Exception:
+    pass
+try:
+    import open_webui.retrieval.utils as rutils
+    rutils.get_model_path = lambda model, auto_update=False: ""
 except Exception:
     pass
 PYCUSTOM

gateway/hermes_standalone.py

@@ -239,6 +239,13 @@ async def chat_completions(request: Request):
     duration_ms = (time.time() - t0) * 1000
     record_hermes_telemetry("chat_completions_fallback", duration_ms, {"model": model, "stream": stream})
 
+    stream_headers = {
+        "Cache-Control": "no-cache",
+        "Connection": "keep-alive",
+        "X-Accel-Buffering": "no",
+        "Content-Type": "text/event-stream"
+    }
+
     if stream:
         async def stream_generator():
             chunk = {
@@ -246,7 +253,7 @@ async def chat_completions(request: Request):
                 "object": "chat.completion.chunk",
                 "created": int(time.time()),
                 "model": model,
-                "choices": [{"index": 0, "delta": {"role": "assistant", "content": reply_text}, "finish_reason": None}]
+                "choices": [{"index": 0, "delta": {"role": "assistant", "content": reply_text}, "logprobs": None, "finish_reason": None}]
             }
             yield f"data: {json.dumps(chunk)}\n\n"
             end_chunk = {
@@ -254,11 +261,11 @@ async def chat_completions(request: Request):
                 "object": "chat.completion.chunk",
                 "created": int(time.time()),
                 "model": model,
-                "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]
+                "choices": [{"index": 0, "delta": {}, "logprobs": None, "finish_reason": "stop"}]
             }
             yield f"data: {json.dumps(end_chunk)}\n\n"
             yield "data: [DONE]\n\n"
-        return StreamingResponse(stream_generator(), media_type="text/event-stream")
+        return StreamingResponse(stream_generator(), media_type="text/event-stream", headers=stream_headers)
 
     return JSONResponse(content={
         "id": f"chatcmpl-{int(time.time()*1000)}",

nginx.conf

@@ -37,6 +37,7 @@ http {
             proxy_set_header X-Forwarded-Host $host;
             proxy_set_header X-Forwarded-Port $server_port;
 
+            proxy_buffering off;
             proxy_connect_timeout 60s;
             proxy_send_timeout 300s;
             proxy_read_timeout 300s;