Spaces:
Running
Running
Wimmboo commited on
Commit ·
09e4b39
1
Parent(s): a689f42
chore: remove Google provider entirely
Browse filesGoogle streaming through HF Spaces HTTP/2 edge never worked reliably
with Janitor AI. Strip all Google-specific code, providers, models,
keys, and docs. Keep only NVIDIA NIM path. google/gemma-4-31b-it
stays in NVIDIA model list (NVIDIA-hosted Gemma).
- .env.example +0 -2
- .opencode/AGENTS.md +1 -2
- PRODUCT.md +1 -1
- config.py +0 -2
- proxy/client.py +4 -107
- proxy/providers.py +0 -8
- proxy/router.py +0 -4
.env.example
CHANGED
|
@@ -1,11 +1,9 @@
|
|
| 1 |
# Real provider API keys (keep these secret)
|
| 2 |
REAL_NVIDIA_KEY=nvapi-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
| 3 |
-
REAL_GOOGLE_KEY=AIzaxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
| 4 |
|
| 5 |
# Proxy keys your client apps will use
|
| 6 |
# Generate with: python -c "import secrets; print('sk-nvidia-' + secrets.token_urlsafe(32))"
|
| 7 |
SK_NVIDIA_KEY=sk-nvidia-xxxxxxxxxxxxxxxxxxxx
|
| 8 |
-
SK_GOOGLE_KEY=sk-google-xxxxxxxxxxxxxxxxxxxx
|
| 9 |
|
| 10 |
# Server
|
| 11 |
HOST=127.0.0.1
|
|
|
|
| 1 |
# Real provider API keys (keep these secret)
|
| 2 |
REAL_NVIDIA_KEY=nvapi-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
|
|
|
| 3 |
|
| 4 |
# Proxy keys your client apps will use
|
| 5 |
# Generate with: python -c "import secrets; print('sk-nvidia-' + secrets.token_urlsafe(32))"
|
| 6 |
SK_NVIDIA_KEY=sk-nvidia-xxxxxxxxxxxxxxxxxxxx
|
|
|
|
| 7 |
|
| 8 |
# Server
|
| 9 |
HOST=127.0.0.1
|
.opencode/AGENTS.md
CHANGED
|
@@ -13,7 +13,7 @@ Rules:
|
|
| 13 |
|
| 14 |
## OpenAI API Proxy
|
| 15 |
|
| 16 |
-
FastAPI proxy that routes OpenAI-compatible chat requests to NVIDIA NIM
|
| 17 |
|
| 18 |
### Setup & run
|
| 19 |
- Install: `pip install -r requirements.txt`
|
|
@@ -30,7 +30,6 @@ FastAPI proxy that routes OpenAI-compatible chat requests to NVIDIA NIM or Googl
|
|
| 30 |
|
| 31 |
### Provider quirks
|
| 32 |
- **NVIDIA**: Bearer token auth, async (202→poll), base URL `https://integrate.api.nvidia.com`
|
| 33 |
-
- **Google**: `x-goog-api-key` header (no Bearer), no 202 handling, base URL `https://generativelanguage.googleapis.com/v1beta/openai`
|
| 34 |
|
| 35 |
### Adding a new provider
|
| 36 |
1. Add `Provider(...)` to `proxy/providers.py:PROVIDERS` dict
|
|
|
|
| 13 |
|
| 14 |
## OpenAI API Proxy
|
| 15 |
|
| 16 |
+
FastAPI proxy that routes OpenAI-compatible chat requests to NVIDIA NIM based on the `sk-nvidia` key the client uses.
|
| 17 |
|
| 18 |
### Setup & run
|
| 19 |
- Install: `pip install -r requirements.txt`
|
|
|
|
| 30 |
|
| 31 |
### Provider quirks
|
| 32 |
- **NVIDIA**: Bearer token auth, async (202→poll), base URL `https://integrate.api.nvidia.com`
|
|
|
|
| 33 |
|
| 34 |
### Adding a new provider
|
| 35 |
1. Add `Provider(...)` to `proxy/providers.py:PROVIDERS` dict
|
PRODUCT.md
CHANGED
|
@@ -10,7 +10,7 @@ A single technical user (the owner) running this proxy locally or on cheap perso
|
|
| 10 |
|
| 11 |
## Product Purpose
|
| 12 |
|
| 13 |
-
A lightweight OpenAI-compatible API gateway that forwards chat/completion requests to
|
| 14 |
|
| 15 |
## Brand Personality
|
| 16 |
|
|
|
|
| 10 |
|
| 11 |
## Product Purpose
|
| 12 |
|
| 13 |
+
A lightweight OpenAI-compatible API gateway that forwards chat/completion requests to NVIDIA NIM based on the `sk-nvidia` proxy key the client uses. One server, one set of routes, one backend. It makes apps that only accept OpenAI-style keys work with non-OpenAI providers.
|
| 14 |
|
| 15 |
## Brand Personality
|
| 16 |
|
config.py
CHANGED
|
@@ -15,12 +15,10 @@ POLL_MAX_ATTEMPTS = int(os.getenv("POLL_MAX_ATTEMPTS", "60"))
|
|
| 15 |
|
| 16 |
PROVIDER_KEYS = {
|
| 17 |
"nvidia": os.getenv("SK_NVIDIA_KEY", ""),
|
| 18 |
-
"google": os.getenv("SK_GOOGLE_KEY", ""),
|
| 19 |
}
|
| 20 |
|
| 21 |
REAL_KEYS = {
|
| 22 |
"nvidia": os.getenv("REAL_NVIDIA_KEY", ""),
|
| 23 |
-
"google": os.getenv("REAL_GOOGLE_KEY", ""),
|
| 24 |
}
|
| 25 |
|
| 26 |
|
|
|
|
| 15 |
|
| 16 |
PROVIDER_KEYS = {
|
| 17 |
"nvidia": os.getenv("SK_NVIDIA_KEY", ""),
|
|
|
|
| 18 |
}
|
| 19 |
|
| 20 |
REAL_KEYS = {
|
| 21 |
"nvidia": os.getenv("REAL_NVIDIA_KEY", ""),
|
|
|
|
| 22 |
}
|
| 23 |
|
| 24 |
|
proxy/client.py
CHANGED
|
@@ -5,7 +5,7 @@ from typing import Any, AsyncIterator
|
|
| 5 |
|
| 6 |
import httpx
|
| 7 |
from fastapi import HTTPException
|
| 8 |
-
from fastapi.responses import
|
| 9 |
from starlette.requests import Request
|
| 10 |
|
| 11 |
from config import POLL_INTERVAL, POLL_MAX_ATTEMPTS
|
|
@@ -24,20 +24,6 @@ def _build_headers(provider: Provider, real_key: str, incoming_headers: dict[str
|
|
| 24 |
return headers
|
| 25 |
|
| 26 |
|
| 27 |
-
_UNSUPPORTED_GOOGLE_PARAMS = {"frequency_penalty", "logprobs", "top_logprobs", "repetition_penalty", "top_k"}
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
def _sanitize_body(provider: Provider, body: dict[str, Any]) -> dict[str, Any]:
|
| 31 |
-
"""Remove parameters that Google's OpenAI-compatible endpoint rejects."""
|
| 32 |
-
if provider.name != "google":
|
| 33 |
-
return body
|
| 34 |
-
stripped = _UNSUPPORTED_GOOGLE_PARAMS & set(body.keys())
|
| 35 |
-
if not stripped:
|
| 36 |
-
return body
|
| 37 |
-
logger.info("Stripping unsupported Google parameters: %s", ", ".join(sorted(stripped)))
|
| 38 |
-
return {k: v for k, v in body.items() if k not in _UNSUPPORTED_GOOGLE_PARAMS}
|
| 39 |
-
|
| 40 |
-
|
| 41 |
def _extract_request_id(response: httpx.Response) -> str | None:
|
| 42 |
try:
|
| 43 |
data = response.json()
|
|
@@ -82,71 +68,6 @@ def _upstream_error_message(status_code: int, content: bytes) -> str:
|
|
| 82 |
return text
|
| 83 |
|
| 84 |
|
| 85 |
-
def _strip_extra_content(data: bytes) -> bytes:
|
| 86 |
-
text = data.decode("utf-8")
|
| 87 |
-
lines = []
|
| 88 |
-
for line in text.split("\n"):
|
| 89 |
-
if line.startswith("data: ") and line != "data: [DONE]":
|
| 90 |
-
try:
|
| 91 |
-
obj = json.loads(line[6:])
|
| 92 |
-
for choice in obj.get("choices", []):
|
| 93 |
-
choice.get("delta", {}).pop("extra_content", None)
|
| 94 |
-
choice.get("message", {}).pop("extra_content", None)
|
| 95 |
-
lines.append("data: " + json.dumps(obj, ensure_ascii=False))
|
| 96 |
-
except (json.JSONDecodeError, KeyError, TypeError):
|
| 97 |
-
lines.append(line)
|
| 98 |
-
else:
|
| 99 |
-
lines.append(line)
|
| 100 |
-
return "\n".join(lines).encode("utf-8")
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
def _strip_extra_content_json(content: bytes) -> bytes:
|
| 104 |
-
try:
|
| 105 |
-
obj = json.loads(content)
|
| 106 |
-
for choice in obj.get("choices", []):
|
| 107 |
-
choice.get("delta", {}).pop("extra_content", None)
|
| 108 |
-
choice.get("message", {}).pop("extra_content", None)
|
| 109 |
-
return json.dumps(obj, ensure_ascii=False).encode("utf-8")
|
| 110 |
-
except (json.JSONDecodeError, KeyError, TypeError, UnicodeDecodeError):
|
| 111 |
-
return content
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
def _json_to_sse(content: bytes) -> bytes:
|
| 115 |
-
"""Convert a non-streaming OpenAI JSON response into SSE chunks."""
|
| 116 |
-
try:
|
| 117 |
-
obj = json.loads(content)
|
| 118 |
-
base = {
|
| 119 |
-
"id": obj.get("id", "chatcmpl-proxy"),
|
| 120 |
-
"object": "chat.completion.chunk",
|
| 121 |
-
"created": obj.get("created", 0),
|
| 122 |
-
"model": obj.get("model", ""),
|
| 123 |
-
}
|
| 124 |
-
events = []
|
| 125 |
-
for choice in obj.get("choices", []):
|
| 126 |
-
idx = choice.get("index", 0)
|
| 127 |
-
message = choice.get("message", {})
|
| 128 |
-
content_text = message.get("content", "")
|
| 129 |
-
finish_reason = choice.get("finish_reason", None)
|
| 130 |
-
|
| 131 |
-
role_chunk = dict(base)
|
| 132 |
-
role_chunk["choices"] = [{"index": idx, "delta": {"role": message.get("role", "assistant")}, "finish_reason": None}]
|
| 133 |
-
events.append(f"data: {json.dumps(role_chunk, ensure_ascii=False)}\n\n")
|
| 134 |
-
|
| 135 |
-
if content_text:
|
| 136 |
-
content_chunk = dict(base)
|
| 137 |
-
content_chunk["choices"] = [{"index": idx, "delta": {"content": content_text}, "finish_reason": None}]
|
| 138 |
-
events.append(f"data: {json.dumps(content_chunk, ensure_ascii=False)}\n\n")
|
| 139 |
-
|
| 140 |
-
final_chunk = dict(base)
|
| 141 |
-
final_chunk["choices"] = [{"index": idx, "delta": {}, "finish_reason": finish_reason}]
|
| 142 |
-
events.append(f"data: {json.dumps(final_chunk, ensure_ascii=False)}\n\n")
|
| 143 |
-
|
| 144 |
-
events.append("data: [DONE]\n\n")
|
| 145 |
-
return "".join(events).encode("utf-8")
|
| 146 |
-
except (json.JSONDecodeError, KeyError, TypeError, UnicodeDecodeError):
|
| 147 |
-
return content
|
| 148 |
-
|
| 149 |
-
|
| 150 |
async def _stream_response(provider: Provider, real_key: str, body: dict[str, Any]) -> AsyncIterator[bytes]:
|
| 151 |
"""Async generator that manages its own httpx client for streaming."""
|
| 152 |
target_url = f"{provider.base_url}/v1/chat/completions"
|
|
@@ -157,30 +78,15 @@ async def _stream_response(provider: Provider, real_key: str, body: dict[str, An
|
|
| 157 |
content = await upstream.aread()
|
| 158 |
detail = _upstream_error_message(upstream.status_code, content)
|
| 159 |
raise HTTPException(status_code=upstream.status_code, detail=detail)
|
| 160 |
-
buffer = b""
|
| 161 |
async for chunk in upstream.aiter_bytes():
|
| 162 |
-
|
| 163 |
-
while b"\n\n" in buffer:
|
| 164 |
-
event, buffer = buffer.split(b"\n\n", 1)
|
| 165 |
-
event += b"\n\n"
|
| 166 |
-
if provider.name == "google":
|
| 167 |
-
event = _strip_extra_content(event)
|
| 168 |
-
yield event
|
| 169 |
-
if buffer:
|
| 170 |
-
if provider.name == "google":
|
| 171 |
-
buffer = _strip_extra_content(buffer)
|
| 172 |
-
yield buffer
|
| 173 |
|
| 174 |
|
| 175 |
async def forward_request(provider: Provider, real_key: str, request: Request, body: dict[str, Any]) -> Any:
|
| 176 |
-
body = _sanitize_body(provider, body)
|
| 177 |
target_url = f"{provider.base_url}/v1/chat/completions"
|
| 178 |
headers = _build_headers(provider, real_key, dict(request.headers))
|
| 179 |
-
original_stream = bool(body.get("stream"))
|
| 180 |
|
| 181 |
-
if
|
| 182 |
-
body["stream"] = False
|
| 183 |
-
elif original_stream:
|
| 184 |
return StreamingResponse(_stream_response(provider, real_key, body), media_type="text/event-stream")
|
| 185 |
|
| 186 |
async with httpx.AsyncClient(timeout=_TIMEOUT, follow_redirects=True) as client:
|
|
@@ -205,17 +111,8 @@ async def forward_request(provider: Provider, real_key: str, request: Request, b
|
|
| 205 |
detail = _upstream_error_message(upstream.status_code, content)
|
| 206 |
raise HTTPException(status_code=upstream.status_code, detail=detail)
|
| 207 |
|
| 208 |
-
content = upstream.content
|
| 209 |
-
if provider.name == "google":
|
| 210 |
-
content = _strip_extra_content_json(content)
|
| 211 |
-
if original_stream:
|
| 212 |
-
content = _json_to_sse(content)
|
| 213 |
-
|
| 214 |
-
if original_stream and provider.name == "google":
|
| 215 |
-
return Response(content=content, media_type="text/event-stream")
|
| 216 |
-
|
| 217 |
return {
|
| 218 |
"status_code": upstream.status_code,
|
| 219 |
"headers": dict(upstream.headers),
|
| 220 |
-
"content": content,
|
| 221 |
}
|
|
|
|
| 5 |
|
| 6 |
import httpx
|
| 7 |
from fastapi import HTTPException
|
| 8 |
+
from fastapi.responses import StreamingResponse
|
| 9 |
from starlette.requests import Request
|
| 10 |
|
| 11 |
from config import POLL_INTERVAL, POLL_MAX_ATTEMPTS
|
|
|
|
| 24 |
return headers
|
| 25 |
|
| 26 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
def _extract_request_id(response: httpx.Response) -> str | None:
|
| 28 |
try:
|
| 29 |
data = response.json()
|
|
|
|
| 68 |
return text
|
| 69 |
|
| 70 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
async def _stream_response(provider: Provider, real_key: str, body: dict[str, Any]) -> AsyncIterator[bytes]:
|
| 72 |
"""Async generator that manages its own httpx client for streaming."""
|
| 73 |
target_url = f"{provider.base_url}/v1/chat/completions"
|
|
|
|
| 78 |
content = await upstream.aread()
|
| 79 |
detail = _upstream_error_message(upstream.status_code, content)
|
| 80 |
raise HTTPException(status_code=upstream.status_code, detail=detail)
|
|
|
|
| 81 |
async for chunk in upstream.aiter_bytes():
|
| 82 |
+
yield chunk
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 83 |
|
| 84 |
|
| 85 |
async def forward_request(provider: Provider, real_key: str, request: Request, body: dict[str, Any]) -> Any:
|
|
|
|
| 86 |
target_url = f"{provider.base_url}/v1/chat/completions"
|
| 87 |
headers = _build_headers(provider, real_key, dict(request.headers))
|
|
|
|
| 88 |
|
| 89 |
+
if bool(body.get("stream")):
|
|
|
|
|
|
|
| 90 |
return StreamingResponse(_stream_response(provider, real_key, body), media_type="text/event-stream")
|
| 91 |
|
| 92 |
async with httpx.AsyncClient(timeout=_TIMEOUT, follow_redirects=True) as client:
|
|
|
|
| 111 |
detail = _upstream_error_message(upstream.status_code, content)
|
| 112 |
raise HTTPException(status_code=upstream.status_code, detail=detail)
|
| 113 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 114 |
return {
|
| 115 |
"status_code": upstream.status_code,
|
| 116 |
"headers": dict(upstream.headers),
|
| 117 |
+
"content": upstream.content,
|
| 118 |
}
|
proxy/providers.py
CHANGED
|
@@ -21,12 +21,4 @@ PROVIDERS: dict[str, Provider] = {
|
|
| 21 |
handles_202=True,
|
| 22 |
env_real_key="REAL_NVIDIA_KEY",
|
| 23 |
),
|
| 24 |
-
"google": Provider(
|
| 25 |
-
name="google",
|
| 26 |
-
base_url="https://generativelanguage.googleapis.com/v1beta/openai",
|
| 27 |
-
auth_header="Authorization",
|
| 28 |
-
auth_value=lambda key: f"Bearer {key}",
|
| 29 |
-
handles_202=False,
|
| 30 |
-
env_real_key="REAL_GOOGLE_KEY",
|
| 31 |
-
),
|
| 32 |
}
|
|
|
|
| 21 |
handles_202=True,
|
| 22 |
env_real_key="REAL_NVIDIA_KEY",
|
| 23 |
),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
}
|
proxy/router.py
CHANGED
|
@@ -24,10 +24,6 @@ MODELS = {
|
|
| 24 |
"moonshotai/kimi-k2.5",
|
| 25 |
"google/gemma-4-31b-it",
|
| 26 |
],
|
| 27 |
-
"google": [
|
| 28 |
-
"gemini-3.1-flash-lite",
|
| 29 |
-
"gemma-4-31b-it",
|
| 30 |
-
],
|
| 31 |
}
|
| 32 |
|
| 33 |
|
|
|
|
| 24 |
"moonshotai/kimi-k2.5",
|
| 25 |
"google/gemma-4-31b-it",
|
| 26 |
],
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
}
|
| 28 |
|
| 29 |
|