Wimmboo commited on
Commit
09e4b39
·
1 Parent(s): a689f42

chore: remove Google provider entirely

Browse files

Google streaming through HF Spaces HTTP/2 edge never worked reliably
with Janitor AI. Strip all Google-specific code, providers, models,
keys, and docs. Keep only NVIDIA NIM path. google/gemma-4-31b-it
stays in NVIDIA model list (NVIDIA-hosted Gemma).

Files changed (7) hide show
  1. .env.example +0 -2
  2. .opencode/AGENTS.md +1 -2
  3. PRODUCT.md +1 -1
  4. config.py +0 -2
  5. proxy/client.py +4 -107
  6. proxy/providers.py +0 -8
  7. proxy/router.py +0 -4
.env.example CHANGED
@@ -1,11 +1,9 @@
1
  # Real provider API keys (keep these secret)
2
  REAL_NVIDIA_KEY=nvapi-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
3
- REAL_GOOGLE_KEY=AIzaxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
4
 
5
  # Proxy keys your client apps will use
6
  # Generate with: python -c "import secrets; print('sk-nvidia-' + secrets.token_urlsafe(32))"
7
  SK_NVIDIA_KEY=sk-nvidia-xxxxxxxxxxxxxxxxxxxx
8
- SK_GOOGLE_KEY=sk-google-xxxxxxxxxxxxxxxxxxxx
9
 
10
  # Server
11
  HOST=127.0.0.1
 
1
  # Real provider API keys (keep these secret)
2
  REAL_NVIDIA_KEY=nvapi-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 
3
 
4
  # Proxy keys your client apps will use
5
  # Generate with: python -c "import secrets; print('sk-nvidia-' + secrets.token_urlsafe(32))"
6
  SK_NVIDIA_KEY=sk-nvidia-xxxxxxxxxxxxxxxxxxxx
 
7
 
8
  # Server
9
  HOST=127.0.0.1
.opencode/AGENTS.md CHANGED
@@ -13,7 +13,7 @@ Rules:
13
 
14
  ## OpenAI API Proxy
15
 
16
- FastAPI proxy that routes OpenAI-compatible chat requests to NVIDIA NIM or Google AI Studio based on which `sk-{provider}` key the client uses.
17
 
18
  ### Setup & run
19
  - Install: `pip install -r requirements.txt`
@@ -30,7 +30,6 @@ FastAPI proxy that routes OpenAI-compatible chat requests to NVIDIA NIM or Googl
30
 
31
  ### Provider quirks
32
  - **NVIDIA**: Bearer token auth, async (202→poll), base URL `https://integrate.api.nvidia.com`
33
- - **Google**: `x-goog-api-key` header (no Bearer), no 202 handling, base URL `https://generativelanguage.googleapis.com/v1beta/openai`
34
 
35
  ### Adding a new provider
36
  1. Add `Provider(...)` to `proxy/providers.py:PROVIDERS` dict
 
13
 
14
  ## OpenAI API Proxy
15
 
16
+ FastAPI proxy that routes OpenAI-compatible chat requests to NVIDIA NIM based on the `sk-nvidia` key the client uses.
17
 
18
  ### Setup & run
19
  - Install: `pip install -r requirements.txt`
 
30
 
31
  ### Provider quirks
32
  - **NVIDIA**: Bearer token auth, async (202→poll), base URL `https://integrate.api.nvidia.com`
 
33
 
34
  ### Adding a new provider
35
  1. Add `Provider(...)` to `proxy/providers.py:PROVIDERS` dict
PRODUCT.md CHANGED
@@ -10,7 +10,7 @@ A single technical user (the owner) running this proxy locally or on cheap perso
10
 
11
  ## Product Purpose
12
 
13
- A lightweight OpenAI-compatible API gateway that forwards chat/completion requests to the right AI provider based on which `sk-` proxy key the client uses. One server, one set of routes, multiple backends (NVIDIA NIM, Google AI Studio, extensible to more). It makes apps that only accept OpenAI-style keys work with non-OpenAI providers.
14
 
15
  ## Brand Personality
16
 
 
10
 
11
  ## Product Purpose
12
 
13
+ A lightweight OpenAI-compatible API gateway that forwards chat/completion requests to NVIDIA NIM based on the `sk-nvidia` proxy key the client uses. One server, one set of routes, one backend. It makes apps that only accept OpenAI-style keys work with non-OpenAI providers.
14
 
15
  ## Brand Personality
16
 
config.py CHANGED
@@ -15,12 +15,10 @@ POLL_MAX_ATTEMPTS = int(os.getenv("POLL_MAX_ATTEMPTS", "60"))
15
 
16
  PROVIDER_KEYS = {
17
  "nvidia": os.getenv("SK_NVIDIA_KEY", ""),
18
- "google": os.getenv("SK_GOOGLE_KEY", ""),
19
  }
20
 
21
  REAL_KEYS = {
22
  "nvidia": os.getenv("REAL_NVIDIA_KEY", ""),
23
- "google": os.getenv("REAL_GOOGLE_KEY", ""),
24
  }
25
 
26
 
 
15
 
16
  PROVIDER_KEYS = {
17
  "nvidia": os.getenv("SK_NVIDIA_KEY", ""),
 
18
  }
19
 
20
  REAL_KEYS = {
21
  "nvidia": os.getenv("REAL_NVIDIA_KEY", ""),
 
22
  }
23
 
24
 
proxy/client.py CHANGED
@@ -5,7 +5,7 @@ from typing import Any, AsyncIterator
5
 
6
  import httpx
7
  from fastapi import HTTPException
8
- from fastapi.responses import Response, StreamingResponse
9
  from starlette.requests import Request
10
 
11
  from config import POLL_INTERVAL, POLL_MAX_ATTEMPTS
@@ -24,20 +24,6 @@ def _build_headers(provider: Provider, real_key: str, incoming_headers: dict[str
24
  return headers
25
 
26
 
27
- _UNSUPPORTED_GOOGLE_PARAMS = {"frequency_penalty", "logprobs", "top_logprobs", "repetition_penalty", "top_k"}
28
-
29
-
30
- def _sanitize_body(provider: Provider, body: dict[str, Any]) -> dict[str, Any]:
31
- """Remove parameters that Google's OpenAI-compatible endpoint rejects."""
32
- if provider.name != "google":
33
- return body
34
- stripped = _UNSUPPORTED_GOOGLE_PARAMS & set(body.keys())
35
- if not stripped:
36
- return body
37
- logger.info("Stripping unsupported Google parameters: %s", ", ".join(sorted(stripped)))
38
- return {k: v for k, v in body.items() if k not in _UNSUPPORTED_GOOGLE_PARAMS}
39
-
40
-
41
  def _extract_request_id(response: httpx.Response) -> str | None:
42
  try:
43
  data = response.json()
@@ -82,71 +68,6 @@ def _upstream_error_message(status_code: int, content: bytes) -> str:
82
  return text
83
 
84
 
85
- def _strip_extra_content(data: bytes) -> bytes:
86
- text = data.decode("utf-8")
87
- lines = []
88
- for line in text.split("\n"):
89
- if line.startswith("data: ") and line != "data: [DONE]":
90
- try:
91
- obj = json.loads(line[6:])
92
- for choice in obj.get("choices", []):
93
- choice.get("delta", {}).pop("extra_content", None)
94
- choice.get("message", {}).pop("extra_content", None)
95
- lines.append("data: " + json.dumps(obj, ensure_ascii=False))
96
- except (json.JSONDecodeError, KeyError, TypeError):
97
- lines.append(line)
98
- else:
99
- lines.append(line)
100
- return "\n".join(lines).encode("utf-8")
101
-
102
-
103
- def _strip_extra_content_json(content: bytes) -> bytes:
104
- try:
105
- obj = json.loads(content)
106
- for choice in obj.get("choices", []):
107
- choice.get("delta", {}).pop("extra_content", None)
108
- choice.get("message", {}).pop("extra_content", None)
109
- return json.dumps(obj, ensure_ascii=False).encode("utf-8")
110
- except (json.JSONDecodeError, KeyError, TypeError, UnicodeDecodeError):
111
- return content
112
-
113
-
114
- def _json_to_sse(content: bytes) -> bytes:
115
- """Convert a non-streaming OpenAI JSON response into SSE chunks."""
116
- try:
117
- obj = json.loads(content)
118
- base = {
119
- "id": obj.get("id", "chatcmpl-proxy"),
120
- "object": "chat.completion.chunk",
121
- "created": obj.get("created", 0),
122
- "model": obj.get("model", ""),
123
- }
124
- events = []
125
- for choice in obj.get("choices", []):
126
- idx = choice.get("index", 0)
127
- message = choice.get("message", {})
128
- content_text = message.get("content", "")
129
- finish_reason = choice.get("finish_reason", None)
130
-
131
- role_chunk = dict(base)
132
- role_chunk["choices"] = [{"index": idx, "delta": {"role": message.get("role", "assistant")}, "finish_reason": None}]
133
- events.append(f"data: {json.dumps(role_chunk, ensure_ascii=False)}\n\n")
134
-
135
- if content_text:
136
- content_chunk = dict(base)
137
- content_chunk["choices"] = [{"index": idx, "delta": {"content": content_text}, "finish_reason": None}]
138
- events.append(f"data: {json.dumps(content_chunk, ensure_ascii=False)}\n\n")
139
-
140
- final_chunk = dict(base)
141
- final_chunk["choices"] = [{"index": idx, "delta": {}, "finish_reason": finish_reason}]
142
- events.append(f"data: {json.dumps(final_chunk, ensure_ascii=False)}\n\n")
143
-
144
- events.append("data: [DONE]\n\n")
145
- return "".join(events).encode("utf-8")
146
- except (json.JSONDecodeError, KeyError, TypeError, UnicodeDecodeError):
147
- return content
148
-
149
-
150
  async def _stream_response(provider: Provider, real_key: str, body: dict[str, Any]) -> AsyncIterator[bytes]:
151
  """Async generator that manages its own httpx client for streaming."""
152
  target_url = f"{provider.base_url}/v1/chat/completions"
@@ -157,30 +78,15 @@ async def _stream_response(provider: Provider, real_key: str, body: dict[str, An
157
  content = await upstream.aread()
158
  detail = _upstream_error_message(upstream.status_code, content)
159
  raise HTTPException(status_code=upstream.status_code, detail=detail)
160
- buffer = b""
161
  async for chunk in upstream.aiter_bytes():
162
- buffer += chunk
163
- while b"\n\n" in buffer:
164
- event, buffer = buffer.split(b"\n\n", 1)
165
- event += b"\n\n"
166
- if provider.name == "google":
167
- event = _strip_extra_content(event)
168
- yield event
169
- if buffer:
170
- if provider.name == "google":
171
- buffer = _strip_extra_content(buffer)
172
- yield buffer
173
 
174
 
175
  async def forward_request(provider: Provider, real_key: str, request: Request, body: dict[str, Any]) -> Any:
176
- body = _sanitize_body(provider, body)
177
  target_url = f"{provider.base_url}/v1/chat/completions"
178
  headers = _build_headers(provider, real_key, dict(request.headers))
179
- original_stream = bool(body.get("stream"))
180
 
181
- if original_stream and provider.name == "google":
182
- body["stream"] = False
183
- elif original_stream:
184
  return StreamingResponse(_stream_response(provider, real_key, body), media_type="text/event-stream")
185
 
186
  async with httpx.AsyncClient(timeout=_TIMEOUT, follow_redirects=True) as client:
@@ -205,17 +111,8 @@ async def forward_request(provider: Provider, real_key: str, request: Request, b
205
  detail = _upstream_error_message(upstream.status_code, content)
206
  raise HTTPException(status_code=upstream.status_code, detail=detail)
207
 
208
- content = upstream.content
209
- if provider.name == "google":
210
- content = _strip_extra_content_json(content)
211
- if original_stream:
212
- content = _json_to_sse(content)
213
-
214
- if original_stream and provider.name == "google":
215
- return Response(content=content, media_type="text/event-stream")
216
-
217
  return {
218
  "status_code": upstream.status_code,
219
  "headers": dict(upstream.headers),
220
- "content": content,
221
  }
 
5
 
6
  import httpx
7
  from fastapi import HTTPException
8
+ from fastapi.responses import StreamingResponse
9
  from starlette.requests import Request
10
 
11
  from config import POLL_INTERVAL, POLL_MAX_ATTEMPTS
 
24
  return headers
25
 
26
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
  def _extract_request_id(response: httpx.Response) -> str | None:
28
  try:
29
  data = response.json()
 
68
  return text
69
 
70
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
71
  async def _stream_response(provider: Provider, real_key: str, body: dict[str, Any]) -> AsyncIterator[bytes]:
72
  """Async generator that manages its own httpx client for streaming."""
73
  target_url = f"{provider.base_url}/v1/chat/completions"
 
78
  content = await upstream.aread()
79
  detail = _upstream_error_message(upstream.status_code, content)
80
  raise HTTPException(status_code=upstream.status_code, detail=detail)
 
81
  async for chunk in upstream.aiter_bytes():
82
+ yield chunk
 
 
 
 
 
 
 
 
 
 
83
 
84
 
85
  async def forward_request(provider: Provider, real_key: str, request: Request, body: dict[str, Any]) -> Any:
 
86
  target_url = f"{provider.base_url}/v1/chat/completions"
87
  headers = _build_headers(provider, real_key, dict(request.headers))
 
88
 
89
+ if bool(body.get("stream")):
 
 
90
  return StreamingResponse(_stream_response(provider, real_key, body), media_type="text/event-stream")
91
 
92
  async with httpx.AsyncClient(timeout=_TIMEOUT, follow_redirects=True) as client:
 
111
  detail = _upstream_error_message(upstream.status_code, content)
112
  raise HTTPException(status_code=upstream.status_code, detail=detail)
113
 
 
 
 
 
 
 
 
 
 
114
  return {
115
  "status_code": upstream.status_code,
116
  "headers": dict(upstream.headers),
117
+ "content": upstream.content,
118
  }
proxy/providers.py CHANGED
@@ -21,12 +21,4 @@ PROVIDERS: dict[str, Provider] = {
21
  handles_202=True,
22
  env_real_key="REAL_NVIDIA_KEY",
23
  ),
24
- "google": Provider(
25
- name="google",
26
- base_url="https://generativelanguage.googleapis.com/v1beta/openai",
27
- auth_header="Authorization",
28
- auth_value=lambda key: f"Bearer {key}",
29
- handles_202=False,
30
- env_real_key="REAL_GOOGLE_KEY",
31
- ),
32
  }
 
21
  handles_202=True,
22
  env_real_key="REAL_NVIDIA_KEY",
23
  ),
 
 
 
 
 
 
 
 
24
  }
proxy/router.py CHANGED
@@ -24,10 +24,6 @@ MODELS = {
24
  "moonshotai/kimi-k2.5",
25
  "google/gemma-4-31b-it",
26
  ],
27
- "google": [
28
- "gemini-3.1-flash-lite",
29
- "gemma-4-31b-it",
30
- ],
31
  }
32
 
33
 
 
24
  "moonshotai/kimi-k2.5",
25
  "google/gemma-4-31b-it",
26
  ],
 
 
 
 
27
  }
28
 
29