tommytracx commited on
Commit
c1e288e
·
verified ·
1 Parent(s): 330d48d

THOX ZeroGPU model Space

Browse files
__pycache__/app.cpython-311.pyc CHANGED
Binary files a/__pycache__/app.cpython-311.pyc and b/__pycache__/app.cpython-311.pyc differ
 
__pycache__/thox_space_config.cpython-311.pyc CHANGED
Binary files a/__pycache__/thox_space_config.cpython-311.pyc and b/__pycache__/thox_space_config.cpython-311.pyc differ
 
app.py CHANGED
@@ -1,330 +1,380 @@
1
- """THOX ZeroGPU model Space — one model behind an OpenAI-compatible /v1 endpoint.
2
-
3
- This single file is deployed verbatim to every THOX ZeroGPU model Space. The only
4
- per-Space difference is `thox_space_config.py`, which names the model to serve.
5
-
6
- Why FastAPI *and* Gradio: Open WebUI consumes this Space as an ordinary "OpenAI
7
- API" connection, which means it needs `/v1/models` and `/v1/chat/completions`.
8
- Gradio alone does not speak that protocol, so the FastAPI app owns the `/v1`
9
- routes and the Gradio Blocks UI is mounted at `/` for humans. ZeroGPU only
10
- schedules Gradio SDK Spaces, so the Gradio app must genuinely be present -- it is
11
- not decoration.
12
- """
13
-
14
- from __future__ import annotations
15
-
16
- import json
17
- import os
18
- import time
19
- import uuid
20
- from threading import Thread
21
- from typing import Any, Iterator
22
-
23
- # Disable Gradio 5 server-side rendering BEFORE gradio is imported.
24
- # On a Spaces Gradio SDK Space, SSR spawns a Node.js front-end server that claims the
25
- # port this app needs, producing "[Errno 98] address already in use" followed by
26
- # "Stopping Node.js server...". We serve the Blocks app from our own uvicorn process, so
27
- # the SSR layer has nothing to add here and everything to collide with.
28
- os.environ.setdefault("GRADIO_SSR_MODE", "false")
29
-
30
- import gradio as gr # noqa: E402
31
- import spaces
32
- import torch
33
- import uvicorn
34
- from fastapi import FastAPI
35
- from fastapi.responses import JSONResponse, StreamingResponse
36
- from pydantic import BaseModel, Field
37
- from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
38
-
39
- from thox_space_config import (
40
- ADAPTER_ID,
41
- DESCRIPTION,
42
- MAX_CONTEXT_TOKENS,
43
- MODEL_ID,
44
- SERVED_MODEL_NAME,
45
- SYSTEM_PROMPT,
46
- )
47
-
48
- # --------------------------------------------------------------------------------------
49
- # Model loading (module scope, eager -- see ZeroGPU rules)
50
- # --------------------------------------------------------------------------------------
51
- # ZeroGPU monkey-patches torch so `cuda` resolves at module scope and the weights are
52
- # registered with the scheduler, then streamed into VRAM when a @spaces.GPU call lands.
53
- # Loading lazily on first request would just move this cost onto the first user.
54
- DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
55
- DTYPE = torch.bfloat16 if DEVICE.type == "cuda" else torch.float32
56
-
57
- print(f"[thox] loading base model: {MODEL_ID}", flush=True)
58
- TOKENIZER = AutoTokenizer.from_pretrained(MODEL_ID)
59
- if TOKENIZER.pad_token_id is None:
60
- TOKENIZER.pad_token = TOKENIZER.eos_token
61
-
62
- MODEL = AutoModelForCausalLM.from_pretrained(MODEL_ID, dtype=DTYPE).to(DEVICE)
63
-
64
- if ADAPTER_ID:
65
- # THOX ships several of its models as LoRA adapters rather than merged weights.
66
- from peft import PeftModel
67
-
68
- print(f"[thox] applying LoRA adapter: {ADAPTER_ID}", flush=True)
69
- MODEL = PeftModel.from_pretrained(MODEL, ADAPTER_ID)
70
- MODEL = MODEL.merge_and_unload() # merge so generation runs at full speed
71
-
72
- MODEL.eval()
73
- print(f"[thox] ready: serving '{SERVED_MODEL_NAME}' on {DEVICE}", flush=True)
74
-
75
-
76
- # --------------------------------------------------------------------------------------
77
- # Generation
78
- # --------------------------------------------------------------------------------------
79
- def _estimate_duration(messages: list[dict], max_new_tokens: int, temperature: float,
80
- top_p: float) -> int:
81
- """Request only the GPU seconds this call plausibly needs.
82
-
83
- ZeroGPU pre-checks the *requested* duration against the caller's remaining quota,
84
- not the actual runtime -- so leaving everything at the 60s default makes short
85
- requests fail with "quota exceeded" once a user drops below 60s. Shorter requests
86
- also rank higher in the node-level queue.
87
- """
88
- return int(min(120, max(15, (max_new_tokens / 12) + 10)))
89
-
90
-
91
- @spaces.GPU(duration=_estimate_duration)
92
- def _stream_tokens(messages: list[dict], max_new_tokens: int, temperature: float,
93
- top_p: float) -> Iterator[str]:
94
- """Yield generated text deltas. Runs in a separate ZeroGPU worker process.
95
-
96
- Everything crossing this boundary is pickled, so only plain strings are yielded --
97
- never CUDA tensors, which would trigger a blocked `torch.cuda._lazy_init()` in the
98
- parent process.
99
- """
100
- # return_dict=True explicitly, and index the result.
101
- # transformers v5 flipped apply_chat_template's default to return_dict=True, so the
102
- # older `...(return_tensors="pt").to(device)` idiom yields a BatchEncoding whose
103
- # `.shape` lookup falls through __getattr__ to KeyError('shape') -- which crosses the
104
- # ZeroGPU process boundary as a bare AttributeError with the cause stripped off.
105
- # Asking for the dict form is correct on both 4.x and 5.x.
106
- enc = TOKENIZER.apply_chat_template(
107
- messages, add_generation_prompt=True, return_tensors="pt", return_dict=True,
108
- )
109
- input_ids = enc["input_ids"]
110
- attention_mask = enc.get("attention_mask")
111
-
112
- # Guard the context window so an over-long conversation degrades predictably rather
113
- # than producing silent garbage.
114
- if input_ids.shape[-1] > MAX_CONTEXT_TOKENS:
115
- input_ids = input_ids[:, -MAX_CONTEXT_TOKENS:]
116
- if attention_mask is not None:
117
- attention_mask = attention_mask[:, -MAX_CONTEXT_TOKENS:]
118
-
119
- input_ids = input_ids.to(MODEL.device)
120
- if attention_mask is not None:
121
- attention_mask = attention_mask.to(MODEL.device)
122
-
123
- streamer = TextIteratorStreamer(TOKENIZER, skip_prompt=True, skip_special_tokens=True)
124
- kwargs: dict[str, Any] = dict(
125
- input_ids=input_ids,
126
- attention_mask=attention_mask,
127
- streamer=streamer,
128
- max_new_tokens=max_new_tokens,
129
- do_sample=temperature > 0,
130
- pad_token_id=TOKENIZER.pad_token_id,
131
- )
132
- if temperature > 0:
133
- kwargs["temperature"] = temperature
134
- kwargs["top_p"] = top_p
135
-
136
- thread = Thread(target=MODEL.generate, kwargs=kwargs)
137
- thread.start()
138
- for delta in streamer:
139
- if delta:
140
- yield delta
141
- thread.join()
142
-
143
-
144
- def _normalise_messages(raw: list[dict]) -> list[dict]:
145
- """Coerce OpenAI message content into the plain strings the chat template expects."""
146
- out: list[dict] = []
147
- if SYSTEM_PROMPT and not (raw and raw[0].get("role") == "system"):
148
- out.append({"role": "system", "content": SYSTEM_PROMPT})
149
- for m in raw:
150
- content = m.get("content", "")
151
- if isinstance(content, list):
152
- # OpenAI multimodal array form -- keep only the text parts.
153
- content = "".join(
154
- p.get("text", "") for p in content if isinstance(p, dict) and p.get("type") == "text"
155
- )
156
- out.append({"role": m.get("role", "user"), "content": content or ""})
157
- return out
158
-
159
-
160
- # --------------------------------------------------------------------------------------
161
- # OpenAI-compatible API
162
- # --------------------------------------------------------------------------------------
163
- class ChatMessage(BaseModel):
164
- role: str
165
- content: Any = ""
166
-
167
-
168
- class ChatRequest(BaseModel):
169
- model: str | None = None
170
- messages: list[ChatMessage]
171
- stream: bool = False
172
- max_tokens: int | None = Field(default=None, ge=1, le=4096)
173
- temperature: float = Field(default=0.7, ge=0.0, le=2.0)
174
- top_p: float = Field(default=0.95, gt=0.0, le=1.0)
175
-
176
-
177
- api = FastAPI(title=f"THOX ZeroGPU — {SERVED_MODEL_NAME}")
178
-
179
-
180
- @api.get("/health")
181
- def health() -> dict:
182
- return {"status": "ok", "model": SERVED_MODEL_NAME, "base": MODEL_ID, "adapter": ADAPTER_ID}
183
-
184
-
185
- @api.get("/v1/models")
186
- def list_models() -> dict:
187
- return {
188
- "object": "list",
189
- "data": [{
190
- "id": SERVED_MODEL_NAME,
191
- "object": "model",
192
- "created": 0,
193
- "owned_by": "thox",
194
- }],
195
- }
196
-
197
-
198
- @api.post("/v1/chat/completions")
199
- def chat_completions(req: ChatRequest):
200
- messages = _normalise_messages([m.model_dump() for m in req.messages])
201
- max_new = req.max_tokens or 512
202
- completion_id = f"chatcmpl-{uuid.uuid4().hex[:24]}"
203
- created = int(time.time())
204
-
205
- if not req.stream:
206
- text = "".join(_stream_tokens(messages, max_new, req.temperature, req.top_p))
207
- # Count via the dict form for the same v5 reason as above: the bare call returns
208
- # a mapping, so len() would count keys rather than tokens.
209
- prompt_tokens = int(TOKENIZER.apply_chat_template(
210
- messages, add_generation_prompt=True, return_tensors="pt", return_dict=True,
211
- )["input_ids"].shape[-1])
212
- completion_tokens = len(TOKENIZER.encode(text))
213
- return JSONResponse({
214
- "id": completion_id,
215
- "object": "chat.completion",
216
- "created": created,
217
- "model": SERVED_MODEL_NAME,
218
- "choices": [{
219
- "index": 0,
220
- "message": {"role": "assistant", "content": text},
221
- "finish_reason": "stop",
222
- }],
223
- "usage": {
224
- "prompt_tokens": prompt_tokens,
225
- "completion_tokens": completion_tokens,
226
- "total_tokens": prompt_tokens + completion_tokens,
227
- },
228
- })
229
-
230
- def sse() -> Iterator[str]:
231
- base = {
232
- "id": completion_id,
233
- "object": "chat.completion.chunk",
234
- "created": created,
235
- "model": SERVED_MODEL_NAME,
236
- }
237
- # Open WebUI expects the role to arrive before any content delta.
238
- yield f"data: {json.dumps({**base, 'choices': [{'index': 0, 'delta': {'role': 'assistant'}, 'finish_reason': None}]})}\n\n"
239
- try:
240
- for delta in _stream_tokens(messages, max_new, req.temperature, req.top_p):
241
- chunk = {**base, "choices": [{"index": 0, "delta": {"content": delta},
242
- "finish_reason": None}]}
243
- yield f"data: {json.dumps(chunk)}\n\n"
244
- except Exception as exc: # noqa: BLE001
245
- # Headers are already sent, so a failure cannot become an HTTP 500. Emit a
246
- # sanitised terminal event instead of tearing the connection down mid-frame,
247
- # which Open WebUI surfaces as a TransferEncodingError.
248
- print(f"[thox] generation failed: {type(exc).__name__}", flush=True)
249
- err = {**base, "choices": [{"index": 0, "delta": {}, "finish_reason": "error"}]}
250
- yield f"data: {json.dumps(err)}\n\n"
251
- yield "data: [DONE]\n\n"
252
- return
253
- done = {**base, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}
254
- yield f"data: {json.dumps(done)}\n\n"
255
- yield "data: [DONE]\n\n"
256
-
257
- return StreamingResponse(sse(), media_type="text/event-stream",
258
- headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})
259
-
260
-
261
- # --------------------------------------------------------------------------------------
262
- # Gradio UI (mounted at /)
263
- # --------------------------------------------------------------------------------------
264
- THOX_GREEN, THOX_ACCENT = "#05A451", "#45e887"
265
-
266
- CSS = f"""
267
- :root {{ --thox-green: {THOX_GREEN}; --thox-accent: {THOX_ACCENT}; }}
268
- .gradio-container {{ background: #09090B !important; }}
269
- #thox-header {{
270
- border-left: 3px solid var(--thox-green); padding: 0.6rem 0 0.6rem 0.9rem; margin-bottom: 0.5rem;
271
- }}
272
- #thox-header h1 {{ color: var(--thox-accent); font-size: 1.15rem; margin: 0; letter-spacing: .02em; }}
273
- #thox-header p {{ color: #a1a1aa; font-size: .85rem; margin: .25rem 0 0; }}
274
- #thox-header code {{ color: var(--thox-accent); }}
275
- """
276
-
277
-
278
- def _ui_respond(message: str, history: list[dict]) -> Iterator[str]:
279
- convo = _normalise_messages([*history, {"role": "user", "content": message}])
280
- acc = ""
281
- for delta in _stream_tokens(convo, 512, 0.7, 0.95):
282
- acc += delta
283
- yield acc
284
-
285
-
286
- with gr.Blocks(css=CSS, title=f"THOX · {SERVED_MODEL_NAME}", theme=gr.themes.Base()) as demo:
287
- gr.HTML(
288
- f"""<div id="thox-header">
289
- <h1>THOX &middot; {SERVED_MODEL_NAME}</h1>
290
- <p>{DESCRIPTION}</p>
291
- <p>OpenAI-compatible endpoint: <code>/v1/chat/completions</code> &middot;
292
- Your AI. Your Data. Your Rules.&trade;</p>
293
- </div>"""
294
- )
295
- gr.ChatInterface(fn=_ui_respond, type="messages")
296
-
297
- # Expose the FastAPI app for anyone importing this module (and for local `uvicorn app:app`).
298
- app = api
299
-
300
- if __name__ == "__main__":
301
- # LET GRADIO OWN THE SERVER.
302
- #
303
- # The obvious construction -- gr.mount_gradio_app(fastapi_app, demo) plus our own
304
- # uvicorn.run() -- does not survive on a Gradio SDK Space. The platform launches the
305
- # Blocks app itself, so our uvicorn came up as a redundant second server and was
306
- # SIGTERM'd about one second after binding ("Uvicorn running ..." immediately
307
- # followed by "Shutting down"). Binding the port the platform wanted instead failed
308
- # with "[Errno 98] address already in use", because the platform's own launch had
309
- # already taken it. Both failures are the same underlying mistake: competing for the
310
- # server rather than joining it.
311
- #
312
- # So Gradio launches (which is also what ZeroGPU expects), and the OpenAI routes are
313
- # grafted onto the FastAPI app Gradio builds. Starlette matches each request against
314
- # a plain list of routes, so inserting ahead of Gradio's catch-all is enough to make
315
- # /v1/* resolve to us while everything else stays Gradio's.
316
- demo.queue(max_size=32)
317
- _, local_url, _ = demo.launch(prevent_thread_lock=True, server_name="0.0.0.0",
318
- ssr_mode=False, show_api=False)
319
- print(f"[thox] gradio launched at {local_url}", flush=True)
320
-
321
- grafted = []
322
- for route in api.routes:
323
- path = getattr(route, "path", "")
324
- if path.startswith("/v1") or path == "/health":
325
- demo.app.router.routes.insert(0, route)
326
- grafted.append(path)
327
- print(f"[thox] grafted OpenAI routes onto gradio app: {grafted}", flush=True)
328
-
329
- # launch(prevent_thread_lock=True) returns immediately; block so PID 1 stays alive.
330
- demo.block_thread()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """THOX ZeroGPU model Space — one model behind an OpenAI-compatible /v1 endpoint.
2
+
3
+ This single file is deployed verbatim to every THOX ZeroGPU model Space. The only
4
+ per-Space difference is `thox_space_config.py`, which names the model to serve.
5
+
6
+ Why FastAPI *and* Gradio: Open WebUI consumes this Space as an ordinary "OpenAI
7
+ API" connection, which means it needs `/v1/models` and `/v1/chat/completions`.
8
+ Gradio alone does not speak that protocol, so the FastAPI app owns the `/v1`
9
+ routes and the Gradio Blocks UI is mounted at `/` for humans. ZeroGPU only
10
+ schedules Gradio SDK Spaces, so the Gradio app must genuinely be present -- it is
11
+ not decoration.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import json
17
+ import os
18
+ import time
19
+ import uuid
20
+ from threading import Thread
21
+ from typing import Any, Iterator
22
+
23
+ # Disable Gradio 5 server-side rendering BEFORE gradio is imported.
24
+ # On a Spaces Gradio SDK Space, SSR spawns a Node.js front-end server that claims the
25
+ # port this app needs, producing "[Errno 98] address already in use" followed by
26
+ # "Stopping Node.js server...". We serve the Blocks app from our own uvicorn process, so
27
+ # the SSR layer has nothing to add here and everything to collide with.
28
+ os.environ.setdefault("GRADIO_SSR_MODE", "false")
29
+
30
+ import gradio as gr # noqa: E402
31
+ import spaces
32
+ import torch
33
+ import uvicorn
34
+ from fastapi import FastAPI
35
+ from fastapi.responses import JSONResponse, StreamingResponse
36
+ from pydantic import BaseModel, Field
37
+ from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
38
+
39
+ from thox_space_config import (
40
+ ADAPTER_ID,
41
+ CHAT_TEMPLATE_KWARGS,
42
+ DESCRIPTION,
43
+ MAX_CONTEXT_TOKENS,
44
+ MODEL_ID,
45
+ SERVED_MODEL_NAME,
46
+ SYSTEM_PROMPT,
47
+ )
48
+
49
+ # --------------------------------------------------------------------------------------
50
+ # Model loading (module scope, eager -- see ZeroGPU rules)
51
+ # --------------------------------------------------------------------------------------
52
+ # ZeroGPU monkey-patches torch so `cuda` resolves at module scope and the weights are
53
+ # registered with the scheduler, then streamed into VRAM when a @spaces.GPU call lands.
54
+ # Loading lazily on first request would just move this cost onto the first user.
55
+ DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
56
+ DTYPE = torch.bfloat16 if DEVICE.type == "cuda" else torch.float32
57
+
58
+ print(f"[thox] loading base model: {MODEL_ID}", flush=True)
59
+ TOKENIZER = AutoTokenizer.from_pretrained(MODEL_ID)
60
+ if TOKENIZER.pad_token_id is None:
61
+ TOKENIZER.pad_token = TOKENIZER.eos_token
62
+
63
+ MODEL = AutoModelForCausalLM.from_pretrained(MODEL_ID, dtype=DTYPE).to(DEVICE)
64
+
65
+ if ADAPTER_ID:
66
+ # THOX ships several of its models as LoRA adapters rather than merged weights.
67
+ from peft import PeftModel
68
+
69
+ print(f"[thox] applying LoRA adapter: {ADAPTER_ID}", flush=True)
70
+ MODEL = PeftModel.from_pretrained(MODEL, ADAPTER_ID)
71
+ MODEL = MODEL.merge_and_unload() # merge so generation runs at full speed
72
+
73
+ MODEL.eval()
74
+ print(f"[thox] ready: serving '{SERVED_MODEL_NAME}' on {DEVICE}", flush=True)
75
+
76
+
77
+ # --------------------------------------------------------------------------------------
78
+ # Generation
79
+ # --------------------------------------------------------------------------------------
80
+ def _encode(messages: list[dict]):
81
+ """Apply the chat template, returning the dict form.
82
+
83
+ return_dict=True is passed explicitly: transformers v5 flipped the default, so the
84
+ older `...(return_tensors="pt").to(device)` idiom yields a BatchEncoding whose
85
+ `.shape` lookup falls through __getattr__ into KeyError('shape') -- which crosses the
86
+ ZeroGPU process boundary as a bare AttributeError with the cause stripped off. Asking
87
+ for the dict form is correct on both 4.x and 5.x.
88
+
89
+ CHAT_TEMPLATE_KWARGS carries per-model template switches (e.g. Qwen3's
90
+ enable_thinking). Unknown keys are simply template variables, so a model whose
91
+ template ignores them is unaffected.
92
+ """
93
+ return TOKENIZER.apply_chat_template(
94
+ messages, add_generation_prompt=True, return_tensors="pt", return_dict=True,
95
+ **CHAT_TEMPLATE_KWARGS,
96
+ )
97
+
98
+
99
+ def _estimate_duration(messages: list[dict], max_new_tokens: int, temperature: float,
100
+ top_p: float) -> int:
101
+ """Request only the GPU seconds this call plausibly needs.
102
+
103
+ ZeroGPU pre-checks the *requested* duration against the caller's remaining quota,
104
+ not the actual runtime -- so leaving everything at the 60s default makes short
105
+ requests fail with "quota exceeded" once a user drops below 60s. Shorter requests
106
+ also rank higher in the node-level queue.
107
+ """
108
+ return int(min(120, max(15, (max_new_tokens / 12) + 10)))
109
+
110
+
111
+ @spaces.GPU(duration=_estimate_duration)
112
+ def _stream_tokens(messages: list[dict], max_new_tokens: int, temperature: float,
113
+ top_p: float) -> Iterator[tuple[str, str]]:
114
+ """Yield ("delta", text) pairs, then exactly one ("finish", reason) pair.
115
+
116
+ Runs in a separate ZeroGPU worker process. Everything crossing this boundary is
117
+ pickled, so only plain strings are yielded -- never CUDA tensors, which would trigger
118
+ a blocked torch.cuda._lazy_init() in the parent process.
119
+
120
+ The trailing ("finish", reason) pair exists because the stop reason is only knowable
121
+ inside the worker: the caller sees text, not token counts, and cannot tell a model
122
+ that chose to stop from one that ran into the max_new_tokens ceiling. Clients rely on
123
+ finish_reason to detect truncation, so reporting "stop" for a truncated response
124
+ makes a cut-off answer look complete.
125
+ """
126
+ # return_dict=True explicitly, and index the result.
127
+ # transformers v5 flipped apply_chat_template's default to return_dict=True, so the
128
+ # older `...(return_tensors="pt").to(device)` idiom yields a BatchEncoding whose
129
+ # `.shape` lookup falls through __getattr__ to KeyError('shape') -- which crosses the
130
+ # ZeroGPU process boundary as a bare AttributeError with the cause stripped off.
131
+ # Asking for the dict form is correct on both 4.x and 5.x.
132
+ enc = _encode(messages)
133
+ input_ids = enc["input_ids"]
134
+ attention_mask = enc.get("attention_mask")
135
+
136
+ # Guard the context window so an over-long conversation degrades predictably rather
137
+ # than producing silent garbage.
138
+ if input_ids.shape[-1] > MAX_CONTEXT_TOKENS:
139
+ input_ids = input_ids[:, -MAX_CONTEXT_TOKENS:]
140
+ if attention_mask is not None:
141
+ attention_mask = attention_mask[:, -MAX_CONTEXT_TOKENS:]
142
+
143
+ input_ids = input_ids.to(MODEL.device)
144
+ if attention_mask is not None:
145
+ attention_mask = attention_mask.to(MODEL.device)
146
+
147
+ streamer = TextIteratorStreamer(TOKENIZER, skip_prompt=True, skip_special_tokens=True)
148
+ kwargs: dict[str, Any] = dict(
149
+ input_ids=input_ids,
150
+ attention_mask=attention_mask,
151
+ streamer=streamer,
152
+ max_new_tokens=max_new_tokens,
153
+ do_sample=temperature > 0,
154
+ pad_token_id=TOKENIZER.pad_token_id,
155
+ )
156
+ if temperature > 0:
157
+ kwargs["temperature"] = temperature
158
+ kwargs["top_p"] = top_p
159
+
160
+ # Capture generate()'s output so the true stop reason is available after the stream
161
+ # drains. Comparing generated length against the ceiling is exact, where inferring it
162
+ # from the decoded text would not be.
163
+ result: dict[str, Any] = {}
164
+
165
+ def _run() -> None:
166
+ try:
167
+ result["ids"] = MODEL.generate(**kwargs)
168
+ except Exception as exc: # noqa: BLE001
169
+ result["error"] = exc
170
+
171
+ thread = Thread(target=_run)
172
+ thread.start()
173
+ for delta in streamer:
174
+ if delta:
175
+ yield "delta", delta
176
+ thread.join()
177
+
178
+ if "error" in result:
179
+ raise result["error"]
180
+
181
+ generated = int(result["ids"].shape[-1] - input_ids.shape[-1]) if "ids" in result else 0
182
+ yield "finish", ("length" if generated >= max_new_tokens else "stop")
183
+
184
+
185
+ def _normalise_messages(raw: list[dict]) -> list[dict]:
186
+ """Coerce OpenAI message content into the plain strings the chat template expects."""
187
+ out: list[dict] = []
188
+ if SYSTEM_PROMPT and not (raw and raw[0].get("role") == "system"):
189
+ out.append({"role": "system", "content": SYSTEM_PROMPT})
190
+ for m in raw:
191
+ content = m.get("content", "")
192
+ if isinstance(content, list):
193
+ # OpenAI multimodal array form -- keep only the text parts.
194
+ content = "".join(
195
+ p.get("text", "") for p in content if isinstance(p, dict) and p.get("type") == "text"
196
+ )
197
+ out.append({"role": m.get("role", "user"), "content": content or ""})
198
+ return out
199
+
200
+
201
+ # --------------------------------------------------------------------------------------
202
+ # OpenAI-compatible API
203
+ # --------------------------------------------------------------------------------------
204
+ class ChatMessage(BaseModel):
205
+ role: str
206
+ content: Any = ""
207
+
208
+
209
+ class ChatRequest(BaseModel):
210
+ model: str | None = None
211
+ messages: list[ChatMessage]
212
+ stream: bool = False
213
+ max_tokens: int | None = Field(default=None, ge=1, le=4096)
214
+ temperature: float = Field(default=0.7, ge=0.0, le=2.0)
215
+ top_p: float = Field(default=0.95, gt=0.0, le=1.0)
216
+
217
+
218
+ api = FastAPI(title=f"THOX ZeroGPU — {SERVED_MODEL_NAME}")
219
+
220
+
221
+ @api.get("/health")
222
+ def health() -> dict:
223
+ return {"status": "ok", "model": SERVED_MODEL_NAME, "base": MODEL_ID, "adapter": ADAPTER_ID}
224
+
225
+
226
+ @api.get("/v1/models")
227
+ def list_models() -> dict:
228
+ return {
229
+ "object": "list",
230
+ "data": [{
231
+ "id": SERVED_MODEL_NAME,
232
+ "object": "model",
233
+ "created": 0,
234
+ "owned_by": "thox",
235
+ }],
236
+ }
237
+
238
+
239
+ @api.post("/v1/chat/completions")
240
+ def chat_completions(req: ChatRequest):
241
+ messages = _normalise_messages([m.model_dump() for m in req.messages])
242
+ max_new = req.max_tokens or 512
243
+ completion_id = f"chatcmpl-{uuid.uuid4().hex[:24]}"
244
+ created = int(time.time())
245
+
246
+ if not req.stream:
247
+ parts: list[str] = []
248
+ finish = "stop"
249
+ for kind, payload in _stream_tokens(messages, max_new, req.temperature, req.top_p):
250
+ if kind == "delta":
251
+ parts.append(payload)
252
+ else:
253
+ finish = payload
254
+ text = "".join(parts)
255
+ prompt_tokens = int(_encode(messages)["input_ids"].shape[-1])
256
+ completion_tokens = len(TOKENIZER.encode(text))
257
+ return JSONResponse({
258
+ "id": completion_id,
259
+ "object": "chat.completion",
260
+ "created": created,
261
+ "model": SERVED_MODEL_NAME,
262
+ "choices": [{
263
+ "index": 0,
264
+ "message": {"role": "assistant", "content": text},
265
+ "finish_reason": finish,
266
+ }],
267
+ "usage": {
268
+ "prompt_tokens": prompt_tokens,
269
+ "completion_tokens": completion_tokens,
270
+ "total_tokens": prompt_tokens + completion_tokens,
271
+ },
272
+ })
273
+
274
+ def sse() -> Iterator[str]:
275
+ base = {
276
+ "id": completion_id,
277
+ "object": "chat.completion.chunk",
278
+ "created": created,
279
+ "model": SERVED_MODEL_NAME,
280
+ }
281
+ # Open WebUI expects the role to arrive before any content delta.
282
+ yield f"data: {json.dumps({**base, 'choices': [{'index': 0, 'delta': {'role': 'assistant'}, 'finish_reason': None}]})}\n\n"
283
+ finish = "stop"
284
+ try:
285
+ for kind, payload in _stream_tokens(messages, max_new, req.temperature,
286
+ req.top_p):
287
+ if kind == "finish":
288
+ finish = payload
289
+ continue
290
+ chunk = {**base, "choices": [{"index": 0, "delta": {"content": payload},
291
+ "finish_reason": None}]}
292
+ yield f"data: {json.dumps(chunk)}\n\n"
293
+ except Exception as exc: # noqa: BLE001
294
+ # Headers are already sent, so a failure cannot become an HTTP 500. Emit a
295
+ # sanitised terminal event instead of tearing the connection down mid-frame,
296
+ # which Open WebUI surfaces as a TransferEncodingError.
297
+ print(f"[thox] generation failed: {type(exc).__name__}", flush=True)
298
+ err = {**base, "choices": [{"index": 0, "delta": {}, "finish_reason": "error"}]}
299
+ yield f"data: {json.dumps(err)}\n\n"
300
+ yield "data: [DONE]\n\n"
301
+ return
302
+ done = {**base, "choices": [{"index": 0, "delta": {}, "finish_reason": finish}]}
303
+ yield f"data: {json.dumps(done)}\n\n"
304
+ yield "data: [DONE]\n\n"
305
+
306
+ return StreamingResponse(sse(), media_type="text/event-stream",
307
+ headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})
308
+
309
+
310
+ # --------------------------------------------------------------------------------------
311
+ # Gradio UI (mounted at /)
312
+ # --------------------------------------------------------------------------------------
313
+ THOX_GREEN, THOX_ACCENT = "#05A451", "#45e887"
314
+
315
+ CSS = f"""
316
+ :root {{ --thox-green: {THOX_GREEN}; --thox-accent: {THOX_ACCENT}; }}
317
+ .gradio-container {{ background: #09090B !important; }}
318
+ #thox-header {{
319
+ border-left: 3px solid var(--thox-green); padding: 0.6rem 0 0.6rem 0.9rem; margin-bottom: 0.5rem;
320
+ }}
321
+ #thox-header h1 {{ color: var(--thox-accent); font-size: 1.15rem; margin: 0; letter-spacing: .02em; }}
322
+ #thox-header p {{ color: #a1a1aa; font-size: .85rem; margin: .25rem 0 0; }}
323
+ #thox-header code {{ color: var(--thox-accent); }}
324
+ """
325
+
326
+
327
+ def _ui_respond(message: str, history: list[dict]) -> Iterator[str]:
328
+ convo = _normalise_messages([*history, {"role": "user", "content": message}])
329
+ acc = ""
330
+ for kind, payload in _stream_tokens(convo, 512, 0.7, 0.95):
331
+ if kind == "delta":
332
+ acc += payload
333
+ yield acc
334
+
335
+
336
+ with gr.Blocks(css=CSS, title=f"THOX · {SERVED_MODEL_NAME}", theme=gr.themes.Base()) as demo:
337
+ gr.HTML(
338
+ f"""<div id="thox-header">
339
+ <h1>THOX &middot; {SERVED_MODEL_NAME}</h1>
340
+ <p>{DESCRIPTION}</p>
341
+ <p>OpenAI-compatible endpoint: <code>/v1/chat/completions</code> &middot;
342
+ Your AI. Your Data. Your Rules.&trade;</p>
343
+ </div>"""
344
+ )
345
+ gr.ChatInterface(fn=_ui_respond, type="messages")
346
+
347
+ # Expose the FastAPI app for anyone importing this module (and for local `uvicorn app:app`).
348
+ app = api
349
+
350
+ if __name__ == "__main__":
351
+ # LET GRADIO OWN THE SERVER.
352
+ #
353
+ # The obvious construction -- gr.mount_gradio_app(fastapi_app, demo) plus our own
354
+ # uvicorn.run() -- does not survive on a Gradio SDK Space. The platform launches the
355
+ # Blocks app itself, so our uvicorn came up as a redundant second server and was
356
+ # SIGTERM'd about one second after binding ("Uvicorn running ..." immediately
357
+ # followed by "Shutting down"). Binding the port the platform wanted instead failed
358
+ # with "[Errno 98] address already in use", because the platform's own launch had
359
+ # already taken it. Both failures are the same underlying mistake: competing for the
360
+ # server rather than joining it.
361
+ #
362
+ # So Gradio launches (which is also what ZeroGPU expects), and the OpenAI routes are
363
+ # grafted onto the FastAPI app Gradio builds. Starlette matches each request against
364
+ # a plain list of routes, so inserting ahead of Gradio's catch-all is enough to make
365
+ # /v1/* resolve to us while everything else stays Gradio's.
366
+ demo.queue(max_size=32)
367
+ _, local_url, _ = demo.launch(prevent_thread_lock=True, server_name="0.0.0.0",
368
+ ssr_mode=False, show_api=False)
369
+ print(f"[thox] gradio launched at {local_url}", flush=True)
370
+
371
+ grafted = []
372
+ for route in api.routes:
373
+ path = getattr(route, "path", "")
374
+ if path.startswith("/v1") or path == "/health":
375
+ demo.app.router.routes.insert(0, route)
376
+ grafted.append(path)
377
+ print(f"[thox] grafted OpenAI routes onto gradio app: {grafted}", flush=True)
378
+
379
+ # launch(prevent_thread_lock=True) returns immediately; block so PID 1 stays alive.
380
+ demo.block_thread()
thox_space_config.py CHANGED
@@ -9,6 +9,9 @@ SERVED_MODEL_NAME = 'thox-coder-7b'
9
  DESCRIPTION = 'Code generation, review and refactoring. Qwen2.5-Coder-7B-Instruct on ZeroGPU.'
10
  SYSTEM_PROMPT = 'You are a THOX coding assistant. Be precise and concise. When you write code, make it complete and runnable.'
11
 
 
 
 
12
  # Truncate prompts beyond this so an over-long conversation degrades predictably
13
  # instead of erroring deep inside generation.
14
  MAX_CONTEXT_TOKENS = 8192
 
9
  DESCRIPTION = 'Code generation, review and refactoring. Qwen2.5-Coder-7B-Instruct on ZeroGPU.'
10
  SYSTEM_PROMPT = 'You are a THOX coding assistant. Be precise and concise. When you write code, make it complete and runnable.'
11
 
12
+ # Extra kwargs handed to tokenizer.apply_chat_template (per-model template switches).
13
+ CHAT_TEMPLATE_KWARGS = {}
14
+
15
  # Truncate prompts beyond this so an over-long conversation degrades predictably
16
  # instead of erroring deep inside generation.
17
  MAX_CONTEXT_TOKENS = 8192