Bound interactive inference backpressure

Exact source: ttracx/thoxroute@4f71bbe0dea6d8f029c06a66f010928f0f363550

Dockerfile

@@ -23,7 +23,8 @@ ENV HF_HOME=/home/mambauser/.cache/huggingface \
     THOX_FAST_MODEL_REPO=Qwen/Qwen2.5-0.5B-Instruct-GGUF \
     THOX_FAST_MODEL_REVISION=9217f5db79a29953eb74d5343926648285ec7e67 \
     THOX_FAST_MODEL_FILE=qwen2.5-0.5b-instruct-q4_k_m.gguf \
-    THOX_FAST_POOL_SIZE=2 \
+    THOX_FAST_POOL_SIZE=1 \
+    THOX_FAST_QUEUE_TIMEOUT_S=6 \
     THOX_MAX_OUTPUT_TOKENS=128
 
 # Bake the immutable public interactive model into the image. A RUNNING Space

README.md

@@ -12,10 +12,14 @@ license: apache-2.0
 
 This Space exposes one OpenAI-compatible endpoint:
 
-- `thox-fast-chat`: eagerly loaded, bounded, interactive Qwen2.5 0.5B target;
+- `thox-fast-chat`: eagerly loaded, bounded, full-CPU Qwen2.5 0.5B target with
+  a six-second queue ceiling;
 - `thox-rust-coder`: lazily loaded 25B/3B-active Rust-specialist target.
 
-`GET /healthz` is ready only after the two-instance interactive pool is loaded
-and each llama context has completed a one-token startup warmup.
+`GET /healthz` is ready only after the interactive context is loaded and has
+completed a one-token startup warmup. Streaming sends a role chunk before model
+evaluation so upstream time-to-first-byte watchdogs do not abandon healthy CPU
+work. First-party 15-second requests are capped at 16 output tokens by
+ThoxRoute.
 The interactive model is pinned to immutable model revision
 `9217f5db79a29953eb74d5343926648285ec7e67` and baked into the image.

app.py

@@ -43,7 +43,10 @@ CODER_MODEL_FILE = os.environ.get(
     "THOX_CODER_MODEL_FILE", "Qwen3-Coder-REAP-25B-A3B-Rust-Q4_K_M.gguf"
 )
 N_CTX = int(os.environ.get("THOX_N_CTX", "4096"))
-FAST_POOL_SIZE = max(1, min(int(os.environ.get("THOX_FAST_POOL_SIZE", "2")), 4))
+FAST_POOL_SIZE = max(1, min(int(os.environ.get("THOX_FAST_POOL_SIZE", "1")), 2))
+FAST_QUEUE_TIMEOUT_S = max(
+    0.1, min(float(os.environ.get("THOX_FAST_QUEUE_TIMEOUT_S", "6")), 10.0)
+)
 MAX_OUTPUT_TOKENS = max(
     1, min(int(os.environ.get("THOX_MAX_OUTPUT_TOKENS", "128")), 512)
 )
@@ -142,7 +145,7 @@ class Runtime:
     def acquire(self, model_id: str) -> Lease:
         if model_id == FAST_MODEL_ID:
             try:
-                model = self._fast.get_nowait()
+                model = self._fast.get(timeout=FAST_QUEUE_TIMEOUT_S)
             except queue.Empty as exc:
                 raise HTTPException(status_code=429, detail="interactive capacity busy") from exc
             return Lease(model=model, release=lambda: self._fast.put(model))
@@ -251,6 +254,23 @@ def _stream_completion(req: ChatRequest, lease: Lease) -> Iterator[bytes]:
     completion_id = _completion_id()
     created = int(time.time())
     try:
+        # Send a valid role chunk before llama prompt evaluation. This proves
+        # the stream is alive inside ThoxRoute's short time-to-first-byte budget
+        # and prevents a healthy CPU inference from being abandoned at 12 s.
+        initial = {
+            "id": completion_id,
+            "object": "chat.completion.chunk",
+            "created": created,
+            "model": req.model,
+            "choices": [
+                {
+                    "index": 0,
+                    "delta": {"role": "assistant", "content": ""},
+                    "finish_reason": None,
+                }
+            ],
+        }
+        yield f"data: {json.dumps(initial, separators=(',', ':'))}\n\n".encode()
         chunks = lease.model.create_chat_completion(
             messages=_messages(req),
             max_tokens=_max_tokens(req),