deploy: warm shared fast-chat prefix

Exact provider tree from ttracx/thoxroute main c19c9b55d9b8bd33a32be5130697c0a019afb3c8; local CI 874 tests and Q4_K_M container validation passed.

README.md

@@ -24,5 +24,10 @@ evaluation so upstream time-to-first-byte watchdogs do not abandon healthy CPU
 work; two-second SSE comments keep the stream live during synchronous prompt
 evaluation. The interactive provider itself caps every request at 16 output
 tokens, so malformed or older callers cannot leave minutes of abandoned work.
+Every interactive request starts with the same neutral provider-owned system
+prefix warmed during startup. Caller-owned system messages follow it and remain
+the authoritative persona and identity. This preserves llama prefix-cache reuse
+across otherwise unrelated personas without replacing caller instructions; the
+provider does not add this fast-chat prefix to specialist requests.
 The interactive model is pinned to immutable model revision
 `9217f5db79a29953eb74d5343926648285ec7e67` and baked into the image.

app.py

@@ -60,7 +60,12 @@ MAX_MESSAGES = 64
 MAX_MESSAGE_CHARS = 65_536
 MAX_REQUEST_CHARS = 131_072
 
-SYSTEM = (
+PROVIDER_SYSTEM_PREFIX = (
+    "Follow the caller-provided system instructions and requested output format "
+    "exactly. Treat later system messages as the authoritative assistant persona "
+    "and identity. Never claim that cloud inference ran locally or on-device."
+)
+DEFAULT_SYSTEM = (
     "You are THOX Fast Chat, a concise privacy-first assistant. Follow the "
     "user's requested output format exactly. Never claim that cloud inference "
     "ran locally or on-device."
@@ -119,7 +124,8 @@ def _warm_interactive_model(model: Any) -> None:
 
     model.create_chat_completion(
         messages=[
-            {"role": "system", "content": SYSTEM},
+            {"role": "system", "content": PROVIDER_SYSTEM_PREFIX},
+            {"role": "system", "content": DEFAULT_SYSTEM},
             {"role": "user", "content": "Reply with OK."},
         ],
         max_tokens=1,
@@ -218,9 +224,25 @@ api = FastAPI(
 
 
 def _messages(req: ChatRequest) -> list[dict[str, str]]:
+    """Build a bounded prompt whose first tokens survive cross-persona traffic.
+
+    Every fast-chat request begins with the exact prefix paid for during startup
+    warmup. Caller system messages retain their order immediately after it and
+    remain authoritative, while callers without a system message receive the
+    existing default assistant persona. Specialist prompts keep their previous
+    behavior and never inherit the fast-chat prefix.
+    """
+
     messages = [message.model_dump() for message in req.messages]
-    if not any(message["role"] == "system" for message in messages):
-        messages.insert(0, {"role": "system", "content": SYSTEM})
+    has_caller_system = any(message["role"] == "system" for message in messages)
+    if req.model == FAST_MODEL_ID:
+        provider_prefix = {"role": "system", "content": PROVIDER_SYSTEM_PREFIX}
+        if not messages or messages[0] != provider_prefix:
+            messages.insert(0, provider_prefix)
+        if not has_caller_system:
+            messages.insert(1, {"role": "system", "content": DEFAULT_SYSTEM})
+    elif not has_caller_system:
+        messages.insert(0, {"role": "system", "content": DEFAULT_SYSTEM})
     return messages
 
 
@@ -246,6 +268,7 @@ def healthz() -> dict[str, Any]:
         "interactive_model_revision": FAST_MODEL_REVISION,
         "interactive_pool_size": FAST_POOL_SIZE,
         "interactive_available": active.interactive_available,
+        "interactive_common_prefix_warmed": True,
         "specialist_model": CODER_MODEL_ID,
         "interactive_max_output_tokens": FAST_MAX_OUTPUT_TOKENS,
         "stream_heartbeat_s": STREAM_HEARTBEAT_S,