Optimize conversational turns on ZeroGPU

README.md

@@ -55,3 +55,7 @@ a slow ZeroGPU generation does not block health checks or web-search requests.
 The aliases `gpt-5.6-luna-max`, `gpt-5.6-luna`, and `qwen-coder` all route to
 the same Qwen backend. Use the first alias for the Luna Max profile; use
 `qwen-coder` when a client requires the historical model name.
+
+Simple greetings such as `ola`/`olá` use a deterministic fast path and do not
+queue a full 30B inference. Tool catalogs are also omitted from ordinary
+conversation turns and retained when the request actually needs a tool.

app.py

@@ -46,6 +46,7 @@ from generation import (
 from openai_compat import (
     analyze_tool_flow,
     indexed_tool_calls,
+    is_simple_greeting,
     normalize_tools,
     resolve_tool_choice,
     select_tools,
@@ -68,6 +69,14 @@ from web_search import SearchUnavailable, search_web
 
 logger = logging.getLogger("qwen-coder-api")
 
+# Prefer the fastest safe CUDA kernels for the mixed-precision checkpoint.
+# These flags do not change the FP8 weights or the public API contract.
+torch.set_float32_matmul_precision("high")
+if hasattr(torch.backends, "cuda"):
+    torch.backends.cuda.matmul.allow_tf32 = True
+if hasattr(torch.backends, "cudnn"):
+    torch.backends.cudnn.allow_tf32 = True
+
 
 # Qwen3-Coder 30B is trained for long-horizon agentic coding and native tool
 # use. Its official fine-grained FP8 checkpoint leaves enough runtime margin on
@@ -129,6 +138,12 @@ MAX_TOOL_CALL_TOKENS = int(os.getenv("MAX_TOOL_CALL_TOKENS", "2048"))
 MAX_TEMPERATURE = float(os.getenv("MAX_TEMPERATURE", "0.2"))
 PRESERVED_PREFIX_TOKENS = int(os.getenv("PRESERVED_PREFIX_TOKENS", "4096"))
 DEVICE = os.getenv("DEVICE", "cuda").strip() or "cuda"
+ENABLE_FAST_GREETING = os.getenv("ENABLE_FAST_GREETING", "1").casefold() not in {
+    "0",
+    "false",
+    "no",
+    "off",
+}
 
 tokenizer = AutoTokenizer.from_pretrained(MODEL)
 
@@ -313,6 +328,7 @@ def gerar(
         "max_new_tokens": output_tokens,
         "do_sample": float(temperature) > 0,
         "pad_token_id": tokenizer.pad_token_id or tokenizer.eos_token_id,
+        "use_cache": True,
     }
     if eos_token_ids is not None:
         generation_kwargs["eos_token_id"] = eos_token_ids
@@ -353,6 +369,48 @@ def _completion_payload(request: ChatCompletionRequest) -> dict[str, Any]:
     if request.model.casefold() not in _MODEL_ALIASES_CASEFOLDED:
         raise HTTPException(status_code=404, detail=f"Model not available: {request.model}")
 
+    # Avoid a full 30B inference for a greeting. OpenClaude includes its
+    # complete tool catalog in these turns, but there is no useful tool work.
+    # Keep this opt-out available for clients that want every turn model-backed.
+    explicit_tool_choice = request.tool_choice
+    has_tool_history = any(
+        isinstance(message, dict)
+        and (
+            message.get("role") == "tool"
+            or bool(message.get("tool_calls"))
+        )
+        for message in request.messages
+    )
+    if (
+        ENABLE_FAST_GREETING
+        and is_simple_greeting(request.messages)
+        and not has_tool_history
+        and (
+            explicit_tool_choice is None
+            or (
+                isinstance(explicit_tool_choice, str)
+                and explicit_tool_choice.casefold() == "auto"
+            )
+        )
+    ):
+        return {
+            "id": f"chatcmpl-{uuid.uuid4().hex}",
+            "object": "chat.completion",
+            "created": int(time.time()),
+            "model": request.model,
+            "choices": [
+                {
+                    "index": 0,
+                    "message": {
+                        "role": "assistant",
+                        "content": "Olá! Como posso ajudar você hoje?",
+                    },
+                    "finish_reason": "stop",
+                }
+            ],
+            "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0},
+        }
+
     already_adapted = has_tool_protocol(request.messages)
     flow_state = analyze_tool_flow(request.messages, request.tools or [])
     state_controls_choice = request.tool_choice is None or (

openai_compat.py

@@ -119,6 +119,10 @@ NO_TOOLS_RE = re.compile(
     r"without"
     r")\s+(?:as?\s+)?(?:ferramentas?|tools?)\b"
 )
+SIMPLE_GREETING_RE = re.compile(
+    r"(?i)^\s*(?:oi|ol[aá]|hello|hi|hey|bom\s+dia|boa\s+tarde|boa\s+noite)"
+    r"[\s!,.?]*$"
+)
 OPENCLAUDE_METADATA_BLOCK_RE = re.compile(
     r"<(?P<tag>available-deferred-tools|system-reminder)\b[^>]*>.*?</(?P=tag)>",
     re.DOTALL | re.IGNORECASE,
@@ -338,6 +342,16 @@ def _latest_user_request(messages: object) -> str:
     return latest
 
 
+def is_simple_greeting(messages: object) -> bool:
+    """Identify a greeting that does not need a model or tool prompt.
+
+    OpenClaude sends its complete tool catalog even for ``ola``.  Calling a
+    30B model for that turn adds tens of seconds on ZeroGPU without adding
+    useful work, so the API can answer it deterministically before inference.
+    """
+    return bool(SIMPLE_GREETING_RE.fullmatch(_latest_user_request(messages)))
+
+
 def _explicitly_disables_tools(messages: object) -> bool:
     if not isinstance(messages, list):
         return False
@@ -632,8 +646,13 @@ def resolve_tool_choice(
     """Override only auto/default choices; explicit client choices win."""
     if (
         state.reason == "no concrete tool action was requested"
-        and isinstance(requested_choice, str)
-        and requested_choice.casefold() == "required"
+        and (
+            requested_choice is None
+            or (
+                isinstance(requested_choice, str)
+                and requested_choice.casefold() in {"auto", "required"}
+            )
+        )
     ):
         return "none"
     is_auto = requested_choice is None or (

test_openai_compat.py

@@ -8,6 +8,7 @@ from openai_compat import (
     _tool_result_events,
     analyze_tool_flow,
     indexed_tool_calls,
+    is_simple_greeting,
     normalize_messages,
     normalize_tools,
     resolve_tool_choice,
@@ -144,8 +145,25 @@ class OpenAICompatibilityTests(unittest.TestCase):
             TOOLS,
         )
         self.assertFalse(state.active)
+        self.assertEqual(resolve_tool_choice(None, state), "none")
+        self.assertEqual(resolve_tool_choice("auto", state), "none")
         self.assertEqual(resolve_tool_choice("required", state), "none")
 
+    def test_openclaude_greeting_metadata_is_fast_path_safe(self) -> None:
+        messages = [
+            {
+                "role": "user",
+                "content": (
+                    "<available-deferred-tools>\nBash\n"
+                    "</available-deferred-tools>\n"
+                    "<system-reminder>Create code and run tests.</system-reminder>\n"
+                    "ola\n<system-reminder>snip_id=x</system-reminder>"
+                ),
+            }
+        ]
+        self.assertTrue(is_simple_greeting(messages))
+        self.assertFalse(is_simple_greeting([{"role": "user", "content": "ola, leia app.py"}]))
+
     def test_openclaude_metadata_does_not_become_user_intent(self) -> None:
         state = analyze_tool_flow(
             [
@@ -921,7 +939,7 @@ class OpenAICompatibilityTests(unittest.TestCase):
                     [*TOOLS, EDIT_TOOL, *WEB_TOOLS],
                 )
                 self.assertFalse(state.active)
-                self.assertIsNone(resolve_tool_choice(None, state))
+                self.assertEqual(resolve_tool_choice(None, state), "none")
 
 
 if __name__ == "__main__":