Update app.py

app.py

@@ -71,7 +71,7 @@ RAM_WARN_GB   = float(os.getenv("RAM_WARN_GB",   "0"))  # 0 = auto 82% of total
 RAM_REJECT_GB = float(os.getenv("RAM_REJECT_GB", "0"))  # 0 = auto 90% of total RAM
 RAM_FLUSH_GB  = float(os.getenv("RAM_FLUSH_GB",  "0"))  # 0 = auto 95% of total RAM
 REQUEST_TIMEOUT  = float(os.getenv("REQUEST_TIMEOUT","90.0"))
-THINKING_TIMEOUT = float(os.getenv("THINKING_TIMEOUT","300.0"))  # thinking mode gets longer budget
+THINKING_TIMEOUT = float(os.getenv("THINKING_TIMEOUT","600.0"))  # thinking mode — CPU needs time
 GENERATE_TIMEOUT = float(os.getenv("GENERATE_TIMEOUT","600.0"))  # file generation (big code files)
 QUEUE_TIMEOUT    = float(os.getenv("QUEUE_TIMEOUT",  "30.0"))
 ENRICH_TIMEOUT   = float(os.getenv("ENRICH_TIMEOUT", "12.0"))
@@ -1211,11 +1211,11 @@ async def chat(request: Request) -> Response | StreamingResponse:
         "chat_template_kwargs": {"enable_thinking": bool(thinking_requested)},
     }
     if thinking_requested:
-        _think_budget = int(body.get("thinking_budget", 4096))
+        _think_budget = int(body.get("thinking_budget", 512))  # 512 default — 4096 is too slow on CPU
         payload["reasoning_budget"] = _think_budget
         # CRITICAL: max_tokens must cover thinking budget + actual answer
         # Without this, the <think> block eats all tokens and content is empty
-        max_tok = max(max_tok, _think_budget + 512)
+        max_tok = max(max_tok, _think_budget + 256)
         payload["max_tokens"] = max_tok
     if body.get("tools"):
         payload["tools"]               = body["tools"]
@@ -1284,6 +1284,7 @@ async def chat(request: Request) -> Response | StreamingResponse:
             resp, deduped = await dedup.run_once(cache_key or rid, _do)
             if deduped: logger.info(f"[{rid}] DEDUP HIT")
         except asyncio.TimeoutError:
+            await llama._reset()  # close broken connection before next request uses it
             return E(f"Inference timeout {_req_timeout:.0f}s", 504, "timeout", rid)
         except httpx.RequestError as e:
             return E(f"Upstream error: {e}",502,"server_error",rid)
@@ -1387,16 +1388,21 @@ async def file_upload(
             "temperature":  0.6,
             "cache_prompt": False,
             "stream":       False,
+            # Always set explicitly — without this Qwen3 may silently enter thinking mode
+            "chat_template_kwargs": {"enable_thinking": thinking.lower() == "true"},
         }
         if thinking.lower() == "true":
-            pl["chat_template_kwargs"] = {"enable_thinking": True}
-            pl["reasoning_budget"]     = 4096
+            _think_budget = 512  # same conservative default as chat endpoint
+            pl["reasoning_budget"] = _think_budget
+            pl["max_tokens"] = max(MAX_NEW_TOKENS, _think_budget + 256)
 
         _file_timeout = THINKING_TIMEOUT if thinking.lower() == "true" else REQUEST_TIMEOUT
         resp = await asyncio.wait_for(
             llama.req("POST","/v1/chat/completions",
                       content=orjson.dumps(pl),
-                      headers={"Content-Type":"application/json"}),
+                      headers={"Content-Type":"application/json"},
+                      timeout=httpx.Timeout(connect=10.0, read=_file_timeout + 30.0,
+                                            write=10.0, pool=5.0)),
             timeout=_file_timeout)
 
         return Response(resp.content, resp.status_code,
@@ -1404,6 +1410,7 @@ async def file_upload(
                         headers={"X-File":fname,"X-Bytes":str(len(data))})
 
     except asyncio.TimeoutError:
+        await llama._reset()
         return E("File processing timed out", 504, "timeout")
     except Exception as e:
         logger.error(f"[files] {e}", exc_info=True)
@@ -1459,13 +1466,14 @@ async def generate_file(request: Request) -> Response:
         "temperature": 0.3,
         "cache_prompt":False,
         "stream":      False,
+        # CRITICAL: always set thinking explicitly — without this Qwen3 may enter thinking
+        # mode silently, consuming all tokens in <think> blocks and producing empty files
+        "chat_template_kwargs": {"enable_thinking": bool(thinking)},
     }
     if thinking:
-        pl["chat_template_kwargs"] = {"enable_thinking": True}
-        pl["reasoning_budget"]     = 4096
+        pl["reasoning_budget"] = 4096
 
     try:
-        # generate can produce long files — use GENERATE_TIMEOUT (600s default)
         resp = await asyncio.wait_for(
             llama.req("POST","/v1/chat/completions",
                       content=orjson.dumps(pl),
@@ -1489,6 +1497,14 @@ async def generate_file(request: Request) -> Response:
         file_content = "\n".join(lines).strip()
 
         tokens_used = data.get("usage",{}).get("completion_tokens","?")
+
+        # Guard: if content is empty the model likely entered thinking mode silently
+        if not file_content:
+            logger.warning(f"[generate] model returned empty content (tokens={tokens_used}) — "
+                           "possible silent thinking mode. Check enable_thinking=False is applied.")
+            return E("Model returned empty file content — the generation produced no output. "
+                     "Try rephrasing the prompt.", 500)
+
         return Response(
             content=file_content.encode("utf-8"),
             media_type="application/octet-stream",
@@ -1499,6 +1515,7 @@ async def generate_file(request: Request) -> Response:
                 "X-Lines":            str(len(file_content.splitlines())),
             })
     except asyncio.TimeoutError:
+        await llama._reset()  # prevent broken connection on next request
         return E("Generation timed out",504)
     except Exception as e:
         logger.error(f"[generate] {e}",exc_info=True)