Unlimited output (max_tokens=-1); raise default N_CTX to 32768

README.md

@@ -22,7 +22,8 @@ On startup it:
 
 - **Tiny + fast:** 0.8B on CPU-basic is quick; great for low-latency chat.
 - **Chain-of-thought:** `<think>` shown in the collapsible panel; the answer stays clean.
-- **Context:** 8,192 tokens by default (model native up to 262,144 — raise `N_CTX` if needed).
+- **Context:** 32,768 tokens by default (model native up to **262,144** — raise `N_CTX`, slower on CPU).
+- **Output length:** unlimited by default (`MAX_TOKENS=-1`) — generates until done or context is full.
 
 Tunable via Space variables (no rebuild): `MODEL_REPO`, `GGUF_FILE` (e.g. `...Q5_K_M.gguf`,
 `...Q8_0.gguf`), `N_CTX`, `MAX_TOKENS`, `NUM_THREADS`, `SYSTEM_PROMPT`, `LLAMA_EXTRA_ARGS`.

app.py

@@ -22,9 +22,10 @@ BIN         = "/opt/llamabin"
 # ALWAYS opens a <think> block; this lets "Fast (no thinking)" actually skip reasoning).
 CHAT_TEMPLATE_FILE = os.environ.get("CHAT_TEMPLATE_FILE", "/home/user/app/chat_template.jinja")
 LLAMA       = f"http://127.0.0.1:{LLAMA_PORT}"
-# Qwen3.5-0.8B native context = 262144. On CPU-basic we keep a smaller window for fast
-# startup + low memory; raise N_CTX via a Space variable if you need long context.
-N_CTX       = os.environ.get("N_CTX", "8192")
+# Qwen3.5-0.8B native context = 262144. We default to a generous 32768 window (good for
+# large files / long chats) while keeping startup + memory reasonable on CPU; raise N_CTX up
+# to 262144 via a Space variable if you really need it (much slower prefill on CPU).
+N_CTX       = os.environ.get("N_CTX", "32768")
 
 
 def _env():
@@ -88,8 +89,9 @@ app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_credentials=False,
                    allow_methods=["*"], allow_headers=["*"])
 
 # Sensible defaults (only applied when the caller doesn't set them). Overridable per request.
+# max_tokens = -1 -> UNLIMITED output (generate until EOS or the context window is full).
 DEFAULTS = {"temperature": 0.3, "top_p": 0.9, "top_k": 20, "repeat_penalty": 1.05,
-            "max_tokens": int(os.environ.get("MAX_TOKENS", "1024"))}
+            "max_tokens": int(os.environ.get("MAX_TOKENS", "-1"))}
 
 
 @app.get("/health")