CPU fallback when ZeroGPU quota exhausted; GPU/CPU chip in topbar

app.py

@@ -166,13 +166,53 @@ def world_snapshot() -> dict:
     return result
 
 
-@app.api(name="tick_decisions")
 @GPU(duration=90)
+def _drain_gpu() -> int:
+    """The hot path: drain inside a ZeroGPU fork. RealLLMBackend loads each
+    fork's Llama with n_gpu_layers=-1.
+    """
+    return TOWNLET.drain_decisions()
+
+
+def _drain_cpu() -> int:
+    """Quota-exhausted fallback: drain in the parent process with
+    n_gpu_layers=0. ZeroGPU isn't touched, so this works even when the
+    GPU allocator is empty. ~1-3 tokens/sec on 2 vCPU — slow but the sim
+    keeps running. Llama is cached in the parent across drains, so only
+    the first call pays the model load.
+    """
+    import os
+    os.environ["TOWNLET_FORCE_CPU"] = "1"
+    try:
+        return TOWNLET.drain_decisions()
+    finally:
+        # Keep the env var set for the rest of the process — once we've
+        # paid the CPU load cost we want all subsequent drains to reuse
+        # the same cached Llama. The next GPU attempt is still cheap to
+        # try (~1s for the fork to fail), so the wrapper's try/except
+        # remains the source of truth on which mode each call uses.
+        pass
+
+
+@app.api(name="tick_decisions")
 def tick_decisions_api() -> dict:
-    """Drain pending agent decisions inside the GPU boundary."""
+    """Try ZeroGPU first; on quota-exhaustion (or any GPU init error) fall
+    back to a CPU drain in the parent process. The sim never stalls; it
+    just runs slower when ZeroGPU is empty.
+    """
     _log_req("tick_decisions")
-    n = TOWNLET.drain_decisions()
-    result = {"decisions_run": n}
+    try:
+        n = _drain_gpu()
+        mode = "gpu"
+    except RuntimeError as e:
+        msg = str(e)
+        if "No CUDA GPUs" in msg or "quota" in msg.lower() or "ZeroGPU" in msg:
+            print(f"[townlet] GPU unavailable ({msg[:80]}); CPU fallback engaged", flush=True)
+            n = _drain_cpu()
+            mode = "cpu"
+        else:
+            raise
+    result = {"decisions_run": n, "mode": mode}
     _log_resp("tick_decisions", result)
     return result
 

backend/factory.py

@@ -14,6 +14,8 @@ as a placeholder if the endpoint ever gets called.
 
 from __future__ import annotations
 
+import os
+
 from game.models import DEFAULT_MODEL_ID, get_spec
 
 from . import config
@@ -23,18 +25,29 @@ _llms: dict[str, LLMBackend] = {}
 _image: ImageBackend | None = None
 
 
+def _cache_key(model_id: str) -> str:
+    """GPU and CPU Llama instances are loaded with different `n_gpu_layers`
+    and can't be reused across modes. Cache them under separate keys so the
+    CPU-fallback path (parent process, TOWNLET_FORCE_CPU=1) and the GPU
+    path (forked workers, no env var) don't collide.
+    """
+    mode = "cpu" if os.getenv("TOWNLET_FORCE_CPU") == "1" else "gpu"
+    return f"{model_id}::{mode}"
+
+
 def get_llm(model_id: str = DEFAULT_MODEL_ID) -> LLMBackend:
-    if model_id not in _llms:
+    key = _cache_key(model_id)
+    if key not in _llms:
         get_spec(model_id)  # validates id
         if config.USE_REAL_MODELS:
             from .real import RealLLMBackend
 
-            _llms[model_id] = RealLLMBackend(model_id)
+            _llms[key] = RealLLMBackend(model_id)
         else:
             from .mock import MockLLMBackend
 
-            _llms[model_id] = MockLLMBackend(model_id)
-    return _llms[model_id]
+            _llms[key] = MockLLMBackend(model_id)
+    return _llms[key]
 
 
 def get_image() -> ImageBackend:

backend/real.py

@@ -17,6 +17,7 @@ Model card references (per CLAUDE.md vendor-citation rule):
 
 from __future__ import annotations
 
+import os
 from typing import TYPE_CHECKING
 
 from game.models import get_spec
@@ -27,6 +28,13 @@ if TYPE_CHECKING:
     from llama_cpp import Llama
 
 
+def force_cpu_mode() -> bool:
+    """When set, RealLLMBackend loads with `n_gpu_layers=0` and reduced
+    context — usable on the Space's 2 vCPU when ZeroGPU quota is gone.
+    Toggled by app.py's tick_decisions fallback path."""
+    return os.getenv("TOWNLET_FORCE_CPU") == "1"
+
+
 class RealLLMBackend:
     """A small GGUF LLM via llama-cpp-python, keyed by `model_id` in the roster.
 
@@ -54,19 +62,39 @@ class RealLLMBackend:
             # n_ctx: smolagents' CodeAgent system prompt is large (tool defs
             # + examples). Combined with our composite perception prompt this
             # blows past 4096 tokens. Nemotron 3 4B trained at 1M ctx;
-            # 16384 is safe headroom on Spaces; smaller locally.
+            # 16384 is safe headroom on Spaces; smaller locally / on CPU.
             # n_gpu_layers: -1 (all on GPU) is what Zero-GPU needs. On a
             # Mac (Intel especially) the Metal integrated GPU times out on
             # batches this size — kIOAccelCommandBufferCallbackErrorTimeout
             # surfaces as `llama_decode -3`. Fall back to CPU there.
+            # When TOWNLET_FORCE_CPU=1 (the quota-exhausted fallback path
+            # set by app.py's tick_decisions wrapper) we force 0 GPU
+            # layers and shrink the context — slower per token but the
+            # sim keeps running without the GPU.
             on_spaces = config.IS_SPACES
+            cpu_only = force_cpu_mode()
+            if cpu_only:
+                gpu_layers = 0
+                n_ctx = 4096
+                n_batch = 256
+                use_flash = False
+            elif on_spaces:
+                gpu_layers = -1
+                n_ctx = 16384
+                n_batch = 16384
+                use_flash = True
+            else:
+                gpu_layers = 0
+                n_ctx = 8192
+                n_batch = 512
+                use_flash = False
             self._llm = Llama.from_pretrained(
                 repo_id=self._spec.repo_id,
                 filename=self._spec.gguf_filename,
-                n_gpu_layers=-1 if on_spaces else 0,
-                n_ctx=16384 if on_spaces else 8192,
-                n_batch=16384 if on_spaces else 512,
-                flash_attn=on_spaces,
+                n_gpu_layers=gpu_layers,
+                n_ctx=n_ctx,
+                n_batch=n_batch,
+                flash_attn=use_flash,
                 verbose=False,
             )
         return self._llm

frontend/index.html

@@ -146,6 +146,16 @@
     .tl-sent-negative .tl-sent-pip { background: var(--red); }
     .tl-sent-positive { color: var(--green); }
     .tl-sent-positive .tl-sent-pip { background: var(--green); }
+    .tl-mode {
+      display: inline-flex; align-items: center; gap: 6px;
+      padding: 2px 9px; border-radius: 8px;
+      border: 1.5px solid currentColor;
+      font-size: 11px; letter-spacing: 0.08em; text-transform: uppercase;
+    }
+    .tl-mode-pip { width: 7px; height: 7px; border-radius: 50%; background: currentColor; }
+    .tl-mode-unknown { color: var(--ink-soft); }
+    .tl-mode-gpu     { color: var(--green); }
+    .tl-mode-cpu     { color: var(--red); }
 
     /* ====== middle band ====== */
     .tl-mid { flex: 1; display: flex; gap: 14px; padding: 14px; min-height: 0; }
@@ -701,6 +711,10 @@
         <span class="tl-sentiment tl-sent-neutral" id="sent-chip">
           <span class="tl-sent-pip"></span><span id="sent-text">neutral</span>
         </span>
+        <span class="tl-meta-dot">·</span>
+        <span class="tl-mode tl-mode-unknown" id="mode-chip" title="Which compute the last tick used. CPU appears when ZeroGPU quota is empty — the sim keeps running, just slower.">
+          <span class="tl-mode-pip"></span><span id="mode-text">—</span>
+        </span>
       </div>
     </div>
 
@@ -1223,6 +1237,24 @@
     const elInfoModal = document.getElementById("info-modal");
     const elInfoClose = document.getElementById("info-close");
     const elInfoDotText = document.getElementById("info-dot-text");
+    const elModeChip = document.getElementById("mode-chip");
+    const elModeText = document.getElementById("mode-text");
+
+    function setModeChip(mode) {
+      // mode is "gpu", "cpu", or null/undefined for unknown.
+      if (!elModeChip || !elModeText) return;
+      elModeChip.classList.remove("tl-mode-unknown", "tl-mode-gpu", "tl-mode-cpu");
+      if (mode === "gpu") {
+        elModeChip.classList.add("tl-mode-gpu");
+        elModeText.textContent = "GPU";
+      } else if (mode === "cpu") {
+        elModeChip.classList.add("tl-mode-cpu");
+        elModeText.textContent = "CPU";
+      } else {
+        elModeChip.classList.add("tl-mode-unknown");
+        elModeText.textContent = "—";
+      }
+    }
     const elReset = document.getElementById("reset-btn");
     const elPanelEmpty = document.getElementById("panel-empty");
     const elPanelBody = document.getElementById("panel-body");
@@ -1873,12 +1905,26 @@
       // previous to resolve, which clamps the effective cadence to the
       // call duration itself.
       let tickInFlight = false;
+      let lastMode = null;
       setInterval(async () => {
         if (tickInFlight) return;
         tickInFlight = true;
-        try { await client.predict("/tick_decisions", {}); }
-        catch (e) { console.warn("tick_decisions failed", e); }
-        finally { tickInFlight = false; }
+        try {
+          const res = await client.predict("/tick_decisions", {});
+          const payload = res?.data?.[0];
+          const mode = payload?.mode;
+          if (mode) {
+            setModeChip(mode);
+            if (mode !== lastMode) {
+              console.log("[townlet] tick mode:", mode, mode === "cpu" ? "(ZeroGPU quota empty — CPU fallback active)" : "");
+              lastMode = mode;
+            }
+          }
+        } catch (e) {
+          console.warn("tick_decisions failed", e);
+        } finally {
+          tickInFlight = false;
+        }
       }, 5000);
       refresh();
     }