zero-gpu: size the token budget to the measured decode rate

BUILD.json

@@ -1,6 +1,6 @@
 {
-  "built_at": "2026-10-29T17:07:28",
-  "commit": "1be6abe",
+  "built_at": "2026-10-29T17:14:25",
+  "commit": "b187ff2",
   "files": 53,
   "with_store": false,
   "with_card": true

traceforge/space/local_llm.py

@@ -255,9 +255,10 @@ def _fused_moe_forward(self, hidden_states, top_k_index, top_k_weights):
     transformers' Lfm2MoeExperts.forward loops over the experts that fired and
     calls .nonzero(), torch.where() and parameter indexing inside that loop.
     Every one of those is a device-to-host sync, so a single decoded token
-    costs ~6 syncs per expert per layer -- ~480 for this 40-layer top-2 model.
-    On a ZeroGPU container, whose CPU share is small, that is the decode
-    ceiling, not the GPU: 16 tok/s measured on a full RTX PRO 6000 Blackwell.
+    costs ~6 syncs per expert per layer -- ~960 for this 40-layer top-4 model
+    (64 experts, 4 per token).  At the ~60 us a sync costs on a ZeroGPU
+    container that is 58 ms per token, i.e. the 16 tok/s we measured on a full
+    RTX PRO 6000 Blackwell, where the weights alone allow hundreds.
 
     For a handful of tokens we gather the selected experts' weights and run two
     batched GEMMs instead, which needs no sync at all.  Prefill keeps the
@@ -386,6 +387,12 @@ def generate(messages: list[dict], *, max_tokens: int = 512,
     enc = tok(prompt, return_tensors="pt", truncation=True,
               max_length=LOCAL_MAX_INPUT_TOKENS).to(LOCAL_DEVICE)
     n = max(1, min(int(max_tokens or 512), LOCAL_MAX_NEW_TOKENS))
+    # Size the budget to what this slice can actually decode.  The first call
+    # measures tok/s; later calls trim the ceiling to ~60 s of generation so a
+    # slow slice fails fast instead of burning the visitor's whole quota.
+    measured = float(LAST_GEN.get("tok_s") or 0)
+    if measured > 0:
+        n = max(64, min(n, int(measured * 60)))
     t0 = time.time()
     with torch.inference_mode():
         out = model.generate(

traceforge/space/pipeline.py

@@ -88,7 +88,9 @@ def space_config(data_dir: Optional[str] = None, **overrides: Any) -> Config:
         cfg.triage_max_chars = cfg.splice_max_chars
         # Labels are compact JSON, but LFM2.5 always thinks first and ignores
         # a pre-closed think block: 600 tokens only ever bought reasoning.
-        cfg.triage_max_tokens = _env_int("TRACEFORGE_LOCAL_TRIAGE_TOKENS", 1200)
+        # The ceiling is generous because generate() trims it to what the
+        # measured decode rate can actually deliver inside one slice.
+        cfg.triage_max_tokens = _env_int("TRACEFORGE_LOCAL_TRIAGE_TOKENS", 4000)
         cfg.rewrite_max_tokens = _env_int("TRACEFORGE_LOCAL_REWRITE_TOKENS",
                                           1000)
         cfg.judge_max_tokens = _env_int("TRACEFORGE_LOCAL_JUDGE_TOKENS", 1200)