Expand live context selection and sync backend source

data/benchmark_bundle/backend_truth_source.json

@@ -0,0 +1,218 @@
+{
+  "qwen35_27b_hf": {
+    "1024": {
+      "output_match": {
+        "learned_selector": true,
+        "shortlist_base": true
+      },
+      "profiles": {
+        "exact": {
+          "latency_ms": 485.96312350127846,
+          "m3_fraction": 0.0,
+          "resident_bytes": 38670336,
+          "selector_us": 0.0,
+          "source_records": 7,
+          "text": " matters for fast decoding.\n\n<think>\n",
+          "token_count": 8
+        },
+        "learned_selector": {
+          "latency_ms": 149.39251463511027,
+          "m3_fraction": 0.99462890625,
+          "resident_bytes": 99929088,
+          "selector_us": 24.829375597335,
+          "source_records": 7,
+          "text": " matters for fast decoding.\n\n<think>\n",
+          "token_count": 8
+        },
+        "shortlist_base": {
+          "latency_ms": 504.25705371890217,
+          "m3_fraction": 0.0,
+          "resident_bytes": 38670336,
+          "selector_us": 0.0,
+          "source_records": 7,
+          "text": " matters for fast decoding.\n\n<think>\n",
+          "token_count": 8
+        }
+      }
+    },
+    "2048": {
+      "output_match": {
+        "learned_selector": true,
+        "shortlist_base": true
+      },
+      "profiles": {
+        "exact": {
+          "latency_ms": 821.4117659954354,
+          "m3_fraction": 0.0,
+          "resident_bytes": 76288000,
+          "selector_us": 0.0,
+          "source_records": 7,
+          "text": " fast decoding.\n\n<think>\nThinking Process",
+          "token_count": 8
+        },
+        "learned_selector": {
+          "latency_ms": 236.0101520025637,
+          "m3_fraction": 0.9954833984375,
+          "resident_bytes": 189485056,
+          "selector_us": 24.959413796718113,
+          "source_records": 7,
+          "text": " fast decoding.\n\n<think>\nThinking Process",
+          "token_count": 8
+        },
+        "shortlist_base": {
+          "latency_ms": 626.3046781823505,
+          "m3_fraction": 0.0,
+          "resident_bytes": 75898880,
+          "selector_us": 0.0,
+          "source_records": 7,
+          "text": " fast decoding.\n\n<think>\nThinking Process",
+          "token_count": 8
+        }
+      }
+    }
+  },
+  "qwen35_4b_hf": {
+    "1024": {
+      "output_match": {
+        "learned_selector": true,
+        "shortlist_base": true
+      },
+      "profiles": {
+        "exact": {
+          "latency_ms": 232.18651674687862,
+          "m3_fraction": 0.0,
+          "resident_bytes": 19337216,
+          "selector_us": 0.0,
+          "source_records": 7,
+          "text": " matters for fast decoding.\n\nCache locality",
+          "token_count": 8
+        },
+        "learned_selector": {
+          "latency_ms": 71.47038698894903,
+          "m3_fraction": 0.982421875,
+          "resident_bytes": 50163712,
+          "selector_us": 25.20372302683427,
+          "source_records": 7,
+          "text": " matters for fast decoding.\n\nCache locality",
+          "token_count": 8
+        },
+        "shortlist_base": {
+          "latency_ms": 242.41085114772432,
+          "m3_fraction": 0.0,
+          "resident_bytes": 19337216,
+          "selector_us": 0.0,
+          "source_records": 7,
+          "text": " matters for fast decoding.\n\nCache locality",
+          "token_count": 8
+        }
+      }
+    },
+    "2048": {
+      "output_match": {
+        "learned_selector": true,
+        "shortlist_base": true
+      },
+      "profiles": {
+        "exact": {
+          "latency_ms": 400.1560862525366,
+          "m3_fraction": 0.0,
+          "resident_bytes": 38146048,
+          "selector_us": 0.0,
+          "source_records": 7,
+          "text": " fast decoding.Cache locality matters for fast",
+          "token_count": 8
+        },
+        "learned_selector": {
+          "latency_ms": 122.57132897502743,
+          "m3_fraction": 0.9710693359375,
+          "resident_bytes": 99615744,
+          "selector_us": 25.1929690234322,
+          "source_records": 7,
+          "text": " fast decoding.Cache locality matters for fast",
+          "token_count": 8
+        },
+        "shortlist_base": {
+          "latency_ms": 300.46495210262947,
+          "m3_fraction": 0.0,
+          "resident_bytes": 37134336,
+          "selector_us": 0.0,
+          "source_records": 7,
+          "text": " fast decoding.Cache locality matters for fast",
+          "token_count": 8
+        }
+      }
+    }
+  },
+  "qwen35_9b_hf": {
+    "1024": {
+      "output_match": {
+        "learned_selector": true,
+        "shortlist_base": true
+      },
+      "profiles": {
+        "exact": {
+          "latency_ms": 242.51539027318358,
+          "m3_fraction": 0.0,
+          "resident_bytes": 19337216,
+          "selector_us": 0.0,
+          "source_records": 7,
+          "text": " matters for fast decoding.Cache locality matters",
+          "token_count": 8
+        },
+        "learned_selector": {
+          "latency_ms": 74.78160472237505,
+          "m3_fraction": 0.98779296875,
+          "resident_bytes": 50642944,
+          "selector_us": 25.876024622562,
+          "source_records": 7,
+          "text": " matters for fast decoding.Cache locality matters",
+          "token_count": 8
+        },
+        "shortlist_base": {
+          "latency_ms": 242.637697578175,
+          "m3_fraction": 0.0,
+          "resident_bytes": 19337216,
+          "selector_us": 0.0,
+          "source_records": 7,
+          "text": " matters for fast decoding.Cache locality matters",
+          "token_count": 8
+        }
+      }
+    },
+    "2048": {
+      "output_match": {
+        "learned_selector": true,
+        "shortlist_base": true
+      },
+      "profiles": {
+        "exact": {
+          "latency_ms": 404.7011856455356,
+          "m3_fraction": 0.0,
+          "resident_bytes": 38146048,
+          "selector_us": 0.0,
+          "source_records": 7,
+          "text": " fast decoding.Cache locality matters for fast",
+          "token_count": 8
+        },
+        "learned_selector": {
+          "latency_ms": 105.71580115356483,
+          "m3_fraction": 0.99853515625,
+          "resident_bytes": 101265408,
+          "selector_us": 25.52354613629047,
+          "source_records": 7,
+          "text": " fast decoding.Cache locality matters for fast",
+          "token_count": 8
+        },
+        "shortlist_base": {
+          "latency_ms": 307.73281096480787,
+          "m3_fraction": 0.0,
+          "resident_bytes": 37134336,
+          "selector_us": 0.0,
+          "source_records": 7,
+          "text": " fast decoding.Cache locality matters for fast",
+          "token_count": 8
+        }
+      }
+    }
+  }
+}

engines/fixture_builder.py

@@ -9,11 +9,12 @@ from typing import Any, Mapping
 from engines.compare import build_summary_sentence
 from engines.presets import MODEL_BY_KEY, PRESET_BY_KEY
 
-FIXTURE_VERSION = "v6"
+FIXTURE_VERSION = "v7"
 
 REPO_ROOT = Path(__file__).resolve().parents[1]
 BUNDLE_ROOT = REPO_ROOT / "data" / "benchmark_bundle"
 COMPACT_BUNDLE_PATH = BUNDLE_ROOT / "space_benchmark_bundle.json"
+BACKEND_TRUTH_BUNDLE_PATH = BUNDLE_ROOT / "backend_truth_source.json"
 
 MODEL_DIR_BY_KEY = {
     "qwen35_4b_hf": "qwen35_4b",
@@ -85,6 +86,13 @@ def _space_bundle() -> dict[str, Any]:
     return json.loads(COMPACT_BUNDLE_PATH.read_text(encoding="utf-8"))
 
 
+@lru_cache(maxsize=None)
+def _backend_truth_bundle() -> dict[str, Any]:
+    if BACKEND_TRUTH_BUNDLE_PATH.exists():
+        return json.loads(BACKEND_TRUTH_BUNDLE_PATH.read_text(encoding="utf-8"))
+    return _space_bundle().get("backend_truth", {})
+
+
 def _clean_text(value: str) -> str:
     cleaned = str(value or "").replace("\r\n", "\n")
     cleaned = re.sub(r"(?is)<think>.*?</think>", " ", cleaned)
@@ -293,7 +301,7 @@ def _longbench_result(request: Mapping[str, Any], *, model_key: str, context_len
 
 
 def _backend_truth_result(request: Mapping[str, Any], *, model_key: str, context_length: int, mode: str) -> dict[str, Any]:
-    row_bundle = _space_bundle()["backend_truth"].get(model_key, {}).get(str(context_length))
+    row_bundle = _backend_truth_bundle().get(model_key, {}).get(str(context_length))
     if row_bundle is None:
         raise ValueError(f"No backend-truth benchmark row is bundled for model={model_key} at {context_length} tokens.")
     candidate_variant = BACKEND_MODE_TO_VARIANT[mode]
@@ -352,7 +360,8 @@ def _backend_truth_result(request: Mapping[str, Any], *, model_key: str, context
         "paper_subtitle": f"Exact-length serving benchmark at {context_length} prompt tokens.",
         "paper_summary": (
             f"Decode latency moves from {baseline_latency:.1f} ms/token to {candidate_latency:.1f} ms/token "
-            f"with resident KV {baseline_kv_bytes / (1024 ** 2):.1f} MiB versus {candidate_kv_bytes / (1024 ** 2):.1f} MiB."
+            f"with resident KV {baseline_kv_bytes / (1024 ** 2):.1f} MiB versus {candidate_kv_bytes / (1024 ** 2):.1f} MiB. "
+            f"The recorded {exact_token_count}-token decode sample is identical across exact, shortlist, and learned rows on this benchmark."
         ),
         "paper_metric_badge": (
             f"M3 fraction {float(candidate_stats.get('m3_fraction') or 0.0):.3f}, "

space_app.py

@@ -1131,6 +1131,12 @@ def _live_context_guard_copy(mode: str) -> str:
             "<strong>⚡ Live mode (ZeroGPU)</strong><span>Your prompt runs on the real Space system using the selected preset's benchmark configuration. When the runtime allows it, custom live prompts and example buttons can use the 4K context lane to sanity-check that behavior stays in the same ballpark as the cached paper row.</span>"
             "</div>"
         )
+    if mode == "live":
+        return (
+            "<div class='live-context-note active'>"
+            "<strong>⚡ Live mode (ZeroGPU)</strong><span>Live mode exposes the full runtime context ladder up to the current Space cap. Empty-prompt replay is still only bundled for the benchmark rows, so use a custom prompt or example button when you want to run the 4K live lane.</span>"
+            "</div>"
+        )
     if mode == "benchmark":
         return (
             "<div class='live-context-note active'>"
@@ -1170,23 +1176,19 @@ def _preferred_live_example_context(current_context_length: int) -> int:
 
 def _guard_live_context(prompt_mode: str, custom_prompt: str, current_context_length: int) -> tuple[Any, str]:
     if _is_live_prompt_mode(prompt_mode):
+        available_choices = _available_live_context_choices()
+        guarded_value = int(current_context_length)
+        if guarded_value not in available_choices:
+            guarded_value = available_choices[min(len(available_choices) - 1, 1)]
         has_custom_prompt = bool(custom_prompt.strip())
         if has_custom_prompt:
-            available_choices = _available_live_context_choices()
-            guarded_value = int(current_context_length)
-            if guarded_value not in available_choices:
-                guarded_value = available_choices[min(len(available_choices) - 1, 1)]
             return (
                 gr.update(choices=available_choices, value=guarded_value),
                 _live_context_guard_copy("custom"),
             )
-        benchmark_choices = _available_live_benchmark_context_choices()
-        guarded_value = int(current_context_length)
-        if guarded_value not in benchmark_choices:
-            guarded_value = int(benchmark_choices[-1])
         return (
-            gr.update(choices=benchmark_choices, value=guarded_value),
-            _live_context_guard_copy("benchmark"),
+            gr.update(choices=available_choices, value=guarded_value),
+            _live_context_guard_copy("live"),
         )
     has_custom_prompt = bool(custom_prompt.strip())
     restored_value = int(current_context_length)