Initial release: Gemma 4 E2B INT4 .pte for Pi 5 via ExecuTorch

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+gemma4_e2b_text_int4_extcache.pte filter=lfs diff=lfs merge=lfs -text
+tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,3 +1,138 @@
 ---
 license: apache-2.0
+language:
+  - en
+base_model: google/gemma-4-e2b-it
+tags:
+  - executorch
+  - quantized
+  - int4
+  - raspberry-pi
+  - on-device
+  - edge
+pipeline_tag: text-generation
 ---
+
+# Gemma 4 E2B — INT4 ExecuTorch `.pte` for Raspberry Pi 5
+
+INT4-quantized, ExecuTorch-lowered `.pte` of [`google/gemma-4-e2b-it`](https://huggingface.co/google/gemma-4-e2b-it), packaged for **Raspberry Pi 5 (Cortex-A76, 8 GB)** deployment via the [ExecuTorch](https://pytorch.org/executorch) 1.2.0 Python runtime with the XNNPACK backend.
+
+This artifact is the deployable output of the full export → quantize → lower → runtime recipe documented at:
+
+**Source code & full recipe:** https://github.com/bamb00boy/Gemma4_executorch_deployment
+
+## Contents
+
+| File | Size | Purpose |
+|---|---|---|
+| `gemma4_e2b_text_int4_extcache.pte` | 5.14 GB | The ExecuTorch program — load + run with `executorch==1.2.0` |
+| `tokenizer/tokenizer.json` | ~5 MB | HF fast tokenizer for Gemma 4 |
+| `tokenizer/tokenizer_config.json` | small | Tokenizer config (special tokens, chat template ref) |
+| `tokenizer/chat_template.jinja` | small | Gemma 4 chat template (used by `gemma4_terminal_chat.py`) |
+| `pi_runner.py` | ~250 lines | Self-contained one-shot runner: tokenize → generate → exit |
+| `gemma4_terminal_chat.py` | ~325 lines | Interactive multi-turn REPL with KV-cache reuse across turns |
+| `LICENSE` | — | Apache 2.0 (covers the weights; see [License](#license) below) |
+
+## Measured performance
+
+Identical 14-token prompt + 9-token decode for `"The capital of France is"`, bit-exact output across all rows.
+
+| Host | Role | Prompt feed | Decode | Total wall |
+|---|---|---|---|---|
+| **Raspberry Pi 5** — 8 GB, Cortex-A76 @ 2.4 GHz, Ubuntu Server 24.04 LTS, microSD | deployment target | 0.77 tok/s | **0.87 tok/s** | 28.6 s |
+| **MacBook Pro 14"** — Apple M1 Pro (6P+2E), 16 GB unified, macOS 26.3.1 | development reference | 7.20 tok/s | **8.66 tok/s** | 2.99 s |
+| [potato-os/core llama.cpp on Pi 5](https://github.com/potato-os/core/blob/main/docs/benchmarks/gemma4-pi-benchmark-2026-04-04.md) | external reference (different runtime) | n/a | **6.71 tok/s** | n/a |
+
+**Output quality:** bit-exact 9/9 token match against the FP32 reference on the canonical prompt.
+
+The Pi 5 decode is approximately 7.7× slower than `llama.cpp` on identical hardware. The cause is fully attributable to a known ARM-side XNNPACK bug in ExecuTorch 1.2.0 that forces the `XnnpackPartitioner(per_op_mode=True)` workaround, which neutralizes the fused-subgraph path that KleidiAI's INT4 matmul fast-path depends on. Full diagnosis at [KNOWN_ISSUES.md #1](https://github.com/bamb00boy/Gemma4_executorch_deployment/blob/master/KNOWN_ISSUES.md) in the source repo. If maximum Pi 5 decode throughput is the priority, `llama.cpp` is the appropriate tool today.
+
+## Quick use on a Raspberry Pi 5
+
+```bash
+# 1. Download the bundle (~5.2 GB)
+pip install --user huggingface_hub
+hf download bamb00boy/gemma4-e2b-int4-executorch-pi5 --local-dir ~/gemma4
+
+# 2. Set up the runtime environment
+cd ~/gemma4
+python3 -m venv .venv && source .venv/bin/activate
+pip install --upgrade pip
+pip install torch==2.11.0 executorch==1.2.0 transformers==5.5.3
+
+# 3. Verify (should print "RESULT: PASS" and "The capital of France is **Paris**.")
+python pi_runner.py --verify
+
+# 4a. One-shot generation
+python pi_runner.py "Your prompt here" --max-new-tokens 50
+
+# 4b. Or an interactive multi-turn chat (KV-cache reused across turns)
+python gemma4_terminal_chat.py
+# Type a message + Enter. /help for commands. Ctrl+C or Ctrl+D to exit.
+```
+
+The Pi setup guide (OS install, performance tuning, SSH) lives in [docs/pi5_setup.md](https://github.com/bamb00boy/Gemma4_executorch_deployment/blob/master/docs/pi5_setup.md) in the source repo.
+
+## Use on other hosts
+
+The `.pte` runs on any host with ExecuTorch 1.2.0 + XNNPACK. It has been validated on:
+
+- aarch64 Linux (Raspberry Pi 5, Ubuntu Server 24.04)
+- macOS arm64 (Apple Silicon, used as the development reference)
+
+x86_64 Linux is expected to work (XNNPACK supports it) but is untested by this project.
+
+## What's quantized, what's not
+
+| Component | Treatment |
+|---|---|
+| `nn.Linear` weights (~3.1 B params) | INT4 weight-only via torchao's `Int8DynamicActivationIntxWeightConfig` (stored unpacked as INT8 bytes on disk) |
+| `embed_tokens_per_layer` (~2.35 B params, the "E2B" trick) | INT8 per-row via a hand-rolled `Int8Embedding` module (see source repo's `scripts/_int8_embedding.py`) |
+| `embed_tokens` (~0.4 B params) | FP32 — Gemma 4's model code performs direct weight slicing, which is incompatible with quantized tensor wrappers |
+| Layer norms, RoPE buffers, biases | FP32 |
+| Runtime K/V cache | FP32, externalized as program inputs/outputs (see source repo's `scripts/_external_cache.py`) |
+
+Disk size: 5.14 GB. Runtime cache footprint: 18.9 MB across 15 layers (12 sliding-window @ `head_dim=256`, 3 full-attention @ `head_dim=512`).
+
+## Architecture & shape constraints
+
+- **Sequence length:** padded to 511 tokens at runtime (the `.pte` shape-specializes to the upper bound of the dynamic dim).
+- **Decode:** token-by-token (no batched prefill in this build).
+- **Maximum total context:** 511 tokens (prompt + generated combined).
+- **Cache:** externalized — 90 cache tensors are passed as graph inputs and 45 are returned as graph outputs each call (one K + one V per layer × 15 layers + sentinel for prefill vs decode).
+
+## License
+
+The weights in this file are derived from [`google/gemma-4-e2b-it`](https://huggingface.co/google/gemma-4-e2b-it) and are licensed under **Apache License 2.0** by Google DeepMind. Use is subject to:
+
+- [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) (text included in this repo as `LICENSE`)
+- [Gemma Prohibited Use Policy](https://ai.google.dev/gemma/prohibited_use_policy)
+- [Gemma 4 Apache 2.0 announcement](https://ai.google.dev/gemma/apache_2)
+
+This is a derivative work: INT4 weight-only quantization of `nn.Linear` weights and INT8 per-row quantization of `embed_tokens_per_layer`, followed by ExecuTorch program lowering with the XNNPACK backend. No additional fine-tuning has been performed.
+
+The packaging code (`pi_runner.py`, `gemma4_terminal_chat.py`, and the export/quantize/lower pipeline) is released under **MIT** — see the [source GitHub repo](https://github.com/bamb00boy/Gemma4_executorch_deployment).
+
+## Attribution
+
+```
+Original Gemma 4 weights © Google DeepMind, released under Apache 2.0.
+INT4 quantization + ExecuTorch lowering: derivative work by the
+Gemma4_executorch_deployment contributors (https://github.com/bamb00boy/Gemma4_executorch_deployment).
+```
+
+## Citation
+
+If this artifact is useful in research, please cite both the original Gemma 4 release and this packaging:
+
+```bibtex
+@misc{gemma4-e2b-int4-executorch-pi5,
+  title  = {Gemma 4 E2B INT4 ExecuTorch for Raspberry Pi 5},
+  author = {bamb00boy and Gemma4_executorch_deployment contributors},
+  year   = {2026},
+  url    = {https://huggingface.co/bamb00boy/gemma4-e2b-int4-executorch-pi5},
+  note   = {Source recipe: https://github.com/bamb00boy/Gemma4_executorch_deployment}
+}
+```
+
+For the upstream Gemma 4 model, see [`google/gemma-4-e2b-it`](https://huggingface.co/google/gemma-4-e2b-it).

gemma4_e2b_text_int4_extcache.pte

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb5b414c8dda8624a36be998dc3e9816c798bb953f160deee70889867c230543
+size 5139050496

gemma4_terminal_chat.py

@@ -0,0 +1,325 @@
+"""
+Gemma 4 E2B — interactive terminal chat.
+
+Self-contained chat REPL that runs the same .pte as pi_runner.py but in
+multi-turn mode. KV-cache is reused across turns — only the *new* tokens
+in each chat round are fed to the model, so latency stays bounded by the
+size of each new user message + the response, not by the cumulative
+conversation length (until cache fills).
+
+Dependencies (install once on the deployment host):
+    pip install torch==2.11.0 executorch==1.2.0 transformers==5.5.3
+
+Expected files in the same directory (or pass --pte / --tokenizer paths):
+    gemma4_e2b_text_int4_extcache.pte
+    tokenizer/                          # tokenizer.json + tokenizer_config.json + chat_template.jinja
+
+Usage:
+    python gemma4_terminal_chat.py
+    python gemma4_terminal_chat.py --max-new-tokens 200
+    python gemma4_terminal_chat.py --pte /path/to/.pte --tokenizer /path/to/tokenizer/
+
+Controls:
+    Type a message + Enter        → model replies
+    Ctrl+C  or  Ctrl+D            → exit
+    /reset                         → wipe cache + history, start fresh
+    /help                          → show controls
+"""
+
+import argparse
+import os
+import signal
+import sys
+import time
+
+import torch
+from transformers import AutoTokenizer
+from executorch.runtime import Runtime, Verification
+
+# -------------------- paths + model layout (must match pi_runner.py) --------------------
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+DEFAULT_PTE = os.path.join(HERE, "gemma4_e2b_text_int4_extcache.pte")
+DEFAULT_TOK = os.path.join(HERE, "tokenizer")
+
+MAX_CACHE_LEN = 512
+MASK_LEN = MAX_CACHE_LEN - 1            # 511; .pte specialized to this upper bound
+DTYPE = torch.float32
+
+# Same hardcoded layout as pi_runner.py — must match what 04_quantize.py exported.
+GEMMA4_E2B_LAYER_SHAPES = [
+    # (head_dim, is_sliding)
+    (256, True),  (256, True),  (256, True),  (256, True),  (512, False),   # 0..4
+    (256, True),  (256, True),  (256, True),  (256, True),  (512, False),   # 5..9
+    (256, True),  (256, True),  (256, True),  (256, True),  (512, False),   # 10..14
+]
+NUM_KV_HEADS = 1
+BATCH = 1
+
+# Gemma 4 end-of-turn / EOS tokens — stop generation when we see any of these
+EOS_TOKEN_IDS = {106, 1, 2}  # <end_of_turn>, <eos>, <bos>-as-sentinel
+
+
+# -------------------- ExecuTorch-side helpers --------------------
+
+def allocate_cache_tensors():
+    """One set of K, V, cumulative_length tensors per cache layer (zero-filled)."""
+    k_caches, v_caches, cumlen_caches = [], [], []
+    for head_dim, _is_sliding in GEMMA4_E2B_LAYER_SHAPES:
+        shape = (BATCH, NUM_KV_HEADS, MAX_CACHE_LEN, head_dim)
+        k_caches.append(torch.zeros(shape, dtype=DTYPE))
+        v_caches.append(torch.zeros(shape, dtype=DTYPE))
+        cumlen_caches.append(torch.zeros(1, dtype=torch.int64))
+    return k_caches, v_caches, cumlen_caches
+
+
+def step(method, token_id, pos, k_caches, v_caches, cumlen_caches):
+    """One forward call: feed `token_id` at position `pos`, return (logits, updated caches)."""
+    input_ids = torch.tensor([[token_id]], dtype=torch.long)
+    attention_mask = torch.zeros(1, MASK_LEN, dtype=torch.long)
+    attention_mask[:, :pos + 1] = 1
+    position_ids = torch.tensor([[pos]], dtype=torch.long)
+    cache_position = torch.tensor([pos], dtype=torch.long)
+
+    args = (input_ids, attention_mask, position_ids, cache_position,
+            *k_caches, *v_caches, *cumlen_caches)
+    outputs = method.execute(args)
+
+    # See pi_runner.py for the .pte's 91-output layout. We use indices [46..90].
+    n = len(GEMMA4_E2B_LAYER_SHAPES)
+    logits = outputs[45]
+    base = 46
+    k_new = list(outputs[base:base + n])
+    v_new = list(outputs[base + n:base + 2 * n])
+    cumlen_new = list(outputs[base + 2 * n:base + 3 * n])
+    return logits, k_new, v_new, cumlen_new
+
+
+# -------------------- chat session state --------------------
+
+class ChatSession:
+    """Tracks conversation history + the tokens already fed to the .pte cache.
+
+    Key trick: every turn, we re-render the full conversation via the chat
+    template, find the longest common prefix with `fed_ids` (what's already
+    in the cache), and only feed the new tail. Avoids paying token-by-token
+    cost for the same context twice.
+    """
+
+    def __init__(self, tokenizer, method, max_new_tokens):
+        self.tokenizer = tokenizer
+        self.method = method
+        self.max_new_tokens = max_new_tokens
+        self.history = []           # list of {"role": "user"/"model", "content": "..."}
+        self.fed_ids = []           # tokens already in cache
+        self.k, self.v, self.cumlen = allocate_cache_tensors()
+
+    def reset(self):
+        """Wipe cache and history."""
+        self.history = []
+        self.fed_ids = []
+        self.k, self.v, self.cumlen = allocate_cache_tensors()
+
+    def _render(self):
+        """Render the full conversation (with chat template + generation prompt)."""
+        messages = [
+            {"role": h["role"], "content": [{"type": "text", "text": h["content"]}]}
+            for h in self.history
+        ]
+        enc = self.tokenizer.apply_chat_template(
+            messages, add_generation_prompt=True, tokenize=True,
+            return_dict=True, return_tensors="pt",
+        )
+        return enc["input_ids"][0].tolist()
+
+    def _feed(self, token_ids, start_pos):
+        """Feed a sequence of tokens to the model, updating cache as we go.
+        Returns the logits from the LAST token (for next-token prediction)."""
+        last_logits = None
+        for i, tok in enumerate(token_ids):
+            last_logits, self.k, self.v, self.cumlen = step(
+                self.method, tok, start_pos + i,
+                self.k, self.v, self.cumlen,
+            )
+        return last_logits
+
+    def turn(self, user_text):
+        """Process one user message → assistant response. Returns (response_text, timing_dict)."""
+        self.history.append({"role": "user", "content": user_text})
+
+        # Render full conversation + find what's new
+        full_ids = self._render()
+        # Verify the existing cache is still a prefix of the new render
+        # (chat template should always extend, not modify earlier tokens).
+        n_existing = len(self.fed_ids)
+        prefix_match = (n_existing <= len(full_ids)
+                        and full_ids[:n_existing] == self.fed_ids)
+        if not prefix_match:
+            # Shouldn't happen with normal chat use, but if it does, reset.
+            print("\n  [warn] chat template re-rendered prefix differently; resetting cache.",
+                  file=sys.stderr)
+            self.k, self.v, self.cumlen = allocate_cache_tensors()
+            self.fed_ids = []
+            n_existing = 0
+
+        new_tail = full_ids[n_existing:]
+
+        # Cache overflow check
+        if n_existing + len(new_tail) + self.max_new_tokens > MASK_LEN:
+            tokens_left = MASK_LEN - (n_existing + len(new_tail))
+            if tokens_left < 4:
+                raise RuntimeError(
+                    f"context window full (cache holds {n_existing + len(new_tail)}/{MASK_LEN}). "
+                    f"Type /reset to start a new conversation."
+                )
+
+        # Feed the new tokens
+        t_prefill = time.time()
+        last_logits = self._feed(new_tail, start_pos=n_existing)
+        self.fed_ids.extend(new_tail)
+        t_prefill = time.time() - t_prefill
+        n_prefill = len(new_tail)
+
+        # Greedy decode loop
+        next_id = int(last_logits[0, -1].argmax())
+        generated = []
+        t_decode = time.time()
+        for step_idx in range(self.max_new_tokens):
+            if next_id in EOS_TOKEN_IDS:
+                # Don't add the EOS to history's text, but include in fed_ids
+                # so cache_position stays aligned.
+                self.fed_ids.append(next_id)
+                # Feed the EOS so the cache reflects model's own output marker
+                _, self.k, self.v, self.cumlen = step(
+                    self.method, next_id, len(self.fed_ids) - 1,
+                    self.k, self.v, self.cumlen,
+                )
+                break
+            generated.append(next_id)
+            pos = len(self.fed_ids)
+            last_logits, self.k, self.v, self.cumlen = step(
+                self.method, next_id, pos,
+                self.k, self.v, self.cumlen,
+            )
+            self.fed_ids.append(next_id)
+            next_id = int(last_logits[0, -1].argmax())
+        t_decode = time.time() - t_decode
+
+        response_text = self.tokenizer.decode(generated, skip_special_tokens=True)
+        self.history.append({"role": "model", "content": response_text})
+
+        return response_text, {
+            "prefill_ms": t_prefill * 1000,
+            "prefill_tokens": n_prefill,
+            "prefill_tok_s": (n_prefill / t_prefill) if t_prefill > 0 else 0,
+            "decode_ms": t_decode * 1000,
+            "decode_tokens": len(generated),
+            "decode_tok_s": (len(generated) / t_decode) if t_decode > 0 else 0,
+            "context_used": len(self.fed_ids),
+            "context_max": MASK_LEN,
+        }
+
+
+# -------------------- terminal UI --------------------
+
+HELP_TEXT = """
+Commands:
+  (just type)   send a message to the model
+  /reset        wipe conversation history + cache, start fresh
+  /stats        show timing for the last turn
+  /help         show this help
+  Ctrl+C/D      exit
+"""
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Interactive terminal chat with Gemma 4 E2B (ExecuTorch .pte runtime)",
+    )
+    parser.add_argument("--pte", default=DEFAULT_PTE, help="path to .pte (default: alongside this script)")
+    parser.add_argument("--tokenizer", default=DEFAULT_TOK, help="path to tokenizer dir")
+    parser.add_argument("--max-new-tokens", type=int, default=200,
+                        help="max tokens to generate per response (default 200)")
+    parser.add_argument("--quiet", action="store_true",
+                        help="don't print per-turn timing")
+    args = parser.parse_args()
+
+    # Validate files
+    for path, label in [(args.pte, ".pte"), (args.tokenizer, "tokenizer dir")]:
+        if not os.path.exists(path):
+            print(f"error: {label} not found at {path}", file=sys.stderr)
+            sys.exit(1)
+
+    print(f"loading tokenizer from {args.tokenizer}...")
+    tokenizer = AutoTokenizer.from_pretrained(args.tokenizer)
+
+    print(f"loading .pte from {args.pte}...")
+    t0 = time.time()
+    rt = Runtime.get()
+    program = rt.load_program(args.pte, verification=Verification.Minimal)
+    method = program.load_method("forward")
+    print(f"  loaded in {time.time() - t0:.1f}s")
+
+    session = ChatSession(tokenizer, method, args.max_new_tokens)
+    last_stats = None
+
+    print()
+    print("=" * 60)
+    print("  Gemma 4 E2B — terminal chat")
+    print("  /help for commands · Ctrl+C or Ctrl+D to exit")
+    print("=" * 60)
+
+    def goodbye(*_args):
+        print("\nbye.")
+        sys.exit(0)
+    signal.signal(signal.SIGINT, goodbye)
+
+    while True:
+        try:
+            user = input("\nyou> ").strip()
+        except EOFError:
+            goodbye()
+
+        if not user:
+            continue
+
+        # Commands
+        if user.startswith("/"):
+            cmd = user.lower()
+            if cmd == "/help":
+                print(HELP_TEXT)
+            elif cmd == "/reset":
+                session.reset()
+                print("  (history + cache reset)")
+            elif cmd == "/stats":
+                if last_stats is None:
+                    print("  (no turn yet)")
+                else:
+                    s = last_stats
+                    print(f"  prefill: {s['prefill_ms']:.0f} ms / {s['prefill_tokens']} tok "
+                          f"= {s['prefill_tok_s']:.2f} tok/s")
+                    print(f"  decode:  {s['decode_ms']:.0f} ms / {s['decode_tokens']} tok "
+                          f"= {s['decode_tok_s']:.2f} tok/s")
+                    print(f"  context: {s['context_used']}/{s['context_max']} tokens used")
+            else:
+                print(f"  unknown command: {user}. type /help for the list.")
+            continue
+
+        # Normal turn
+        try:
+            response, stats = session.turn(user)
+        except RuntimeError as e:
+            print(f"  [error] {e}")
+            continue
+        except KeyboardInterrupt:
+            goodbye()
+
+        last_stats = stats
+        print(f"\nmodel> {response}")
+        if not args.quiet:
+            print(f"  [{stats['decode_tokens']} tok @ {stats['decode_tok_s']:.2f} tok/s · "
+                  f"context {stats['context_used']}/{stats['context_max']}]")
+
+
+if __name__ == "__main__":
+    main()

pi_runner.py

@@ -0,0 +1,209 @@
+"""
+Self-contained Gemma 4 E2B INT4 runner for Raspberry Pi 5 (or any ARM64).
+
+Loads ONE .pte (external-cache variant), tokenizes a prompt with the
+Gemma chat template, runs token-by-token (prompt feed + decode) threading
+KV cache tensors across calls, prints generated text + timing.
+
+Designed to run on the Pi with NO project codebase — just the .pte,
+the tokenizer files, and this script. Only Python deps:
+    pip install executorch transformers
+
+Files expected next to this script (or pass paths via flags):
+    gemma4_e2b_text_int4_extcache.pte
+    tokenizer/                          # dir with tokenizer.json + tokenizer_config.json + chat_template.jinja
+
+Usage:
+    python pi_runner.py "The capital of France is"
+    python pi_runner.py "Why is the sky blue?" --max-new-tokens 50
+    python pi_runner.py "Hello" --verify  # asserts output matches reference
+"""
+
+import argparse
+import os
+import time
+
+import torch
+from transformers import AutoTokenizer
+from executorch.runtime import Runtime, Verification
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+DEFAULT_PTE = os.path.join(HERE, "gemma4_e2b_text_int4_extcache.pte")
+DEFAULT_TOK = os.path.join(HERE, "tokenizer")
+
+MAX_CACHE_LEN = 512
+MASK_LEN = MAX_CACHE_LEN - 1  # 511; .pte specialized to this upper bound
+DTYPE = torch.float32  # matches the model's quantize-time dtype
+
+# Hardcoded cache layout for Gemma 4 E2B. Matches what
+# scripts/_external_cache.py:compute_layer_specs derives from the model
+# config. 35 decoder layers minus num_kv_shared_layers=20 = 15 cache layers.
+# Layer-type pattern (repeats every 5): [sliding, sliding, sliding, sliding, full].
+# Sliding layers: head_dim=256. Full layers: global_head_dim=512.
+GEMMA4_E2B_LAYER_SHAPES = [
+    # (head_dim, is_sliding)
+    (256, True),  (256, True),  (256, True),  (256, True),  (512, False),  # layers 0-4
+    (256, True),  (256, True),  (256, True),  (256, True),  (512, False),  # layers 5-9
+    (256, True),  (256, True),  (256, True),  (256, True),  (512, False),  # layers 10-14
+]
+NUM_KV_HEADS = 1
+BATCH = 1
+
+# Reference for --verify mode (FP32/INT4 token-id sequence for "The capital of France is")
+REFERENCE_PROMPT = "The capital of France is"
+REFERENCE_IDS = [818, 5279, 529, 7001, 563, 5213, 50429, 84750, 106]
+REFERENCE_TEXT = "The capital of France is **Paris**."
+
+# Gemma 4 end-of-turn token id (model stops here in chat)
+EOS_TOKEN_IDS = {106, 1, 2}  # <end_of_turn>, <eos>, <bos>-as-sentinel
+
+
+def allocate_cache_tensors():
+    """Allocate one set of K, V, cumulative_length tensors per cache layer."""
+    k_caches, v_caches, cumlen_caches = [], [], []
+    for head_dim, _is_sliding in GEMMA4_E2B_LAYER_SHAPES:
+        shape = (BATCH, NUM_KV_HEADS, MAX_CACHE_LEN, head_dim)
+        k_caches.append(torch.zeros(shape, dtype=DTYPE))
+        v_caches.append(torch.zeros(shape, dtype=DTYPE))
+        cumlen_caches.append(torch.zeros(1, dtype=torch.int64))
+    return k_caches, v_caches, cumlen_caches
+
+
+def step(method, token_id, pos, k_caches, v_caches, cumlen_caches):
+    """One forward call: feed `token_id` at position `pos`, get logits +
+    updated cache tensors back."""
+    input_ids = torch.tensor([[token_id]], dtype=torch.long)
+    attention_mask = torch.zeros(1, MASK_LEN, dtype=torch.long)
+    attention_mask[:, :pos + 1] = 1
+    position_ids = torch.tensor([[pos]], dtype=torch.long)
+    cache_position = torch.tensor([pos], dtype=torch.long)
+
+    # The .pte's execute() takes flat positional inputs.
+    # Order matches the wrapper's forward signature:
+    #   input_ids, attention_mask, position_ids, cache_position, *k_caches, *v_caches, *cumlen_caches
+    args = (input_ids, attention_mask, position_ids, cache_position,
+            *k_caches, *v_caches, *cumlen_caches)
+    if os.environ.get("DEBUG_SHAPES"):
+        for i, a in enumerate(args):
+            if hasattr(a, "shape"):
+                print(f"    arg[{i:2d}]: shape={tuple(a.shape)} dtype={a.dtype}", flush=True)
+    outputs = method.execute(args)
+    # The .pte emits 91 outputs:
+    #   [0..14]   K mutations (auto-emitted by torch.export)
+    #   [15..29]  V mutations
+    #   [30..44]  cumlen mutations
+    #   [45]      logits
+    #   [46..60]  K (from wrapper's explicit return — same tensors)
+    #   [61..75]  V (from wrapper's explicit return)
+    #   [76..90]  cumlen (from wrapper's explicit return)
+    # Either copy works; we use the explicit-return half because indices
+    # align with the (logits, k, v, cumlen) ordering the wrapper declared.
+    n = len(GEMMA4_E2B_LAYER_SHAPES)
+    logits = outputs[45]
+    base = 46
+    k_new = list(outputs[base:base + n])
+    v_new = list(outputs[base + n:base + 2 * n])
+    cumlen_new = list(outputs[base + 2 * n:base + 3 * n])
+    return logits, k_new, v_new, cumlen_new
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("prompt", nargs="?", default=REFERENCE_PROMPT,
+                        help=f"Prompt text (default: {REFERENCE_PROMPT!r})")
+    parser.add_argument("--pte", default=DEFAULT_PTE, help="Path to .pte file")
+    parser.add_argument("--tokenizer", default=DEFAULT_TOK, help="Path to tokenizer dir")
+    parser.add_argument("--max-new-tokens", type=int, default=20)
+    parser.add_argument("--verify", action="store_true",
+                        help="Assert prompt+output match the reference (smoke test)")
+    args = parser.parse_args()
+
+    if args.verify:
+        args.prompt = REFERENCE_PROMPT
+        args.max_new_tokens = max(args.max_new_tokens, len(REFERENCE_IDS))
+
+    print(f"Loading tokenizer from {args.tokenizer}...")
+    tokenizer = AutoTokenizer.from_pretrained(args.tokenizer)
+
+    print(f"Tokenizing prompt: {args.prompt!r}")
+    messages = [{"role": "user", "content": [{"type": "text", "text": args.prompt}]}]
+    enc = tokenizer.apply_chat_template(
+        messages, add_generation_prompt=True, tokenize=True,
+        return_dict=True, return_tensors="pt",
+    )
+    prompt_ids = enc["input_ids"][0].tolist()
+    n_prompt = len(prompt_ids)
+    if n_prompt + args.max_new_tokens > MASK_LEN:
+        raise SystemExit(
+            f"prompt ({n_prompt}) + max_new_tokens ({args.max_new_tokens}) "
+            f"exceeds mask_len ({MASK_LEN})"
+        )
+    print(f"  prompt_len = {n_prompt}")
+
+    print(f"\nLoading {args.pte}...")
+    t0 = time.time()
+    rt = Runtime.get()
+    program = rt.load_program(args.pte, verification=Verification.Minimal)
+    method = program.load_method("forward")
+    print(f"  loaded in {time.time() - t0:.1f}s")
+
+    print("Allocating cache tensors...")
+    k_caches, v_caches, cumlen_caches = allocate_cache_tensors()
+    cache_mb = sum(t.numel() * t.element_size() for t in k_caches + v_caches) / 1e6
+    print(f"  total cache size: {cache_mb:.1f} MB across {len(k_caches)} layers")
+
+    # --- Prompt token-by-token feed ("slow prefill") ---
+    print(f"\nFeeding {n_prompt} prompt tokens (token-by-token; no batched prefill in this design)...")
+    t_prefill_start = time.time()
+    last_logits = None
+    for i, tok in enumerate(prompt_ids):
+        last_logits, k_caches, v_caches, cumlen_caches = step(
+            method, tok, i, k_caches, v_caches, cumlen_caches
+        )
+    t_prefill = time.time() - t_prefill_start
+    print(f"  prompt feed: {t_prefill:.2f}s ({n_prompt} tokens, "
+          f"{n_prompt / t_prefill:.2f} tok/s, ttft equivalent)")
+
+    # Next-token prediction from last prompt position's logits
+    next_id = int(last_logits[0, -1].argmax())
+    generated = [next_id]
+    print(f"  first generated token: id={next_id} text={tokenizer.decode([next_id])!r}")
+
+    # --- Decode loop ---
+    print(f"\nDecoding up to {args.max_new_tokens - 1} more tokens...")
+    t_decode_start = time.time()
+    n_decoded = 1
+    for step_idx in range(args.max_new_tokens - 1):
+        if next_id in EOS_TOKEN_IDS:
+            print(f"  hit EOS (id={next_id}) at decode step {step_idx}")
+            break
+        pos = n_prompt + step_idx  # position of the token we just produced
+        last_logits, k_caches, v_caches, cumlen_caches = step(
+            method, next_id, pos, k_caches, v_caches, cumlen_caches
+        )
+        next_id = int(last_logits[0, -1].argmax())
+        generated.append(next_id)
+        n_decoded += 1
+    t_decode = time.time() - t_decode_start
+
+    text = tokenizer.decode(generated, skip_special_tokens=True)
+    print(f"\nGenerated ({len(generated)} tokens): {text!r}")
+    print(f"\n=== Timing ===")
+    print(f"  prompt feed:   {t_prefill*1000:7.0f} ms  ({n_prompt} tok @ {n_prompt/t_prefill:5.2f} tok/s)")
+    print(f"  decode:        {t_decode*1000:7.0f} ms  ({n_decoded} tok @ {n_decoded/t_decode:5.2f} tok/s)")
+    print(f"  total:         {(t_prefill + t_decode)*1000:7.0f} ms")
+
+    if args.verify:
+        compare_len = min(len(generated), len(REFERENCE_IDS))
+        match = generated[:compare_len] == REFERENCE_IDS[:compare_len]
+        print(f"\n=== Verify ===")
+        print(f"  reference text: {REFERENCE_TEXT!r}")
+        print(f"  generated text: {text!r}")
+        print(f"  reference ids:  {REFERENCE_IDS[:compare_len]}")
+        print(f"  generated ids:  {generated[:compare_len]}")
+        print(f"  RESULT: {'PASS' if match else 'FAIL'}")
+        raise SystemExit(0 if match else 1)
+
+
+if __name__ == "__main__":
+    main()

tokenizer/chat_template.jinja

@@ -0,0 +1,360 @@
+{%- macro format_parameters(properties, required, filter_keys=false) -%}
+    {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%}
+    {%- set ns = namespace(found_first=false) -%}
+    {%- for key, value in properties | dictsort -%}
+        {%- set add_comma = false -%}
+        {%- if not filter_keys or key not in standard_keys -%}
+            {%- if ns.found_first %},{% endif -%}
+            {%- set ns.found_first = true -%}
+            {{ key }}:{
+            {%- if value['description'] -%}
+                description:<|"|>{{ value['description'] }}<|"|>
+                {%- set add_comma = true -%}
+            {%- endif -%}
+            {%- if value['type'] | upper == 'STRING' -%}
+                {%- if value['enum'] -%}
+                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                    enum:{{ format_argument(value['enum']) }}
+                {%- endif -%}
+            {%- elif value['type'] | upper == 'ARRAY' -%}
+                {%- if value['items'] is mapping and value['items'] -%}
+                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                    items:{
+                    {%- set ns_items = namespace(found_first=false) -%}
+                    {%- for item_key, item_value in value['items'] | dictsort -%}
+                        {%- if item_value is not none -%}
+                            {%- if ns_items.found_first %},{% endif -%}
+                            {%- set ns_items.found_first = true -%}
+                            {%- if item_key == 'properties' -%}
+                                properties:{
+                                {%- if item_value is mapping -%}
+                                    {{- format_parameters(item_value, value['items']['required'] | default([])) -}}
+                                {%- endif -%}
+                                }
+                            {%- elif item_key == 'required' -%}
+                                required:[
+                                {%- for req_item in item_value -%}
+                                    <|"|>{{- req_item -}}<|"|>
+                                    {%- if not loop.last %},{% endif -%}
+                                {%- endfor -%}
+                                ]
+                            {%- elif item_key == 'type' -%}
+                                {%- if item_value is string -%}
+                                    type:{{ format_argument(item_value | upper) }}
+                                {%- else -%}
+                                    type:{{ format_argument(item_value | map('upper') | list) }}
+                                {%- endif -%}
+                            {%- else -%}
+                                {{ item_key }}:{{ format_argument(item_value) }}
+                            {%- endif -%}
+                        {%- endif -%}
+                    {%- endfor -%}
+                    }
+                {%- endif -%}
+            {%- endif -%}
+            {%- if value['nullable'] %}
+                {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                nullable:true
+            {%- endif -%}
+            {%- if value['type'] | upper == 'OBJECT' -%}
+                {%- if value['properties'] is defined and value['properties'] is mapping -%}
+                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                    properties:{
+                    {{- format_parameters(value['properties'], value['required'] | default([])) -}}
+                    }
+                {%- elif value is mapping -%}
+                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                    properties:{
+                    {{- format_parameters(value, value['required'] | default([]), filter_keys=true) -}}
+                    }
+                {%- endif -%}
+                {%- if value['required'] -%}
+                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                    required:[
+                    {%- for item in value['required'] | default([]) -%}
+                        <|"|>{{- item -}}<|"|>
+                        {%- if not loop.last %},{% endif -%}
+                    {%- endfor -%}
+                    ]
+                {%- endif -%}
+            {%- endif -%}
+            {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+            type:<|"|>{{ value['type'] | upper }}<|"|>}
+        {%- endif -%}
+    {%- endfor -%}
+{%- endmacro -%}
+{%- macro format_function_declaration(tool_data) -%}
+    declaration:{{- tool_data['function']['name'] -}}{description:<|"|>{{- tool_data['function']['description'] -}}<|"|>
+    {%- set params = tool_data['function']['parameters'] -%}
+    {%- if params -%}
+        ,parameters:{
+        {%- if params['properties'] -%}
+            properties:{ {{- format_parameters(params['properties'], params['required']) -}} },
+        {%- endif -%}
+        {%- if params['required'] -%}
+            required:[
+            {%- for item in params['required'] -%}
+                <|"|>{{- item -}}<|"|>
+                {{- ',' if not loop.last -}}
+            {%- endfor -%}
+            ],
+        {%- endif -%}
+        {%- if params['type'] -%}
+            type:<|"|>{{- params['type'] | upper -}}<|"|>}
+        {%- endif -%}
+    {%- endif -%}
+    {%- if 'response' in tool_data['function'] -%}
+        {%- set response_declaration = tool_data['function']['response'] -%}
+        ,response:{
+        {%- if response_declaration['description'] -%}
+            description:<|"|>{{- response_declaration['description'] -}}<|"|>,
+        {%- endif -%}
+        {%- if response_declaration['type'] | upper == 'OBJECT' -%}
+            type:<|"|>{{- response_declaration['type'] | upper -}}<|"|>}
+        {%- endif -%}
+    {%- endif -%}
+    }
+{%- endmacro -%}
+{%- macro format_argument(argument, escape_keys=True) -%}
+    {%- if argument is string -%}
+        {{- '<|"|>' + argument + '<|"|>' -}}
+    {%- elif argument is boolean -%}
+        {{- 'true' if argument else 'false' -}}
+    {%- elif argument is mapping -%}
+        {{- '{' -}}
+        {%- set ns = namespace(found_first=false) -%}
+        {%- for key, value in argument | dictsort -%}
+            {%- if ns.found_first %},{% endif -%}
+            {%- set ns.found_first = true -%}
+            {%- if escape_keys -%}
+                {{- '<|"|>' + key + '<|"|>' -}}
+            {%- else -%}
+                {{- key -}}
+            {%- endif -%}
+            :{{- format_argument(value, escape_keys=escape_keys) -}}
+        {%- endfor -%}
+        {{- '}' -}}
+    {%- elif argument is sequence -%}
+        {{- '[' -}}
+        {%- for item in argument -%}
+            {{- format_argument(item, escape_keys=escape_keys) -}}
+            {%- if not loop.last %},{% endif -%}
+        {%- endfor -%}
+        {{- ']' -}}
+    {%- else -%}
+        {{- argument -}}
+    {%- endif -%}
+{%- endmacro -%}
+{%- macro strip_thinking(text) -%}
+    {%- set ns = namespace(result='') -%}
+    {%- for part in text.split('<channel|>') -%}
+        {%- if '<|channel>' in part -%}
+            {%- set ns.result = ns.result + part.split('<|channel>')[0] -%}
+        {%- else -%}
+            {%- set ns.result = ns.result + part -%}
+        {%- endif -%}
+    {%- endfor -%}
+    {{- ns.result | trim -}}
+{%- endmacro -%}
+
+{%- macro format_tool_response_block(tool_name, response) -%}
+    {{- '<|tool_response>' -}}
+    {%- if response is mapping -%}
+        {{- 'response:' + tool_name + '{' -}}
+        {%- for key, value in response | dictsort -%}
+            {{- key -}}:{{- format_argument(value, escape_keys=False) -}}
+            {%- if not loop.last %},{% endif -%}
+        {%- endfor -%}
+        {{- '}' -}}
+    {%- else -%}
+        {{- 'response:' + tool_name + '{value:' + format_argument(response, escape_keys=False) + '}' -}}
+    {%- endif -%}
+    {{- '<tool_response|>' -}}
+{%- endmacro -%}
+
+{%- set ns = namespace(prev_message_type=None) -%}
+{%- set loop_messages = messages -%}
+{{- bos_token -}}
+{#- Handle System/Tool Definitions Block -#}
+{%- if (enable_thinking is defined and enable_thinking) or tools or messages[0]['role'] in ['system', 'developer'] -%}
+    {{- '<|turn>system\n' -}}
+    {#- Inject Thinking token at the very top of the FIRST system turn -#}
+    {%- if enable_thinking is defined and enable_thinking -%}
+        {{- '<|think|>\n' -}}
+        {%- set ns.prev_message_type = 'think' -%}
+    {%- endif -%}
+    {%- if messages[0]['role'] in ['system', 'developer'] -%}
+        {%- if messages[0]['content'] is string -%}
+            {{- messages[0]['content'] | trim -}}
+        {%- elif messages[0]['content'] is sequence -%}
+            {%- for item in messages[0]['content'] -%}
+                {{- item['text'] | trim + ' '-}}
+            {%- endfor -%}
+        {%- endif -%}
+        {%- set loop_messages = messages[1:] -%}
+    {%- endif -%}
+    {%- if tools -%}
+        {%- for tool in tools %}
+            {{- '<|tool>' -}}
+            {{- format_function_declaration(tool) | trim -}}
+            {{- '<tool|>' -}}
+        {%- endfor %}
+        {%- set ns.prev_message_type = 'tool' -%}
+    {%- endif -%}
+    {{- '<turn|>\n' -}}
+{%- endif %}
+
+{#- Pre-scan: find last user message index for reasoning guard -#}
+{%- set ns_turn = namespace(last_user_idx=-1) -%}
+{%- for i in range(loop_messages | length) -%}
+    {%- if loop_messages[i]['role'] == 'user' -%}
+        {%- set ns_turn.last_user_idx = i -%}
+    {%- endif -%}
+{%- endfor -%}
+
+{#- Loop through messages -#}
+{%- for message in loop_messages -%}
+    {%- if message['role'] != 'tool' -%}
+    {%- set ns.prev_message_type = None -%}
+    {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}
+    {#- Detect continuation: suppress duplicate <|turn>model when previous non-tool message was also assistant -#}
+    {%- set prev_nt = namespace(role=None, found=false) -%}
+    {%- if loop.index0 > 0 -%}
+        {%- for j in range(loop.index0 - 1, -1, -1) -%}
+            {%- if not prev_nt.found -%}
+                {%- if loop_messages[j]['role'] != 'tool' -%}
+                    {%- set prev_nt.role = loop_messages[j]['role'] -%}
+                    {%- set prev_nt.found = true -%}
+                {%- endif -%}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+    {%- set continue_same_model_turn = (role == 'model' and prev_nt.role == 'assistant') -%}
+    {%- if not continue_same_model_turn -%}
+        {{- '<|turn>' + role + '\n' }}
+    {%- endif -%}
+
+    {#- Render reasoning/reasoning_content as thinking channel -#}
+    {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%}
+    {%- if thinking_text and loop.index0 > ns_turn.last_user_idx and message.get('tool_calls') -%}
+        {{- '<|channel>thought\n' + thinking_text + '\n<channel|>' -}}
+    {%- endif -%}
+
+            {%- if message['tool_calls'] -%}
+                {%- for tool_call in message['tool_calls'] -%}
+                    {%- set function = tool_call['function'] -%}
+                    {{- '<|tool_call>call:' + function['name'] + '{' -}}
+                    {%- if function['arguments'] is mapping -%}
+                        {%- set ns_args = namespace(found_first=false) -%}
+                        {%- for key, value in function['arguments'] | dictsort -%}
+                            {%- if ns_args.found_first %},{% endif -%}
+                            {%- set ns_args.found_first = true -%}
+                            {{- key -}}:{{- format_argument(value, escape_keys=False) -}}
+                        {%- endfor -%}
+                    {%- elif function['arguments'] is string -%}
+                        {{- function['arguments'] -}}
+                    {%- endif -%}
+                    {{- '}<tool_call|>' -}}
+                {%- endfor -%}
+                {%- set ns.prev_message_type = 'tool_call' -%}
+            {%- endif -%}
+
+            {%- set ns_tr_out = namespace(flag=false) -%}
+            {%- if message.get('tool_responses') -%}
+                {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#}
+                {%- for tool_response in message['tool_responses'] -%}
+                    {{- format_tool_response_block(tool_response['name'] | default('unknown'), tool_response['response']) -}}
+                    {%- set ns_tr_out.flag = true -%}
+                    {%- set ns.prev_message_type = 'tool_response' -%}
+                {%- endfor -%}
+            {%- elif message.get('tool_calls') -%}
+                {#- OpenAI Chat Completions: forward-scan consecutive role:tool messages -#}
+                {%- set ns_tool_scan = namespace(stopped=false) -%}
+                {%- for k in range(loop.index0 + 1, loop_messages | length) -%}
+                    {%- if ns_tool_scan.stopped -%}
+                    {%- elif loop_messages[k]['role'] != 'tool' -%}
+                        {%- set ns_tool_scan.stopped = true -%}
+                    {%- else -%}
+                        {%- set follow = loop_messages[k] -%}
+                        {#- Resolve tool_call_id to function name -#}
+                        {%- set ns_tname = namespace(name=follow.get('name') | default('unknown')) -%}
+                        {%- for tc in message['tool_calls'] -%}
+                            {%- if tc.get('id') == follow.get('tool_call_id') -%}
+                                {%- set ns_tname.name = tc['function']['name'] -%}
+                            {%- endif -%}
+                        {%- endfor -%}
+                        {#- Handle content as string or content-parts array -#}
+                        {%- set tool_body = follow.get('content') -%}
+                        {%- if tool_body is string -%}
+                            {{- format_tool_response_block(ns_tname.name, tool_body) -}}
+                        {%- elif tool_body is sequence and tool_body is not string -%}
+                            {%- set ns_txt = namespace(s='') -%}
+                            {%- for part in tool_body -%}
+                                {%- if part.get('type') == 'text' -%}
+                                    {%- set ns_txt.s = ns_txt.s + (part.get('text') | default('')) -%}
+                                {%- endif -%}
+                            {%- endfor -%}
+                            {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}}
+                            {%- for part in tool_body -%}
+                                {%- if part.get('type') == 'image' -%}
+                                    {{- '<|image|>' -}}
+                                {%- elif part.get('type') == 'audio' -%}
+                                    {{- '<|audio|>' -}}
+                                {%- elif part.get('type') == 'video' -%}
+                                    {{- '<|video|>' -}}
+                                {%- endif -%}
+                            {%- endfor -%}
+                        {%- else -%}
+                            {{- format_tool_response_block(ns_tname.name, tool_body) -}}
+                        {%- endif -%}
+                        {%- set ns_tr_out.flag = true -%}
+                        {%- set ns.prev_message_type = 'tool_response' -%}
+                    {%- endif -%}
+                {%- endfor -%}
+            {%- endif -%}
+
+            {%- set captured_content -%}
+            {%- if message['content'] is string -%}
+                {%- if role == 'model' -%}
+                    {{- strip_thinking(message['content']) -}}
+                {%- else -%}
+                    {{- message['content'] | trim -}}
+                {%- endif -%}
+            {%- elif message['content'] is sequence -%}
+                {%- for item in message['content'] -%}
+                    {%- if item['type'] == 'text' -%}
+                        {%- if role == 'model' -%}
+                            {{- strip_thinking(item['text']) -}}
+                        {%- else -%}
+                            {{- item['text'] | trim -}}
+                        {%- endif -%}
+                    {%- elif item['type'] == 'image' -%}
+                        {{- '<|image|>' -}}
+                        {%- set ns.prev_message_type = 'image' -%}
+                    {%- elif item['type'] == 'audio' -%}
+                        {{- '<|audio|>' -}}
+                        {%- set ns.prev_message_type = 'audio' -%}
+                    {%- elif item['type'] == 'video' -%}
+                        {{- '<|video|>' -}}
+                        {%- set ns.prev_message_type = 'video' -%}
+                    {%- endif -%}
+                {%- endfor -%}
+            {%- endif -%}
+            {%- endset -%}
+
+            {{- captured_content -}}
+            {%- set has_content = captured_content | trim | length > 0 -%}
+
+        {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%}
+            {{- '<|tool_response>' -}}
+        {%- elif not (ns_tr_out.flag and not has_content) -%}
+            {{- '<turn|>\n' -}}
+        {%- endif -%}
+    {%- endif -%}
+{%- endfor -%}
+
+{%- if add_generation_prompt -%}
+    {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%}
+        {{- '<|turn>model\n' -}}
+    {%- endif -%}
+{%- endif -%}

tokenizer/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f
+size 32169626

tokenizer/tokenizer_config.json

@@ -0,0 +1,74 @@
+{
+  "audio_token": "<|audio|>",
+  "backend": "tokenizers",
+  "boa_token": "<|audio>",
+  "boi_token": "<|image>",
+  "bos_token": "<bos>",
+  "eoa_token": "<audio|>",
+  "eoc_token": "<channel|>",
+  "eoi_token": "<image|>",
+  "eos_token": "<eos>",
+  "eot_token": "<turn|>",
+  "escape_token": "<|\"|>",
+  "etc_token": "<tool_call|>",
+  "etd_token": "<tool|>",
+  "etr_token": "<tool_response|>",
+  "extra_special_tokens": [
+    "<|video|>"
+  ],
+  "image_token": "<|image|>",
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "processor_class": "Gemma4Processor",
+  "response_schema": {
+    "type": "object",
+    "properties": {
+      "role": {
+        "const": "assistant"
+      },
+      "thinking": {
+        "type": "string"
+      },
+      "content": {
+        "type": "string"
+      },
+      "tool_calls": {
+        "x-regex-iterator": "<\\|tool_call>(.*?)<tool_call\\|>",
+        "type": "array",
+        "items": {
+          "type": "object",
+          "properties": {
+            "type": {
+              "const": "function"
+            },
+            "function": {
+              "type": "object",
+              "x-regex": "call\\:(?P<name>\\w+)(?P<arguments>\\{.*\\})",
+              "properties": {
+                "name": {
+                  "type": "string"
+                },
+                "arguments": {
+                  "type": "object",
+                  "x-parser": "gemma4-tool-call",
+                  "additionalProperties": {}
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "x-regex": "(\\<\\|channel\\>thought\\n(?P<thinking>.*?)\\<channel\\|\\>)?(?P<tool_calls>\\<\\|tool_call\\>.*\\<tool_call\\|\\>)?(?P<content>(?:(?!\\<turn\\|\\>)(?!\\<\\|tool_response\\>).)+)?(?:\\<turn\\|\\>|\\<\\|tool_response\\>)?"
+  },
+  "soc_token": "<|channel>",
+  "sot_token": "<|turn>",
+  "stc_token": "<|tool_call>",
+  "std_token": "<|tool>",
+  "str_token": "<|tool_response>",
+  "think_token": "<|think|>",
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>"
+}