Initial E4B CoreML bundle — chunks, embeds, PLE, RoPE, tokenizer

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+hf_model/tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,83 @@
+---
+license: gemma
+base_model: google/gemma-4-E4B-it
+tags:
+  - coreml
+  - apple-neural-engine
+  - gemma
+  - text-generation
+pipeline_tag: text-generation
+---
+
+# Gemma 4 E4B — CoreML (INT4, Apple Neural Engine)
+
+CoreML port of [`google/gemma-4-E4B-it`](https://huggingface.co/google/gemma-4-E4B-it) (the
+4B-effective Gemma 4 decoder), chunked into 4 sliding-window-attention (SWA) pieces
+for Apple Neural Engine execution. Produced by
+[john-rocky/CoreML-LLM](https://github.com/john-rocky/CoreML-LLM) via
+`conversion/build_gemma4_bundle.py --model gemma4-e4b --ctx 2048`.
+
+## Layout
+
+- `chunk{1..4}.mlmodelc/` — multi-function chunks (`decode_q1` + `verify_qK`), INT4 palettized
+  - chunk1: L0–11 (own KV), chunk2: L12–23 (emits `kv13_*`/`kv14_*` aliases for producer
+    layers L22/L23), chunk3: L24–32 (KV-shared), chunk4: L33–41 + LM head
+- `embed_tokens_q8.bin` + `embed_tokens_scales.bin` — INT8 token embeddings (262144 × 2560)
+- `embed_tokens_per_layer_q8.bin` + `embed_tokens_per_layer_scales.bin` — INT8 PLE
+  (262144 × 11776 = nlayers × per_layer_dim)
+- `per_layer_projection.bin`, `per_layer_norm_weight.bin` — fp16 per-layer projection weights
+- `cos_{sliding,full}.npy`, `sin_{sliding,full}.npy` — precomputed RoPE tables
+- `hf_model/tokenizer.json`, `tokenizer_config.json`, `config.json`,
+  `generation_config.json` — swift-transformers tokenizer input
+- `model_config.json` — runtime config (hidden_size=2560, num_hidden_layers=42,
+  num_key_value_heads=2, sliding_window=512, context_length=2048)
+
+## Architecture (vs E2B)
+
+| | E2B | E4B |
+|---|---:|---:|
+| num_hidden_layers | 35 | **42** |
+| hidden_size | 1536 | **2560** |
+| num_key_value_heads | 1 | **2** |
+| intermediate_size | 6144 | **10240** |
+| num_kv_shared_layers | 20 | 18 |
+| KV producers (sliding/full) | L13 / L14 | **L22 / L23** |
+| Chunk boundaries | L0-7, L8-14, L15-24, L25-34 | L0-11, L12-23, L24-32, L33-41 |
+
+The Swift runtime names the producer-layer KV outputs `kv13_*` / `kv14_*` regardless of
+actual layer index, so the iOS side needs no model-specific wiring.
+
+## Usage (Swift)
+
+Use via the [john-rocky/CoreML-LLM](https://github.com/john-rocky/CoreML-LLM) app. The
+model picker in `CoreMLLLMChat` will auto-download and run this repo when you select
+"Gemma 4 E4B".
+
+Direct load in CoreMLLLM Swift:
+
+```swift
+import CoreMLLLM
+
+let modelURL = // Documents/Models/gemma4-e4b
+let llm = try await CoreMLLLM.load(from: modelURL)
+for await token in try await llm.stream([.init(role: .user, content: "Hi")]) {
+    print(token, terminator: "")
+}
+```
+
+## Benchmarks
+
+iPhone 17 Pro, INT4 palettized, ctx=2048, baseline (no speculative decoding):
+
+| Metric | Value |
+|---|---:|
+| Decode tok/s | **~14 tok/s** |
+| Per-step latency | ~71 ms |
+| Memory (`phys_footprint`) | ~4.5 GB |
+| ANE placement | 100% |
+
+## Context length
+
+Shipping bundle is ctx=2048. Rebuild with `--ctx 4096` or higher on a sufficiently
+large Mac to extend; the ANE will reject chunks whose declared context differs from
+`model_config.json`.

chunk1.mlmodelc/analytics/coremldata.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb7d0c074925f5e2b23d70754135276d3b38e5bb2ebf89df153a401e37ef2f57
+size 243

chunk1.mlmodelc/coremldata.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b53fc92f6d11bf88eb63b9a7af4a7211180e3c031115a9abfa20655814727d4
+size 1333

chunk1.mlmodelc/weights/weight.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67c868123b2e7b2182d97a0aaca1d4e33f861ee446eac5b038bdb9f0e2c6e787
+size 585970432

chunk2.mlmodelc/analytics/coremldata.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c032a454d2eaeea9fd5bfdfe3a2caf53e1a79d401fe8007c986abcc44469a19
+size 243

chunk2.mlmodelc/coremldata.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8046d35bb019573a7541c830b6b712d5176c199f51e2b1fec3a17342d86a3ac6
+size 1471

chunk2.mlmodelc/weights/weight.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6cd92e3945e5e809a15df7a1d9e648fb651e859d733d9589eba817805e2d96d
+size 572196992

chunk3.mlmodelc/analytics/coremldata.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83e107e7d0f531fa20c6861ea0483120a4246adc98daba2bdc9ec015f77bc7ac
+size 243

chunk3.mlmodelc/coremldata.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d43e7d6694b27bec11ece3eb5bb8b3b5b185fa8bc0b668980e154952fd36bf0b
+size 940

chunk3.mlmodelc/weights/weight.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db45f9ce7443de57765ba412a0158ac2ad46a9d2f735fba9376bbdb0aa357b88
+size 412740736

chunk4.mlmodelc/analytics/coremldata.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58c1fb89f6c05774b2ac875839fcc1e5c153cc195cfe223ad9ffb42d2d30ea48
+size 243

chunk4.mlmodelc/coremldata.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b78c0ef3198782c4e4060bdf45f682dcb566666e05fee15858a5c2467c05965b
+size 1014

chunk4.mlmodelc/weights/weight.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f044d109750fec5d781baada3c070cd3d524b674ea68a00c1c99b5e8015cfbb
+size 753797440

cos_full.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:767b3a21305a67e3a3fd22e256f2e7385014b32374442b6103fb820c7d9ef1fc
+size 4194432

cos_sliding.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a27afac2d0282008c59736cb498f0b49e6f775e0b3847811fdd06be09c6df4a1
+size 2097280

embed_tokens_per_layer_q8.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:269eb54aa366e2d73474d7521b376025fa53bd2d72c6d3bc0301b0882c6ae681
+size 2818572288

embed_tokens_per_layer_scales.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc32925fb678b96d96cf804b77c1d137f157a6136b76e6f8a003d69f4e976fea
+size 524288

embed_tokens_q8.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8921fcbb6d8a79c7b304b929f357cdb41905fec75a90bad00dfed071c76fb82
+size 671088640

embed_tokens_scales.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5c20211cbfbb5e5a1059d91c5c1e0493e7630505028459e7843a9b1b41ee854
+size 524288

hf_model/config.json

@@ -0,0 +1,197 @@
+{
+  "architectures": [
+    "Gemma4ForConditionalGeneration"
+  ],
+  "audio_config": {
+    "_name_or_path": "",
+    "architectures": null,
+    "attention_chunk_size": 12,
+    "attention_context_left": 13,
+    "attention_context_right": 0,
+    "attention_invalid_logits_value": -1000000000.0,
+    "attention_logit_cap": 50.0,
+    "chunk_size_feed_forward": 0,
+    "conv_kernel_size": 5,
+    "dtype": "bfloat16",
+    "gradient_clipping": 10000000000.0,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "model_type": "gemma4_audio",
+    "num_attention_heads": 8,
+    "num_hidden_layers": 12,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_proj_dims": 1536,
+    "problem_type": null,
+    "residual_weight": 0.5,
+    "return_dict": true,
+    "rms_norm_eps": 1e-06,
+    "subsampling_conv_channels": [
+      128,
+      32
+    ],
+    "use_clipped_linears": true
+  },
+  "audio_token_id": 258881,
+  "boa_token_id": 256000,
+  "boi_token_id": 255999,
+  "dtype": "bfloat16",
+  "eoa_token_id": 258883,
+  "eoa_token_index": 258883,
+  "eoi_token_id": 258882,
+  "eos_token_id": [
+    1,
+    106
+  ],
+  "image_token_id": 258880,
+  "initializer_range": 0.02,
+  "model_type": "gemma4",
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "attention_k_eq_v": false,
+    "bos_token_id": 2,
+    "dtype": "bfloat16",
+    "enable_moe_block": false,
+    "eos_token_id": 1,
+    "expert_intermediate_size": null,
+    "final_logit_softcapping": 30.0,
+    "global_head_dim": 512,
+    "head_dim": 256,
+    "hidden_activation": "gelu_pytorch_tanh",
+    "hidden_size": 2560,
+    "hidden_size_per_layer_input": 256,
+    "initializer_range": 0.02,
+    "intermediate_size": 10240,
+    "layer_types": [
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "model_type": "gemma4_text",
+    "num_attention_heads": 8,
+    "num_experts": null,
+    "num_global_key_value_heads": null,
+    "num_hidden_layers": 42,
+    "num_key_value_heads": 2,
+    "num_kv_shared_layers": 18,
+    "pad_token_id": 0,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "full_attention": {
+        "partial_rotary_factor": 0.25,
+        "rope_theta": 1000000.0,
+        "rope_type": "proportional"
+      },
+      "sliding_attention": {
+        "rope_theta": 10000.0,
+        "rope_type": "default"
+      }
+    },
+    "sliding_window": 512,
+    "tie_word_embeddings": true,
+    "top_k_experts": null,
+    "use_bidirectional_attention": null,
+    "use_cache": true,
+    "use_double_wide_mlp": false,
+    "vocab_size": 262144,
+    "vocab_size_per_layer_input": 262144
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.5.0.dev0",
+  "video_token_id": 258884,
+  "vision_config": {
+    "_name_or_path": "",
+    "architectures": null,
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "chunk_size_feed_forward": 0,
+    "default_output_length": 280,
+    "dtype": "bfloat16",
+    "global_head_dim": 64,
+    "head_dim": 64,
+    "hidden_activation": "gelu_pytorch_tanh",
+    "hidden_size": 768,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "max_position_embeddings": 131072,
+    "model_type": "gemma4_vision",
+    "num_attention_heads": 12,
+    "num_hidden_layers": 16,
+    "num_key_value_heads": 12,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "patch_size": 16,
+    "pooling_kernel_size": 3,
+    "position_embedding_size": 10240,
+    "problem_type": null,
+    "return_dict": true,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 100.0,
+      "rope_type": "default"
+    },
+    "standardize": false,
+    "use_clipped_linears": true
+  },
+  "vision_soft_tokens_per_image": 280
+}

hf_model/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 2,
+  "do_sample": true,
+  "eos_token_id": [
+    1,
+    106,
+    50
+  ],
+  "pad_token_id": 0,
+  "temperature": 1.0,
+  "top_k": 64,
+  "top_p": 0.95,
+  "transformers_version": "5.5.0.dev0"
+}

hf_model/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f
+size 32169626

hf_model/tokenizer_config.json

@@ -0,0 +1,74 @@
+{
+  "audio_token": "<|audio|>",
+  "backend": "tokenizers",
+  "boa_token": "<|audio>",
+  "boi_token": "<|image>",
+  "bos_token": "<bos>",
+  "eoa_token": "<audio|>",
+  "eoc_token": "<channel|>",
+  "eoi_token": "<image|>",
+  "eos_token": "<eos>",
+  "eot_token": "<turn|>",
+  "escape_token": "<|\"|>",
+  "etc_token": "<tool_call|>",
+  "etd_token": "<tool|>",
+  "etr_token": "<tool_response|>",
+  "extra_special_tokens": [
+    "<|video|>"
+  ],
+  "image_token": "<|image|>",
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "processor_class": "Gemma4Processor",
+  "response_schema": {
+    "type": "object",
+    "properties": {
+      "role": {
+        "const": "assistant"
+      },
+      "thinking": {
+        "type": "string"
+      },
+      "content": {
+        "type": "string"
+      },
+      "tool_calls": {
+        "x-regex-iterator": "<\\|tool_call>(.*?)<tool_call\\|>",
+        "type": "array",
+        "items": {
+          "type": "object",
+          "properties": {
+            "type": {
+              "const": "function"
+            },
+            "function": {
+              "type": "object",
+              "x-regex": "call\\:(?P<name>\\w+)(?P<arguments>\\{.*\\})",
+              "properties": {
+                "name": {
+                  "type": "string"
+                },
+                "arguments": {
+                  "type": "object",
+                  "x-parser": "gemma4-tool-call",
+                  "additionalProperties": {}
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "x-regex": "(\\<\\|channel\\>thought\\n(?P<thinking>.*?)\\<channel\\|\\>)?(?P<tool_calls>\\<\\|tool_call\\>.*\\<tool_call\\|\\>)?(?P<content>(?:(?!\\<turn\\|\\>)(?!\\<\\|tool_response\\>).)+)?(?:\\<turn\\|\\>|\\<\\|tool_response\\>)?"
+  },
+  "soc_token": "<|channel>",
+  "sot_token": "<|turn>",
+  "stc_token": "<|tool_call>",
+  "std_token": "<|tool>",
+  "str_token": "<|tool_response>",
+  "think_token": "<|think|>",
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>"
+}

model_config.json

@@ -0,0 +1,26 @@
+{
+  "model_name": "gemma4-e4b",
+  "architecture": "gemma4",
+  "hidden_size": 2560,
+  "num_hidden_layers": 42,
+  "num_attention_heads": 8,
+  "num_key_value_heads": 2,
+  "head_dim": 256,
+  "global_head_dim": 512,
+  "vocab_size": 262144,
+  "context_length": 2048,
+  "sliding_window": 512,
+  "per_layer_dim": 256,
+  "num_layers": 42,
+  "embed_scale": 50.59644256269407,
+  "per_layer_embed_scale": 16.0,
+  "per_layer_model_projection_scale": 0.01976423537605237,
+  "per_layer_input_scale": 0.7071067811865475,
+  "rms_norm_eps": 1e-06,
+  "bos_token_id": 2,
+  "eos_token_id": 1,
+  "final_logit_softcapping": 30.0,
+  "quantization": "int4",
+  "compute_units": "CPU_AND_NE",
+  "tokenizer_repo": "google/gemma-4-E4B-it"
+}

per_layer_norm_weight.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff67a6ed2e1ac597c28467653c4d67ecd4018f668b1d667af95e564539bd4c10
+size 512

per_layer_projection.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8889e468a6c7e0a43f6ea4a7df18435cf75a5e496415146790f9e0a8a9cd63f
+size 55050240

sin_full.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa9d6aa40ca6b4f02f2be7563be801bc8ed77bcbc06f3aeb46050587401f2b4e
+size 4194432

sin_sliding.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8a996e1172dbb9748de65e1b7cede880e07f89310719215c627cf26faebcd55
+size 2097280