init

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+.DS_Store

README.md

@@ -1,3 +1,84 @@
----
-license: mit
----
+# gemma-3-4b-it-qat-4bit-lite
+
+Optimized version of [gemma-3-4b-it-qat-4bit](https://huggingface.co/google/gemma-3-4b-it-qat-q4_0_4_8) for Apple Silicon edge devices. Reduces model size from 2.8 GB to 2.3 GB with lower runtime memory and significantly reduced thermal output, while preserving text and image understanding quality.
+
+## Optimizations Applied
+
+| Step | Optimization | Effect |
+|------|-------------|--------|
+| 1 | Vocabulary pruning 262K → 144K tokens | -170 MB disk, token_map remapping |
+| 2 | Vision fc2 bf16 → 4-bit (pad 4304 → 4352) | -191 MB disk |
+| 3 | Remove text layers 31, 32, 33 (34 → 31 layers) | -159 MB disk, faster inference |
+| 4 | Image resolution 896 → 672 | ~3x less vision attention compute |
+
+## Architecture
+
+```
+Text model:
+  vocab_size: 262,208 (token_map → 144,257 compact embeddings)
+  hidden_size: 2560
+  intermediate_size: 10240
+  num_hidden_layers: 31
+  num_attention_heads: 8 (GQA, 4 KV heads)
+  head_dim: 256
+  quantization: 4-bit, group_size=64
+
+Vision model (SigLIP):
+  hidden_size: 1152
+  intermediate_size: 4352 (padded from 4304, fc2 4-bit quantized)
+  num_hidden_layers: 27
+  image_size: 672
+  patch_size: 14
+  mm_tokens_per_image: 144
+```
+
+## Model Files
+
+| File | Size | Description |
+|------|------|-------------|
+| `model.safetensors` | 2.3 GB | All weights (language + vision) |
+| `config.json` | - | Model configuration with `vocab_pruning` metadata |
+
+## Requirements
+
+This model uses **token_map** for vocabulary pruning. The inference engine must:
+
+1. Read `vocab_pruning.compact_vocab_size` from config.json
+2. Initialize embedding layer with compact size (144,257) instead of original vocab size (262,208)
+3. Load `language_model.model.embed_tokens.token_map` (int32[262208]) from weights
+4. Remap token IDs before embedding lookup: `embedding(token_map[input_ids])`
+
+Standard `mlx_vlm.generate` works for models **without** vocab pruning. For this model, use the custom Swift CLI or a compatible inference engine.
+
+## Usage
+
+### Swift (gemma-cli with MLX Swift)
+
+```bash
+# Text generation
+swift run -c release gemma-cli /path/to/gemma-3-4b-it-qat-4bit-lite \
+  --prompt "Hello, how are you?" --max-tokens 100 --temperature 0.0
+
+# Image understanding
+swift run -c release gemma-cli /path/to/gemma-3-4b-it-qat-4bit-lite \
+  --image photo.jpg \
+  --prompt "Describe this image in detail." --max-tokens 200 --temperature 0.0
+```
+
+## Benchmarks (Apple Silicon)
+
+| Metric | Original | This Model |
+|--------|----------|------------|
+| Disk size | 2.8 GB | 2.3 GB |
+| Prompt speed (text) | 109 t/s | ~120 t/s |
+| Generation speed (text) | 90 t/s | ~110 t/s |
+| Image understanding | Correct | Correct |
+| Text quality | Perfect | Good |
+
+## Base Model
+
+[google/gemma-3-4b-it-qat-q4_0_4_8](https://huggingface.co/google/gemma-3-4b-it-qat-q4_0_4_8)
+
+## License
+
+Same as the base model. See [Gemma Terms of Use](https://ai.google.dev/gemma/terms).

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<image_soft_token>": 262144
+}

chat_template.json

@@ -0,0 +1,3 @@
+{
+  "chat_template": "{{ bos_token }}\n{%- if messages[0]['role'] == 'system' -%}\n    {%- if messages[0]['content'] is string -%}\n        {%- set first_user_prefix = messages[0]['content'] + '\n\n' -%}\n    {%- else -%}\n        {%- set first_user_prefix = messages[0]['content'][0]['text'] + '\n\n' -%}\n    {%- endif -%}\n    {%- set loop_messages = messages[1:] -%}\n{%- else -%}\n    {%- set first_user_prefix = \"\" -%}\n    {%- set loop_messages = messages -%}\n{%- endif -%}\n{%- for message in loop_messages -%}\n    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}\n        {{ raise_exception(\"Conversation roles must alternate user/assistant/user/assistant/...\") }}\n    {%- endif -%}\n    {%- if (message['role'] == 'assistant') -%}\n        {%- set role = \"model\" -%}\n    {%- else -%}\n        {%- set role = message['role'] -%}\n    {%- endif -%}\n    {{ '<start_of_turn>' + role + '\n' + (first_user_prefix if loop.first else \"\") }}\n    {%- if message['content'] is string -%}\n        {{ message['content'] | trim }}\n    {%- elif message['content'] is iterable -%}\n        {%- for item in message['content'] -%}\n            {%- if item['type'] == 'image' -%}\n                {{ '<start_of_image>' }}\n            {%- elif item['type'] == 'text' -%}\n                {{ item['text'] | trim }}\n            {%- endif -%}\n        {%- endfor -%}\n    {%- else -%}\n        {{ raise_exception(\"Invalid content type\") }}\n    {%- endif -%}\n    {{ '<end_of_turn>\n' }}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n    {{'<start_of_turn>model\n'}}\n{%- endif -%}\n"
+}

config.json

@@ -0,0 +1,308 @@
+{
+    "_attn_implementation_autoset": false,
+    "add_cross_attention": false,
+    "architectures": [
+        "Gemma3ForConditionalGeneration"
+    ],
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "boi_token_index": 255999,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eoi_token_index": 256000,
+    "eos_token_id": [
+        1,
+        106
+    ],
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+    },
+    "image_token_index": 262144,
+    "initializer_range": 0.02,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "mm_tokens_per_image": 144,
+    "model_type": "gemma3",
+    "no_repeat_ngram_size": 0,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "quantization": {
+        "group_size": 64,
+        "bits": 4
+    },
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "text_config": {
+        "return_dict": true,
+        "output_hidden_states": false,
+        "output_attentions": false,
+        "torchscript": false,
+        "torch_dtype": "bfloat16",
+        "use_bfloat16": false,
+        "tf_legacy_loss": false,
+        "pruned_heads": {},
+        "tie_word_embeddings": true,
+        "chunk_size_feed_forward": 0,
+        "is_encoder_decoder": false,
+        "is_decoder": false,
+        "cross_attention_hidden_size": null,
+        "add_cross_attention": false,
+        "tie_encoder_decoder": false,
+        "max_length": 20,
+        "min_length": 0,
+        "do_sample": false,
+        "early_stopping": false,
+        "num_beams": 1,
+        "num_beam_groups": 1,
+        "diversity_penalty": 0.0,
+        "temperature": 1.0,
+        "top_k": 50,
+        "top_p": 1.0,
+        "typical_p": 1.0,
+        "repetition_penalty": 1.0,
+        "length_penalty": 1.0,
+        "no_repeat_ngram_size": 0,
+        "encoder_no_repeat_ngram_size": 0,
+        "bad_words_ids": null,
+        "num_return_sequences": 1,
+        "output_scores": false,
+        "return_dict_in_generate": false,
+        "forced_bos_token_id": null,
+        "forced_eos_token_id": null,
+        "remove_invalid_values": false,
+        "exponential_decay_length_penalty": null,
+        "suppress_tokens": null,
+        "begin_suppress_tokens": null,
+        "architectures": null,
+        "finetuning_task": null,
+        "id2label": {
+            "0": "LABEL_0",
+            "1": "LABEL_1"
+        },
+        "label2id": {
+            "LABEL_0": 0,
+            "LABEL_1": 1
+        },
+        "tokenizer_class": null,
+        "prefix": null,
+        "bos_token_id": 2,
+        "pad_token_id": 0,
+        "eos_token_id": 1,
+        "sep_token_id": null,
+        "decoder_start_token_id": null,
+        "task_specific_params": null,
+        "problem_type": null,
+        "_name_or_path": "",
+        "_attn_implementation_autoset": false,
+        "model_type": "gemma3_text",
+        "vocab_size": 262208,
+        "max_position_embeddings": 131072,
+        "hidden_size": 2560,
+        "intermediate_size": 10240,
+        "num_hidden_layers": 31,
+        "num_attention_heads": 8,
+        "head_dim": 256,
+        "num_key_value_heads": 4,
+        "initializer_range": 0.02,
+        "rms_norm_eps": 1e-06,
+        "use_cache": true,
+        "rope_theta": 1000000,
+        "attention_bias": false,
+        "attention_dropout": 0.0,
+        "hidden_activation": "gelu_pytorch_tanh",
+        "query_pre_attn_scalar": 256,
+        "sliding_window": 1024,
+        "final_logit_softcapping": null,
+        "attn_logit_softcapping": null,
+        "cache_implementation": "hybrid",
+        "rope_local_base_freq": 10000,
+        "sliding_window_pattern": 6,
+        "rope_scaling": {
+            "factor": 8.0,
+            "rope_type": "linear"
+        }
+    },
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torchscript": false,
+    "transformers_version": "4.51.3",
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "vision_config": {
+        "return_dict": true,
+        "output_hidden_states": false,
+        "output_attentions": false,
+        "torchscript": false,
+        "torch_dtype": "bfloat16",
+        "use_bfloat16": false,
+        "tf_legacy_loss": false,
+        "pruned_heads": {},
+        "tie_word_embeddings": true,
+        "chunk_size_feed_forward": 0,
+        "is_encoder_decoder": false,
+        "is_decoder": false,
+        "cross_attention_hidden_size": null,
+        "add_cross_attention": false,
+        "tie_encoder_decoder": false,
+        "max_length": 20,
+        "min_length": 0,
+        "do_sample": false,
+        "early_stopping": false,
+        "num_beams": 1,
+        "num_beam_groups": 1,
+        "diversity_penalty": 0.0,
+        "temperature": 1.0,
+        "top_k": 50,
+        "top_p": 1.0,
+        "typical_p": 1.0,
+        "repetition_penalty": 1.0,
+        "length_penalty": 1.0,
+        "no_repeat_ngram_size": 0,
+        "encoder_no_repeat_ngram_size": 0,
+        "bad_words_ids": null,
+        "num_return_sequences": 1,
+        "output_scores": false,
+        "return_dict_in_generate": false,
+        "forced_bos_token_id": null,
+        "forced_eos_token_id": null,
+        "remove_invalid_values": false,
+        "exponential_decay_length_penalty": null,
+        "suppress_tokens": null,
+        "begin_suppress_tokens": null,
+        "architectures": null,
+        "finetuning_task": null,
+        "id2label": {
+            "0": "LABEL_0",
+            "1": "LABEL_1"
+        },
+        "label2id": {
+            "LABEL_0": 0,
+            "LABEL_1": 1
+        },
+        "tokenizer_class": null,
+        "prefix": null,
+        "bos_token_id": null,
+        "pad_token_id": null,
+        "eos_token_id": null,
+        "sep_token_id": null,
+        "decoder_start_token_id": null,
+        "task_specific_params": null,
+        "problem_type": null,
+        "_name_or_path": "",
+        "_attn_implementation_autoset": false,
+        "model_type": "siglip_vision_model",
+        "vision_use_head": false,
+        "hidden_size": 1152,
+        "intermediate_size": 4352,
+        "num_hidden_layers": 27,
+        "num_attention_heads": 16,
+        "num_channels": 3,
+        "patch_size": 14,
+        "image_size": 672,
+        "attention_dropout": 0.0,
+        "layer_norm_eps": 1e-06,
+        "hidden_act": "gelu_pytorch_tanh"
+    },
+    "vocab_pruning": {
+        "original_vocab_size": 262208,
+        "compact_vocab_size": 144257,
+        "keep_ids_count": 144256,
+        "has_token_map": true,
+        "token_map_key": "language_model.model.embed_tokens.token_map"
+    },
+    "vision_fc2_quantization": {
+        "bits": 4,
+        "group_size": 64,
+        "quantized_layers": 27,
+        "original_intermediate_size": 4304,
+        "padded_intermediate_size": 4352,
+        "pad_amount": 48,
+        "saved_mb": 191.6
+    },
+    "text_layer_pruning": {
+        "removed_layers": [
+            31,
+            32,
+            33
+        ],
+        "keep_layers": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30
+        ],
+        "old_num_layers": 34,
+        "new_num_layers": 31
+    },
+    "resolution_reduction": {
+        "original_image_size": 896,
+        "target_image_size": 672,
+        "original_patches": 4096,
+        "new_patches": 2304,
+        "original_mm_tokens": 256,
+        "new_mm_tokens": 144
+    }
+}

generation_config.json

@@ -0,0 +1,11 @@
+{
+  "cache_implementation": "hybrid",
+  "do_sample": true,
+  "eos_token_id": [
+    1,
+    106
+  ],
+  "top_k": 64,
+  "top_p": 0.95,
+  "transformers_version": "4.52.0.dev0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90b33a6839e1b2bac46a3278405ec300415f274c2cbf7a62c0be9d9630f9dbdd
+size 2475476883

preprocessor_config.json

@@ -0,0 +1,29 @@
+{
+  "do_convert_rgb": null,
+  "do_normalize": true,
+  "do_pan_and_scan": null,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "Gemma3ImageProcessor",
+  "image_seq_length": 144,
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "pan_and_scan_max_num_crops": null,
+  "pan_and_scan_min_crop_size": null,
+  "pan_and_scan_min_ratio_to_activate": null,
+  "processor_class": "Gemma3Processor",
+  "resample": 2,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 672,
+    "width": 672
+  }
+}

processor_config.json

@@ -0,0 +1,4 @@
+{
+  "image_seq_length": 256,
+  "processor_class": "Gemma3Processor"
+}

special_tokens_map.json

@@ -0,0 +1,33 @@
+{
+  "boi_token": "<start_of_image>",
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eoi_token": "<end_of_image>",
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "image_token": "<image_soft_token>",
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4667f2089529e8e7657cfb6d1c19910ae71ff5f28aa7ab2ff2763330affad795
+size 33384568

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1299c11d7cf632ef3b4e11937501358ada021bbdf7c47638d13c0ee982f2e79c
+size 4689074