Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

.meta.json

@@ -0,0 +1,4 @@
+{
+  "torch": "2.12.0",
+  "transformers": "5.9.0"
+}

README.md

@@ -0,0 +1,223 @@
+---
+library_name: transformers
+base_model:
+- google/gemma-4-31B-it-assistant
+---
+
+This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from [google/gemma-4-31B-it-assistant](https://huggingface.co/google/gemma-4-31B-it-assistant).
+
+| File path | Size |
+|------|------|
+| model.safetensors | 4.3MB |
+
+
+### Example usage:
+
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoProcessor, AutoModelForMultimodalLM
+
+model_id = "tiny-random/gemma-4-assistant"
+target_model_id = "tiny-random/gemma-4-moe"
+
+processor = AutoProcessor.from_pretrained(target_model_id)
+target_model = AutoModelForMultimodalLM.from_pretrained(
+    target_model_id,
+    dtype=torch.bfloat16,
+    device_map="auto",
+)
+assistant_model = AutoModelForCausalLM.from_pretrained(
+    model_id,
+    dtype=torch.bfloat16,
+    device_map="auto",
+)
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "image",
+                "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG",
+            },
+            {"type": "text", "text": "What is shown in this image?"},
+        ],
+    },
+    # {
+    #     "role": "assistant",
+    #     "content": [{"type": "text", "text": "Dummy response for image"}],
+    # },
+    # {
+    #     "role": "user",
+    #     "content": [
+    #         {
+    #             "type": "video",
+    #             "video": "https://github.com/bebechien/gemma/raw/refs/heads/main/videos/ForBiggerBlazes.mp4",
+    #         },
+    #         {"type": "text", "text": "Describe this video."},
+    #     ],
+    # },
+]
+inputs = processor.apply_chat_template(
+    messages,
+    tokenize=True,
+    return_dict=True,
+    return_tensors="pt",
+    add_generation_prompt=True,
+).to(target_model.device)
+input_len = inputs["input_ids"].shape[-1]
+print("input_len:", input_len)
+outputs = target_model.generate(
+    **inputs,
+    assistant_model=assistant_model,
+    max_new_tokens=32,
+)
+response = processor.decode(outputs[0], skip_special_tokens=False)
+response = response.replace("<|image|>", "I")
+response = response.replace("<|video|>", "V")
+print(response)
+```
+
+### Codes to create this repo:
+
+<details>
+<summary>Click to expand</summary>
+
+```python
+import json
+from pathlib import Path
+
+import torch
+from huggingface_hub import file_exists, hf_hub_download
+
+from transformers import (
+    AutoConfig,
+    AutoModelForCausalLM,
+    AutoProcessor,
+    AutoTokenizer,
+    Gemma4AssistantForCausalLM,
+    Gemma4ForConditionalGeneration,
+    GenerationConfig,
+    set_seed,
+)
+
+source_model_id = "google/gemma-4-31B-it-assistant"
+save_folder = "/tmp/tiny-random/gemma-4-assistant"
+
+processor = AutoProcessor.from_pretrained(source_model_id)
+processor.save_pretrained(save_folder)
+
+with open(
+    hf_hub_download(source_model_id, filename="config.json", repo_type="model"),
+    "r",
+    encoding="utf-8",
+) as f:
+    config_json = json.load(f)
+
+config_json["backbone_hidden_size"] = 8
+config_json["text_config"].update(
+    {
+        "global_head_dim": 64,
+        "head_dim": 32,
+        "hidden_size": 8,
+        "intermediate_size": 64,
+        "layer_types": [
+            "sliding_attention",
+            "sliding_attention",
+            "sliding_attention",
+            "full_attention",
+        ],
+        "moe_intermediate_size": 32,
+        "num_attention_heads": 8,
+        "num_hidden_layers": 4,
+        "num_key_value_heads": 4,
+    }
+)
+
+with open(f"{save_folder}/config.json", "w", encoding="utf-8") as f:
+    json.dump(config_json, f, indent=2)
+
+config = AutoConfig.from_pretrained(
+    save_folder,
+    trust_remote_code=True,
+)
+print(config)
+
+torch.set_default_dtype(torch.bfloat16)
+model = Gemma4AssistantForCausalLM(config)
+torch.set_default_dtype(torch.float32)
+if file_exists(filename="generation_config.json", repo_id=source_model_id, repo_type="model"):
+    model.generation_config = GenerationConfig.from_pretrained(
+        source_model_id,
+        trust_remote_code=True,
+    )
+set_seed(42)
+model = model.cpu()
+all_numels = sum(p.numel() for p in model.parameters())
+with torch.no_grad():
+    for name, p in sorted(model.named_parameters()):
+        torch.nn.init.normal_(p, 0, 0.2)
+        print(name, p.shape, f"{p.numel() / all_numels * 100: .4f}%")
+model.save_pretrained(save_folder)
+```
+
+</details>
+
+### Printing the model:
+
+<details><summary>Click to expand</summary>
+
+```text
+Gemma4AssistantForCausalLM(
+  (model): Gemma4TextModel(
+    (embed_tokens): Gemma4TextScaledWordEmbedding(262144, 8, padding_idx=0)
+    (layers): ModuleList(
+      (0-2): 3 x Gemma4TextDecoderLayer(
+        (self_attn): Gemma4TextAttention(
+          (q_proj): Linear(in_features=8, out_features=256, bias=False)
+          (q_norm): Gemma4RMSNorm()
+          (o_proj): Linear(in_features=256, out_features=8, bias=False)
+        )
+        (mlp): Gemma4TextMLP(
+          (gate_proj): Linear(in_features=8, out_features=64, bias=False)
+          (up_proj): Linear(in_features=8, out_features=64, bias=False)
+          (down_proj): Linear(in_features=64, out_features=8, bias=False)
+          (act_fn): GELUTanh()
+        )
+        (input_layernorm): Gemma4RMSNorm()
+        (post_attention_layernorm): Gemma4RMSNorm()
+        (pre_feedforward_layernorm): Gemma4RMSNorm()
+        (post_feedforward_layernorm): Gemma4RMSNorm()
+      )
+      (3): Gemma4TextDecoderLayer(
+        (self_attn): Gemma4TextAttention(
+          (q_proj): Linear(in_features=8, out_features=512, bias=False)
+          (q_norm): Gemma4RMSNorm()
+          (o_proj): Linear(in_features=512, out_features=8, bias=False)
+        )
+        (mlp): Gemma4TextMLP(
+          (gate_proj): Linear(in_features=8, out_features=64, bias=False)
+          (up_proj): Linear(in_features=8, out_features=64, bias=False)
+          (down_proj): Linear(in_features=64, out_features=8, bias=False)
+          (act_fn): GELUTanh()
+        )
+        (input_layernorm): Gemma4RMSNorm()
+        (post_attention_layernorm): Gemma4RMSNorm()
+        (pre_feedforward_layernorm): Gemma4RMSNorm()
+        (post_feedforward_layernorm): Gemma4RMSNorm()
+      )
+    )
+    (norm): Gemma4RMSNorm()
+    (rotary_emb): Gemma4TextRotaryEmbedding()
+  )
+  (lm_head): Linear(in_features=8, out_features=262144, bias=False)
+  (pre_projection): Linear(in_features=16, out_features=8, bias=False)
+  (post_projection): Linear(in_features=8, out_features=8, bias=False)
+)
+```
+
+</details>
+
+### Test environment:
+
+- torch: 2.12.0
+- transformers: 5.9.0

config.json

@@ -0,0 +1,88 @@
+{
+  "architectures": [
+    "Gemma4AssistantForCausalLM"
+  ],
+  "audio_token_id": 258881,
+  "backbone_hidden_size": 8,
+  "boa_token_id": 256000,
+  "boi_token_id": 255999,
+  "centroid_intermediate_top_k": 32,
+  "dtype": "bfloat16",
+  "eoa_token_id": 258883,
+  "eoi_token_id": 258882,
+  "image_token_id": 258880,
+  "model_type": "gemma4_assistant",
+  "num_centroids": 2048,
+  "text_config": {
+    "_name_or_path": "",
+    "architectures": null,
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "attention_k_eq_v": true,
+    "bos_token_id": 2,
+    "chunk_size_feed_forward": 0,
+    "dtype": "bfloat16",
+    "enable_moe_block": false,
+    "eos_token_id": 1,
+    "final_logit_softcapping": null,
+    "global_head_dim": 64,
+    "head_dim": 32,
+    "hidden_activation": "gelu_pytorch_tanh",
+    "hidden_size": 8,
+    "hidden_size_per_layer_input": 0,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 64,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_types": [
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 262144,
+    "model_type": "gemma4_text",
+    "moe_intermediate_size": 32,
+    "num_attention_heads": 8,
+    "num_experts": null,
+    "num_global_key_value_heads": 4,
+    "num_hidden_layers": 4,
+    "num_key_value_heads": 4,
+    "num_kv_shared_layers": 4,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "pad_token_id": 0,
+    "problem_type": null,
+    "return_dict": true,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "full_attention": {
+        "partial_rotary_factor": 0.25,
+        "rope_theta": 1000000.0,
+        "rope_type": "proportional"
+      },
+      "sliding_attention": {
+        "rope_theta": 10000.0,
+        "rope_type": "default"
+      }
+    },
+    "sliding_window": 1024,
+    "tie_word_embeddings": true,
+    "top_k_experts": null,
+    "use_bidirectional_attention": null,
+    "use_cache": true,
+    "use_double_wide_mlp": false,
+    "vocab_size": 262144,
+    "vocab_size_per_layer_input": 0
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.9.0",
+  "use_ordered_embeddings": false
+}

generation_config.json

@@ -0,0 +1,18 @@
+{
+  "bos_token_id": 2,
+  "do_sample": true,
+  "eos_token_id": [
+    1,
+    106,
+    50
+  ],
+  "is_assistant": true,
+  "num_assistant_tokens": 6,
+  "num_assistant_tokens_schedule": "constant",
+  "pad_token_id": 0,
+  "temperature": 1.0,
+  "top_k": 64,
+  "top_p": 0.95,
+  "transformers_version": "5.9.0",
+  "trust_remote_code": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2d24ea45870ac8e6a035df98973bb49244814eabe093b8fbf5d85ab040d6348
+size 4253552

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75a6583c1a418e2bbd79c60d95d28e0f5bf549ad3f2990b5bdb5238c6c2bf70c
+size 32169440

tokenizer_config.json

@@ -0,0 +1,52 @@
+{
+  "audio_token": "<|audio|>",
+  "backend": "tokenizers",
+  "boa_token": "<|audio>",
+  "boi_token": "<|image>",
+  "bos_token": "<bos>",
+  "eoa_token": "<audio|>",
+  "eoc_token": "<channel|>",
+  "eoi_token": "<image|>",
+  "eos_token": "<eos>",
+  "eot_token": "<turn|>",
+  "escape_token": "<|\"|>",
+  "etc_token": "<tool_call|>",
+  "etd_token": "<tool|>",
+  "etr_token": "<tool_response|>",
+  "extra_special_tokens": [],
+  "image_token": "<|image|>",
+  "is_local": false,
+  "local_files_only": false,
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "model_specific_special_tokens": {
+    "audio_token": "<|audio|>",
+    "boa_token": "<|audio>",
+    "boi_token": "<|image>",
+    "eoa_token": "<audio|>",
+    "eoc_token": "<channel|>",
+    "eoi_token": "<image|>",
+    "eot_token": "<turn|>",
+    "escape_token": "<|\"|>",
+    "etc_token": "<tool_call|>",
+    "etd_token": "<tool|>",
+    "etr_token": "<tool_response|>",
+    "image_token": "<|image|>",
+    "soc_token": "<|channel>",
+    "sot_token": "<|turn>",
+    "stc_token": "<|tool_call>",
+    "std_token": "<|tool>",
+    "str_token": "<|tool_response>",
+    "think_token": "<|think|>"
+  },
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "soc_token": "<|channel>",
+  "sot_token": "<|turn>",
+  "stc_token": "<|tool_call>",
+  "std_token": "<|tool>",
+  "str_token": "<|tool_response>",
+  "think_token": "<|think|>",
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>"
+}