Upload task output c08fd258-a4d3-438c-a1aa-a39a294c3c3c

chat_template.jinja

@@ -1 +0,0 @@
-{% for message in messages %}{% set role = message['role'] | lower %}{% if role == 'user' %}{% set role = 'HUMAN' %}{% endif %}{% set role = role | upper %}{{ '<role>' + role + '</role>' + message['content'] }}{% if role == 'ASSISTANT' %}{{ '<|endoftext|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<role>ASSISTANT</role>' }}{% endif %}

config.json

@@ -5,17 +5,15 @@
   "attention_bias": false,
   "attention_dropout": 0.0,
   "attn_output_gate": true,
-  "auto_map": {
-    "AutoConfig": "configuration_qwen3_5.QuasarConfig",
-    "AutoModelForCausalLM": "modeling_qwen3_5.QuasarForCausalLM"
-  },
-  "bos_token_id": null,
   "dtype": "bfloat16",
-  "eos_token_id": 248044,
   "full_attention_interval": 4,
   "head_dim": 256,
   "hidden_act": "silu",
   "hidden_size": 4096,
+  "ignore_keys_at_rope_validation": [
+    "mrope_interleaved",
+    "mrope_section"
+  ],
   "initializer_range": 0.02,
   "intermediate_size": 12288,
   "layer_types": [
@@ -66,7 +64,6 @@
   "num_attention_heads": 16,
   "num_hidden_layers": 32,
   "num_key_value_heads": 4,
-  "pad_token_id": 248044,
   "partial_rotary_factor": 0.25,
   "rms_norm_eps": 1e-06,
   "rope_parameters": {
@@ -80,8 +77,8 @@
     "rope_theta": 10000000,
     "rope_type": "default"
   },
-  "tie_word_embeddings": false,
-  "transformers_version": "5.9.0",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
   "use_cache": false,
   "use_gla": true,
   "use_nope": true,

configuration_qwen3_5.py

@@ -1,311 +0,0 @@
-#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
-#           This file was automatically generated from src/transformers/models/quasar/modular_quasar.py.
-#               Do NOT edit this file manually as any edits will be overwritten by the generation of
-#             the file from the modular. If any change should be done, please apply the change to the
-#                          modular_quasar.py file directly. One of our CI enforces this.
-#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
-# Copyright 2025 The Qwen Team and The HuggingFace Inc. team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from transformers.configuration_utils import PreTrainedConfig, layer_type_validation
-from transformers.modeling_rope_utils import RopeParameters
-
-
-class QuasarTextConfig(PreTrainedConfig):
-    r"""
-    This is the configuration class to store the configuration of a [`QuasarTextModel`]. It is used to instantiate a
-    Quasar model according to the specified arguments, defining the model architecture.
-    Instantiating a configuration with the defaults will yield a similar configuration to that of
-    Qwen3.5-9B-Instruct [Qwen/Qwen3.5-9B-Instruct](https://huggingface.co/Qwen/Qwen3.5-9B-Instruct).
-
-    Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
-    documentation from [`PreTrainedConfig`] for more information.
-
-
-    Args:
-        vocab_size (`int`, *optional*, defaults to 248320):
-            Vocabulary size of the model. Defines the number of different tokens that can be represented by the
-            `inputs_ids`.
-        hidden_size (`int`, *optional*, defaults to 4096):
-            Dimension of the hidden representations.
-        intermediate_size (`int`, *optional*, defaults to 12288):
-            Dimension of the MLP representations.
-        num_hidden_layers (`int`, *optional*, defaults to 32):
-            Number of hidden layers in the Transformer encoder.
-        num_attention_heads (`int`, *optional*, defaults to 16):
-            Number of attention heads for each attention layer in the Transformer encoder.
-        num_key_value_heads (`int`, *optional*, defaults to 4):
-            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
-            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
-            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
-            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
-            by meanpooling all the original heads within that group. For more details checkout [this
-            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `32`.
-        hidden_act (`str`, *optional*, defaults to `"silu"`):
-            The non-linear activation function in the decoder.
-        max_position_embeddings (`int`, *optional*, defaults to 32768):
-            The maximum sequence length that this model might ever be used with.
-        initializer_range (`float`, *optional*, defaults to 0.02):
-            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
-        rms_norm_eps (`float`, *optional*, defaults to 1e-06):
-            The epsilon used by the rms normalization layers.
-        use_cache (`bool`, *optional*, defaults to `True`):
-            Whether or not the model should return the last key/values attentions (not used by all models). Only
-            relevant if `config.is_decoder=True`.
-        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
-            Whether the model's input and output word embeddings should be tied.
-        rope_parameters (`RopeParameters`, *optional*):
-            Dictionary containing the configuration parameters for the RoPE embeddings. The dictionary should contain
-            a value for `rope_theta` and optionally parameters used for scaling in case you want to use RoPE
-            with longer `max_position_embeddings`.
-        attention_bias (`bool`, *optional*, defaults to `False`):
-            Whether to use a bias in the query, key, value and output projection layers during self-attention.
-        attention_dropout (`float`, *optional*, defaults to 0.0):
-            The dropout ratio for the attention probabilities.
-        head_dim (`int`, *optional*, defaults to 256):
-            Projection weights dimension in multi-head attention.
-        linear_conv_kernel_dim (`int`, *optional*, defaults to 4):
-            Kernel size of the convolution used in linear attention layers.
-        linear_key_head_dim (`int`, *optional*, defaults to 128):
-            Dimension of each key head in linear attention.
-        linear_value_head_dim (`int`, *optional*, defaults to 128):
-            Dimension of each value head in linear attention.
-        linear_num_key_heads (`int`, *optional*, defaults to 16):
-            Number of key heads used in linear attention layers.
-        linear_num_value_heads (`int`, *optional*, defaults to 32):
-            Number of value heads used in linear attention layers.
-        layer_types (`list[str]`, *optional*):
-            Types of each layer (attention or linear).
-        pad_token_id (`int`, *optional*):
-            Padding token id.
-        bos_token_id (`int`, *optional*):
-            Beginning of stream token id.
-        eos_token_id (`int`, *optional*):
-            End of stream token id.
-
-    ```python
-    >>> from transformers import QuasarTextModel, QuasarTextConfig
-
-    >>> # Initializing a Qwen3.5 style configuration
-    >>> configuration =  QuasarTextConfig()
-
-    >>> # Initializing a model from the Qwen3.5-9B style configuration
-    >>> model = QuasarTextModel(configuration)
-
-    >>> # Accessing the model configuration
-    >>> configuration = model.config
-    ```
-    """
-
-    model_type = "quasar_text"
-    keys_to_ignore_at_inference = ["past_key_values"]
-
-    base_model_tp_plan = {
-        "layers.*.self_attn.q_proj": "colwise",
-        "layers.*.self_attn.k_proj": "colwise",
-        "layers.*.self_attn.v_proj": "colwise",
-        "layers.*.self_attn.o_proj": "rowwise",
-        "layers.*.mlp.gate_proj": "colwise",
-        "layers.*.mlp.up_proj": "colwise",
-        "layers.*.mlp.down_proj": "rowwise",
-    }
-    base_model_pp_plan = {
-        "embed_tokens": (["input_ids"], ["inputs_embeds"]),
-        "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
-        "norm": (["hidden_states"], ["hidden_states"]),
-    }
-    base_config_key = "text_config"
-
-    def __init__(
-        self,
-        vocab_size=248320,
-        hidden_size=4096,
-        intermediate_size=12288,
-        num_hidden_layers=32,
-        num_attention_heads=16,
-        num_key_value_heads=4,
-        hidden_act="silu",
-        max_position_embeddings=32768,
-        initializer_range=0.02,
-        rms_norm_eps=1e-6,
-        use_cache=True,
-        tie_word_embeddings=False,
-        rope_parameters: RopeParameters | dict[str, RopeParameters] | None = None,
-        attention_bias=False,
-        attention_dropout=0.0,
-        head_dim=256,
-        linear_conv_kernel_dim=4,
-        linear_key_head_dim=128,
-        linear_value_head_dim=128,
-        linear_num_key_heads=16,
-        linear_num_value_heads=32,
-        layer_types=None,
-        pad_token_id: int | None = None,
-        bos_token_id: int | None = None,
-        eos_token_id: int | None = None,
-        use_gla: bool = False,
-        use_nope: bool = False,
-        **kwargs,
-    ):
-        kwargs["ignore_keys_at_rope_validation"] = {"mrope_section", "mrope_interleaved"}
-        self.pad_token_id = pad_token_id
-        self.bos_token_id = bos_token_id
-        self.eos_token_id = eos_token_id
-        self.tie_word_embeddings = tie_word_embeddings
-        self.vocab_size = vocab_size
-        self.max_position_embeddings = max_position_embeddings
-        self.hidden_size = hidden_size
-        self.intermediate_size = intermediate_size
-        self.num_hidden_layers = num_hidden_layers
-        self.num_attention_heads = num_attention_heads
-        self.num_key_value_heads = num_key_value_heads
-        self.hidden_act = hidden_act
-        self.initializer_range = initializer_range
-        self.rms_norm_eps = rms_norm_eps
-        self.use_cache = use_cache
-        self.attention_bias = attention_bias
-        self.attention_dropout = attention_dropout
-        self.head_dim = head_dim
-        self.rope_parameters = rope_parameters
-        self.use_gla = use_gla
-        self.use_nope = use_nope
-        kwargs.setdefault("partial_rotary_factor", 0.25)  # assign default for BC
-
-        self.layer_types = layer_types
-        if self.layer_types is None:
-            interval_pattern = kwargs.get("full_attention_interval", 4)
-            self.layer_types = [
-                "linear_attention" if bool((i + 1) % interval_pattern) else "full_attention"
-                for i in range(self.num_hidden_layers)
-            ]
-        layer_type_validation(self.layer_types, self.num_hidden_layers)
-
-        # linear attention part
-        self.linear_conv_kernel_dim = linear_conv_kernel_dim
-        self.linear_key_head_dim = linear_key_head_dim
-        self.linear_value_head_dim = linear_value_head_dim
-        self.linear_num_key_heads = linear_num_key_heads
-        self.linear_num_value_heads = linear_num_value_heads
-        super().__init__(**kwargs)
-
-
-class QuasarVisionConfig(PreTrainedConfig):
-    model_type = "quasar"
-    base_config_key = "vision_config"
-
-    def __init__(
-        self,
-        depth=27,
-        hidden_size=1152,
-        hidden_act="gelu_pytorch_tanh",
-        intermediate_size=4304,
-        num_heads=16,
-        in_channels=3,
-        patch_size=16,
-        spatial_merge_size=2,
-        temporal_patch_size=2,
-        out_hidden_size=3584,
-        num_position_embeddings=2304,
-        initializer_range=0.02,
-        **kwargs,
-    ):
-        super().__init__(**kwargs)
-
-        self.depth = depth
-        self.hidden_size = hidden_size
-        self.hidden_act = hidden_act
-        self.intermediate_size = intermediate_size
-        self.num_heads = num_heads
-        self.in_channels = in_channels
-        self.patch_size = patch_size
-        self.spatial_merge_size = spatial_merge_size
-        self.temporal_patch_size = temporal_patch_size
-        self.out_hidden_size = out_hidden_size
-        self.num_position_embeddings = num_position_embeddings
-        self.initializer_range = initializer_range
-
-
-class QuasarConfig(PreTrainedConfig):
-    r"""
-    This is the configuration class to store the configuration of a [`QuasarModel`]. It is used to instantiate a
-    Qwen3.5 model according to the specified arguments, defining the model architecture. Instantiating a configuration
-    with the defaults will yield a similar configuration to that of
-    Qwen3.5-9B-Instruct [Qwen/Qwen3.5-9B-Instruct](https://huggingface.co/Qwen/Qwen3.5-9B-Instruct).
-
-    Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
-    documentation from [`PreTrainedConfig`] for more information.
-
-
-    Args:
-        text_config (`Union[PreTrainedConfig, dict]`, *optional*, defaults to `QuasarTextConfig`):
-            The config object or dictionary of the text backbone.
-        vision_config (`Union[PreTrainedConfig, dict]`,  *optional*, defaults to `QuasarVisionConfig`):
-            The config object or dictionary of the vision backbone.
-        image_token_id (`int`, *optional*, defaults to 248056):
-            The image token index to encode the image prompt.
-        video_token_id (`int`, *optional*, defaults to 248057):
-            The video token index to encode the image prompt.
-        vision_start_token_id (`int`, *optional*, defaults to 248053):
-            The start token index to encode the image prompt.
-        vision_end_token_id (`int`, *optional*, defaults to 248054):
-            The end token index to encode the image prompt.
-        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
-            Whether to tie the word embeddings.
-
-    ```python
-    >>> from transformers import QuasarForConditionalGeneration, QuasarConfig
-
-    >>> # Initializing a Qwen3.5 style configuration
-    >>> configuration = QuasarConfig()
-
-    >>> # Initializing a model from the Qwen3.5-9B style configuration
-    >>> model = QuasarForConditionalGeneration(configuration)
-
-    >>> # Accessing the model configuration
-    >>> configuration = model.config
-    ```"""
-
-    model_type = "quasar"
-    sub_configs = {"vision_config": QuasarVisionConfig, "text_config": QuasarTextConfig}
-    keys_to_ignore_at_inference = ["past_key_values"]
-
-    def __init__(
-        self,
-        text_config=None,
-        vision_config=None,
-        image_token_id=248056,
-        video_token_id=248057,
-        vision_start_token_id=248053,
-        vision_end_token_id=248054,
-        tie_word_embeddings=False,
-        **kwargs,
-    ):
-        if isinstance(vision_config, dict):
-            self.vision_config = self.sub_configs["vision_config"](**vision_config)
-        elif vision_config is None:
-            self.vision_config = self.sub_configs["vision_config"]()
-
-        if isinstance(text_config, dict):
-            self.text_config = self.sub_configs["text_config"](**text_config)
-        elif text_config is None:
-            self.text_config = self.sub_configs["text_config"]()
-
-        self.image_token_id = image_token_id
-        self.video_token_id = video_token_id
-        self.vision_start_token_id = vision_start_token_id
-        self.vision_end_token_id = vision_end_token_id
-        self.tie_word_embeddings = tie_word_embeddings
-        super().__init__(**kwargs)
-
-
-__all__ = ["QuasarConfig", "QuasarTextConfig"]

generation_config.json

@@ -3,9 +3,6 @@
   "eos_token_id": [
     248044
   ],
-  "output_attentions": false,
-  "output_hidden_states": false,
   "pad_token_id": 248044,
-  "transformers_version": "5.3.0",
-  "use_cache": true
+  "transformers_version": "4.51.3"
 }

loss.txt

@@ -0,0 +1 @@
+624,1.4197497367858887

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:acf7a4341e0d4c746295b6ee23c3188f22a60b0c8f6e788e1e296bc5f840f5d3
-size 4942705592
+oid sha256:5f68af5bf61dd775b4a09bfb84e2c23fad6e6dbc251e12e5984b51e692635ef5
+size 4999223784

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5e4c06f828d1c2d2d253e5c683ea230b6f8dc3b38ca2fd319d4b1e85420f66b3
-size 4992975304
+oid sha256:361ad3efb23feba3b04643d6b15aea69ec4e8c9ac788d5a4625b32d1c18f7eb9
+size 4942784240

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a8e8d9db1cafd02c56e688e611ef0452761dd9429b9cf611f6fc19c8b225ed18
-size 4992975960
+oid sha256:dd46008c9ee7ee681180894acd1c405e1841738e440cbc35105b444bd13c8140
+size 4925866480

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fefaf66a9445c0fe99d8d6cb7cd167392af8f2676c22c8baf5577f20b118b394
-size 2275470288
+oid sha256:a3a93a50cb6c2bde5ddb36143d9e273d6d30afff887b3ab08f51a078c524a07e
+size 2336252704

model.safetensors.index.json

@@ -1,10 +1,9 @@
 {
   "metadata": {
-    "total_parameters": 8602037248,
     "total_size": 17204074496
   },
   "weight_map": {
-    "lm_head.weight": "model-00001-of-00004.safetensors",
+    "lm_head.weight": "model-00004-of-00004.safetensors",
     "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
     "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
     "model.layers.0.linear_attn.A_log": "model-00001-of-00004.safetensors",
@@ -97,97 +96,97 @@
     "model.layers.14.linear_attn.in_proj_a.weight": "model-00002-of-00004.safetensors",
     "model.layers.14.linear_attn.in_proj_b.weight": "model-00002-of-00004.safetensors",
     "model.layers.14.linear_attn.in_proj_qkv.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.linear_attn.in_proj_z.weight": "model-00003-of-00004.safetensors",
-    "model.layers.14.linear_attn.norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.14.linear_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.14.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.14.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.14.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.15.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.15.self_attn.gla.g_norm_swish_gate.weight": "model-00003-of-00004.safetensors",
-    "model.layers.15.self_attn.gla.g_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.15.self_attn.gla.gk_proj.0.weight": "model-00003-of-00004.safetensors",
-    "model.layers.15.self_attn.gla.gk_proj.1.bias": "model-00003-of-00004.safetensors",
-    "model.layers.15.self_attn.gla.gk_proj.1.weight": "model-00003-of-00004.safetensors",
-    "model.layers.15.self_attn.gla.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.15.self_attn.gla.o_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.15.self_attn.gla.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.15.self_attn.gla.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.16.linear_attn.A_log": "model-00003-of-00004.safetensors",
-    "model.layers.16.linear_attn.conv1d.weight": "model-00003-of-00004.safetensors",
-    "model.layers.16.linear_attn.dt_bias": "model-00003-of-00004.safetensors",
-    "model.layers.16.linear_attn.in_proj_a.weight": "model-00003-of-00004.safetensors",
-    "model.layers.16.linear_attn.in_proj_b.weight": "model-00003-of-00004.safetensors",
-    "model.layers.16.linear_attn.in_proj_qkv.weight": "model-00003-of-00004.safetensors",
-    "model.layers.16.linear_attn.in_proj_z.weight": "model-00003-of-00004.safetensors",
-    "model.layers.16.linear_attn.norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.16.linear_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.16.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.16.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.17.linear_attn.A_log": "model-00003-of-00004.safetensors",
-    "model.layers.17.linear_attn.conv1d.weight": "model-00003-of-00004.safetensors",
-    "model.layers.17.linear_attn.dt_bias": "model-00003-of-00004.safetensors",
-    "model.layers.17.linear_attn.in_proj_a.weight": "model-00003-of-00004.safetensors",
-    "model.layers.17.linear_attn.in_proj_b.weight": "model-00003-of-00004.safetensors",
-    "model.layers.17.linear_attn.in_proj_qkv.weight": "model-00003-of-00004.safetensors",
-    "model.layers.17.linear_attn.in_proj_z.weight": "model-00003-of-00004.safetensors",
-    "model.layers.17.linear_attn.norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.17.linear_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.linear_attn.A_log": "model-00003-of-00004.safetensors",
-    "model.layers.18.linear_attn.conv1d.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.linear_attn.dt_bias": "model-00003-of-00004.safetensors",
-    "model.layers.18.linear_attn.in_proj_a.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.linear_attn.in_proj_b.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.linear_attn.in_proj_qkv.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.linear_attn.in_proj_z.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.linear_attn.norm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.linear_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.14.linear_attn.in_proj_z.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.linear_attn.norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.linear_attn.out_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.gla.g_norm_swish_gate.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.gla.g_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.gla.gk_proj.0.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.gla.gk_proj.1.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.gla.gk_proj.1.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.gla.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.gla.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.gla.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.gla.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.linear_attn.A_log": "model-00002-of-00004.safetensors",
+    "model.layers.16.linear_attn.conv1d.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.linear_attn.dt_bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.linear_attn.in_proj_a.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.linear_attn.in_proj_b.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.linear_attn.in_proj_qkv.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.linear_attn.in_proj_z.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.linear_attn.norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.linear_attn.out_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.linear_attn.A_log": "model-00002-of-00004.safetensors",
+    "model.layers.17.linear_attn.conv1d.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.linear_attn.dt_bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.linear_attn.in_proj_a.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.linear_attn.in_proj_b.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.linear_attn.in_proj_qkv.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.linear_attn.in_proj_z.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.linear_attn.norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.linear_attn.out_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.linear_attn.A_log": "model-00002-of-00004.safetensors",
+    "model.layers.18.linear_attn.conv1d.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.linear_attn.dt_bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.linear_attn.in_proj_a.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.linear_attn.in_proj_b.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.linear_attn.in_proj_qkv.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.linear_attn.in_proj_z.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.linear_attn.norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.linear_attn.out_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
     "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
     "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
     "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
     "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
     "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.self_attn.gla.g_norm_swish_gate.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.self_attn.gla.g_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.self_attn.gla.gk_proj.0.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.self_attn.gla.gk_proj.1.bias": "model-00003-of-00004.safetensors",
-    "model.layers.19.self_attn.gla.gk_proj.1.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.self_attn.gla.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.self_attn.gla.o_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.self_attn.gla.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.self_attn.gla.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.gla.g_norm_swish_gate.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.gla.g_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.gla.gk_proj.0.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.gla.gk_proj.1.bias": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.gla.gk_proj.1.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.gla.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.gla.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.gla.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.gla.v_proj.weight": "model-00002-of-00004.safetensors",
     "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
     "model.layers.2.linear_attn.A_log": "model-00001-of-00004.safetensors",
     "model.layers.2.linear_attn.conv1d.weight": "model-00001-of-00004.safetensors",
     "model.layers.2.linear_attn.dt_bias": "model-00001-of-00004.safetensors",
     "model.layers.2.linear_attn.in_proj_a.weight": "model-00001-of-00004.safetensors",
     "model.layers.2.linear_attn.in_proj_b.weight": "model-00001-of-00004.safetensors",
-    "model.layers.2.linear_attn.in_proj_qkv.weight": "model-00002-of-00004.safetensors",
-    "model.layers.2.linear_attn.in_proj_z.weight": "model-00002-of-00004.safetensors",
-    "model.layers.2.linear_attn.norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.2.linear_attn.out_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.2.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.2.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.2.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.2.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.linear_attn.in_proj_qkv.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.linear_attn.in_proj_z.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.linear_attn.norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.linear_attn.out_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
     "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
     "model.layers.20.linear_attn.A_log": "model-00003-of-00004.safetensors",
     "model.layers.20.linear_attn.conv1d.weight": "model-00003-of-00004.safetensors",
@@ -282,136 +281,136 @@
     "model.layers.26.linear_attn.in_proj_z.weight": "model-00003-of-00004.safetensors",
     "model.layers.26.linear_attn.norm.weight": "model-00003-of-00004.safetensors",
     "model.layers.26.linear_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.26.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.26.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.26.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.27.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.27.self_attn.gla.g_norm_swish_gate.weight": "model-00004-of-00004.safetensors",
-    "model.layers.27.self_attn.gla.g_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.27.self_attn.gla.gk_proj.0.weight": "model-00004-of-00004.safetensors",
-    "model.layers.27.self_attn.gla.gk_proj.1.bias": "model-00004-of-00004.safetensors",
-    "model.layers.27.self_attn.gla.gk_proj.1.weight": "model-00004-of-00004.safetensors",
-    "model.layers.27.self_attn.gla.k_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.27.self_attn.gla.o_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.27.self_attn.gla.q_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.27.self_attn.gla.v_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.28.input_layernorm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.28.linear_attn.A_log": "model-00004-of-00004.safetensors",
-    "model.layers.28.linear_attn.conv1d.weight": "model-00004-of-00004.safetensors",
-    "model.layers.28.linear_attn.dt_bias": "model-00004-of-00004.safetensors",
-    "model.layers.28.linear_attn.in_proj_a.weight": "model-00004-of-00004.safetensors",
-    "model.layers.28.linear_attn.in_proj_b.weight": "model-00004-of-00004.safetensors",
-    "model.layers.28.linear_attn.in_proj_qkv.weight": "model-00004-of-00004.safetensors",
-    "model.layers.28.linear_attn.in_proj_z.weight": "model-00004-of-00004.safetensors",
-    "model.layers.28.linear_attn.norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.28.linear_attn.out_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.28.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.28.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.28.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.28.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.29.input_layernorm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.29.linear_attn.A_log": "model-00004-of-00004.safetensors",
-    "model.layers.29.linear_attn.conv1d.weight": "model-00004-of-00004.safetensors",
-    "model.layers.29.linear_attn.dt_bias": "model-00004-of-00004.safetensors",
-    "model.layers.29.linear_attn.in_proj_a.weight": "model-00004-of-00004.safetensors",
-    "model.layers.29.linear_attn.in_proj_b.weight": "model-00004-of-00004.safetensors",
-    "model.layers.29.linear_attn.in_proj_qkv.weight": "model-00004-of-00004.safetensors",
-    "model.layers.29.linear_attn.in_proj_z.weight": "model-00004-of-00004.safetensors",
-    "model.layers.29.linear_attn.norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.29.linear_attn.out_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.29.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.29.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.29.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.29.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.3.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.3.self_attn.gla.g_norm_swish_gate.weight": "model-00002-of-00004.safetensors",
-    "model.layers.3.self_attn.gla.g_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.3.self_attn.gla.gk_proj.0.weight": "model-00002-of-00004.safetensors",
-    "model.layers.3.self_attn.gla.gk_proj.1.bias": "model-00002-of-00004.safetensors",
-    "model.layers.3.self_attn.gla.gk_proj.1.weight": "model-00002-of-00004.safetensors",
-    "model.layers.3.self_attn.gla.k_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.3.self_attn.gla.o_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.3.self_attn.gla.q_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.3.self_attn.gla.v_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.30.input_layernorm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.30.linear_attn.A_log": "model-00004-of-00004.safetensors",
-    "model.layers.30.linear_attn.conv1d.weight": "model-00004-of-00004.safetensors",
-    "model.layers.30.linear_attn.dt_bias": "model-00004-of-00004.safetensors",
-    "model.layers.30.linear_attn.in_proj_a.weight": "model-00004-of-00004.safetensors",
-    "model.layers.30.linear_attn.in_proj_b.weight": "model-00004-of-00004.safetensors",
-    "model.layers.30.linear_attn.in_proj_qkv.weight": "model-00004-of-00004.safetensors",
-    "model.layers.30.linear_attn.in_proj_z.weight": "model-00004-of-00004.safetensors",
-    "model.layers.30.linear_attn.norm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.30.linear_attn.out_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.30.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.30.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.30.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.30.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.gla.g_norm_swish_gate.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.gla.g_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.gla.gk_proj.0.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.gla.gk_proj.1.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.gla.gk_proj.1.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.gla.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.gla.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.gla.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.gla.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.linear_attn.A_log": "model-00003-of-00004.safetensors",
+    "model.layers.28.linear_attn.conv1d.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.linear_attn.dt_bias": "model-00003-of-00004.safetensors",
+    "model.layers.28.linear_attn.in_proj_a.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.linear_attn.in_proj_b.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.linear_attn.in_proj_qkv.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.linear_attn.in_proj_z.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.linear_attn.norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.linear_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.linear_attn.A_log": "model-00003-of-00004.safetensors",
+    "model.layers.29.linear_attn.conv1d.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.linear_attn.dt_bias": "model-00003-of-00004.safetensors",
+    "model.layers.29.linear_attn.in_proj_a.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.linear_attn.in_proj_b.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.linear_attn.in_proj_qkv.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.linear_attn.in_proj_z.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.linear_attn.norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.linear_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.gla.g_norm_swish_gate.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.gla.g_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.gla.gk_proj.0.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.gla.gk_proj.1.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.gla.gk_proj.1.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.gla.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.gla.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.gla.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.gla.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.linear_attn.A_log": "model-00003-of-00004.safetensors",
+    "model.layers.30.linear_attn.conv1d.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.linear_attn.dt_bias": "model-00003-of-00004.safetensors",
+    "model.layers.30.linear_attn.in_proj_a.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.linear_attn.in_proj_b.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.linear_attn.in_proj_qkv.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.linear_attn.in_proj_z.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.linear_attn.norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.linear_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
     "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors",
     "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
     "model.layers.31.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
     "model.layers.31.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
     "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
-    "model.layers.31.self_attn.gla.g_norm_swish_gate.weight": "model-00004-of-00004.safetensors",
-    "model.layers.31.self_attn.gla.g_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.31.self_attn.gla.gk_proj.0.weight": "model-00004-of-00004.safetensors",
-    "model.layers.31.self_attn.gla.gk_proj.1.bias": "model-00004-of-00004.safetensors",
-    "model.layers.31.self_attn.gla.gk_proj.1.weight": "model-00004-of-00004.safetensors",
-    "model.layers.31.self_attn.gla.k_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.31.self_attn.gla.o_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.31.self_attn.gla.q_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.31.self_attn.gla.v_proj.weight": "model-00004-of-00004.safetensors",
-    "model.layers.4.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.4.linear_attn.A_log": "model-00002-of-00004.safetensors",
-    "model.layers.4.linear_attn.conv1d.weight": "model-00002-of-00004.safetensors",
-    "model.layers.4.linear_attn.dt_bias": "model-00002-of-00004.safetensors",
-    "model.layers.4.linear_attn.in_proj_a.weight": "model-00002-of-00004.safetensors",
-    "model.layers.4.linear_attn.in_proj_b.weight": "model-00002-of-00004.safetensors",
-    "model.layers.4.linear_attn.in_proj_qkv.weight": "model-00002-of-00004.safetensors",
-    "model.layers.4.linear_attn.in_proj_z.weight": "model-00002-of-00004.safetensors",
-    "model.layers.4.linear_attn.norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.4.linear_attn.out_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.5.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.5.linear_attn.A_log": "model-00002-of-00004.safetensors",
-    "model.layers.5.linear_attn.conv1d.weight": "model-00002-of-00004.safetensors",
-    "model.layers.5.linear_attn.dt_bias": "model-00002-of-00004.safetensors",
-    "model.layers.5.linear_attn.in_proj_a.weight": "model-00002-of-00004.safetensors",
-    "model.layers.5.linear_attn.in_proj_b.weight": "model-00002-of-00004.safetensors",
-    "model.layers.5.linear_attn.in_proj_qkv.weight": "model-00002-of-00004.safetensors",
-    "model.layers.5.linear_attn.in_proj_z.weight": "model-00002-of-00004.safetensors",
-    "model.layers.5.linear_attn.norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.5.linear_attn.out_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.6.linear_attn.A_log": "model-00002-of-00004.safetensors",
-    "model.layers.6.linear_attn.conv1d.weight": "model-00002-of-00004.safetensors",
-    "model.layers.6.linear_attn.dt_bias": "model-00002-of-00004.safetensors",
-    "model.layers.6.linear_attn.in_proj_a.weight": "model-00002-of-00004.safetensors",
-    "model.layers.6.linear_attn.in_proj_b.weight": "model-00002-of-00004.safetensors",
-    "model.layers.6.linear_attn.in_proj_qkv.weight": "model-00002-of-00004.safetensors",
-    "model.layers.6.linear_attn.in_proj_z.weight": "model-00002-of-00004.safetensors",
-    "model.layers.6.linear_attn.norm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.6.linear_attn.out_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.31.self_attn.gla.g_norm_swish_gate.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.gla.g_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.gla.gk_proj.0.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.gla.gk_proj.1.bias": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.gla.gk_proj.1.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.gla.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.gla.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.gla.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.gla.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.linear_attn.A_log": "model-00001-of-00004.safetensors",
+    "model.layers.4.linear_attn.conv1d.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.linear_attn.dt_bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.linear_attn.in_proj_a.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.linear_attn.in_proj_b.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.linear_attn.in_proj_qkv.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.linear_attn.in_proj_z.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.linear_attn.norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.linear_attn.out_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.linear_attn.A_log": "model-00001-of-00004.safetensors",
+    "model.layers.5.linear_attn.conv1d.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.linear_attn.dt_bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.linear_attn.in_proj_a.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.linear_attn.in_proj_b.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.linear_attn.in_proj_qkv.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.linear_attn.in_proj_z.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.linear_attn.norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.linear_attn.out_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.linear_attn.A_log": "model-00001-of-00004.safetensors",
+    "model.layers.6.linear_attn.conv1d.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.linear_attn.dt_bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.linear_attn.in_proj_a.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.linear_attn.in_proj_b.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.linear_attn.in_proj_qkv.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.linear_attn.in_proj_z.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.linear_attn.norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.linear_attn.out_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
     "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
     "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
     "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
@@ -422,10 +421,10 @@
     "model.layers.7.self_attn.gla.gk_proj.0.weight": "model-00002-of-00004.safetensors",
     "model.layers.7.self_attn.gla.gk_proj.1.bias": "model-00002-of-00004.safetensors",
     "model.layers.7.self_attn.gla.gk_proj.1.weight": "model-00002-of-00004.safetensors",
-    "model.layers.7.self_attn.gla.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.gla.k_proj.weight": "model-00001-of-00004.safetensors",
     "model.layers.7.self_attn.gla.o_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.7.self_attn.gla.q_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.7.self_attn.gla.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.gla.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.gla.v_proj.weight": "model-00001-of-00004.safetensors",
     "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
     "model.layers.8.linear_attn.A_log": "model-00002-of-00004.safetensors",
     "model.layers.8.linear_attn.conv1d.weight": "model-00002-of-00004.safetensors",

special_tokens_map.json

@@ -1,4 +1,16 @@
 {
-  "eos_token": "<|endoftext|>",
-  "pad_token": "<|endoftext|>"
-}
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -1,13 +1,281 @@
 {
   "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "248044": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248045": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248046": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248047": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248048": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248049": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248050": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248051": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248052": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248053": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248054": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248055": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248056": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248057": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248058": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248059": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248060": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248061": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248062": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248063": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248064": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248065": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248066": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248067": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248068": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248069": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248070": {
+      "content": "<|audio_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248071": {
+      "content": "<|audio_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248072": {
+      "content": "<tts_pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248073": {
+      "content": "<tts_text_bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248074": {
+      "content": "<tts_text_eod>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248075": {
+      "content": "<tts_text_bos_single>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248076": {
+      "content": "<|audio_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
   "audio_bos_token": "<|audio_start|>",
   "audio_eos_token": "<|audio_end|>",
   "audio_token": "<|audio_pad|>",
   "backend": "tokenizers",
   "bos_token": null,
+  "chat_template": "{% for message in messages %}{% set role = message['role'] | lower %}{% if role == 'user' %}{% set role = 'HUMAN' %}{% endif %}{% set role = role | upper %}{{ '<role>' + role + '</role>' + message['content'] }}{% if role == 'ASSISTANT' %}{{ '<|endoftext|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<role>ASSISTANT</role>' }}{% endif %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "errors": "replace",
+  "extra_special_tokens": {},
   "image_token": "<|image_pad|>",
   "is_local": false,
   "model_max_length": 262144,
@@ -23,10 +291,9 @@
   "pad_token": "<|endoftext|>",
   "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
   "split_special_tokens": false,
-  "tokenizer_class": "PreTrainedTokenizerFast",
+  "tokenizer_class": "PreTrainedTokenizer",
   "unk_token": null,
   "video_token": "<|video_pad|>",
   "vision_bos_token": "<|vision_start|>",
-  "vision_eos_token": "<|vision_end|>",
-  "chat_template": "{% for message in messages %}{% set role = message['role'] | lower %}{% if role == 'user' %}{% set role = 'HUMAN' %}{% endif %}{% set role = role | upper %}{{ '<role>' + role + '</role>' + message['content'] }}{% if role == 'ASSISTANT' %}{{ '<|endoftext|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<role>ASSISTANT</role>' }}{% endif %}"
-}
+  "vision_eos_token": "<|vision_end|>"
+}