diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..52373fe24473b1aa44333d318f578ae6bf04b49b 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
diff --git a/chat_template.jinja b/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..736f54af62e0cf07b107882bab67b0f491180367
--- /dev/null
+++ b/chat_template.jinja
@@ -0,0 +1,157 @@
+{%- macro visible_text(content) -%}
+ {%- if content is string -%}
+ {{- content -}}
+ {%- elif content is iterable and content is not mapping -%}
+ {%- for item in content -%}
+ {%- if item is mapping and item.type == 'text' -%}
+ {{- item.text -}}
+ {%- elif item is string -%}
+ {{- item -}}
+ {%- endif -%}
+ {%- endfor -%}
+ {%- else -%}
+ {{- content -}}
+ {%- endif -%}
+{%- endmacro -%}
+
+{{- '<|beginoftext|>' -}}
+
+{%- set ns = namespace(has_system=false, last_user_index=-1, last_assistant_index=-1) -%}
+{%- if messages | length > 0 and messages[0].role == 'system' -%}
+ {%- set ns.has_system = true -%}
+{%- endif -%}
+{%- for m in messages -%}
+ {%- if m.role == 'user' -%}
+ {%- set ns.last_user_index = loop.index0 -%}
+ {%- elif m.role == 'assistant' -%}
+ {%- set ns.last_assistant_index = loop.index0 -%}
+ {%- endif -%}
+{%- endfor -%}
+
+{#- ── System / Tools block ── -#}
+{%- if tools is iterable and tools | length > 0 -%}
+ {{- '<|startofturn|><|system|>' -}}
+ {{- '# Tools\n\nYou may call one or more functions to assist with the user query.\n\n' -}}
+ {{- 'You are provided with function signatures within XML tags:\n\n' -}}
+ {%- for tool in tools -%}
+ {%- if tool.function is defined -%}
+ {%- set tool = tool.function -%}
+ {%- endif -%}
+ {{- '\n' ~ (tool | tojson) -}}
+ {%- endfor -%}
+ {{- '\n' -}}
+ {{- '\n\nFor each function call, output in JSON within tags:\n' -}}
+ {%- for tool in tools -%}
+ {%- if tool.function is defined -%}
+ {%- set tool = tool.function -%}
+ {%- endif -%}
+ {%- set _props = tool.parameters.properties if (tool.parameters is defined and tool.parameters.properties is defined) else {} -%}
+ {{- '\n{"name": "' ~ tool.name ~ '", "arguments": {' -}}
+ {%- set _keys = _props | list -%}
+ {%- for k in _keys -%}
+ {{- '"' ~ k ~ '": <' ~ k ~ '>' -}}
+ {%- if not loop.last -%}{{- ', ' -}}{%- endif -%}
+ {%- endfor -%}
+ {{- '}}' -}}
+ {%- endfor -%}
+ {%- if ns.has_system -%}
+ {{- '\n\n' ~ visible_text(messages[0].content) -}}
+ {%- endif -%}
+ {{- '<|endofturn|>' -}}
+{%- elif ns.has_system -%}
+ {{- '<|startofturn|><|system|>' ~ visible_text(messages[0].content) ~ '<|endofturn|>' -}}
+{%- endif -%}
+
+{#- ── Conversation turns ── -#}
+{%- for m in messages -%}
+
+ {#- [Fix 1] continue 대신 if/elif 체인으로 첫 system 스킵 -#}
+ {%- if loop.index0 == 0 and m.role == 'system' -%}
+ {#- already rendered above, skip -#}
+
+ {#- [Fix 2] 중간에 나오는 system 메시지도 처리 -#}
+ {%- elif m.role == 'system' -%}
+ {{- '<|startofturn|><|system|>' ~ visible_text(m.content) ~ '<|endofturn|>' -}}
+
+ {%- elif m.role == 'user' -%}
+ {{- '<|startofturn|><|user|>' -}}
+ {%- if m.references is defined and m.references -%}
+ {{- '<|reference|>' ~ m.references ~ '\n' -}}
+ {%- endif -%}
+ {{- visible_text(m.content) -}}
+ {{- '<|endofturn|>' -}}
+
+ {%- elif m.role == 'assistant' -%}
+ {{- '<|startofturn|><|assistant|>' -}}
+
+ {#- [순서 정책] think → plan → content → tool_calls -#}
+
+ {#- Reasoning block -#}
+ {%- set _content = visible_text(m.content) -%}
+ {%- set _reasoning = '' -%}
+ {%- if m.reasoning_content is string -%}
+ {%- set _reasoning = m.reasoning_content -%}
+ {%- elif '' in _content -%}
+ {%- set _reasoning = _content.split('')[0].split('')[-1].strip() -%}
+ {%- set _content = _content.split('', 1)[-1].lstrip('\n') -%}
+ {%- endif -%}
+ {%- set _has_tools = (tools is defined and tools is iterable and tools | length > 0) -%}
+ {%- set _emit_think = _reasoning and (_has_tools or loop.index0 == ns.last_assistant_index) -%}
+ {%- if _emit_think -%}
+ {{- '' ~ _reasoning.strip() ~ '' -}}
+ {%- endif -%}
+
+ {#- Text content -#}
+ {%- if _content.strip() -%}
+ {{- _content.strip() -}}
+ {%- endif -%}
+
+ {#- Tool calls — IDs are rendered for intermediate assistant turns
+ (context for call↔response correlation) but omitted for the last
+ assistant turn (prediction target — model should not learn to
+ generate IDs). After multi-turn data expansion, intermediate turns
+ become GRAY context and the last turn is GREEN. -#}
+ {%- if m.tool_calls is defined and m.tool_calls -%}
+ {%- set _is_last_assistant = (loop.index0 == ns.last_assistant_index) and not add_generation_prompt -%}
+ {%- for tc in m.tool_calls -%}
+ {%- set _tc_id = tc.id if (tc.id is defined and not _is_last_assistant) else none -%}
+ {%- if tc.function is defined -%}
+ {%- set tc = tc.function -%}
+ {%- endif -%}
+ {%- set _id_suffix = ', "id": ' ~ (_tc_id | tojson) if _tc_id is not none else '' -%}
+ {%- if tc.arguments is not defined or not tc.arguments or tc.arguments == "" -%}
+ {{- '\n' ~ '{"name": "' ~ tc.name ~ '", "arguments": ' ~ null ~ _id_suffix ~ '}' ~ '' -}}
+ {%- elif tc.arguments is string -%}
+ {{- '\n' ~ '{"name": "' ~ tc.name ~ '", "arguments": ' ~ tc.arguments ~ _id_suffix ~ '}' ~ '' -}}
+ {%- else -%}
+ {{- '\n' ~ '{"name": "' ~ tc.name ~ '", "arguments": ' ~ (tc.arguments | tojson) ~ _id_suffix ~ '}' ~ '' -}}
+ {%- endif -%}
+ {%- endfor -%}
+ {%- endif -%}
+
+ {{- '<|endofturn|>' -}}
+
+ {%- elif m.role == 'tool' -%}
+ {#- [Fix 3] loop.previtem/nextitem으로 인덱스 오버플로 제거 -#}
+ {%- if loop.first or loop.previtem.role != 'tool' -%}
+ {{- '<|startofturn|><|tool|>' -}}
+ {%- endif -%}
+ {{- '' ~ ({"tool_call_id": m.tool_call_id, "content": m.content} | tojson) ~ '' -}}
+ {%- if loop.last or loop.nextitem.role != 'tool' -%}
+ {{- '<|endofturn|>' -}}
+ {%- endif -%}
+
+ {%- endif -%}
+{%- endfor -%}
+
+{#- ── Generation prompt ── -#}
+{%- if add_generation_prompt -%}
+ {{- '<|startofturn|><|assistant|>' -}}
+ {%- if enable_thinking is defined and not enable_thinking -%}
+ {{- '' -}}
+ {%- else -%}
+ {{- '' -}}
+ {%- endif -%}
+{%- else -%}
+ {{- '<|endoftext|>' -}}
+{%- endif -%}
\ No newline at end of file
diff --git a/config.json b/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..975cd9f7085b49f530204708bb1bfbe86379befc
--- /dev/null
+++ b/config.json
@@ -0,0 +1,81 @@
+{
+ "_debug_force_load_balance": false,
+ "architectures": [
+ "MotifForCausalLM"
+ ],
+ "attention_cls": "gdla",
+ "attention_dropout": 0.0,
+ "auto_map": {
+ "AutoConfig": "configuration_motif.MotifConfig",
+ "AutoModel": "modeling_motif.MotifForCausalLM",
+ "AutoModelForCausalLM": "modeling_motif.MotifForCausalLM"
+ },
+ "diff_v2": true,
+ "dtype": "bfloat16",
+ "elementwise_attn_output_gate": true,
+ "eos_token_id": 0,
+ "experts_top_k": 8,
+ "head_dim": 192,
+ "headwise_attn_output_gate": false,
+ "hidden_act": "poly_norm",
+ "hidden_size": 4096,
+ "initializer_range": 0.02,
+ "interleave_moe_layer_step": 1,
+ "intermediate_size": 12288,
+ "k_ratio": 1,
+ "kv_lora_rank": 512,
+ "load_balance_coeff": 0.0001,
+ "max_position_embeddings": 262144,
+ "max_window_layers": 9,
+ "mhc_enabled": true,
+ "mhc_expansion_rate": 4,
+ "mhc_identity_init": false,
+ "mhc_sinkhorn_iters": 20,
+ "model_type": "Motif",
+ "moe_intermediate_size": 1280,
+ "mscale": 1.0,
+ "n_dense_first_layers": 2,
+ "num_attention_heads": 80,
+ "num_experts": 384,
+ "num_hidden_layers": 53,
+ "num_key_value_heads": 16,
+ "num_noise_heads": 16,
+ "num_shared_experts": 1,
+ "output_router_logits": false,
+ "q_lora_rank": 1024,
+ "qk_rope_head_dim": 64,
+ "rms_norm_eps": 1e-05,
+ "rope_theta": 10000.0,
+ "route_norm": true,
+ "route_scale": 2.0,
+ "router_aux_loss_coef": 0.0,
+ "score_before_experts": false,
+ "score_func": "sigmoid",
+ "sliding_window": 128,
+ "sliding_window_pattern": "interleave",
+ "sliding_window_period": 4,
+ "swa_rope_theta": 10000.0,
+ "tie_word_embeddings": false,
+ "transformers_version": "5.7.0",
+ "use_cache": true,
+ "use_sliding_window": true,
+ "v_head_dim": 128,
+ "vocab_size": 220160,
+ "rope_factor": 64.0,
+ "original_seq_len": 4096,
+ "rope_scaling": {
+ "original_max_position_embeddings": 4096,
+ "factor": 64.0,
+ "mscale": 1.0,
+ "rope_type": "yarn",
+ "rope_theta": 10000.0,
+ "beta_fast": 32.0,
+ "beta_slow": 1.0,
+ "apply_yarn_scaling": false
+ },
+ "polynorm_output_scale": 0.5,
+ "polynorm_output_scale_per_layer": {},
+ "polynorm_bias_clamp": 0.5,
+ "hidden_clamp": 1000000.0,
+ "num_nextn_predict_layers": 1
+}
\ No newline at end of file
diff --git a/configuration_motif.py b/configuration_motif.py
new file mode 100644
index 0000000000000000000000000000000000000000..af77a03003641b1805753d222a79015c65f9c2fb
--- /dev/null
+++ b/configuration_motif.py
@@ -0,0 +1,294 @@
+import torch
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+
+class MotifConfig(PretrainedConfig):
+ r"""
+ This is the configuration class to store the configuration of a [`MotifModel`]. It is used to instantiate a
+ Motif model according to the specified arguments, defining the model architecture.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+ Args:
+ vocab_size (`int`, *optional*, defaults to 151936):
+ Vocabulary size of the Motif model. Defines the number of different tokens that can be represented by the
+ `inputs_ids` passed when calling [`MotifModel`]
+ hidden_size (`int`, *optional*, defaults to 4096):
+ Dimension of the hidden representations.
+ intermediate_size (`int`, *optional*, defaults to 22016):
+ Dimension of the MLP representations.
+ num_hidden_layers (`int`, *optional*, defaults to 32):
+ Number of hidden layers in the Transformer encoder.
+ num_attention_heads (`int`, *optional*, defaults to 32):
+ Number of attention heads for each attention layer in the Transformer encoder.
+ num_key_value_heads (`int`, *optional*, defaults to 32):
+ This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+ `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+ `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+ converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+ by meanpooling all the original heads within that group. For more details checkout [this
+ paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `32`.
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+ The non-linear activation function (function or string) in the decoder.
+ max_position_embeddings (`int`, *optional*, defaults to 32768):
+ The maximum sequence length that this model might ever be used with.
+ initializer_range (`float`, *optional*, defaults to 0.02):
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+ rms_norm_eps (`float`, *optional*, defaults to 1e-06):
+ The epsilon used by the rms normalization layers.
+ use_cache (`bool`, *optional*, defaults to `True`):
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
+ relevant if `config.is_decoder=True`.
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+ Whether the model's input and output word embeddings should be tied.
+ rope_theta (`float`, *optional*, defaults to 1000000.0):
+ The base period of the RoPE embeddings.
+ rope_scaling (`Dict`, *optional*):
+ Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
+ and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
+ accordingly.
+ Expected contents:
+ `rope_type` (`str`):
+ The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
+ 'llama3'], with 'default' being the original RoPE implementation.
+ `factor` (`float`, *optional*):
+ Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
+ most scaling types, a `factor` of x will enable the model to handle sequences of length x *
+ original maximum pre-trained length.
+ `original_max_position_embeddings` (`int`, *optional*):
+ Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
+ pretraining.
+ `attention_factor` (`float`, *optional*):
+ Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
+ computation. If unspecified, it defaults to value recommended by the implementation, using the
+ `factor` field to infer the suggested value.
+ `beta_fast` (`float`, *optional*):
+ Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
+ ramp function. If unspecified, it defaults to 32.
+ `beta_slow` (`float`, *optional*):
+ Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
+ ramp function. If unspecified, it defaults to 1.
+ `short_factor` (`List[float]`, *optional*):
+ Only used with 'longrope'. The scaling factor to be applied to short contexts (<
+ `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
+ size divided by the number of attention heads divided by 2
+ `long_factor` (`List[float]`, *optional*):
+ Only used with 'longrope'. The scaling factor to be applied to long contexts (<
+ `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
+ size divided by the number of attention heads divided by 2
+ `low_freq_factor` (`float`, *optional*):
+ Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
+ `high_freq_factor` (`float`, *optional*):
+ Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
+ use_sliding_window (`bool`, *optional*, defaults to `False`):
+ Whether to use sliding window attention.
+ sliding_window (`int`, *optional*, defaults to 4096):
+ Sliding window attention (SWA) window size. If not specified, will default to `4096`.
+ max_window_layers (`int`, *optional*, defaults to 28):
+ The number of layers that use SWA (Sliding Window Attention). The bottom layers use SWA while the top use full attention.
+ attention_dropout (`float`, *optional*, defaults to 0.0):
+ The dropout ratio for the attention probabilities.
+ ```python
+ >>> from transformers import MotifModel, MotifConfig
+ >>> # Initializing a Motif style configuration
+ >>> configuration = MotifConfig()
+ >>> # Initializing a model from the Motif-102B style configuration
+ >>> model = MotifModel(configuration)
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+ ```"""
+
+ model_type = "Motif"
+ keys_to_ignore_at_inference = ["past_key_values"]
+
+ base_model_tp_plan = {
+ # Attention
+ "layers.*.self_attn.q_proj": "colwise",
+ "layers.*.self_attn.k_proj": "colwise",
+ "layers.*.self_attn.v_proj": "colwise",
+ "layers.*.self_attn.o_proj": "rowwise",
+ # Dense MLP
+ "layers.*.mlp.gate_proj": "colwise",
+ "layers.*.mlp.up_proj": "colwise",
+ "layers.*.mlp.down_proj": "rowwise",
+ # MoE experts (fused gate+up)
+ "layers.*.moe.experts.gate_up_proj": "packed_colwise",
+ "layers.*.moe.experts.down_proj": "rowwise",
+ # Shared experts
+ "layers.*.moe.shared_experts.gate_proj": "colwise",
+ "layers.*.moe.shared_experts.up_proj": "colwise",
+ "layers.*.moe.shared_experts.down_proj": "rowwise",
+ }
+
+ base_model_pp_plan = {
+ "embed_tokens": (["input_ids"], ["inputs_embeds"]),
+ "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
+ "norm": (["hidden_states"], ["hidden_states"]),
+ }
+
+ def __init__(
+ self,
+ vocab_size=151936,
+ hidden_size=4096,
+ intermediate_size=22016,
+ num_hidden_layers=32,
+ num_attention_heads=32,
+ num_key_value_heads=32,
+ hidden_act="silu",
+ max_position_embeddings=32768,
+ initializer_range=0.02,
+ rms_norm_eps=1e-6,
+ use_cache=True,
+ tie_word_embeddings=False,
+ rope_theta=1000000.0,
+ rope_scaling=None,
+ use_sliding_window=False,
+ sliding_window=4096,
+ max_window_layers=28,
+ sliding_window_pattern="interleave",
+ sliding_window_period=2,
+ attention_dropout=0.0,
+ # Differential Attention parameters
+ head_dim=None,
+ num_noise_heads=0,
+ k_ratio=1,
+ # MoE parameters
+ num_experts=0,
+ experts_top_k=2,
+ num_shared_experts=0,
+ interleave_moe_layer_step=0,
+ moe_intermediate_size=None,
+ score_func="softmax",
+ route_norm=False,
+ route_scale=1.0,
+ load_balance_coeff=None,
+ score_before_experts=False,
+ _debug_force_load_balance=False,
+ output_router_logits=False,
+ router_aux_loss_coef=0.0,
+ # MHC (Manifold-constrained Hyper-Connections) parameters
+ mhc_enabled=False,
+ mhc_expansion_rate=4,
+ mhc_identity_init=False,
+ mhc_sinkhorn_iters=20,
+ # DiffAttention V2 / Attention class
+ diff_v2=False,
+ attention_cls="basic",
+ # GDLA (Grouped Differential Latent Attention) parameters
+ q_lora_rank=0,
+ kv_lora_rank=0,
+ qk_rope_head_dim=None,
+ v_head_dim=None,
+ original_seq_len=32768,
+ rope_factor=1.0,
+ mscale=1.0,
+ swa_rope_theta=None,
+ # Attention output gating
+ headwise_attn_output_gate=False,
+ elementwise_attn_output_gate=False,
+ # MoE: first N layers always dense (no MoE), regardless of interleave schedule
+ n_dense_first_layers=0,
+ # MTP (Multi-Token Prediction) speculative decoding
+ num_nextn_predict_layers=0,
+ **kwargs,
+ ):
+ self.vocab_size = vocab_size
+ self.max_position_embeddings = max_position_embeddings
+ self.hidden_size = hidden_size
+ self.intermediate_size = intermediate_size
+ self.num_hidden_layers = num_hidden_layers
+ self.num_attention_heads = num_attention_heads
+ self.use_sliding_window = use_sliding_window
+ self.sliding_window = sliding_window if use_sliding_window else None
+ self.max_window_layers = max_window_layers
+ self.sliding_window_pattern = sliding_window_pattern
+ self.sliding_window_period = sliding_window_period
+
+ # for backward compatibility
+ if num_key_value_heads is None:
+ num_key_value_heads = num_attention_heads
+
+ self.num_key_value_heads = num_key_value_heads
+ self.hidden_act = hidden_act
+ self.initializer_range = initializer_range
+ self.rms_norm_eps = rms_norm_eps
+ self.use_cache = use_cache
+ self.rope_theta = rope_theta
+ self.rope_scaling = rope_scaling
+ self.attention_dropout = attention_dropout
+
+ # Differential Attention configuration
+ self.head_dim = head_dim
+ self.num_noise_heads = num_noise_heads
+ self.k_ratio = k_ratio
+
+ # MoE configuration
+ self.num_experts = num_experts
+ self.experts_top_k = experts_top_k
+ self.num_shared_experts = num_shared_experts
+ self.interleave_moe_layer_step = interleave_moe_layer_step
+ self.moe_intermediate_size = moe_intermediate_size if moe_intermediate_size is not None else intermediate_size
+ self.score_func = score_func
+ self.route_norm = route_norm
+ self.route_scale = route_scale
+ self.load_balance_coeff = load_balance_coeff
+ self.score_before_experts = score_before_experts
+ self._debug_force_load_balance = _debug_force_load_balance
+ self.output_router_logits = output_router_logits
+ self.router_aux_loss_coef = router_aux_loss_coef
+
+ # MHC configuration
+ self.mhc_enabled = mhc_enabled
+ self.mhc_expansion_rate = mhc_expansion_rate
+ self.mhc_identity_init = mhc_identity_init
+ self.mhc_sinkhorn_iters = mhc_sinkhorn_iters
+
+ # DiffAttention V2 / Attention class
+ self.diff_v2 = diff_v2
+ self.attention_cls = attention_cls
+
+ # GDLA parameters
+ self.q_lora_rank = q_lora_rank
+ self.kv_lora_rank = kv_lora_rank
+ self.qk_rope_head_dim = qk_rope_head_dim
+ self.v_head_dim = v_head_dim
+ self.original_seq_len = original_seq_len
+ self.rope_factor = rope_factor
+ self.mscale = mscale
+ self.swa_rope_theta = swa_rope_theta
+
+ # Attention output gating
+ self.headwise_attn_output_gate = headwise_attn_output_gate
+ self.elementwise_attn_output_gate = elementwise_attn_output_gate
+
+ # MoE dense-first layers
+ self.n_dense_first_layers = n_dense_first_layers
+
+ # MTP speculative decoding
+ self.num_nextn_predict_layers = num_nextn_predict_layers
+
+ # Validate the correctness of rotary position embeddings parameters
+ # BC: if there is a 'type' field, move it to 'rope_type'.
+ if self.rope_scaling is not None and "type" in self.rope_scaling:
+ self.rope_scaling["rope_type"] = self.rope_scaling["type"]
+ if callable(getattr(type(self), "validate_rope", None)):
+ self.validate_rope()
+
+ # On ROCm, torch._grouped_mm is not supported at runtime even though
+ # transformers auto-selects the grouped_mm expert backend for torch>=2.9.
+ # Force eager (for-loop) dispatch so MoE models work on ROCm.
+ if (
+ self.num_experts > 0
+ and hasattr(torch.version, "hip")
+ and torch.version.hip is not None
+ and "experts_implementation" not in kwargs
+ ):
+ kwargs["experts_implementation"] = "eager"
+
+ super().__init__(
+ tie_word_embeddings=tie_word_embeddings,
+ **kwargs,
+ )
+ logger.info(f" kwargs : {kwargs}")
diff --git a/generation_config.json b/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0d31bc676035f33634f3e77c9d4889679ae240c
--- /dev/null
+++ b/generation_config.json
@@ -0,0 +1,13 @@
+{
+ "_from_model_config": true,
+ "bos_token_id": 1,
+ "pad_token_id": 0,
+ "eos_token_id": [0, 3, 6],
+ "output_attentions": false,
+ "output_hidden_states": false,
+ "transformers_version": "5.7.0",
+ "use_cache": true,
+ "do_sample": true,
+ "temperature": 1.0,
+ "top_p": 0.95
+}
diff --git a/model-00001-of-00155.safetensors b/model-00001-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0ceca74c942f11369c060d54844e126eda83bc99
--- /dev/null
+++ b/model-00001-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d448f19cf7d6e5deb7bc4eb33c11b6a29e3ba38ed06f36161739673d482325a
+size 2474208740
diff --git a/model-00002-of-00155.safetensors b/model-00002-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..f34b748287debc760358545b216ddcecc005daf9
--- /dev/null
+++ b/model-00002-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1eb1652398b1cd781e63ba7212ba871857896d927b49a9ff039aa5d450750c8
+size 336158060
diff --git a/model-00003-of-00155.safetensors b/model-00003-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2a1f715c08ad7469e20780f9a585596d7bf41e55
--- /dev/null
+++ b/model-00003-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:779acccca99ba6820889f6dfa9c975ccb0a8ef0f6850f6565543525d9f7e8b9b
+size 4026531992
diff --git a/model-00004-of-00155.safetensors b/model-00004-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..49932e676bd156c47b8cf3f528a55f44bbc5d350
--- /dev/null
+++ b/model-00004-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efbdffbdc2a0b0f08b69c1b79dfdbacbc60d489d8b69d442d9b35103139ff20c
+size 169438244
diff --git a/model-00005-of-00155.safetensors b/model-00005-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d07940224fae7860f3ceeeacb29d6ec08fc59981
--- /dev/null
+++ b/model-00005-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a261c05852c76d8aea3f007595914b79616bafc5d8b610871248cbe6424c66d8
+size 4026531992
diff --git a/model-00006-of-00155.safetensors b/model-00006-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a9220d69608c546a04e24384d55fb54d9dc94a30
--- /dev/null
+++ b/model-00006-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c735774006a755ba75deb7a111107133c746905878ce1682f2d6de9f66128cb
+size 169438244
diff --git a/model-00007-of-00155.safetensors b/model-00007-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..29fe3742bc6c9cabe5ddb35bca37eb2de2f2dd1c
--- /dev/null
+++ b/model-00007-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2692b0a753bbf3b43eeef9c8bc10bb01324d0c1d49dd488d13362b0f3d836b2
+size 4026531992
diff --git a/model-00008-of-00155.safetensors b/model-00008-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..be559b863f12ad9090a07fc2f91ecb2b6041a8b8
--- /dev/null
+++ b/model-00008-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da1c4e051658c8acd142485621d3e113d948d7caf99048c38b4f47d2a20fc837
+size 169438244
diff --git a/model-00009-of-00155.safetensors b/model-00009-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..407ee53f0c4d255da6f2cc0ab06109c56971daf8
--- /dev/null
+++ b/model-00009-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f660e9baa2f5caa5ffee4b3fdd5c734c87726c6f68a51f5bf36287fb05b7adce
+size 4026531992
diff --git a/model-00010-of-00155.safetensors b/model-00010-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..60ad8e696b53c4e546d7cefd77411aa4ebd2b6c0
--- /dev/null
+++ b/model-00010-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08c389d5a08e7a7effef1c25d687f086dce81857b5ac51aef567514c9cb1b6f8
+size 169438244
diff --git a/model-00011-of-00155.safetensors b/model-00011-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..3520f7693badcfce7577102a4f2572cab289f932
--- /dev/null
+++ b/model-00011-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb2d91468b652ad09c70665e4ed9a29557deddcbf7c6cd9a70f57a59b03d87f1
+size 4026531992
diff --git a/model-00012-of-00155.safetensors b/model-00012-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..11db45f3463a402ef9b8cd9b3dc1f86f49edd04c
--- /dev/null
+++ b/model-00012-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e36df01f27ef3e663552b27216fdb9a46955f4936b7e39cd246422a2a10731b
+size 169438244
diff --git a/model-00013-of-00155.safetensors b/model-00013-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ad4f622d9f01e2c6a4ac821ea1388e50cb2c8c24
--- /dev/null
+++ b/model-00013-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ae9f93e85aa948b05c3e3030fddf1ed10c80300db4c65061da6b522fb6f056e
+size 4026531992
diff --git a/model-00014-of-00155.safetensors b/model-00014-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..3fe2f8a09b0763031beab64baf8bd5f4d05adfce
--- /dev/null
+++ b/model-00014-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52b26ded92e96ee7dd743d35c977ffb2b2758bb5ee8a1d58f740009396ec2b45
+size 169438244
diff --git a/model-00015-of-00155.safetensors b/model-00015-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..42f739c20c11b67924dfbc604d2ef72ecc38bcf7
--- /dev/null
+++ b/model-00015-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa0724f6c5426337f429855b4c59e0bf95d8da2fca7f44e509164bf1aa317d46
+size 4026531992
diff --git a/model-00016-of-00155.safetensors b/model-00016-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..3763e6b7d7eee47e5b22bf01f6705fc269af0eb7
--- /dev/null
+++ b/model-00016-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e58c2cf0ca136addd3f5c129ab6b5f2266c121a97a9256ed3198b9aad422f71
+size 169438244
diff --git a/model-00017-of-00155.safetensors b/model-00017-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..bcf4bbafc22eea205208346c2bc5ceeb2d535563
--- /dev/null
+++ b/model-00017-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:618b62820764ac2ee205816a4efefd28b8b3b7b5c5a1ffed4d2ca3959db64e4e
+size 4026531992
diff --git a/model-00018-of-00155.safetensors b/model-00018-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..3527ffdfcc419ce594b512ca73916b83bc95eedf
--- /dev/null
+++ b/model-00018-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c20761c60352617d41c9277e9bd0cceabb69523ce538788c46fced0cf91ca75
+size 169438396
diff --git a/model-00019-of-00155.safetensors b/model-00019-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..52948940d7e265d3dfd8a4917c53f96e3e4eacab
--- /dev/null
+++ b/model-00019-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d57eaf08774e881ed4ebc286292f273384b0e2a24f58a320d47e49857c96b96a
+size 4026531992
diff --git a/model-00020-of-00155.safetensors b/model-00020-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..77f6e57990243ec6e51d9e35c6b849e5a858ba89
--- /dev/null
+++ b/model-00020-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83f8879330ec2bfb707e36d18b5183f5a588cc2a7e1cc813c0b8dc0cffb841b9
+size 169438284
diff --git a/model-00021-of-00155.safetensors b/model-00021-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..50159051dfda5c6ec20bd254985cd885f4fc1e91
--- /dev/null
+++ b/model-00021-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:160ca8b2a98bc7b63a9107c7e6dafbb041bc109e668c16313374773a74401f5e
+size 4026531992
diff --git a/model-00022-of-00155.safetensors b/model-00022-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..59aeec2f7a0fb79aa7cc008e7b63684bd95eb094
--- /dev/null
+++ b/model-00022-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:950be62cc981471239196a31e1393c05b5c07710f8773d32f197b4002e3c111d
+size 169438284
diff --git a/model-00023-of-00155.safetensors b/model-00023-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..3d24a5daf83b2084ab579227ab7185f700739c44
--- /dev/null
+++ b/model-00023-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ca72ad4dabe121f4fde880ae081355c921a616e13481ab4cb1d091e11d30ac8
+size 4026531992
diff --git a/model-00024-of-00155.safetensors b/model-00024-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..895be36c04c6b52b5d60874d0401d0a06573e66f
--- /dev/null
+++ b/model-00024-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d052825b6c7eb3d35d61bb1a7431989b84fa507a99b0c5eba9351d246178fac
+size 169438284
diff --git a/model-00025-of-00155.safetensors b/model-00025-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b0ebf78eecd51f47037e070e9c323c706bf33635
--- /dev/null
+++ b/model-00025-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64471596c543df81d52d0bf691d4ca21b28ec19f03a78774725f58846bc636b3
+size 4026531992
diff --git a/model-00026-of-00155.safetensors b/model-00026-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9840de9ad8e2b7b5b92677f5c93fa4b4bf6dbfec
--- /dev/null
+++ b/model-00026-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:416e57726a13ba8715b6d5a5e13851ba95ae0166786883b7f50a71d9e204e6f4
+size 169438284
diff --git a/model-00027-of-00155.safetensors b/model-00027-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..f70f6cee508cdf546b55d01d897dce082469fc7b
--- /dev/null
+++ b/model-00027-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b27ccdf68027deb2d89b96733f44ae02c28205957951e1d5a0ccb19d15af05f
+size 4026531992
diff --git a/model-00028-of-00155.safetensors b/model-00028-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d8c7d63cff8c1daaca4c1211e9a0675105a1e269
--- /dev/null
+++ b/model-00028-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd66917ed5061a086da6053190b2484a49ec3012d3021ef75c03450b454886d5
+size 169438284
diff --git a/model-00029-of-00155.safetensors b/model-00029-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..83d0efb4b31b7217e6d02703b523da60887abc28
--- /dev/null
+++ b/model-00029-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:353ad9a5ed504382fd80555ead5cb247dcf9ab52060b014274af8061b82b6b15
+size 4026531992
diff --git a/model-00030-of-00155.safetensors b/model-00030-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9783d16f5449348077837e6890a496b8b823047c
--- /dev/null
+++ b/model-00030-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:449d4df572c901fd3c09afa890198bf10e96006788374475557dd958c8278cc9
+size 169438284
diff --git a/model-00031-of-00155.safetensors b/model-00031-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..f9ff199476cc985328ec6f1d652730b2b46286cc
--- /dev/null
+++ b/model-00031-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c599890131f8c3bec1d4c7683a724ed80ecc3f826246d5e9b60f11eb1ac490d9
+size 4026531992
diff --git a/model-00032-of-00155.safetensors b/model-00032-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ba5e929ca57558f92a1944bc2af04873ea3c75b1
--- /dev/null
+++ b/model-00032-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e57ccd45eecf0809bacfd82d7ea216280206d1edc9ba48b4eba42b21be4ef2d
+size 169438284
diff --git a/model-00033-of-00155.safetensors b/model-00033-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a562c9654a3cfb0b59b7193b5db6459e3de0736f
--- /dev/null
+++ b/model-00033-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4e22c1094e1ea7ca8a8ab4615e8ccc5b2210c58743c796147df0072dbeb22a1
+size 4026531992
diff --git a/model-00034-of-00155.safetensors b/model-00034-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9cf4ee451f154b43c0619fc549374992f4ca33b5
--- /dev/null
+++ b/model-00034-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa271bb63ac57361f39beb626681ea6ebb48fd78dd94a842d6588371adec8eb5
+size 169438284
diff --git a/model-00035-of-00155.safetensors b/model-00035-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..17c16c717980240501778960ebeb0db09860d0b4
--- /dev/null
+++ b/model-00035-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bedbbdf120394fe75565200234475ea4f1b29928cdba87b3a9f8a593e06dc96d
+size 4026531992
diff --git a/model-00036-of-00155.safetensors b/model-00036-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..df7573d157c3596d2ef29d641e9d262311d38ed5
--- /dev/null
+++ b/model-00036-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2461806db82a88db31f2886ea5702c85e9e43e74b61941a3087c73fadbb92c34
+size 169438284
diff --git a/model-00037-of-00155.safetensors b/model-00037-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c3396dee05db6071f0a3282a0e366e990f802eeb
--- /dev/null
+++ b/model-00037-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a077ad2d9b8924da94efb1dd6691e248ee4ad1271468409fd522611b792cead1
+size 4026531992
diff --git a/model-00038-of-00155.safetensors b/model-00038-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9fb5a9cebefa59cd697e09a60443549182209ca6
--- /dev/null
+++ b/model-00038-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:989273cc306fd1e32c538fc2490db41de7927c31f71c444ab9c494ee68508ca3
+size 169438284
diff --git a/model-00039-of-00155.safetensors b/model-00039-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2d79b3da67915d3c3bb9cfc8d60d8b655cb73fa9
--- /dev/null
+++ b/model-00039-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7cb963f713ac084448b6fc767f15a9a3ca1b1318361f7030770afc4928f17cdb
+size 4026531992
diff --git a/model-00040-of-00155.safetensors b/model-00040-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c7deb70e59666f2fdd61b7461b6c8ee93b741c90
--- /dev/null
+++ b/model-00040-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8b8ab9ba70831205339498bfe49055f5194b9349f0e0cda1986b93b14fdcebb
+size 169438284
diff --git a/model-00041-of-00155.safetensors b/model-00041-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..be425961c49820ad8a9995c6652519be6dc22e38
--- /dev/null
+++ b/model-00041-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45fcfb10ee7bfd84baa379efdcfb62aab78fc190d1a4e837de9ea77be26d5d27
+size 4026531992
diff --git a/model-00042-of-00155.safetensors b/model-00042-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..effd9b1394ad71192e47ef89734bf5d9198a4844
--- /dev/null
+++ b/model-00042-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc71f5151978672b0f47de152faf30dd34628ac883c6800187ced6e09c81cfb1
+size 169438284
diff --git a/model-00043-of-00155.safetensors b/model-00043-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..de2a1a82f3731a85980d7fd8c947d80bfea66111
--- /dev/null
+++ b/model-00043-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38f43a3fdb639d9af510f02a3d48c66668cbe6850c9e36897eeadbc0a28a52fa
+size 4026531992
diff --git a/model-00044-of-00155.safetensors b/model-00044-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..412e84679c3aafdf1a0ecb5cf6733a646c8b9765
--- /dev/null
+++ b/model-00044-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b69854117c9f9d93fd4cb969b0b1a5e927bbe300a4530306e04335c1ccb04470
+size 169438284
diff --git a/model-00045-of-00155.safetensors b/model-00045-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c03f273659e14ca127eb6fa463d2b947080f71c0
--- /dev/null
+++ b/model-00045-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ea007d3929f682ab85db150dffc3f1cd9753dd2420fff791d37bcca9d9a26bf
+size 4026531992
diff --git a/model-00046-of-00155.safetensors b/model-00046-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1f5864aff726771cc05b76c4558228bc43bd852d
--- /dev/null
+++ b/model-00046-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1efb81309fc3800e0ecbd8f54c6066fbeb14cf3c4a07375ac22391fdacbabd3
+size 169438284
diff --git a/model-00047-of-00155.safetensors b/model-00047-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..36370820497b43e3b0b036bf97b4190d6d5b663c
--- /dev/null
+++ b/model-00047-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b477be0f8f69ec38c4248b8debe572092ac7ecb0c839de647ee3efc3f316069
+size 4026531992
diff --git a/model-00048-of-00155.safetensors b/model-00048-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2ebdbb23e39d1030b2f5d116478165b094594804
--- /dev/null
+++ b/model-00048-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76474ca0d30c1634632623678933f27fc321adc212d0933e74ac610b5301eafe
+size 169438284
diff --git a/model-00049-of-00155.safetensors b/model-00049-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ffb100f770b25930265e31d942638e0470aae966
--- /dev/null
+++ b/model-00049-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1107093e8fab81fa9c360c38df7b84331a8cd338106ee5ddae406529151bdedf
+size 4026531992
diff --git a/model-00050-of-00155.safetensors b/model-00050-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..07a7b45eff1ba0e174fc1334fc5c868f19b3d6ab
--- /dev/null
+++ b/model-00050-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8168dc36ced491759dc9e53f55baf5354ce33fdd371fd0046a5b177d21d83880
+size 169438284
diff --git a/model-00051-of-00155.safetensors b/model-00051-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e177b1b4c6dc92366de5863c427ae5a179cd09be
--- /dev/null
+++ b/model-00051-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a87eee8f9ab945deac66f61d1adfd41734a702fcf81995a511f12bcb965337d
+size 4026531992
diff --git a/model-00052-of-00155.safetensors b/model-00052-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..35a7d826df0238c39ddf9ff69b579a20342efced
--- /dev/null
+++ b/model-00052-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb1aa77a67db7df3d709589cedbe15190584c7ab47c5eca79666e26a041652db
+size 169438284
diff --git a/model-00053-of-00155.safetensors b/model-00053-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b3c75e78d14c1edb8b68963d365b7de1d86d85e9
--- /dev/null
+++ b/model-00053-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06ab00d77300b566c1939a4181fd47839fa827188ffa72f3d39624d47861a4bd
+size 4026531992
diff --git a/model-00054-of-00155.safetensors b/model-00054-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..080d2b674f93fcc942e9aac542831d9b3040559b
--- /dev/null
+++ b/model-00054-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ad4262e1a9d038490b76a131eb8195e7a2302fc3af929aaefa768d53a8c552b
+size 169438284
diff --git a/model-00055-of-00155.safetensors b/model-00055-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..4cdc4891e50add045e99dec00ec1a48e689c4c2d
--- /dev/null
+++ b/model-00055-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ef3aacfa51de69061b48fbba984cc3e823ab3118f2c7566227201b2b327051d
+size 4026531992
diff --git a/model-00056-of-00155.safetensors b/model-00056-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..10ec92904daefafb61911efa5919cd13fb4bf497
--- /dev/null
+++ b/model-00056-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66f7676a785a179bf5e655f26a5ff40517ac6578af8b1387c10456a4d74b1d34
+size 169438284
diff --git a/model-00057-of-00155.safetensors b/model-00057-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9224e53db1997c81c260334b008a47cc4519115b
--- /dev/null
+++ b/model-00057-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfbdf8c7529e4225c624df04f0455ee6454cd7400d2f6766a0ea96ff04813445
+size 4026531992
diff --git a/model-00058-of-00155.safetensors b/model-00058-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..5c2c4c6d8b19a1fe7620b2a6afd2738b87bd2fac
--- /dev/null
+++ b/model-00058-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:395172cb3cd57d5dac8a0bdf3ae0397aabd0986bb9ca208b5fa517dd3eee9253
+size 169438284
diff --git a/model-00059-of-00155.safetensors b/model-00059-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..3e3a1d13143d2fb0185486fec3e69ef60e752ce9
--- /dev/null
+++ b/model-00059-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:809d048bae80aa0d32d78969f0154f2efb3157ee34e25d15023904152dde8d6d
+size 4026531992
diff --git a/model-00060-of-00155.safetensors b/model-00060-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e6870313e8c2c6958766bcd7f2cafe2cf070f0d3
--- /dev/null
+++ b/model-00060-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b172cab10545a2b7d5c7e6d52f0b5dd0230f884240fd6e23e79d0c845a13603
+size 169438284
diff --git a/model-00061-of-00155.safetensors b/model-00061-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..f597aaa4ded345c890840d0bb084c19ee0dd7ce5
--- /dev/null
+++ b/model-00061-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9841ff4f8d9dd69b333a6025ca367d0ef854b319561d9cee804c8fea1dc7b06b
+size 4026531992
diff --git a/model-00062-of-00155.safetensors b/model-00062-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..89c6e3acea8a6515e9d9e4b6a96d57664727d7d7
--- /dev/null
+++ b/model-00062-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b82be6c2ee58afef25cc6b4e6018cb9081a9551090a1188a4565c87aee143495
+size 169438284
diff --git a/model-00063-of-00155.safetensors b/model-00063-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e49ea293960eeb0581292045d722ebe3baa1e4f8
--- /dev/null
+++ b/model-00063-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75e78de494af7663707ae81e9da1e0e11ac1136269dd299a99b64656cfd65351
+size 4026531992
diff --git a/model-00064-of-00155.safetensors b/model-00064-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..79a6f70a5767ac527c0bbfd3bdbe641dae1219d7
--- /dev/null
+++ b/model-00064-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6e6651710b59c78c7865b57f52bfc57a90f29eb0ce738fc00b4d33d2d69f8c5
+size 169438284
diff --git a/model-00065-of-00155.safetensors b/model-00065-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d491df27391c5087212c5741bdcd41284cf307f5
--- /dev/null
+++ b/model-00065-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af63ba70e31bfb969197dbc77077e8bd7510678f9fd6a992ddea9920b63aa970
+size 4026531992
diff --git a/model-00066-of-00155.safetensors b/model-00066-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..701e23e6f6f92ecc524948bb191a792e663e82f1
--- /dev/null
+++ b/model-00066-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:276c6096245e2f0a9789808c7521195a94b402c71daa4967198a29dbcb00617f
+size 169438284
diff --git a/model-00067-of-00155.safetensors b/model-00067-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..43aee33bc35a4b724ff9e6765cebd87a6afbd24c
--- /dev/null
+++ b/model-00067-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7381d36a3ae2038f8ce2680a1c4a04ad1bb608832e05dd24b769e2cb69b7826
+size 4026531992
diff --git a/model-00068-of-00155.safetensors b/model-00068-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..8594cf280d55b3b142aea4380df1c9eb463f1da3
--- /dev/null
+++ b/model-00068-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4a9b9da9b6c83bcbb88103c86aed1f0aa17789d0962dc536bc361ffe639d7e7
+size 169438284
diff --git a/model-00069-of-00155.safetensors b/model-00069-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..41792499d837265fb665d1d3353f4e9cd9c9948b
--- /dev/null
+++ b/model-00069-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efd66d43ccc3688839d6364c1881e1e17180318c2a4f0125df4fe2df6512cf1b
+size 4026531992
diff --git a/model-00070-of-00155.safetensors b/model-00070-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e197a9f80c3643612a4a90c4730fcb8a534318ab
--- /dev/null
+++ b/model-00070-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0613fc561c9145684b0a50a5a0212c95460036a406458b076eba307c93f1a345
+size 169438284
diff --git a/model-00071-of-00155.safetensors b/model-00071-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0628f6e21007450a3ef25c79df61cc6a61cc9d12
--- /dev/null
+++ b/model-00071-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52b1843ce646853f9d9a3e48e76789991b66628a5d5e4f4555e5830f7dec96c5
+size 4026531992
diff --git a/model-00072-of-00155.safetensors b/model-00072-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..97276988b7253bcf0634ef2cd1a3207600dd9840
--- /dev/null
+++ b/model-00072-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91d1c529dc069a3563f3aba83c9829ab8172431ec243eeea9a85e3a101c5378d
+size 169438284
diff --git a/model-00073-of-00155.safetensors b/model-00073-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c35b6e361a7ea7f28d1b60b94b01dac327a3fca1
--- /dev/null
+++ b/model-00073-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ea9fd4d2c48e81f9a9eba03d7155d20a95019fb0a216cd935b1f9aacbc241ef
+size 4026531992
diff --git a/model-00074-of-00155.safetensors b/model-00074-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e8f9aaf79c9c1ff2f535fed4c54764e64cc3c307
--- /dev/null
+++ b/model-00074-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:616d2e8d497da4ee27c98124c06d2a7db5b679181b5db9909f31040e251eaf96
+size 169438284
diff --git a/model-00075-of-00155.safetensors b/model-00075-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..48879cc63e1a07d7f0eaa7623c86bb9203fd38e3
--- /dev/null
+++ b/model-00075-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2476514413013280e8b2989722d10b14a25bce968ded07e1f6c435f45f78f89b
+size 4026531992
diff --git a/model-00076-of-00155.safetensors b/model-00076-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..8e82db7b6a1f630f1540621fcfc9afbcd8eed3b0
--- /dev/null
+++ b/model-00076-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:518388e89a8be326d9b6d02d93a2f3afa474b9ade13d72d72bd9b7397ab69f76
+size 169438284
diff --git a/model-00077-of-00155.safetensors b/model-00077-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b4e14b97e7d070d34f54bc1c5a1481432f5a0acf
--- /dev/null
+++ b/model-00077-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30dace558939b1feb66d7c621f84644388faa88003eb0467ebb8c44197044f58
+size 4026531992
diff --git a/model-00078-of-00155.safetensors b/model-00078-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..7a59c8e14ccf4e7f22abb8eb3ad7392ac9c3e2ee
--- /dev/null
+++ b/model-00078-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac54795cc016ecb11e3f93b33d89d5d4d4a9aa12dd8ad044ec8968bb1f994299
+size 169438284
diff --git a/model-00079-of-00155.safetensors b/model-00079-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9989595147172c9c25906d4133514b01e09281ad
--- /dev/null
+++ b/model-00079-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91fd648ec8327009002a73edc8650020312783f0f9d09a27550a32e7ce821942
+size 4026531992
diff --git a/model-00080-of-00155.safetensors b/model-00080-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..12cc8415d3e47f8ba83ddfe1a5bfea28e4c34de8
--- /dev/null
+++ b/model-00080-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:022e85c3bbaeee23e7d3daa07a8e77db3eddf7cd39a3f9befec0ac196d948357
+size 169438284
diff --git a/model-00081-of-00155.safetensors b/model-00081-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e78c0e45f65c621a77400cb3288c28f00303b2cc
--- /dev/null
+++ b/model-00081-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4979d456a7c4c482934dbfba51a14406d96a9ef70b04243546bb7ba6e07cfe6d
+size 4026531992
diff --git a/model-00082-of-00155.safetensors b/model-00082-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..28dd9ead803f58073e80ad87fca9ff9751a62e82
--- /dev/null
+++ b/model-00082-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d48101a88434ba13aa8b990c909cbb838060607eeb9b3dac384b9b711a265623
+size 169438284
diff --git a/model-00083-of-00155.safetensors b/model-00083-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0e6680a72899db30a69415ef55a79a44b592decc
--- /dev/null
+++ b/model-00083-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04f198a27c4a0d099832fc83230a91923721ef341aa8d670f59c3d5cc9cbfff0
+size 4026531992
diff --git a/model-00084-of-00155.safetensors b/model-00084-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..62d8f18d0714983d121b9234d76008132ae2e755
--- /dev/null
+++ b/model-00084-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04db15e4a9f4283d0ad4732a0c97cdf9839ae7440c9d768e465d75f9e9ec01eb
+size 169438284
diff --git a/model-00085-of-00155.safetensors b/model-00085-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..bd3d7886630ec32845ce8b9c76b9d92efd1ba1d0
--- /dev/null
+++ b/model-00085-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc14214082ea041c0fb2259581a8f448ed88774ce8f17a8decf37215e7605b0c
+size 4026531992
diff --git a/model-00086-of-00155.safetensors b/model-00086-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c1c58537dacb8e030a43abffbd4390736656468e
--- /dev/null
+++ b/model-00086-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ff5acac84bf03cf2c005a2040e0d091affafe95bd20a1772d7e8b24d6319551
+size 169438284
diff --git a/model-00087-of-00155.safetensors b/model-00087-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..371d8c1893869984d0762f2e1cb4e7b69e31a1d7
--- /dev/null
+++ b/model-00087-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdacccdd42ff615eb563d9c002a02304d0b31ea2c2304af249fa7a2f6321a6c8
+size 4026531992
diff --git a/model-00088-of-00155.safetensors b/model-00088-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..802b4c4d014a723f225f1558ea1286d9e8d1516c
--- /dev/null
+++ b/model-00088-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:478ea0fdc4129d06521c5872c742ff4901ad8592e0d8677acf965a68a95e06b1
+size 169438284
diff --git a/model-00089-of-00155.safetensors b/model-00089-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..f49386a14b92b55ee2c43e35ccd261d3a6a7d5e9
--- /dev/null
+++ b/model-00089-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ac155ae0d9c9148146a451c870d2d8638ef26ead892890969ebc07bf83a8522
+size 4026531992
diff --git a/model-00090-of-00155.safetensors b/model-00090-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e51773ea860a5e6eed587f23d6c88f5df51897d4
--- /dev/null
+++ b/model-00090-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b41b0394f20b86d2483f1949012c3d53e25b73da34148adefab0f3096105fc08
+size 169438284
diff --git a/model-00091-of-00155.safetensors b/model-00091-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..52207ca37d45493c4cd9ca6eb23f3ce5d69d772f
--- /dev/null
+++ b/model-00091-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f054dc5a455020b386f51b2f29d77a0896164f975a0a7f8c7a78f5d5a7b7fdd
+size 4026531992
diff --git a/model-00092-of-00155.safetensors b/model-00092-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..4077d67d47a686a08f7876d256beed78e37edf2e
--- /dev/null
+++ b/model-00092-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f9a1285e5e0b2b22e77e84bc1cf148de3cd30734e9ce507c2be457e196b0228
+size 169438284
diff --git a/model-00093-of-00155.safetensors b/model-00093-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b18ae10d8b0ec42ae55486b4582119c25d9ea1b7
--- /dev/null
+++ b/model-00093-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ce99670ae67b8dbff89244456f56c8f3e5ec10540f59aeefb929c535a8f2d0d
+size 4026531992
diff --git a/model-00094-of-00155.safetensors b/model-00094-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6eb5525a949da73fc63b5dac26bb6bec868699fd
--- /dev/null
+++ b/model-00094-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67673b6c6a4aa5e7b946a9aadc4991542bfe3bc99a02c3d25397403196a1e60a
+size 169438284
diff --git a/model-00095-of-00155.safetensors b/model-00095-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..76b06c94f7651d1fba9df5cc0684ceba987f2164
--- /dev/null
+++ b/model-00095-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e26be94794634851c455a026c6e37537194911112d8ec6e37670f13c1f01b92
+size 4026531992
diff --git a/model-00096-of-00155.safetensors b/model-00096-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..28716af027b2c3e2f5ea2bdddf776e86c5c44f55
--- /dev/null
+++ b/model-00096-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:355d21a7ac67edad8a19a11e94241ba4e1ddf14ec6f10d56914ee08829cf9930
+size 169438284
diff --git a/model-00097-of-00155.safetensors b/model-00097-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0579b44be563f96edc2f83cf9be9768fc20c9465
--- /dev/null
+++ b/model-00097-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9349da1955d5e42697062b7fe2e1b4f99e2a7560e27d896af84f5b8aec139d2e
+size 4026531992
diff --git a/model-00098-of-00155.safetensors b/model-00098-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..fd6dcc9b050cc7855f3901f2af1baecdf59342ec
--- /dev/null
+++ b/model-00098-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a915ccdbca688719776a8e40b4a98468ef244b44edbf9805b262a0b20e91ba6b
+size 169438284
diff --git a/model-00099-of-00155.safetensors b/model-00099-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..55bcb11673148ceabec5ec8371377931b500256a
--- /dev/null
+++ b/model-00099-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b595e6bbf39316039ac3d16b2f2baec0b592f4e133b616265587ff44f4d42f9
+size 4026531992
diff --git a/model-00100-of-00155.safetensors b/model-00100-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..be0723d3cddadbc03d505d0f871c57c75ee51474
--- /dev/null
+++ b/model-00100-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04c3d71aaf630b6e4563b4cb905bb8a5e6acfc487564955f9be0d79800b459c8
+size 169438284
diff --git a/model-00101-of-00155.safetensors b/model-00101-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0ad4ea668b8a50cc2709ffae06aa2b6d7a2b02f6
--- /dev/null
+++ b/model-00101-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa278454204f5007f80876d0a2a642256afdbd6b86b10c6e39d952947eae91fd
+size 4026531992
diff --git a/model-00102-of-00155.safetensors b/model-00102-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..829f866d1015211c5754d79dd35daf57f4b780ee
--- /dev/null
+++ b/model-00102-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ae9785cc180228fa920e303bf87af037bf66e97fafc90c77187dde2c14a785b
+size 169438284
diff --git a/model-00103-of-00155.safetensors b/model-00103-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..23e1a4bb732c3c7a6d29bd44a48ce0b554aea6ce
--- /dev/null
+++ b/model-00103-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a337b2b39273015fafd3b5fa215a9c85e87b0d68311904ad92376eef38b4cc26
+size 4026531992
diff --git a/model-00104-of-00155.safetensors b/model-00104-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..cd4b2eda7cb34e9cf29d5eb6afe3ad1da398119b
--- /dev/null
+++ b/model-00104-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2153498182d0d827f9baea953fa5c3ca4b7ea256ff514c5440cbcf0b2278cd50
+size 2342129476
diff --git a/model-00105-of-00155.safetensors b/model-00105-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1b3ff9c29150d5e4bee4320ee07638e11fcc6dcd
--- /dev/null
+++ b/model-00105-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d40f91d1c4f6df0459736f62cea5d4fea5eab2e24d50e54ecf3c3d47c5fa8f11
+size 8053063840
diff --git a/model-00106-of-00155.safetensors b/model-00106-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..39b88e14315b448bea33c3e7cd8db3a26b2c6938
--- /dev/null
+++ b/model-00106-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40b51c186f5fba460262fc98c980941d3e3b30e622c8408732590f9ca4d46497
+size 8053063840
diff --git a/model-00107-of-00155.safetensors b/model-00107-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..8ee0fdac48bf96d8d377e1718298c024b4e066aa
--- /dev/null
+++ b/model-00107-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44f67577d8e26a21ee46beb45ad18781e66d8e5010024f28f2a7126860f9c681
+size 8053063840
diff --git a/model-00108-of-00155.safetensors b/model-00108-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..5600c1569425809ff8c4ca9b8dc85d5c4afd471d
--- /dev/null
+++ b/model-00108-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58cccfe9cb739dcb954aa75dcf26b39e7e34b6b1b75de3b419a485c97ea3f9be
+size 8053063840
diff --git a/model-00109-of-00155.safetensors b/model-00109-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0d57da9769f23887fa73fe2adebcc4db7ab88cee
--- /dev/null
+++ b/model-00109-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16fc0c69203c05cf90d99b21b2d0e91eeeb5b32f616a9ecdded7c84de011afc7
+size 8053063840
diff --git a/model-00110-of-00155.safetensors b/model-00110-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e1a628bcf20877439fb548e6dbddc1f71b7c666f
--- /dev/null
+++ b/model-00110-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d4b7a4694410d58def84e53e79240efb6b0f5babc4b9770a87b547a5cffc44a
+size 8053063840
diff --git a/model-00111-of-00155.safetensors b/model-00111-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..efc01792de9f3bca9bf0bd421d63e87b944291bb
--- /dev/null
+++ b/model-00111-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd9e5f68ab62bc60879c1ac515c11a89faf9d29ac676696abbe3d55522373700
+size 8053063840
diff --git a/model-00112-of-00155.safetensors b/model-00112-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..f0332bbef8a29a59496d30fae3f244be1c648af8
--- /dev/null
+++ b/model-00112-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37fd30d7170ab9a6995d8dce7c602fec6bebec98dd1694f3ff07b924eff1070c
+size 8053063840
diff --git a/model-00113-of-00155.safetensors b/model-00113-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..86a983ec5f895c00e56849ca142415e2a960fee4
--- /dev/null
+++ b/model-00113-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99b496e34eb0c3ba5517a2f3f6f15e0afabb736c796f9c3e19350eb839518863
+size 8053063840
diff --git a/model-00114-of-00155.safetensors b/model-00114-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..25d5f757105b34077469a24005160492ffc8cdfd
--- /dev/null
+++ b/model-00114-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:465bb716aee58a3783d68132f35298f402cc4fd768fbe80969681054368f3474
+size 8053063840
diff --git a/model-00115-of-00155.safetensors b/model-00115-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..cfc5ce812577fa37535a4184bb85b31f80bef5cc
--- /dev/null
+++ b/model-00115-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e3599c97f269ff790bf83b86c20953ced31041214ebb129041e6a25ba39eddd
+size 8053063840
diff --git a/model-00116-of-00155.safetensors b/model-00116-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..61d06e2cd823590c36ec9a172bc464ee9b2e01dd
--- /dev/null
+++ b/model-00116-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85b40f02742c9b09c3cfe2f794e29b2a8f58b414587de94cb3639bdf57e8af39
+size 8053063840
diff --git a/model-00117-of-00155.safetensors b/model-00117-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ada673c940cce9f241faae81a605c134010f339b
--- /dev/null
+++ b/model-00117-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92154208d08a1aad664e8f0db0e3356c162f79407e05cf0a532fd107b91c43d3
+size 8053063840
diff --git a/model-00118-of-00155.safetensors b/model-00118-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9591153711f207dc5d2f9fc84185a1424e1703e0
--- /dev/null
+++ b/model-00118-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5694c7c45e5f00e9d6f57b429dfab817cef76808db2743923bec98581d81c424
+size 8053063840
diff --git a/model-00119-of-00155.safetensors b/model-00119-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..87a0a536227864618731424da36ec3e0c63c3ff1
--- /dev/null
+++ b/model-00119-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89dcc8a507040b8ea5dd1a8dd82874ef32d8e1ed557e1f3392243341085cc323
+size 8053063840
diff --git a/model-00120-of-00155.safetensors b/model-00120-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c65b5e74e099e5002d321d6a40f29ce05ef8dbb8
--- /dev/null
+++ b/model-00120-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcebfe96bae0362c82db6ec9de80e402c7ba990b698e9d17a2f2c83f75452b7d
+size 8053063840
diff --git a/model-00121-of-00155.safetensors b/model-00121-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0a57f7a0bed9b4ec6f5194f9c5abc1814cedc02b
--- /dev/null
+++ b/model-00121-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7a13ef79b394185af440e0b9e02dd914b4025fadc2457e40cb7beab14913f3e
+size 8053063840
diff --git a/model-00122-of-00155.safetensors b/model-00122-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..18c2a206107a79a01a6b6f3dbee8839501ed3bdc
--- /dev/null
+++ b/model-00122-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7ce6ed94ad768f84eb4ec857f39abb1d567afbee4ea58cf45a9856539b05360
+size 8053063840
diff --git a/model-00123-of-00155.safetensors b/model-00123-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2980c3443a5126bd878c2ea80b4c1e09227f9f4e
--- /dev/null
+++ b/model-00123-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ef730e07c789f14a190b8e09d5e55e4fd41522417a1517cd29430836fef840a
+size 8053063840
diff --git a/model-00124-of-00155.safetensors b/model-00124-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6f64379f238feed7689b97645775f32b91638a92
--- /dev/null
+++ b/model-00124-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c15b200e1329dd2db5704d43fe3e5d59962e1902514e47cd3f5165a262ebc8e
+size 8053063840
diff --git a/model-00125-of-00155.safetensors b/model-00125-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..888e0bd1b64e55f746da89e305777ca10c02c2bd
--- /dev/null
+++ b/model-00125-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2feec6744eec67e1a1f25ed1fa2189e43ef8cf5e6170cccad66035ed1aa748e
+size 8053063840
diff --git a/model-00126-of-00155.safetensors b/model-00126-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..693e817a5fd5f6d1dae8f39f679bcb7b133bd19f
--- /dev/null
+++ b/model-00126-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8676919c4624f18e1a874ce0f30e69745817cc55e961fa4e28a5618e6f34d905
+size 8053063840
diff --git a/model-00127-of-00155.safetensors b/model-00127-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b8560ab595e14b911b9a560e20e65942999d3b48
--- /dev/null
+++ b/model-00127-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ea9f3a669234bb08b0ea055f38addd373c39573cd10d1968c785ab1fb0dc641
+size 8053063840
diff --git a/model-00128-of-00155.safetensors b/model-00128-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..bb0b663fd7376f42cd3924d7e43d97697788d66d
--- /dev/null
+++ b/model-00128-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc05d7a8a69610a92721720e44597bf0abbe22d7a33bc8a53e38357ab8d6afe5
+size 8053063840
diff --git a/model-00129-of-00155.safetensors b/model-00129-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6d0dd4f83dab5966f4d3faf98be62d1e1f862498
--- /dev/null
+++ b/model-00129-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3bbc4133f61b62c001a9400a0f45714b9d495b588c6eb57a7c203050fd0975e
+size 8053063840
diff --git a/model-00130-of-00155.safetensors b/model-00130-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..52a35d02c4b45ec34791f71c999939e9f056977b
--- /dev/null
+++ b/model-00130-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14387dfcded2b33cec457710ba648911cef46822f62b15da209180e0c5bedfb1
+size 8053063840
diff --git a/model-00131-of-00155.safetensors b/model-00131-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a66e4650c591c41a6540be5d21b13ee33a91922f
--- /dev/null
+++ b/model-00131-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c99f730c83fc928e29005e3efc3a4a63f1c5e642f15dd6ed9e91e53a377ced04
+size 8053063840
diff --git a/model-00132-of-00155.safetensors b/model-00132-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..fa480901a169096114a364c65362986765d3bfaf
--- /dev/null
+++ b/model-00132-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:891a8042b7157fe4a1c6fbe648749194b1275e04c0bab95b0141a24325cec1ab
+size 8053063840
diff --git a/model-00133-of-00155.safetensors b/model-00133-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..686fa70914099569457c627cfab00eb4942775cb
--- /dev/null
+++ b/model-00133-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64c47f9871a37ba1f856ff50dfeebdf90dfa93effc84cbe15f7cef2fab27f669
+size 8053063840
diff --git a/model-00134-of-00155.safetensors b/model-00134-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0318698fba5c0930dd54e2caa75bb94870f5a279
--- /dev/null
+++ b/model-00134-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec225e0939b440e17a05795ee5d322acefdddbdef0e574be6fc76976cac37a18
+size 8053063840
diff --git a/model-00135-of-00155.safetensors b/model-00135-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e49ee21c8be65013b5da9439feb5b58c1c0ed7ec
--- /dev/null
+++ b/model-00135-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bada08f46b3b28999d3e49b48f69e398029330303fcc12f8842c93c38bf223b5
+size 8053063840
diff --git a/model-00136-of-00155.safetensors b/model-00136-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2ac6196bcadf7647095d3f0e6a93f0f75770941c
--- /dev/null
+++ b/model-00136-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:240b1089e7ee4ccd0a8aad613365594b77eb28bfb902cd53839d61aaf17b05af
+size 8053063840
diff --git a/model-00137-of-00155.safetensors b/model-00137-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..7b0f66b382acfc2a7450f128fd14df5ff859936b
--- /dev/null
+++ b/model-00137-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb28072e075398d4229268d90b0e281e96c5f5323efd8f19b451b43983002351
+size 8053063840
diff --git a/model-00138-of-00155.safetensors b/model-00138-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c38cd870044c4443ec417297031d4eaf8e86c864
--- /dev/null
+++ b/model-00138-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e10652d0f03d6d0e29ae6a0a6f46ec9d964abd64e2f60c0f4b050ed336c7b5dc
+size 8053063840
diff --git a/model-00139-of-00155.safetensors b/model-00139-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..afefd4a3b664c47977301a1c3fe6ad93e8e9a992
--- /dev/null
+++ b/model-00139-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82047a3a0dfd45641ac1313761a156c20399b4abe998eecfe0f153d130b59f3b
+size 8053063840
diff --git a/model-00140-of-00155.safetensors b/model-00140-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6d2b23d986790667dd4e165ee5170227b6cbd9cc
--- /dev/null
+++ b/model-00140-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b150e276c0a7b9b8cb8573fa36d2d62ea8464000cb5684b7ca937249b184de10
+size 8053063840
diff --git a/model-00141-of-00155.safetensors b/model-00141-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a09ab857dc36379f738572a6e9eb9d40db22739c
--- /dev/null
+++ b/model-00141-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7805bd272cfdc761967793b0932065998a0d80c89b6b3417ed3f41ae66019b6
+size 8053063840
diff --git a/model-00142-of-00155.safetensors b/model-00142-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..3d0062111602146285267cff369312b21bfccc1f
--- /dev/null
+++ b/model-00142-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:068e39aa2b82a2db43fab2318f00567e72c24c5acd1de0605428dafcb3487fd4
+size 8053063840
diff --git a/model-00143-of-00155.safetensors b/model-00143-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b9eeaec882a8ecb6b852efe491bc6dea2c012f63
--- /dev/null
+++ b/model-00143-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4801532d462c86490e286e2a8cf6fd7e76a7366c223c46aa6a7ddf24189a6d2a
+size 8053063840
diff --git a/model-00144-of-00155.safetensors b/model-00144-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..4b4ee42b628d10c77ec6d3d082a664163a5ea302
--- /dev/null
+++ b/model-00144-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1b776e1a68c0110a7607ebcedb9e1f31af2b312684b0fde38c3a187a6ef007f
+size 8053063840
diff --git a/model-00145-of-00155.safetensors b/model-00145-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..be754987dd94ffe1ece0b7e23355b61ab2a1cc4e
--- /dev/null
+++ b/model-00145-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b6f1cf56e0e4c7e717878b1c7e8231ef3ed276904e47f8c7dc986bd3ce99661
+size 8053063840
diff --git a/model-00146-of-00155.safetensors b/model-00146-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..319dc44d5d6464e2b4043f9d91dc9b98984e88a6
--- /dev/null
+++ b/model-00146-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:705efc65e74ecda5bcda8a9a8b9889b85fde9691a2e153675186d1cc6474c62e
+size 8053063840
diff --git a/model-00147-of-00155.safetensors b/model-00147-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..44fcec264f9dd75b30d0cf96a98a2f8092e6fd9c
--- /dev/null
+++ b/model-00147-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6601df2f44c4db63b13c08660e7e64cef004dc0d41ec2be89352321d7f00a22
+size 8053063840
diff --git a/model-00148-of-00155.safetensors b/model-00148-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..3995b448d9df37599e8422157bd093abbc3e7a0d
--- /dev/null
+++ b/model-00148-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0424d6738198f3928ac677ea3e16847e3d9f74f5d380f5cd3d59bc9ba783a73
+size 8053063840
diff --git a/model-00149-of-00155.safetensors b/model-00149-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..75af554183dd747956107b01826a596c3d2b6daf
--- /dev/null
+++ b/model-00149-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6faabe3e298e6bbba2e8b384e1721b5cd569c7070eea37bf43731e0a1b9afa6b
+size 8053063840
diff --git a/model-00150-of-00155.safetensors b/model-00150-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d78331f2578ea42d93d76226c92484c44965d968
--- /dev/null
+++ b/model-00150-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b79b0d0bae48946f1a2f80aee1878569632461aaafb3f2d3bf89bd9176144b88
+size 8053063840
diff --git a/model-00151-of-00155.safetensors b/model-00151-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c54c8bc880ff0b8c485b930073aa51a8fafed78a
--- /dev/null
+++ b/model-00151-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72b8366e476ef269a96c10aba08fca587b88c9f0d351165458bd78046ebea458
+size 8053063840
diff --git a/model-00152-of-00155.safetensors b/model-00152-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a3db8216025c229fb0820b4d9722327bcd81692d
--- /dev/null
+++ b/model-00152-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07b29b5062a95bf315efaca00fb91abc8809a1d5c75b7ce7fe276ec6e9905da9
+size 8053063840
diff --git a/model-00153-of-00155.safetensors b/model-00153-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..eb592dbbbdf5a5392b96e55950bc333e6b67fc7f
--- /dev/null
+++ b/model-00153-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d63225ae7b5753b45fc8bd8f0c5dc74337e0775f63f484d4065236b37c60d41
+size 8053063840
diff --git a/model-00154-of-00155.safetensors b/model-00154-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..7356aaea5ccb76f722baefc81446f9ac44bb9230
--- /dev/null
+++ b/model-00154-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdf42bfd514a990465fbad2609547ab84aeca27ad760ec0891a4c92d6d278800
+size 8053063840
diff --git a/model-00155-of-00155.safetensors b/model-00155-of-00155.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c0f2ed9bdcd6efe6151da6f4f8cdd7b2aeeee0f9
--- /dev/null
+++ b/model-00155-of-00155.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80f0d1cf67441043c7577760ed2cfd6feb6ab1a155fd476938fb21252cd1a55b
+size 8053063840
diff --git a/model.safetensors.index.json b/model.safetensors.index.json
new file mode 100644
index 0000000000000000000000000000000000000000..696197cb12f975ca481628a5ebf4aa84d0b283c1
--- /dev/null
+++ b/model.safetensors.index.json
@@ -0,0 +1,2243 @@
+{
+ "metadata": {
+ "total_size": 629683551500
+ },
+ "weight_map": {
+ "lm_head.weight": "model-00104-of-00155.safetensors",
+ "model.embed_tokens.weight": "model-00001-of-00155.safetensors",
+ "model.layers.0.input_layernorm.weight": "model-00001-of-00155.safetensors",
+ "model.layers.0.mhc_attn.alpha_post": "model-00001-of-00155.safetensors",
+ "model.layers.0.mhc_attn.alpha_pre": "model-00001-of-00155.safetensors",
+ "model.layers.0.mhc_attn.alpha_res": "model-00001-of-00155.safetensors",
+ "model.layers.0.mhc_attn.bias_post": "model-00001-of-00155.safetensors",
+ "model.layers.0.mhc_attn.bias_pre": "model-00001-of-00155.safetensors",
+ "model.layers.0.mhc_attn.bias_res": "model-00001-of-00155.safetensors",
+ "model.layers.0.mhc_attn.proj_post.weight": "model-00001-of-00155.safetensors",
+ "model.layers.0.mhc_attn.proj_pre.weight": "model-00001-of-00155.safetensors",
+ "model.layers.0.mhc_attn.proj_res.weight": "model-00001-of-00155.safetensors",
+ "model.layers.0.mhc_attn.rms_norm.weight": "model-00001-of-00155.safetensors",
+ "model.layers.0.mhc_ffn.alpha_post": "model-00001-of-00155.safetensors",
+ "model.layers.0.mhc_ffn.alpha_pre": "model-00001-of-00155.safetensors",
+ "model.layers.0.mhc_ffn.alpha_res": "model-00001-of-00155.safetensors",
+ "model.layers.0.mhc_ffn.bias_post": "model-00001-of-00155.safetensors",
+ "model.layers.0.mhc_ffn.bias_pre": "model-00001-of-00155.safetensors",
+ "model.layers.0.mhc_ffn.bias_res": "model-00001-of-00155.safetensors",
+ "model.layers.0.mhc_ffn.proj_post.weight": "model-00001-of-00155.safetensors",
+ "model.layers.0.mhc_ffn.proj_pre.weight": "model-00001-of-00155.safetensors",
+ "model.layers.0.mhc_ffn.proj_res.weight": "model-00001-of-00155.safetensors",
+ "model.layers.0.mhc_ffn.rms_norm.weight": "model-00001-of-00155.safetensors",
+ "model.layers.0.mlp.act_fn.bias": "model-00001-of-00155.safetensors",
+ "model.layers.0.mlp.act_fn.weight": "model-00001-of-00155.safetensors",
+ "model.layers.0.mlp.down_proj.weight": "model-00001-of-00155.safetensors",
+ "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00155.safetensors",
+ "model.layers.0.mlp.up_proj.weight": "model-00001-of-00155.safetensors",
+ "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00155.safetensors",
+ "model.layers.0.self_attn.kv_norm.weight": "model-00001-of-00155.safetensors",
+ "model.layers.0.self_attn.lambda_proj.weight": "model-00001-of-00155.safetensors",
+ "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00155.safetensors",
+ "model.layers.0.self_attn.wkv_a.weight": "model-00001-of-00155.safetensors",
+ "model.layers.0.self_attn.wkv_b.weight": "model-00001-of-00155.safetensors",
+ "model.layers.0.self_attn.wo.weight": "model-00001-of-00155.safetensors",
+ "model.layers.0.self_attn.wq_a.weight": "model-00001-of-00155.safetensors",
+ "model.layers.0.self_attn.wq_b.weight": "model-00001-of-00155.safetensors",
+ "model.layers.0.self_attn.wq_b_gate.weight": "model-00001-of-00155.safetensors",
+ "model.layers.1.input_layernorm.weight": "model-00002-of-00155.safetensors",
+ "model.layers.1.mhc_attn.alpha_post": "model-00002-of-00155.safetensors",
+ "model.layers.1.mhc_attn.alpha_pre": "model-00002-of-00155.safetensors",
+ "model.layers.1.mhc_attn.alpha_res": "model-00002-of-00155.safetensors",
+ "model.layers.1.mhc_attn.bias_post": "model-00002-of-00155.safetensors",
+ "model.layers.1.mhc_attn.bias_pre": "model-00002-of-00155.safetensors",
+ "model.layers.1.mhc_attn.bias_res": "model-00002-of-00155.safetensors",
+ "model.layers.1.mhc_attn.proj_post.weight": "model-00002-of-00155.safetensors",
+ "model.layers.1.mhc_attn.proj_pre.weight": "model-00002-of-00155.safetensors",
+ "model.layers.1.mhc_attn.proj_res.weight": "model-00002-of-00155.safetensors",
+ "model.layers.1.mhc_attn.rms_norm.weight": "model-00002-of-00155.safetensors",
+ "model.layers.1.mhc_ffn.alpha_post": "model-00002-of-00155.safetensors",
+ "model.layers.1.mhc_ffn.alpha_pre": "model-00002-of-00155.safetensors",
+ "model.layers.1.mhc_ffn.alpha_res": "model-00002-of-00155.safetensors",
+ "model.layers.1.mhc_ffn.bias_post": "model-00002-of-00155.safetensors",
+ "model.layers.1.mhc_ffn.bias_pre": "model-00002-of-00155.safetensors",
+ "model.layers.1.mhc_ffn.bias_res": "model-00002-of-00155.safetensors",
+ "model.layers.1.mhc_ffn.proj_post.weight": "model-00002-of-00155.safetensors",
+ "model.layers.1.mhc_ffn.proj_pre.weight": "model-00002-of-00155.safetensors",
+ "model.layers.1.mhc_ffn.proj_res.weight": "model-00002-of-00155.safetensors",
+ "model.layers.1.mhc_ffn.rms_norm.weight": "model-00002-of-00155.safetensors",
+ "model.layers.1.mlp.act_fn.bias": "model-00001-of-00155.safetensors",
+ "model.layers.1.mlp.act_fn.weight": "model-00001-of-00155.safetensors",
+ "model.layers.1.mlp.down_proj.weight": "model-00002-of-00155.safetensors",
+ "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00155.safetensors",
+ "model.layers.1.mlp.up_proj.weight": "model-00002-of-00155.safetensors",
+ "model.layers.1.post_attention_layernorm.weight": "model-00002-of-00155.safetensors",
+ "model.layers.1.self_attn.kv_norm.weight": "model-00001-of-00155.safetensors",
+ "model.layers.1.self_attn.lambda_proj.weight": "model-00001-of-00155.safetensors",
+ "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00155.safetensors",
+ "model.layers.1.self_attn.wkv_a.weight": "model-00001-of-00155.safetensors",
+ "model.layers.1.self_attn.wkv_b.weight": "model-00001-of-00155.safetensors",
+ "model.layers.1.self_attn.wo.weight": "model-00001-of-00155.safetensors",
+ "model.layers.1.self_attn.wq_a.weight": "model-00001-of-00155.safetensors",
+ "model.layers.1.self_attn.wq_b.weight": "model-00001-of-00155.safetensors",
+ "model.layers.1.self_attn.wq_b_gate.weight": "model-00001-of-00155.safetensors",
+ "model.layers.10.input_layernorm.weight": "model-00020-of-00155.safetensors",
+ "model.layers.10.mhc_attn.alpha_post": "model-00020-of-00155.safetensors",
+ "model.layers.10.mhc_attn.alpha_pre": "model-00020-of-00155.safetensors",
+ "model.layers.10.mhc_attn.alpha_res": "model-00020-of-00155.safetensors",
+ "model.layers.10.mhc_attn.bias_post": "model-00020-of-00155.safetensors",
+ "model.layers.10.mhc_attn.bias_pre": "model-00020-of-00155.safetensors",
+ "model.layers.10.mhc_attn.bias_res": "model-00020-of-00155.safetensors",
+ "model.layers.10.mhc_attn.proj_post.weight": "model-00020-of-00155.safetensors",
+ "model.layers.10.mhc_attn.proj_pre.weight": "model-00020-of-00155.safetensors",
+ "model.layers.10.mhc_attn.proj_res.weight": "model-00020-of-00155.safetensors",
+ "model.layers.10.mhc_attn.rms_norm.weight": "model-00020-of-00155.safetensors",
+ "model.layers.10.mhc_ffn.alpha_post": "model-00020-of-00155.safetensors",
+ "model.layers.10.mhc_ffn.alpha_pre": "model-00020-of-00155.safetensors",
+ "model.layers.10.mhc_ffn.alpha_res": "model-00020-of-00155.safetensors",
+ "model.layers.10.mhc_ffn.bias_post": "model-00020-of-00155.safetensors",
+ "model.layers.10.mhc_ffn.bias_pre": "model-00020-of-00155.safetensors",
+ "model.layers.10.mhc_ffn.bias_res": "model-00020-of-00155.safetensors",
+ "model.layers.10.mhc_ffn.proj_post.weight": "model-00020-of-00155.safetensors",
+ "model.layers.10.mhc_ffn.proj_pre.weight": "model-00020-of-00155.safetensors",
+ "model.layers.10.mhc_ffn.proj_res.weight": "model-00020-of-00155.safetensors",
+ "model.layers.10.mhc_ffn.rms_norm.weight": "model-00020-of-00155.safetensors",
+ "model.layers.10.moe.expert_bias": "model-00018-of-00155.safetensors",
+ "model.layers.10.moe.experts.act_fn.bias": "model-00020-of-00155.safetensors",
+ "model.layers.10.moe.experts.act_fn.weight": "model-00020-of-00155.safetensors",
+ "model.layers.10.moe.experts.down_proj": "model-00019-of-00155.safetensors",
+ "model.layers.10.moe.experts.gate_up_proj": "model-00113-of-00155.safetensors",
+ "model.layers.10.moe.router.gate.weight": "model-00020-of-00155.safetensors",
+ "model.layers.10.moe.shared_experts.act_fn.bias": "model-00020-of-00155.safetensors",
+ "model.layers.10.moe.shared_experts.act_fn.weight": "model-00020-of-00155.safetensors",
+ "model.layers.10.moe.shared_experts.down_proj.weight": "model-00020-of-00155.safetensors",
+ "model.layers.10.moe.shared_experts.gate_proj.weight": "model-00020-of-00155.safetensors",
+ "model.layers.10.moe.shared_experts.up_proj.weight": "model-00020-of-00155.safetensors",
+ "model.layers.10.post_attention_layernorm.weight": "model-00020-of-00155.safetensors",
+ "model.layers.10.self_attn.kv_norm.weight": "model-00018-of-00155.safetensors",
+ "model.layers.10.self_attn.lambda_proj.weight": "model-00018-of-00155.safetensors",
+ "model.layers.10.self_attn.q_norm.weight": "model-00018-of-00155.safetensors",
+ "model.layers.10.self_attn.wkv_a.weight": "model-00018-of-00155.safetensors",
+ "model.layers.10.self_attn.wkv_b.weight": "model-00018-of-00155.safetensors",
+ "model.layers.10.self_attn.wo.weight": "model-00018-of-00155.safetensors",
+ "model.layers.10.self_attn.wq_a.weight": "model-00018-of-00155.safetensors",
+ "model.layers.10.self_attn.wq_b.weight": "model-00018-of-00155.safetensors",
+ "model.layers.10.self_attn.wq_b_gate.weight": "model-00018-of-00155.safetensors",
+ "model.layers.11.input_layernorm.weight": "model-00022-of-00155.safetensors",
+ "model.layers.11.mhc_attn.alpha_post": "model-00022-of-00155.safetensors",
+ "model.layers.11.mhc_attn.alpha_pre": "model-00022-of-00155.safetensors",
+ "model.layers.11.mhc_attn.alpha_res": "model-00022-of-00155.safetensors",
+ "model.layers.11.mhc_attn.bias_post": "model-00022-of-00155.safetensors",
+ "model.layers.11.mhc_attn.bias_pre": "model-00022-of-00155.safetensors",
+ "model.layers.11.mhc_attn.bias_res": "model-00022-of-00155.safetensors",
+ "model.layers.11.mhc_attn.proj_post.weight": "model-00022-of-00155.safetensors",
+ "model.layers.11.mhc_attn.proj_pre.weight": "model-00022-of-00155.safetensors",
+ "model.layers.11.mhc_attn.proj_res.weight": "model-00022-of-00155.safetensors",
+ "model.layers.11.mhc_attn.rms_norm.weight": "model-00022-of-00155.safetensors",
+ "model.layers.11.mhc_ffn.alpha_post": "model-00022-of-00155.safetensors",
+ "model.layers.11.mhc_ffn.alpha_pre": "model-00022-of-00155.safetensors",
+ "model.layers.11.mhc_ffn.alpha_res": "model-00022-of-00155.safetensors",
+ "model.layers.11.mhc_ffn.bias_post": "model-00022-of-00155.safetensors",
+ "model.layers.11.mhc_ffn.bias_pre": "model-00022-of-00155.safetensors",
+ "model.layers.11.mhc_ffn.bias_res": "model-00022-of-00155.safetensors",
+ "model.layers.11.mhc_ffn.proj_post.weight": "model-00022-of-00155.safetensors",
+ "model.layers.11.mhc_ffn.proj_pre.weight": "model-00022-of-00155.safetensors",
+ "model.layers.11.mhc_ffn.proj_res.weight": "model-00022-of-00155.safetensors",
+ "model.layers.11.mhc_ffn.rms_norm.weight": "model-00022-of-00155.safetensors",
+ "model.layers.11.moe.expert_bias": "model-00020-of-00155.safetensors",
+ "model.layers.11.moe.experts.act_fn.bias": "model-00022-of-00155.safetensors",
+ "model.layers.11.moe.experts.act_fn.weight": "model-00022-of-00155.safetensors",
+ "model.layers.11.moe.experts.down_proj": "model-00021-of-00155.safetensors",
+ "model.layers.11.moe.experts.gate_up_proj": "model-00114-of-00155.safetensors",
+ "model.layers.11.moe.router.gate.weight": "model-00022-of-00155.safetensors",
+ "model.layers.11.moe.shared_experts.act_fn.bias": "model-00022-of-00155.safetensors",
+ "model.layers.11.moe.shared_experts.act_fn.weight": "model-00022-of-00155.safetensors",
+ "model.layers.11.moe.shared_experts.down_proj.weight": "model-00022-of-00155.safetensors",
+ "model.layers.11.moe.shared_experts.gate_proj.weight": "model-00022-of-00155.safetensors",
+ "model.layers.11.moe.shared_experts.up_proj.weight": "model-00022-of-00155.safetensors",
+ "model.layers.11.post_attention_layernorm.weight": "model-00022-of-00155.safetensors",
+ "model.layers.11.self_attn.kv_norm.weight": "model-00020-of-00155.safetensors",
+ "model.layers.11.self_attn.lambda_proj.weight": "model-00020-of-00155.safetensors",
+ "model.layers.11.self_attn.q_norm.weight": "model-00020-of-00155.safetensors",
+ "model.layers.11.self_attn.wkv_a.weight": "model-00020-of-00155.safetensors",
+ "model.layers.11.self_attn.wkv_b.weight": "model-00020-of-00155.safetensors",
+ "model.layers.11.self_attn.wo.weight": "model-00020-of-00155.safetensors",
+ "model.layers.11.self_attn.wq_a.weight": "model-00020-of-00155.safetensors",
+ "model.layers.11.self_attn.wq_b.weight": "model-00020-of-00155.safetensors",
+ "model.layers.11.self_attn.wq_b_gate.weight": "model-00020-of-00155.safetensors",
+ "model.layers.12.input_layernorm.weight": "model-00024-of-00155.safetensors",
+ "model.layers.12.mhc_attn.alpha_post": "model-00024-of-00155.safetensors",
+ "model.layers.12.mhc_attn.alpha_pre": "model-00024-of-00155.safetensors",
+ "model.layers.12.mhc_attn.alpha_res": "model-00024-of-00155.safetensors",
+ "model.layers.12.mhc_attn.bias_post": "model-00024-of-00155.safetensors",
+ "model.layers.12.mhc_attn.bias_pre": "model-00024-of-00155.safetensors",
+ "model.layers.12.mhc_attn.bias_res": "model-00024-of-00155.safetensors",
+ "model.layers.12.mhc_attn.proj_post.weight": "model-00024-of-00155.safetensors",
+ "model.layers.12.mhc_attn.proj_pre.weight": "model-00024-of-00155.safetensors",
+ "model.layers.12.mhc_attn.proj_res.weight": "model-00024-of-00155.safetensors",
+ "model.layers.12.mhc_attn.rms_norm.weight": "model-00024-of-00155.safetensors",
+ "model.layers.12.mhc_ffn.alpha_post": "model-00024-of-00155.safetensors",
+ "model.layers.12.mhc_ffn.alpha_pre": "model-00024-of-00155.safetensors",
+ "model.layers.12.mhc_ffn.alpha_res": "model-00024-of-00155.safetensors",
+ "model.layers.12.mhc_ffn.bias_post": "model-00024-of-00155.safetensors",
+ "model.layers.12.mhc_ffn.bias_pre": "model-00024-of-00155.safetensors",
+ "model.layers.12.mhc_ffn.bias_res": "model-00024-of-00155.safetensors",
+ "model.layers.12.mhc_ffn.proj_post.weight": "model-00024-of-00155.safetensors",
+ "model.layers.12.mhc_ffn.proj_pre.weight": "model-00024-of-00155.safetensors",
+ "model.layers.12.mhc_ffn.proj_res.weight": "model-00024-of-00155.safetensors",
+ "model.layers.12.mhc_ffn.rms_norm.weight": "model-00024-of-00155.safetensors",
+ "model.layers.12.moe.expert_bias": "model-00022-of-00155.safetensors",
+ "model.layers.12.moe.experts.act_fn.bias": "model-00024-of-00155.safetensors",
+ "model.layers.12.moe.experts.act_fn.weight": "model-00024-of-00155.safetensors",
+ "model.layers.12.moe.experts.down_proj": "model-00023-of-00155.safetensors",
+ "model.layers.12.moe.experts.gate_up_proj": "model-00115-of-00155.safetensors",
+ "model.layers.12.moe.router.gate.weight": "model-00024-of-00155.safetensors",
+ "model.layers.12.moe.shared_experts.act_fn.bias": "model-00024-of-00155.safetensors",
+ "model.layers.12.moe.shared_experts.act_fn.weight": "model-00024-of-00155.safetensors",
+ "model.layers.12.moe.shared_experts.down_proj.weight": "model-00024-of-00155.safetensors",
+ "model.layers.12.moe.shared_experts.gate_proj.weight": "model-00024-of-00155.safetensors",
+ "model.layers.12.moe.shared_experts.up_proj.weight": "model-00024-of-00155.safetensors",
+ "model.layers.12.post_attention_layernorm.weight": "model-00024-of-00155.safetensors",
+ "model.layers.12.self_attn.kv_norm.weight": "model-00022-of-00155.safetensors",
+ "model.layers.12.self_attn.lambda_proj.weight": "model-00022-of-00155.safetensors",
+ "model.layers.12.self_attn.q_norm.weight": "model-00022-of-00155.safetensors",
+ "model.layers.12.self_attn.wkv_a.weight": "model-00022-of-00155.safetensors",
+ "model.layers.12.self_attn.wkv_b.weight": "model-00022-of-00155.safetensors",
+ "model.layers.12.self_attn.wo.weight": "model-00022-of-00155.safetensors",
+ "model.layers.12.self_attn.wq_a.weight": "model-00022-of-00155.safetensors",
+ "model.layers.12.self_attn.wq_b.weight": "model-00022-of-00155.safetensors",
+ "model.layers.12.self_attn.wq_b_gate.weight": "model-00022-of-00155.safetensors",
+ "model.layers.13.input_layernorm.weight": "model-00026-of-00155.safetensors",
+ "model.layers.13.mhc_attn.alpha_post": "model-00026-of-00155.safetensors",
+ "model.layers.13.mhc_attn.alpha_pre": "model-00026-of-00155.safetensors",
+ "model.layers.13.mhc_attn.alpha_res": "model-00026-of-00155.safetensors",
+ "model.layers.13.mhc_attn.bias_post": "model-00026-of-00155.safetensors",
+ "model.layers.13.mhc_attn.bias_pre": "model-00026-of-00155.safetensors",
+ "model.layers.13.mhc_attn.bias_res": "model-00026-of-00155.safetensors",
+ "model.layers.13.mhc_attn.proj_post.weight": "model-00026-of-00155.safetensors",
+ "model.layers.13.mhc_attn.proj_pre.weight": "model-00026-of-00155.safetensors",
+ "model.layers.13.mhc_attn.proj_res.weight": "model-00026-of-00155.safetensors",
+ "model.layers.13.mhc_attn.rms_norm.weight": "model-00026-of-00155.safetensors",
+ "model.layers.13.mhc_ffn.alpha_post": "model-00026-of-00155.safetensors",
+ "model.layers.13.mhc_ffn.alpha_pre": "model-00026-of-00155.safetensors",
+ "model.layers.13.mhc_ffn.alpha_res": "model-00026-of-00155.safetensors",
+ "model.layers.13.mhc_ffn.bias_post": "model-00026-of-00155.safetensors",
+ "model.layers.13.mhc_ffn.bias_pre": "model-00026-of-00155.safetensors",
+ "model.layers.13.mhc_ffn.bias_res": "model-00026-of-00155.safetensors",
+ "model.layers.13.mhc_ffn.proj_post.weight": "model-00026-of-00155.safetensors",
+ "model.layers.13.mhc_ffn.proj_pre.weight": "model-00026-of-00155.safetensors",
+ "model.layers.13.mhc_ffn.proj_res.weight": "model-00026-of-00155.safetensors",
+ "model.layers.13.mhc_ffn.rms_norm.weight": "model-00026-of-00155.safetensors",
+ "model.layers.13.moe.expert_bias": "model-00024-of-00155.safetensors",
+ "model.layers.13.moe.experts.act_fn.bias": "model-00026-of-00155.safetensors",
+ "model.layers.13.moe.experts.act_fn.weight": "model-00026-of-00155.safetensors",
+ "model.layers.13.moe.experts.down_proj": "model-00025-of-00155.safetensors",
+ "model.layers.13.moe.experts.gate_up_proj": "model-00116-of-00155.safetensors",
+ "model.layers.13.moe.router.gate.weight": "model-00026-of-00155.safetensors",
+ "model.layers.13.moe.shared_experts.act_fn.bias": "model-00026-of-00155.safetensors",
+ "model.layers.13.moe.shared_experts.act_fn.weight": "model-00026-of-00155.safetensors",
+ "model.layers.13.moe.shared_experts.down_proj.weight": "model-00026-of-00155.safetensors",
+ "model.layers.13.moe.shared_experts.gate_proj.weight": "model-00026-of-00155.safetensors",
+ "model.layers.13.moe.shared_experts.up_proj.weight": "model-00026-of-00155.safetensors",
+ "model.layers.13.post_attention_layernorm.weight": "model-00026-of-00155.safetensors",
+ "model.layers.13.self_attn.kv_norm.weight": "model-00024-of-00155.safetensors",
+ "model.layers.13.self_attn.lambda_proj.weight": "model-00024-of-00155.safetensors",
+ "model.layers.13.self_attn.q_norm.weight": "model-00024-of-00155.safetensors",
+ "model.layers.13.self_attn.wkv_a.weight": "model-00024-of-00155.safetensors",
+ "model.layers.13.self_attn.wkv_b.weight": "model-00024-of-00155.safetensors",
+ "model.layers.13.self_attn.wo.weight": "model-00024-of-00155.safetensors",
+ "model.layers.13.self_attn.wq_a.weight": "model-00024-of-00155.safetensors",
+ "model.layers.13.self_attn.wq_b.weight": "model-00024-of-00155.safetensors",
+ "model.layers.13.self_attn.wq_b_gate.weight": "model-00024-of-00155.safetensors",
+ "model.layers.14.input_layernorm.weight": "model-00028-of-00155.safetensors",
+ "model.layers.14.mhc_attn.alpha_post": "model-00028-of-00155.safetensors",
+ "model.layers.14.mhc_attn.alpha_pre": "model-00028-of-00155.safetensors",
+ "model.layers.14.mhc_attn.alpha_res": "model-00028-of-00155.safetensors",
+ "model.layers.14.mhc_attn.bias_post": "model-00028-of-00155.safetensors",
+ "model.layers.14.mhc_attn.bias_pre": "model-00028-of-00155.safetensors",
+ "model.layers.14.mhc_attn.bias_res": "model-00028-of-00155.safetensors",
+ "model.layers.14.mhc_attn.proj_post.weight": "model-00028-of-00155.safetensors",
+ "model.layers.14.mhc_attn.proj_pre.weight": "model-00028-of-00155.safetensors",
+ "model.layers.14.mhc_attn.proj_res.weight": "model-00028-of-00155.safetensors",
+ "model.layers.14.mhc_attn.rms_norm.weight": "model-00028-of-00155.safetensors",
+ "model.layers.14.mhc_ffn.alpha_post": "model-00028-of-00155.safetensors",
+ "model.layers.14.mhc_ffn.alpha_pre": "model-00028-of-00155.safetensors",
+ "model.layers.14.mhc_ffn.alpha_res": "model-00028-of-00155.safetensors",
+ "model.layers.14.mhc_ffn.bias_post": "model-00028-of-00155.safetensors",
+ "model.layers.14.mhc_ffn.bias_pre": "model-00028-of-00155.safetensors",
+ "model.layers.14.mhc_ffn.bias_res": "model-00028-of-00155.safetensors",
+ "model.layers.14.mhc_ffn.proj_post.weight": "model-00028-of-00155.safetensors",
+ "model.layers.14.mhc_ffn.proj_pre.weight": "model-00028-of-00155.safetensors",
+ "model.layers.14.mhc_ffn.proj_res.weight": "model-00028-of-00155.safetensors",
+ "model.layers.14.mhc_ffn.rms_norm.weight": "model-00028-of-00155.safetensors",
+ "model.layers.14.moe.expert_bias": "model-00026-of-00155.safetensors",
+ "model.layers.14.moe.experts.act_fn.bias": "model-00028-of-00155.safetensors",
+ "model.layers.14.moe.experts.act_fn.weight": "model-00028-of-00155.safetensors",
+ "model.layers.14.moe.experts.down_proj": "model-00027-of-00155.safetensors",
+ "model.layers.14.moe.experts.gate_up_proj": "model-00117-of-00155.safetensors",
+ "model.layers.14.moe.router.gate.weight": "model-00028-of-00155.safetensors",
+ "model.layers.14.moe.shared_experts.act_fn.bias": "model-00028-of-00155.safetensors",
+ "model.layers.14.moe.shared_experts.act_fn.weight": "model-00028-of-00155.safetensors",
+ "model.layers.14.moe.shared_experts.down_proj.weight": "model-00028-of-00155.safetensors",
+ "model.layers.14.moe.shared_experts.gate_proj.weight": "model-00028-of-00155.safetensors",
+ "model.layers.14.moe.shared_experts.up_proj.weight": "model-00028-of-00155.safetensors",
+ "model.layers.14.post_attention_layernorm.weight": "model-00028-of-00155.safetensors",
+ "model.layers.14.self_attn.kv_norm.weight": "model-00026-of-00155.safetensors",
+ "model.layers.14.self_attn.lambda_proj.weight": "model-00026-of-00155.safetensors",
+ "model.layers.14.self_attn.q_norm.weight": "model-00026-of-00155.safetensors",
+ "model.layers.14.self_attn.wkv_a.weight": "model-00026-of-00155.safetensors",
+ "model.layers.14.self_attn.wkv_b.weight": "model-00026-of-00155.safetensors",
+ "model.layers.14.self_attn.wo.weight": "model-00026-of-00155.safetensors",
+ "model.layers.14.self_attn.wq_a.weight": "model-00026-of-00155.safetensors",
+ "model.layers.14.self_attn.wq_b.weight": "model-00026-of-00155.safetensors",
+ "model.layers.14.self_attn.wq_b_gate.weight": "model-00026-of-00155.safetensors",
+ "model.layers.15.input_layernorm.weight": "model-00030-of-00155.safetensors",
+ "model.layers.15.mhc_attn.alpha_post": "model-00030-of-00155.safetensors",
+ "model.layers.15.mhc_attn.alpha_pre": "model-00030-of-00155.safetensors",
+ "model.layers.15.mhc_attn.alpha_res": "model-00030-of-00155.safetensors",
+ "model.layers.15.mhc_attn.bias_post": "model-00030-of-00155.safetensors",
+ "model.layers.15.mhc_attn.bias_pre": "model-00030-of-00155.safetensors",
+ "model.layers.15.mhc_attn.bias_res": "model-00030-of-00155.safetensors",
+ "model.layers.15.mhc_attn.proj_post.weight": "model-00030-of-00155.safetensors",
+ "model.layers.15.mhc_attn.proj_pre.weight": "model-00030-of-00155.safetensors",
+ "model.layers.15.mhc_attn.proj_res.weight": "model-00030-of-00155.safetensors",
+ "model.layers.15.mhc_attn.rms_norm.weight": "model-00030-of-00155.safetensors",
+ "model.layers.15.mhc_ffn.alpha_post": "model-00030-of-00155.safetensors",
+ "model.layers.15.mhc_ffn.alpha_pre": "model-00030-of-00155.safetensors",
+ "model.layers.15.mhc_ffn.alpha_res": "model-00030-of-00155.safetensors",
+ "model.layers.15.mhc_ffn.bias_post": "model-00030-of-00155.safetensors",
+ "model.layers.15.mhc_ffn.bias_pre": "model-00030-of-00155.safetensors",
+ "model.layers.15.mhc_ffn.bias_res": "model-00030-of-00155.safetensors",
+ "model.layers.15.mhc_ffn.proj_post.weight": "model-00030-of-00155.safetensors",
+ "model.layers.15.mhc_ffn.proj_pre.weight": "model-00030-of-00155.safetensors",
+ "model.layers.15.mhc_ffn.proj_res.weight": "model-00030-of-00155.safetensors",
+ "model.layers.15.mhc_ffn.rms_norm.weight": "model-00030-of-00155.safetensors",
+ "model.layers.15.moe.expert_bias": "model-00028-of-00155.safetensors",
+ "model.layers.15.moe.experts.act_fn.bias": "model-00030-of-00155.safetensors",
+ "model.layers.15.moe.experts.act_fn.weight": "model-00030-of-00155.safetensors",
+ "model.layers.15.moe.experts.down_proj": "model-00029-of-00155.safetensors",
+ "model.layers.15.moe.experts.gate_up_proj": "model-00118-of-00155.safetensors",
+ "model.layers.15.moe.router.gate.weight": "model-00030-of-00155.safetensors",
+ "model.layers.15.moe.shared_experts.act_fn.bias": "model-00030-of-00155.safetensors",
+ "model.layers.15.moe.shared_experts.act_fn.weight": "model-00030-of-00155.safetensors",
+ "model.layers.15.moe.shared_experts.down_proj.weight": "model-00030-of-00155.safetensors",
+ "model.layers.15.moe.shared_experts.gate_proj.weight": "model-00030-of-00155.safetensors",
+ "model.layers.15.moe.shared_experts.up_proj.weight": "model-00030-of-00155.safetensors",
+ "model.layers.15.post_attention_layernorm.weight": "model-00030-of-00155.safetensors",
+ "model.layers.15.self_attn.kv_norm.weight": "model-00028-of-00155.safetensors",
+ "model.layers.15.self_attn.lambda_proj.weight": "model-00028-of-00155.safetensors",
+ "model.layers.15.self_attn.q_norm.weight": "model-00028-of-00155.safetensors",
+ "model.layers.15.self_attn.wkv_a.weight": "model-00028-of-00155.safetensors",
+ "model.layers.15.self_attn.wkv_b.weight": "model-00028-of-00155.safetensors",
+ "model.layers.15.self_attn.wo.weight": "model-00028-of-00155.safetensors",
+ "model.layers.15.self_attn.wq_a.weight": "model-00028-of-00155.safetensors",
+ "model.layers.15.self_attn.wq_b.weight": "model-00028-of-00155.safetensors",
+ "model.layers.15.self_attn.wq_b_gate.weight": "model-00028-of-00155.safetensors",
+ "model.layers.16.input_layernorm.weight": "model-00032-of-00155.safetensors",
+ "model.layers.16.mhc_attn.alpha_post": "model-00032-of-00155.safetensors",
+ "model.layers.16.mhc_attn.alpha_pre": "model-00032-of-00155.safetensors",
+ "model.layers.16.mhc_attn.alpha_res": "model-00032-of-00155.safetensors",
+ "model.layers.16.mhc_attn.bias_post": "model-00032-of-00155.safetensors",
+ "model.layers.16.mhc_attn.bias_pre": "model-00032-of-00155.safetensors",
+ "model.layers.16.mhc_attn.bias_res": "model-00032-of-00155.safetensors",
+ "model.layers.16.mhc_attn.proj_post.weight": "model-00032-of-00155.safetensors",
+ "model.layers.16.mhc_attn.proj_pre.weight": "model-00032-of-00155.safetensors",
+ "model.layers.16.mhc_attn.proj_res.weight": "model-00032-of-00155.safetensors",
+ "model.layers.16.mhc_attn.rms_norm.weight": "model-00032-of-00155.safetensors",
+ "model.layers.16.mhc_ffn.alpha_post": "model-00032-of-00155.safetensors",
+ "model.layers.16.mhc_ffn.alpha_pre": "model-00032-of-00155.safetensors",
+ "model.layers.16.mhc_ffn.alpha_res": "model-00032-of-00155.safetensors",
+ "model.layers.16.mhc_ffn.bias_post": "model-00032-of-00155.safetensors",
+ "model.layers.16.mhc_ffn.bias_pre": "model-00032-of-00155.safetensors",
+ "model.layers.16.mhc_ffn.bias_res": "model-00032-of-00155.safetensors",
+ "model.layers.16.mhc_ffn.proj_post.weight": "model-00032-of-00155.safetensors",
+ "model.layers.16.mhc_ffn.proj_pre.weight": "model-00032-of-00155.safetensors",
+ "model.layers.16.mhc_ffn.proj_res.weight": "model-00032-of-00155.safetensors",
+ "model.layers.16.mhc_ffn.rms_norm.weight": "model-00032-of-00155.safetensors",
+ "model.layers.16.moe.expert_bias": "model-00030-of-00155.safetensors",
+ "model.layers.16.moe.experts.act_fn.bias": "model-00032-of-00155.safetensors",
+ "model.layers.16.moe.experts.act_fn.weight": "model-00032-of-00155.safetensors",
+ "model.layers.16.moe.experts.down_proj": "model-00031-of-00155.safetensors",
+ "model.layers.16.moe.experts.gate_up_proj": "model-00119-of-00155.safetensors",
+ "model.layers.16.moe.router.gate.weight": "model-00032-of-00155.safetensors",
+ "model.layers.16.moe.shared_experts.act_fn.bias": "model-00032-of-00155.safetensors",
+ "model.layers.16.moe.shared_experts.act_fn.weight": "model-00032-of-00155.safetensors",
+ "model.layers.16.moe.shared_experts.down_proj.weight": "model-00032-of-00155.safetensors",
+ "model.layers.16.moe.shared_experts.gate_proj.weight": "model-00032-of-00155.safetensors",
+ "model.layers.16.moe.shared_experts.up_proj.weight": "model-00032-of-00155.safetensors",
+ "model.layers.16.post_attention_layernorm.weight": "model-00032-of-00155.safetensors",
+ "model.layers.16.self_attn.kv_norm.weight": "model-00030-of-00155.safetensors",
+ "model.layers.16.self_attn.lambda_proj.weight": "model-00030-of-00155.safetensors",
+ "model.layers.16.self_attn.q_norm.weight": "model-00030-of-00155.safetensors",
+ "model.layers.16.self_attn.wkv_a.weight": "model-00030-of-00155.safetensors",
+ "model.layers.16.self_attn.wkv_b.weight": "model-00030-of-00155.safetensors",
+ "model.layers.16.self_attn.wo.weight": "model-00030-of-00155.safetensors",
+ "model.layers.16.self_attn.wq_a.weight": "model-00030-of-00155.safetensors",
+ "model.layers.16.self_attn.wq_b.weight": "model-00030-of-00155.safetensors",
+ "model.layers.16.self_attn.wq_b_gate.weight": "model-00030-of-00155.safetensors",
+ "model.layers.17.input_layernorm.weight": "model-00034-of-00155.safetensors",
+ "model.layers.17.mhc_attn.alpha_post": "model-00034-of-00155.safetensors",
+ "model.layers.17.mhc_attn.alpha_pre": "model-00034-of-00155.safetensors",
+ "model.layers.17.mhc_attn.alpha_res": "model-00034-of-00155.safetensors",
+ "model.layers.17.mhc_attn.bias_post": "model-00034-of-00155.safetensors",
+ "model.layers.17.mhc_attn.bias_pre": "model-00034-of-00155.safetensors",
+ "model.layers.17.mhc_attn.bias_res": "model-00034-of-00155.safetensors",
+ "model.layers.17.mhc_attn.proj_post.weight": "model-00034-of-00155.safetensors",
+ "model.layers.17.mhc_attn.proj_pre.weight": "model-00034-of-00155.safetensors",
+ "model.layers.17.mhc_attn.proj_res.weight": "model-00034-of-00155.safetensors",
+ "model.layers.17.mhc_attn.rms_norm.weight": "model-00034-of-00155.safetensors",
+ "model.layers.17.mhc_ffn.alpha_post": "model-00034-of-00155.safetensors",
+ "model.layers.17.mhc_ffn.alpha_pre": "model-00034-of-00155.safetensors",
+ "model.layers.17.mhc_ffn.alpha_res": "model-00034-of-00155.safetensors",
+ "model.layers.17.mhc_ffn.bias_post": "model-00034-of-00155.safetensors",
+ "model.layers.17.mhc_ffn.bias_pre": "model-00034-of-00155.safetensors",
+ "model.layers.17.mhc_ffn.bias_res": "model-00034-of-00155.safetensors",
+ "model.layers.17.mhc_ffn.proj_post.weight": "model-00034-of-00155.safetensors",
+ "model.layers.17.mhc_ffn.proj_pre.weight": "model-00034-of-00155.safetensors",
+ "model.layers.17.mhc_ffn.proj_res.weight": "model-00034-of-00155.safetensors",
+ "model.layers.17.mhc_ffn.rms_norm.weight": "model-00034-of-00155.safetensors",
+ "model.layers.17.moe.expert_bias": "model-00032-of-00155.safetensors",
+ "model.layers.17.moe.experts.act_fn.bias": "model-00034-of-00155.safetensors",
+ "model.layers.17.moe.experts.act_fn.weight": "model-00034-of-00155.safetensors",
+ "model.layers.17.moe.experts.down_proj": "model-00033-of-00155.safetensors",
+ "model.layers.17.moe.experts.gate_up_proj": "model-00120-of-00155.safetensors",
+ "model.layers.17.moe.router.gate.weight": "model-00034-of-00155.safetensors",
+ "model.layers.17.moe.shared_experts.act_fn.bias": "model-00034-of-00155.safetensors",
+ "model.layers.17.moe.shared_experts.act_fn.weight": "model-00034-of-00155.safetensors",
+ "model.layers.17.moe.shared_experts.down_proj.weight": "model-00034-of-00155.safetensors",
+ "model.layers.17.moe.shared_experts.gate_proj.weight": "model-00034-of-00155.safetensors",
+ "model.layers.17.moe.shared_experts.up_proj.weight": "model-00034-of-00155.safetensors",
+ "model.layers.17.post_attention_layernorm.weight": "model-00034-of-00155.safetensors",
+ "model.layers.17.self_attn.kv_norm.weight": "model-00032-of-00155.safetensors",
+ "model.layers.17.self_attn.lambda_proj.weight": "model-00032-of-00155.safetensors",
+ "model.layers.17.self_attn.q_norm.weight": "model-00032-of-00155.safetensors",
+ "model.layers.17.self_attn.wkv_a.weight": "model-00032-of-00155.safetensors",
+ "model.layers.17.self_attn.wkv_b.weight": "model-00032-of-00155.safetensors",
+ "model.layers.17.self_attn.wo.weight": "model-00032-of-00155.safetensors",
+ "model.layers.17.self_attn.wq_a.weight": "model-00032-of-00155.safetensors",
+ "model.layers.17.self_attn.wq_b.weight": "model-00032-of-00155.safetensors",
+ "model.layers.17.self_attn.wq_b_gate.weight": "model-00032-of-00155.safetensors",
+ "model.layers.18.input_layernorm.weight": "model-00036-of-00155.safetensors",
+ "model.layers.18.mhc_attn.alpha_post": "model-00036-of-00155.safetensors",
+ "model.layers.18.mhc_attn.alpha_pre": "model-00036-of-00155.safetensors",
+ "model.layers.18.mhc_attn.alpha_res": "model-00036-of-00155.safetensors",
+ "model.layers.18.mhc_attn.bias_post": "model-00036-of-00155.safetensors",
+ "model.layers.18.mhc_attn.bias_pre": "model-00036-of-00155.safetensors",
+ "model.layers.18.mhc_attn.bias_res": "model-00036-of-00155.safetensors",
+ "model.layers.18.mhc_attn.proj_post.weight": "model-00036-of-00155.safetensors",
+ "model.layers.18.mhc_attn.proj_pre.weight": "model-00036-of-00155.safetensors",
+ "model.layers.18.mhc_attn.proj_res.weight": "model-00036-of-00155.safetensors",
+ "model.layers.18.mhc_attn.rms_norm.weight": "model-00036-of-00155.safetensors",
+ "model.layers.18.mhc_ffn.alpha_post": "model-00036-of-00155.safetensors",
+ "model.layers.18.mhc_ffn.alpha_pre": "model-00036-of-00155.safetensors",
+ "model.layers.18.mhc_ffn.alpha_res": "model-00036-of-00155.safetensors",
+ "model.layers.18.mhc_ffn.bias_post": "model-00036-of-00155.safetensors",
+ "model.layers.18.mhc_ffn.bias_pre": "model-00036-of-00155.safetensors",
+ "model.layers.18.mhc_ffn.bias_res": "model-00036-of-00155.safetensors",
+ "model.layers.18.mhc_ffn.proj_post.weight": "model-00036-of-00155.safetensors",
+ "model.layers.18.mhc_ffn.proj_pre.weight": "model-00036-of-00155.safetensors",
+ "model.layers.18.mhc_ffn.proj_res.weight": "model-00036-of-00155.safetensors",
+ "model.layers.18.mhc_ffn.rms_norm.weight": "model-00036-of-00155.safetensors",
+ "model.layers.18.moe.expert_bias": "model-00034-of-00155.safetensors",
+ "model.layers.18.moe.experts.act_fn.bias": "model-00036-of-00155.safetensors",
+ "model.layers.18.moe.experts.act_fn.weight": "model-00036-of-00155.safetensors",
+ "model.layers.18.moe.experts.down_proj": "model-00035-of-00155.safetensors",
+ "model.layers.18.moe.experts.gate_up_proj": "model-00121-of-00155.safetensors",
+ "model.layers.18.moe.router.gate.weight": "model-00036-of-00155.safetensors",
+ "model.layers.18.moe.shared_experts.act_fn.bias": "model-00036-of-00155.safetensors",
+ "model.layers.18.moe.shared_experts.act_fn.weight": "model-00036-of-00155.safetensors",
+ "model.layers.18.moe.shared_experts.down_proj.weight": "model-00036-of-00155.safetensors",
+ "model.layers.18.moe.shared_experts.gate_proj.weight": "model-00036-of-00155.safetensors",
+ "model.layers.18.moe.shared_experts.up_proj.weight": "model-00036-of-00155.safetensors",
+ "model.layers.18.post_attention_layernorm.weight": "model-00036-of-00155.safetensors",
+ "model.layers.18.self_attn.kv_norm.weight": "model-00034-of-00155.safetensors",
+ "model.layers.18.self_attn.lambda_proj.weight": "model-00034-of-00155.safetensors",
+ "model.layers.18.self_attn.q_norm.weight": "model-00034-of-00155.safetensors",
+ "model.layers.18.self_attn.wkv_a.weight": "model-00034-of-00155.safetensors",
+ "model.layers.18.self_attn.wkv_b.weight": "model-00034-of-00155.safetensors",
+ "model.layers.18.self_attn.wo.weight": "model-00034-of-00155.safetensors",
+ "model.layers.18.self_attn.wq_a.weight": "model-00034-of-00155.safetensors",
+ "model.layers.18.self_attn.wq_b.weight": "model-00034-of-00155.safetensors",
+ "model.layers.18.self_attn.wq_b_gate.weight": "model-00034-of-00155.safetensors",
+ "model.layers.19.input_layernorm.weight": "model-00038-of-00155.safetensors",
+ "model.layers.19.mhc_attn.alpha_post": "model-00038-of-00155.safetensors",
+ "model.layers.19.mhc_attn.alpha_pre": "model-00038-of-00155.safetensors",
+ "model.layers.19.mhc_attn.alpha_res": "model-00038-of-00155.safetensors",
+ "model.layers.19.mhc_attn.bias_post": "model-00038-of-00155.safetensors",
+ "model.layers.19.mhc_attn.bias_pre": "model-00038-of-00155.safetensors",
+ "model.layers.19.mhc_attn.bias_res": "model-00038-of-00155.safetensors",
+ "model.layers.19.mhc_attn.proj_post.weight": "model-00038-of-00155.safetensors",
+ "model.layers.19.mhc_attn.proj_pre.weight": "model-00038-of-00155.safetensors",
+ "model.layers.19.mhc_attn.proj_res.weight": "model-00038-of-00155.safetensors",
+ "model.layers.19.mhc_attn.rms_norm.weight": "model-00038-of-00155.safetensors",
+ "model.layers.19.mhc_ffn.alpha_post": "model-00038-of-00155.safetensors",
+ "model.layers.19.mhc_ffn.alpha_pre": "model-00038-of-00155.safetensors",
+ "model.layers.19.mhc_ffn.alpha_res": "model-00038-of-00155.safetensors",
+ "model.layers.19.mhc_ffn.bias_post": "model-00038-of-00155.safetensors",
+ "model.layers.19.mhc_ffn.bias_pre": "model-00038-of-00155.safetensors",
+ "model.layers.19.mhc_ffn.bias_res": "model-00038-of-00155.safetensors",
+ "model.layers.19.mhc_ffn.proj_post.weight": "model-00038-of-00155.safetensors",
+ "model.layers.19.mhc_ffn.proj_pre.weight": "model-00038-of-00155.safetensors",
+ "model.layers.19.mhc_ffn.proj_res.weight": "model-00038-of-00155.safetensors",
+ "model.layers.19.mhc_ffn.rms_norm.weight": "model-00038-of-00155.safetensors",
+ "model.layers.19.moe.expert_bias": "model-00036-of-00155.safetensors",
+ "model.layers.19.moe.experts.act_fn.bias": "model-00038-of-00155.safetensors",
+ "model.layers.19.moe.experts.act_fn.weight": "model-00038-of-00155.safetensors",
+ "model.layers.19.moe.experts.down_proj": "model-00037-of-00155.safetensors",
+ "model.layers.19.moe.experts.gate_up_proj": "model-00122-of-00155.safetensors",
+ "model.layers.19.moe.router.gate.weight": "model-00038-of-00155.safetensors",
+ "model.layers.19.moe.shared_experts.act_fn.bias": "model-00038-of-00155.safetensors",
+ "model.layers.19.moe.shared_experts.act_fn.weight": "model-00038-of-00155.safetensors",
+ "model.layers.19.moe.shared_experts.down_proj.weight": "model-00038-of-00155.safetensors",
+ "model.layers.19.moe.shared_experts.gate_proj.weight": "model-00038-of-00155.safetensors",
+ "model.layers.19.moe.shared_experts.up_proj.weight": "model-00038-of-00155.safetensors",
+ "model.layers.19.post_attention_layernorm.weight": "model-00038-of-00155.safetensors",
+ "model.layers.19.self_attn.kv_norm.weight": "model-00036-of-00155.safetensors",
+ "model.layers.19.self_attn.lambda_proj.weight": "model-00036-of-00155.safetensors",
+ "model.layers.19.self_attn.q_norm.weight": "model-00036-of-00155.safetensors",
+ "model.layers.19.self_attn.wkv_a.weight": "model-00036-of-00155.safetensors",
+ "model.layers.19.self_attn.wkv_b.weight": "model-00036-of-00155.safetensors",
+ "model.layers.19.self_attn.wo.weight": "model-00036-of-00155.safetensors",
+ "model.layers.19.self_attn.wq_a.weight": "model-00036-of-00155.safetensors",
+ "model.layers.19.self_attn.wq_b.weight": "model-00036-of-00155.safetensors",
+ "model.layers.19.self_attn.wq_b_gate.weight": "model-00036-of-00155.safetensors",
+ "model.layers.2.input_layernorm.weight": "model-00004-of-00155.safetensors",
+ "model.layers.2.mhc_attn.alpha_post": "model-00004-of-00155.safetensors",
+ "model.layers.2.mhc_attn.alpha_pre": "model-00004-of-00155.safetensors",
+ "model.layers.2.mhc_attn.alpha_res": "model-00004-of-00155.safetensors",
+ "model.layers.2.mhc_attn.bias_post": "model-00004-of-00155.safetensors",
+ "model.layers.2.mhc_attn.bias_pre": "model-00004-of-00155.safetensors",
+ "model.layers.2.mhc_attn.bias_res": "model-00004-of-00155.safetensors",
+ "model.layers.2.mhc_attn.proj_post.weight": "model-00004-of-00155.safetensors",
+ "model.layers.2.mhc_attn.proj_pre.weight": "model-00004-of-00155.safetensors",
+ "model.layers.2.mhc_attn.proj_res.weight": "model-00004-of-00155.safetensors",
+ "model.layers.2.mhc_attn.rms_norm.weight": "model-00004-of-00155.safetensors",
+ "model.layers.2.mhc_ffn.alpha_post": "model-00004-of-00155.safetensors",
+ "model.layers.2.mhc_ffn.alpha_pre": "model-00004-of-00155.safetensors",
+ "model.layers.2.mhc_ffn.alpha_res": "model-00004-of-00155.safetensors",
+ "model.layers.2.mhc_ffn.bias_post": "model-00004-of-00155.safetensors",
+ "model.layers.2.mhc_ffn.bias_pre": "model-00004-of-00155.safetensors",
+ "model.layers.2.mhc_ffn.bias_res": "model-00004-of-00155.safetensors",
+ "model.layers.2.mhc_ffn.proj_post.weight": "model-00004-of-00155.safetensors",
+ "model.layers.2.mhc_ffn.proj_pre.weight": "model-00004-of-00155.safetensors",
+ "model.layers.2.mhc_ffn.proj_res.weight": "model-00004-of-00155.safetensors",
+ "model.layers.2.mhc_ffn.rms_norm.weight": "model-00004-of-00155.safetensors",
+ "model.layers.2.moe.expert_bias": "model-00002-of-00155.safetensors",
+ "model.layers.2.moe.experts.act_fn.bias": "model-00004-of-00155.safetensors",
+ "model.layers.2.moe.experts.act_fn.weight": "model-00004-of-00155.safetensors",
+ "model.layers.2.moe.experts.down_proj": "model-00003-of-00155.safetensors",
+ "model.layers.2.moe.experts.gate_up_proj": "model-00105-of-00155.safetensors",
+ "model.layers.2.moe.router.gate.weight": "model-00004-of-00155.safetensors",
+ "model.layers.2.moe.shared_experts.act_fn.bias": "model-00004-of-00155.safetensors",
+ "model.layers.2.moe.shared_experts.act_fn.weight": "model-00004-of-00155.safetensors",
+ "model.layers.2.moe.shared_experts.down_proj.weight": "model-00004-of-00155.safetensors",
+ "model.layers.2.moe.shared_experts.gate_proj.weight": "model-00004-of-00155.safetensors",
+ "model.layers.2.moe.shared_experts.up_proj.weight": "model-00004-of-00155.safetensors",
+ "model.layers.2.post_attention_layernorm.weight": "model-00004-of-00155.safetensors",
+ "model.layers.2.self_attn.kv_norm.weight": "model-00002-of-00155.safetensors",
+ "model.layers.2.self_attn.lambda_proj.weight": "model-00002-of-00155.safetensors",
+ "model.layers.2.self_attn.q_norm.weight": "model-00002-of-00155.safetensors",
+ "model.layers.2.self_attn.wkv_a.weight": "model-00002-of-00155.safetensors",
+ "model.layers.2.self_attn.wkv_b.weight": "model-00002-of-00155.safetensors",
+ "model.layers.2.self_attn.wo.weight": "model-00002-of-00155.safetensors",
+ "model.layers.2.self_attn.wq_a.weight": "model-00002-of-00155.safetensors",
+ "model.layers.2.self_attn.wq_b.weight": "model-00002-of-00155.safetensors",
+ "model.layers.2.self_attn.wq_b_gate.weight": "model-00002-of-00155.safetensors",
+ "model.layers.20.input_layernorm.weight": "model-00040-of-00155.safetensors",
+ "model.layers.20.mhc_attn.alpha_post": "model-00040-of-00155.safetensors",
+ "model.layers.20.mhc_attn.alpha_pre": "model-00040-of-00155.safetensors",
+ "model.layers.20.mhc_attn.alpha_res": "model-00040-of-00155.safetensors",
+ "model.layers.20.mhc_attn.bias_post": "model-00040-of-00155.safetensors",
+ "model.layers.20.mhc_attn.bias_pre": "model-00040-of-00155.safetensors",
+ "model.layers.20.mhc_attn.bias_res": "model-00040-of-00155.safetensors",
+ "model.layers.20.mhc_attn.proj_post.weight": "model-00040-of-00155.safetensors",
+ "model.layers.20.mhc_attn.proj_pre.weight": "model-00040-of-00155.safetensors",
+ "model.layers.20.mhc_attn.proj_res.weight": "model-00040-of-00155.safetensors",
+ "model.layers.20.mhc_attn.rms_norm.weight": "model-00040-of-00155.safetensors",
+ "model.layers.20.mhc_ffn.alpha_post": "model-00040-of-00155.safetensors",
+ "model.layers.20.mhc_ffn.alpha_pre": "model-00040-of-00155.safetensors",
+ "model.layers.20.mhc_ffn.alpha_res": "model-00040-of-00155.safetensors",
+ "model.layers.20.mhc_ffn.bias_post": "model-00040-of-00155.safetensors",
+ "model.layers.20.mhc_ffn.bias_pre": "model-00040-of-00155.safetensors",
+ "model.layers.20.mhc_ffn.bias_res": "model-00040-of-00155.safetensors",
+ "model.layers.20.mhc_ffn.proj_post.weight": "model-00040-of-00155.safetensors",
+ "model.layers.20.mhc_ffn.proj_pre.weight": "model-00040-of-00155.safetensors",
+ "model.layers.20.mhc_ffn.proj_res.weight": "model-00040-of-00155.safetensors",
+ "model.layers.20.mhc_ffn.rms_norm.weight": "model-00040-of-00155.safetensors",
+ "model.layers.20.moe.expert_bias": "model-00038-of-00155.safetensors",
+ "model.layers.20.moe.experts.act_fn.bias": "model-00040-of-00155.safetensors",
+ "model.layers.20.moe.experts.act_fn.weight": "model-00040-of-00155.safetensors",
+ "model.layers.20.moe.experts.down_proj": "model-00039-of-00155.safetensors",
+ "model.layers.20.moe.experts.gate_up_proj": "model-00123-of-00155.safetensors",
+ "model.layers.20.moe.router.gate.weight": "model-00040-of-00155.safetensors",
+ "model.layers.20.moe.shared_experts.act_fn.bias": "model-00040-of-00155.safetensors",
+ "model.layers.20.moe.shared_experts.act_fn.weight": "model-00040-of-00155.safetensors",
+ "model.layers.20.moe.shared_experts.down_proj.weight": "model-00040-of-00155.safetensors",
+ "model.layers.20.moe.shared_experts.gate_proj.weight": "model-00040-of-00155.safetensors",
+ "model.layers.20.moe.shared_experts.up_proj.weight": "model-00040-of-00155.safetensors",
+ "model.layers.20.post_attention_layernorm.weight": "model-00040-of-00155.safetensors",
+ "model.layers.20.self_attn.kv_norm.weight": "model-00038-of-00155.safetensors",
+ "model.layers.20.self_attn.lambda_proj.weight": "model-00038-of-00155.safetensors",
+ "model.layers.20.self_attn.q_norm.weight": "model-00038-of-00155.safetensors",
+ "model.layers.20.self_attn.wkv_a.weight": "model-00038-of-00155.safetensors",
+ "model.layers.20.self_attn.wkv_b.weight": "model-00038-of-00155.safetensors",
+ "model.layers.20.self_attn.wo.weight": "model-00038-of-00155.safetensors",
+ "model.layers.20.self_attn.wq_a.weight": "model-00038-of-00155.safetensors",
+ "model.layers.20.self_attn.wq_b.weight": "model-00038-of-00155.safetensors",
+ "model.layers.20.self_attn.wq_b_gate.weight": "model-00038-of-00155.safetensors",
+ "model.layers.21.input_layernorm.weight": "model-00042-of-00155.safetensors",
+ "model.layers.21.mhc_attn.alpha_post": "model-00042-of-00155.safetensors",
+ "model.layers.21.mhc_attn.alpha_pre": "model-00042-of-00155.safetensors",
+ "model.layers.21.mhc_attn.alpha_res": "model-00042-of-00155.safetensors",
+ "model.layers.21.mhc_attn.bias_post": "model-00042-of-00155.safetensors",
+ "model.layers.21.mhc_attn.bias_pre": "model-00042-of-00155.safetensors",
+ "model.layers.21.mhc_attn.bias_res": "model-00042-of-00155.safetensors",
+ "model.layers.21.mhc_attn.proj_post.weight": "model-00042-of-00155.safetensors",
+ "model.layers.21.mhc_attn.proj_pre.weight": "model-00042-of-00155.safetensors",
+ "model.layers.21.mhc_attn.proj_res.weight": "model-00042-of-00155.safetensors",
+ "model.layers.21.mhc_attn.rms_norm.weight": "model-00042-of-00155.safetensors",
+ "model.layers.21.mhc_ffn.alpha_post": "model-00042-of-00155.safetensors",
+ "model.layers.21.mhc_ffn.alpha_pre": "model-00042-of-00155.safetensors",
+ "model.layers.21.mhc_ffn.alpha_res": "model-00042-of-00155.safetensors",
+ "model.layers.21.mhc_ffn.bias_post": "model-00042-of-00155.safetensors",
+ "model.layers.21.mhc_ffn.bias_pre": "model-00042-of-00155.safetensors",
+ "model.layers.21.mhc_ffn.bias_res": "model-00042-of-00155.safetensors",
+ "model.layers.21.mhc_ffn.proj_post.weight": "model-00042-of-00155.safetensors",
+ "model.layers.21.mhc_ffn.proj_pre.weight": "model-00042-of-00155.safetensors",
+ "model.layers.21.mhc_ffn.proj_res.weight": "model-00042-of-00155.safetensors",
+ "model.layers.21.mhc_ffn.rms_norm.weight": "model-00042-of-00155.safetensors",
+ "model.layers.21.moe.expert_bias": "model-00040-of-00155.safetensors",
+ "model.layers.21.moe.experts.act_fn.bias": "model-00042-of-00155.safetensors",
+ "model.layers.21.moe.experts.act_fn.weight": "model-00042-of-00155.safetensors",
+ "model.layers.21.moe.experts.down_proj": "model-00041-of-00155.safetensors",
+ "model.layers.21.moe.experts.gate_up_proj": "model-00124-of-00155.safetensors",
+ "model.layers.21.moe.router.gate.weight": "model-00042-of-00155.safetensors",
+ "model.layers.21.moe.shared_experts.act_fn.bias": "model-00042-of-00155.safetensors",
+ "model.layers.21.moe.shared_experts.act_fn.weight": "model-00042-of-00155.safetensors",
+ "model.layers.21.moe.shared_experts.down_proj.weight": "model-00042-of-00155.safetensors",
+ "model.layers.21.moe.shared_experts.gate_proj.weight": "model-00042-of-00155.safetensors",
+ "model.layers.21.moe.shared_experts.up_proj.weight": "model-00042-of-00155.safetensors",
+ "model.layers.21.post_attention_layernorm.weight": "model-00042-of-00155.safetensors",
+ "model.layers.21.self_attn.kv_norm.weight": "model-00040-of-00155.safetensors",
+ "model.layers.21.self_attn.lambda_proj.weight": "model-00040-of-00155.safetensors",
+ "model.layers.21.self_attn.q_norm.weight": "model-00040-of-00155.safetensors",
+ "model.layers.21.self_attn.wkv_a.weight": "model-00040-of-00155.safetensors",
+ "model.layers.21.self_attn.wkv_b.weight": "model-00040-of-00155.safetensors",
+ "model.layers.21.self_attn.wo.weight": "model-00040-of-00155.safetensors",
+ "model.layers.21.self_attn.wq_a.weight": "model-00040-of-00155.safetensors",
+ "model.layers.21.self_attn.wq_b.weight": "model-00040-of-00155.safetensors",
+ "model.layers.21.self_attn.wq_b_gate.weight": "model-00040-of-00155.safetensors",
+ "model.layers.22.input_layernorm.weight": "model-00044-of-00155.safetensors",
+ "model.layers.22.mhc_attn.alpha_post": "model-00044-of-00155.safetensors",
+ "model.layers.22.mhc_attn.alpha_pre": "model-00044-of-00155.safetensors",
+ "model.layers.22.mhc_attn.alpha_res": "model-00044-of-00155.safetensors",
+ "model.layers.22.mhc_attn.bias_post": "model-00044-of-00155.safetensors",
+ "model.layers.22.mhc_attn.bias_pre": "model-00044-of-00155.safetensors",
+ "model.layers.22.mhc_attn.bias_res": "model-00044-of-00155.safetensors",
+ "model.layers.22.mhc_attn.proj_post.weight": "model-00044-of-00155.safetensors",
+ "model.layers.22.mhc_attn.proj_pre.weight": "model-00044-of-00155.safetensors",
+ "model.layers.22.mhc_attn.proj_res.weight": "model-00044-of-00155.safetensors",
+ "model.layers.22.mhc_attn.rms_norm.weight": "model-00044-of-00155.safetensors",
+ "model.layers.22.mhc_ffn.alpha_post": "model-00044-of-00155.safetensors",
+ "model.layers.22.mhc_ffn.alpha_pre": "model-00044-of-00155.safetensors",
+ "model.layers.22.mhc_ffn.alpha_res": "model-00044-of-00155.safetensors",
+ "model.layers.22.mhc_ffn.bias_post": "model-00044-of-00155.safetensors",
+ "model.layers.22.mhc_ffn.bias_pre": "model-00044-of-00155.safetensors",
+ "model.layers.22.mhc_ffn.bias_res": "model-00044-of-00155.safetensors",
+ "model.layers.22.mhc_ffn.proj_post.weight": "model-00044-of-00155.safetensors",
+ "model.layers.22.mhc_ffn.proj_pre.weight": "model-00044-of-00155.safetensors",
+ "model.layers.22.mhc_ffn.proj_res.weight": "model-00044-of-00155.safetensors",
+ "model.layers.22.mhc_ffn.rms_norm.weight": "model-00044-of-00155.safetensors",
+ "model.layers.22.moe.expert_bias": "model-00042-of-00155.safetensors",
+ "model.layers.22.moe.experts.act_fn.bias": "model-00044-of-00155.safetensors",
+ "model.layers.22.moe.experts.act_fn.weight": "model-00044-of-00155.safetensors",
+ "model.layers.22.moe.experts.down_proj": "model-00043-of-00155.safetensors",
+ "model.layers.22.moe.experts.gate_up_proj": "model-00125-of-00155.safetensors",
+ "model.layers.22.moe.router.gate.weight": "model-00044-of-00155.safetensors",
+ "model.layers.22.moe.shared_experts.act_fn.bias": "model-00044-of-00155.safetensors",
+ "model.layers.22.moe.shared_experts.act_fn.weight": "model-00044-of-00155.safetensors",
+ "model.layers.22.moe.shared_experts.down_proj.weight": "model-00044-of-00155.safetensors",
+ "model.layers.22.moe.shared_experts.gate_proj.weight": "model-00044-of-00155.safetensors",
+ "model.layers.22.moe.shared_experts.up_proj.weight": "model-00044-of-00155.safetensors",
+ "model.layers.22.post_attention_layernorm.weight": "model-00044-of-00155.safetensors",
+ "model.layers.22.self_attn.kv_norm.weight": "model-00042-of-00155.safetensors",
+ "model.layers.22.self_attn.lambda_proj.weight": "model-00042-of-00155.safetensors",
+ "model.layers.22.self_attn.q_norm.weight": "model-00042-of-00155.safetensors",
+ "model.layers.22.self_attn.wkv_a.weight": "model-00042-of-00155.safetensors",
+ "model.layers.22.self_attn.wkv_b.weight": "model-00042-of-00155.safetensors",
+ "model.layers.22.self_attn.wo.weight": "model-00042-of-00155.safetensors",
+ "model.layers.22.self_attn.wq_a.weight": "model-00042-of-00155.safetensors",
+ "model.layers.22.self_attn.wq_b.weight": "model-00042-of-00155.safetensors",
+ "model.layers.22.self_attn.wq_b_gate.weight": "model-00042-of-00155.safetensors",
+ "model.layers.23.input_layernorm.weight": "model-00046-of-00155.safetensors",
+ "model.layers.23.mhc_attn.alpha_post": "model-00046-of-00155.safetensors",
+ "model.layers.23.mhc_attn.alpha_pre": "model-00046-of-00155.safetensors",
+ "model.layers.23.mhc_attn.alpha_res": "model-00046-of-00155.safetensors",
+ "model.layers.23.mhc_attn.bias_post": "model-00046-of-00155.safetensors",
+ "model.layers.23.mhc_attn.bias_pre": "model-00046-of-00155.safetensors",
+ "model.layers.23.mhc_attn.bias_res": "model-00046-of-00155.safetensors",
+ "model.layers.23.mhc_attn.proj_post.weight": "model-00046-of-00155.safetensors",
+ "model.layers.23.mhc_attn.proj_pre.weight": "model-00046-of-00155.safetensors",
+ "model.layers.23.mhc_attn.proj_res.weight": "model-00046-of-00155.safetensors",
+ "model.layers.23.mhc_attn.rms_norm.weight": "model-00046-of-00155.safetensors",
+ "model.layers.23.mhc_ffn.alpha_post": "model-00046-of-00155.safetensors",
+ "model.layers.23.mhc_ffn.alpha_pre": "model-00046-of-00155.safetensors",
+ "model.layers.23.mhc_ffn.alpha_res": "model-00046-of-00155.safetensors",
+ "model.layers.23.mhc_ffn.bias_post": "model-00046-of-00155.safetensors",
+ "model.layers.23.mhc_ffn.bias_pre": "model-00046-of-00155.safetensors",
+ "model.layers.23.mhc_ffn.bias_res": "model-00046-of-00155.safetensors",
+ "model.layers.23.mhc_ffn.proj_post.weight": "model-00046-of-00155.safetensors",
+ "model.layers.23.mhc_ffn.proj_pre.weight": "model-00046-of-00155.safetensors",
+ "model.layers.23.mhc_ffn.proj_res.weight": "model-00046-of-00155.safetensors",
+ "model.layers.23.mhc_ffn.rms_norm.weight": "model-00046-of-00155.safetensors",
+ "model.layers.23.moe.expert_bias": "model-00044-of-00155.safetensors",
+ "model.layers.23.moe.experts.act_fn.bias": "model-00046-of-00155.safetensors",
+ "model.layers.23.moe.experts.act_fn.weight": "model-00046-of-00155.safetensors",
+ "model.layers.23.moe.experts.down_proj": "model-00045-of-00155.safetensors",
+ "model.layers.23.moe.experts.gate_up_proj": "model-00126-of-00155.safetensors",
+ "model.layers.23.moe.router.gate.weight": "model-00046-of-00155.safetensors",
+ "model.layers.23.moe.shared_experts.act_fn.bias": "model-00046-of-00155.safetensors",
+ "model.layers.23.moe.shared_experts.act_fn.weight": "model-00046-of-00155.safetensors",
+ "model.layers.23.moe.shared_experts.down_proj.weight": "model-00046-of-00155.safetensors",
+ "model.layers.23.moe.shared_experts.gate_proj.weight": "model-00046-of-00155.safetensors",
+ "model.layers.23.moe.shared_experts.up_proj.weight": "model-00046-of-00155.safetensors",
+ "model.layers.23.post_attention_layernorm.weight": "model-00046-of-00155.safetensors",
+ "model.layers.23.self_attn.kv_norm.weight": "model-00044-of-00155.safetensors",
+ "model.layers.23.self_attn.lambda_proj.weight": "model-00044-of-00155.safetensors",
+ "model.layers.23.self_attn.q_norm.weight": "model-00044-of-00155.safetensors",
+ "model.layers.23.self_attn.wkv_a.weight": "model-00044-of-00155.safetensors",
+ "model.layers.23.self_attn.wkv_b.weight": "model-00044-of-00155.safetensors",
+ "model.layers.23.self_attn.wo.weight": "model-00044-of-00155.safetensors",
+ "model.layers.23.self_attn.wq_a.weight": "model-00044-of-00155.safetensors",
+ "model.layers.23.self_attn.wq_b.weight": "model-00044-of-00155.safetensors",
+ "model.layers.23.self_attn.wq_b_gate.weight": "model-00044-of-00155.safetensors",
+ "model.layers.24.input_layernorm.weight": "model-00048-of-00155.safetensors",
+ "model.layers.24.mhc_attn.alpha_post": "model-00048-of-00155.safetensors",
+ "model.layers.24.mhc_attn.alpha_pre": "model-00048-of-00155.safetensors",
+ "model.layers.24.mhc_attn.alpha_res": "model-00048-of-00155.safetensors",
+ "model.layers.24.mhc_attn.bias_post": "model-00048-of-00155.safetensors",
+ "model.layers.24.mhc_attn.bias_pre": "model-00048-of-00155.safetensors",
+ "model.layers.24.mhc_attn.bias_res": "model-00048-of-00155.safetensors",
+ "model.layers.24.mhc_attn.proj_post.weight": "model-00048-of-00155.safetensors",
+ "model.layers.24.mhc_attn.proj_pre.weight": "model-00048-of-00155.safetensors",
+ "model.layers.24.mhc_attn.proj_res.weight": "model-00048-of-00155.safetensors",
+ "model.layers.24.mhc_attn.rms_norm.weight": "model-00048-of-00155.safetensors",
+ "model.layers.24.mhc_ffn.alpha_post": "model-00048-of-00155.safetensors",
+ "model.layers.24.mhc_ffn.alpha_pre": "model-00048-of-00155.safetensors",
+ "model.layers.24.mhc_ffn.alpha_res": "model-00048-of-00155.safetensors",
+ "model.layers.24.mhc_ffn.bias_post": "model-00048-of-00155.safetensors",
+ "model.layers.24.mhc_ffn.bias_pre": "model-00048-of-00155.safetensors",
+ "model.layers.24.mhc_ffn.bias_res": "model-00048-of-00155.safetensors",
+ "model.layers.24.mhc_ffn.proj_post.weight": "model-00048-of-00155.safetensors",
+ "model.layers.24.mhc_ffn.proj_pre.weight": "model-00048-of-00155.safetensors",
+ "model.layers.24.mhc_ffn.proj_res.weight": "model-00048-of-00155.safetensors",
+ "model.layers.24.mhc_ffn.rms_norm.weight": "model-00048-of-00155.safetensors",
+ "model.layers.24.moe.expert_bias": "model-00046-of-00155.safetensors",
+ "model.layers.24.moe.experts.act_fn.bias": "model-00048-of-00155.safetensors",
+ "model.layers.24.moe.experts.act_fn.weight": "model-00048-of-00155.safetensors",
+ "model.layers.24.moe.experts.down_proj": "model-00047-of-00155.safetensors",
+ "model.layers.24.moe.experts.gate_up_proj": "model-00127-of-00155.safetensors",
+ "model.layers.24.moe.router.gate.weight": "model-00048-of-00155.safetensors",
+ "model.layers.24.moe.shared_experts.act_fn.bias": "model-00048-of-00155.safetensors",
+ "model.layers.24.moe.shared_experts.act_fn.weight": "model-00048-of-00155.safetensors",
+ "model.layers.24.moe.shared_experts.down_proj.weight": "model-00048-of-00155.safetensors",
+ "model.layers.24.moe.shared_experts.gate_proj.weight": "model-00048-of-00155.safetensors",
+ "model.layers.24.moe.shared_experts.up_proj.weight": "model-00048-of-00155.safetensors",
+ "model.layers.24.post_attention_layernorm.weight": "model-00048-of-00155.safetensors",
+ "model.layers.24.self_attn.kv_norm.weight": "model-00046-of-00155.safetensors",
+ "model.layers.24.self_attn.lambda_proj.weight": "model-00046-of-00155.safetensors",
+ "model.layers.24.self_attn.q_norm.weight": "model-00046-of-00155.safetensors",
+ "model.layers.24.self_attn.wkv_a.weight": "model-00046-of-00155.safetensors",
+ "model.layers.24.self_attn.wkv_b.weight": "model-00046-of-00155.safetensors",
+ "model.layers.24.self_attn.wo.weight": "model-00046-of-00155.safetensors",
+ "model.layers.24.self_attn.wq_a.weight": "model-00046-of-00155.safetensors",
+ "model.layers.24.self_attn.wq_b.weight": "model-00046-of-00155.safetensors",
+ "model.layers.24.self_attn.wq_b_gate.weight": "model-00046-of-00155.safetensors",
+ "model.layers.25.input_layernorm.weight": "model-00050-of-00155.safetensors",
+ "model.layers.25.mhc_attn.alpha_post": "model-00050-of-00155.safetensors",
+ "model.layers.25.mhc_attn.alpha_pre": "model-00050-of-00155.safetensors",
+ "model.layers.25.mhc_attn.alpha_res": "model-00050-of-00155.safetensors",
+ "model.layers.25.mhc_attn.bias_post": "model-00050-of-00155.safetensors",
+ "model.layers.25.mhc_attn.bias_pre": "model-00050-of-00155.safetensors",
+ "model.layers.25.mhc_attn.bias_res": "model-00050-of-00155.safetensors",
+ "model.layers.25.mhc_attn.proj_post.weight": "model-00050-of-00155.safetensors",
+ "model.layers.25.mhc_attn.proj_pre.weight": "model-00050-of-00155.safetensors",
+ "model.layers.25.mhc_attn.proj_res.weight": "model-00050-of-00155.safetensors",
+ "model.layers.25.mhc_attn.rms_norm.weight": "model-00050-of-00155.safetensors",
+ "model.layers.25.mhc_ffn.alpha_post": "model-00050-of-00155.safetensors",
+ "model.layers.25.mhc_ffn.alpha_pre": "model-00050-of-00155.safetensors",
+ "model.layers.25.mhc_ffn.alpha_res": "model-00050-of-00155.safetensors",
+ "model.layers.25.mhc_ffn.bias_post": "model-00050-of-00155.safetensors",
+ "model.layers.25.mhc_ffn.bias_pre": "model-00050-of-00155.safetensors",
+ "model.layers.25.mhc_ffn.bias_res": "model-00050-of-00155.safetensors",
+ "model.layers.25.mhc_ffn.proj_post.weight": "model-00050-of-00155.safetensors",
+ "model.layers.25.mhc_ffn.proj_pre.weight": "model-00050-of-00155.safetensors",
+ "model.layers.25.mhc_ffn.proj_res.weight": "model-00050-of-00155.safetensors",
+ "model.layers.25.mhc_ffn.rms_norm.weight": "model-00050-of-00155.safetensors",
+ "model.layers.25.moe.expert_bias": "model-00048-of-00155.safetensors",
+ "model.layers.25.moe.experts.act_fn.bias": "model-00050-of-00155.safetensors",
+ "model.layers.25.moe.experts.act_fn.weight": "model-00050-of-00155.safetensors",
+ "model.layers.25.moe.experts.down_proj": "model-00049-of-00155.safetensors",
+ "model.layers.25.moe.experts.gate_up_proj": "model-00128-of-00155.safetensors",
+ "model.layers.25.moe.router.gate.weight": "model-00050-of-00155.safetensors",
+ "model.layers.25.moe.shared_experts.act_fn.bias": "model-00050-of-00155.safetensors",
+ "model.layers.25.moe.shared_experts.act_fn.weight": "model-00050-of-00155.safetensors",
+ "model.layers.25.moe.shared_experts.down_proj.weight": "model-00050-of-00155.safetensors",
+ "model.layers.25.moe.shared_experts.gate_proj.weight": "model-00050-of-00155.safetensors",
+ "model.layers.25.moe.shared_experts.up_proj.weight": "model-00050-of-00155.safetensors",
+ "model.layers.25.post_attention_layernorm.weight": "model-00050-of-00155.safetensors",
+ "model.layers.25.self_attn.kv_norm.weight": "model-00048-of-00155.safetensors",
+ "model.layers.25.self_attn.lambda_proj.weight": "model-00048-of-00155.safetensors",
+ "model.layers.25.self_attn.q_norm.weight": "model-00048-of-00155.safetensors",
+ "model.layers.25.self_attn.wkv_a.weight": "model-00048-of-00155.safetensors",
+ "model.layers.25.self_attn.wkv_b.weight": "model-00048-of-00155.safetensors",
+ "model.layers.25.self_attn.wo.weight": "model-00048-of-00155.safetensors",
+ "model.layers.25.self_attn.wq_a.weight": "model-00048-of-00155.safetensors",
+ "model.layers.25.self_attn.wq_b.weight": "model-00048-of-00155.safetensors",
+ "model.layers.25.self_attn.wq_b_gate.weight": "model-00048-of-00155.safetensors",
+ "model.layers.26.input_layernorm.weight": "model-00052-of-00155.safetensors",
+ "model.layers.26.mhc_attn.alpha_post": "model-00052-of-00155.safetensors",
+ "model.layers.26.mhc_attn.alpha_pre": "model-00052-of-00155.safetensors",
+ "model.layers.26.mhc_attn.alpha_res": "model-00052-of-00155.safetensors",
+ "model.layers.26.mhc_attn.bias_post": "model-00052-of-00155.safetensors",
+ "model.layers.26.mhc_attn.bias_pre": "model-00052-of-00155.safetensors",
+ "model.layers.26.mhc_attn.bias_res": "model-00052-of-00155.safetensors",
+ "model.layers.26.mhc_attn.proj_post.weight": "model-00052-of-00155.safetensors",
+ "model.layers.26.mhc_attn.proj_pre.weight": "model-00052-of-00155.safetensors",
+ "model.layers.26.mhc_attn.proj_res.weight": "model-00052-of-00155.safetensors",
+ "model.layers.26.mhc_attn.rms_norm.weight": "model-00052-of-00155.safetensors",
+ "model.layers.26.mhc_ffn.alpha_post": "model-00052-of-00155.safetensors",
+ "model.layers.26.mhc_ffn.alpha_pre": "model-00052-of-00155.safetensors",
+ "model.layers.26.mhc_ffn.alpha_res": "model-00052-of-00155.safetensors",
+ "model.layers.26.mhc_ffn.bias_post": "model-00052-of-00155.safetensors",
+ "model.layers.26.mhc_ffn.bias_pre": "model-00052-of-00155.safetensors",
+ "model.layers.26.mhc_ffn.bias_res": "model-00052-of-00155.safetensors",
+ "model.layers.26.mhc_ffn.proj_post.weight": "model-00052-of-00155.safetensors",
+ "model.layers.26.mhc_ffn.proj_pre.weight": "model-00052-of-00155.safetensors",
+ "model.layers.26.mhc_ffn.proj_res.weight": "model-00052-of-00155.safetensors",
+ "model.layers.26.mhc_ffn.rms_norm.weight": "model-00052-of-00155.safetensors",
+ "model.layers.26.moe.expert_bias": "model-00050-of-00155.safetensors",
+ "model.layers.26.moe.experts.act_fn.bias": "model-00052-of-00155.safetensors",
+ "model.layers.26.moe.experts.act_fn.weight": "model-00052-of-00155.safetensors",
+ "model.layers.26.moe.experts.down_proj": "model-00051-of-00155.safetensors",
+ "model.layers.26.moe.experts.gate_up_proj": "model-00129-of-00155.safetensors",
+ "model.layers.26.moe.router.gate.weight": "model-00052-of-00155.safetensors",
+ "model.layers.26.moe.shared_experts.act_fn.bias": "model-00052-of-00155.safetensors",
+ "model.layers.26.moe.shared_experts.act_fn.weight": "model-00052-of-00155.safetensors",
+ "model.layers.26.moe.shared_experts.down_proj.weight": "model-00052-of-00155.safetensors",
+ "model.layers.26.moe.shared_experts.gate_proj.weight": "model-00052-of-00155.safetensors",
+ "model.layers.26.moe.shared_experts.up_proj.weight": "model-00052-of-00155.safetensors",
+ "model.layers.26.post_attention_layernorm.weight": "model-00052-of-00155.safetensors",
+ "model.layers.26.self_attn.kv_norm.weight": "model-00050-of-00155.safetensors",
+ "model.layers.26.self_attn.lambda_proj.weight": "model-00050-of-00155.safetensors",
+ "model.layers.26.self_attn.q_norm.weight": "model-00050-of-00155.safetensors",
+ "model.layers.26.self_attn.wkv_a.weight": "model-00050-of-00155.safetensors",
+ "model.layers.26.self_attn.wkv_b.weight": "model-00050-of-00155.safetensors",
+ "model.layers.26.self_attn.wo.weight": "model-00050-of-00155.safetensors",
+ "model.layers.26.self_attn.wq_a.weight": "model-00050-of-00155.safetensors",
+ "model.layers.26.self_attn.wq_b.weight": "model-00050-of-00155.safetensors",
+ "model.layers.26.self_attn.wq_b_gate.weight": "model-00050-of-00155.safetensors",
+ "model.layers.27.input_layernorm.weight": "model-00054-of-00155.safetensors",
+ "model.layers.27.mhc_attn.alpha_post": "model-00054-of-00155.safetensors",
+ "model.layers.27.mhc_attn.alpha_pre": "model-00054-of-00155.safetensors",
+ "model.layers.27.mhc_attn.alpha_res": "model-00054-of-00155.safetensors",
+ "model.layers.27.mhc_attn.bias_post": "model-00054-of-00155.safetensors",
+ "model.layers.27.mhc_attn.bias_pre": "model-00054-of-00155.safetensors",
+ "model.layers.27.mhc_attn.bias_res": "model-00054-of-00155.safetensors",
+ "model.layers.27.mhc_attn.proj_post.weight": "model-00054-of-00155.safetensors",
+ "model.layers.27.mhc_attn.proj_pre.weight": "model-00054-of-00155.safetensors",
+ "model.layers.27.mhc_attn.proj_res.weight": "model-00054-of-00155.safetensors",
+ "model.layers.27.mhc_attn.rms_norm.weight": "model-00054-of-00155.safetensors",
+ "model.layers.27.mhc_ffn.alpha_post": "model-00054-of-00155.safetensors",
+ "model.layers.27.mhc_ffn.alpha_pre": "model-00054-of-00155.safetensors",
+ "model.layers.27.mhc_ffn.alpha_res": "model-00054-of-00155.safetensors",
+ "model.layers.27.mhc_ffn.bias_post": "model-00054-of-00155.safetensors",
+ "model.layers.27.mhc_ffn.bias_pre": "model-00054-of-00155.safetensors",
+ "model.layers.27.mhc_ffn.bias_res": "model-00054-of-00155.safetensors",
+ "model.layers.27.mhc_ffn.proj_post.weight": "model-00054-of-00155.safetensors",
+ "model.layers.27.mhc_ffn.proj_pre.weight": "model-00054-of-00155.safetensors",
+ "model.layers.27.mhc_ffn.proj_res.weight": "model-00054-of-00155.safetensors",
+ "model.layers.27.mhc_ffn.rms_norm.weight": "model-00054-of-00155.safetensors",
+ "model.layers.27.moe.expert_bias": "model-00052-of-00155.safetensors",
+ "model.layers.27.moe.experts.act_fn.bias": "model-00054-of-00155.safetensors",
+ "model.layers.27.moe.experts.act_fn.weight": "model-00054-of-00155.safetensors",
+ "model.layers.27.moe.experts.down_proj": "model-00053-of-00155.safetensors",
+ "model.layers.27.moe.experts.gate_up_proj": "model-00130-of-00155.safetensors",
+ "model.layers.27.moe.router.gate.weight": "model-00054-of-00155.safetensors",
+ "model.layers.27.moe.shared_experts.act_fn.bias": "model-00054-of-00155.safetensors",
+ "model.layers.27.moe.shared_experts.act_fn.weight": "model-00054-of-00155.safetensors",
+ "model.layers.27.moe.shared_experts.down_proj.weight": "model-00054-of-00155.safetensors",
+ "model.layers.27.moe.shared_experts.gate_proj.weight": "model-00054-of-00155.safetensors",
+ "model.layers.27.moe.shared_experts.up_proj.weight": "model-00054-of-00155.safetensors",
+ "model.layers.27.post_attention_layernorm.weight": "model-00054-of-00155.safetensors",
+ "model.layers.27.self_attn.kv_norm.weight": "model-00052-of-00155.safetensors",
+ "model.layers.27.self_attn.lambda_proj.weight": "model-00052-of-00155.safetensors",
+ "model.layers.27.self_attn.q_norm.weight": "model-00052-of-00155.safetensors",
+ "model.layers.27.self_attn.wkv_a.weight": "model-00052-of-00155.safetensors",
+ "model.layers.27.self_attn.wkv_b.weight": "model-00052-of-00155.safetensors",
+ "model.layers.27.self_attn.wo.weight": "model-00052-of-00155.safetensors",
+ "model.layers.27.self_attn.wq_a.weight": "model-00052-of-00155.safetensors",
+ "model.layers.27.self_attn.wq_b.weight": "model-00052-of-00155.safetensors",
+ "model.layers.27.self_attn.wq_b_gate.weight": "model-00052-of-00155.safetensors",
+ "model.layers.28.input_layernorm.weight": "model-00056-of-00155.safetensors",
+ "model.layers.28.mhc_attn.alpha_post": "model-00056-of-00155.safetensors",
+ "model.layers.28.mhc_attn.alpha_pre": "model-00056-of-00155.safetensors",
+ "model.layers.28.mhc_attn.alpha_res": "model-00056-of-00155.safetensors",
+ "model.layers.28.mhc_attn.bias_post": "model-00056-of-00155.safetensors",
+ "model.layers.28.mhc_attn.bias_pre": "model-00056-of-00155.safetensors",
+ "model.layers.28.mhc_attn.bias_res": "model-00056-of-00155.safetensors",
+ "model.layers.28.mhc_attn.proj_post.weight": "model-00056-of-00155.safetensors",
+ "model.layers.28.mhc_attn.proj_pre.weight": "model-00056-of-00155.safetensors",
+ "model.layers.28.mhc_attn.proj_res.weight": "model-00056-of-00155.safetensors",
+ "model.layers.28.mhc_attn.rms_norm.weight": "model-00056-of-00155.safetensors",
+ "model.layers.28.mhc_ffn.alpha_post": "model-00056-of-00155.safetensors",
+ "model.layers.28.mhc_ffn.alpha_pre": "model-00056-of-00155.safetensors",
+ "model.layers.28.mhc_ffn.alpha_res": "model-00056-of-00155.safetensors",
+ "model.layers.28.mhc_ffn.bias_post": "model-00056-of-00155.safetensors",
+ "model.layers.28.mhc_ffn.bias_pre": "model-00056-of-00155.safetensors",
+ "model.layers.28.mhc_ffn.bias_res": "model-00056-of-00155.safetensors",
+ "model.layers.28.mhc_ffn.proj_post.weight": "model-00056-of-00155.safetensors",
+ "model.layers.28.mhc_ffn.proj_pre.weight": "model-00056-of-00155.safetensors",
+ "model.layers.28.mhc_ffn.proj_res.weight": "model-00056-of-00155.safetensors",
+ "model.layers.28.mhc_ffn.rms_norm.weight": "model-00056-of-00155.safetensors",
+ "model.layers.28.moe.expert_bias": "model-00054-of-00155.safetensors",
+ "model.layers.28.moe.experts.act_fn.bias": "model-00056-of-00155.safetensors",
+ "model.layers.28.moe.experts.act_fn.weight": "model-00056-of-00155.safetensors",
+ "model.layers.28.moe.experts.down_proj": "model-00055-of-00155.safetensors",
+ "model.layers.28.moe.experts.gate_up_proj": "model-00131-of-00155.safetensors",
+ "model.layers.28.moe.router.gate.weight": "model-00056-of-00155.safetensors",
+ "model.layers.28.moe.shared_experts.act_fn.bias": "model-00056-of-00155.safetensors",
+ "model.layers.28.moe.shared_experts.act_fn.weight": "model-00056-of-00155.safetensors",
+ "model.layers.28.moe.shared_experts.down_proj.weight": "model-00056-of-00155.safetensors",
+ "model.layers.28.moe.shared_experts.gate_proj.weight": "model-00056-of-00155.safetensors",
+ "model.layers.28.moe.shared_experts.up_proj.weight": "model-00056-of-00155.safetensors",
+ "model.layers.28.post_attention_layernorm.weight": "model-00056-of-00155.safetensors",
+ "model.layers.28.self_attn.kv_norm.weight": "model-00054-of-00155.safetensors",
+ "model.layers.28.self_attn.lambda_proj.weight": "model-00054-of-00155.safetensors",
+ "model.layers.28.self_attn.q_norm.weight": "model-00054-of-00155.safetensors",
+ "model.layers.28.self_attn.wkv_a.weight": "model-00054-of-00155.safetensors",
+ "model.layers.28.self_attn.wkv_b.weight": "model-00054-of-00155.safetensors",
+ "model.layers.28.self_attn.wo.weight": "model-00054-of-00155.safetensors",
+ "model.layers.28.self_attn.wq_a.weight": "model-00054-of-00155.safetensors",
+ "model.layers.28.self_attn.wq_b.weight": "model-00054-of-00155.safetensors",
+ "model.layers.28.self_attn.wq_b_gate.weight": "model-00054-of-00155.safetensors",
+ "model.layers.29.input_layernorm.weight": "model-00058-of-00155.safetensors",
+ "model.layers.29.mhc_attn.alpha_post": "model-00058-of-00155.safetensors",
+ "model.layers.29.mhc_attn.alpha_pre": "model-00058-of-00155.safetensors",
+ "model.layers.29.mhc_attn.alpha_res": "model-00058-of-00155.safetensors",
+ "model.layers.29.mhc_attn.bias_post": "model-00058-of-00155.safetensors",
+ "model.layers.29.mhc_attn.bias_pre": "model-00058-of-00155.safetensors",
+ "model.layers.29.mhc_attn.bias_res": "model-00058-of-00155.safetensors",
+ "model.layers.29.mhc_attn.proj_post.weight": "model-00058-of-00155.safetensors",
+ "model.layers.29.mhc_attn.proj_pre.weight": "model-00058-of-00155.safetensors",
+ "model.layers.29.mhc_attn.proj_res.weight": "model-00058-of-00155.safetensors",
+ "model.layers.29.mhc_attn.rms_norm.weight": "model-00058-of-00155.safetensors",
+ "model.layers.29.mhc_ffn.alpha_post": "model-00058-of-00155.safetensors",
+ "model.layers.29.mhc_ffn.alpha_pre": "model-00058-of-00155.safetensors",
+ "model.layers.29.mhc_ffn.alpha_res": "model-00058-of-00155.safetensors",
+ "model.layers.29.mhc_ffn.bias_post": "model-00058-of-00155.safetensors",
+ "model.layers.29.mhc_ffn.bias_pre": "model-00058-of-00155.safetensors",
+ "model.layers.29.mhc_ffn.bias_res": "model-00058-of-00155.safetensors",
+ "model.layers.29.mhc_ffn.proj_post.weight": "model-00058-of-00155.safetensors",
+ "model.layers.29.mhc_ffn.proj_pre.weight": "model-00058-of-00155.safetensors",
+ "model.layers.29.mhc_ffn.proj_res.weight": "model-00058-of-00155.safetensors",
+ "model.layers.29.mhc_ffn.rms_norm.weight": "model-00058-of-00155.safetensors",
+ "model.layers.29.moe.expert_bias": "model-00056-of-00155.safetensors",
+ "model.layers.29.moe.experts.act_fn.bias": "model-00058-of-00155.safetensors",
+ "model.layers.29.moe.experts.act_fn.weight": "model-00058-of-00155.safetensors",
+ "model.layers.29.moe.experts.down_proj": "model-00057-of-00155.safetensors",
+ "model.layers.29.moe.experts.gate_up_proj": "model-00132-of-00155.safetensors",
+ "model.layers.29.moe.router.gate.weight": "model-00058-of-00155.safetensors",
+ "model.layers.29.moe.shared_experts.act_fn.bias": "model-00058-of-00155.safetensors",
+ "model.layers.29.moe.shared_experts.act_fn.weight": "model-00058-of-00155.safetensors",
+ "model.layers.29.moe.shared_experts.down_proj.weight": "model-00058-of-00155.safetensors",
+ "model.layers.29.moe.shared_experts.gate_proj.weight": "model-00058-of-00155.safetensors",
+ "model.layers.29.moe.shared_experts.up_proj.weight": "model-00058-of-00155.safetensors",
+ "model.layers.29.post_attention_layernorm.weight": "model-00058-of-00155.safetensors",
+ "model.layers.29.self_attn.kv_norm.weight": "model-00056-of-00155.safetensors",
+ "model.layers.29.self_attn.lambda_proj.weight": "model-00056-of-00155.safetensors",
+ "model.layers.29.self_attn.q_norm.weight": "model-00056-of-00155.safetensors",
+ "model.layers.29.self_attn.wkv_a.weight": "model-00056-of-00155.safetensors",
+ "model.layers.29.self_attn.wkv_b.weight": "model-00056-of-00155.safetensors",
+ "model.layers.29.self_attn.wo.weight": "model-00056-of-00155.safetensors",
+ "model.layers.29.self_attn.wq_a.weight": "model-00056-of-00155.safetensors",
+ "model.layers.29.self_attn.wq_b.weight": "model-00056-of-00155.safetensors",
+ "model.layers.29.self_attn.wq_b_gate.weight": "model-00056-of-00155.safetensors",
+ "model.layers.3.input_layernorm.weight": "model-00006-of-00155.safetensors",
+ "model.layers.3.mhc_attn.alpha_post": "model-00006-of-00155.safetensors",
+ "model.layers.3.mhc_attn.alpha_pre": "model-00006-of-00155.safetensors",
+ "model.layers.3.mhc_attn.alpha_res": "model-00006-of-00155.safetensors",
+ "model.layers.3.mhc_attn.bias_post": "model-00006-of-00155.safetensors",
+ "model.layers.3.mhc_attn.bias_pre": "model-00006-of-00155.safetensors",
+ "model.layers.3.mhc_attn.bias_res": "model-00006-of-00155.safetensors",
+ "model.layers.3.mhc_attn.proj_post.weight": "model-00006-of-00155.safetensors",
+ "model.layers.3.mhc_attn.proj_pre.weight": "model-00006-of-00155.safetensors",
+ "model.layers.3.mhc_attn.proj_res.weight": "model-00006-of-00155.safetensors",
+ "model.layers.3.mhc_attn.rms_norm.weight": "model-00006-of-00155.safetensors",
+ "model.layers.3.mhc_ffn.alpha_post": "model-00006-of-00155.safetensors",
+ "model.layers.3.mhc_ffn.alpha_pre": "model-00006-of-00155.safetensors",
+ "model.layers.3.mhc_ffn.alpha_res": "model-00006-of-00155.safetensors",
+ "model.layers.3.mhc_ffn.bias_post": "model-00006-of-00155.safetensors",
+ "model.layers.3.mhc_ffn.bias_pre": "model-00006-of-00155.safetensors",
+ "model.layers.3.mhc_ffn.bias_res": "model-00006-of-00155.safetensors",
+ "model.layers.3.mhc_ffn.proj_post.weight": "model-00006-of-00155.safetensors",
+ "model.layers.3.mhc_ffn.proj_pre.weight": "model-00006-of-00155.safetensors",
+ "model.layers.3.mhc_ffn.proj_res.weight": "model-00006-of-00155.safetensors",
+ "model.layers.3.mhc_ffn.rms_norm.weight": "model-00006-of-00155.safetensors",
+ "model.layers.3.moe.expert_bias": "model-00004-of-00155.safetensors",
+ "model.layers.3.moe.experts.act_fn.bias": "model-00006-of-00155.safetensors",
+ "model.layers.3.moe.experts.act_fn.weight": "model-00006-of-00155.safetensors",
+ "model.layers.3.moe.experts.down_proj": "model-00005-of-00155.safetensors",
+ "model.layers.3.moe.experts.gate_up_proj": "model-00106-of-00155.safetensors",
+ "model.layers.3.moe.router.gate.weight": "model-00006-of-00155.safetensors",
+ "model.layers.3.moe.shared_experts.act_fn.bias": "model-00006-of-00155.safetensors",
+ "model.layers.3.moe.shared_experts.act_fn.weight": "model-00006-of-00155.safetensors",
+ "model.layers.3.moe.shared_experts.down_proj.weight": "model-00006-of-00155.safetensors",
+ "model.layers.3.moe.shared_experts.gate_proj.weight": "model-00006-of-00155.safetensors",
+ "model.layers.3.moe.shared_experts.up_proj.weight": "model-00006-of-00155.safetensors",
+ "model.layers.3.post_attention_layernorm.weight": "model-00006-of-00155.safetensors",
+ "model.layers.3.self_attn.kv_norm.weight": "model-00004-of-00155.safetensors",
+ "model.layers.3.self_attn.lambda_proj.weight": "model-00004-of-00155.safetensors",
+ "model.layers.3.self_attn.q_norm.weight": "model-00004-of-00155.safetensors",
+ "model.layers.3.self_attn.wkv_a.weight": "model-00004-of-00155.safetensors",
+ "model.layers.3.self_attn.wkv_b.weight": "model-00004-of-00155.safetensors",
+ "model.layers.3.self_attn.wo.weight": "model-00004-of-00155.safetensors",
+ "model.layers.3.self_attn.wq_a.weight": "model-00004-of-00155.safetensors",
+ "model.layers.3.self_attn.wq_b.weight": "model-00004-of-00155.safetensors",
+ "model.layers.3.self_attn.wq_b_gate.weight": "model-00004-of-00155.safetensors",
+ "model.layers.30.input_layernorm.weight": "model-00060-of-00155.safetensors",
+ "model.layers.30.mhc_attn.alpha_post": "model-00060-of-00155.safetensors",
+ "model.layers.30.mhc_attn.alpha_pre": "model-00060-of-00155.safetensors",
+ "model.layers.30.mhc_attn.alpha_res": "model-00060-of-00155.safetensors",
+ "model.layers.30.mhc_attn.bias_post": "model-00060-of-00155.safetensors",
+ "model.layers.30.mhc_attn.bias_pre": "model-00060-of-00155.safetensors",
+ "model.layers.30.mhc_attn.bias_res": "model-00060-of-00155.safetensors",
+ "model.layers.30.mhc_attn.proj_post.weight": "model-00060-of-00155.safetensors",
+ "model.layers.30.mhc_attn.proj_pre.weight": "model-00060-of-00155.safetensors",
+ "model.layers.30.mhc_attn.proj_res.weight": "model-00060-of-00155.safetensors",
+ "model.layers.30.mhc_attn.rms_norm.weight": "model-00060-of-00155.safetensors",
+ "model.layers.30.mhc_ffn.alpha_post": "model-00060-of-00155.safetensors",
+ "model.layers.30.mhc_ffn.alpha_pre": "model-00060-of-00155.safetensors",
+ "model.layers.30.mhc_ffn.alpha_res": "model-00060-of-00155.safetensors",
+ "model.layers.30.mhc_ffn.bias_post": "model-00060-of-00155.safetensors",
+ "model.layers.30.mhc_ffn.bias_pre": "model-00060-of-00155.safetensors",
+ "model.layers.30.mhc_ffn.bias_res": "model-00060-of-00155.safetensors",
+ "model.layers.30.mhc_ffn.proj_post.weight": "model-00060-of-00155.safetensors",
+ "model.layers.30.mhc_ffn.proj_pre.weight": "model-00060-of-00155.safetensors",
+ "model.layers.30.mhc_ffn.proj_res.weight": "model-00060-of-00155.safetensors",
+ "model.layers.30.mhc_ffn.rms_norm.weight": "model-00060-of-00155.safetensors",
+ "model.layers.30.moe.expert_bias": "model-00058-of-00155.safetensors",
+ "model.layers.30.moe.experts.act_fn.bias": "model-00060-of-00155.safetensors",
+ "model.layers.30.moe.experts.act_fn.weight": "model-00060-of-00155.safetensors",
+ "model.layers.30.moe.experts.down_proj": "model-00059-of-00155.safetensors",
+ "model.layers.30.moe.experts.gate_up_proj": "model-00133-of-00155.safetensors",
+ "model.layers.30.moe.router.gate.weight": "model-00060-of-00155.safetensors",
+ "model.layers.30.moe.shared_experts.act_fn.bias": "model-00060-of-00155.safetensors",
+ "model.layers.30.moe.shared_experts.act_fn.weight": "model-00060-of-00155.safetensors",
+ "model.layers.30.moe.shared_experts.down_proj.weight": "model-00060-of-00155.safetensors",
+ "model.layers.30.moe.shared_experts.gate_proj.weight": "model-00060-of-00155.safetensors",
+ "model.layers.30.moe.shared_experts.up_proj.weight": "model-00060-of-00155.safetensors",
+ "model.layers.30.post_attention_layernorm.weight": "model-00060-of-00155.safetensors",
+ "model.layers.30.self_attn.kv_norm.weight": "model-00058-of-00155.safetensors",
+ "model.layers.30.self_attn.lambda_proj.weight": "model-00058-of-00155.safetensors",
+ "model.layers.30.self_attn.q_norm.weight": "model-00058-of-00155.safetensors",
+ "model.layers.30.self_attn.wkv_a.weight": "model-00058-of-00155.safetensors",
+ "model.layers.30.self_attn.wkv_b.weight": "model-00058-of-00155.safetensors",
+ "model.layers.30.self_attn.wo.weight": "model-00058-of-00155.safetensors",
+ "model.layers.30.self_attn.wq_a.weight": "model-00058-of-00155.safetensors",
+ "model.layers.30.self_attn.wq_b.weight": "model-00058-of-00155.safetensors",
+ "model.layers.30.self_attn.wq_b_gate.weight": "model-00058-of-00155.safetensors",
+ "model.layers.31.input_layernorm.weight": "model-00062-of-00155.safetensors",
+ "model.layers.31.mhc_attn.alpha_post": "model-00062-of-00155.safetensors",
+ "model.layers.31.mhc_attn.alpha_pre": "model-00062-of-00155.safetensors",
+ "model.layers.31.mhc_attn.alpha_res": "model-00062-of-00155.safetensors",
+ "model.layers.31.mhc_attn.bias_post": "model-00062-of-00155.safetensors",
+ "model.layers.31.mhc_attn.bias_pre": "model-00062-of-00155.safetensors",
+ "model.layers.31.mhc_attn.bias_res": "model-00062-of-00155.safetensors",
+ "model.layers.31.mhc_attn.proj_post.weight": "model-00062-of-00155.safetensors",
+ "model.layers.31.mhc_attn.proj_pre.weight": "model-00062-of-00155.safetensors",
+ "model.layers.31.mhc_attn.proj_res.weight": "model-00062-of-00155.safetensors",
+ "model.layers.31.mhc_attn.rms_norm.weight": "model-00062-of-00155.safetensors",
+ "model.layers.31.mhc_ffn.alpha_post": "model-00062-of-00155.safetensors",
+ "model.layers.31.mhc_ffn.alpha_pre": "model-00062-of-00155.safetensors",
+ "model.layers.31.mhc_ffn.alpha_res": "model-00062-of-00155.safetensors",
+ "model.layers.31.mhc_ffn.bias_post": "model-00062-of-00155.safetensors",
+ "model.layers.31.mhc_ffn.bias_pre": "model-00062-of-00155.safetensors",
+ "model.layers.31.mhc_ffn.bias_res": "model-00062-of-00155.safetensors",
+ "model.layers.31.mhc_ffn.proj_post.weight": "model-00062-of-00155.safetensors",
+ "model.layers.31.mhc_ffn.proj_pre.weight": "model-00062-of-00155.safetensors",
+ "model.layers.31.mhc_ffn.proj_res.weight": "model-00062-of-00155.safetensors",
+ "model.layers.31.mhc_ffn.rms_norm.weight": "model-00062-of-00155.safetensors",
+ "model.layers.31.moe.expert_bias": "model-00060-of-00155.safetensors",
+ "model.layers.31.moe.experts.act_fn.bias": "model-00062-of-00155.safetensors",
+ "model.layers.31.moe.experts.act_fn.weight": "model-00062-of-00155.safetensors",
+ "model.layers.31.moe.experts.down_proj": "model-00061-of-00155.safetensors",
+ "model.layers.31.moe.experts.gate_up_proj": "model-00134-of-00155.safetensors",
+ "model.layers.31.moe.router.gate.weight": "model-00062-of-00155.safetensors",
+ "model.layers.31.moe.shared_experts.act_fn.bias": "model-00062-of-00155.safetensors",
+ "model.layers.31.moe.shared_experts.act_fn.weight": "model-00062-of-00155.safetensors",
+ "model.layers.31.moe.shared_experts.down_proj.weight": "model-00062-of-00155.safetensors",
+ "model.layers.31.moe.shared_experts.gate_proj.weight": "model-00062-of-00155.safetensors",
+ "model.layers.31.moe.shared_experts.up_proj.weight": "model-00062-of-00155.safetensors",
+ "model.layers.31.post_attention_layernorm.weight": "model-00062-of-00155.safetensors",
+ "model.layers.31.self_attn.kv_norm.weight": "model-00060-of-00155.safetensors",
+ "model.layers.31.self_attn.lambda_proj.weight": "model-00060-of-00155.safetensors",
+ "model.layers.31.self_attn.q_norm.weight": "model-00060-of-00155.safetensors",
+ "model.layers.31.self_attn.wkv_a.weight": "model-00060-of-00155.safetensors",
+ "model.layers.31.self_attn.wkv_b.weight": "model-00060-of-00155.safetensors",
+ "model.layers.31.self_attn.wo.weight": "model-00060-of-00155.safetensors",
+ "model.layers.31.self_attn.wq_a.weight": "model-00060-of-00155.safetensors",
+ "model.layers.31.self_attn.wq_b.weight": "model-00060-of-00155.safetensors",
+ "model.layers.31.self_attn.wq_b_gate.weight": "model-00060-of-00155.safetensors",
+ "model.layers.32.input_layernorm.weight": "model-00064-of-00155.safetensors",
+ "model.layers.32.mhc_attn.alpha_post": "model-00064-of-00155.safetensors",
+ "model.layers.32.mhc_attn.alpha_pre": "model-00064-of-00155.safetensors",
+ "model.layers.32.mhc_attn.alpha_res": "model-00064-of-00155.safetensors",
+ "model.layers.32.mhc_attn.bias_post": "model-00064-of-00155.safetensors",
+ "model.layers.32.mhc_attn.bias_pre": "model-00064-of-00155.safetensors",
+ "model.layers.32.mhc_attn.bias_res": "model-00064-of-00155.safetensors",
+ "model.layers.32.mhc_attn.proj_post.weight": "model-00064-of-00155.safetensors",
+ "model.layers.32.mhc_attn.proj_pre.weight": "model-00064-of-00155.safetensors",
+ "model.layers.32.mhc_attn.proj_res.weight": "model-00064-of-00155.safetensors",
+ "model.layers.32.mhc_attn.rms_norm.weight": "model-00064-of-00155.safetensors",
+ "model.layers.32.mhc_ffn.alpha_post": "model-00064-of-00155.safetensors",
+ "model.layers.32.mhc_ffn.alpha_pre": "model-00064-of-00155.safetensors",
+ "model.layers.32.mhc_ffn.alpha_res": "model-00064-of-00155.safetensors",
+ "model.layers.32.mhc_ffn.bias_post": "model-00064-of-00155.safetensors",
+ "model.layers.32.mhc_ffn.bias_pre": "model-00064-of-00155.safetensors",
+ "model.layers.32.mhc_ffn.bias_res": "model-00064-of-00155.safetensors",
+ "model.layers.32.mhc_ffn.proj_post.weight": "model-00064-of-00155.safetensors",
+ "model.layers.32.mhc_ffn.proj_pre.weight": "model-00064-of-00155.safetensors",
+ "model.layers.32.mhc_ffn.proj_res.weight": "model-00064-of-00155.safetensors",
+ "model.layers.32.mhc_ffn.rms_norm.weight": "model-00064-of-00155.safetensors",
+ "model.layers.32.moe.expert_bias": "model-00062-of-00155.safetensors",
+ "model.layers.32.moe.experts.act_fn.bias": "model-00064-of-00155.safetensors",
+ "model.layers.32.moe.experts.act_fn.weight": "model-00064-of-00155.safetensors",
+ "model.layers.32.moe.experts.down_proj": "model-00063-of-00155.safetensors",
+ "model.layers.32.moe.experts.gate_up_proj": "model-00135-of-00155.safetensors",
+ "model.layers.32.moe.router.gate.weight": "model-00064-of-00155.safetensors",
+ "model.layers.32.moe.shared_experts.act_fn.bias": "model-00064-of-00155.safetensors",
+ "model.layers.32.moe.shared_experts.act_fn.weight": "model-00064-of-00155.safetensors",
+ "model.layers.32.moe.shared_experts.down_proj.weight": "model-00064-of-00155.safetensors",
+ "model.layers.32.moe.shared_experts.gate_proj.weight": "model-00064-of-00155.safetensors",
+ "model.layers.32.moe.shared_experts.up_proj.weight": "model-00064-of-00155.safetensors",
+ "model.layers.32.post_attention_layernorm.weight": "model-00064-of-00155.safetensors",
+ "model.layers.32.self_attn.kv_norm.weight": "model-00062-of-00155.safetensors",
+ "model.layers.32.self_attn.lambda_proj.weight": "model-00062-of-00155.safetensors",
+ "model.layers.32.self_attn.q_norm.weight": "model-00062-of-00155.safetensors",
+ "model.layers.32.self_attn.wkv_a.weight": "model-00062-of-00155.safetensors",
+ "model.layers.32.self_attn.wkv_b.weight": "model-00062-of-00155.safetensors",
+ "model.layers.32.self_attn.wo.weight": "model-00062-of-00155.safetensors",
+ "model.layers.32.self_attn.wq_a.weight": "model-00062-of-00155.safetensors",
+ "model.layers.32.self_attn.wq_b.weight": "model-00062-of-00155.safetensors",
+ "model.layers.32.self_attn.wq_b_gate.weight": "model-00062-of-00155.safetensors",
+ "model.layers.33.input_layernorm.weight": "model-00066-of-00155.safetensors",
+ "model.layers.33.mhc_attn.alpha_post": "model-00066-of-00155.safetensors",
+ "model.layers.33.mhc_attn.alpha_pre": "model-00066-of-00155.safetensors",
+ "model.layers.33.mhc_attn.alpha_res": "model-00066-of-00155.safetensors",
+ "model.layers.33.mhc_attn.bias_post": "model-00066-of-00155.safetensors",
+ "model.layers.33.mhc_attn.bias_pre": "model-00066-of-00155.safetensors",
+ "model.layers.33.mhc_attn.bias_res": "model-00066-of-00155.safetensors",
+ "model.layers.33.mhc_attn.proj_post.weight": "model-00066-of-00155.safetensors",
+ "model.layers.33.mhc_attn.proj_pre.weight": "model-00066-of-00155.safetensors",
+ "model.layers.33.mhc_attn.proj_res.weight": "model-00066-of-00155.safetensors",
+ "model.layers.33.mhc_attn.rms_norm.weight": "model-00066-of-00155.safetensors",
+ "model.layers.33.mhc_ffn.alpha_post": "model-00066-of-00155.safetensors",
+ "model.layers.33.mhc_ffn.alpha_pre": "model-00066-of-00155.safetensors",
+ "model.layers.33.mhc_ffn.alpha_res": "model-00066-of-00155.safetensors",
+ "model.layers.33.mhc_ffn.bias_post": "model-00066-of-00155.safetensors",
+ "model.layers.33.mhc_ffn.bias_pre": "model-00066-of-00155.safetensors",
+ "model.layers.33.mhc_ffn.bias_res": "model-00066-of-00155.safetensors",
+ "model.layers.33.mhc_ffn.proj_post.weight": "model-00066-of-00155.safetensors",
+ "model.layers.33.mhc_ffn.proj_pre.weight": "model-00066-of-00155.safetensors",
+ "model.layers.33.mhc_ffn.proj_res.weight": "model-00066-of-00155.safetensors",
+ "model.layers.33.mhc_ffn.rms_norm.weight": "model-00066-of-00155.safetensors",
+ "model.layers.33.moe.expert_bias": "model-00064-of-00155.safetensors",
+ "model.layers.33.moe.experts.act_fn.bias": "model-00066-of-00155.safetensors",
+ "model.layers.33.moe.experts.act_fn.weight": "model-00066-of-00155.safetensors",
+ "model.layers.33.moe.experts.down_proj": "model-00065-of-00155.safetensors",
+ "model.layers.33.moe.experts.gate_up_proj": "model-00136-of-00155.safetensors",
+ "model.layers.33.moe.router.gate.weight": "model-00066-of-00155.safetensors",
+ "model.layers.33.moe.shared_experts.act_fn.bias": "model-00066-of-00155.safetensors",
+ "model.layers.33.moe.shared_experts.act_fn.weight": "model-00066-of-00155.safetensors",
+ "model.layers.33.moe.shared_experts.down_proj.weight": "model-00066-of-00155.safetensors",
+ "model.layers.33.moe.shared_experts.gate_proj.weight": "model-00066-of-00155.safetensors",
+ "model.layers.33.moe.shared_experts.up_proj.weight": "model-00066-of-00155.safetensors",
+ "model.layers.33.post_attention_layernorm.weight": "model-00066-of-00155.safetensors",
+ "model.layers.33.self_attn.kv_norm.weight": "model-00064-of-00155.safetensors",
+ "model.layers.33.self_attn.lambda_proj.weight": "model-00064-of-00155.safetensors",
+ "model.layers.33.self_attn.q_norm.weight": "model-00064-of-00155.safetensors",
+ "model.layers.33.self_attn.wkv_a.weight": "model-00064-of-00155.safetensors",
+ "model.layers.33.self_attn.wkv_b.weight": "model-00064-of-00155.safetensors",
+ "model.layers.33.self_attn.wo.weight": "model-00064-of-00155.safetensors",
+ "model.layers.33.self_attn.wq_a.weight": "model-00064-of-00155.safetensors",
+ "model.layers.33.self_attn.wq_b.weight": "model-00064-of-00155.safetensors",
+ "model.layers.33.self_attn.wq_b_gate.weight": "model-00064-of-00155.safetensors",
+ "model.layers.34.input_layernorm.weight": "model-00068-of-00155.safetensors",
+ "model.layers.34.mhc_attn.alpha_post": "model-00068-of-00155.safetensors",
+ "model.layers.34.mhc_attn.alpha_pre": "model-00068-of-00155.safetensors",
+ "model.layers.34.mhc_attn.alpha_res": "model-00068-of-00155.safetensors",
+ "model.layers.34.mhc_attn.bias_post": "model-00068-of-00155.safetensors",
+ "model.layers.34.mhc_attn.bias_pre": "model-00068-of-00155.safetensors",
+ "model.layers.34.mhc_attn.bias_res": "model-00068-of-00155.safetensors",
+ "model.layers.34.mhc_attn.proj_post.weight": "model-00068-of-00155.safetensors",
+ "model.layers.34.mhc_attn.proj_pre.weight": "model-00068-of-00155.safetensors",
+ "model.layers.34.mhc_attn.proj_res.weight": "model-00068-of-00155.safetensors",
+ "model.layers.34.mhc_attn.rms_norm.weight": "model-00068-of-00155.safetensors",
+ "model.layers.34.mhc_ffn.alpha_post": "model-00068-of-00155.safetensors",
+ "model.layers.34.mhc_ffn.alpha_pre": "model-00068-of-00155.safetensors",
+ "model.layers.34.mhc_ffn.alpha_res": "model-00068-of-00155.safetensors",
+ "model.layers.34.mhc_ffn.bias_post": "model-00068-of-00155.safetensors",
+ "model.layers.34.mhc_ffn.bias_pre": "model-00068-of-00155.safetensors",
+ "model.layers.34.mhc_ffn.bias_res": "model-00068-of-00155.safetensors",
+ "model.layers.34.mhc_ffn.proj_post.weight": "model-00068-of-00155.safetensors",
+ "model.layers.34.mhc_ffn.proj_pre.weight": "model-00068-of-00155.safetensors",
+ "model.layers.34.mhc_ffn.proj_res.weight": "model-00068-of-00155.safetensors",
+ "model.layers.34.mhc_ffn.rms_norm.weight": "model-00068-of-00155.safetensors",
+ "model.layers.34.moe.expert_bias": "model-00066-of-00155.safetensors",
+ "model.layers.34.moe.experts.act_fn.bias": "model-00068-of-00155.safetensors",
+ "model.layers.34.moe.experts.act_fn.weight": "model-00068-of-00155.safetensors",
+ "model.layers.34.moe.experts.down_proj": "model-00067-of-00155.safetensors",
+ "model.layers.34.moe.experts.gate_up_proj": "model-00137-of-00155.safetensors",
+ "model.layers.34.moe.router.gate.weight": "model-00068-of-00155.safetensors",
+ "model.layers.34.moe.shared_experts.act_fn.bias": "model-00068-of-00155.safetensors",
+ "model.layers.34.moe.shared_experts.act_fn.weight": "model-00068-of-00155.safetensors",
+ "model.layers.34.moe.shared_experts.down_proj.weight": "model-00068-of-00155.safetensors",
+ "model.layers.34.moe.shared_experts.gate_proj.weight": "model-00068-of-00155.safetensors",
+ "model.layers.34.moe.shared_experts.up_proj.weight": "model-00068-of-00155.safetensors",
+ "model.layers.34.post_attention_layernorm.weight": "model-00068-of-00155.safetensors",
+ "model.layers.34.self_attn.kv_norm.weight": "model-00066-of-00155.safetensors",
+ "model.layers.34.self_attn.lambda_proj.weight": "model-00066-of-00155.safetensors",
+ "model.layers.34.self_attn.q_norm.weight": "model-00066-of-00155.safetensors",
+ "model.layers.34.self_attn.wkv_a.weight": "model-00066-of-00155.safetensors",
+ "model.layers.34.self_attn.wkv_b.weight": "model-00066-of-00155.safetensors",
+ "model.layers.34.self_attn.wo.weight": "model-00066-of-00155.safetensors",
+ "model.layers.34.self_attn.wq_a.weight": "model-00066-of-00155.safetensors",
+ "model.layers.34.self_attn.wq_b.weight": "model-00066-of-00155.safetensors",
+ "model.layers.34.self_attn.wq_b_gate.weight": "model-00066-of-00155.safetensors",
+ "model.layers.35.input_layernorm.weight": "model-00070-of-00155.safetensors",
+ "model.layers.35.mhc_attn.alpha_post": "model-00070-of-00155.safetensors",
+ "model.layers.35.mhc_attn.alpha_pre": "model-00070-of-00155.safetensors",
+ "model.layers.35.mhc_attn.alpha_res": "model-00070-of-00155.safetensors",
+ "model.layers.35.mhc_attn.bias_post": "model-00070-of-00155.safetensors",
+ "model.layers.35.mhc_attn.bias_pre": "model-00070-of-00155.safetensors",
+ "model.layers.35.mhc_attn.bias_res": "model-00070-of-00155.safetensors",
+ "model.layers.35.mhc_attn.proj_post.weight": "model-00070-of-00155.safetensors",
+ "model.layers.35.mhc_attn.proj_pre.weight": "model-00070-of-00155.safetensors",
+ "model.layers.35.mhc_attn.proj_res.weight": "model-00070-of-00155.safetensors",
+ "model.layers.35.mhc_attn.rms_norm.weight": "model-00070-of-00155.safetensors",
+ "model.layers.35.mhc_ffn.alpha_post": "model-00070-of-00155.safetensors",
+ "model.layers.35.mhc_ffn.alpha_pre": "model-00070-of-00155.safetensors",
+ "model.layers.35.mhc_ffn.alpha_res": "model-00070-of-00155.safetensors",
+ "model.layers.35.mhc_ffn.bias_post": "model-00070-of-00155.safetensors",
+ "model.layers.35.mhc_ffn.bias_pre": "model-00070-of-00155.safetensors",
+ "model.layers.35.mhc_ffn.bias_res": "model-00070-of-00155.safetensors",
+ "model.layers.35.mhc_ffn.proj_post.weight": "model-00070-of-00155.safetensors",
+ "model.layers.35.mhc_ffn.proj_pre.weight": "model-00070-of-00155.safetensors",
+ "model.layers.35.mhc_ffn.proj_res.weight": "model-00070-of-00155.safetensors",
+ "model.layers.35.mhc_ffn.rms_norm.weight": "model-00070-of-00155.safetensors",
+ "model.layers.35.moe.expert_bias": "model-00068-of-00155.safetensors",
+ "model.layers.35.moe.experts.act_fn.bias": "model-00070-of-00155.safetensors",
+ "model.layers.35.moe.experts.act_fn.weight": "model-00070-of-00155.safetensors",
+ "model.layers.35.moe.experts.down_proj": "model-00069-of-00155.safetensors",
+ "model.layers.35.moe.experts.gate_up_proj": "model-00138-of-00155.safetensors",
+ "model.layers.35.moe.router.gate.weight": "model-00070-of-00155.safetensors",
+ "model.layers.35.moe.shared_experts.act_fn.bias": "model-00070-of-00155.safetensors",
+ "model.layers.35.moe.shared_experts.act_fn.weight": "model-00070-of-00155.safetensors",
+ "model.layers.35.moe.shared_experts.down_proj.weight": "model-00070-of-00155.safetensors",
+ "model.layers.35.moe.shared_experts.gate_proj.weight": "model-00070-of-00155.safetensors",
+ "model.layers.35.moe.shared_experts.up_proj.weight": "model-00070-of-00155.safetensors",
+ "model.layers.35.post_attention_layernorm.weight": "model-00070-of-00155.safetensors",
+ "model.layers.35.self_attn.kv_norm.weight": "model-00068-of-00155.safetensors",
+ "model.layers.35.self_attn.lambda_proj.weight": "model-00068-of-00155.safetensors",
+ "model.layers.35.self_attn.q_norm.weight": "model-00068-of-00155.safetensors",
+ "model.layers.35.self_attn.wkv_a.weight": "model-00068-of-00155.safetensors",
+ "model.layers.35.self_attn.wkv_b.weight": "model-00068-of-00155.safetensors",
+ "model.layers.35.self_attn.wo.weight": "model-00068-of-00155.safetensors",
+ "model.layers.35.self_attn.wq_a.weight": "model-00068-of-00155.safetensors",
+ "model.layers.35.self_attn.wq_b.weight": "model-00068-of-00155.safetensors",
+ "model.layers.35.self_attn.wq_b_gate.weight": "model-00068-of-00155.safetensors",
+ "model.layers.36.input_layernorm.weight": "model-00072-of-00155.safetensors",
+ "model.layers.36.mhc_attn.alpha_post": "model-00072-of-00155.safetensors",
+ "model.layers.36.mhc_attn.alpha_pre": "model-00072-of-00155.safetensors",
+ "model.layers.36.mhc_attn.alpha_res": "model-00072-of-00155.safetensors",
+ "model.layers.36.mhc_attn.bias_post": "model-00072-of-00155.safetensors",
+ "model.layers.36.mhc_attn.bias_pre": "model-00072-of-00155.safetensors",
+ "model.layers.36.mhc_attn.bias_res": "model-00072-of-00155.safetensors",
+ "model.layers.36.mhc_attn.proj_post.weight": "model-00072-of-00155.safetensors",
+ "model.layers.36.mhc_attn.proj_pre.weight": "model-00072-of-00155.safetensors",
+ "model.layers.36.mhc_attn.proj_res.weight": "model-00072-of-00155.safetensors",
+ "model.layers.36.mhc_attn.rms_norm.weight": "model-00072-of-00155.safetensors",
+ "model.layers.36.mhc_ffn.alpha_post": "model-00072-of-00155.safetensors",
+ "model.layers.36.mhc_ffn.alpha_pre": "model-00072-of-00155.safetensors",
+ "model.layers.36.mhc_ffn.alpha_res": "model-00072-of-00155.safetensors",
+ "model.layers.36.mhc_ffn.bias_post": "model-00072-of-00155.safetensors",
+ "model.layers.36.mhc_ffn.bias_pre": "model-00072-of-00155.safetensors",
+ "model.layers.36.mhc_ffn.bias_res": "model-00072-of-00155.safetensors",
+ "model.layers.36.mhc_ffn.proj_post.weight": "model-00072-of-00155.safetensors",
+ "model.layers.36.mhc_ffn.proj_pre.weight": "model-00072-of-00155.safetensors",
+ "model.layers.36.mhc_ffn.proj_res.weight": "model-00072-of-00155.safetensors",
+ "model.layers.36.mhc_ffn.rms_norm.weight": "model-00072-of-00155.safetensors",
+ "model.layers.36.moe.expert_bias": "model-00070-of-00155.safetensors",
+ "model.layers.36.moe.experts.act_fn.bias": "model-00072-of-00155.safetensors",
+ "model.layers.36.moe.experts.act_fn.weight": "model-00072-of-00155.safetensors",
+ "model.layers.36.moe.experts.down_proj": "model-00071-of-00155.safetensors",
+ "model.layers.36.moe.experts.gate_up_proj": "model-00139-of-00155.safetensors",
+ "model.layers.36.moe.router.gate.weight": "model-00072-of-00155.safetensors",
+ "model.layers.36.moe.shared_experts.act_fn.bias": "model-00072-of-00155.safetensors",
+ "model.layers.36.moe.shared_experts.act_fn.weight": "model-00072-of-00155.safetensors",
+ "model.layers.36.moe.shared_experts.down_proj.weight": "model-00072-of-00155.safetensors",
+ "model.layers.36.moe.shared_experts.gate_proj.weight": "model-00072-of-00155.safetensors",
+ "model.layers.36.moe.shared_experts.up_proj.weight": "model-00072-of-00155.safetensors",
+ "model.layers.36.post_attention_layernorm.weight": "model-00072-of-00155.safetensors",
+ "model.layers.36.self_attn.kv_norm.weight": "model-00070-of-00155.safetensors",
+ "model.layers.36.self_attn.lambda_proj.weight": "model-00070-of-00155.safetensors",
+ "model.layers.36.self_attn.q_norm.weight": "model-00070-of-00155.safetensors",
+ "model.layers.36.self_attn.wkv_a.weight": "model-00070-of-00155.safetensors",
+ "model.layers.36.self_attn.wkv_b.weight": "model-00070-of-00155.safetensors",
+ "model.layers.36.self_attn.wo.weight": "model-00070-of-00155.safetensors",
+ "model.layers.36.self_attn.wq_a.weight": "model-00070-of-00155.safetensors",
+ "model.layers.36.self_attn.wq_b.weight": "model-00070-of-00155.safetensors",
+ "model.layers.36.self_attn.wq_b_gate.weight": "model-00070-of-00155.safetensors",
+ "model.layers.37.input_layernorm.weight": "model-00074-of-00155.safetensors",
+ "model.layers.37.mhc_attn.alpha_post": "model-00074-of-00155.safetensors",
+ "model.layers.37.mhc_attn.alpha_pre": "model-00074-of-00155.safetensors",
+ "model.layers.37.mhc_attn.alpha_res": "model-00074-of-00155.safetensors",
+ "model.layers.37.mhc_attn.bias_post": "model-00074-of-00155.safetensors",
+ "model.layers.37.mhc_attn.bias_pre": "model-00074-of-00155.safetensors",
+ "model.layers.37.mhc_attn.bias_res": "model-00074-of-00155.safetensors",
+ "model.layers.37.mhc_attn.proj_post.weight": "model-00074-of-00155.safetensors",
+ "model.layers.37.mhc_attn.proj_pre.weight": "model-00074-of-00155.safetensors",
+ "model.layers.37.mhc_attn.proj_res.weight": "model-00074-of-00155.safetensors",
+ "model.layers.37.mhc_attn.rms_norm.weight": "model-00074-of-00155.safetensors",
+ "model.layers.37.mhc_ffn.alpha_post": "model-00074-of-00155.safetensors",
+ "model.layers.37.mhc_ffn.alpha_pre": "model-00074-of-00155.safetensors",
+ "model.layers.37.mhc_ffn.alpha_res": "model-00074-of-00155.safetensors",
+ "model.layers.37.mhc_ffn.bias_post": "model-00074-of-00155.safetensors",
+ "model.layers.37.mhc_ffn.bias_pre": "model-00074-of-00155.safetensors",
+ "model.layers.37.mhc_ffn.bias_res": "model-00074-of-00155.safetensors",
+ "model.layers.37.mhc_ffn.proj_post.weight": "model-00074-of-00155.safetensors",
+ "model.layers.37.mhc_ffn.proj_pre.weight": "model-00074-of-00155.safetensors",
+ "model.layers.37.mhc_ffn.proj_res.weight": "model-00074-of-00155.safetensors",
+ "model.layers.37.mhc_ffn.rms_norm.weight": "model-00074-of-00155.safetensors",
+ "model.layers.37.moe.expert_bias": "model-00072-of-00155.safetensors",
+ "model.layers.37.moe.experts.act_fn.bias": "model-00074-of-00155.safetensors",
+ "model.layers.37.moe.experts.act_fn.weight": "model-00074-of-00155.safetensors",
+ "model.layers.37.moe.experts.down_proj": "model-00073-of-00155.safetensors",
+ "model.layers.37.moe.experts.gate_up_proj": "model-00140-of-00155.safetensors",
+ "model.layers.37.moe.router.gate.weight": "model-00074-of-00155.safetensors",
+ "model.layers.37.moe.shared_experts.act_fn.bias": "model-00074-of-00155.safetensors",
+ "model.layers.37.moe.shared_experts.act_fn.weight": "model-00074-of-00155.safetensors",
+ "model.layers.37.moe.shared_experts.down_proj.weight": "model-00074-of-00155.safetensors",
+ "model.layers.37.moe.shared_experts.gate_proj.weight": "model-00074-of-00155.safetensors",
+ "model.layers.37.moe.shared_experts.up_proj.weight": "model-00074-of-00155.safetensors",
+ "model.layers.37.post_attention_layernorm.weight": "model-00074-of-00155.safetensors",
+ "model.layers.37.self_attn.kv_norm.weight": "model-00072-of-00155.safetensors",
+ "model.layers.37.self_attn.lambda_proj.weight": "model-00072-of-00155.safetensors",
+ "model.layers.37.self_attn.q_norm.weight": "model-00072-of-00155.safetensors",
+ "model.layers.37.self_attn.wkv_a.weight": "model-00072-of-00155.safetensors",
+ "model.layers.37.self_attn.wkv_b.weight": "model-00072-of-00155.safetensors",
+ "model.layers.37.self_attn.wo.weight": "model-00072-of-00155.safetensors",
+ "model.layers.37.self_attn.wq_a.weight": "model-00072-of-00155.safetensors",
+ "model.layers.37.self_attn.wq_b.weight": "model-00072-of-00155.safetensors",
+ "model.layers.37.self_attn.wq_b_gate.weight": "model-00072-of-00155.safetensors",
+ "model.layers.38.input_layernorm.weight": "model-00076-of-00155.safetensors",
+ "model.layers.38.mhc_attn.alpha_post": "model-00076-of-00155.safetensors",
+ "model.layers.38.mhc_attn.alpha_pre": "model-00076-of-00155.safetensors",
+ "model.layers.38.mhc_attn.alpha_res": "model-00076-of-00155.safetensors",
+ "model.layers.38.mhc_attn.bias_post": "model-00076-of-00155.safetensors",
+ "model.layers.38.mhc_attn.bias_pre": "model-00076-of-00155.safetensors",
+ "model.layers.38.mhc_attn.bias_res": "model-00076-of-00155.safetensors",
+ "model.layers.38.mhc_attn.proj_post.weight": "model-00076-of-00155.safetensors",
+ "model.layers.38.mhc_attn.proj_pre.weight": "model-00076-of-00155.safetensors",
+ "model.layers.38.mhc_attn.proj_res.weight": "model-00076-of-00155.safetensors",
+ "model.layers.38.mhc_attn.rms_norm.weight": "model-00076-of-00155.safetensors",
+ "model.layers.38.mhc_ffn.alpha_post": "model-00076-of-00155.safetensors",
+ "model.layers.38.mhc_ffn.alpha_pre": "model-00076-of-00155.safetensors",
+ "model.layers.38.mhc_ffn.alpha_res": "model-00076-of-00155.safetensors",
+ "model.layers.38.mhc_ffn.bias_post": "model-00076-of-00155.safetensors",
+ "model.layers.38.mhc_ffn.bias_pre": "model-00076-of-00155.safetensors",
+ "model.layers.38.mhc_ffn.bias_res": "model-00076-of-00155.safetensors",
+ "model.layers.38.mhc_ffn.proj_post.weight": "model-00076-of-00155.safetensors",
+ "model.layers.38.mhc_ffn.proj_pre.weight": "model-00076-of-00155.safetensors",
+ "model.layers.38.mhc_ffn.proj_res.weight": "model-00076-of-00155.safetensors",
+ "model.layers.38.mhc_ffn.rms_norm.weight": "model-00076-of-00155.safetensors",
+ "model.layers.38.moe.expert_bias": "model-00074-of-00155.safetensors",
+ "model.layers.38.moe.experts.act_fn.bias": "model-00076-of-00155.safetensors",
+ "model.layers.38.moe.experts.act_fn.weight": "model-00076-of-00155.safetensors",
+ "model.layers.38.moe.experts.down_proj": "model-00075-of-00155.safetensors",
+ "model.layers.38.moe.experts.gate_up_proj": "model-00141-of-00155.safetensors",
+ "model.layers.38.moe.router.gate.weight": "model-00076-of-00155.safetensors",
+ "model.layers.38.moe.shared_experts.act_fn.bias": "model-00076-of-00155.safetensors",
+ "model.layers.38.moe.shared_experts.act_fn.weight": "model-00076-of-00155.safetensors",
+ "model.layers.38.moe.shared_experts.down_proj.weight": "model-00076-of-00155.safetensors",
+ "model.layers.38.moe.shared_experts.gate_proj.weight": "model-00076-of-00155.safetensors",
+ "model.layers.38.moe.shared_experts.up_proj.weight": "model-00076-of-00155.safetensors",
+ "model.layers.38.post_attention_layernorm.weight": "model-00076-of-00155.safetensors",
+ "model.layers.38.self_attn.kv_norm.weight": "model-00074-of-00155.safetensors",
+ "model.layers.38.self_attn.lambda_proj.weight": "model-00074-of-00155.safetensors",
+ "model.layers.38.self_attn.q_norm.weight": "model-00074-of-00155.safetensors",
+ "model.layers.38.self_attn.wkv_a.weight": "model-00074-of-00155.safetensors",
+ "model.layers.38.self_attn.wkv_b.weight": "model-00074-of-00155.safetensors",
+ "model.layers.38.self_attn.wo.weight": "model-00074-of-00155.safetensors",
+ "model.layers.38.self_attn.wq_a.weight": "model-00074-of-00155.safetensors",
+ "model.layers.38.self_attn.wq_b.weight": "model-00074-of-00155.safetensors",
+ "model.layers.38.self_attn.wq_b_gate.weight": "model-00074-of-00155.safetensors",
+ "model.layers.39.input_layernorm.weight": "model-00078-of-00155.safetensors",
+ "model.layers.39.mhc_attn.alpha_post": "model-00078-of-00155.safetensors",
+ "model.layers.39.mhc_attn.alpha_pre": "model-00078-of-00155.safetensors",
+ "model.layers.39.mhc_attn.alpha_res": "model-00078-of-00155.safetensors",
+ "model.layers.39.mhc_attn.bias_post": "model-00078-of-00155.safetensors",
+ "model.layers.39.mhc_attn.bias_pre": "model-00078-of-00155.safetensors",
+ "model.layers.39.mhc_attn.bias_res": "model-00078-of-00155.safetensors",
+ "model.layers.39.mhc_attn.proj_post.weight": "model-00078-of-00155.safetensors",
+ "model.layers.39.mhc_attn.proj_pre.weight": "model-00078-of-00155.safetensors",
+ "model.layers.39.mhc_attn.proj_res.weight": "model-00078-of-00155.safetensors",
+ "model.layers.39.mhc_attn.rms_norm.weight": "model-00078-of-00155.safetensors",
+ "model.layers.39.mhc_ffn.alpha_post": "model-00078-of-00155.safetensors",
+ "model.layers.39.mhc_ffn.alpha_pre": "model-00078-of-00155.safetensors",
+ "model.layers.39.mhc_ffn.alpha_res": "model-00078-of-00155.safetensors",
+ "model.layers.39.mhc_ffn.bias_post": "model-00078-of-00155.safetensors",
+ "model.layers.39.mhc_ffn.bias_pre": "model-00078-of-00155.safetensors",
+ "model.layers.39.mhc_ffn.bias_res": "model-00078-of-00155.safetensors",
+ "model.layers.39.mhc_ffn.proj_post.weight": "model-00078-of-00155.safetensors",
+ "model.layers.39.mhc_ffn.proj_pre.weight": "model-00078-of-00155.safetensors",
+ "model.layers.39.mhc_ffn.proj_res.weight": "model-00078-of-00155.safetensors",
+ "model.layers.39.mhc_ffn.rms_norm.weight": "model-00078-of-00155.safetensors",
+ "model.layers.39.moe.expert_bias": "model-00076-of-00155.safetensors",
+ "model.layers.39.moe.experts.act_fn.bias": "model-00078-of-00155.safetensors",
+ "model.layers.39.moe.experts.act_fn.weight": "model-00078-of-00155.safetensors",
+ "model.layers.39.moe.experts.down_proj": "model-00077-of-00155.safetensors",
+ "model.layers.39.moe.experts.gate_up_proj": "model-00142-of-00155.safetensors",
+ "model.layers.39.moe.router.gate.weight": "model-00078-of-00155.safetensors",
+ "model.layers.39.moe.shared_experts.act_fn.bias": "model-00078-of-00155.safetensors",
+ "model.layers.39.moe.shared_experts.act_fn.weight": "model-00078-of-00155.safetensors",
+ "model.layers.39.moe.shared_experts.down_proj.weight": "model-00078-of-00155.safetensors",
+ "model.layers.39.moe.shared_experts.gate_proj.weight": "model-00078-of-00155.safetensors",
+ "model.layers.39.moe.shared_experts.up_proj.weight": "model-00078-of-00155.safetensors",
+ "model.layers.39.post_attention_layernorm.weight": "model-00078-of-00155.safetensors",
+ "model.layers.39.self_attn.kv_norm.weight": "model-00076-of-00155.safetensors",
+ "model.layers.39.self_attn.lambda_proj.weight": "model-00076-of-00155.safetensors",
+ "model.layers.39.self_attn.q_norm.weight": "model-00076-of-00155.safetensors",
+ "model.layers.39.self_attn.wkv_a.weight": "model-00076-of-00155.safetensors",
+ "model.layers.39.self_attn.wkv_b.weight": "model-00076-of-00155.safetensors",
+ "model.layers.39.self_attn.wo.weight": "model-00076-of-00155.safetensors",
+ "model.layers.39.self_attn.wq_a.weight": "model-00076-of-00155.safetensors",
+ "model.layers.39.self_attn.wq_b.weight": "model-00076-of-00155.safetensors",
+ "model.layers.39.self_attn.wq_b_gate.weight": "model-00076-of-00155.safetensors",
+ "model.layers.4.input_layernorm.weight": "model-00008-of-00155.safetensors",
+ "model.layers.4.mhc_attn.alpha_post": "model-00008-of-00155.safetensors",
+ "model.layers.4.mhc_attn.alpha_pre": "model-00008-of-00155.safetensors",
+ "model.layers.4.mhc_attn.alpha_res": "model-00008-of-00155.safetensors",
+ "model.layers.4.mhc_attn.bias_post": "model-00008-of-00155.safetensors",
+ "model.layers.4.mhc_attn.bias_pre": "model-00008-of-00155.safetensors",
+ "model.layers.4.mhc_attn.bias_res": "model-00008-of-00155.safetensors",
+ "model.layers.4.mhc_attn.proj_post.weight": "model-00008-of-00155.safetensors",
+ "model.layers.4.mhc_attn.proj_pre.weight": "model-00008-of-00155.safetensors",
+ "model.layers.4.mhc_attn.proj_res.weight": "model-00008-of-00155.safetensors",
+ "model.layers.4.mhc_attn.rms_norm.weight": "model-00008-of-00155.safetensors",
+ "model.layers.4.mhc_ffn.alpha_post": "model-00008-of-00155.safetensors",
+ "model.layers.4.mhc_ffn.alpha_pre": "model-00008-of-00155.safetensors",
+ "model.layers.4.mhc_ffn.alpha_res": "model-00008-of-00155.safetensors",
+ "model.layers.4.mhc_ffn.bias_post": "model-00008-of-00155.safetensors",
+ "model.layers.4.mhc_ffn.bias_pre": "model-00008-of-00155.safetensors",
+ "model.layers.4.mhc_ffn.bias_res": "model-00008-of-00155.safetensors",
+ "model.layers.4.mhc_ffn.proj_post.weight": "model-00008-of-00155.safetensors",
+ "model.layers.4.mhc_ffn.proj_pre.weight": "model-00008-of-00155.safetensors",
+ "model.layers.4.mhc_ffn.proj_res.weight": "model-00008-of-00155.safetensors",
+ "model.layers.4.mhc_ffn.rms_norm.weight": "model-00008-of-00155.safetensors",
+ "model.layers.4.moe.expert_bias": "model-00006-of-00155.safetensors",
+ "model.layers.4.moe.experts.act_fn.bias": "model-00008-of-00155.safetensors",
+ "model.layers.4.moe.experts.act_fn.weight": "model-00008-of-00155.safetensors",
+ "model.layers.4.moe.experts.down_proj": "model-00007-of-00155.safetensors",
+ "model.layers.4.moe.experts.gate_up_proj": "model-00107-of-00155.safetensors",
+ "model.layers.4.moe.router.gate.weight": "model-00008-of-00155.safetensors",
+ "model.layers.4.moe.shared_experts.act_fn.bias": "model-00008-of-00155.safetensors",
+ "model.layers.4.moe.shared_experts.act_fn.weight": "model-00008-of-00155.safetensors",
+ "model.layers.4.moe.shared_experts.down_proj.weight": "model-00008-of-00155.safetensors",
+ "model.layers.4.moe.shared_experts.gate_proj.weight": "model-00008-of-00155.safetensors",
+ "model.layers.4.moe.shared_experts.up_proj.weight": "model-00008-of-00155.safetensors",
+ "model.layers.4.post_attention_layernorm.weight": "model-00008-of-00155.safetensors",
+ "model.layers.4.self_attn.kv_norm.weight": "model-00006-of-00155.safetensors",
+ "model.layers.4.self_attn.lambda_proj.weight": "model-00006-of-00155.safetensors",
+ "model.layers.4.self_attn.q_norm.weight": "model-00006-of-00155.safetensors",
+ "model.layers.4.self_attn.wkv_a.weight": "model-00006-of-00155.safetensors",
+ "model.layers.4.self_attn.wkv_b.weight": "model-00006-of-00155.safetensors",
+ "model.layers.4.self_attn.wo.weight": "model-00006-of-00155.safetensors",
+ "model.layers.4.self_attn.wq_a.weight": "model-00006-of-00155.safetensors",
+ "model.layers.4.self_attn.wq_b.weight": "model-00006-of-00155.safetensors",
+ "model.layers.4.self_attn.wq_b_gate.weight": "model-00006-of-00155.safetensors",
+ "model.layers.40.input_layernorm.weight": "model-00080-of-00155.safetensors",
+ "model.layers.40.mhc_attn.alpha_post": "model-00080-of-00155.safetensors",
+ "model.layers.40.mhc_attn.alpha_pre": "model-00080-of-00155.safetensors",
+ "model.layers.40.mhc_attn.alpha_res": "model-00080-of-00155.safetensors",
+ "model.layers.40.mhc_attn.bias_post": "model-00080-of-00155.safetensors",
+ "model.layers.40.mhc_attn.bias_pre": "model-00080-of-00155.safetensors",
+ "model.layers.40.mhc_attn.bias_res": "model-00080-of-00155.safetensors",
+ "model.layers.40.mhc_attn.proj_post.weight": "model-00080-of-00155.safetensors",
+ "model.layers.40.mhc_attn.proj_pre.weight": "model-00080-of-00155.safetensors",
+ "model.layers.40.mhc_attn.proj_res.weight": "model-00080-of-00155.safetensors",
+ "model.layers.40.mhc_attn.rms_norm.weight": "model-00080-of-00155.safetensors",
+ "model.layers.40.mhc_ffn.alpha_post": "model-00080-of-00155.safetensors",
+ "model.layers.40.mhc_ffn.alpha_pre": "model-00080-of-00155.safetensors",
+ "model.layers.40.mhc_ffn.alpha_res": "model-00080-of-00155.safetensors",
+ "model.layers.40.mhc_ffn.bias_post": "model-00080-of-00155.safetensors",
+ "model.layers.40.mhc_ffn.bias_pre": "model-00080-of-00155.safetensors",
+ "model.layers.40.mhc_ffn.bias_res": "model-00080-of-00155.safetensors",
+ "model.layers.40.mhc_ffn.proj_post.weight": "model-00080-of-00155.safetensors",
+ "model.layers.40.mhc_ffn.proj_pre.weight": "model-00080-of-00155.safetensors",
+ "model.layers.40.mhc_ffn.proj_res.weight": "model-00080-of-00155.safetensors",
+ "model.layers.40.mhc_ffn.rms_norm.weight": "model-00080-of-00155.safetensors",
+ "model.layers.40.moe.expert_bias": "model-00078-of-00155.safetensors",
+ "model.layers.40.moe.experts.act_fn.bias": "model-00080-of-00155.safetensors",
+ "model.layers.40.moe.experts.act_fn.weight": "model-00080-of-00155.safetensors",
+ "model.layers.40.moe.experts.down_proj": "model-00079-of-00155.safetensors",
+ "model.layers.40.moe.experts.gate_up_proj": "model-00143-of-00155.safetensors",
+ "model.layers.40.moe.router.gate.weight": "model-00080-of-00155.safetensors",
+ "model.layers.40.moe.shared_experts.act_fn.bias": "model-00080-of-00155.safetensors",
+ "model.layers.40.moe.shared_experts.act_fn.weight": "model-00080-of-00155.safetensors",
+ "model.layers.40.moe.shared_experts.down_proj.weight": "model-00080-of-00155.safetensors",
+ "model.layers.40.moe.shared_experts.gate_proj.weight": "model-00080-of-00155.safetensors",
+ "model.layers.40.moe.shared_experts.up_proj.weight": "model-00080-of-00155.safetensors",
+ "model.layers.40.post_attention_layernorm.weight": "model-00080-of-00155.safetensors",
+ "model.layers.40.self_attn.kv_norm.weight": "model-00078-of-00155.safetensors",
+ "model.layers.40.self_attn.lambda_proj.weight": "model-00078-of-00155.safetensors",
+ "model.layers.40.self_attn.q_norm.weight": "model-00078-of-00155.safetensors",
+ "model.layers.40.self_attn.wkv_a.weight": "model-00078-of-00155.safetensors",
+ "model.layers.40.self_attn.wkv_b.weight": "model-00078-of-00155.safetensors",
+ "model.layers.40.self_attn.wo.weight": "model-00078-of-00155.safetensors",
+ "model.layers.40.self_attn.wq_a.weight": "model-00078-of-00155.safetensors",
+ "model.layers.40.self_attn.wq_b.weight": "model-00078-of-00155.safetensors",
+ "model.layers.40.self_attn.wq_b_gate.weight": "model-00078-of-00155.safetensors",
+ "model.layers.41.input_layernorm.weight": "model-00082-of-00155.safetensors",
+ "model.layers.41.mhc_attn.alpha_post": "model-00082-of-00155.safetensors",
+ "model.layers.41.mhc_attn.alpha_pre": "model-00082-of-00155.safetensors",
+ "model.layers.41.mhc_attn.alpha_res": "model-00082-of-00155.safetensors",
+ "model.layers.41.mhc_attn.bias_post": "model-00082-of-00155.safetensors",
+ "model.layers.41.mhc_attn.bias_pre": "model-00082-of-00155.safetensors",
+ "model.layers.41.mhc_attn.bias_res": "model-00082-of-00155.safetensors",
+ "model.layers.41.mhc_attn.proj_post.weight": "model-00082-of-00155.safetensors",
+ "model.layers.41.mhc_attn.proj_pre.weight": "model-00082-of-00155.safetensors",
+ "model.layers.41.mhc_attn.proj_res.weight": "model-00082-of-00155.safetensors",
+ "model.layers.41.mhc_attn.rms_norm.weight": "model-00082-of-00155.safetensors",
+ "model.layers.41.mhc_ffn.alpha_post": "model-00082-of-00155.safetensors",
+ "model.layers.41.mhc_ffn.alpha_pre": "model-00082-of-00155.safetensors",
+ "model.layers.41.mhc_ffn.alpha_res": "model-00082-of-00155.safetensors",
+ "model.layers.41.mhc_ffn.bias_post": "model-00082-of-00155.safetensors",
+ "model.layers.41.mhc_ffn.bias_pre": "model-00082-of-00155.safetensors",
+ "model.layers.41.mhc_ffn.bias_res": "model-00082-of-00155.safetensors",
+ "model.layers.41.mhc_ffn.proj_post.weight": "model-00082-of-00155.safetensors",
+ "model.layers.41.mhc_ffn.proj_pre.weight": "model-00082-of-00155.safetensors",
+ "model.layers.41.mhc_ffn.proj_res.weight": "model-00082-of-00155.safetensors",
+ "model.layers.41.mhc_ffn.rms_norm.weight": "model-00082-of-00155.safetensors",
+ "model.layers.41.moe.expert_bias": "model-00080-of-00155.safetensors",
+ "model.layers.41.moe.experts.act_fn.bias": "model-00082-of-00155.safetensors",
+ "model.layers.41.moe.experts.act_fn.weight": "model-00082-of-00155.safetensors",
+ "model.layers.41.moe.experts.down_proj": "model-00081-of-00155.safetensors",
+ "model.layers.41.moe.experts.gate_up_proj": "model-00144-of-00155.safetensors",
+ "model.layers.41.moe.router.gate.weight": "model-00082-of-00155.safetensors",
+ "model.layers.41.moe.shared_experts.act_fn.bias": "model-00082-of-00155.safetensors",
+ "model.layers.41.moe.shared_experts.act_fn.weight": "model-00082-of-00155.safetensors",
+ "model.layers.41.moe.shared_experts.down_proj.weight": "model-00082-of-00155.safetensors",
+ "model.layers.41.moe.shared_experts.gate_proj.weight": "model-00082-of-00155.safetensors",
+ "model.layers.41.moe.shared_experts.up_proj.weight": "model-00082-of-00155.safetensors",
+ "model.layers.41.post_attention_layernorm.weight": "model-00082-of-00155.safetensors",
+ "model.layers.41.self_attn.kv_norm.weight": "model-00080-of-00155.safetensors",
+ "model.layers.41.self_attn.lambda_proj.weight": "model-00080-of-00155.safetensors",
+ "model.layers.41.self_attn.q_norm.weight": "model-00080-of-00155.safetensors",
+ "model.layers.41.self_attn.wkv_a.weight": "model-00080-of-00155.safetensors",
+ "model.layers.41.self_attn.wkv_b.weight": "model-00080-of-00155.safetensors",
+ "model.layers.41.self_attn.wo.weight": "model-00080-of-00155.safetensors",
+ "model.layers.41.self_attn.wq_a.weight": "model-00080-of-00155.safetensors",
+ "model.layers.41.self_attn.wq_b.weight": "model-00080-of-00155.safetensors",
+ "model.layers.41.self_attn.wq_b_gate.weight": "model-00080-of-00155.safetensors",
+ "model.layers.42.input_layernorm.weight": "model-00084-of-00155.safetensors",
+ "model.layers.42.mhc_attn.alpha_post": "model-00084-of-00155.safetensors",
+ "model.layers.42.mhc_attn.alpha_pre": "model-00084-of-00155.safetensors",
+ "model.layers.42.mhc_attn.alpha_res": "model-00084-of-00155.safetensors",
+ "model.layers.42.mhc_attn.bias_post": "model-00084-of-00155.safetensors",
+ "model.layers.42.mhc_attn.bias_pre": "model-00084-of-00155.safetensors",
+ "model.layers.42.mhc_attn.bias_res": "model-00084-of-00155.safetensors",
+ "model.layers.42.mhc_attn.proj_post.weight": "model-00084-of-00155.safetensors",
+ "model.layers.42.mhc_attn.proj_pre.weight": "model-00084-of-00155.safetensors",
+ "model.layers.42.mhc_attn.proj_res.weight": "model-00084-of-00155.safetensors",
+ "model.layers.42.mhc_attn.rms_norm.weight": "model-00084-of-00155.safetensors",
+ "model.layers.42.mhc_ffn.alpha_post": "model-00084-of-00155.safetensors",
+ "model.layers.42.mhc_ffn.alpha_pre": "model-00084-of-00155.safetensors",
+ "model.layers.42.mhc_ffn.alpha_res": "model-00084-of-00155.safetensors",
+ "model.layers.42.mhc_ffn.bias_post": "model-00084-of-00155.safetensors",
+ "model.layers.42.mhc_ffn.bias_pre": "model-00084-of-00155.safetensors",
+ "model.layers.42.mhc_ffn.bias_res": "model-00084-of-00155.safetensors",
+ "model.layers.42.mhc_ffn.proj_post.weight": "model-00084-of-00155.safetensors",
+ "model.layers.42.mhc_ffn.proj_pre.weight": "model-00084-of-00155.safetensors",
+ "model.layers.42.mhc_ffn.proj_res.weight": "model-00084-of-00155.safetensors",
+ "model.layers.42.mhc_ffn.rms_norm.weight": "model-00084-of-00155.safetensors",
+ "model.layers.42.moe.expert_bias": "model-00082-of-00155.safetensors",
+ "model.layers.42.moe.experts.act_fn.bias": "model-00084-of-00155.safetensors",
+ "model.layers.42.moe.experts.act_fn.weight": "model-00084-of-00155.safetensors",
+ "model.layers.42.moe.experts.down_proj": "model-00083-of-00155.safetensors",
+ "model.layers.42.moe.experts.gate_up_proj": "model-00145-of-00155.safetensors",
+ "model.layers.42.moe.router.gate.weight": "model-00084-of-00155.safetensors",
+ "model.layers.42.moe.shared_experts.act_fn.bias": "model-00084-of-00155.safetensors",
+ "model.layers.42.moe.shared_experts.act_fn.weight": "model-00084-of-00155.safetensors",
+ "model.layers.42.moe.shared_experts.down_proj.weight": "model-00084-of-00155.safetensors",
+ "model.layers.42.moe.shared_experts.gate_proj.weight": "model-00084-of-00155.safetensors",
+ "model.layers.42.moe.shared_experts.up_proj.weight": "model-00084-of-00155.safetensors",
+ "model.layers.42.post_attention_layernorm.weight": "model-00084-of-00155.safetensors",
+ "model.layers.42.self_attn.kv_norm.weight": "model-00082-of-00155.safetensors",
+ "model.layers.42.self_attn.lambda_proj.weight": "model-00082-of-00155.safetensors",
+ "model.layers.42.self_attn.q_norm.weight": "model-00082-of-00155.safetensors",
+ "model.layers.42.self_attn.wkv_a.weight": "model-00082-of-00155.safetensors",
+ "model.layers.42.self_attn.wkv_b.weight": "model-00082-of-00155.safetensors",
+ "model.layers.42.self_attn.wo.weight": "model-00082-of-00155.safetensors",
+ "model.layers.42.self_attn.wq_a.weight": "model-00082-of-00155.safetensors",
+ "model.layers.42.self_attn.wq_b.weight": "model-00082-of-00155.safetensors",
+ "model.layers.42.self_attn.wq_b_gate.weight": "model-00082-of-00155.safetensors",
+ "model.layers.43.input_layernorm.weight": "model-00086-of-00155.safetensors",
+ "model.layers.43.mhc_attn.alpha_post": "model-00086-of-00155.safetensors",
+ "model.layers.43.mhc_attn.alpha_pre": "model-00086-of-00155.safetensors",
+ "model.layers.43.mhc_attn.alpha_res": "model-00086-of-00155.safetensors",
+ "model.layers.43.mhc_attn.bias_post": "model-00086-of-00155.safetensors",
+ "model.layers.43.mhc_attn.bias_pre": "model-00086-of-00155.safetensors",
+ "model.layers.43.mhc_attn.bias_res": "model-00086-of-00155.safetensors",
+ "model.layers.43.mhc_attn.proj_post.weight": "model-00086-of-00155.safetensors",
+ "model.layers.43.mhc_attn.proj_pre.weight": "model-00086-of-00155.safetensors",
+ "model.layers.43.mhc_attn.proj_res.weight": "model-00086-of-00155.safetensors",
+ "model.layers.43.mhc_attn.rms_norm.weight": "model-00086-of-00155.safetensors",
+ "model.layers.43.mhc_ffn.alpha_post": "model-00086-of-00155.safetensors",
+ "model.layers.43.mhc_ffn.alpha_pre": "model-00086-of-00155.safetensors",
+ "model.layers.43.mhc_ffn.alpha_res": "model-00086-of-00155.safetensors",
+ "model.layers.43.mhc_ffn.bias_post": "model-00086-of-00155.safetensors",
+ "model.layers.43.mhc_ffn.bias_pre": "model-00086-of-00155.safetensors",
+ "model.layers.43.mhc_ffn.bias_res": "model-00086-of-00155.safetensors",
+ "model.layers.43.mhc_ffn.proj_post.weight": "model-00086-of-00155.safetensors",
+ "model.layers.43.mhc_ffn.proj_pre.weight": "model-00086-of-00155.safetensors",
+ "model.layers.43.mhc_ffn.proj_res.weight": "model-00086-of-00155.safetensors",
+ "model.layers.43.mhc_ffn.rms_norm.weight": "model-00086-of-00155.safetensors",
+ "model.layers.43.moe.expert_bias": "model-00084-of-00155.safetensors",
+ "model.layers.43.moe.experts.act_fn.bias": "model-00086-of-00155.safetensors",
+ "model.layers.43.moe.experts.act_fn.weight": "model-00086-of-00155.safetensors",
+ "model.layers.43.moe.experts.down_proj": "model-00085-of-00155.safetensors",
+ "model.layers.43.moe.experts.gate_up_proj": "model-00146-of-00155.safetensors",
+ "model.layers.43.moe.router.gate.weight": "model-00086-of-00155.safetensors",
+ "model.layers.43.moe.shared_experts.act_fn.bias": "model-00086-of-00155.safetensors",
+ "model.layers.43.moe.shared_experts.act_fn.weight": "model-00086-of-00155.safetensors",
+ "model.layers.43.moe.shared_experts.down_proj.weight": "model-00086-of-00155.safetensors",
+ "model.layers.43.moe.shared_experts.gate_proj.weight": "model-00086-of-00155.safetensors",
+ "model.layers.43.moe.shared_experts.up_proj.weight": "model-00086-of-00155.safetensors",
+ "model.layers.43.post_attention_layernorm.weight": "model-00086-of-00155.safetensors",
+ "model.layers.43.self_attn.kv_norm.weight": "model-00084-of-00155.safetensors",
+ "model.layers.43.self_attn.lambda_proj.weight": "model-00084-of-00155.safetensors",
+ "model.layers.43.self_attn.q_norm.weight": "model-00084-of-00155.safetensors",
+ "model.layers.43.self_attn.wkv_a.weight": "model-00084-of-00155.safetensors",
+ "model.layers.43.self_attn.wkv_b.weight": "model-00084-of-00155.safetensors",
+ "model.layers.43.self_attn.wo.weight": "model-00084-of-00155.safetensors",
+ "model.layers.43.self_attn.wq_a.weight": "model-00084-of-00155.safetensors",
+ "model.layers.43.self_attn.wq_b.weight": "model-00084-of-00155.safetensors",
+ "model.layers.43.self_attn.wq_b_gate.weight": "model-00084-of-00155.safetensors",
+ "model.layers.44.input_layernorm.weight": "model-00088-of-00155.safetensors",
+ "model.layers.44.mhc_attn.alpha_post": "model-00088-of-00155.safetensors",
+ "model.layers.44.mhc_attn.alpha_pre": "model-00088-of-00155.safetensors",
+ "model.layers.44.mhc_attn.alpha_res": "model-00088-of-00155.safetensors",
+ "model.layers.44.mhc_attn.bias_post": "model-00088-of-00155.safetensors",
+ "model.layers.44.mhc_attn.bias_pre": "model-00088-of-00155.safetensors",
+ "model.layers.44.mhc_attn.bias_res": "model-00088-of-00155.safetensors",
+ "model.layers.44.mhc_attn.proj_post.weight": "model-00088-of-00155.safetensors",
+ "model.layers.44.mhc_attn.proj_pre.weight": "model-00088-of-00155.safetensors",
+ "model.layers.44.mhc_attn.proj_res.weight": "model-00088-of-00155.safetensors",
+ "model.layers.44.mhc_attn.rms_norm.weight": "model-00088-of-00155.safetensors",
+ "model.layers.44.mhc_ffn.alpha_post": "model-00088-of-00155.safetensors",
+ "model.layers.44.mhc_ffn.alpha_pre": "model-00088-of-00155.safetensors",
+ "model.layers.44.mhc_ffn.alpha_res": "model-00088-of-00155.safetensors",
+ "model.layers.44.mhc_ffn.bias_post": "model-00088-of-00155.safetensors",
+ "model.layers.44.mhc_ffn.bias_pre": "model-00088-of-00155.safetensors",
+ "model.layers.44.mhc_ffn.bias_res": "model-00088-of-00155.safetensors",
+ "model.layers.44.mhc_ffn.proj_post.weight": "model-00088-of-00155.safetensors",
+ "model.layers.44.mhc_ffn.proj_pre.weight": "model-00088-of-00155.safetensors",
+ "model.layers.44.mhc_ffn.proj_res.weight": "model-00088-of-00155.safetensors",
+ "model.layers.44.mhc_ffn.rms_norm.weight": "model-00088-of-00155.safetensors",
+ "model.layers.44.moe.expert_bias": "model-00086-of-00155.safetensors",
+ "model.layers.44.moe.experts.act_fn.bias": "model-00088-of-00155.safetensors",
+ "model.layers.44.moe.experts.act_fn.weight": "model-00088-of-00155.safetensors",
+ "model.layers.44.moe.experts.down_proj": "model-00087-of-00155.safetensors",
+ "model.layers.44.moe.experts.gate_up_proj": "model-00147-of-00155.safetensors",
+ "model.layers.44.moe.router.gate.weight": "model-00088-of-00155.safetensors",
+ "model.layers.44.moe.shared_experts.act_fn.bias": "model-00088-of-00155.safetensors",
+ "model.layers.44.moe.shared_experts.act_fn.weight": "model-00088-of-00155.safetensors",
+ "model.layers.44.moe.shared_experts.down_proj.weight": "model-00088-of-00155.safetensors",
+ "model.layers.44.moe.shared_experts.gate_proj.weight": "model-00088-of-00155.safetensors",
+ "model.layers.44.moe.shared_experts.up_proj.weight": "model-00088-of-00155.safetensors",
+ "model.layers.44.post_attention_layernorm.weight": "model-00088-of-00155.safetensors",
+ "model.layers.44.self_attn.kv_norm.weight": "model-00086-of-00155.safetensors",
+ "model.layers.44.self_attn.lambda_proj.weight": "model-00086-of-00155.safetensors",
+ "model.layers.44.self_attn.q_norm.weight": "model-00086-of-00155.safetensors",
+ "model.layers.44.self_attn.wkv_a.weight": "model-00086-of-00155.safetensors",
+ "model.layers.44.self_attn.wkv_b.weight": "model-00086-of-00155.safetensors",
+ "model.layers.44.self_attn.wo.weight": "model-00086-of-00155.safetensors",
+ "model.layers.44.self_attn.wq_a.weight": "model-00086-of-00155.safetensors",
+ "model.layers.44.self_attn.wq_b.weight": "model-00086-of-00155.safetensors",
+ "model.layers.44.self_attn.wq_b_gate.weight": "model-00086-of-00155.safetensors",
+ "model.layers.45.input_layernorm.weight": "model-00090-of-00155.safetensors",
+ "model.layers.45.mhc_attn.alpha_post": "model-00090-of-00155.safetensors",
+ "model.layers.45.mhc_attn.alpha_pre": "model-00090-of-00155.safetensors",
+ "model.layers.45.mhc_attn.alpha_res": "model-00090-of-00155.safetensors",
+ "model.layers.45.mhc_attn.bias_post": "model-00090-of-00155.safetensors",
+ "model.layers.45.mhc_attn.bias_pre": "model-00090-of-00155.safetensors",
+ "model.layers.45.mhc_attn.bias_res": "model-00090-of-00155.safetensors",
+ "model.layers.45.mhc_attn.proj_post.weight": "model-00090-of-00155.safetensors",
+ "model.layers.45.mhc_attn.proj_pre.weight": "model-00090-of-00155.safetensors",
+ "model.layers.45.mhc_attn.proj_res.weight": "model-00090-of-00155.safetensors",
+ "model.layers.45.mhc_attn.rms_norm.weight": "model-00090-of-00155.safetensors",
+ "model.layers.45.mhc_ffn.alpha_post": "model-00090-of-00155.safetensors",
+ "model.layers.45.mhc_ffn.alpha_pre": "model-00090-of-00155.safetensors",
+ "model.layers.45.mhc_ffn.alpha_res": "model-00090-of-00155.safetensors",
+ "model.layers.45.mhc_ffn.bias_post": "model-00090-of-00155.safetensors",
+ "model.layers.45.mhc_ffn.bias_pre": "model-00090-of-00155.safetensors",
+ "model.layers.45.mhc_ffn.bias_res": "model-00090-of-00155.safetensors",
+ "model.layers.45.mhc_ffn.proj_post.weight": "model-00090-of-00155.safetensors",
+ "model.layers.45.mhc_ffn.proj_pre.weight": "model-00090-of-00155.safetensors",
+ "model.layers.45.mhc_ffn.proj_res.weight": "model-00090-of-00155.safetensors",
+ "model.layers.45.mhc_ffn.rms_norm.weight": "model-00090-of-00155.safetensors",
+ "model.layers.45.moe.expert_bias": "model-00088-of-00155.safetensors",
+ "model.layers.45.moe.experts.act_fn.bias": "model-00090-of-00155.safetensors",
+ "model.layers.45.moe.experts.act_fn.weight": "model-00090-of-00155.safetensors",
+ "model.layers.45.moe.experts.down_proj": "model-00089-of-00155.safetensors",
+ "model.layers.45.moe.experts.gate_up_proj": "model-00148-of-00155.safetensors",
+ "model.layers.45.moe.router.gate.weight": "model-00090-of-00155.safetensors",
+ "model.layers.45.moe.shared_experts.act_fn.bias": "model-00090-of-00155.safetensors",
+ "model.layers.45.moe.shared_experts.act_fn.weight": "model-00090-of-00155.safetensors",
+ "model.layers.45.moe.shared_experts.down_proj.weight": "model-00090-of-00155.safetensors",
+ "model.layers.45.moe.shared_experts.gate_proj.weight": "model-00090-of-00155.safetensors",
+ "model.layers.45.moe.shared_experts.up_proj.weight": "model-00090-of-00155.safetensors",
+ "model.layers.45.post_attention_layernorm.weight": "model-00090-of-00155.safetensors",
+ "model.layers.45.self_attn.kv_norm.weight": "model-00088-of-00155.safetensors",
+ "model.layers.45.self_attn.lambda_proj.weight": "model-00088-of-00155.safetensors",
+ "model.layers.45.self_attn.q_norm.weight": "model-00088-of-00155.safetensors",
+ "model.layers.45.self_attn.wkv_a.weight": "model-00088-of-00155.safetensors",
+ "model.layers.45.self_attn.wkv_b.weight": "model-00088-of-00155.safetensors",
+ "model.layers.45.self_attn.wo.weight": "model-00088-of-00155.safetensors",
+ "model.layers.45.self_attn.wq_a.weight": "model-00088-of-00155.safetensors",
+ "model.layers.45.self_attn.wq_b.weight": "model-00088-of-00155.safetensors",
+ "model.layers.45.self_attn.wq_b_gate.weight": "model-00088-of-00155.safetensors",
+ "model.layers.46.input_layernorm.weight": "model-00092-of-00155.safetensors",
+ "model.layers.46.mhc_attn.alpha_post": "model-00092-of-00155.safetensors",
+ "model.layers.46.mhc_attn.alpha_pre": "model-00092-of-00155.safetensors",
+ "model.layers.46.mhc_attn.alpha_res": "model-00092-of-00155.safetensors",
+ "model.layers.46.mhc_attn.bias_post": "model-00092-of-00155.safetensors",
+ "model.layers.46.mhc_attn.bias_pre": "model-00092-of-00155.safetensors",
+ "model.layers.46.mhc_attn.bias_res": "model-00092-of-00155.safetensors",
+ "model.layers.46.mhc_attn.proj_post.weight": "model-00092-of-00155.safetensors",
+ "model.layers.46.mhc_attn.proj_pre.weight": "model-00092-of-00155.safetensors",
+ "model.layers.46.mhc_attn.proj_res.weight": "model-00092-of-00155.safetensors",
+ "model.layers.46.mhc_attn.rms_norm.weight": "model-00092-of-00155.safetensors",
+ "model.layers.46.mhc_ffn.alpha_post": "model-00092-of-00155.safetensors",
+ "model.layers.46.mhc_ffn.alpha_pre": "model-00092-of-00155.safetensors",
+ "model.layers.46.mhc_ffn.alpha_res": "model-00092-of-00155.safetensors",
+ "model.layers.46.mhc_ffn.bias_post": "model-00092-of-00155.safetensors",
+ "model.layers.46.mhc_ffn.bias_pre": "model-00092-of-00155.safetensors",
+ "model.layers.46.mhc_ffn.bias_res": "model-00092-of-00155.safetensors",
+ "model.layers.46.mhc_ffn.proj_post.weight": "model-00092-of-00155.safetensors",
+ "model.layers.46.mhc_ffn.proj_pre.weight": "model-00092-of-00155.safetensors",
+ "model.layers.46.mhc_ffn.proj_res.weight": "model-00092-of-00155.safetensors",
+ "model.layers.46.mhc_ffn.rms_norm.weight": "model-00092-of-00155.safetensors",
+ "model.layers.46.moe.expert_bias": "model-00090-of-00155.safetensors",
+ "model.layers.46.moe.experts.act_fn.bias": "model-00092-of-00155.safetensors",
+ "model.layers.46.moe.experts.act_fn.weight": "model-00092-of-00155.safetensors",
+ "model.layers.46.moe.experts.down_proj": "model-00091-of-00155.safetensors",
+ "model.layers.46.moe.experts.gate_up_proj": "model-00149-of-00155.safetensors",
+ "model.layers.46.moe.router.gate.weight": "model-00092-of-00155.safetensors",
+ "model.layers.46.moe.shared_experts.act_fn.bias": "model-00092-of-00155.safetensors",
+ "model.layers.46.moe.shared_experts.act_fn.weight": "model-00092-of-00155.safetensors",
+ "model.layers.46.moe.shared_experts.down_proj.weight": "model-00092-of-00155.safetensors",
+ "model.layers.46.moe.shared_experts.gate_proj.weight": "model-00092-of-00155.safetensors",
+ "model.layers.46.moe.shared_experts.up_proj.weight": "model-00092-of-00155.safetensors",
+ "model.layers.46.post_attention_layernorm.weight": "model-00092-of-00155.safetensors",
+ "model.layers.46.self_attn.kv_norm.weight": "model-00090-of-00155.safetensors",
+ "model.layers.46.self_attn.lambda_proj.weight": "model-00090-of-00155.safetensors",
+ "model.layers.46.self_attn.q_norm.weight": "model-00090-of-00155.safetensors",
+ "model.layers.46.self_attn.wkv_a.weight": "model-00090-of-00155.safetensors",
+ "model.layers.46.self_attn.wkv_b.weight": "model-00090-of-00155.safetensors",
+ "model.layers.46.self_attn.wo.weight": "model-00090-of-00155.safetensors",
+ "model.layers.46.self_attn.wq_a.weight": "model-00090-of-00155.safetensors",
+ "model.layers.46.self_attn.wq_b.weight": "model-00090-of-00155.safetensors",
+ "model.layers.46.self_attn.wq_b_gate.weight": "model-00090-of-00155.safetensors",
+ "model.layers.47.input_layernorm.weight": "model-00094-of-00155.safetensors",
+ "model.layers.47.mhc_attn.alpha_post": "model-00094-of-00155.safetensors",
+ "model.layers.47.mhc_attn.alpha_pre": "model-00094-of-00155.safetensors",
+ "model.layers.47.mhc_attn.alpha_res": "model-00094-of-00155.safetensors",
+ "model.layers.47.mhc_attn.bias_post": "model-00094-of-00155.safetensors",
+ "model.layers.47.mhc_attn.bias_pre": "model-00094-of-00155.safetensors",
+ "model.layers.47.mhc_attn.bias_res": "model-00094-of-00155.safetensors",
+ "model.layers.47.mhc_attn.proj_post.weight": "model-00094-of-00155.safetensors",
+ "model.layers.47.mhc_attn.proj_pre.weight": "model-00094-of-00155.safetensors",
+ "model.layers.47.mhc_attn.proj_res.weight": "model-00094-of-00155.safetensors",
+ "model.layers.47.mhc_attn.rms_norm.weight": "model-00094-of-00155.safetensors",
+ "model.layers.47.mhc_ffn.alpha_post": "model-00094-of-00155.safetensors",
+ "model.layers.47.mhc_ffn.alpha_pre": "model-00094-of-00155.safetensors",
+ "model.layers.47.mhc_ffn.alpha_res": "model-00094-of-00155.safetensors",
+ "model.layers.47.mhc_ffn.bias_post": "model-00094-of-00155.safetensors",
+ "model.layers.47.mhc_ffn.bias_pre": "model-00094-of-00155.safetensors",
+ "model.layers.47.mhc_ffn.bias_res": "model-00094-of-00155.safetensors",
+ "model.layers.47.mhc_ffn.proj_post.weight": "model-00094-of-00155.safetensors",
+ "model.layers.47.mhc_ffn.proj_pre.weight": "model-00094-of-00155.safetensors",
+ "model.layers.47.mhc_ffn.proj_res.weight": "model-00094-of-00155.safetensors",
+ "model.layers.47.mhc_ffn.rms_norm.weight": "model-00094-of-00155.safetensors",
+ "model.layers.47.moe.expert_bias": "model-00092-of-00155.safetensors",
+ "model.layers.47.moe.experts.act_fn.bias": "model-00094-of-00155.safetensors",
+ "model.layers.47.moe.experts.act_fn.weight": "model-00094-of-00155.safetensors",
+ "model.layers.47.moe.experts.down_proj": "model-00093-of-00155.safetensors",
+ "model.layers.47.moe.experts.gate_up_proj": "model-00150-of-00155.safetensors",
+ "model.layers.47.moe.router.gate.weight": "model-00094-of-00155.safetensors",
+ "model.layers.47.moe.shared_experts.act_fn.bias": "model-00094-of-00155.safetensors",
+ "model.layers.47.moe.shared_experts.act_fn.weight": "model-00094-of-00155.safetensors",
+ "model.layers.47.moe.shared_experts.down_proj.weight": "model-00094-of-00155.safetensors",
+ "model.layers.47.moe.shared_experts.gate_proj.weight": "model-00094-of-00155.safetensors",
+ "model.layers.47.moe.shared_experts.up_proj.weight": "model-00094-of-00155.safetensors",
+ "model.layers.47.post_attention_layernorm.weight": "model-00094-of-00155.safetensors",
+ "model.layers.47.self_attn.kv_norm.weight": "model-00092-of-00155.safetensors",
+ "model.layers.47.self_attn.lambda_proj.weight": "model-00092-of-00155.safetensors",
+ "model.layers.47.self_attn.q_norm.weight": "model-00092-of-00155.safetensors",
+ "model.layers.47.self_attn.wkv_a.weight": "model-00092-of-00155.safetensors",
+ "model.layers.47.self_attn.wkv_b.weight": "model-00092-of-00155.safetensors",
+ "model.layers.47.self_attn.wo.weight": "model-00092-of-00155.safetensors",
+ "model.layers.47.self_attn.wq_a.weight": "model-00092-of-00155.safetensors",
+ "model.layers.47.self_attn.wq_b.weight": "model-00092-of-00155.safetensors",
+ "model.layers.47.self_attn.wq_b_gate.weight": "model-00092-of-00155.safetensors",
+ "model.layers.48.input_layernorm.weight": "model-00096-of-00155.safetensors",
+ "model.layers.48.mhc_attn.alpha_post": "model-00096-of-00155.safetensors",
+ "model.layers.48.mhc_attn.alpha_pre": "model-00096-of-00155.safetensors",
+ "model.layers.48.mhc_attn.alpha_res": "model-00096-of-00155.safetensors",
+ "model.layers.48.mhc_attn.bias_post": "model-00096-of-00155.safetensors",
+ "model.layers.48.mhc_attn.bias_pre": "model-00096-of-00155.safetensors",
+ "model.layers.48.mhc_attn.bias_res": "model-00096-of-00155.safetensors",
+ "model.layers.48.mhc_attn.proj_post.weight": "model-00096-of-00155.safetensors",
+ "model.layers.48.mhc_attn.proj_pre.weight": "model-00096-of-00155.safetensors",
+ "model.layers.48.mhc_attn.proj_res.weight": "model-00096-of-00155.safetensors",
+ "model.layers.48.mhc_attn.rms_norm.weight": "model-00096-of-00155.safetensors",
+ "model.layers.48.mhc_ffn.alpha_post": "model-00096-of-00155.safetensors",
+ "model.layers.48.mhc_ffn.alpha_pre": "model-00096-of-00155.safetensors",
+ "model.layers.48.mhc_ffn.alpha_res": "model-00096-of-00155.safetensors",
+ "model.layers.48.mhc_ffn.bias_post": "model-00096-of-00155.safetensors",
+ "model.layers.48.mhc_ffn.bias_pre": "model-00096-of-00155.safetensors",
+ "model.layers.48.mhc_ffn.bias_res": "model-00096-of-00155.safetensors",
+ "model.layers.48.mhc_ffn.proj_post.weight": "model-00096-of-00155.safetensors",
+ "model.layers.48.mhc_ffn.proj_pre.weight": "model-00096-of-00155.safetensors",
+ "model.layers.48.mhc_ffn.proj_res.weight": "model-00096-of-00155.safetensors",
+ "model.layers.48.mhc_ffn.rms_norm.weight": "model-00096-of-00155.safetensors",
+ "model.layers.48.moe.expert_bias": "model-00094-of-00155.safetensors",
+ "model.layers.48.moe.experts.act_fn.bias": "model-00096-of-00155.safetensors",
+ "model.layers.48.moe.experts.act_fn.weight": "model-00096-of-00155.safetensors",
+ "model.layers.48.moe.experts.down_proj": "model-00095-of-00155.safetensors",
+ "model.layers.48.moe.experts.gate_up_proj": "model-00151-of-00155.safetensors",
+ "model.layers.48.moe.router.gate.weight": "model-00096-of-00155.safetensors",
+ "model.layers.48.moe.shared_experts.act_fn.bias": "model-00096-of-00155.safetensors",
+ "model.layers.48.moe.shared_experts.act_fn.weight": "model-00096-of-00155.safetensors",
+ "model.layers.48.moe.shared_experts.down_proj.weight": "model-00096-of-00155.safetensors",
+ "model.layers.48.moe.shared_experts.gate_proj.weight": "model-00096-of-00155.safetensors",
+ "model.layers.48.moe.shared_experts.up_proj.weight": "model-00096-of-00155.safetensors",
+ "model.layers.48.post_attention_layernorm.weight": "model-00096-of-00155.safetensors",
+ "model.layers.48.self_attn.kv_norm.weight": "model-00094-of-00155.safetensors",
+ "model.layers.48.self_attn.lambda_proj.weight": "model-00094-of-00155.safetensors",
+ "model.layers.48.self_attn.q_norm.weight": "model-00094-of-00155.safetensors",
+ "model.layers.48.self_attn.wkv_a.weight": "model-00094-of-00155.safetensors",
+ "model.layers.48.self_attn.wkv_b.weight": "model-00094-of-00155.safetensors",
+ "model.layers.48.self_attn.wo.weight": "model-00094-of-00155.safetensors",
+ "model.layers.48.self_attn.wq_a.weight": "model-00094-of-00155.safetensors",
+ "model.layers.48.self_attn.wq_b.weight": "model-00094-of-00155.safetensors",
+ "model.layers.48.self_attn.wq_b_gate.weight": "model-00094-of-00155.safetensors",
+ "model.layers.49.input_layernorm.weight": "model-00098-of-00155.safetensors",
+ "model.layers.49.mhc_attn.alpha_post": "model-00098-of-00155.safetensors",
+ "model.layers.49.mhc_attn.alpha_pre": "model-00098-of-00155.safetensors",
+ "model.layers.49.mhc_attn.alpha_res": "model-00098-of-00155.safetensors",
+ "model.layers.49.mhc_attn.bias_post": "model-00098-of-00155.safetensors",
+ "model.layers.49.mhc_attn.bias_pre": "model-00098-of-00155.safetensors",
+ "model.layers.49.mhc_attn.bias_res": "model-00098-of-00155.safetensors",
+ "model.layers.49.mhc_attn.proj_post.weight": "model-00098-of-00155.safetensors",
+ "model.layers.49.mhc_attn.proj_pre.weight": "model-00098-of-00155.safetensors",
+ "model.layers.49.mhc_attn.proj_res.weight": "model-00098-of-00155.safetensors",
+ "model.layers.49.mhc_attn.rms_norm.weight": "model-00098-of-00155.safetensors",
+ "model.layers.49.mhc_ffn.alpha_post": "model-00098-of-00155.safetensors",
+ "model.layers.49.mhc_ffn.alpha_pre": "model-00098-of-00155.safetensors",
+ "model.layers.49.mhc_ffn.alpha_res": "model-00098-of-00155.safetensors",
+ "model.layers.49.mhc_ffn.bias_post": "model-00098-of-00155.safetensors",
+ "model.layers.49.mhc_ffn.bias_pre": "model-00098-of-00155.safetensors",
+ "model.layers.49.mhc_ffn.bias_res": "model-00098-of-00155.safetensors",
+ "model.layers.49.mhc_ffn.proj_post.weight": "model-00098-of-00155.safetensors",
+ "model.layers.49.mhc_ffn.proj_pre.weight": "model-00098-of-00155.safetensors",
+ "model.layers.49.mhc_ffn.proj_res.weight": "model-00098-of-00155.safetensors",
+ "model.layers.49.mhc_ffn.rms_norm.weight": "model-00098-of-00155.safetensors",
+ "model.layers.49.moe.expert_bias": "model-00096-of-00155.safetensors",
+ "model.layers.49.moe.experts.act_fn.bias": "model-00098-of-00155.safetensors",
+ "model.layers.49.moe.experts.act_fn.weight": "model-00098-of-00155.safetensors",
+ "model.layers.49.moe.experts.down_proj": "model-00097-of-00155.safetensors",
+ "model.layers.49.moe.experts.gate_up_proj": "model-00152-of-00155.safetensors",
+ "model.layers.49.moe.router.gate.weight": "model-00098-of-00155.safetensors",
+ "model.layers.49.moe.shared_experts.act_fn.bias": "model-00098-of-00155.safetensors",
+ "model.layers.49.moe.shared_experts.act_fn.weight": "model-00098-of-00155.safetensors",
+ "model.layers.49.moe.shared_experts.down_proj.weight": "model-00098-of-00155.safetensors",
+ "model.layers.49.moe.shared_experts.gate_proj.weight": "model-00098-of-00155.safetensors",
+ "model.layers.49.moe.shared_experts.up_proj.weight": "model-00098-of-00155.safetensors",
+ "model.layers.49.post_attention_layernorm.weight": "model-00098-of-00155.safetensors",
+ "model.layers.49.self_attn.kv_norm.weight": "model-00096-of-00155.safetensors",
+ "model.layers.49.self_attn.lambda_proj.weight": "model-00096-of-00155.safetensors",
+ "model.layers.49.self_attn.q_norm.weight": "model-00096-of-00155.safetensors",
+ "model.layers.49.self_attn.wkv_a.weight": "model-00096-of-00155.safetensors",
+ "model.layers.49.self_attn.wkv_b.weight": "model-00096-of-00155.safetensors",
+ "model.layers.49.self_attn.wo.weight": "model-00096-of-00155.safetensors",
+ "model.layers.49.self_attn.wq_a.weight": "model-00096-of-00155.safetensors",
+ "model.layers.49.self_attn.wq_b.weight": "model-00096-of-00155.safetensors",
+ "model.layers.49.self_attn.wq_b_gate.weight": "model-00096-of-00155.safetensors",
+ "model.layers.5.input_layernorm.weight": "model-00010-of-00155.safetensors",
+ "model.layers.5.mhc_attn.alpha_post": "model-00010-of-00155.safetensors",
+ "model.layers.5.mhc_attn.alpha_pre": "model-00010-of-00155.safetensors",
+ "model.layers.5.mhc_attn.alpha_res": "model-00010-of-00155.safetensors",
+ "model.layers.5.mhc_attn.bias_post": "model-00010-of-00155.safetensors",
+ "model.layers.5.mhc_attn.bias_pre": "model-00010-of-00155.safetensors",
+ "model.layers.5.mhc_attn.bias_res": "model-00010-of-00155.safetensors",
+ "model.layers.5.mhc_attn.proj_post.weight": "model-00010-of-00155.safetensors",
+ "model.layers.5.mhc_attn.proj_pre.weight": "model-00010-of-00155.safetensors",
+ "model.layers.5.mhc_attn.proj_res.weight": "model-00010-of-00155.safetensors",
+ "model.layers.5.mhc_attn.rms_norm.weight": "model-00010-of-00155.safetensors",
+ "model.layers.5.mhc_ffn.alpha_post": "model-00010-of-00155.safetensors",
+ "model.layers.5.mhc_ffn.alpha_pre": "model-00010-of-00155.safetensors",
+ "model.layers.5.mhc_ffn.alpha_res": "model-00010-of-00155.safetensors",
+ "model.layers.5.mhc_ffn.bias_post": "model-00010-of-00155.safetensors",
+ "model.layers.5.mhc_ffn.bias_pre": "model-00010-of-00155.safetensors",
+ "model.layers.5.mhc_ffn.bias_res": "model-00010-of-00155.safetensors",
+ "model.layers.5.mhc_ffn.proj_post.weight": "model-00010-of-00155.safetensors",
+ "model.layers.5.mhc_ffn.proj_pre.weight": "model-00010-of-00155.safetensors",
+ "model.layers.5.mhc_ffn.proj_res.weight": "model-00010-of-00155.safetensors",
+ "model.layers.5.mhc_ffn.rms_norm.weight": "model-00010-of-00155.safetensors",
+ "model.layers.5.moe.expert_bias": "model-00008-of-00155.safetensors",
+ "model.layers.5.moe.experts.act_fn.bias": "model-00010-of-00155.safetensors",
+ "model.layers.5.moe.experts.act_fn.weight": "model-00010-of-00155.safetensors",
+ "model.layers.5.moe.experts.down_proj": "model-00009-of-00155.safetensors",
+ "model.layers.5.moe.experts.gate_up_proj": "model-00108-of-00155.safetensors",
+ "model.layers.5.moe.router.gate.weight": "model-00010-of-00155.safetensors",
+ "model.layers.5.moe.shared_experts.act_fn.bias": "model-00010-of-00155.safetensors",
+ "model.layers.5.moe.shared_experts.act_fn.weight": "model-00010-of-00155.safetensors",
+ "model.layers.5.moe.shared_experts.down_proj.weight": "model-00010-of-00155.safetensors",
+ "model.layers.5.moe.shared_experts.gate_proj.weight": "model-00010-of-00155.safetensors",
+ "model.layers.5.moe.shared_experts.up_proj.weight": "model-00010-of-00155.safetensors",
+ "model.layers.5.post_attention_layernorm.weight": "model-00010-of-00155.safetensors",
+ "model.layers.5.self_attn.kv_norm.weight": "model-00008-of-00155.safetensors",
+ "model.layers.5.self_attn.lambda_proj.weight": "model-00008-of-00155.safetensors",
+ "model.layers.5.self_attn.q_norm.weight": "model-00008-of-00155.safetensors",
+ "model.layers.5.self_attn.wkv_a.weight": "model-00008-of-00155.safetensors",
+ "model.layers.5.self_attn.wkv_b.weight": "model-00008-of-00155.safetensors",
+ "model.layers.5.self_attn.wo.weight": "model-00008-of-00155.safetensors",
+ "model.layers.5.self_attn.wq_a.weight": "model-00008-of-00155.safetensors",
+ "model.layers.5.self_attn.wq_b.weight": "model-00008-of-00155.safetensors",
+ "model.layers.5.self_attn.wq_b_gate.weight": "model-00008-of-00155.safetensors",
+ "model.layers.50.input_layernorm.weight": "model-00100-of-00155.safetensors",
+ "model.layers.50.mhc_attn.alpha_post": "model-00100-of-00155.safetensors",
+ "model.layers.50.mhc_attn.alpha_pre": "model-00100-of-00155.safetensors",
+ "model.layers.50.mhc_attn.alpha_res": "model-00100-of-00155.safetensors",
+ "model.layers.50.mhc_attn.bias_post": "model-00100-of-00155.safetensors",
+ "model.layers.50.mhc_attn.bias_pre": "model-00100-of-00155.safetensors",
+ "model.layers.50.mhc_attn.bias_res": "model-00100-of-00155.safetensors",
+ "model.layers.50.mhc_attn.proj_post.weight": "model-00100-of-00155.safetensors",
+ "model.layers.50.mhc_attn.proj_pre.weight": "model-00100-of-00155.safetensors",
+ "model.layers.50.mhc_attn.proj_res.weight": "model-00100-of-00155.safetensors",
+ "model.layers.50.mhc_attn.rms_norm.weight": "model-00100-of-00155.safetensors",
+ "model.layers.50.mhc_ffn.alpha_post": "model-00100-of-00155.safetensors",
+ "model.layers.50.mhc_ffn.alpha_pre": "model-00100-of-00155.safetensors",
+ "model.layers.50.mhc_ffn.alpha_res": "model-00100-of-00155.safetensors",
+ "model.layers.50.mhc_ffn.bias_post": "model-00100-of-00155.safetensors",
+ "model.layers.50.mhc_ffn.bias_pre": "model-00100-of-00155.safetensors",
+ "model.layers.50.mhc_ffn.bias_res": "model-00100-of-00155.safetensors",
+ "model.layers.50.mhc_ffn.proj_post.weight": "model-00100-of-00155.safetensors",
+ "model.layers.50.mhc_ffn.proj_pre.weight": "model-00100-of-00155.safetensors",
+ "model.layers.50.mhc_ffn.proj_res.weight": "model-00100-of-00155.safetensors",
+ "model.layers.50.mhc_ffn.rms_norm.weight": "model-00100-of-00155.safetensors",
+ "model.layers.50.moe.expert_bias": "model-00098-of-00155.safetensors",
+ "model.layers.50.moe.experts.act_fn.bias": "model-00100-of-00155.safetensors",
+ "model.layers.50.moe.experts.act_fn.weight": "model-00100-of-00155.safetensors",
+ "model.layers.50.moe.experts.down_proj": "model-00099-of-00155.safetensors",
+ "model.layers.50.moe.experts.gate_up_proj": "model-00153-of-00155.safetensors",
+ "model.layers.50.moe.router.gate.weight": "model-00100-of-00155.safetensors",
+ "model.layers.50.moe.shared_experts.act_fn.bias": "model-00100-of-00155.safetensors",
+ "model.layers.50.moe.shared_experts.act_fn.weight": "model-00100-of-00155.safetensors",
+ "model.layers.50.moe.shared_experts.down_proj.weight": "model-00100-of-00155.safetensors",
+ "model.layers.50.moe.shared_experts.gate_proj.weight": "model-00100-of-00155.safetensors",
+ "model.layers.50.moe.shared_experts.up_proj.weight": "model-00100-of-00155.safetensors",
+ "model.layers.50.post_attention_layernorm.weight": "model-00100-of-00155.safetensors",
+ "model.layers.50.self_attn.kv_norm.weight": "model-00098-of-00155.safetensors",
+ "model.layers.50.self_attn.lambda_proj.weight": "model-00098-of-00155.safetensors",
+ "model.layers.50.self_attn.q_norm.weight": "model-00098-of-00155.safetensors",
+ "model.layers.50.self_attn.wkv_a.weight": "model-00098-of-00155.safetensors",
+ "model.layers.50.self_attn.wkv_b.weight": "model-00098-of-00155.safetensors",
+ "model.layers.50.self_attn.wo.weight": "model-00098-of-00155.safetensors",
+ "model.layers.50.self_attn.wq_a.weight": "model-00098-of-00155.safetensors",
+ "model.layers.50.self_attn.wq_b.weight": "model-00098-of-00155.safetensors",
+ "model.layers.50.self_attn.wq_b_gate.weight": "model-00098-of-00155.safetensors",
+ "model.layers.51.input_layernorm.weight": "model-00102-of-00155.safetensors",
+ "model.layers.51.mhc_attn.alpha_post": "model-00102-of-00155.safetensors",
+ "model.layers.51.mhc_attn.alpha_pre": "model-00102-of-00155.safetensors",
+ "model.layers.51.mhc_attn.alpha_res": "model-00102-of-00155.safetensors",
+ "model.layers.51.mhc_attn.bias_post": "model-00102-of-00155.safetensors",
+ "model.layers.51.mhc_attn.bias_pre": "model-00102-of-00155.safetensors",
+ "model.layers.51.mhc_attn.bias_res": "model-00102-of-00155.safetensors",
+ "model.layers.51.mhc_attn.proj_post.weight": "model-00102-of-00155.safetensors",
+ "model.layers.51.mhc_attn.proj_pre.weight": "model-00102-of-00155.safetensors",
+ "model.layers.51.mhc_attn.proj_res.weight": "model-00102-of-00155.safetensors",
+ "model.layers.51.mhc_attn.rms_norm.weight": "model-00102-of-00155.safetensors",
+ "model.layers.51.mhc_ffn.alpha_post": "model-00102-of-00155.safetensors",
+ "model.layers.51.mhc_ffn.alpha_pre": "model-00102-of-00155.safetensors",
+ "model.layers.51.mhc_ffn.alpha_res": "model-00102-of-00155.safetensors",
+ "model.layers.51.mhc_ffn.bias_post": "model-00102-of-00155.safetensors",
+ "model.layers.51.mhc_ffn.bias_pre": "model-00102-of-00155.safetensors",
+ "model.layers.51.mhc_ffn.bias_res": "model-00102-of-00155.safetensors",
+ "model.layers.51.mhc_ffn.proj_post.weight": "model-00102-of-00155.safetensors",
+ "model.layers.51.mhc_ffn.proj_pre.weight": "model-00102-of-00155.safetensors",
+ "model.layers.51.mhc_ffn.proj_res.weight": "model-00102-of-00155.safetensors",
+ "model.layers.51.mhc_ffn.rms_norm.weight": "model-00102-of-00155.safetensors",
+ "model.layers.51.moe.expert_bias": "model-00100-of-00155.safetensors",
+ "model.layers.51.moe.experts.act_fn.bias": "model-00102-of-00155.safetensors",
+ "model.layers.51.moe.experts.act_fn.weight": "model-00102-of-00155.safetensors",
+ "model.layers.51.moe.experts.down_proj": "model-00101-of-00155.safetensors",
+ "model.layers.51.moe.experts.gate_up_proj": "model-00154-of-00155.safetensors",
+ "model.layers.51.moe.router.gate.weight": "model-00102-of-00155.safetensors",
+ "model.layers.51.moe.shared_experts.act_fn.bias": "model-00102-of-00155.safetensors",
+ "model.layers.51.moe.shared_experts.act_fn.weight": "model-00102-of-00155.safetensors",
+ "model.layers.51.moe.shared_experts.down_proj.weight": "model-00102-of-00155.safetensors",
+ "model.layers.51.moe.shared_experts.gate_proj.weight": "model-00102-of-00155.safetensors",
+ "model.layers.51.moe.shared_experts.up_proj.weight": "model-00102-of-00155.safetensors",
+ "model.layers.51.post_attention_layernorm.weight": "model-00102-of-00155.safetensors",
+ "model.layers.51.self_attn.kv_norm.weight": "model-00100-of-00155.safetensors",
+ "model.layers.51.self_attn.lambda_proj.weight": "model-00100-of-00155.safetensors",
+ "model.layers.51.self_attn.q_norm.weight": "model-00100-of-00155.safetensors",
+ "model.layers.51.self_attn.wkv_a.weight": "model-00100-of-00155.safetensors",
+ "model.layers.51.self_attn.wkv_b.weight": "model-00100-of-00155.safetensors",
+ "model.layers.51.self_attn.wo.weight": "model-00100-of-00155.safetensors",
+ "model.layers.51.self_attn.wq_a.weight": "model-00100-of-00155.safetensors",
+ "model.layers.51.self_attn.wq_b.weight": "model-00100-of-00155.safetensors",
+ "model.layers.51.self_attn.wq_b_gate.weight": "model-00100-of-00155.safetensors",
+ "model.layers.52.input_layernorm.weight": "model-00104-of-00155.safetensors",
+ "model.layers.52.mhc_attn.alpha_post": "model-00104-of-00155.safetensors",
+ "model.layers.52.mhc_attn.alpha_pre": "model-00104-of-00155.safetensors",
+ "model.layers.52.mhc_attn.alpha_res": "model-00104-of-00155.safetensors",
+ "model.layers.52.mhc_attn.bias_post": "model-00104-of-00155.safetensors",
+ "model.layers.52.mhc_attn.bias_pre": "model-00104-of-00155.safetensors",
+ "model.layers.52.mhc_attn.bias_res": "model-00104-of-00155.safetensors",
+ "model.layers.52.mhc_attn.proj_post.weight": "model-00104-of-00155.safetensors",
+ "model.layers.52.mhc_attn.proj_pre.weight": "model-00104-of-00155.safetensors",
+ "model.layers.52.mhc_attn.proj_res.weight": "model-00104-of-00155.safetensors",
+ "model.layers.52.mhc_attn.rms_norm.weight": "model-00104-of-00155.safetensors",
+ "model.layers.52.mhc_ffn.alpha_post": "model-00104-of-00155.safetensors",
+ "model.layers.52.mhc_ffn.alpha_pre": "model-00104-of-00155.safetensors",
+ "model.layers.52.mhc_ffn.alpha_res": "model-00104-of-00155.safetensors",
+ "model.layers.52.mhc_ffn.bias_post": "model-00104-of-00155.safetensors",
+ "model.layers.52.mhc_ffn.bias_pre": "model-00104-of-00155.safetensors",
+ "model.layers.52.mhc_ffn.bias_res": "model-00104-of-00155.safetensors",
+ "model.layers.52.mhc_ffn.proj_post.weight": "model-00104-of-00155.safetensors",
+ "model.layers.52.mhc_ffn.proj_pre.weight": "model-00104-of-00155.safetensors",
+ "model.layers.52.mhc_ffn.proj_res.weight": "model-00104-of-00155.safetensors",
+ "model.layers.52.mhc_ffn.rms_norm.weight": "model-00104-of-00155.safetensors",
+ "model.layers.52.moe.expert_bias": "model-00102-of-00155.safetensors",
+ "model.layers.52.moe.experts.act_fn.bias": "model-00104-of-00155.safetensors",
+ "model.layers.52.moe.experts.act_fn.weight": "model-00104-of-00155.safetensors",
+ "model.layers.52.moe.experts.down_proj": "model-00103-of-00155.safetensors",
+ "model.layers.52.moe.experts.gate_up_proj": "model-00155-of-00155.safetensors",
+ "model.layers.52.moe.router.gate.weight": "model-00104-of-00155.safetensors",
+ "model.layers.52.moe.shared_experts.act_fn.bias": "model-00104-of-00155.safetensors",
+ "model.layers.52.moe.shared_experts.act_fn.weight": "model-00104-of-00155.safetensors",
+ "model.layers.52.moe.shared_experts.down_proj.weight": "model-00104-of-00155.safetensors",
+ "model.layers.52.moe.shared_experts.gate_proj.weight": "model-00104-of-00155.safetensors",
+ "model.layers.52.moe.shared_experts.up_proj.weight": "model-00104-of-00155.safetensors",
+ "model.layers.52.post_attention_layernorm.weight": "model-00104-of-00155.safetensors",
+ "model.layers.52.self_attn.kv_norm.weight": "model-00102-of-00155.safetensors",
+ "model.layers.52.self_attn.lambda_proj.weight": "model-00102-of-00155.safetensors",
+ "model.layers.52.self_attn.q_norm.weight": "model-00102-of-00155.safetensors",
+ "model.layers.52.self_attn.wkv_a.weight": "model-00102-of-00155.safetensors",
+ "model.layers.52.self_attn.wkv_b.weight": "model-00102-of-00155.safetensors",
+ "model.layers.52.self_attn.wo.weight": "model-00102-of-00155.safetensors",
+ "model.layers.52.self_attn.wq_a.weight": "model-00102-of-00155.safetensors",
+ "model.layers.52.self_attn.wq_b.weight": "model-00102-of-00155.safetensors",
+ "model.layers.52.self_attn.wq_b_gate.weight": "model-00102-of-00155.safetensors",
+ "model.layers.6.input_layernorm.weight": "model-00012-of-00155.safetensors",
+ "model.layers.6.mhc_attn.alpha_post": "model-00012-of-00155.safetensors",
+ "model.layers.6.mhc_attn.alpha_pre": "model-00012-of-00155.safetensors",
+ "model.layers.6.mhc_attn.alpha_res": "model-00012-of-00155.safetensors",
+ "model.layers.6.mhc_attn.bias_post": "model-00012-of-00155.safetensors",
+ "model.layers.6.mhc_attn.bias_pre": "model-00012-of-00155.safetensors",
+ "model.layers.6.mhc_attn.bias_res": "model-00012-of-00155.safetensors",
+ "model.layers.6.mhc_attn.proj_post.weight": "model-00012-of-00155.safetensors",
+ "model.layers.6.mhc_attn.proj_pre.weight": "model-00012-of-00155.safetensors",
+ "model.layers.6.mhc_attn.proj_res.weight": "model-00012-of-00155.safetensors",
+ "model.layers.6.mhc_attn.rms_norm.weight": "model-00012-of-00155.safetensors",
+ "model.layers.6.mhc_ffn.alpha_post": "model-00012-of-00155.safetensors",
+ "model.layers.6.mhc_ffn.alpha_pre": "model-00012-of-00155.safetensors",
+ "model.layers.6.mhc_ffn.alpha_res": "model-00012-of-00155.safetensors",
+ "model.layers.6.mhc_ffn.bias_post": "model-00012-of-00155.safetensors",
+ "model.layers.6.mhc_ffn.bias_pre": "model-00012-of-00155.safetensors",
+ "model.layers.6.mhc_ffn.bias_res": "model-00012-of-00155.safetensors",
+ "model.layers.6.mhc_ffn.proj_post.weight": "model-00012-of-00155.safetensors",
+ "model.layers.6.mhc_ffn.proj_pre.weight": "model-00012-of-00155.safetensors",
+ "model.layers.6.mhc_ffn.proj_res.weight": "model-00012-of-00155.safetensors",
+ "model.layers.6.mhc_ffn.rms_norm.weight": "model-00012-of-00155.safetensors",
+ "model.layers.6.moe.expert_bias": "model-00010-of-00155.safetensors",
+ "model.layers.6.moe.experts.act_fn.bias": "model-00012-of-00155.safetensors",
+ "model.layers.6.moe.experts.act_fn.weight": "model-00012-of-00155.safetensors",
+ "model.layers.6.moe.experts.down_proj": "model-00011-of-00155.safetensors",
+ "model.layers.6.moe.experts.gate_up_proj": "model-00109-of-00155.safetensors",
+ "model.layers.6.moe.router.gate.weight": "model-00012-of-00155.safetensors",
+ "model.layers.6.moe.shared_experts.act_fn.bias": "model-00012-of-00155.safetensors",
+ "model.layers.6.moe.shared_experts.act_fn.weight": "model-00012-of-00155.safetensors",
+ "model.layers.6.moe.shared_experts.down_proj.weight": "model-00012-of-00155.safetensors",
+ "model.layers.6.moe.shared_experts.gate_proj.weight": "model-00012-of-00155.safetensors",
+ "model.layers.6.moe.shared_experts.up_proj.weight": "model-00012-of-00155.safetensors",
+ "model.layers.6.post_attention_layernorm.weight": "model-00012-of-00155.safetensors",
+ "model.layers.6.self_attn.kv_norm.weight": "model-00010-of-00155.safetensors",
+ "model.layers.6.self_attn.lambda_proj.weight": "model-00010-of-00155.safetensors",
+ "model.layers.6.self_attn.q_norm.weight": "model-00010-of-00155.safetensors",
+ "model.layers.6.self_attn.wkv_a.weight": "model-00010-of-00155.safetensors",
+ "model.layers.6.self_attn.wkv_b.weight": "model-00010-of-00155.safetensors",
+ "model.layers.6.self_attn.wo.weight": "model-00010-of-00155.safetensors",
+ "model.layers.6.self_attn.wq_a.weight": "model-00010-of-00155.safetensors",
+ "model.layers.6.self_attn.wq_b.weight": "model-00010-of-00155.safetensors",
+ "model.layers.6.self_attn.wq_b_gate.weight": "model-00010-of-00155.safetensors",
+ "model.layers.7.input_layernorm.weight": "model-00014-of-00155.safetensors",
+ "model.layers.7.mhc_attn.alpha_post": "model-00014-of-00155.safetensors",
+ "model.layers.7.mhc_attn.alpha_pre": "model-00014-of-00155.safetensors",
+ "model.layers.7.mhc_attn.alpha_res": "model-00014-of-00155.safetensors",
+ "model.layers.7.mhc_attn.bias_post": "model-00014-of-00155.safetensors",
+ "model.layers.7.mhc_attn.bias_pre": "model-00014-of-00155.safetensors",
+ "model.layers.7.mhc_attn.bias_res": "model-00014-of-00155.safetensors",
+ "model.layers.7.mhc_attn.proj_post.weight": "model-00014-of-00155.safetensors",
+ "model.layers.7.mhc_attn.proj_pre.weight": "model-00014-of-00155.safetensors",
+ "model.layers.7.mhc_attn.proj_res.weight": "model-00014-of-00155.safetensors",
+ "model.layers.7.mhc_attn.rms_norm.weight": "model-00014-of-00155.safetensors",
+ "model.layers.7.mhc_ffn.alpha_post": "model-00014-of-00155.safetensors",
+ "model.layers.7.mhc_ffn.alpha_pre": "model-00014-of-00155.safetensors",
+ "model.layers.7.mhc_ffn.alpha_res": "model-00014-of-00155.safetensors",
+ "model.layers.7.mhc_ffn.bias_post": "model-00014-of-00155.safetensors",
+ "model.layers.7.mhc_ffn.bias_pre": "model-00014-of-00155.safetensors",
+ "model.layers.7.mhc_ffn.bias_res": "model-00014-of-00155.safetensors",
+ "model.layers.7.mhc_ffn.proj_post.weight": "model-00014-of-00155.safetensors",
+ "model.layers.7.mhc_ffn.proj_pre.weight": "model-00014-of-00155.safetensors",
+ "model.layers.7.mhc_ffn.proj_res.weight": "model-00014-of-00155.safetensors",
+ "model.layers.7.mhc_ffn.rms_norm.weight": "model-00014-of-00155.safetensors",
+ "model.layers.7.moe.expert_bias": "model-00012-of-00155.safetensors",
+ "model.layers.7.moe.experts.act_fn.bias": "model-00014-of-00155.safetensors",
+ "model.layers.7.moe.experts.act_fn.weight": "model-00014-of-00155.safetensors",
+ "model.layers.7.moe.experts.down_proj": "model-00013-of-00155.safetensors",
+ "model.layers.7.moe.experts.gate_up_proj": "model-00110-of-00155.safetensors",
+ "model.layers.7.moe.router.gate.weight": "model-00014-of-00155.safetensors",
+ "model.layers.7.moe.shared_experts.act_fn.bias": "model-00014-of-00155.safetensors",
+ "model.layers.7.moe.shared_experts.act_fn.weight": "model-00014-of-00155.safetensors",
+ "model.layers.7.moe.shared_experts.down_proj.weight": "model-00014-of-00155.safetensors",
+ "model.layers.7.moe.shared_experts.gate_proj.weight": "model-00014-of-00155.safetensors",
+ "model.layers.7.moe.shared_experts.up_proj.weight": "model-00014-of-00155.safetensors",
+ "model.layers.7.post_attention_layernorm.weight": "model-00014-of-00155.safetensors",
+ "model.layers.7.self_attn.kv_norm.weight": "model-00012-of-00155.safetensors",
+ "model.layers.7.self_attn.lambda_proj.weight": "model-00012-of-00155.safetensors",
+ "model.layers.7.self_attn.q_norm.weight": "model-00012-of-00155.safetensors",
+ "model.layers.7.self_attn.wkv_a.weight": "model-00012-of-00155.safetensors",
+ "model.layers.7.self_attn.wkv_b.weight": "model-00012-of-00155.safetensors",
+ "model.layers.7.self_attn.wo.weight": "model-00012-of-00155.safetensors",
+ "model.layers.7.self_attn.wq_a.weight": "model-00012-of-00155.safetensors",
+ "model.layers.7.self_attn.wq_b.weight": "model-00012-of-00155.safetensors",
+ "model.layers.7.self_attn.wq_b_gate.weight": "model-00012-of-00155.safetensors",
+ "model.layers.8.input_layernorm.weight": "model-00016-of-00155.safetensors",
+ "model.layers.8.mhc_attn.alpha_post": "model-00016-of-00155.safetensors",
+ "model.layers.8.mhc_attn.alpha_pre": "model-00016-of-00155.safetensors",
+ "model.layers.8.mhc_attn.alpha_res": "model-00016-of-00155.safetensors",
+ "model.layers.8.mhc_attn.bias_post": "model-00016-of-00155.safetensors",
+ "model.layers.8.mhc_attn.bias_pre": "model-00016-of-00155.safetensors",
+ "model.layers.8.mhc_attn.bias_res": "model-00016-of-00155.safetensors",
+ "model.layers.8.mhc_attn.proj_post.weight": "model-00016-of-00155.safetensors",
+ "model.layers.8.mhc_attn.proj_pre.weight": "model-00016-of-00155.safetensors",
+ "model.layers.8.mhc_attn.proj_res.weight": "model-00016-of-00155.safetensors",
+ "model.layers.8.mhc_attn.rms_norm.weight": "model-00016-of-00155.safetensors",
+ "model.layers.8.mhc_ffn.alpha_post": "model-00016-of-00155.safetensors",
+ "model.layers.8.mhc_ffn.alpha_pre": "model-00016-of-00155.safetensors",
+ "model.layers.8.mhc_ffn.alpha_res": "model-00016-of-00155.safetensors",
+ "model.layers.8.mhc_ffn.bias_post": "model-00016-of-00155.safetensors",
+ "model.layers.8.mhc_ffn.bias_pre": "model-00016-of-00155.safetensors",
+ "model.layers.8.mhc_ffn.bias_res": "model-00016-of-00155.safetensors",
+ "model.layers.8.mhc_ffn.proj_post.weight": "model-00016-of-00155.safetensors",
+ "model.layers.8.mhc_ffn.proj_pre.weight": "model-00016-of-00155.safetensors",
+ "model.layers.8.mhc_ffn.proj_res.weight": "model-00016-of-00155.safetensors",
+ "model.layers.8.mhc_ffn.rms_norm.weight": "model-00016-of-00155.safetensors",
+ "model.layers.8.moe.expert_bias": "model-00014-of-00155.safetensors",
+ "model.layers.8.moe.experts.act_fn.bias": "model-00016-of-00155.safetensors",
+ "model.layers.8.moe.experts.act_fn.weight": "model-00016-of-00155.safetensors",
+ "model.layers.8.moe.experts.down_proj": "model-00015-of-00155.safetensors",
+ "model.layers.8.moe.experts.gate_up_proj": "model-00111-of-00155.safetensors",
+ "model.layers.8.moe.router.gate.weight": "model-00016-of-00155.safetensors",
+ "model.layers.8.moe.shared_experts.act_fn.bias": "model-00016-of-00155.safetensors",
+ "model.layers.8.moe.shared_experts.act_fn.weight": "model-00016-of-00155.safetensors",
+ "model.layers.8.moe.shared_experts.down_proj.weight": "model-00016-of-00155.safetensors",
+ "model.layers.8.moe.shared_experts.gate_proj.weight": "model-00016-of-00155.safetensors",
+ "model.layers.8.moe.shared_experts.up_proj.weight": "model-00016-of-00155.safetensors",
+ "model.layers.8.post_attention_layernorm.weight": "model-00016-of-00155.safetensors",
+ "model.layers.8.self_attn.kv_norm.weight": "model-00014-of-00155.safetensors",
+ "model.layers.8.self_attn.lambda_proj.weight": "model-00014-of-00155.safetensors",
+ "model.layers.8.self_attn.q_norm.weight": "model-00014-of-00155.safetensors",
+ "model.layers.8.self_attn.wkv_a.weight": "model-00014-of-00155.safetensors",
+ "model.layers.8.self_attn.wkv_b.weight": "model-00014-of-00155.safetensors",
+ "model.layers.8.self_attn.wo.weight": "model-00014-of-00155.safetensors",
+ "model.layers.8.self_attn.wq_a.weight": "model-00014-of-00155.safetensors",
+ "model.layers.8.self_attn.wq_b.weight": "model-00014-of-00155.safetensors",
+ "model.layers.8.self_attn.wq_b_gate.weight": "model-00014-of-00155.safetensors",
+ "model.layers.9.input_layernorm.weight": "model-00018-of-00155.safetensors",
+ "model.layers.9.mhc_attn.alpha_post": "model-00018-of-00155.safetensors",
+ "model.layers.9.mhc_attn.alpha_pre": "model-00018-of-00155.safetensors",
+ "model.layers.9.mhc_attn.alpha_res": "model-00018-of-00155.safetensors",
+ "model.layers.9.mhc_attn.bias_post": "model-00018-of-00155.safetensors",
+ "model.layers.9.mhc_attn.bias_pre": "model-00018-of-00155.safetensors",
+ "model.layers.9.mhc_attn.bias_res": "model-00018-of-00155.safetensors",
+ "model.layers.9.mhc_attn.proj_post.weight": "model-00018-of-00155.safetensors",
+ "model.layers.9.mhc_attn.proj_pre.weight": "model-00018-of-00155.safetensors",
+ "model.layers.9.mhc_attn.proj_res.weight": "model-00018-of-00155.safetensors",
+ "model.layers.9.mhc_attn.rms_norm.weight": "model-00018-of-00155.safetensors",
+ "model.layers.9.mhc_ffn.alpha_post": "model-00018-of-00155.safetensors",
+ "model.layers.9.mhc_ffn.alpha_pre": "model-00018-of-00155.safetensors",
+ "model.layers.9.mhc_ffn.alpha_res": "model-00018-of-00155.safetensors",
+ "model.layers.9.mhc_ffn.bias_post": "model-00018-of-00155.safetensors",
+ "model.layers.9.mhc_ffn.bias_pre": "model-00018-of-00155.safetensors",
+ "model.layers.9.mhc_ffn.bias_res": "model-00018-of-00155.safetensors",
+ "model.layers.9.mhc_ffn.proj_post.weight": "model-00018-of-00155.safetensors",
+ "model.layers.9.mhc_ffn.proj_pre.weight": "model-00018-of-00155.safetensors",
+ "model.layers.9.mhc_ffn.proj_res.weight": "model-00018-of-00155.safetensors",
+ "model.layers.9.mhc_ffn.rms_norm.weight": "model-00018-of-00155.safetensors",
+ "model.layers.9.moe.expert_bias": "model-00016-of-00155.safetensors",
+ "model.layers.9.moe.experts.act_fn.bias": "model-00018-of-00155.safetensors",
+ "model.layers.9.moe.experts.act_fn.weight": "model-00018-of-00155.safetensors",
+ "model.layers.9.moe.experts.down_proj": "model-00017-of-00155.safetensors",
+ "model.layers.9.moe.experts.gate_up_proj": "model-00112-of-00155.safetensors",
+ "model.layers.9.moe.router.gate.weight": "model-00018-of-00155.safetensors",
+ "model.layers.9.moe.shared_experts.act_fn.bias": "model-00018-of-00155.safetensors",
+ "model.layers.9.moe.shared_experts.act_fn.weight": "model-00018-of-00155.safetensors",
+ "model.layers.9.moe.shared_experts.down_proj.weight": "model-00018-of-00155.safetensors",
+ "model.layers.9.moe.shared_experts.gate_proj.weight": "model-00018-of-00155.safetensors",
+ "model.layers.9.moe.shared_experts.up_proj.weight": "model-00018-of-00155.safetensors",
+ "model.layers.9.post_attention_layernorm.weight": "model-00018-of-00155.safetensors",
+ "model.layers.9.self_attn.kv_norm.weight": "model-00016-of-00155.safetensors",
+ "model.layers.9.self_attn.lambda_proj.weight": "model-00016-of-00155.safetensors",
+ "model.layers.9.self_attn.q_norm.weight": "model-00016-of-00155.safetensors",
+ "model.layers.9.self_attn.wkv_a.weight": "model-00016-of-00155.safetensors",
+ "model.layers.9.self_attn.wkv_b.weight": "model-00016-of-00155.safetensors",
+ "model.layers.9.self_attn.wo.weight": "model-00016-of-00155.safetensors",
+ "model.layers.9.self_attn.wq_a.weight": "model-00016-of-00155.safetensors",
+ "model.layers.9.self_attn.wq_b.weight": "model-00016-of-00155.safetensors",
+ "model.layers.9.self_attn.wq_b_gate.weight": "model-00016-of-00155.safetensors",
+ "model.mtp_layers.0.embed_norm.weight": "model-00104-of-00155.safetensors",
+ "model.mtp_layers.0.final_layernorm.weight": "model-00104-of-00155.safetensors",
+ "model.mtp_layers.0.input_layernorm.weight": "model-00104-of-00155.safetensors",
+ "model.mtp_layers.0.input_proj.weight": "model-00104-of-00155.safetensors",
+ "model.mtp_layers.0.mlp.act_fn.bias": "model-00104-of-00155.safetensors",
+ "model.mtp_layers.0.mlp.act_fn.weight": "model-00104-of-00155.safetensors",
+ "model.mtp_layers.0.mlp.down_proj.weight": "model-00104-of-00155.safetensors",
+ "model.mtp_layers.0.mlp.gate_proj.weight": "model-00104-of-00155.safetensors",
+ "model.mtp_layers.0.mlp.up_proj.weight": "model-00104-of-00155.safetensors",
+ "model.mtp_layers.0.post_attention_layernorm.weight": "model-00104-of-00155.safetensors",
+ "model.mtp_layers.0.self_attn.kv_norm.weight": "model-00104-of-00155.safetensors",
+ "model.mtp_layers.0.self_attn.lambda_proj.weight": "model-00104-of-00155.safetensors",
+ "model.mtp_layers.0.self_attn.q_norm.weight": "model-00104-of-00155.safetensors",
+ "model.mtp_layers.0.self_attn.wkv_a.weight": "model-00104-of-00155.safetensors",
+ "model.mtp_layers.0.self_attn.wkv_b.weight": "model-00104-of-00155.safetensors",
+ "model.mtp_layers.0.self_attn.wo.weight": "model-00104-of-00155.safetensors",
+ "model.mtp_layers.0.self_attn.wq_a.weight": "model-00104-of-00155.safetensors",
+ "model.mtp_layers.0.self_attn.wq_b.weight": "model-00104-of-00155.safetensors",
+ "model.mtp_layers.0.self_attn.wq_b_gate.weight": "model-00104-of-00155.safetensors",
+ "model.norm.weight": "model-00104-of-00155.safetensors"
+ }
+}
\ No newline at end of file
diff --git a/modeling_motif.py b/modeling_motif.py
new file mode 100644
index 0000000000000000000000000000000000000000..ea75c5f080d665df2527ca29e59caa1891439bac
--- /dev/null
+++ b/modeling_motif.py
@@ -0,0 +1,1943 @@
+import math
+from typing import Callable, Literal, Optional, Tuple
+
+import einops
+import torch
+import torch.nn.functional as F
+import torch.utils.checkpoint
+from torch import nn
+from torch.nn import CrossEntropyLoss
+from transformers.activations import ACT2CLS as _ACT2CLS
+from transformers.activations import ClassInstantier
+from transformers.cache_utils import Cache, DynamicCache, StaticCache
+from transformers.generation import GenerationMixin
+from transformers.integrations import use_experts_implementation
+from transformers.modeling_attn_mask_utils import AttentionMaskConverter
+from transformers.modeling_layers import GradientCheckpointingLayer
+from transformers.modeling_outputs import MoeCausalLMOutputWithPast, MoeModelOutputWithPast
+from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS
+from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
+from transformers.utils import auto_docstring, can_return_tuple, logging
+
+from .configuration_motif import MotifConfig
+
+logger = logging.get_logger(__name__)
+
+if hasattr(torch.version, "hip") and torch.version.hip is not None:
+ activation = None
+ logger.warning_once("Using HIP")
+ logger.warning_once("Due to the HIP, we do not utilize the kernel ops for precision.")
+ logger.warning_once("Using torch ops")
+ kernelRMSNorm = None
+ PolyNormKernel = None
+else:
+ logger.warning_once("Using CUDA")
+ try:
+ import kernels
+
+ activation = kernels.get_kernel("Motif-Technologies/activation")
+ kernelRMSNorm = activation.layers.RMSNorm
+ PolyNormKernel = activation.layers.PolyNorm
+ except Exception as e:
+ activation = None
+ kernelRMSNorm = None
+ PolyNormKernel = None
+ logger.warning_once(f"Failed to import kernel ops: {e}")
+ logger.warning_once("Using torch ops")
+
+
+class PolyNormTorch(torch.nn.Module):
+ """
+ A trainable activation function introduced in https://arxiv.org/html/2411.03884v1.
+ The code is copied from https://github.com/BryceZhuo/PolyCom?tab=readme-ov-file/README.md,
+ with the change `* torch.rsqrt` => `/ torch.sqrt`.
+ """
+
+ def __init__(self, eps=1e-6):
+ super(PolyNormTorch, self).__init__()
+ self.weight = torch.nn.Parameter(torch.ones(3) / 3)
+ self.bias = torch.nn.Parameter(torch.zeros(1))
+ self.eps = eps
+
+ def _norm(self, x):
+ return x / torch.sqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
+
+ def forward(self, x):
+ return (
+ self.weight[0] * self._norm(x**3)
+ + self.weight[1] * self._norm(x**2)
+ + self.weight[2] * self._norm(x)
+ + self.bias
+ )
+
+
+PolyNorm = PolyNormKernel if PolyNormKernel is not None else PolyNormTorch
+
+
+class GroupedPolyNorm(nn.Module):
+ """Per-expert PolyNorm: weight [num_experts, 3], bias [num_experts, 1].
+
+ Mirrors titan's GroupedExpertsPolyNorm — each expert has independent
+ polynomial normalization coefficients.
+ """
+
+ def __init__(self, num_experts: int, eps: float = 1e-6):
+ super().__init__()
+ self.num_experts = num_experts
+ self.eps = eps
+ self.weight = nn.Parameter(torch.ones(num_experts, 3) / 3)
+ self.bias = nn.Parameter(torch.zeros(num_experts, 1))
+
+ def _norm(self, x: torch.Tensor) -> torch.Tensor:
+ return x / torch.sqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
+
+ def forward_single(self, x: torch.Tensor, expert_idx: int) -> torch.Tensor:
+ w = self.weight[expert_idx] # [3]
+ b = self.bias[expert_idx] # [1]
+ return w[0] * self._norm(x**3) + w[1] * self._norm(x**2) + w[2] * self._norm(x) + b
+
+
+CUSTOM_ACT2CLS = {"poly_norm": PolyNorm}
+ACT2CLS = {**_ACT2CLS, **CUSTOM_ACT2CLS}
+ACT2FN = ClassInstantier(ACT2CLS)
+
+
+class MotifRMSNorm(nn.Module):
+ def __init__(self, hidden_size, eps=1e-6):
+ """
+ MotifRMSNorm is equivalent to T5LayerNorm
+ """
+ super().__init__()
+ self.weight = nn.Parameter(torch.ones(hidden_size))
+ self.variance_epsilon = eps
+
+ def forward(self, hidden_states):
+ input_dtype = hidden_states.dtype
+ hidden_states = hidden_states.to(torch.float32)
+ variance = hidden_states.pow(2).mean(-1, keepdim=True)
+ hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+ return self.weight * hidden_states.to(input_dtype)
+
+ def extra_repr(self):
+ return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
+
+
+class MHCLayer(nn.Module):
+ """Manifold-constrained Hyper-Connections (MHC) layer.
+
+ Written inline in the HF model (no llm_training.layers.mhc dependency).
+ apply_h_res uses a pure-PyTorch einsum instead of the Triton kernel.
+
+ Reference: https://arxiv.org/abs/2512.24880
+ """
+
+ def __init__(
+ self,
+ expansion_rate: int,
+ num_dim: int,
+ identity_init: bool = False,
+ sinkhorn_iters: int = 20,
+ ):
+ super().__init__()
+ self.expansion_rate = expansion_rate
+ self.num_dim = num_dim
+ self.sinkhorn_iters = sinkhorn_iters
+
+ E, D = expansion_rate, num_dim
+ self.proj_pre = nn.Linear(E * D, E, bias=False)
+ self.proj_post = nn.Linear(E * D, E, bias=False)
+ self.proj_res = nn.Linear(E * D, E * E, bias=False)
+
+ RMSNorm = kernelRMSNorm if kernelRMSNorm is not None else MotifRMSNorm
+ self.rms_norm = RMSNorm(E * D, eps=1e-6)
+
+ self.bias_pre = nn.Parameter(torch.empty(E))
+ self.bias_post = nn.Parameter(torch.empty(E))
+ self.bias_res = nn.Parameter(torch.empty(E, E))
+ self.alpha_pre = nn.Parameter(torch.empty(1))
+ self.alpha_post = nn.Parameter(torch.empty(1))
+ self.alpha_res = nn.Parameter(torch.empty(1))
+
+ self._init_weights(identity_init)
+
+ def _init_weights(self, identity_init: bool) -> None:
+ if hasattr(self.rms_norm, "reset_parameters"):
+ self.rms_norm.reset_parameters()
+ if identity_init:
+ nn.init.zeros_(self.alpha_pre)
+ nn.init.zeros_(self.alpha_post)
+ nn.init.zeros_(self.alpha_res)
+ nn.init.xavier_uniform_(self.proj_pre.weight)
+ nn.init.xavier_uniform_(self.proj_post.weight)
+ nn.init.xavier_uniform_(self.proj_res.weight)
+ uniform_weight = 1.0 / self.expansion_rate
+ bias_pre_value = math.log(uniform_weight / (1 - uniform_weight)) if 0 < uniform_weight < 1 else 0.0
+ nn.init.constant_(self.bias_pre, bias_pre_value)
+ nn.init.zeros_(self.bias_post)
+ nn.init.constant_(self.bias_res, -10.0)
+ self.bias_res.data.fill_diagonal_(0.0)
+ else:
+ nn.init.normal_(self.alpha_pre, mean=0.0, std=0.1)
+ nn.init.normal_(self.alpha_post, mean=0.0, std=0.1)
+ nn.init.normal_(self.alpha_res, mean=0.0, std=0.1)
+ nn.init.xavier_uniform_(self.proj_pre.weight)
+ nn.init.xavier_uniform_(self.proj_post.weight)
+ nn.init.xavier_uniform_(self.proj_res.weight)
+ nn.init.zeros_(self.bias_pre)
+ nn.init.zeros_(self.bias_post)
+ nn.init.normal_(self.bias_res, mean=0.0, std=0.1)
+
+ def _sinkhorn_knopp_batch(self, matrix: torch.Tensor) -> torch.Tensor:
+ orig_dtype = matrix.dtype
+ # Run Sinkhorn-Knopp in float32 (bf16/fp16 exp() is numerically unstable)
+ m = matrix.float().clamp(-20.0, 20.0).exp()
+ for _ in range(self.sinkhorn_iters):
+ m = m / m.sum(dim=-1, keepdim=True).clamp(min=1e-8)
+ m = m / m.sum(dim=-2, keepdim=True).clamp(min=1e-8)
+ return m.to(orig_dtype)
+
+ def forward(self, x: torch.Tensor):
+ batch_size, seq_len, expansion_rate, dim = x.shape
+ x_reshaped = x.reshape(batch_size, seq_len, expansion_rate * dim)
+ x_norm = self.rms_norm(x_reshaped)
+
+ # Cast projection outputs to float32 (paper §4.3.1)
+ proj_pre_out = self.proj_pre(x_norm).float()
+ proj_post_out = self.proj_post(x_norm).float()
+ proj_res_out = self.proj_res(x_norm).float().reshape(batch_size, seq_len, expansion_rate, expansion_rate)
+
+ h_pre = torch.sigmoid((self.alpha_pre * proj_pre_out + self.bias_pre).clamp(-10.0, 10.0))
+ h_post = 2 * torch.sigmoid((self.alpha_post * proj_post_out + self.bias_post).clamp(-10.0, 10.0))
+ h_res = self._sinkhorn_knopp_batch(self.alpha_res * proj_res_out + self.bias_res)
+
+ return h_pre, h_post, h_res
+
+ @classmethod
+ def apply_h_res(cls, x: torch.Tensor, h_res: torch.Tensor) -> torch.Tensor:
+ """h_res: (B, S, E, E), x: (B, S, E, D) -> (B, S, E, D)."""
+ return torch.einsum("bsij,bsjd->bsid", h_res, x.float()).to(x.dtype)
+
+ @classmethod
+ def apply_h_pre(cls, x: torch.Tensor, h_pre: torch.Tensor) -> torch.Tensor:
+ """Weighted sum over expansion dim: (B, S, E, D) -> (B, S, D)."""
+ return (x * h_pre.unsqueeze(-1)).sum(dim=2).to(x.dtype)
+
+ @classmethod
+ def apply_h_post(cls, x: torch.Tensor, h_post: torch.Tensor) -> torch.Tensor:
+ """Expand: (B, S, D) -> (B, S, E, D)."""
+ return (h_post.unsqueeze(-1) * x.unsqueeze(2)).to(x.dtype)
+
+ def extra_repr(self) -> str:
+ return f"expansion_rate={self.expansion_rate}, sinkhorn_iters={self.sinkhorn_iters}"
+
+
+class MotifRotaryEmbedding(nn.Module):
+ inv_freq: torch.Tensor
+
+ def __init__(self, config: MotifConfig, device=None, rope_head_dim: Optional[int] = None):
+ super().__init__()
+ # BC: "rope_type" was originally "type"
+ if hasattr(config, "rope_scaling") and isinstance(config.rope_scaling, dict):
+ self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type"))
+ else:
+ self.rope_type = "default"
+ self.max_seq_len_cached = config.max_position_embeddings
+ self.original_max_seq_len = config.max_position_embeddings
+
+ self.config = config
+ # Use rope_head_dim if provided (e.g. for GDLA which only applies RoPE to qk_rope_head_dim dims)
+ effective_head_dim = (
+ rope_head_dim
+ if rope_head_dim is not None
+ else (config.head_dim if config.head_dim is not None else config.hidden_size // config.num_attention_heads)
+ )
+ if self.rope_type == "default":
+ self.rope_init_fn = None
+ inv_freq = 1.0 / (
+ config.rope_theta
+ ** (torch.arange(0, effective_head_dim, 2, dtype=torch.int64).float().to(device) / effective_head_dim)
+ )
+ self.attention_scaling = 1.0
+
+ else:
+ self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
+ inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
+ self.original_inv_freq = self.inv_freq
+
+ def _dynamic_frequency_update(self, position_ids, device):
+ seq_len = torch.max(position_ids) + 1
+ if seq_len > self.max_seq_len_cached:
+ if self.rope_init_fn is not None:
+ inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device, seq_len=seq_len)
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
+ self.max_seq_len_cached = seq_len
+
+ if seq_len < self.original_max_seq_len and self.max_seq_len_cached > self.original_max_seq_len:
+ self.register_buffer("inv_freq", self.original_inv_freq, persistent=False)
+ self.max_seq_len_cached = self.original_max_seq_len
+
+ @torch.no_grad()
+ def forward(self, x, position_ids):
+ if "dynamic" in self.rope_type:
+ self._dynamic_frequency_update(position_ids, device=x.device)
+
+ inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device)
+ position_ids_expanded = position_ids[:, None, :].float()
+
+ device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
+ with torch.autocast(device_type=device_type, enabled=False):
+ freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
+ emb = torch.cat((freqs, freqs), dim=-1)
+ cos = emb.cos() * self.attention_scaling
+ sin = emb.sin() * self.attention_scaling
+
+ return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
+
+
+def rotate_half(x):
+ """
+ Rotates half of the dimensions of the input tensor using torch.roll and in-place negation.
+
+ Args:
+ x (torch.Tensor): The input tensor.
+
+ Returns:
+ torch.Tensor: A tensor where the latter half of the dimensions are negated
+ and moved before the first half.
+ """
+ half_size = x.shape[-1] // 2
+ rotated_tensor = torch.roll(x, shifts=-half_size, dims=-1)
+ rotated_tensor[..., :half_size] *= -1
+
+ return rotated_tensor
+
+
+def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
+ """
+ Applies rotary position embeddings to the input tensors.
+ Args:
+ q (torch.Tensor): Query tensor of shape (B, NH, S, D_KV).
+ k (torch.Tensor): Key tensor of shape (B, NH, S, D_KV).
+ cos (torch.Tensor): Cosine values for rotary embedding, shape (B, S, D) from MotifRotaryEmbedding.
+ sin (torch.Tensor): Sine values for rotary embedding, shape (B, S, D) from MotifRotaryEmbedding.
+ position_ids: Unused, kept for API compatibility.
+ unsqueeze_dim (int, optional): Dimension along which `cos` and `sin` are unsqueezed.
+ Defaults to 1 (head dimension).
+ Returns:
+ Tuple[torch.Tensor, torch.Tensor]: Transformed query and key tensors.
+ """
+ # cos/sin shape: (B, S, D) -> unsqueeze to (B, 1, S, D) for broadcasting with (B, NH, S, D)
+ cos = cos.unsqueeze(unsqueeze_dim)
+ sin = sin.unsqueeze(unsqueeze_dim)
+ q_embed = (q * cos) + (rotate_half(q) * sin)
+ k_embed = (k * cos) + (rotate_half(k) * sin)
+ return q_embed, k_embed
+
+
+def apply_rotary_pos_emb_single(
+ x: torch.Tensor,
+ cos: torch.Tensor,
+ sin: torch.Tensor,
+) -> torch.Tensor:
+ """Apply RoPE to a single tensor in (B, S, NH, D) format.
+
+ Used by GDLA to apply positional encoding only to the rope portion of Q/K.
+ cos/sin shape: (B, S, D) — broadcast over NH dimension.
+ """
+ cos = cos.unsqueeze(2) # (B, S, 1, D)
+ sin = sin.unsqueeze(2) # (B, S, 1, D)
+ return x * cos + rotate_half(x) * sin
+
+
+class MotifMLP(nn.Module):
+ def __init__(self, config, intermediate_size: int | None = None):
+ super().__init__()
+ self.hidden_size = config.hidden_size
+ self.intermediate_size = intermediate_size if intermediate_size is not None else config.intermediate_size
+
+ self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
+ self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
+ self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
+ self.act_fn = ACT2FN[config.hidden_act]
+
+ def forward(self, hidden_state):
+ hidden_state = self.act_fn(self.gate_proj(hidden_state)) * self.up_proj(hidden_state)
+ return self.down_proj(hidden_state)
+
+
+def repeat_kv(hidden_states: torch.Tensor, dim: int, n_rep: int) -> torch.Tensor:
+ return torch.repeat_interleave(hidden_states, dim=dim, repeats=n_rep)
+
+
+def eager_attention_forward(
+ module: nn.Module,
+ query: torch.Tensor,
+ key: torch.Tensor,
+ value: torch.Tensor,
+ attention_mask: Optional[torch.Tensor],
+ scaling: float,
+ dropout: float = 0.0,
+ **kwargs,
+):
+ """Eager attention forward compatible with ALL_ATTENTION_FUNCTIONS interface.
+ Expects query/key/value in [batch, num_heads, seq_len, head_dim] format.
+ """
+ attn_weights = torch.matmul(query, key.transpose(2, 3)) * scaling
+ if attention_mask is not None:
+ causal_mask = attention_mask[:, :, :, : key.shape[-2]]
+ attn_weights = attn_weights + causal_mask
+
+ attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
+ attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
+ attn_output = torch.matmul(attn_weights, value)
+ attn_output = attn_output.transpose(1, 2).contiguous()
+
+ return attn_output, attn_weights
+
+
+class MotifAttention(nn.Module):
+ """
+ Grouped Differential Attention module.
+
+ Implements Grouped Differential Attention (https://arxiv.org/pdf/2510.06949)
+ with support for eager, Flash Attention, and SDPA backends via the attention
+ function registry.
+ """
+
+ def __init__(self, config: MotifConfig, layer_idx: Optional[int] = None):
+ super().__init__()
+ self.config = config
+ self.layer_idx = layer_idx
+ if layer_idx is None:
+ logger.warning_once(
+ f"Instantiating {self.__class__.__name__} without passing `layer_idx` is not recommended and will "
+ "lead to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` "
+ "when creating this class."
+ )
+
+ self.hidden_size = config.hidden_size
+ self.num_heads = config.num_attention_heads
+ self.head_dim = self.hidden_size // self.num_heads if config.head_dim is None else config.head_dim
+ self.num_key_value_heads = config.num_key_value_heads
+ self.is_causal = True
+ self.attention_dropout = config.attention_dropout
+ self.scaling = 1.0 / math.sqrt(self.head_dim)
+
+ # Grouped Differential Transformer
+ self.num_noise_heads = config.num_noise_heads
+ self.grouped_ratio = (self.num_heads - self.num_noise_heads) // self.num_noise_heads
+ self.q_heads = (self.grouped_ratio + 1) * self.num_noise_heads
+ self.n_signal_heads = self.grouped_ratio * self.num_noise_heads # = q_heads - noise_heads
+ self.expanded = getattr(config, "expanded", False)
+
+ self.diff_v2 = getattr(config, "diff_v2", False)
+ self.elementwise_attn_output_gate = getattr(config, "elementwise_attn_output_gate", False)
+ self.headwise_attn_output_gate = getattr(config, "headwise_attn_output_gate", False)
+
+ if (self.head_dim * self.num_heads) != self.hidden_size and not self.expanded:
+ raise ValueError(
+ f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
+ f" and `num_heads`: {self.num_heads})."
+ )
+
+ # q_proj size depends on gating mode
+ if self.elementwise_attn_output_gate:
+ self.q_proj = nn.Linear(
+ self.hidden_size, (self.num_heads + self.n_signal_heads * 2) * self.head_dim, bias=False
+ )
+ elif self.headwise_attn_output_gate:
+ self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim + self.n_signal_heads, bias=False)
+ else:
+ self.q_proj = nn.Linear(self.hidden_size, self.q_heads * self.head_dim, bias=False)
+
+ self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
+
+ if self.diff_v2:
+ # V2: single V projection, smaller O projection, lambda from projection
+ self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
+ self.o_proj = nn.Linear(self.n_signal_heads * self.head_dim, self.hidden_size, bias=False)
+ self.lambda_proj = nn.Linear(self.hidden_size, self.n_signal_heads, bias=False)
+ else:
+ # V1: split V projection, subln, learnable lambda scalars
+ self.k_ratio = config.k_ratio
+ k_noise_heads = self.num_key_value_heads // (self.k_ratio + 1)
+ self.kv_repeat = self.num_noise_heads // k_noise_heads
+ self.v_proj = nn.Linear(self.hidden_size, 2 * k_noise_heads * self.head_dim, bias=False)
+ self.o_proj = nn.Linear(
+ 2 * self.grouped_ratio * self.num_noise_heads * self.head_dim, self.hidden_size, bias=False
+ )
+ self.lambda_proj = None
+ for name in ["lambda_q1", "lambda_k1", "lambda_q2", "lambda_k2"]:
+ setattr(self, name, nn.Parameter(torch.zeros(self.head_dim, dtype=torch.float32)))
+ getattr(self, name).data.normal_(mean=0.0, std=0.1)
+ RMSNorm = kernelRMSNorm if kernelRMSNorm is not None else MotifRMSNorm
+ self.subln = RMSNorm(2 * self.head_dim, eps=1e-5)
+ self.lambda_init = 0.8 - 0.6 * math.exp(-0.3 * (layer_idx - 1))
+
+ # Sliding window config for this layer (interleave pattern matching Titan)
+ if config.use_sliding_window and getattr(config, "sliding_window", None) is not None:
+ pattern = getattr(config, "sliding_window_pattern", "interleave")
+ period = getattr(config, "sliding_window_period", 2)
+ # +1 to match torchtitan convention: torchtitan passes window_size=(W, 0)
+ # directly to flash_attn (right=0 since attention is causal), but HF
+ # converts sliding_window=X to window_size=(X-1, X-1). Adding 1
+ # compensates the left side so X+1 → (X, X). The right side differs
+ # (W vs 0) but is irrelevant because causal masking already prevents
+ # attending to future tokens.
+ effective_window = config.sliding_window + 1
+ if pattern == "all":
+ self.sliding_window = effective_window
+ elif pattern == "interleave" and layer_idx % period != 0:
+ self.sliding_window = effective_window
+ else:
+ self.sliding_window = None
+ else:
+ self.sliding_window = None
+
+ def _reshape_heads(self, tensor, grouped_ratio, num_groups):
+ """2-way head split tensor reshape"""
+ tensor = einops.rearrange(
+ tensor,
+ "... (num_groups group_size) D -> ... num_groups group_size D",
+ num_groups=num_groups,
+ group_size=grouped_ratio + 1,
+ )
+ tensor1 = tensor[..., :grouped_ratio, :]
+ tensor2 = tensor[..., grouped_ratio:, :]
+ return tensor1.contiguous(), tensor2.contiguous()
+
+ def _restore_shape(self, tensor, batch_size, seq_len):
+ """restore tensor"""
+ return tensor.reshape(batch_size, seq_len, -1, self.head_dim)
+
+ def _compute_attention_via_interface(
+ self,
+ attention_interface: Callable,
+ query_states,
+ key_states,
+ value_states,
+ attention_mask,
+ dropout_rate,
+ **kwargs,
+ ):
+ """Compute attention via the registered attention interface."""
+ # Transpose to [batch, num_heads, seq_len, head_dim] for attention interface
+ q = query_states.transpose(1, 2)
+ k = key_states.transpose(1, 2)
+ v = value_states.transpose(1, 2)
+
+ attn_output, _ = attention_interface(
+ self,
+ q,
+ k,
+ v,
+ attention_mask,
+ dropout=dropout_rate,
+ scaling=self.scaling,
+ sliding_window=self.sliding_window,
+ is_causal=self.is_causal,
+ **kwargs,
+ )
+
+ # attn_output from interface is [batch, seq_len, num_heads, head_dim] (already transposed)
+ # But some interfaces return [batch, num_heads, seq_len, head_dim] - handle both
+ if attn_output.dim() == 4 and attn_output.shape[1] != query_states.shape[1]:
+ attn_output = attn_output.transpose(1, 2)
+ return attn_output
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_value: Optional[Cache] = None,
+ output_attentions: bool = False,
+ use_cache: bool = False,
+ cache_position: Optional[torch.LongTensor] = None,
+ position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+ **kwargs,
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+ bsz, q_len, _ = hidden_states.size()
+
+ # Lambda for V2 (input-dependent)
+ lambda_full = None
+ if self.diff_v2:
+ lambda_full = self.lambda_proj(hidden_states) # (bsz, q_len, n_signal_heads)
+
+ # Project Q, K, V
+ query_states = self.q_proj(hidden_states)
+ key_states = self.k_proj(hidden_states)
+ value_states = self.v_proj(hidden_states)
+
+ # Extract gate score from Q if gating is enabled
+ gate_score = None
+ if self.headwise_attn_output_gate:
+ q_flat = query_states[..., : self.num_heads * self.head_dim]
+ gate_flat = query_states[..., self.num_heads * self.head_dim :] # (bsz, q_len, n_signal_heads)
+ query_states = q_flat.view(bsz, q_len, self.num_heads, self.head_dim)
+ gate_score = gate_flat.unsqueeze(-1) # (bsz, q_len, n_signal_heads, 1)
+ elif self.elementwise_attn_output_gate:
+ query_states = query_states.view(bsz, q_len, -1, self.head_dim)
+ q_main = query_states[:, :, : self.num_heads, :]
+ gate_heads = query_states[:, :, self.num_heads :, :] # (bsz, q_len, n_signal_heads*2, head_dim)
+ gate_score = gate_heads.view(bsz, q_len, self.n_signal_heads, self.head_dim * 2)
+ query_states = q_main
+ else:
+ query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim)
+
+ key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim)
+ value_states = value_states.view(bsz, q_len, -1, self.head_dim)
+
+ # Transpose to [batch, num_heads, seq_len, head_dim] for RoPE
+ query_states = query_states.transpose(1, 2)
+ key_states = key_states.transpose(1, 2)
+ value_states = value_states.transpose(1, 2)
+
+ # Apply RoPE
+ cos, sin = position_embeddings
+ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids=position_ids)
+
+ # Handle KV cache
+ if past_key_value is not None:
+ cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
+ key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+
+ kv_seq_len = key_states.shape[-2]
+ dropout_rate = 0.0 if not self.training else self.attention_dropout
+
+ # Cast dtype if needed (PEFT float32 workaround)
+ input_dtype = query_states.dtype
+ if input_dtype == torch.float32:
+ if torch.is_autocast_enabled():
+ target_dtype = torch.get_autocast_gpu_dtype()
+ elif hasattr(self.config, "_pre_quantization_dtype"):
+ target_dtype = self.config._pre_quantization_dtype
+ else:
+ target_dtype = self.q_proj.weight.dtype
+ logger.warning_once(
+ f"The input hidden states seems to be silently casted in float32, this might be related to"
+ f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in"
+ f" {target_dtype}."
+ )
+ query_states = query_states.to(target_dtype)
+ key_states = key_states.to(target_dtype)
+ value_states = value_states.to(target_dtype)
+
+ attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(
+ self.config._attn_implementation, eager_attention_forward
+ )
+
+ if self.diff_v2:
+ attn_output = self._forward_v2(
+ attention_interface,
+ query_states,
+ key_states,
+ value_states,
+ lambda_full,
+ gate_score,
+ attention_mask,
+ dropout_rate,
+ bsz,
+ q_len,
+ **kwargs,
+ )
+ else:
+ attn_output = self._forward_v1(
+ attention_interface,
+ query_states,
+ key_states,
+ value_states,
+ gate_score,
+ attention_mask,
+ dropout_rate,
+ bsz,
+ q_len,
+ kv_seq_len,
+ **kwargs,
+ )
+
+ attn_output = attn_output.reshape(bsz, q_len, -1)
+ attn_output = self.o_proj(attn_output)
+ return attn_output, None, past_key_value
+
+ def _forward_v1(
+ self,
+ attention_interface,
+ query_states,
+ key_states,
+ value_states,
+ gate_score,
+ attention_mask,
+ dropout_rate,
+ bsz,
+ q_len,
+ kv_seq_len,
+ **kwargs,
+ ):
+ """Differential Attention V1: two flash-attn calls with learnable scalar lambda."""
+ # forward() transposes to [B, H, S, D] for RoPE; undo to [B, S, H, D]
+ # so _reshape_heads (splits H) and _compute_attention_via_interface work correctly.
+ query_states = query_states.transpose(1, 2)
+ key_states = key_states.transpose(1, 2)
+ value_states = value_states.transpose(1, 2)
+ num_groups = self.q_heads // (self.grouped_ratio + 1)
+ q1, q2 = self._reshape_heads(query_states, self.grouped_ratio, num_groups)
+
+ num_kv_groups = self.num_key_value_heads // (self.k_ratio + 1)
+ k1, k2 = self._reshape_heads(key_states, self.k_ratio, num_kv_groups)
+ v1, v2 = self._reshape_heads(value_states, 1, num_kv_groups)
+
+ q1, q2 = self._restore_shape(q1, bsz, q_len), self._restore_shape(q2, bsz, q_len)
+ k1, k2 = self._restore_shape(k1, bsz, kv_seq_len), self._restore_shape(k2, bsz, kv_seq_len)
+ v1, v2 = self._restore_shape(v1, bsz, kv_seq_len), self._restore_shape(v2, bsz, kv_seq_len)
+
+ q_f = torch.cat([q1, q2], dim=2)
+
+ k1 = repeat_kv(k1, 2, self.kv_repeat)
+ k2 = repeat_kv(k2, 2, self.kv_repeat)
+ v1 = repeat_kv(v1, 2, self.kv_repeat)
+ v2 = repeat_kv(v2, 2, self.kv_repeat)
+
+ if self.k_ratio == 1:
+ k_f = torch.cat([repeat_kv(k1, 2, self.grouped_ratio), k2], dim=2)
+ else:
+ k_f = torch.cat([k1, k2], dim=2)
+ v1_f = torch.cat([repeat_kv(v1, 2, self.grouped_ratio), v1], dim=2)
+ v2_f = torch.cat([repeat_kv(v2, 2, self.grouped_ratio), v2], dim=2)
+
+ attn_1 = self._compute_attention_via_interface(
+ attention_interface, q_f, k_f, v1_f, attention_mask, dropout_rate, **kwargs
+ )
+ attn_2 = self._compute_attention_via_interface(
+ attention_interface, q_f, k_f, v2_f, attention_mask, dropout_rate, **kwargs
+ )
+
+ merged_attn = torch.cat([attn_1, attn_2], dim=-1)
+ attn_o = merged_attn[..., :-num_kv_groups, :]
+ attn_n_group = merged_attn[..., -num_kv_groups:, :]
+ attn_n = repeat_kv(attn_n_group, 2, self.grouped_ratio)
+
+ lambda_q1 = self.lambda_q1.unsqueeze(0).expand([bsz, self.lambda_q1.shape[0]])
+ lambda_q2 = self.lambda_q2.unsqueeze(0).expand([bsz, self.lambda_q2.shape[0]])
+ lambda_1 = torch.exp(torch.sum(lambda_q1 * self.lambda_k1, dim=-1).float()).type_as(attn_o)
+ lambda_2 = torch.exp(torch.sum(lambda_q2 * self.lambda_k2, dim=-1).float()).type_as(attn_n)
+ lambda_full = lambda_1 - lambda_2 + self.lambda_init
+
+ attn_output = attn_o - lambda_full.view([bsz, 1, 1, 1]) * attn_n
+ attn_output = self.subln(attn_output)
+ attn_output = attn_output * (1 - self.lambda_init)
+
+ if gate_score is not None:
+ attn_output = attn_output * torch.sigmoid(gate_score)
+
+ expected = (bsz, q_len, self.grouped_ratio * self.num_noise_heads, self.head_dim * 2)
+ if attn_output.size() != expected:
+ raise ValueError(f"`attn_output` should be of size {expected}, but is {attn_output.size()}")
+ return attn_output
+
+ def _forward_v2(
+ self,
+ attention_interface,
+ query_states,
+ key_states,
+ value_states,
+ lambda_full,
+ gate_score,
+ attention_mask,
+ dropout_rate,
+ bsz,
+ q_len,
+ **kwargs,
+ ):
+ """Differential Attention V2: single attention call with input-dependent lambda."""
+ # forward() transposes to [B, H, S, D] for RoPE; undo to [B, S, H, D]
+ # so _compute_attention_via_interface and einops rearrange work correctly.
+ query_states = query_states.transpose(1, 2)
+ key_states = key_states.transpose(1, 2)
+ value_states = value_states.transpose(1, 2)
+ attn_output = self._compute_attention_via_interface(
+ attention_interface,
+ query_states,
+ key_states,
+ value_states,
+ attention_mask,
+ dropout_rate,
+ **kwargs,
+ )
+ # attn_output: (bsz, q_len, n_heads, head_dim)
+
+ # Split heads into signal and noise groups
+ num_groups = self.num_noise_heads # n_heads // (grouped_ratio + 1)
+ attn_reshaped = einops.rearrange(
+ attn_output,
+ "b s (g gs) d -> b s g gs d",
+ g=num_groups,
+ gs=self.grouped_ratio + 1,
+ )
+ attn1 = attn_reshaped[:, :, :, : self.grouped_ratio, :].reshape(bsz, q_len, -1, self.head_dim)
+ attn2_group = attn_reshaped[:, :, :, self.grouped_ratio :, :].reshape(bsz, q_len, num_groups, self.head_dim)
+ attn2 = repeat_kv(attn2_group, 2, self.grouped_ratio) # (bsz, q_len, n_signal_heads, head_dim)
+
+ # Differential: signal - sigmoid(lambda) * noise
+ attn_output = attn1 - torch.sigmoid(lambda_full).unsqueeze(-1) * attn2
+
+ if gate_score is not None:
+ attn_output = attn_output * torch.sigmoid(gate_score)
+
+ return attn_output
+
+
+class MotifGDLAttention(nn.Module):
+ """Grouped Differential Latent Attention (GDLA) for HF Transformers.
+
+ Ports GDLAttention from torchtitan. Uses low-rank Q and KV projections
+ (MLA-style) with RoPE applied only to the qk_rope_head_dim dimensions.
+ Only diff_v2=True is fully supported (the motif3 configuration).
+ """
+
+ def __init__(self, config: MotifConfig, layer_idx: Optional[int] = None):
+ super().__init__()
+ self.config = config
+ self.layer_idx = layer_idx
+
+ self.hidden_size = config.hidden_size
+ self.num_heads = config.num_attention_heads
+ self.num_key_value_heads = config.num_key_value_heads
+ self.head_dim = config.head_dim if config.head_dim is not None else self.hidden_size // self.num_heads
+ self.is_causal = True
+ self.attention_dropout = config.attention_dropout
+
+ # Head split
+ self.num_noise_heads = config.num_noise_heads
+ self.grouped_ratio = (self.num_heads - self.num_noise_heads) // self.num_noise_heads
+ self.n_signal_heads = self.grouped_ratio * self.num_noise_heads
+
+ # GDLA dimensions
+ self.q_lora_rank = config.q_lora_rank
+ self.kv_lora_rank = config.kv_lora_rank
+ self.qk_rope_head_dim = config.qk_rope_head_dim if config.qk_rope_head_dim is not None else self.head_dim // 2
+ self.qk_nope_head_dim = self.head_dim - self.qk_rope_head_dim
+ self.v_head_dim = config.v_head_dim if config.v_head_dim is not None else self.head_dim
+ self.diff_v2 = getattr(config, "diff_v2", True)
+
+ # Softmax scaling with optional mscale correction (DeepSeek-style)
+ self.scaling = self.head_dim**-0.5
+ original_seq_len = getattr(config, "original_seq_len", 32768)
+ rope_factor = getattr(config, "rope_factor", 1.0)
+ mscale = getattr(config, "mscale", 1.0)
+ if config.max_position_embeddings > original_seq_len:
+ mscale_val = 0.1 * mscale * math.log(rope_factor) + 1.0
+ self.scaling = self.scaling * mscale_val * mscale_val
+
+ # Output gating
+ self.elementwise_attn_output_gate = getattr(config, "elementwise_attn_output_gate", False)
+ self.headwise_attn_output_gate = getattr(config, "headwise_attn_output_gate", False)
+
+ # Required by transformers SDPA interface for GQA repeat_kv dispatch
+ self.num_key_value_groups = self.num_heads // self.num_key_value_heads
+
+ RMSNorm = kernelRMSNorm if kernelRMSNorm is not None else MotifRMSNorm
+
+ # Query LoRA
+ self.wq_a = nn.Linear(self.hidden_size, self.q_lora_rank, bias=False)
+ self.q_norm = RMSNorm(self.q_lora_rank, eps=config.rms_norm_eps)
+
+ if self.elementwise_attn_output_gate:
+ # Separate gate projection; for V2: n_signal_heads gates; for V1: n_signal_heads*2
+ gate_extra = self.n_signal_heads if self.diff_v2 else self.n_signal_heads * 2
+ self.wq_b = nn.Linear(self.q_lora_rank, self.num_heads * self.head_dim, bias=False)
+ self.wq_b_gate = nn.Linear(self.q_lora_rank, gate_extra * self.v_head_dim, bias=False)
+ else:
+ self.wq_b = nn.Linear(self.q_lora_rank, self.num_heads * self.head_dim, bias=False)
+ self.wq_b_gate = None
+
+ # KV LoRA: output kv_lora_rank + qk_rope_head_dim
+ self.wkv_a = nn.Linear(self.hidden_size, self.kv_lora_rank + self.qk_rope_head_dim, bias=False)
+
+ if self.diff_v2:
+ self.kv_norm = RMSNorm(self.kv_lora_rank, eps=config.rms_norm_eps)
+ self.wkv_b = nn.Linear(
+ self.kv_lora_rank,
+ self.num_key_value_heads * (self.qk_nope_head_dim + self.v_head_dim),
+ bias=False,
+ )
+ self.lambda_proj = nn.Linear(self.hidden_size, self.n_signal_heads, bias=False)
+ self.wo = nn.Linear(self.n_signal_heads * self.v_head_dim, self.hidden_size, bias=False)
+ else:
+ raise NotImplementedError(
+ "GDLA V1 (diff_v2=False) is not yet implemented in HF. Use attention_cls='gdla' only with diff_v2=True."
+ )
+
+ # Sliding window (same interleave pattern as basic Attention)
+ self.sliding_window = None
+ if config.use_sliding_window and getattr(config, "sliding_window", None) is not None:
+ pattern = getattr(config, "sliding_window_pattern", "interleave")
+ period = getattr(config, "sliding_window_period", 2)
+ effective_window = config.sliding_window + 1
+ if pattern == "all":
+ self.sliding_window = effective_window
+ elif pattern == "interleave" and (layer_idx + 1) % period != 0:
+ self.sliding_window = effective_window
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_value: Optional[Cache] = None,
+ output_attentions: bool = False,
+ use_cache: bool = False,
+ cache_position: Optional[torch.LongTensor] = None,
+ position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+ **kwargs,
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+ bsz, q_len, _ = hidden_states.size()
+
+ # Q path: down-project -> norm -> up-project
+ q_latent = self.q_norm(self.wq_a(hidden_states)) # (bsz, q_len, q_lora_rank)
+ q = self.wq_b(q_latent).view(bsz, q_len, self.num_heads, self.head_dim)
+
+ # Gate score (elementwise)
+ gate_score = None
+ if self.elementwise_attn_output_gate and self.wq_b_gate is not None:
+ gate_score = self.wq_b_gate(q_latent).view(bsz, q_len, -1, self.v_head_dim)
+
+ # Split Q into nope (no-positional) and rope parts
+ q_nope, q_pe = torch.split(q, [self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
+
+ # KV path: project to kv_lora_rank + qk_rope_head_dim, then split
+ kv_raw = self.wkv_a(hidden_states) # (bsz, q_len, kv_lora_rank + qk_rope_head_dim)
+ kv_latent, k_pe = torch.split(kv_raw, [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
+
+ # Apply RoPE only to the rope parts (qk_rope_head_dim dims)
+ # position_embeddings were computed with qk_rope_head_dim by MotifModel
+ cos, sin = position_embeddings # (bsz, q_len, qk_rope_head_dim)
+ q_pe = apply_rotary_pos_emb_single(q_pe, cos, sin)
+ # k_pe: (bsz, q_len, qk_rope_head_dim) -> add head dim for apply_rotary_pos_emb_single
+ k_pe = apply_rotary_pos_emb_single(k_pe.unsqueeze(2), cos, sin) # (bsz, q_len, 1, qk_rope_head_dim)
+
+ # Reconstruct full Q with nope + rope
+ q_total = torch.cat([q_nope, q_pe], dim=-1) # (bsz, q_len, n_heads, head_dim)
+
+ # KV projection: norm -> project -> split k_nope and v
+ kv_latent = kv_latent.contiguous()
+ kv_proj = self.wkv_b(self.kv_norm(kv_latent))
+ kv_proj = kv_proj.view(bsz, q_len, self.num_key_value_heads, self.qk_nope_head_dim + self.v_head_dim)
+ k_nope, v = torch.split(kv_proj, [self.qk_nope_head_dim, self.v_head_dim], dim=-1)
+
+ # Assemble full K: k_nope + k_pe (broadcast shared rope over all kv heads)
+ k_full = torch.cat([k_nope, k_pe.expand(-1, -1, self.num_key_value_heads, -1)], dim=-1)
+
+ # Lambda (input-dependent for V2)
+ lambda_full = self.lambda_proj(hidden_states) # (bsz, q_len, n_signal_heads)
+
+ # Transpose to (B, H, S, D) before cache (DynamicCache expects this format)
+ k_full = k_full.transpose(1, 2) # (bsz, n_kv_heads, q_len, head_dim)
+ v = v.transpose(1, 2) # (bsz, n_kv_heads, q_len, v_head_dim)
+
+ # KV cache
+ if past_key_value is not None:
+ cache_kwargs = {"cache_position": cache_position}
+ k_full, v = past_key_value.update(k_full, v, self.layer_idx, cache_kwargs)
+
+ dropout_rate = 0.0 if not self.training else self.attention_dropout
+
+ # Cast dtype if needed
+ input_dtype = q_total.dtype
+ if input_dtype == torch.float32:
+ if torch.is_autocast_enabled():
+ target_dtype = torch.get_autocast_gpu_dtype()
+ elif hasattr(self.config, "_pre_quantization_dtype"):
+ target_dtype = self.config._pre_quantization_dtype
+ else:
+ target_dtype = self.wq_b.weight.dtype
+ q_total = q_total.to(target_dtype)
+ k_full = k_full.to(target_dtype)
+ v = v.to(target_dtype)
+
+ # Attention: pad v to head_dim if v_head_dim != head_dim (flash_attn compatibility)
+ need_v_pad = self.v_head_dim != self.head_dim
+
+ attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(
+ self.config._attn_implementation, eager_attention_forward
+ )
+
+ # k_full and v are already (B, H, S, D); transpose q for interface
+ q_t = q_total.transpose(1, 2) # (bsz, n_heads, q_len, head_dim)
+ # q_t = q_total
+ k_t = k_full # (bsz, n_kv_heads, kv_seq_len, head_dim)
+ v_t = v # (bsz, n_kv_heads, kv_seq_len, v_head_dim)
+
+ if need_v_pad:
+ v_t = F.pad(v_t, [0, self.head_dim - self.v_head_dim])
+
+ # Truncate mask to actual kv length (sliding window cache may return fewer tokens than target_length)
+ if attention_mask is not None:
+ attention_mask = attention_mask[:, -k_t.shape[-2] :]
+
+ attn_out, _ = attention_interface(
+ self,
+ q_t,
+ k_t,
+ v_t,
+ attention_mask,
+ dropout=dropout_rate,
+ scaling=self.scaling,
+ sliding_window=self.sliding_window,
+ is_causal=self.is_causal,
+ **kwargs,
+ )
+
+ # Normalize to (B, S, H, D)
+ if attn_out.shape[1] == self.num_heads: # (B, H, S, D) from SDPA
+ attn_out = attn_out.transpose(1, 2)
+ # Now (B, S, H, D)
+
+ if need_v_pad:
+ attn_out = attn_out[..., : self.v_head_dim].contiguous()
+
+ # Split heads into signal and noise groups
+ num_groups = self.num_noise_heads # n_heads // (grouped_ratio + 1)
+ attn_reshaped = einops.rearrange(
+ attn_out,
+ "b s (g gs) d -> b s g gs d",
+ g=num_groups,
+ gs=self.grouped_ratio + 1,
+ )
+ attn1 = attn_reshaped[:, :, :, : self.grouped_ratio, :].reshape(bsz, q_len, -1, self.v_head_dim)
+ attn2_group = attn_reshaped[:, :, :, self.grouped_ratio :, :].reshape(bsz, q_len, num_groups, self.v_head_dim)
+ attn2 = repeat_kv(attn2_group, 2, self.grouped_ratio) # (bsz, q_len, n_signal_heads, v_head_dim)
+
+ # Differential combination: signal - sigmoid(lambda) * noise
+ attn_output = attn1 - torch.sigmoid(lambda_full).unsqueeze(-1) * attn2
+
+ if gate_score is not None:
+ attn_output = attn_output * torch.sigmoid(gate_score)
+
+ # Output projection
+ attn_output = attn_output.reshape(bsz, q_len, -1)
+ attn_output = self.wo(attn_output)
+
+ return attn_output, None, past_key_value
+
+
+class TokenChoiceTopKRouter(nn.Module):
+ """This class implements token-choice routing. In token-choice top-K routing, each token is
+ routed to top K experts based on the router scores.
+
+ Args:
+ dim (int): Dimension of input tokens.
+ num_experts (int): Number of experts in each moe layer.
+ experts_top_k (int): Number of experts each token will be routed to in token-choice routing.
+ score_func (Literal["softmax", "sigmoid"]): Whether to use sigmoid or softmax for router scores.
+ route_norm (bool): Whether to normalize the routing scores when using sigmoid.
+ route_scale (float): Scaling factor applied to the routing scores.
+ """
+
+ def __init__(
+ self,
+ hidden_size: int,
+ num_experts: int,
+ experts_top_k: int,
+ score_func: Literal["softmax", "sigmoid"],
+ route_norm: bool,
+ route_scale: float,
+ _debug_force_load_balance: bool = False,
+ ):
+ super().__init__()
+ self.gate = nn.Linear(hidden_size, num_experts, bias=False)
+ self.num_experts = num_experts
+ self.experts_top_k = experts_top_k
+ self.score_func = score_func
+ self.route_norm = route_norm
+ self.route_scale = route_scale
+ self._debug_force_load_balance = _debug_force_load_balance
+
+ def _debug_force_load_balance_routing(self, scores: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+ """Balanced round-robin expert assignment.
+ Returns (selected_experts_indices [N, K] LongTensor, top_scores [N, K] FloatTensor).
+ """
+ n_tokens = scores.size(0)
+ # Round-robin indices with exact balance
+ selected_experts_indices = (
+ torch.arange(n_tokens * self.experts_top_k, device=scores.device, dtype=torch.int64).reshape(
+ n_tokens, self.experts_top_k
+ )
+ % self.num_experts
+ )
+ top_scores = scores.gather(dim=1, index=selected_experts_indices) # [N,K]
+ return selected_experts_indices, top_scores
+
+ def forward(
+ self, x: torch.Tensor, expert_bias: torch.Tensor | None = None
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ """
+ Args:
+ x (torch.Tensor): Input tensor with shape ``(bs*slen, dim)``.
+ expert_bias (torch.Tensor | None, optional): Optional bias tensor for experts with shape ``(num_experts,)``.
+ Used for load balancing. Defaults to None.
+
+ Returns:
+ tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ - top_scores (torch.Tensor):
+ Routing scores for selected experts with shape ``(bs*slen, experts_top_k)``.
+ - selected_experts_indices (torch.Tensor):
+ Expert indices selected for each token with shape ``(bs*slen, experts_top_k)``.
+ - num_tokens_per_expert (torch.Tensor):
+ Number of tokens assigned to each expert with shape ``(num_experts,)``.
+ """
+ # scores shape (bs*slen, num_experts)
+ scores = self.gate(x)
+
+ # By default, sigmoid or softmax is performed in float32 to avoid loss explosion
+ if self.score_func == "sigmoid":
+ scores = torch.sigmoid(scores.to(torch.float32))
+ elif self.score_func == "softmax":
+ scores = F.softmax(scores.to(torch.float32), dim=1)
+ else:
+ raise NotImplementedError(f"Unknown score function {self.score_func}")
+
+ # top scores shape (bs*slen, experts_top_k)
+ # NOTE: The expert_bias is only used for routing. The gating value
+ # top_scores is still derived from the original scores.
+
+ if expert_bias is not None:
+ _, selected_experts_indices = torch.topk(scores + expert_bias, k=self.experts_top_k, dim=1)
+ top_scores = scores.gather(dim=1, index=selected_experts_indices)
+ else:
+ top_scores, selected_experts_indices = torch.topk(scores, k=self.experts_top_k, dim=1)
+
+ # debug override: balanced round-robin routing
+ if self._debug_force_load_balance:
+ (
+ selected_experts_indices,
+ top_scores,
+ ) = self._debug_force_load_balance_routing(scores)
+
+ if self.route_norm:
+ denominator = top_scores.sum(dim=-1, keepdim=True) + 1e-20
+ top_scores = top_scores / denominator
+ top_scores = top_scores * self.route_scale
+
+ num_tokens_per_expert = torch.bincount(selected_experts_indices.view(-1).long(), minlength=self.num_experts).to(
+ dtype=torch.float32
+ )
+
+ return top_scores, selected_experts_indices, num_tokens_per_expert
+
+ def init_weights(self, init_std: float):
+ nn.init.trunc_normal_(self.gate.weight, mean=0.0, std=init_std)
+
+
+@use_experts_implementation
+class MotifExperts(nn.Module):
+ """Collection of expert weights stored as fused 3D tensors."""
+
+ def __init__(self, config):
+ super().__init__()
+ self.num_experts = config.num_experts
+ self.hidden_size = config.hidden_size
+ moe_intermediate = getattr(config, "moe_intermediate_size", config.intermediate_size)
+ self.intermediate_dim = moe_intermediate
+
+ # Fused gate+up: [num_experts, 2*intermediate, hidden_size]
+ self.gate_up_proj = nn.Parameter(torch.empty(self.num_experts, 2 * self.intermediate_dim, self.hidden_size))
+ # Down projection: [num_experts, hidden_size, intermediate]
+ self.down_proj = nn.Parameter(torch.empty(self.num_experts, self.hidden_size, self.intermediate_dim))
+ # Per-expert poly norm (grouped) or shared activation
+ if config.hidden_act == "poly_norm":
+ self.act_fn = GroupedPolyNorm(self.num_experts)
+ else:
+ self.act_fn = ACT2FN[config.hidden_act]
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ top_k_index: torch.Tensor,
+ top_k_weights: torch.Tensor,
+ ) -> torch.Tensor:
+ """Eager expert dispatch (loops over experts).
+
+ Args:
+ hidden_states: [total_tokens, hidden_size]
+ top_k_index: [total_tokens, top_k] expert indices
+ top_k_weights: [total_tokens, top_k] routing weights
+
+ Returns:
+ [total_tokens, hidden_size]
+ """
+ final_hidden_states = torch.zeros_like(hidden_states)
+ expert_mask = F.one_hot(top_k_index, num_classes=self.num_experts).permute(2, 1, 0)
+
+ for expert_idx in range(self.num_experts):
+ top_k_pos, token_idx = torch.where(expert_mask[expert_idx])
+ if token_idx.shape[0] == 0:
+ continue
+
+ current_state = hidden_states[token_idx]
+ gate_up = current_state @ self.gate_up_proj[expert_idx].T # [T, 2*I]
+ current_hidden = self._apply_gate(gate_up, expert_idx) @ self.down_proj[expert_idx].T # [T, H]
+
+ current_hidden = current_hidden * top_k_weights[token_idx, top_k_pos, None]
+ final_hidden_states.index_add_(0, token_idx, current_hidden.to(final_hidden_states.dtype))
+
+ return final_hidden_states
+
+ def _apply_gate(self, gate_up_output: torch.Tensor, expert_idx: int = 0) -> torch.Tensor:
+ gate, up = gate_up_output.chunk(2, dim=-1)
+ # .chunk() returns non-contiguous views; kernel act_fn requires contiguous input
+ gate = gate.contiguous()
+ if isinstance(self.act_fn, GroupedPolyNorm):
+ return self.act_fn.forward_single(gate, expert_idx) * up
+ return self.act_fn(gate) * up
+
+
+class MoE(nn.Module):
+ def __init__(self, config):
+ super().__init__()
+
+ self.num_experts = config.num_experts
+ self.experts_top_k = config.experts_top_k
+
+ self.experts = MotifExperts(config)
+
+ self.router = TokenChoiceTopKRouter(
+ hidden_size=config.hidden_size,
+ num_experts=config.num_experts,
+ experts_top_k=config.experts_top_k,
+ score_func=config.score_func,
+ route_norm=config.route_norm,
+ route_scale=config.route_scale,
+ _debug_force_load_balance=config._debug_force_load_balance,
+ )
+
+ moe_intermediate = getattr(config, "moe_intermediate_size", config.intermediate_size)
+ self.shared_experts = (
+ MotifMLP(config, intermediate_size=moe_intermediate) if config.num_shared_experts > 0 else None
+ )
+ self.score_before_experts = config.score_before_experts
+
+ # Auxiliary-loss-free load balancing (https://arxiv.org/abs/2408.15664)
+ self.load_balance_coeff = config.load_balance_coeff
+ if self.load_balance_coeff is not None:
+ assert self.load_balance_coeff > 0.0
+ self.expert_bias = nn.Parameter(torch.zeros(config.num_experts, dtype=torch.float32), requires_grad=False)
+ else:
+ self.expert_bias = None
+
+ def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+ """
+ Args:
+ x (torch.Tensor): Input tensor with shape ``(bs, slen, dim)``.
+
+ Returns:
+ tuple: (output tensor (bs, slen, dim), router_logits (bs*slen, num_experts))
+ """
+ bs, slen, dim = x.shape
+ x = x.view(-1, dim)
+
+ # Route tokens
+ top_scores, selected_experts_indices, num_tokens_per_expert = self.router(x, self.expert_bias)
+
+ # Track expert usage for load balancing
+
+ # Dispatch to experts
+ if self.score_before_experts:
+ final_hidden_states = self._score_before_forward(x, selected_experts_indices, top_scores)
+ else:
+ final_hidden_states = self.experts(x, selected_experts_indices, top_scores)
+
+ # Shared experts
+ if self.shared_experts is not None:
+ final_hidden_states = final_hidden_states + self.shared_experts(x)
+
+ # Return router logits for auxiliary loss computation
+ router_logits = self.router.gate(x.view(-1, dim)) if hasattr(self.router, "gate") else None
+
+ return final_hidden_states.reshape(bs, slen, dim), router_logits
+
+ def _score_before_forward(
+ self,
+ hidden_states: torch.Tensor,
+ top_k_index: torch.Tensor,
+ top_k_weights: torch.Tensor,
+ ) -> torch.Tensor:
+ """Custom dispatch for score_before_experts mode.
+
+ Pre-weights inputs by routing scores before expert computation,
+ rather than weighting expert outputs (the standard approach).
+ """
+ final_hidden_states = torch.zeros_like(hidden_states)
+ expert_mask = F.one_hot(top_k_index, num_classes=self.num_experts).permute(2, 1, 0)
+
+ for expert_idx in range(self.num_experts):
+ top_k_pos, token_idx = torch.where(expert_mask[expert_idx])
+ if token_idx.shape[0] == 0:
+ continue
+
+ current_state = hidden_states[token_idx]
+ weights = top_k_weights[token_idx, top_k_pos, None]
+ # Pre-weight input
+ current_state = (current_state.to(torch.float32) * weights).to(hidden_states.dtype)
+
+ gate_up = current_state @ self.experts.gate_up_proj[expert_idx].T # [T, 2*I]
+ current_hidden = self.experts._apply_gate(gate_up) @ self.experts.down_proj[expert_idx].T # [T, H]
+
+ final_hidden_states.index_add_(0, token_idx, current_hidden.to(final_hidden_states.dtype))
+
+ return final_hidden_states
+
+ def init_weights(self, init_std: float, buffer_device: torch.device):
+ nn.init.trunc_normal_(self.experts.gate_up_proj, mean=0.0, std=0.02)
+ nn.init.trunc_normal_(self.experts.down_proj, mean=0.0, std=init_std)
+ self.router.init_weights(init_std)
+ if self.shared_experts is not None:
+ nn.init.trunc_normal_(self.shared_experts.gate_proj.weight, mean=0.0, std=0.02)
+ nn.init.trunc_normal_(self.shared_experts.up_proj.weight, mean=0.0, std=init_std)
+ nn.init.trunc_normal_(self.shared_experts.down_proj.weight, mean=0.0, std=init_std)
+ nn.init.zeros_(self.expert_bias)
+
+
+class MotifDecoderLayer(GradientCheckpointingLayer):
+ _ATTN_CLS = {
+ "basic": MotifAttention,
+ "gdla": MotifGDLAttention,
+ }
+
+ def __init__(self, config: MotifConfig, layer_idx: int):
+ super().__init__()
+ self.hidden_size = config.hidden_size
+
+ attention_cls_name = getattr(config, "attention_cls", "basic")
+ attn_cls = self._ATTN_CLS.get(attention_cls_name)
+ if attn_cls is None:
+ raise ValueError(f"Unknown attention_cls={attention_cls_name!r}, expected one of {list(self._ATTN_CLS)}")
+ self.self_attn = attn_cls(config, layer_idx)
+
+ # n_dense_first_layers: first N layers always dense (no MoE)
+ n_dense_first = getattr(config, "n_dense_first_layers", 0)
+ self.moe_enabled = (
+ layer_idx >= n_dense_first and (layer_idx + 1) % config.interleave_moe_layer_step == 0
+ if config.interleave_moe_layer_step != 0
+ else False
+ )
+
+ if self.moe_enabled:
+ self.moe = MoE(config)
+ else:
+ self.mlp = MotifMLP(config)
+
+ RMSNorm = kernelRMSNorm if kernelRMSNorm is not None else MotifRMSNorm
+ self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+ self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+ # MHC (Manifold-constrained Hyper-Connections) layers
+ self.mhc_enabled = getattr(config, "mhc_enabled", False)
+ if self.mhc_enabled:
+ mhc_expansion_rate = config.mhc_expansion_rate
+ self.mhc_attn = MHCLayer(
+ expansion_rate=mhc_expansion_rate,
+ num_dim=config.hidden_size,
+ identity_init=getattr(config, "mhc_identity_init", False),
+ sinkhorn_iters=getattr(config, "mhc_sinkhorn_iters", 20),
+ )
+ self.mhc_ffn = MHCLayer(
+ expansion_rate=mhc_expansion_rate,
+ num_dim=config.hidden_size,
+ identity_init=getattr(config, "mhc_identity_init", False),
+ sinkhorn_iters=getattr(config, "mhc_sinkhorn_iters", 20),
+ )
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_value: Optional[Tuple[torch.Tensor]] = None,
+ output_attentions: Optional[bool] = False,
+ use_cache: Optional[bool] = False,
+ cache_position: Optional[torch.LongTensor] = None,
+ position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46
+ **kwargs,
+ ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
+ """
+ Args:
+ hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
+ attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
+ `(batch, sequence_length)` where padding elements are indicated by 0.
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+ returned tensors for more detail.
+ use_cache (`bool`, *optional*):
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
+ (see `past_key_values`).
+ past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
+ cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
+ Indices depicting the position of the input sequence tokens in the sequence.
+ position_embeddings (`Tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
+ Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
+ with `head_dim` being the embedding dimension of each attention head.
+ kwargs (`dict`, *optional*):
+ Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code
+ into the model
+ """
+
+ if self.mhc_enabled:
+ return self._forward_with_mhc(
+ hidden_states,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_value=past_key_value,
+ output_attentions=output_attentions,
+ use_cache=use_cache,
+ cache_position=cache_position,
+ position_embeddings=position_embeddings,
+ )
+
+ residual = hidden_states
+
+ hidden_states = self.input_layernorm(hidden_states)
+
+ # Self Attention
+ hidden_states, self_attn_weights, present_key_value = self.self_attn(
+ hidden_states=hidden_states,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_value=past_key_value,
+ output_attentions=output_attentions,
+ use_cache=use_cache,
+ cache_position=cache_position,
+ position_embeddings=position_embeddings,
+ )
+ hidden_states = residual + hidden_states
+
+ # Fully Connected
+ residual = hidden_states
+ hidden_states = self.post_attention_layernorm(hidden_states)
+
+ router_logits = None
+ if self.moe_enabled:
+ hidden_states, router_logits = self.moe(hidden_states)
+ else:
+ hidden_states = self.mlp(hidden_states)
+ hidden_states = residual + hidden_states
+
+ outputs = (hidden_states,)
+
+ if output_attentions:
+ outputs += (self_attn_weights,)
+
+ if use_cache:
+ outputs += (present_key_value,)
+
+ outputs += (router_logits,)
+
+ return outputs
+
+ def _forward_with_mhc(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_value: Optional[Tuple[torch.Tensor]] = None,
+ output_attentions: Optional[bool] = False,
+ use_cache: Optional[bool] = False,
+ cache_position: Optional[torch.LongTensor] = None,
+ position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+ ) -> Tuple:
+ """MHC residual path. hidden_states: (batch, seq_len, expansion_rate, dim)."""
+ x = hidden_states
+
+ # === Attention sublayer with MHC ===
+ h_pre_attn, h_post_attn, h_res_attn = self.mhc_attn(x)
+
+ # Reduce for attention input: (B, S, E, D) -> (B, S, D)
+ x_reduced = MHCLayer.apply_h_pre(x, h_pre_attn)
+
+ attn_in = self.input_layernorm(x_reduced)
+ attn_out, self_attn_weights, present_key_value = self.self_attn(
+ hidden_states=attn_in,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_value=past_key_value,
+ output_attentions=output_attentions,
+ use_cache=use_cache,
+ cache_position=cache_position,
+ position_embeddings=position_embeddings,
+ )
+
+ # Expand attention output: (B, S, D) -> (B, S, E, D)
+ attn_expanded = MHCLayer.apply_h_post(attn_out, h_post_attn)
+ # Apply H_res to residual stream
+ x_res_attn = MHCLayer.apply_h_res(x, h_res_attn)
+ h = x_res_attn + attn_expanded
+
+ # === FFN sublayer with MHC ===
+ h_pre_ffn, h_post_ffn, h_res_ffn = self.mhc_ffn(h)
+
+ # Reduce for FFN input: (B, S, E, D) -> (B, S, D)
+ h_reduced = MHCLayer.apply_h_pre(h, h_pre_ffn)
+ n_out = self.post_attention_layernorm(h_reduced)
+
+ router_logits = None
+ if self.moe_enabled:
+ ffn_out, router_logits = self.moe(n_out)
+ else:
+ ffn_out = self.mlp(n_out)
+
+ # Expand FFN output: (B, S, D) -> (B, S, E, D)
+ ffn_expanded = MHCLayer.apply_h_post(ffn_out, h_post_ffn)
+ h_res_ffn_out = MHCLayer.apply_h_res(h, h_res_ffn)
+ out = h_res_ffn_out + ffn_expanded
+
+ outputs = (out,)
+
+ if output_attentions:
+ outputs += (self_attn_weights,)
+
+ if use_cache:
+ outputs += (present_key_value,)
+
+ outputs += (router_logits,)
+
+ return outputs
+
+
+@auto_docstring
+class MotifPreTrainedModel(PreTrainedModel):
+ config_class = MotifConfig
+ base_model_prefix = "model"
+ supports_gradient_checkpointing = True
+ _no_split_modules = ["MotifDecoderLayer"]
+ _skip_keys_device_placement = "past_key_values"
+ _supports_flash_attn = True
+ _supports_sdpa = True
+ _supports_flex_attn = True
+ _supports_attention_backend = True
+ _supports_cache_class = True
+ _supports_quantized_cache = True
+ _supports_static_cache = True
+
+ def _init_weights(self, module):
+ std = self.config.initializer_range
+ if isinstance(module, nn.Linear):
+ module.weight.data = torch.where(abs(module.weight.data) > 3 * std, 0, module.weight.data)
+ if module.bias is not None:
+ module.bias.data.zero_()
+ elif isinstance(module, nn.Embedding):
+ module.weight.data = torch.where(abs(module.weight.data) > 3 * std, 0, module.weight.data)
+ if module.padding_idx is not None:
+ module.weight.data[module.padding_idx].zero_()
+ elif isinstance(module, MoE):
+ module.init_weights(std, buffer_device=torch.device("cpu"))
+ elif isinstance(module, MotifRotaryEmbedding):
+ effective_head_dim = module.inv_freq.shape[0] * 2
+ inv_freq = 1.0 / (
+ self.config.rope_theta
+ ** (torch.arange(0, effective_head_dim, 2, dtype=torch.int64).float() / effective_head_dim)
+ )
+ module.register_buffer("inv_freq", inv_freq, persistent=False)
+
+
+@auto_docstring
+class MotifModel(MotifPreTrainedModel):
+ """
+ Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MotifDecoderLayer`]
+
+ Args:
+ config: MotifConfig
+ """
+
+ def __init__(self, config: MotifConfig):
+ super().__init__(config)
+ self.padding_idx = getattr(config, "pad_token_id", None)
+ self.vocab_size = config.vocab_size
+
+ self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
+ self.layers = nn.ModuleList(
+ [MotifDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
+ )
+ self._attn_implementation = config._attn_implementation
+ RMSNorm = kernelRMSNorm if kernelRMSNorm is not None else MotifRMSNorm
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+ # GDLA applies RoPE only to qk_rope_head_dim dimensions
+ rope_head_dim = (
+ getattr(config, "qk_rope_head_dim", None) if getattr(config, "attention_cls", "basic") == "gdla" else None
+ )
+ self.rotary_emb = MotifRotaryEmbedding(config=config, rope_head_dim=rope_head_dim)
+
+ self.mhc_enabled = getattr(config, "mhc_enabled", False)
+ self.mhc_expansion_rate = getattr(config, "mhc_expansion_rate", 4)
+
+ self.gradient_checkpointing = False
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self):
+ return self.embed_tokens
+
+ def set_input_embeddings(self, value):
+ self.embed_tokens = value
+
+ @can_return_tuple
+ @auto_docstring
+ def forward(
+ self,
+ input_ids: torch.LongTensor = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_values: Optional[Cache] = None,
+ inputs_embeds: Optional[torch.FloatTensor] = None,
+ use_cache: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ output_router_logits: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ cache_position: Optional[torch.LongTensor] = None,
+ ) -> MoeModelOutputWithPast:
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ output_router_logits = (
+ output_router_logits
+ if output_router_logits is not None
+ else getattr(self.config, "output_router_logits", False)
+ )
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
+
+ if (input_ids is None) ^ (inputs_embeds is not None):
+ raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
+
+ if self.gradient_checkpointing and self.training:
+ if use_cache:
+ logger.warning_once(
+ "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
+ )
+ use_cache = False
+
+ if use_cache and past_key_values is None:
+ past_key_values = DynamicCache()
+
+ if inputs_embeds is None:
+ inputs_embeds = self.embed_tokens(input_ids)
+
+ if cache_position is None:
+ past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
+ cache_position = torch.arange(
+ past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device
+ )
+ if position_ids is None:
+ position_ids = cache_position.unsqueeze(0)
+
+ causal_mask = self._update_causal_mask(
+ attention_mask, inputs_embeds, cache_position, past_key_values, output_attentions
+ )
+
+ hidden_states = inputs_embeds
+
+ # Create position embeddings BEFORE MHC expansion (uses (B, S, D) for dtype/device)
+ position_embeddings = self.rotary_emb(hidden_states, position_ids)
+
+ # Expand to (B, S, E, D) for MHC
+ if self.mhc_enabled:
+ hidden_states = hidden_states.unsqueeze(2).expand(-1, -1, self.mhc_expansion_rate, -1).contiguous()
+
+ # Decoder layers
+ all_hidden_states = () if output_hidden_states else None
+ all_self_attns = () if output_attentions else None
+ all_router_logits = () if output_router_logits else None
+ next_decoder_cache = None
+
+ for decoder_layer in self.layers:
+ if output_hidden_states:
+ all_hidden_states += (hidden_states,)
+
+ layer_outputs = decoder_layer(
+ hidden_states,
+ attention_mask=causal_mask,
+ position_ids=position_ids,
+ past_key_value=past_key_values,
+ output_attentions=output_attentions,
+ use_cache=use_cache,
+ cache_position=cache_position,
+ position_embeddings=position_embeddings,
+ )
+
+ hidden_states = layer_outputs[0]
+
+ if use_cache:
+ next_decoder_cache = layer_outputs[2 if output_attentions else 1]
+
+ if output_attentions:
+ all_self_attns += (layer_outputs[1],)
+
+ # Router logits are always the last element
+ if output_router_logits:
+ all_router_logits += (layer_outputs[-1],)
+
+ # Reduce from (B, S, E, D) back to (B, S, D) for MHC
+ if self.mhc_enabled:
+ hidden_states = hidden_states.mean(dim=2)
+
+ hidden_states = self.norm(hidden_states)
+
+ # Add hidden states from the last decoder layer
+ if output_hidden_states:
+ all_hidden_states += (hidden_states,)
+
+ next_cache = next_decoder_cache if use_cache else None
+
+ return MoeModelOutputWithPast(
+ last_hidden_state=hidden_states,
+ past_key_values=next_cache,
+ hidden_states=all_hidden_states,
+ attentions=all_self_attns,
+ router_logits=all_router_logits,
+ )
+
+ def _update_causal_mask(
+ self,
+ attention_mask: torch.Tensor,
+ input_tensor: torch.Tensor,
+ cache_position: torch.Tensor,
+ past_key_values: Cache,
+ output_attentions: bool,
+ ):
+ if self.config._attn_implementation == "flash_attention_2":
+ if attention_mask is not None and 0.0 in attention_mask:
+ return attention_mask
+ return None
+
+ # For SDPA, when possible, we will rely on its `is_causal` argument instead of its `attn_mask` argument, in
+ # order to dispatch on Flash Attention 2. This feature is not compatible with static cache, as SDPA will fail
+ # to infer the attention mask.
+ past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
+ using_static_cache = isinstance(past_key_values, StaticCache)
+
+ # When output attentions is True, sdpa implementation's forward method calls the eager implementation's forward
+ if self.config._attn_implementation == "sdpa" and not using_static_cache and not output_attentions:
+ if AttentionMaskConverter._ignore_causal_mask_sdpa(
+ attention_mask,
+ inputs_embeds=input_tensor,
+ past_key_values_length=past_seen_tokens,
+ sliding_window=self.config.sliding_window,
+ is_training=self.training,
+ ):
+ return None
+
+ dtype, device = input_tensor.dtype, input_tensor.device
+ min_dtype = torch.finfo(dtype).min
+ sequence_length = input_tensor.shape[1]
+ # StaticCache
+ if using_static_cache:
+ target_length = past_key_values.get_max_cache_shape()
+ # DynamicCache or no cache
+ else:
+ target_length = (
+ attention_mask.shape[-1]
+ if isinstance(attention_mask, torch.Tensor)
+ else past_seen_tokens + sequence_length
+ )
+
+ # In case the provided `attention` mask is 2D, we generate a causal mask here (4D).
+ causal_mask = self._prepare_4d_causal_attention_mask_with_cache_position(
+ attention_mask,
+ sequence_length=sequence_length,
+ target_length=target_length,
+ dtype=dtype,
+ device=device,
+ cache_position=cache_position,
+ batch_size=input_tensor.shape[0],
+ config=self.config,
+ past_key_values=past_key_values,
+ )
+
+ if (
+ self.config._attn_implementation == "sdpa"
+ and attention_mask is not None
+ and attention_mask.device.type == "cuda"
+ and not output_attentions
+ ):
+ # Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when
+ # using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path.
+ # Details: https://github.com/pytorch/pytorch/issues/110213
+ causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype)
+
+ return causal_mask
+
+ @staticmethod
+ def _prepare_4d_causal_attention_mask_with_cache_position(
+ attention_mask: torch.Tensor,
+ sequence_length: int,
+ target_length: int,
+ dtype: torch.dtype,
+ device: torch.device,
+ cache_position: torch.Tensor,
+ batch_size: int,
+ config: MotifConfig,
+ past_key_values: Cache,
+ ):
+ """
+ Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
+ `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.
+
+ Args:
+ attention_mask (`torch.Tensor`):
+ A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape `(batch_size, 1, query_length, key_value_length)`.
+ sequence_length (`int`):
+ The sequence length being processed.
+ target_length (`int`):
+ The target length: when generating with static cache, the mask should be as long as the static cache, to account for the 0 padding, the part of the cache that is not filled yet.
+ dtype (`torch.dtype`):
+ The dtype to use for the 4D attention mask.
+ device (`torch.device`):
+ The device to plcae the 4D attention mask on.
+ cache_position (`torch.Tensor`):
+ Indices depicting the position of the input sequence tokens in the sequence.
+ batch_size (`torch.Tensor`):
+ Batch size.
+ config (`MotifConfig`):
+ The model's configuration class
+ past_key_values (`Cache`):
+ The cache class that is being used currently to generate
+ """
+ if attention_mask is not None and attention_mask.dim() == 4:
+ # In this case we assume that the mask comes already in inverted form and requires no inversion or slicing.
+ causal_mask = attention_mask
+ else:
+ min_dtype = torch.finfo(dtype).min
+ causal_mask = torch.full(
+ (sequence_length, target_length), fill_value=min_dtype, dtype=dtype, device=cache_position.device
+ )
+ diagonal_attend_mask = torch.arange(target_length, device=cache_position.device) > cache_position.reshape(
+ -1, 1
+ )
+ if config.sliding_window is not None:
+ # if we have sliding window, we should not attend to tokens beyond sliding window length, so we mask them out also
+ # the check is needed to verify is current checkpoint was trained with sliding window or not
+ if sequence_length > target_length:
+ sliding_attend_mask = torch.arange(target_length, device=device) <= (
+ cache_position.reshape(-1, 1) - config.sliding_window
+ )
+ diagonal_attend_mask.bitwise_or_(sliding_attend_mask)
+ causal_mask *= diagonal_attend_mask
+ causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1)
+ if attention_mask is not None:
+ causal_mask = causal_mask.clone() # copy to contiguous memory for in-place edit
+ if attention_mask.shape[-1] > target_length:
+ attention_mask = attention_mask[:, :target_length]
+ mask_length = attention_mask.shape[-1]
+ padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :]
+ padding_mask = padding_mask == 0
+ causal_mask[:, :, :, :mask_length] = causal_mask[:, :, :, :mask_length].masked_fill(
+ padding_mask, min_dtype
+ )
+ return causal_mask
+
+
+class MotifForCausalLM(MotifPreTrainedModel, GenerationMixin):
+ _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"}
+ _tp_plan = {"lm_head": "colwise_gather_output"}
+ _pp_plan = {"lm_head": (["hidden_states"], ["logits"])}
+
+ def __init__(self, config):
+ super().__init__(config)
+ self.model = MotifModel(config)
+ self.vocab_size = config.vocab_size
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ if config.tie_word_embeddings:
+ self.tie_weights()
+
+ def get_input_embeddings(self):
+ return self.model.embed_tokens
+
+ def set_input_embeddings(self, value):
+ self.model.embed_tokens = value
+
+ def get_output_embeddings(self):
+ return self.lm_head
+
+ def set_output_embeddings(self, new_embeddings):
+ self.lm_head = new_embeddings
+
+ def set_decoder(self, decoder):
+ self.model = decoder
+
+ def get_decoder(self):
+ return self.model
+
+ @can_return_tuple
+ @auto_docstring
+ def forward(
+ self,
+ input_ids: torch.LongTensor = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_values: Optional[Cache] = None,
+ inputs_embeds: Optional[torch.FloatTensor] = None,
+ labels: Optional[torch.LongTensor] = None,
+ use_cache: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ output_router_logits: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ cache_position: Optional[torch.LongTensor] = None,
+ logits_to_keep: int = 0,
+ **kwargs,
+ ) -> MoeCausalLMOutputWithPast:
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ output_router_logits = (
+ output_router_logits
+ if output_router_logits is not None
+ else getattr(self.config, "output_router_logits", False)
+ )
+
+ # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+ outputs = self.model(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_values=past_key_values,
+ inputs_embeds=inputs_embeds,
+ use_cache=use_cache,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ output_router_logits=output_router_logits,
+ cache_position=cache_position,
+ )
+
+ hidden_states = outputs[0]
+ # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
+ logits = self.lm_head(hidden_states[:, -logits_to_keep:, :])
+ logits = logits.float()
+
+ loss = None
+ if labels is not None:
+ # Shift so that tokens < n predict n
+ shift_logits = logits[..., :-1, :].contiguous()
+ shift_labels = labels[..., 1:].contiguous()
+ # Flatten the tokens
+ loss_fct = CrossEntropyLoss()
+ shift_logits = shift_logits.view(-1, self.config.vocab_size)
+ shift_labels = shift_labels.view(-1)
+ # Enable model parallelism
+ shift_labels = shift_labels.to(shift_logits.device)
+ loss = loss_fct(shift_logits, shift_labels)
+
+ return MoeCausalLMOutputWithPast(
+ loss=loss,
+ logits=logits,
+ past_key_values=outputs.past_key_values,
+ hidden_states=outputs.hidden_states,
+ attentions=outputs.attentions,
+ router_logits=outputs.router_logits,
+ )
diff --git a/nvfp4_act_scales.safetensors b/nvfp4_act_scales.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..5bbcd96e83ecc21a7d152d6946dde7ce751b4925
--- /dev/null
+++ b/nvfp4_act_scales.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7cbea201c128079fdbe6b106a11c49499e65d07b963619900544d14b670c24b
+size 166872
diff --git a/tokenizer.json b/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..14eecbd297baafcafa452972f9dc274ad9622546
--- /dev/null
+++ b/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:956d8a693e0ede5283c803aba3fe1d4b46b202d9faac007f947c6f5b61ce7864
+size 17548202
diff --git a/tokenizer_config.json b/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..dc2c2f123b7c5768d156b584234915ecb37546d4
--- /dev/null
+++ b/tokenizer_config.json
@@ -0,0 +1,27 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|beginoftext|>",
+ "eos_token": "<|endoftext|>",
+ "extra_special_tokens": [
+ "<|system|>",
+ "<|user|>",
+ "<|assistant|>",
+ "<|startofturn|>",
+ "<|endofturn|>",
+ "<|tool|>",
+ "<|reference|>",
+ "<|plan|>",
+ "<|endofplan|>",
+ "",
+ "",
+ "",
+ "",
+ ""
+ ],
+ "model_max_length": 1000000000000000019884624838656,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "left",
+ "split_special_tokens": false,
+ "tokenizer_class": "TokenizersBackend",
+ "truncation_side": "left"
+}