| { |
| "format": "native_hf_identity_v3", |
| "translation_applied": false, |
| "source": "GDNMCAForCausalLM.state_dict() trained directly", |
| "tensor_count": 578, |
| "tensors": { |
| "model.embed_tokens.proj": { |
| "hf_key": "model.embed_tokens.proj", |
| "shape": [ |
| 768, |
| 256 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.embed_tokens.token_weight.weight": { |
| "hf_key": "model.embed_tokens.token_weight.weight", |
| "shape": [ |
| 151665, |
| 256 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.norm1.weight": { |
| "hf_key": "model.layers.0.norm1.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.mixer.gdn.A_log": { |
| "hf_key": "model.layers.0.mixer.gdn.A_log", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.mixer.gdn.dt_bias": { |
| "hf_key": "model.layers.0.mixer.gdn.dt_bias", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.mixer.gdn.q_proj.weight": { |
| "hf_key": "model.layers.0.mixer.gdn.q_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.mixer.gdn.k_proj.weight": { |
| "hf_key": "model.layers.0.mixer.gdn.k_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.mixer.gdn.v_proj.weight": { |
| "hf_key": "model.layers.0.mixer.gdn.v_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.mixer.gdn.a_proj.weight": { |
| "hf_key": "model.layers.0.mixer.gdn.a_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.mixer.gdn.b_proj.weight": { |
| "hf_key": "model.layers.0.mixer.gdn.b_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.mixer.gdn.q_conv1d.weight": { |
| "hf_key": "model.layers.0.mixer.gdn.q_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.mixer.gdn.k_conv1d.weight": { |
| "hf_key": "model.layers.0.mixer.gdn.k_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.mixer.gdn.v_conv1d.weight": { |
| "hf_key": "model.layers.0.mixer.gdn.v_conv1d.weight", |
| "shape": [ |
| 6144, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.mixer.gdn.g_proj.weight": { |
| "hf_key": "model.layers.0.mixer.gdn.g_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.mixer.gdn.o_norm.weight": { |
| "hf_key": "model.layers.0.mixer.gdn.o_norm.weight", |
| "shape": [ |
| 512 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.mixer.gdn.o_proj.weight": { |
| "hf_key": "model.layers.0.mixer.gdn.o_proj.weight", |
| "shape": [ |
| 768, |
| 6144 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.residual1.mix_logits": { |
| "hf_key": "model.layers.0.residual1.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.residual1.update_gain": { |
| "hf_key": "model.layers.0.residual1.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.residual1.context_gate.weight": { |
| "hf_key": "model.layers.0.residual1.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.residual1.context_gate.bias": { |
| "hf_key": "model.layers.0.residual1.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.norm2.weight": { |
| "hf_key": "model.layers.0.norm2.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.mlp.gate_proj.weight": { |
| "hf_key": "model.layers.0.mlp.gate_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.mlp.up_proj.weight": { |
| "hf_key": "model.layers.0.mlp.up_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.mlp.down_proj.weight": { |
| "hf_key": "model.layers.0.mlp.down_proj.weight", |
| "shape": [ |
| 768, |
| 2816 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.residual2.mix_logits": { |
| "hf_key": "model.layers.0.residual2.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.residual2.update_gain": { |
| "hf_key": "model.layers.0.residual2.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.residual2.context_gate.weight": { |
| "hf_key": "model.layers.0.residual2.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.0.residual2.context_gate.bias": { |
| "hf_key": "model.layers.0.residual2.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.norm1.weight": { |
| "hf_key": "model.layers.1.norm1.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.mixer.gdn.A_log": { |
| "hf_key": "model.layers.1.mixer.gdn.A_log", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.mixer.gdn.dt_bias": { |
| "hf_key": "model.layers.1.mixer.gdn.dt_bias", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.mixer.gdn.q_proj.weight": { |
| "hf_key": "model.layers.1.mixer.gdn.q_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.mixer.gdn.k_proj.weight": { |
| "hf_key": "model.layers.1.mixer.gdn.k_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.mixer.gdn.v_proj.weight": { |
| "hf_key": "model.layers.1.mixer.gdn.v_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.mixer.gdn.a_proj.weight": { |
| "hf_key": "model.layers.1.mixer.gdn.a_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.mixer.gdn.b_proj.weight": { |
| "hf_key": "model.layers.1.mixer.gdn.b_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.mixer.gdn.q_conv1d.weight": { |
| "hf_key": "model.layers.1.mixer.gdn.q_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.mixer.gdn.k_conv1d.weight": { |
| "hf_key": "model.layers.1.mixer.gdn.k_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.mixer.gdn.v_conv1d.weight": { |
| "hf_key": "model.layers.1.mixer.gdn.v_conv1d.weight", |
| "shape": [ |
| 6144, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.mixer.gdn.g_proj.weight": { |
| "hf_key": "model.layers.1.mixer.gdn.g_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.mixer.gdn.o_norm.weight": { |
| "hf_key": "model.layers.1.mixer.gdn.o_norm.weight", |
| "shape": [ |
| 512 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.mixer.gdn.o_proj.weight": { |
| "hf_key": "model.layers.1.mixer.gdn.o_proj.weight", |
| "shape": [ |
| 768, |
| 6144 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.residual1.mix_logits": { |
| "hf_key": "model.layers.1.residual1.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.residual1.update_gain": { |
| "hf_key": "model.layers.1.residual1.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.residual1.context_gate.weight": { |
| "hf_key": "model.layers.1.residual1.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.residual1.context_gate.bias": { |
| "hf_key": "model.layers.1.residual1.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.norm2.weight": { |
| "hf_key": "model.layers.1.norm2.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.mlp.gate_proj.weight": { |
| "hf_key": "model.layers.1.mlp.gate_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.mlp.up_proj.weight": { |
| "hf_key": "model.layers.1.mlp.up_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.mlp.down_proj.weight": { |
| "hf_key": "model.layers.1.mlp.down_proj.weight", |
| "shape": [ |
| 768, |
| 2816 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.residual2.mix_logits": { |
| "hf_key": "model.layers.1.residual2.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.residual2.update_gain": { |
| "hf_key": "model.layers.1.residual2.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.residual2.context_gate.weight": { |
| "hf_key": "model.layers.1.residual2.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.1.residual2.context_gate.bias": { |
| "hf_key": "model.layers.1.residual2.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.norm1.weight": { |
| "hf_key": "model.layers.2.norm1.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.mixer.gdn.A_log": { |
| "hf_key": "model.layers.2.mixer.gdn.A_log", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.mixer.gdn.dt_bias": { |
| "hf_key": "model.layers.2.mixer.gdn.dt_bias", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.mixer.gdn.q_proj.weight": { |
| "hf_key": "model.layers.2.mixer.gdn.q_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.mixer.gdn.k_proj.weight": { |
| "hf_key": "model.layers.2.mixer.gdn.k_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.mixer.gdn.v_proj.weight": { |
| "hf_key": "model.layers.2.mixer.gdn.v_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.mixer.gdn.a_proj.weight": { |
| "hf_key": "model.layers.2.mixer.gdn.a_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.mixer.gdn.b_proj.weight": { |
| "hf_key": "model.layers.2.mixer.gdn.b_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.mixer.gdn.q_conv1d.weight": { |
| "hf_key": "model.layers.2.mixer.gdn.q_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.mixer.gdn.k_conv1d.weight": { |
| "hf_key": "model.layers.2.mixer.gdn.k_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.mixer.gdn.v_conv1d.weight": { |
| "hf_key": "model.layers.2.mixer.gdn.v_conv1d.weight", |
| "shape": [ |
| 6144, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.mixer.gdn.g_proj.weight": { |
| "hf_key": "model.layers.2.mixer.gdn.g_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.mixer.gdn.o_norm.weight": { |
| "hf_key": "model.layers.2.mixer.gdn.o_norm.weight", |
| "shape": [ |
| 512 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.mixer.gdn.o_proj.weight": { |
| "hf_key": "model.layers.2.mixer.gdn.o_proj.weight", |
| "shape": [ |
| 768, |
| 6144 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.residual1.mix_logits": { |
| "hf_key": "model.layers.2.residual1.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.residual1.update_gain": { |
| "hf_key": "model.layers.2.residual1.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.residual1.context_gate.weight": { |
| "hf_key": "model.layers.2.residual1.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.residual1.context_gate.bias": { |
| "hf_key": "model.layers.2.residual1.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.norm2.weight": { |
| "hf_key": "model.layers.2.norm2.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.mlp.gate_proj.weight": { |
| "hf_key": "model.layers.2.mlp.gate_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.mlp.up_proj.weight": { |
| "hf_key": "model.layers.2.mlp.up_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.mlp.down_proj.weight": { |
| "hf_key": "model.layers.2.mlp.down_proj.weight", |
| "shape": [ |
| 768, |
| 2816 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.residual2.mix_logits": { |
| "hf_key": "model.layers.2.residual2.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.residual2.update_gain": { |
| "hf_key": "model.layers.2.residual2.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.residual2.context_gate.weight": { |
| "hf_key": "model.layers.2.residual2.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.2.residual2.context_gate.bias": { |
| "hf_key": "model.layers.2.residual2.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.norm1.weight": { |
| "hf_key": "model.layers.3.norm1.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.route_bias": { |
| "hf_key": "model.layers.3.mixer.route_bias", |
| "shape": [ |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.route_load_ema": { |
| "hf_key": "model.layers.3.mixer.route_load_ema", |
| "shape": [ |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.mla.q_a.weight": { |
| "hf_key": "model.layers.3.mixer.mla.q_a.weight", |
| "shape": [ |
| 192, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.mla.q_norm.weight": { |
| "hf_key": "model.layers.3.mixer.mla.q_norm.weight", |
| "shape": [ |
| 192 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.mla.q_b.weight": { |
| "hf_key": "model.layers.3.mixer.mla.q_b.weight", |
| "shape": [ |
| 768, |
| 192 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.mla.kv_a.weight": { |
| "hf_key": "model.layers.3.mixer.mla.kv_a.weight", |
| "shape": [ |
| 192, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.mla.kv_norm.weight": { |
| "hf_key": "model.layers.3.mixer.mla.kv_norm.weight", |
| "shape": [ |
| 192 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.mla.kv_b.weight": { |
| "hf_key": "model.layers.3.mixer.mla.kv_b.weight", |
| "shape": [ |
| 1152, |
| 192 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.mla.k_rope.weight": { |
| "hf_key": "model.layers.3.mixer.mla.k_rope.weight", |
| "shape": [ |
| 32, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.mla.out_proj.weight": { |
| "hf_key": "model.layers.3.mixer.mla.out_proj.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.mem_q.weight": { |
| "hf_key": "model.layers.3.mixer.mem_q.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.memory.initial_fast": { |
| "hf_key": "model.layers.3.mixer.memory.initial_fast", |
| "shape": [ |
| 32, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.memory.initial_slow": { |
| "hf_key": "model.layers.3.mixer.memory.initial_slow", |
| "shape": [ |
| 16, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.memory.write_query.weight": { |
| "hf_key": "model.layers.3.mixer.memory.write_query.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.memory.fast_slot_key.weight": { |
| "hf_key": "model.layers.3.mixer.memory.fast_slot_key.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.memory.slow_slot_key.weight": { |
| "hf_key": "model.layers.3.mixer.memory.slow_slot_key.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.memory.value_proj.weight": { |
| "hf_key": "model.layers.3.mixer.memory.value_proj.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.memory.fast_gate.weight": { |
| "hf_key": "model.layers.3.mixer.memory.fast_gate.weight", |
| "shape": [ |
| 768, |
| 1536 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.memory.fast_gate.bias": { |
| "hf_key": "model.layers.3.mixer.memory.fast_gate.bias", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.memory.slow_gate.weight": { |
| "hf_key": "model.layers.3.mixer.memory.slow_gate.weight", |
| "shape": [ |
| 768, |
| 1536 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.memory.slow_gate.bias": { |
| "hf_key": "model.layers.3.mixer.memory.slow_gate.bias", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.memory.read_k.weight": { |
| "hf_key": "model.layers.3.mixer.memory.read_k.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.memory.read_v.weight": { |
| "hf_key": "model.layers.3.mixer.memory.read_v.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mixer.router.weight": { |
| "hf_key": "model.layers.3.mixer.router.weight", |
| "shape": [ |
| 2, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.residual1.mix_logits": { |
| "hf_key": "model.layers.3.residual1.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.residual1.update_gain": { |
| "hf_key": "model.layers.3.residual1.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.residual1.context_gate.weight": { |
| "hf_key": "model.layers.3.residual1.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.residual1.context_gate.bias": { |
| "hf_key": "model.layers.3.residual1.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.norm2.weight": { |
| "hf_key": "model.layers.3.norm2.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mlp.gate_proj.weight": { |
| "hf_key": "model.layers.3.mlp.gate_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mlp.up_proj.weight": { |
| "hf_key": "model.layers.3.mlp.up_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.mlp.down_proj.weight": { |
| "hf_key": "model.layers.3.mlp.down_proj.weight", |
| "shape": [ |
| 768, |
| 2816 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.residual2.mix_logits": { |
| "hf_key": "model.layers.3.residual2.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.residual2.update_gain": { |
| "hf_key": "model.layers.3.residual2.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.residual2.context_gate.weight": { |
| "hf_key": "model.layers.3.residual2.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.3.residual2.context_gate.bias": { |
| "hf_key": "model.layers.3.residual2.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.norm1.weight": { |
| "hf_key": "model.layers.4.norm1.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.mixer.gdn.A_log": { |
| "hf_key": "model.layers.4.mixer.gdn.A_log", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.mixer.gdn.dt_bias": { |
| "hf_key": "model.layers.4.mixer.gdn.dt_bias", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.mixer.gdn.q_proj.weight": { |
| "hf_key": "model.layers.4.mixer.gdn.q_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.mixer.gdn.k_proj.weight": { |
| "hf_key": "model.layers.4.mixer.gdn.k_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.mixer.gdn.v_proj.weight": { |
| "hf_key": "model.layers.4.mixer.gdn.v_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.mixer.gdn.a_proj.weight": { |
| "hf_key": "model.layers.4.mixer.gdn.a_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.mixer.gdn.b_proj.weight": { |
| "hf_key": "model.layers.4.mixer.gdn.b_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.mixer.gdn.q_conv1d.weight": { |
| "hf_key": "model.layers.4.mixer.gdn.q_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.mixer.gdn.k_conv1d.weight": { |
| "hf_key": "model.layers.4.mixer.gdn.k_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.mixer.gdn.v_conv1d.weight": { |
| "hf_key": "model.layers.4.mixer.gdn.v_conv1d.weight", |
| "shape": [ |
| 6144, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.mixer.gdn.g_proj.weight": { |
| "hf_key": "model.layers.4.mixer.gdn.g_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.mixer.gdn.o_norm.weight": { |
| "hf_key": "model.layers.4.mixer.gdn.o_norm.weight", |
| "shape": [ |
| 512 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.mixer.gdn.o_proj.weight": { |
| "hf_key": "model.layers.4.mixer.gdn.o_proj.weight", |
| "shape": [ |
| 768, |
| 6144 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.residual1.mix_logits": { |
| "hf_key": "model.layers.4.residual1.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.residual1.update_gain": { |
| "hf_key": "model.layers.4.residual1.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.residual1.context_gate.weight": { |
| "hf_key": "model.layers.4.residual1.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.residual1.context_gate.bias": { |
| "hf_key": "model.layers.4.residual1.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.norm2.weight": { |
| "hf_key": "model.layers.4.norm2.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.mlp.gate_proj.weight": { |
| "hf_key": "model.layers.4.mlp.gate_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.mlp.up_proj.weight": { |
| "hf_key": "model.layers.4.mlp.up_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.mlp.down_proj.weight": { |
| "hf_key": "model.layers.4.mlp.down_proj.weight", |
| "shape": [ |
| 768, |
| 2816 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.residual2.mix_logits": { |
| "hf_key": "model.layers.4.residual2.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.residual2.update_gain": { |
| "hf_key": "model.layers.4.residual2.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.residual2.context_gate.weight": { |
| "hf_key": "model.layers.4.residual2.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.4.residual2.context_gate.bias": { |
| "hf_key": "model.layers.4.residual2.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.norm1.weight": { |
| "hf_key": "model.layers.5.norm1.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.mixer.gdn.A_log": { |
| "hf_key": "model.layers.5.mixer.gdn.A_log", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.mixer.gdn.dt_bias": { |
| "hf_key": "model.layers.5.mixer.gdn.dt_bias", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.mixer.gdn.q_proj.weight": { |
| "hf_key": "model.layers.5.mixer.gdn.q_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.mixer.gdn.k_proj.weight": { |
| "hf_key": "model.layers.5.mixer.gdn.k_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.mixer.gdn.v_proj.weight": { |
| "hf_key": "model.layers.5.mixer.gdn.v_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.mixer.gdn.a_proj.weight": { |
| "hf_key": "model.layers.5.mixer.gdn.a_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.mixer.gdn.b_proj.weight": { |
| "hf_key": "model.layers.5.mixer.gdn.b_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.mixer.gdn.q_conv1d.weight": { |
| "hf_key": "model.layers.5.mixer.gdn.q_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.mixer.gdn.k_conv1d.weight": { |
| "hf_key": "model.layers.5.mixer.gdn.k_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.mixer.gdn.v_conv1d.weight": { |
| "hf_key": "model.layers.5.mixer.gdn.v_conv1d.weight", |
| "shape": [ |
| 6144, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.mixer.gdn.g_proj.weight": { |
| "hf_key": "model.layers.5.mixer.gdn.g_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.mixer.gdn.o_norm.weight": { |
| "hf_key": "model.layers.5.mixer.gdn.o_norm.weight", |
| "shape": [ |
| 512 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.mixer.gdn.o_proj.weight": { |
| "hf_key": "model.layers.5.mixer.gdn.o_proj.weight", |
| "shape": [ |
| 768, |
| 6144 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.residual1.mix_logits": { |
| "hf_key": "model.layers.5.residual1.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.residual1.update_gain": { |
| "hf_key": "model.layers.5.residual1.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.residual1.context_gate.weight": { |
| "hf_key": "model.layers.5.residual1.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.residual1.context_gate.bias": { |
| "hf_key": "model.layers.5.residual1.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.norm2.weight": { |
| "hf_key": "model.layers.5.norm2.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.mlp.gate_proj.weight": { |
| "hf_key": "model.layers.5.mlp.gate_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.mlp.up_proj.weight": { |
| "hf_key": "model.layers.5.mlp.up_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.mlp.down_proj.weight": { |
| "hf_key": "model.layers.5.mlp.down_proj.weight", |
| "shape": [ |
| 768, |
| 2816 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.residual2.mix_logits": { |
| "hf_key": "model.layers.5.residual2.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.residual2.update_gain": { |
| "hf_key": "model.layers.5.residual2.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.residual2.context_gate.weight": { |
| "hf_key": "model.layers.5.residual2.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.5.residual2.context_gate.bias": { |
| "hf_key": "model.layers.5.residual2.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.norm1.weight": { |
| "hf_key": "model.layers.6.norm1.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.mixer.gdn.A_log": { |
| "hf_key": "model.layers.6.mixer.gdn.A_log", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.mixer.gdn.dt_bias": { |
| "hf_key": "model.layers.6.mixer.gdn.dt_bias", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.mixer.gdn.q_proj.weight": { |
| "hf_key": "model.layers.6.mixer.gdn.q_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.mixer.gdn.k_proj.weight": { |
| "hf_key": "model.layers.6.mixer.gdn.k_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.mixer.gdn.v_proj.weight": { |
| "hf_key": "model.layers.6.mixer.gdn.v_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.mixer.gdn.a_proj.weight": { |
| "hf_key": "model.layers.6.mixer.gdn.a_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.mixer.gdn.b_proj.weight": { |
| "hf_key": "model.layers.6.mixer.gdn.b_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.mixer.gdn.q_conv1d.weight": { |
| "hf_key": "model.layers.6.mixer.gdn.q_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.mixer.gdn.k_conv1d.weight": { |
| "hf_key": "model.layers.6.mixer.gdn.k_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.mixer.gdn.v_conv1d.weight": { |
| "hf_key": "model.layers.6.mixer.gdn.v_conv1d.weight", |
| "shape": [ |
| 6144, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.mixer.gdn.g_proj.weight": { |
| "hf_key": "model.layers.6.mixer.gdn.g_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.mixer.gdn.o_norm.weight": { |
| "hf_key": "model.layers.6.mixer.gdn.o_norm.weight", |
| "shape": [ |
| 512 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.mixer.gdn.o_proj.weight": { |
| "hf_key": "model.layers.6.mixer.gdn.o_proj.weight", |
| "shape": [ |
| 768, |
| 6144 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.residual1.mix_logits": { |
| "hf_key": "model.layers.6.residual1.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.residual1.update_gain": { |
| "hf_key": "model.layers.6.residual1.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.residual1.context_gate.weight": { |
| "hf_key": "model.layers.6.residual1.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.residual1.context_gate.bias": { |
| "hf_key": "model.layers.6.residual1.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.norm2.weight": { |
| "hf_key": "model.layers.6.norm2.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.mlp.gate_proj.weight": { |
| "hf_key": "model.layers.6.mlp.gate_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.mlp.up_proj.weight": { |
| "hf_key": "model.layers.6.mlp.up_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.mlp.down_proj.weight": { |
| "hf_key": "model.layers.6.mlp.down_proj.weight", |
| "shape": [ |
| 768, |
| 2816 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.residual2.mix_logits": { |
| "hf_key": "model.layers.6.residual2.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.residual2.update_gain": { |
| "hf_key": "model.layers.6.residual2.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.residual2.context_gate.weight": { |
| "hf_key": "model.layers.6.residual2.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.6.residual2.context_gate.bias": { |
| "hf_key": "model.layers.6.residual2.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.norm1.weight": { |
| "hf_key": "model.layers.7.norm1.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.route_bias": { |
| "hf_key": "model.layers.7.mixer.route_bias", |
| "shape": [ |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.route_load_ema": { |
| "hf_key": "model.layers.7.mixer.route_load_ema", |
| "shape": [ |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.mla.q_a.weight": { |
| "hf_key": "model.layers.7.mixer.mla.q_a.weight", |
| "shape": [ |
| 192, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.mla.q_norm.weight": { |
| "hf_key": "model.layers.7.mixer.mla.q_norm.weight", |
| "shape": [ |
| 192 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.mla.q_b.weight": { |
| "hf_key": "model.layers.7.mixer.mla.q_b.weight", |
| "shape": [ |
| 768, |
| 192 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.mla.kv_a.weight": { |
| "hf_key": "model.layers.7.mixer.mla.kv_a.weight", |
| "shape": [ |
| 192, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.mla.kv_norm.weight": { |
| "hf_key": "model.layers.7.mixer.mla.kv_norm.weight", |
| "shape": [ |
| 192 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.mla.kv_b.weight": { |
| "hf_key": "model.layers.7.mixer.mla.kv_b.weight", |
| "shape": [ |
| 1152, |
| 192 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.mla.k_rope.weight": { |
| "hf_key": "model.layers.7.mixer.mla.k_rope.weight", |
| "shape": [ |
| 32, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.mla.out_proj.weight": { |
| "hf_key": "model.layers.7.mixer.mla.out_proj.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.mem_q.weight": { |
| "hf_key": "model.layers.7.mixer.mem_q.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.memory.initial_fast": { |
| "hf_key": "model.layers.7.mixer.memory.initial_fast", |
| "shape": [ |
| 32, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.memory.initial_slow": { |
| "hf_key": "model.layers.7.mixer.memory.initial_slow", |
| "shape": [ |
| 16, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.memory.write_query.weight": { |
| "hf_key": "model.layers.7.mixer.memory.write_query.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.memory.fast_slot_key.weight": { |
| "hf_key": "model.layers.7.mixer.memory.fast_slot_key.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.memory.slow_slot_key.weight": { |
| "hf_key": "model.layers.7.mixer.memory.slow_slot_key.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.memory.value_proj.weight": { |
| "hf_key": "model.layers.7.mixer.memory.value_proj.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.memory.fast_gate.weight": { |
| "hf_key": "model.layers.7.mixer.memory.fast_gate.weight", |
| "shape": [ |
| 768, |
| 1536 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.memory.fast_gate.bias": { |
| "hf_key": "model.layers.7.mixer.memory.fast_gate.bias", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.memory.slow_gate.weight": { |
| "hf_key": "model.layers.7.mixer.memory.slow_gate.weight", |
| "shape": [ |
| 768, |
| 1536 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.memory.slow_gate.bias": { |
| "hf_key": "model.layers.7.mixer.memory.slow_gate.bias", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.memory.read_k.weight": { |
| "hf_key": "model.layers.7.mixer.memory.read_k.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.memory.read_v.weight": { |
| "hf_key": "model.layers.7.mixer.memory.read_v.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mixer.router.weight": { |
| "hf_key": "model.layers.7.mixer.router.weight", |
| "shape": [ |
| 2, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.residual1.mix_logits": { |
| "hf_key": "model.layers.7.residual1.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.residual1.update_gain": { |
| "hf_key": "model.layers.7.residual1.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.residual1.context_gate.weight": { |
| "hf_key": "model.layers.7.residual1.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.residual1.context_gate.bias": { |
| "hf_key": "model.layers.7.residual1.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.norm2.weight": { |
| "hf_key": "model.layers.7.norm2.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mlp.gate_proj.weight": { |
| "hf_key": "model.layers.7.mlp.gate_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mlp.up_proj.weight": { |
| "hf_key": "model.layers.7.mlp.up_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.mlp.down_proj.weight": { |
| "hf_key": "model.layers.7.mlp.down_proj.weight", |
| "shape": [ |
| 768, |
| 2816 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.residual2.mix_logits": { |
| "hf_key": "model.layers.7.residual2.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.residual2.update_gain": { |
| "hf_key": "model.layers.7.residual2.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.residual2.context_gate.weight": { |
| "hf_key": "model.layers.7.residual2.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.7.residual2.context_gate.bias": { |
| "hf_key": "model.layers.7.residual2.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.norm1.weight": { |
| "hf_key": "model.layers.8.norm1.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.mixer.gdn.A_log": { |
| "hf_key": "model.layers.8.mixer.gdn.A_log", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.mixer.gdn.dt_bias": { |
| "hf_key": "model.layers.8.mixer.gdn.dt_bias", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.mixer.gdn.q_proj.weight": { |
| "hf_key": "model.layers.8.mixer.gdn.q_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.mixer.gdn.k_proj.weight": { |
| "hf_key": "model.layers.8.mixer.gdn.k_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.mixer.gdn.v_proj.weight": { |
| "hf_key": "model.layers.8.mixer.gdn.v_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.mixer.gdn.a_proj.weight": { |
| "hf_key": "model.layers.8.mixer.gdn.a_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.mixer.gdn.b_proj.weight": { |
| "hf_key": "model.layers.8.mixer.gdn.b_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.mixer.gdn.q_conv1d.weight": { |
| "hf_key": "model.layers.8.mixer.gdn.q_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.mixer.gdn.k_conv1d.weight": { |
| "hf_key": "model.layers.8.mixer.gdn.k_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.mixer.gdn.v_conv1d.weight": { |
| "hf_key": "model.layers.8.mixer.gdn.v_conv1d.weight", |
| "shape": [ |
| 6144, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.mixer.gdn.g_proj.weight": { |
| "hf_key": "model.layers.8.mixer.gdn.g_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.mixer.gdn.o_norm.weight": { |
| "hf_key": "model.layers.8.mixer.gdn.o_norm.weight", |
| "shape": [ |
| 512 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.mixer.gdn.o_proj.weight": { |
| "hf_key": "model.layers.8.mixer.gdn.o_proj.weight", |
| "shape": [ |
| 768, |
| 6144 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.residual1.mix_logits": { |
| "hf_key": "model.layers.8.residual1.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.residual1.update_gain": { |
| "hf_key": "model.layers.8.residual1.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.residual1.context_gate.weight": { |
| "hf_key": "model.layers.8.residual1.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.residual1.context_gate.bias": { |
| "hf_key": "model.layers.8.residual1.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.norm2.weight": { |
| "hf_key": "model.layers.8.norm2.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.mlp.gate_proj.weight": { |
| "hf_key": "model.layers.8.mlp.gate_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.mlp.up_proj.weight": { |
| "hf_key": "model.layers.8.mlp.up_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.mlp.down_proj.weight": { |
| "hf_key": "model.layers.8.mlp.down_proj.weight", |
| "shape": [ |
| 768, |
| 2816 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.residual2.mix_logits": { |
| "hf_key": "model.layers.8.residual2.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.residual2.update_gain": { |
| "hf_key": "model.layers.8.residual2.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.residual2.context_gate.weight": { |
| "hf_key": "model.layers.8.residual2.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.8.residual2.context_gate.bias": { |
| "hf_key": "model.layers.8.residual2.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.norm1.weight": { |
| "hf_key": "model.layers.9.norm1.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.mixer.gdn.A_log": { |
| "hf_key": "model.layers.9.mixer.gdn.A_log", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.mixer.gdn.dt_bias": { |
| "hf_key": "model.layers.9.mixer.gdn.dt_bias", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.mixer.gdn.q_proj.weight": { |
| "hf_key": "model.layers.9.mixer.gdn.q_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.mixer.gdn.k_proj.weight": { |
| "hf_key": "model.layers.9.mixer.gdn.k_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.mixer.gdn.v_proj.weight": { |
| "hf_key": "model.layers.9.mixer.gdn.v_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.mixer.gdn.a_proj.weight": { |
| "hf_key": "model.layers.9.mixer.gdn.a_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.mixer.gdn.b_proj.weight": { |
| "hf_key": "model.layers.9.mixer.gdn.b_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.mixer.gdn.q_conv1d.weight": { |
| "hf_key": "model.layers.9.mixer.gdn.q_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.mixer.gdn.k_conv1d.weight": { |
| "hf_key": "model.layers.9.mixer.gdn.k_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.mixer.gdn.v_conv1d.weight": { |
| "hf_key": "model.layers.9.mixer.gdn.v_conv1d.weight", |
| "shape": [ |
| 6144, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.mixer.gdn.g_proj.weight": { |
| "hf_key": "model.layers.9.mixer.gdn.g_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.mixer.gdn.o_norm.weight": { |
| "hf_key": "model.layers.9.mixer.gdn.o_norm.weight", |
| "shape": [ |
| 512 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.mixer.gdn.o_proj.weight": { |
| "hf_key": "model.layers.9.mixer.gdn.o_proj.weight", |
| "shape": [ |
| 768, |
| 6144 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.residual1.mix_logits": { |
| "hf_key": "model.layers.9.residual1.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.residual1.update_gain": { |
| "hf_key": "model.layers.9.residual1.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.residual1.context_gate.weight": { |
| "hf_key": "model.layers.9.residual1.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.residual1.context_gate.bias": { |
| "hf_key": "model.layers.9.residual1.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.norm2.weight": { |
| "hf_key": "model.layers.9.norm2.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.mlp.gate_proj.weight": { |
| "hf_key": "model.layers.9.mlp.gate_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.mlp.up_proj.weight": { |
| "hf_key": "model.layers.9.mlp.up_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.mlp.down_proj.weight": { |
| "hf_key": "model.layers.9.mlp.down_proj.weight", |
| "shape": [ |
| 768, |
| 2816 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.residual2.mix_logits": { |
| "hf_key": "model.layers.9.residual2.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.residual2.update_gain": { |
| "hf_key": "model.layers.9.residual2.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.residual2.context_gate.weight": { |
| "hf_key": "model.layers.9.residual2.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.9.residual2.context_gate.bias": { |
| "hf_key": "model.layers.9.residual2.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.norm1.weight": { |
| "hf_key": "model.layers.10.norm1.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.mixer.gdn.A_log": { |
| "hf_key": "model.layers.10.mixer.gdn.A_log", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.mixer.gdn.dt_bias": { |
| "hf_key": "model.layers.10.mixer.gdn.dt_bias", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.mixer.gdn.q_proj.weight": { |
| "hf_key": "model.layers.10.mixer.gdn.q_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.mixer.gdn.k_proj.weight": { |
| "hf_key": "model.layers.10.mixer.gdn.k_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.mixer.gdn.v_proj.weight": { |
| "hf_key": "model.layers.10.mixer.gdn.v_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.mixer.gdn.a_proj.weight": { |
| "hf_key": "model.layers.10.mixer.gdn.a_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.mixer.gdn.b_proj.weight": { |
| "hf_key": "model.layers.10.mixer.gdn.b_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.mixer.gdn.q_conv1d.weight": { |
| "hf_key": "model.layers.10.mixer.gdn.q_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.mixer.gdn.k_conv1d.weight": { |
| "hf_key": "model.layers.10.mixer.gdn.k_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.mixer.gdn.v_conv1d.weight": { |
| "hf_key": "model.layers.10.mixer.gdn.v_conv1d.weight", |
| "shape": [ |
| 6144, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.mixer.gdn.g_proj.weight": { |
| "hf_key": "model.layers.10.mixer.gdn.g_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.mixer.gdn.o_norm.weight": { |
| "hf_key": "model.layers.10.mixer.gdn.o_norm.weight", |
| "shape": [ |
| 512 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.mixer.gdn.o_proj.weight": { |
| "hf_key": "model.layers.10.mixer.gdn.o_proj.weight", |
| "shape": [ |
| 768, |
| 6144 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.residual1.mix_logits": { |
| "hf_key": "model.layers.10.residual1.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.residual1.update_gain": { |
| "hf_key": "model.layers.10.residual1.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.residual1.context_gate.weight": { |
| "hf_key": "model.layers.10.residual1.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.residual1.context_gate.bias": { |
| "hf_key": "model.layers.10.residual1.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.norm2.weight": { |
| "hf_key": "model.layers.10.norm2.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.mlp.gate_proj.weight": { |
| "hf_key": "model.layers.10.mlp.gate_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.mlp.up_proj.weight": { |
| "hf_key": "model.layers.10.mlp.up_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.mlp.down_proj.weight": { |
| "hf_key": "model.layers.10.mlp.down_proj.weight", |
| "shape": [ |
| 768, |
| 2816 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.residual2.mix_logits": { |
| "hf_key": "model.layers.10.residual2.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.residual2.update_gain": { |
| "hf_key": "model.layers.10.residual2.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.residual2.context_gate.weight": { |
| "hf_key": "model.layers.10.residual2.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.10.residual2.context_gate.bias": { |
| "hf_key": "model.layers.10.residual2.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.norm1.weight": { |
| "hf_key": "model.layers.11.norm1.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.route_bias": { |
| "hf_key": "model.layers.11.mixer.route_bias", |
| "shape": [ |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.route_load_ema": { |
| "hf_key": "model.layers.11.mixer.route_load_ema", |
| "shape": [ |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.mla.q_a.weight": { |
| "hf_key": "model.layers.11.mixer.mla.q_a.weight", |
| "shape": [ |
| 192, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.mla.q_norm.weight": { |
| "hf_key": "model.layers.11.mixer.mla.q_norm.weight", |
| "shape": [ |
| 192 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.mla.q_b.weight": { |
| "hf_key": "model.layers.11.mixer.mla.q_b.weight", |
| "shape": [ |
| 768, |
| 192 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.mla.kv_a.weight": { |
| "hf_key": "model.layers.11.mixer.mla.kv_a.weight", |
| "shape": [ |
| 192, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.mla.kv_norm.weight": { |
| "hf_key": "model.layers.11.mixer.mla.kv_norm.weight", |
| "shape": [ |
| 192 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.mla.kv_b.weight": { |
| "hf_key": "model.layers.11.mixer.mla.kv_b.weight", |
| "shape": [ |
| 1152, |
| 192 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.mla.k_rope.weight": { |
| "hf_key": "model.layers.11.mixer.mla.k_rope.weight", |
| "shape": [ |
| 32, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.mla.out_proj.weight": { |
| "hf_key": "model.layers.11.mixer.mla.out_proj.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.mem_q.weight": { |
| "hf_key": "model.layers.11.mixer.mem_q.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.memory.initial_fast": { |
| "hf_key": "model.layers.11.mixer.memory.initial_fast", |
| "shape": [ |
| 32, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.memory.initial_slow": { |
| "hf_key": "model.layers.11.mixer.memory.initial_slow", |
| "shape": [ |
| 16, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.memory.write_query.weight": { |
| "hf_key": "model.layers.11.mixer.memory.write_query.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.memory.fast_slot_key.weight": { |
| "hf_key": "model.layers.11.mixer.memory.fast_slot_key.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.memory.slow_slot_key.weight": { |
| "hf_key": "model.layers.11.mixer.memory.slow_slot_key.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.memory.value_proj.weight": { |
| "hf_key": "model.layers.11.mixer.memory.value_proj.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.memory.fast_gate.weight": { |
| "hf_key": "model.layers.11.mixer.memory.fast_gate.weight", |
| "shape": [ |
| 768, |
| 1536 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.memory.fast_gate.bias": { |
| "hf_key": "model.layers.11.mixer.memory.fast_gate.bias", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.memory.slow_gate.weight": { |
| "hf_key": "model.layers.11.mixer.memory.slow_gate.weight", |
| "shape": [ |
| 768, |
| 1536 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.memory.slow_gate.bias": { |
| "hf_key": "model.layers.11.mixer.memory.slow_gate.bias", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.memory.read_k.weight": { |
| "hf_key": "model.layers.11.mixer.memory.read_k.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.memory.read_v.weight": { |
| "hf_key": "model.layers.11.mixer.memory.read_v.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mixer.router.weight": { |
| "hf_key": "model.layers.11.mixer.router.weight", |
| "shape": [ |
| 2, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.residual1.mix_logits": { |
| "hf_key": "model.layers.11.residual1.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.residual1.update_gain": { |
| "hf_key": "model.layers.11.residual1.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.residual1.context_gate.weight": { |
| "hf_key": "model.layers.11.residual1.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.residual1.context_gate.bias": { |
| "hf_key": "model.layers.11.residual1.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.norm2.weight": { |
| "hf_key": "model.layers.11.norm2.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mlp.gate_proj.weight": { |
| "hf_key": "model.layers.11.mlp.gate_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mlp.up_proj.weight": { |
| "hf_key": "model.layers.11.mlp.up_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.mlp.down_proj.weight": { |
| "hf_key": "model.layers.11.mlp.down_proj.weight", |
| "shape": [ |
| 768, |
| 2816 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.residual2.mix_logits": { |
| "hf_key": "model.layers.11.residual2.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.residual2.update_gain": { |
| "hf_key": "model.layers.11.residual2.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.residual2.context_gate.weight": { |
| "hf_key": "model.layers.11.residual2.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.11.residual2.context_gate.bias": { |
| "hf_key": "model.layers.11.residual2.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.norm1.weight": { |
| "hf_key": "model.layers.12.norm1.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.mixer.gdn.A_log": { |
| "hf_key": "model.layers.12.mixer.gdn.A_log", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.mixer.gdn.dt_bias": { |
| "hf_key": "model.layers.12.mixer.gdn.dt_bias", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.mixer.gdn.q_proj.weight": { |
| "hf_key": "model.layers.12.mixer.gdn.q_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.mixer.gdn.k_proj.weight": { |
| "hf_key": "model.layers.12.mixer.gdn.k_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.mixer.gdn.v_proj.weight": { |
| "hf_key": "model.layers.12.mixer.gdn.v_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.mixer.gdn.a_proj.weight": { |
| "hf_key": "model.layers.12.mixer.gdn.a_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.mixer.gdn.b_proj.weight": { |
| "hf_key": "model.layers.12.mixer.gdn.b_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.mixer.gdn.q_conv1d.weight": { |
| "hf_key": "model.layers.12.mixer.gdn.q_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.mixer.gdn.k_conv1d.weight": { |
| "hf_key": "model.layers.12.mixer.gdn.k_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.mixer.gdn.v_conv1d.weight": { |
| "hf_key": "model.layers.12.mixer.gdn.v_conv1d.weight", |
| "shape": [ |
| 6144, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.mixer.gdn.g_proj.weight": { |
| "hf_key": "model.layers.12.mixer.gdn.g_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.mixer.gdn.o_norm.weight": { |
| "hf_key": "model.layers.12.mixer.gdn.o_norm.weight", |
| "shape": [ |
| 512 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.mixer.gdn.o_proj.weight": { |
| "hf_key": "model.layers.12.mixer.gdn.o_proj.weight", |
| "shape": [ |
| 768, |
| 6144 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.residual1.mix_logits": { |
| "hf_key": "model.layers.12.residual1.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.residual1.update_gain": { |
| "hf_key": "model.layers.12.residual1.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.residual1.context_gate.weight": { |
| "hf_key": "model.layers.12.residual1.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.residual1.context_gate.bias": { |
| "hf_key": "model.layers.12.residual1.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.norm2.weight": { |
| "hf_key": "model.layers.12.norm2.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.mlp.gate_proj.weight": { |
| "hf_key": "model.layers.12.mlp.gate_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.mlp.up_proj.weight": { |
| "hf_key": "model.layers.12.mlp.up_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.mlp.down_proj.weight": { |
| "hf_key": "model.layers.12.mlp.down_proj.weight", |
| "shape": [ |
| 768, |
| 2816 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.residual2.mix_logits": { |
| "hf_key": "model.layers.12.residual2.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.residual2.update_gain": { |
| "hf_key": "model.layers.12.residual2.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.residual2.context_gate.weight": { |
| "hf_key": "model.layers.12.residual2.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.12.residual2.context_gate.bias": { |
| "hf_key": "model.layers.12.residual2.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.norm1.weight": { |
| "hf_key": "model.layers.13.norm1.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.mixer.gdn.A_log": { |
| "hf_key": "model.layers.13.mixer.gdn.A_log", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.mixer.gdn.dt_bias": { |
| "hf_key": "model.layers.13.mixer.gdn.dt_bias", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.mixer.gdn.q_proj.weight": { |
| "hf_key": "model.layers.13.mixer.gdn.q_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.mixer.gdn.k_proj.weight": { |
| "hf_key": "model.layers.13.mixer.gdn.k_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.mixer.gdn.v_proj.weight": { |
| "hf_key": "model.layers.13.mixer.gdn.v_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.mixer.gdn.a_proj.weight": { |
| "hf_key": "model.layers.13.mixer.gdn.a_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.mixer.gdn.b_proj.weight": { |
| "hf_key": "model.layers.13.mixer.gdn.b_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.mixer.gdn.q_conv1d.weight": { |
| "hf_key": "model.layers.13.mixer.gdn.q_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.mixer.gdn.k_conv1d.weight": { |
| "hf_key": "model.layers.13.mixer.gdn.k_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.mixer.gdn.v_conv1d.weight": { |
| "hf_key": "model.layers.13.mixer.gdn.v_conv1d.weight", |
| "shape": [ |
| 6144, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.mixer.gdn.g_proj.weight": { |
| "hf_key": "model.layers.13.mixer.gdn.g_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.mixer.gdn.o_norm.weight": { |
| "hf_key": "model.layers.13.mixer.gdn.o_norm.weight", |
| "shape": [ |
| 512 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.mixer.gdn.o_proj.weight": { |
| "hf_key": "model.layers.13.mixer.gdn.o_proj.weight", |
| "shape": [ |
| 768, |
| 6144 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.residual1.mix_logits": { |
| "hf_key": "model.layers.13.residual1.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.residual1.update_gain": { |
| "hf_key": "model.layers.13.residual1.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.residual1.context_gate.weight": { |
| "hf_key": "model.layers.13.residual1.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.residual1.context_gate.bias": { |
| "hf_key": "model.layers.13.residual1.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.norm2.weight": { |
| "hf_key": "model.layers.13.norm2.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.mlp.gate_proj.weight": { |
| "hf_key": "model.layers.13.mlp.gate_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.mlp.up_proj.weight": { |
| "hf_key": "model.layers.13.mlp.up_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.mlp.down_proj.weight": { |
| "hf_key": "model.layers.13.mlp.down_proj.weight", |
| "shape": [ |
| 768, |
| 2816 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.residual2.mix_logits": { |
| "hf_key": "model.layers.13.residual2.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.residual2.update_gain": { |
| "hf_key": "model.layers.13.residual2.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.residual2.context_gate.weight": { |
| "hf_key": "model.layers.13.residual2.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.13.residual2.context_gate.bias": { |
| "hf_key": "model.layers.13.residual2.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.norm1.weight": { |
| "hf_key": "model.layers.14.norm1.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.mixer.gdn.A_log": { |
| "hf_key": "model.layers.14.mixer.gdn.A_log", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.mixer.gdn.dt_bias": { |
| "hf_key": "model.layers.14.mixer.gdn.dt_bias", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.mixer.gdn.q_proj.weight": { |
| "hf_key": "model.layers.14.mixer.gdn.q_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.mixer.gdn.k_proj.weight": { |
| "hf_key": "model.layers.14.mixer.gdn.k_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.mixer.gdn.v_proj.weight": { |
| "hf_key": "model.layers.14.mixer.gdn.v_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.mixer.gdn.a_proj.weight": { |
| "hf_key": "model.layers.14.mixer.gdn.a_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.mixer.gdn.b_proj.weight": { |
| "hf_key": "model.layers.14.mixer.gdn.b_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.mixer.gdn.q_conv1d.weight": { |
| "hf_key": "model.layers.14.mixer.gdn.q_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.mixer.gdn.k_conv1d.weight": { |
| "hf_key": "model.layers.14.mixer.gdn.k_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.mixer.gdn.v_conv1d.weight": { |
| "hf_key": "model.layers.14.mixer.gdn.v_conv1d.weight", |
| "shape": [ |
| 6144, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.mixer.gdn.g_proj.weight": { |
| "hf_key": "model.layers.14.mixer.gdn.g_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.mixer.gdn.o_norm.weight": { |
| "hf_key": "model.layers.14.mixer.gdn.o_norm.weight", |
| "shape": [ |
| 512 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.mixer.gdn.o_proj.weight": { |
| "hf_key": "model.layers.14.mixer.gdn.o_proj.weight", |
| "shape": [ |
| 768, |
| 6144 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.residual1.mix_logits": { |
| "hf_key": "model.layers.14.residual1.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.residual1.update_gain": { |
| "hf_key": "model.layers.14.residual1.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.residual1.context_gate.weight": { |
| "hf_key": "model.layers.14.residual1.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.residual1.context_gate.bias": { |
| "hf_key": "model.layers.14.residual1.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.norm2.weight": { |
| "hf_key": "model.layers.14.norm2.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.mlp.gate_proj.weight": { |
| "hf_key": "model.layers.14.mlp.gate_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.mlp.up_proj.weight": { |
| "hf_key": "model.layers.14.mlp.up_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.mlp.down_proj.weight": { |
| "hf_key": "model.layers.14.mlp.down_proj.weight", |
| "shape": [ |
| 768, |
| 2816 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.residual2.mix_logits": { |
| "hf_key": "model.layers.14.residual2.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.residual2.update_gain": { |
| "hf_key": "model.layers.14.residual2.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.residual2.context_gate.weight": { |
| "hf_key": "model.layers.14.residual2.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.14.residual2.context_gate.bias": { |
| "hf_key": "model.layers.14.residual2.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.norm1.weight": { |
| "hf_key": "model.layers.15.norm1.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.route_bias": { |
| "hf_key": "model.layers.15.mixer.route_bias", |
| "shape": [ |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.route_load_ema": { |
| "hf_key": "model.layers.15.mixer.route_load_ema", |
| "shape": [ |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.mla.q_a.weight": { |
| "hf_key": "model.layers.15.mixer.mla.q_a.weight", |
| "shape": [ |
| 192, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.mla.q_norm.weight": { |
| "hf_key": "model.layers.15.mixer.mla.q_norm.weight", |
| "shape": [ |
| 192 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.mla.q_b.weight": { |
| "hf_key": "model.layers.15.mixer.mla.q_b.weight", |
| "shape": [ |
| 768, |
| 192 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.mla.kv_a.weight": { |
| "hf_key": "model.layers.15.mixer.mla.kv_a.weight", |
| "shape": [ |
| 192, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.mla.kv_norm.weight": { |
| "hf_key": "model.layers.15.mixer.mla.kv_norm.weight", |
| "shape": [ |
| 192 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.mla.kv_b.weight": { |
| "hf_key": "model.layers.15.mixer.mla.kv_b.weight", |
| "shape": [ |
| 1152, |
| 192 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.mla.k_rope.weight": { |
| "hf_key": "model.layers.15.mixer.mla.k_rope.weight", |
| "shape": [ |
| 32, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.mla.out_proj.weight": { |
| "hf_key": "model.layers.15.mixer.mla.out_proj.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.mem_q.weight": { |
| "hf_key": "model.layers.15.mixer.mem_q.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.memory.initial_fast": { |
| "hf_key": "model.layers.15.mixer.memory.initial_fast", |
| "shape": [ |
| 32, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.memory.initial_slow": { |
| "hf_key": "model.layers.15.mixer.memory.initial_slow", |
| "shape": [ |
| 16, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.memory.write_query.weight": { |
| "hf_key": "model.layers.15.mixer.memory.write_query.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.memory.fast_slot_key.weight": { |
| "hf_key": "model.layers.15.mixer.memory.fast_slot_key.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.memory.slow_slot_key.weight": { |
| "hf_key": "model.layers.15.mixer.memory.slow_slot_key.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.memory.value_proj.weight": { |
| "hf_key": "model.layers.15.mixer.memory.value_proj.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.memory.fast_gate.weight": { |
| "hf_key": "model.layers.15.mixer.memory.fast_gate.weight", |
| "shape": [ |
| 768, |
| 1536 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.memory.fast_gate.bias": { |
| "hf_key": "model.layers.15.mixer.memory.fast_gate.bias", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.memory.slow_gate.weight": { |
| "hf_key": "model.layers.15.mixer.memory.slow_gate.weight", |
| "shape": [ |
| 768, |
| 1536 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.memory.slow_gate.bias": { |
| "hf_key": "model.layers.15.mixer.memory.slow_gate.bias", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.memory.read_k.weight": { |
| "hf_key": "model.layers.15.mixer.memory.read_k.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.memory.read_v.weight": { |
| "hf_key": "model.layers.15.mixer.memory.read_v.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mixer.router.weight": { |
| "hf_key": "model.layers.15.mixer.router.weight", |
| "shape": [ |
| 2, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.residual1.mix_logits": { |
| "hf_key": "model.layers.15.residual1.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.residual1.update_gain": { |
| "hf_key": "model.layers.15.residual1.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.residual1.context_gate.weight": { |
| "hf_key": "model.layers.15.residual1.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.residual1.context_gate.bias": { |
| "hf_key": "model.layers.15.residual1.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.norm2.weight": { |
| "hf_key": "model.layers.15.norm2.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mlp.gate_proj.weight": { |
| "hf_key": "model.layers.15.mlp.gate_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mlp.up_proj.weight": { |
| "hf_key": "model.layers.15.mlp.up_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.mlp.down_proj.weight": { |
| "hf_key": "model.layers.15.mlp.down_proj.weight", |
| "shape": [ |
| 768, |
| 2816 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.residual2.mix_logits": { |
| "hf_key": "model.layers.15.residual2.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.residual2.update_gain": { |
| "hf_key": "model.layers.15.residual2.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.residual2.context_gate.weight": { |
| "hf_key": "model.layers.15.residual2.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.15.residual2.context_gate.bias": { |
| "hf_key": "model.layers.15.residual2.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.norm1.weight": { |
| "hf_key": "model.layers.16.norm1.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.mixer.gdn.A_log": { |
| "hf_key": "model.layers.16.mixer.gdn.A_log", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.mixer.gdn.dt_bias": { |
| "hf_key": "model.layers.16.mixer.gdn.dt_bias", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.mixer.gdn.q_proj.weight": { |
| "hf_key": "model.layers.16.mixer.gdn.q_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.mixer.gdn.k_proj.weight": { |
| "hf_key": "model.layers.16.mixer.gdn.k_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.mixer.gdn.v_proj.weight": { |
| "hf_key": "model.layers.16.mixer.gdn.v_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.mixer.gdn.a_proj.weight": { |
| "hf_key": "model.layers.16.mixer.gdn.a_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.mixer.gdn.b_proj.weight": { |
| "hf_key": "model.layers.16.mixer.gdn.b_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.mixer.gdn.q_conv1d.weight": { |
| "hf_key": "model.layers.16.mixer.gdn.q_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.mixer.gdn.k_conv1d.weight": { |
| "hf_key": "model.layers.16.mixer.gdn.k_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.mixer.gdn.v_conv1d.weight": { |
| "hf_key": "model.layers.16.mixer.gdn.v_conv1d.weight", |
| "shape": [ |
| 6144, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.mixer.gdn.g_proj.weight": { |
| "hf_key": "model.layers.16.mixer.gdn.g_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.mixer.gdn.o_norm.weight": { |
| "hf_key": "model.layers.16.mixer.gdn.o_norm.weight", |
| "shape": [ |
| 512 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.mixer.gdn.o_proj.weight": { |
| "hf_key": "model.layers.16.mixer.gdn.o_proj.weight", |
| "shape": [ |
| 768, |
| 6144 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.residual1.mix_logits": { |
| "hf_key": "model.layers.16.residual1.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.residual1.update_gain": { |
| "hf_key": "model.layers.16.residual1.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.residual1.context_gate.weight": { |
| "hf_key": "model.layers.16.residual1.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.residual1.context_gate.bias": { |
| "hf_key": "model.layers.16.residual1.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.norm2.weight": { |
| "hf_key": "model.layers.16.norm2.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.mlp.gate_proj.weight": { |
| "hf_key": "model.layers.16.mlp.gate_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.mlp.up_proj.weight": { |
| "hf_key": "model.layers.16.mlp.up_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.mlp.down_proj.weight": { |
| "hf_key": "model.layers.16.mlp.down_proj.weight", |
| "shape": [ |
| 768, |
| 2816 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.residual2.mix_logits": { |
| "hf_key": "model.layers.16.residual2.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.residual2.update_gain": { |
| "hf_key": "model.layers.16.residual2.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.residual2.context_gate.weight": { |
| "hf_key": "model.layers.16.residual2.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.16.residual2.context_gate.bias": { |
| "hf_key": "model.layers.16.residual2.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.norm1.weight": { |
| "hf_key": "model.layers.17.norm1.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.mixer.gdn.A_log": { |
| "hf_key": "model.layers.17.mixer.gdn.A_log", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.mixer.gdn.dt_bias": { |
| "hf_key": "model.layers.17.mixer.gdn.dt_bias", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.mixer.gdn.q_proj.weight": { |
| "hf_key": "model.layers.17.mixer.gdn.q_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.mixer.gdn.k_proj.weight": { |
| "hf_key": "model.layers.17.mixer.gdn.k_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.mixer.gdn.v_proj.weight": { |
| "hf_key": "model.layers.17.mixer.gdn.v_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.mixer.gdn.a_proj.weight": { |
| "hf_key": "model.layers.17.mixer.gdn.a_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.mixer.gdn.b_proj.weight": { |
| "hf_key": "model.layers.17.mixer.gdn.b_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.mixer.gdn.q_conv1d.weight": { |
| "hf_key": "model.layers.17.mixer.gdn.q_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.mixer.gdn.k_conv1d.weight": { |
| "hf_key": "model.layers.17.mixer.gdn.k_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.mixer.gdn.v_conv1d.weight": { |
| "hf_key": "model.layers.17.mixer.gdn.v_conv1d.weight", |
| "shape": [ |
| 6144, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.mixer.gdn.g_proj.weight": { |
| "hf_key": "model.layers.17.mixer.gdn.g_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.mixer.gdn.o_norm.weight": { |
| "hf_key": "model.layers.17.mixer.gdn.o_norm.weight", |
| "shape": [ |
| 512 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.mixer.gdn.o_proj.weight": { |
| "hf_key": "model.layers.17.mixer.gdn.o_proj.weight", |
| "shape": [ |
| 768, |
| 6144 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.residual1.mix_logits": { |
| "hf_key": "model.layers.17.residual1.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.residual1.update_gain": { |
| "hf_key": "model.layers.17.residual1.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.residual1.context_gate.weight": { |
| "hf_key": "model.layers.17.residual1.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.residual1.context_gate.bias": { |
| "hf_key": "model.layers.17.residual1.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.norm2.weight": { |
| "hf_key": "model.layers.17.norm2.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.mlp.gate_proj.weight": { |
| "hf_key": "model.layers.17.mlp.gate_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.mlp.up_proj.weight": { |
| "hf_key": "model.layers.17.mlp.up_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.mlp.down_proj.weight": { |
| "hf_key": "model.layers.17.mlp.down_proj.weight", |
| "shape": [ |
| 768, |
| 2816 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.residual2.mix_logits": { |
| "hf_key": "model.layers.17.residual2.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.residual2.update_gain": { |
| "hf_key": "model.layers.17.residual2.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.residual2.context_gate.weight": { |
| "hf_key": "model.layers.17.residual2.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.17.residual2.context_gate.bias": { |
| "hf_key": "model.layers.17.residual2.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.norm1.weight": { |
| "hf_key": "model.layers.18.norm1.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.mixer.gdn.A_log": { |
| "hf_key": "model.layers.18.mixer.gdn.A_log", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.mixer.gdn.dt_bias": { |
| "hf_key": "model.layers.18.mixer.gdn.dt_bias", |
| "shape": [ |
| 12 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.mixer.gdn.q_proj.weight": { |
| "hf_key": "model.layers.18.mixer.gdn.q_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.mixer.gdn.k_proj.weight": { |
| "hf_key": "model.layers.18.mixer.gdn.k_proj.weight", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.mixer.gdn.v_proj.weight": { |
| "hf_key": "model.layers.18.mixer.gdn.v_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.mixer.gdn.a_proj.weight": { |
| "hf_key": "model.layers.18.mixer.gdn.a_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.mixer.gdn.b_proj.weight": { |
| "hf_key": "model.layers.18.mixer.gdn.b_proj.weight", |
| "shape": [ |
| 12, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.mixer.gdn.q_conv1d.weight": { |
| "hf_key": "model.layers.18.mixer.gdn.q_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.mixer.gdn.k_conv1d.weight": { |
| "hf_key": "model.layers.18.mixer.gdn.k_conv1d.weight", |
| "shape": [ |
| 3072, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.mixer.gdn.v_conv1d.weight": { |
| "hf_key": "model.layers.18.mixer.gdn.v_conv1d.weight", |
| "shape": [ |
| 6144, |
| 1, |
| 4 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.mixer.gdn.g_proj.weight": { |
| "hf_key": "model.layers.18.mixer.gdn.g_proj.weight", |
| "shape": [ |
| 6144, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.mixer.gdn.o_norm.weight": { |
| "hf_key": "model.layers.18.mixer.gdn.o_norm.weight", |
| "shape": [ |
| 512 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.mixer.gdn.o_proj.weight": { |
| "hf_key": "model.layers.18.mixer.gdn.o_proj.weight", |
| "shape": [ |
| 768, |
| 6144 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.residual1.mix_logits": { |
| "hf_key": "model.layers.18.residual1.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.residual1.update_gain": { |
| "hf_key": "model.layers.18.residual1.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.residual1.context_gate.weight": { |
| "hf_key": "model.layers.18.residual1.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.residual1.context_gate.bias": { |
| "hf_key": "model.layers.18.residual1.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.norm2.weight": { |
| "hf_key": "model.layers.18.norm2.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.mlp.gate_proj.weight": { |
| "hf_key": "model.layers.18.mlp.gate_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.mlp.up_proj.weight": { |
| "hf_key": "model.layers.18.mlp.up_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.mlp.down_proj.weight": { |
| "hf_key": "model.layers.18.mlp.down_proj.weight", |
| "shape": [ |
| 768, |
| 2816 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.residual2.mix_logits": { |
| "hf_key": "model.layers.18.residual2.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.residual2.update_gain": { |
| "hf_key": "model.layers.18.residual2.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.residual2.context_gate.weight": { |
| "hf_key": "model.layers.18.residual2.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.18.residual2.context_gate.bias": { |
| "hf_key": "model.layers.18.residual2.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.norm1.weight": { |
| "hf_key": "model.layers.19.norm1.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.route_bias": { |
| "hf_key": "model.layers.19.mixer.route_bias", |
| "shape": [ |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.route_load_ema": { |
| "hf_key": "model.layers.19.mixer.route_load_ema", |
| "shape": [ |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.mla.q_a.weight": { |
| "hf_key": "model.layers.19.mixer.mla.q_a.weight", |
| "shape": [ |
| 192, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.mla.q_norm.weight": { |
| "hf_key": "model.layers.19.mixer.mla.q_norm.weight", |
| "shape": [ |
| 192 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.mla.q_b.weight": { |
| "hf_key": "model.layers.19.mixer.mla.q_b.weight", |
| "shape": [ |
| 768, |
| 192 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.mla.kv_a.weight": { |
| "hf_key": "model.layers.19.mixer.mla.kv_a.weight", |
| "shape": [ |
| 192, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.mla.kv_norm.weight": { |
| "hf_key": "model.layers.19.mixer.mla.kv_norm.weight", |
| "shape": [ |
| 192 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.mla.kv_b.weight": { |
| "hf_key": "model.layers.19.mixer.mla.kv_b.weight", |
| "shape": [ |
| 1152, |
| 192 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.mla.k_rope.weight": { |
| "hf_key": "model.layers.19.mixer.mla.k_rope.weight", |
| "shape": [ |
| 32, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.mla.out_proj.weight": { |
| "hf_key": "model.layers.19.mixer.mla.out_proj.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.mem_q.weight": { |
| "hf_key": "model.layers.19.mixer.mem_q.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.memory.initial_fast": { |
| "hf_key": "model.layers.19.mixer.memory.initial_fast", |
| "shape": [ |
| 32, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.memory.initial_slow": { |
| "hf_key": "model.layers.19.mixer.memory.initial_slow", |
| "shape": [ |
| 16, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.memory.write_query.weight": { |
| "hf_key": "model.layers.19.mixer.memory.write_query.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.memory.fast_slot_key.weight": { |
| "hf_key": "model.layers.19.mixer.memory.fast_slot_key.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.memory.slow_slot_key.weight": { |
| "hf_key": "model.layers.19.mixer.memory.slow_slot_key.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.memory.value_proj.weight": { |
| "hf_key": "model.layers.19.mixer.memory.value_proj.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.memory.fast_gate.weight": { |
| "hf_key": "model.layers.19.mixer.memory.fast_gate.weight", |
| "shape": [ |
| 768, |
| 1536 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.memory.fast_gate.bias": { |
| "hf_key": "model.layers.19.mixer.memory.fast_gate.bias", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.memory.slow_gate.weight": { |
| "hf_key": "model.layers.19.mixer.memory.slow_gate.weight", |
| "shape": [ |
| 768, |
| 1536 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.memory.slow_gate.bias": { |
| "hf_key": "model.layers.19.mixer.memory.slow_gate.bias", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.memory.read_k.weight": { |
| "hf_key": "model.layers.19.mixer.memory.read_k.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.memory.read_v.weight": { |
| "hf_key": "model.layers.19.mixer.memory.read_v.weight", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mixer.router.weight": { |
| "hf_key": "model.layers.19.mixer.router.weight", |
| "shape": [ |
| 2, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.residual1.mix_logits": { |
| "hf_key": "model.layers.19.residual1.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.residual1.update_gain": { |
| "hf_key": "model.layers.19.residual1.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.residual1.context_gate.weight": { |
| "hf_key": "model.layers.19.residual1.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.residual1.context_gate.bias": { |
| "hf_key": "model.layers.19.residual1.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.norm2.weight": { |
| "hf_key": "model.layers.19.norm2.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mlp.gate_proj.weight": { |
| "hf_key": "model.layers.19.mlp.gate_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mlp.up_proj.weight": { |
| "hf_key": "model.layers.19.mlp.up_proj.weight", |
| "shape": [ |
| 2816, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.mlp.down_proj.weight": { |
| "hf_key": "model.layers.19.mlp.down_proj.weight", |
| "shape": [ |
| 768, |
| 2816 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.residual2.mix_logits": { |
| "hf_key": "model.layers.19.residual2.mix_logits", |
| "shape": [ |
| 2, |
| 2 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.residual2.update_gain": { |
| "hf_key": "model.layers.19.residual2.update_gain", |
| "shape": [], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.residual2.context_gate.weight": { |
| "hf_key": "model.layers.19.residual2.context_gate.weight", |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.layers.19.residual2.context_gate.bias": { |
| "hf_key": "model.layers.19.residual2.context_gate.bias", |
| "shape": [ |
| 1 |
| ], |
| "source_dtype": "float32" |
| }, |
| "model.norm.weight": { |
| "hf_key": "model.norm.weight", |
| "shape": [ |
| 768 |
| ], |
| "source_dtype": "float32" |
| } |
| } |
| } |
