config.json

{
  "architectures": [
    "KimiK3ForConditionalGeneration"
  ],
  "auto_map": {
    "AutoConfig": "configuration_kimi_k3.KimiK3Config",
    "AutoModel": "modeling_kimi_k3.KimiK3ForConditionalGeneration",
    "AutoModelForCausalLM": "modeling_kimi_k3.KimiK3ForConditionalGeneration"
  },
  "bos_token_id": 163584,
  "dtype": "bfloat16",
  "eos_token_id": 163586,
  "ignore_index": -100,
  "image_placeholder": "<|kimi_image_placeholder|>",
  "media_placeholder_token_id": 163605,
  "model_type": "kimi_k3",
  "pad_token_id": 163839,
  "quantization_config": {
    "config_groups": {
      "group_0": {
        "format": "nvfp4-pack-quantized",
        "input_activations": {
          "actorder": null,
          "block_structure": null,
          "dynamic": "local",
          "group_size": 16,
          "num_bits": 4,
          "observer": "static_minmax",
          "observer_kwargs": {},
          "scale_dtype": "torch.float8_e4m3fn",
          "strategy": "tensor_group",
          "symmetric": true,
          "type": "float",
          "zp_dtype": null
        },
        "output_activations": null,
        "targets": [
          "re:.*block_sparse_moe.*"
        ],
        "weights": {
          "actorder": null,
          "block_structure": null,
          "dynamic": false,
          "group_size": 16,
          "num_bits": 4,
          "observer": "memoryless_minmax",
          "observer_kwargs": {},
          "scale_dtype": "torch.float8_e4m3fn",
          "strategy": "tensor_group",
          "symmetric": true,
          "type": "float",
          "zp_dtype": null
        }
      }
    },
    "format": "nvfp4-pack-quantized",
    "global_compression_ratio": null,
    "ignore": [
      "vision_tower.encoder.blocks.0.mlp.fc0",
      "vision_tower.encoder.blocks.0.mlp.fc1",
      "vision_tower.encoder.blocks.0.wqkv",
      "vision_tower.encoder.blocks.0.wo",
      "vision_tower.encoder.blocks.1.mlp.fc0",
      "vision_tower.encoder.blocks.1.mlp.fc1",
      "vision_tower.encoder.blocks.1.wqkv",
      "vision_tower.encoder.blocks.1.wo",
      "vision_tower.encoder.blocks.2.mlp.fc0",
      "vision_tower.encoder.blocks.2.mlp.fc1",
      "vision_tower.encoder.blocks.2.wqkv",
      "vision_tower.encoder.blocks.2.wo",
      "vision_tower.encoder.blocks.3.mlp.fc0",
      "vision_tower.encoder.blocks.3.mlp.fc1",
      "vision_tower.encoder.blocks.3.wqkv",
      "vision_tower.encoder.blocks.3.wo",
      "vision_tower.encoder.blocks.4.mlp.fc0",
      "vision_tower.encoder.blocks.4.mlp.fc1",
      "vision_tower.encoder.blocks.4.wqkv",
      "vision_tower.encoder.blocks.4.wo",
      "vision_tower.encoder.blocks.5.mlp.fc0",
      "vision_tower.encoder.blocks.5.mlp.fc1",
      "vision_tower.encoder.blocks.5.wqkv",
      "vision_tower.encoder.blocks.5.wo",
      "vision_tower.encoder.blocks.6.mlp.fc0",
      "vision_tower.encoder.blocks.6.mlp.fc1",
      "vision_tower.encoder.blocks.6.wqkv",
      "vision_tower.encoder.blocks.6.wo",
      "vision_tower.encoder.blocks.7.mlp.fc0",
      "vision_tower.encoder.blocks.7.mlp.fc1",
      "vision_tower.encoder.blocks.7.wqkv",
      "vision_tower.encoder.blocks.7.wo",
      "vision_tower.encoder.blocks.8.mlp.fc0",
      "vision_tower.encoder.blocks.8.mlp.fc1",
      "vision_tower.encoder.blocks.8.wqkv",
      "vision_tower.encoder.blocks.8.wo",
      "vision_tower.encoder.blocks.9.mlp.fc0",
      "vision_tower.encoder.blocks.9.mlp.fc1",
      "vision_tower.encoder.blocks.9.wqkv",
      "vision_tower.encoder.blocks.9.wo",
      "vision_tower.encoder.blocks.10.mlp.fc0",
      "vision_tower.encoder.blocks.10.mlp.fc1",
      "vision_tower.encoder.blocks.10.wqkv",
      "vision_tower.encoder.blocks.10.wo",
      "vision_tower.encoder.blocks.11.mlp.fc0",
      "vision_tower.encoder.blocks.11.mlp.fc1",
      "vision_tower.encoder.blocks.11.wqkv",
      "vision_tower.encoder.blocks.11.wo",
      "vision_tower.encoder.blocks.12.mlp.fc0",
      "vision_tower.encoder.blocks.12.mlp.fc1",
      "vision_tower.encoder.blocks.12.wqkv",
      "vision_tower.encoder.blocks.12.wo",
      "vision_tower.encoder.blocks.13.mlp.fc0",
      "vision_tower.encoder.blocks.13.mlp.fc1",
      "vision_tower.encoder.blocks.13.wqkv",
      "vision_tower.encoder.blocks.13.wo",
      "vision_tower.encoder.blocks.14.mlp.fc0",
      "vision_tower.encoder.blocks.14.mlp.fc1",
      "vision_tower.encoder.blocks.14.wqkv",
      "vision_tower.encoder.blocks.14.wo",
      "vision_tower.encoder.blocks.15.mlp.fc0",
      "vision_tower.encoder.blocks.15.mlp.fc1",
      "vision_tower.encoder.blocks.15.wqkv",
      "vision_tower.encoder.blocks.15.wo",
      "vision_tower.encoder.blocks.16.mlp.fc0",
      "vision_tower.encoder.blocks.16.mlp.fc1",
      "vision_tower.encoder.blocks.16.wqkv",
      "vision_tower.encoder.blocks.16.wo",
      "vision_tower.encoder.blocks.17.mlp.fc0",
      "vision_tower.encoder.blocks.17.mlp.fc1",
      "vision_tower.encoder.blocks.17.wqkv",
      "vision_tower.encoder.blocks.17.wo",
      "vision_tower.encoder.blocks.18.mlp.fc0",
      "vision_tower.encoder.blocks.18.mlp.fc1",
      "vision_tower.encoder.blocks.18.wqkv",
      "vision_tower.encoder.blocks.18.wo",
      "vision_tower.encoder.blocks.19.mlp.fc0",
      "vision_tower.encoder.blocks.19.mlp.fc1",
      "vision_tower.encoder.blocks.19.wqkv",
      "vision_tower.encoder.blocks.19.wo",
      "vision_tower.encoder.blocks.20.mlp.fc0",
      "vision_tower.encoder.blocks.20.mlp.fc1",
      "vision_tower.encoder.blocks.20.wqkv",
      "vision_tower.encoder.blocks.20.wo",
      "vision_tower.encoder.blocks.21.mlp.fc0",
      "vision_tower.encoder.blocks.21.mlp.fc1",
      "vision_tower.encoder.blocks.21.wqkv",
      "vision_tower.encoder.blocks.21.wo",
      "vision_tower.encoder.blocks.22.mlp.fc0",
      "vision_tower.encoder.blocks.22.mlp.fc1",
      "vision_tower.encoder.blocks.22.wqkv",
      "vision_tower.encoder.blocks.22.wo",
      "vision_tower.encoder.blocks.23.mlp.fc0",
      "vision_tower.encoder.blocks.23.mlp.fc1",
      "vision_tower.encoder.blocks.23.wqkv",
      "vision_tower.encoder.blocks.23.wo",
      "vision_tower.encoder.blocks.24.mlp.fc0",
      "vision_tower.encoder.blocks.24.mlp.fc1",
      "vision_tower.encoder.blocks.24.wqkv",
      "vision_tower.encoder.blocks.24.wo",
      "vision_tower.encoder.blocks.25.mlp.fc0",
      "vision_tower.encoder.blocks.25.mlp.fc1",
      "vision_tower.encoder.blocks.25.wqkv",
      "vision_tower.encoder.blocks.25.wo",
      "vision_tower.encoder.blocks.26.mlp.fc0",
      "vision_tower.encoder.blocks.26.mlp.fc1",
      "vision_tower.encoder.blocks.26.wqkv",
      "vision_tower.encoder.blocks.26.wo",
      "mm_projector.proj.0",
      "mm_projector.proj.2",
      "language_model.model.layers.0.self_attn.q_proj",
      "language_model.model.layers.0.self_attn.k_proj",
      "language_model.model.layers.0.self_attn.v_proj",
      "language_model.model.layers.0.self_attn.f_a_proj",
      "language_model.model.layers.0.self_attn.f_b_proj",
      "language_model.model.layers.0.self_attn.b_proj",
      "language_model.model.layers.0.self_attn.g_proj",
      "language_model.model.layers.0.self_attn.o_norm",
      "language_model.model.layers.0.self_attn.o_proj",
      "language_model.model.layers.0.mlp.gate_proj",
      "language_model.model.layers.0.mlp.up_proj",
      "language_model.model.layers.0.mlp.down_proj",
      "language_model.model.layers.0.self_attention_res_proj",
      "language_model.model.layers.0.mlp_res_proj",
      "language_model.model.layers.1.self_attn.q_proj",
      "language_model.model.layers.1.self_attn.k_proj",
      "language_model.model.layers.1.self_attn.v_proj",
      "language_model.model.layers.1.self_attn.f_a_proj",
      "language_model.model.layers.1.self_attn.f_b_proj",
      "language_model.model.layers.1.self_attn.b_proj",
      "language_model.model.layers.1.self_attn.g_proj",
      "language_model.model.layers.1.self_attn.o_norm",
      "language_model.model.layers.1.self_attn.o_proj",
      "language_model.model.layers.1.block_sparse_moe.gate",
      "language_model.model.layers.1.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.1.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.1.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.1.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.1.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.1.self_attention_res_proj",
      "language_model.model.layers.1.mlp_res_proj",
      "language_model.model.layers.2.self_attn.q_proj",
      "language_model.model.layers.2.self_attn.k_proj",
      "language_model.model.layers.2.self_attn.v_proj",
      "language_model.model.layers.2.self_attn.f_a_proj",
      "language_model.model.layers.2.self_attn.f_b_proj",
      "language_model.model.layers.2.self_attn.b_proj",
      "language_model.model.layers.2.self_attn.g_proj",
      "language_model.model.layers.2.self_attn.o_norm",
      "language_model.model.layers.2.self_attn.o_proj",
      "language_model.model.layers.2.block_sparse_moe.gate",
      "language_model.model.layers.2.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.2.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.2.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.2.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.2.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.2.self_attention_res_proj",
      "language_model.model.layers.2.mlp_res_proj",
      "language_model.model.layers.3.self_attn.q_a_proj",
      "language_model.model.layers.3.self_attn.q_b_proj",
      "language_model.model.layers.3.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.3.self_attn.kv_b_proj",
      "language_model.model.layers.3.self_attn.o_proj",
      "language_model.model.layers.3.self_attn.g_proj",
      "language_model.model.layers.3.block_sparse_moe.gate",
      "language_model.model.layers.3.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.3.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.3.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.3.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.3.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.3.self_attention_res_proj",
      "language_model.model.layers.3.mlp_res_proj",
      "language_model.model.layers.4.self_attn.q_proj",
      "language_model.model.layers.4.self_attn.k_proj",
      "language_model.model.layers.4.self_attn.v_proj",
      "language_model.model.layers.4.self_attn.f_a_proj",
      "language_model.model.layers.4.self_attn.f_b_proj",
      "language_model.model.layers.4.self_attn.b_proj",
      "language_model.model.layers.4.self_attn.g_proj",
      "language_model.model.layers.4.self_attn.o_norm",
      "language_model.model.layers.4.self_attn.o_proj",
      "language_model.model.layers.4.block_sparse_moe.gate",
      "language_model.model.layers.4.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.4.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.4.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.4.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.4.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.4.self_attention_res_proj",
      "language_model.model.layers.4.mlp_res_proj",
      "language_model.model.layers.5.self_attn.q_proj",
      "language_model.model.layers.5.self_attn.k_proj",
      "language_model.model.layers.5.self_attn.v_proj",
      "language_model.model.layers.5.self_attn.f_a_proj",
      "language_model.model.layers.5.self_attn.f_b_proj",
      "language_model.model.layers.5.self_attn.b_proj",
      "language_model.model.layers.5.self_attn.g_proj",
      "language_model.model.layers.5.self_attn.o_norm",
      "language_model.model.layers.5.self_attn.o_proj",
      "language_model.model.layers.5.block_sparse_moe.gate",
      "language_model.model.layers.5.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.5.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.5.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.5.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.5.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.5.self_attention_res_proj",
      "language_model.model.layers.5.mlp_res_proj",
      "language_model.model.layers.6.self_attn.q_proj",
      "language_model.model.layers.6.self_attn.k_proj",
      "language_model.model.layers.6.self_attn.v_proj",
      "language_model.model.layers.6.self_attn.f_a_proj",
      "language_model.model.layers.6.self_attn.f_b_proj",
      "language_model.model.layers.6.self_attn.b_proj",
      "language_model.model.layers.6.self_attn.g_proj",
      "language_model.model.layers.6.self_attn.o_norm",
      "language_model.model.layers.6.self_attn.o_proj",
      "language_model.model.layers.6.block_sparse_moe.gate",
      "language_model.model.layers.6.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.6.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.6.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.6.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.6.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.6.self_attention_res_proj",
      "language_model.model.layers.6.mlp_res_proj",
      "language_model.model.layers.7.self_attn.q_a_proj",
      "language_model.model.layers.7.self_attn.q_b_proj",
      "language_model.model.layers.7.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.7.self_attn.kv_b_proj",
      "language_model.model.layers.7.self_attn.o_proj",
      "language_model.model.layers.7.self_attn.g_proj",
      "language_model.model.layers.7.block_sparse_moe.gate",
      "language_model.model.layers.7.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.7.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.7.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.7.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.7.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.7.self_attention_res_proj",
      "language_model.model.layers.7.mlp_res_proj",
      "language_model.model.layers.8.self_attn.q_proj",
      "language_model.model.layers.8.self_attn.k_proj",
      "language_model.model.layers.8.self_attn.v_proj",
      "language_model.model.layers.8.self_attn.f_a_proj",
      "language_model.model.layers.8.self_attn.f_b_proj",
      "language_model.model.layers.8.self_attn.b_proj",
      "language_model.model.layers.8.self_attn.g_proj",
      "language_model.model.layers.8.self_attn.o_norm",
      "language_model.model.layers.8.self_attn.o_proj",
      "language_model.model.layers.8.block_sparse_moe.gate",
      "language_model.model.layers.8.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.8.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.8.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.8.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.8.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.8.self_attention_res_proj",
      "language_model.model.layers.8.mlp_res_proj",
      "language_model.model.layers.9.self_attn.q_proj",
      "language_model.model.layers.9.self_attn.k_proj",
      "language_model.model.layers.9.self_attn.v_proj",
      "language_model.model.layers.9.self_attn.f_a_proj",
      "language_model.model.layers.9.self_attn.f_b_proj",
      "language_model.model.layers.9.self_attn.b_proj",
      "language_model.model.layers.9.self_attn.g_proj",
      "language_model.model.layers.9.self_attn.o_norm",
      "language_model.model.layers.9.self_attn.o_proj",
      "language_model.model.layers.9.block_sparse_moe.gate",
      "language_model.model.layers.9.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.9.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.9.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.9.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.9.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.9.self_attention_res_proj",
      "language_model.model.layers.9.mlp_res_proj",
      "language_model.model.layers.10.self_attn.q_proj",
      "language_model.model.layers.10.self_attn.k_proj",
      "language_model.model.layers.10.self_attn.v_proj",
      "language_model.model.layers.10.self_attn.f_a_proj",
      "language_model.model.layers.10.self_attn.f_b_proj",
      "language_model.model.layers.10.self_attn.b_proj",
      "language_model.model.layers.10.self_attn.g_proj",
      "language_model.model.layers.10.self_attn.o_norm",
      "language_model.model.layers.10.self_attn.o_proj",
      "language_model.model.layers.10.block_sparse_moe.gate",
      "language_model.model.layers.10.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.10.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.10.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.10.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.10.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.10.self_attention_res_proj",
      "language_model.model.layers.10.mlp_res_proj",
      "language_model.model.layers.11.self_attn.q_a_proj",
      "language_model.model.layers.11.self_attn.q_b_proj",
      "language_model.model.layers.11.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.11.self_attn.kv_b_proj",
      "language_model.model.layers.11.self_attn.o_proj",
      "language_model.model.layers.11.self_attn.g_proj",
      "language_model.model.layers.11.block_sparse_moe.gate",
      "language_model.model.layers.11.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.11.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.11.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.11.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.11.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.11.self_attention_res_proj",
      "language_model.model.layers.11.mlp_res_proj",
      "language_model.model.layers.12.self_attn.q_proj",
      "language_model.model.layers.12.self_attn.k_proj",
      "language_model.model.layers.12.self_attn.v_proj",
      "language_model.model.layers.12.self_attn.f_a_proj",
      "language_model.model.layers.12.self_attn.f_b_proj",
      "language_model.model.layers.12.self_attn.b_proj",
      "language_model.model.layers.12.self_attn.g_proj",
      "language_model.model.layers.12.self_attn.o_norm",
      "language_model.model.layers.12.self_attn.o_proj",
      "language_model.model.layers.12.block_sparse_moe.gate",
      "language_model.model.layers.12.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.12.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.12.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.12.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.12.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.12.self_attention_res_proj",
      "language_model.model.layers.12.mlp_res_proj",
      "language_model.model.layers.13.self_attn.q_proj",
      "language_model.model.layers.13.self_attn.k_proj",
      "language_model.model.layers.13.self_attn.v_proj",
      "language_model.model.layers.13.self_attn.f_a_proj",
      "language_model.model.layers.13.self_attn.f_b_proj",
      "language_model.model.layers.13.self_attn.b_proj",
      "language_model.model.layers.13.self_attn.g_proj",
      "language_model.model.layers.13.self_attn.o_norm",
      "language_model.model.layers.13.self_attn.o_proj",
      "language_model.model.layers.13.block_sparse_moe.gate",
      "language_model.model.layers.13.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.13.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.13.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.13.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.13.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.13.self_attention_res_proj",
      "language_model.model.layers.13.mlp_res_proj",
      "language_model.model.layers.14.self_attn.q_proj",
      "language_model.model.layers.14.self_attn.k_proj",
      "language_model.model.layers.14.self_attn.v_proj",
      "language_model.model.layers.14.self_attn.f_a_proj",
      "language_model.model.layers.14.self_attn.f_b_proj",
      "language_model.model.layers.14.self_attn.b_proj",
      "language_model.model.layers.14.self_attn.g_proj",
      "language_model.model.layers.14.self_attn.o_norm",
      "language_model.model.layers.14.self_attn.o_proj",
      "language_model.model.layers.14.block_sparse_moe.gate",
      "language_model.model.layers.14.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.14.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.14.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.14.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.14.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.14.self_attention_res_proj",
      "language_model.model.layers.14.mlp_res_proj",
      "language_model.model.layers.15.self_attn.q_a_proj",
      "language_model.model.layers.15.self_attn.q_b_proj",
      "language_model.model.layers.15.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.15.self_attn.kv_b_proj",
      "language_model.model.layers.15.self_attn.o_proj",
      "language_model.model.layers.15.self_attn.g_proj",
      "language_model.model.layers.15.block_sparse_moe.gate",
      "language_model.model.layers.15.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.15.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.15.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.15.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.15.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.15.self_attention_res_proj",
      "language_model.model.layers.15.mlp_res_proj",
      "language_model.model.layers.16.self_attn.q_proj",
      "language_model.model.layers.16.self_attn.k_proj",
      "language_model.model.layers.16.self_attn.v_proj",
      "language_model.model.layers.16.self_attn.f_a_proj",
      "language_model.model.layers.16.self_attn.f_b_proj",
      "language_model.model.layers.16.self_attn.b_proj",
      "language_model.model.layers.16.self_attn.g_proj",
      "language_model.model.layers.16.self_attn.o_norm",
      "language_model.model.layers.16.self_attn.o_proj",
      "language_model.model.layers.16.block_sparse_moe.gate",
      "language_model.model.layers.16.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.16.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.16.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.16.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.16.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.16.self_attention_res_proj",
      "language_model.model.layers.16.mlp_res_proj",
      "language_model.model.layers.17.self_attn.q_proj",
      "language_model.model.layers.17.self_attn.k_proj",
      "language_model.model.layers.17.self_attn.v_proj",
      "language_model.model.layers.17.self_attn.f_a_proj",
      "language_model.model.layers.17.self_attn.f_b_proj",
      "language_model.model.layers.17.self_attn.b_proj",
      "language_model.model.layers.17.self_attn.g_proj",
      "language_model.model.layers.17.self_attn.o_norm",
      "language_model.model.layers.17.self_attn.o_proj",
      "language_model.model.layers.17.block_sparse_moe.gate",
      "language_model.model.layers.17.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.17.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.17.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.17.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.17.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.17.self_attention_res_proj",
      "language_model.model.layers.17.mlp_res_proj",
      "language_model.model.layers.18.self_attn.q_proj",
      "language_model.model.layers.18.self_attn.k_proj",
      "language_model.model.layers.18.self_attn.v_proj",
      "language_model.model.layers.18.self_attn.f_a_proj",
      "language_model.model.layers.18.self_attn.f_b_proj",
      "language_model.model.layers.18.self_attn.b_proj",
      "language_model.model.layers.18.self_attn.g_proj",
      "language_model.model.layers.18.self_attn.o_norm",
      "language_model.model.layers.18.self_attn.o_proj",
      "language_model.model.layers.18.block_sparse_moe.gate",
      "language_model.model.layers.18.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.18.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.18.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.18.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.18.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.18.self_attention_res_proj",
      "language_model.model.layers.18.mlp_res_proj",
      "language_model.model.layers.19.self_attn.q_a_proj",
      "language_model.model.layers.19.self_attn.q_b_proj",
      "language_model.model.layers.19.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.19.self_attn.kv_b_proj",
      "language_model.model.layers.19.self_attn.o_proj",
      "language_model.model.layers.19.self_attn.g_proj",
      "language_model.model.layers.19.block_sparse_moe.gate",
      "language_model.model.layers.19.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.19.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.19.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.19.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.19.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.19.self_attention_res_proj",
      "language_model.model.layers.19.mlp_res_proj",
      "language_model.model.layers.20.self_attn.q_proj",
      "language_model.model.layers.20.self_attn.k_proj",
      "language_model.model.layers.20.self_attn.v_proj",
      "language_model.model.layers.20.self_attn.f_a_proj",
      "language_model.model.layers.20.self_attn.f_b_proj",
      "language_model.model.layers.20.self_attn.b_proj",
      "language_model.model.layers.20.self_attn.g_proj",
      "language_model.model.layers.20.self_attn.o_norm",
      "language_model.model.layers.20.self_attn.o_proj",
      "language_model.model.layers.20.block_sparse_moe.gate",
      "language_model.model.layers.20.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.20.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.20.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.20.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.20.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.20.self_attention_res_proj",
      "language_model.model.layers.20.mlp_res_proj",
      "language_model.model.layers.21.self_attn.q_proj",
      "language_model.model.layers.21.self_attn.k_proj",
      "language_model.model.layers.21.self_attn.v_proj",
      "language_model.model.layers.21.self_attn.f_a_proj",
      "language_model.model.layers.21.self_attn.f_b_proj",
      "language_model.model.layers.21.self_attn.b_proj",
      "language_model.model.layers.21.self_attn.g_proj",
      "language_model.model.layers.21.self_attn.o_norm",
      "language_model.model.layers.21.self_attn.o_proj",
      "language_model.model.layers.21.block_sparse_moe.gate",
      "language_model.model.layers.21.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.21.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.21.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.21.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.21.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.21.self_attention_res_proj",
      "language_model.model.layers.21.mlp_res_proj",
      "language_model.model.layers.22.self_attn.q_proj",
      "language_model.model.layers.22.self_attn.k_proj",
      "language_model.model.layers.22.self_attn.v_proj",
      "language_model.model.layers.22.self_attn.f_a_proj",
      "language_model.model.layers.22.self_attn.f_b_proj",
      "language_model.model.layers.22.self_attn.b_proj",
      "language_model.model.layers.22.self_attn.g_proj",
      "language_model.model.layers.22.self_attn.o_norm",
      "language_model.model.layers.22.self_attn.o_proj",
      "language_model.model.layers.22.block_sparse_moe.gate",
      "language_model.model.layers.22.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.22.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.22.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.22.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.22.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.22.self_attention_res_proj",
      "language_model.model.layers.22.mlp_res_proj",
      "language_model.model.layers.23.self_attn.q_a_proj",
      "language_model.model.layers.23.self_attn.q_b_proj",
      "language_model.model.layers.23.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.23.self_attn.kv_b_proj",
      "language_model.model.layers.23.self_attn.o_proj",
      "language_model.model.layers.23.self_attn.g_proj",
      "language_model.model.layers.23.block_sparse_moe.gate",
      "language_model.model.layers.23.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.23.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.23.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.23.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.23.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.23.self_attention_res_proj",
      "language_model.model.layers.23.mlp_res_proj",
      "language_model.model.layers.24.self_attn.q_proj",
      "language_model.model.layers.24.self_attn.k_proj",
      "language_model.model.layers.24.self_attn.v_proj",
      "language_model.model.layers.24.self_attn.f_a_proj",
      "language_model.model.layers.24.self_attn.f_b_proj",
      "language_model.model.layers.24.self_attn.b_proj",
      "language_model.model.layers.24.self_attn.g_proj",
      "language_model.model.layers.24.self_attn.o_norm",
      "language_model.model.layers.24.self_attn.o_proj",
      "language_model.model.layers.24.block_sparse_moe.gate",
      "language_model.model.layers.24.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.24.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.24.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.24.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.24.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.24.self_attention_res_proj",
      "language_model.model.layers.24.mlp_res_proj",
      "language_model.model.layers.25.self_attn.q_proj",
      "language_model.model.layers.25.self_attn.k_proj",
      "language_model.model.layers.25.self_attn.v_proj",
      "language_model.model.layers.25.self_attn.f_a_proj",
      "language_model.model.layers.25.self_attn.f_b_proj",
      "language_model.model.layers.25.self_attn.b_proj",
      "language_model.model.layers.25.self_attn.g_proj",
      "language_model.model.layers.25.self_attn.o_norm",
      "language_model.model.layers.25.self_attn.o_proj",
      "language_model.model.layers.25.block_sparse_moe.gate",
      "language_model.model.layers.25.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.25.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.25.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.25.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.25.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.25.self_attention_res_proj",
      "language_model.model.layers.25.mlp_res_proj",
      "language_model.model.layers.26.self_attn.q_proj",
      "language_model.model.layers.26.self_attn.k_proj",
      "language_model.model.layers.26.self_attn.v_proj",
      "language_model.model.layers.26.self_attn.f_a_proj",
      "language_model.model.layers.26.self_attn.f_b_proj",
      "language_model.model.layers.26.self_attn.b_proj",
      "language_model.model.layers.26.self_attn.g_proj",
      "language_model.model.layers.26.self_attn.o_norm",
      "language_model.model.layers.26.self_attn.o_proj",
      "language_model.model.layers.26.block_sparse_moe.gate",
      "language_model.model.layers.26.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.26.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.26.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.26.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.26.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.26.self_attention_res_proj",
      "language_model.model.layers.26.mlp_res_proj",
      "language_model.model.layers.27.self_attn.q_a_proj",
      "language_model.model.layers.27.self_attn.q_b_proj",
      "language_model.model.layers.27.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.27.self_attn.kv_b_proj",
      "language_model.model.layers.27.self_attn.o_proj",
      "language_model.model.layers.27.self_attn.g_proj",
      "language_model.model.layers.27.block_sparse_moe.gate",
      "language_model.model.layers.27.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.27.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.27.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.27.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.27.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.27.self_attention_res_proj",
      "language_model.model.layers.27.mlp_res_proj",
      "language_model.model.layers.28.self_attn.q_proj",
      "language_model.model.layers.28.self_attn.k_proj",
      "language_model.model.layers.28.self_attn.v_proj",
      "language_model.model.layers.28.self_attn.f_a_proj",
      "language_model.model.layers.28.self_attn.f_b_proj",
      "language_model.model.layers.28.self_attn.b_proj",
      "language_model.model.layers.28.self_attn.g_proj",
      "language_model.model.layers.28.self_attn.o_norm",
      "language_model.model.layers.28.self_attn.o_proj",
      "language_model.model.layers.28.block_sparse_moe.gate",
      "language_model.model.layers.28.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.28.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.28.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.28.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.28.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.28.self_attention_res_proj",
      "language_model.model.layers.28.mlp_res_proj",
      "language_model.model.layers.29.self_attn.q_proj",
      "language_model.model.layers.29.self_attn.k_proj",
      "language_model.model.layers.29.self_attn.v_proj",
      "language_model.model.layers.29.self_attn.f_a_proj",
      "language_model.model.layers.29.self_attn.f_b_proj",
      "language_model.model.layers.29.self_attn.b_proj",
      "language_model.model.layers.29.self_attn.g_proj",
      "language_model.model.layers.29.self_attn.o_norm",
      "language_model.model.layers.29.self_attn.o_proj",
      "language_model.model.layers.29.block_sparse_moe.gate",
      "language_model.model.layers.29.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.29.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.29.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.29.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.29.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.29.self_attention_res_proj",
      "language_model.model.layers.29.mlp_res_proj",
      "language_model.model.layers.30.self_attn.q_proj",
      "language_model.model.layers.30.self_attn.k_proj",
      "language_model.model.layers.30.self_attn.v_proj",
      "language_model.model.layers.30.self_attn.f_a_proj",
      "language_model.model.layers.30.self_attn.f_b_proj",
      "language_model.model.layers.30.self_attn.b_proj",
      "language_model.model.layers.30.self_attn.g_proj",
      "language_model.model.layers.30.self_attn.o_norm",
      "language_model.model.layers.30.self_attn.o_proj",
      "language_model.model.layers.30.block_sparse_moe.gate",
      "language_model.model.layers.30.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.30.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.30.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.30.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.30.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.30.self_attention_res_proj",
      "language_model.model.layers.30.mlp_res_proj",
      "language_model.model.layers.31.self_attn.q_a_proj",
      "language_model.model.layers.31.self_attn.q_b_proj",
      "language_model.model.layers.31.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.31.self_attn.kv_b_proj",
      "language_model.model.layers.31.self_attn.o_proj",
      "language_model.model.layers.31.self_attn.g_proj",
      "language_model.model.layers.31.block_sparse_moe.gate",
      "language_model.model.layers.31.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.31.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.31.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.31.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.31.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.31.self_attention_res_proj",
      "language_model.model.layers.31.mlp_res_proj",
      "language_model.model.layers.32.self_attn.q_proj",
      "language_model.model.layers.32.self_attn.k_proj",
      "language_model.model.layers.32.self_attn.v_proj",
      "language_model.model.layers.32.self_attn.f_a_proj",
      "language_model.model.layers.32.self_attn.f_b_proj",
      "language_model.model.layers.32.self_attn.b_proj",
      "language_model.model.layers.32.self_attn.g_proj",
      "language_model.model.layers.32.self_attn.o_norm",
      "language_model.model.layers.32.self_attn.o_proj",
      "language_model.model.layers.32.block_sparse_moe.gate",
      "language_model.model.layers.32.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.32.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.32.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.32.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.32.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.32.self_attention_res_proj",
      "language_model.model.layers.32.mlp_res_proj",
      "language_model.model.layers.33.self_attn.q_proj",
      "language_model.model.layers.33.self_attn.k_proj",
      "language_model.model.layers.33.self_attn.v_proj",
      "language_model.model.layers.33.self_attn.f_a_proj",
      "language_model.model.layers.33.self_attn.f_b_proj",
      "language_model.model.layers.33.self_attn.b_proj",
      "language_model.model.layers.33.self_attn.g_proj",
      "language_model.model.layers.33.self_attn.o_norm",
      "language_model.model.layers.33.self_attn.o_proj",
      "language_model.model.layers.33.block_sparse_moe.gate",
      "language_model.model.layers.33.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.33.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.33.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.33.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.33.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.33.self_attention_res_proj",
      "language_model.model.layers.33.mlp_res_proj",
      "language_model.model.layers.34.self_attn.q_proj",
      "language_model.model.layers.34.self_attn.k_proj",
      "language_model.model.layers.34.self_attn.v_proj",
      "language_model.model.layers.34.self_attn.f_a_proj",
      "language_model.model.layers.34.self_attn.f_b_proj",
      "language_model.model.layers.34.self_attn.b_proj",
      "language_model.model.layers.34.self_attn.g_proj",
      "language_model.model.layers.34.self_attn.o_norm",
      "language_model.model.layers.34.self_attn.o_proj",
      "language_model.model.layers.34.block_sparse_moe.gate",
      "language_model.model.layers.34.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.34.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.34.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.34.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.34.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.34.self_attention_res_proj",
      "language_model.model.layers.34.mlp_res_proj",
      "language_model.model.layers.35.self_attn.q_a_proj",
      "language_model.model.layers.35.self_attn.q_b_proj",
      "language_model.model.layers.35.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.35.self_attn.kv_b_proj",
      "language_model.model.layers.35.self_attn.o_proj",
      "language_model.model.layers.35.self_attn.g_proj",
      "language_model.model.layers.35.block_sparse_moe.gate",
      "language_model.model.layers.35.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.35.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.35.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.35.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.35.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.35.self_attention_res_proj",
      "language_model.model.layers.35.mlp_res_proj",
      "language_model.model.layers.36.self_attn.q_proj",
      "language_model.model.layers.36.self_attn.k_proj",
      "language_model.model.layers.36.self_attn.v_proj",
      "language_model.model.layers.36.self_attn.f_a_proj",
      "language_model.model.layers.36.self_attn.f_b_proj",
      "language_model.model.layers.36.self_attn.b_proj",
      "language_model.model.layers.36.self_attn.g_proj",
      "language_model.model.layers.36.self_attn.o_norm",
      "language_model.model.layers.36.self_attn.o_proj",
      "language_model.model.layers.36.block_sparse_moe.gate",
      "language_model.model.layers.36.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.36.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.36.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.36.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.36.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.36.self_attention_res_proj",
      "language_model.model.layers.36.mlp_res_proj",
      "language_model.model.layers.37.self_attn.q_proj",
      "language_model.model.layers.37.self_attn.k_proj",
      "language_model.model.layers.37.self_attn.v_proj",
      "language_model.model.layers.37.self_attn.f_a_proj",
      "language_model.model.layers.37.self_attn.f_b_proj",
      "language_model.model.layers.37.self_attn.b_proj",
      "language_model.model.layers.37.self_attn.g_proj",
      "language_model.model.layers.37.self_attn.o_norm",
      "language_model.model.layers.37.self_attn.o_proj",
      "language_model.model.layers.37.block_sparse_moe.gate",
      "language_model.model.layers.37.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.37.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.37.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.37.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.37.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.37.self_attention_res_proj",
      "language_model.model.layers.37.mlp_res_proj",
      "language_model.model.layers.38.self_attn.q_proj",
      "language_model.model.layers.38.self_attn.k_proj",
      "language_model.model.layers.38.self_attn.v_proj",
      "language_model.model.layers.38.self_attn.f_a_proj",
      "language_model.model.layers.38.self_attn.f_b_proj",
      "language_model.model.layers.38.self_attn.b_proj",
      "language_model.model.layers.38.self_attn.g_proj",
      "language_model.model.layers.38.self_attn.o_norm",
      "language_model.model.layers.38.self_attn.o_proj",
      "language_model.model.layers.38.block_sparse_moe.gate",
      "language_model.model.layers.38.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.38.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.38.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.38.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.38.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.38.self_attention_res_proj",
      "language_model.model.layers.38.mlp_res_proj",
      "language_model.model.layers.39.self_attn.q_a_proj",
      "language_model.model.layers.39.self_attn.q_b_proj",
      "language_model.model.layers.39.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.39.self_attn.kv_b_proj",
      "language_model.model.layers.39.self_attn.o_proj",
      "language_model.model.layers.39.self_attn.g_proj",
      "language_model.model.layers.39.block_sparse_moe.gate",
      "language_model.model.layers.39.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.39.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.39.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.39.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.39.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.39.self_attention_res_proj",
      "language_model.model.layers.39.mlp_res_proj",
      "language_model.model.layers.40.self_attn.q_proj",
      "language_model.model.layers.40.self_attn.k_proj",
      "language_model.model.layers.40.self_attn.v_proj",
      "language_model.model.layers.40.self_attn.f_a_proj",
      "language_model.model.layers.40.self_attn.f_b_proj",
      "language_model.model.layers.40.self_attn.b_proj",
      "language_model.model.layers.40.self_attn.g_proj",
      "language_model.model.layers.40.self_attn.o_norm",
      "language_model.model.layers.40.self_attn.o_proj",
      "language_model.model.layers.40.block_sparse_moe.gate",
      "language_model.model.layers.40.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.40.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.40.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.40.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.40.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.40.self_attention_res_proj",
      "language_model.model.layers.40.mlp_res_proj",
      "language_model.model.layers.41.self_attn.q_proj",
      "language_model.model.layers.41.self_attn.k_proj",
      "language_model.model.layers.41.self_attn.v_proj",
      "language_model.model.layers.41.self_attn.f_a_proj",
      "language_model.model.layers.41.self_attn.f_b_proj",
      "language_model.model.layers.41.self_attn.b_proj",
      "language_model.model.layers.41.self_attn.g_proj",
      "language_model.model.layers.41.self_attn.o_norm",
      "language_model.model.layers.41.self_attn.o_proj",
      "language_model.model.layers.41.block_sparse_moe.gate",
      "language_model.model.layers.41.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.41.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.41.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.41.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.41.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.41.self_attention_res_proj",
      "language_model.model.layers.41.mlp_res_proj",
      "language_model.model.layers.42.self_attn.q_proj",
      "language_model.model.layers.42.self_attn.k_proj",
      "language_model.model.layers.42.self_attn.v_proj",
      "language_model.model.layers.42.self_attn.f_a_proj",
      "language_model.model.layers.42.self_attn.f_b_proj",
      "language_model.model.layers.42.self_attn.b_proj",
      "language_model.model.layers.42.self_attn.g_proj",
      "language_model.model.layers.42.self_attn.o_norm",
      "language_model.model.layers.42.self_attn.o_proj",
      "language_model.model.layers.42.block_sparse_moe.gate",
      "language_model.model.layers.42.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.42.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.42.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.42.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.42.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.42.self_attention_res_proj",
      "language_model.model.layers.42.mlp_res_proj",
      "language_model.model.layers.43.self_attn.q_a_proj",
      "language_model.model.layers.43.self_attn.q_b_proj",
      "language_model.model.layers.43.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.43.self_attn.kv_b_proj",
      "language_model.model.layers.43.self_attn.o_proj",
      "language_model.model.layers.43.self_attn.g_proj",
      "language_model.model.layers.43.block_sparse_moe.gate",
      "language_model.model.layers.43.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.43.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.43.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.43.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.43.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.43.self_attention_res_proj",
      "language_model.model.layers.43.mlp_res_proj",
      "language_model.model.layers.44.self_attn.q_proj",
      "language_model.model.layers.44.self_attn.k_proj",
      "language_model.model.layers.44.self_attn.v_proj",
      "language_model.model.layers.44.self_attn.f_a_proj",
      "language_model.model.layers.44.self_attn.f_b_proj",
      "language_model.model.layers.44.self_attn.b_proj",
      "language_model.model.layers.44.self_attn.g_proj",
      "language_model.model.layers.44.self_attn.o_norm",
      "language_model.model.layers.44.self_attn.o_proj",
      "language_model.model.layers.44.block_sparse_moe.gate",
      "language_model.model.layers.44.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.44.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.44.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.44.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.44.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.44.self_attention_res_proj",
      "language_model.model.layers.44.mlp_res_proj",
      "language_model.model.layers.45.self_attn.q_proj",
      "language_model.model.layers.45.self_attn.k_proj",
      "language_model.model.layers.45.self_attn.v_proj",
      "language_model.model.layers.45.self_attn.f_a_proj",
      "language_model.model.layers.45.self_attn.f_b_proj",
      "language_model.model.layers.45.self_attn.b_proj",
      "language_model.model.layers.45.self_attn.g_proj",
      "language_model.model.layers.45.self_attn.o_norm",
      "language_model.model.layers.45.self_attn.o_proj",
      "language_model.model.layers.45.block_sparse_moe.gate",
      "language_model.model.layers.45.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.45.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.45.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.45.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.45.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.45.self_attention_res_proj",
      "language_model.model.layers.45.mlp_res_proj",
      "language_model.model.layers.46.self_attn.q_proj",
      "language_model.model.layers.46.self_attn.k_proj",
      "language_model.model.layers.46.self_attn.v_proj",
      "language_model.model.layers.46.self_attn.f_a_proj",
      "language_model.model.layers.46.self_attn.f_b_proj",
      "language_model.model.layers.46.self_attn.b_proj",
      "language_model.model.layers.46.self_attn.g_proj",
      "language_model.model.layers.46.self_attn.o_norm",
      "language_model.model.layers.46.self_attn.o_proj",
      "language_model.model.layers.46.block_sparse_moe.gate",
      "language_model.model.layers.46.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.46.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.46.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.46.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.46.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.46.self_attention_res_proj",
      "language_model.model.layers.46.mlp_res_proj",
      "language_model.model.layers.47.self_attn.q_a_proj",
      "language_model.model.layers.47.self_attn.q_b_proj",
      "language_model.model.layers.47.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.47.self_attn.kv_b_proj",
      "language_model.model.layers.47.self_attn.o_proj",
      "language_model.model.layers.47.self_attn.g_proj",
      "language_model.model.layers.47.block_sparse_moe.gate",
      "language_model.model.layers.47.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.47.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.47.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.47.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.47.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.47.self_attention_res_proj",
      "language_model.model.layers.47.mlp_res_proj",
      "language_model.model.layers.48.self_attn.q_proj",
      "language_model.model.layers.48.self_attn.k_proj",
      "language_model.model.layers.48.self_attn.v_proj",
      "language_model.model.layers.48.self_attn.f_a_proj",
      "language_model.model.layers.48.self_attn.f_b_proj",
      "language_model.model.layers.48.self_attn.b_proj",
      "language_model.model.layers.48.self_attn.g_proj",
      "language_model.model.layers.48.self_attn.o_norm",
      "language_model.model.layers.48.self_attn.o_proj",
      "language_model.model.layers.48.block_sparse_moe.gate",
      "language_model.model.layers.48.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.48.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.48.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.48.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.48.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.48.self_attention_res_proj",
      "language_model.model.layers.48.mlp_res_proj",
      "language_model.model.layers.49.self_attn.q_proj",
      "language_model.model.layers.49.self_attn.k_proj",
      "language_model.model.layers.49.self_attn.v_proj",
      "language_model.model.layers.49.self_attn.f_a_proj",
      "language_model.model.layers.49.self_attn.f_b_proj",
      "language_model.model.layers.49.self_attn.b_proj",
      "language_model.model.layers.49.self_attn.g_proj",
      "language_model.model.layers.49.self_attn.o_norm",
      "language_model.model.layers.49.self_attn.o_proj",
      "language_model.model.layers.49.block_sparse_moe.gate",
      "language_model.model.layers.49.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.49.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.49.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.49.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.49.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.49.self_attention_res_proj",
      "language_model.model.layers.49.mlp_res_proj",
      "language_model.model.layers.50.self_attn.q_proj",
      "language_model.model.layers.50.self_attn.k_proj",
      "language_model.model.layers.50.self_attn.v_proj",
      "language_model.model.layers.50.self_attn.f_a_proj",
      "language_model.model.layers.50.self_attn.f_b_proj",
      "language_model.model.layers.50.self_attn.b_proj",
      "language_model.model.layers.50.self_attn.g_proj",
      "language_model.model.layers.50.self_attn.o_norm",
      "language_model.model.layers.50.self_attn.o_proj",
      "language_model.model.layers.50.block_sparse_moe.gate",
      "language_model.model.layers.50.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.50.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.50.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.50.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.50.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.50.self_attention_res_proj",
      "language_model.model.layers.50.mlp_res_proj",
      "language_model.model.layers.51.self_attn.q_a_proj",
      "language_model.model.layers.51.self_attn.q_b_proj",
      "language_model.model.layers.51.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.51.self_attn.kv_b_proj",
      "language_model.model.layers.51.self_attn.o_proj",
      "language_model.model.layers.51.self_attn.g_proj",
      "language_model.model.layers.51.block_sparse_moe.gate",
      "language_model.model.layers.51.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.51.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.51.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.51.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.51.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.51.self_attention_res_proj",
      "language_model.model.layers.51.mlp_res_proj",
      "language_model.model.layers.52.self_attn.q_proj",
      "language_model.model.layers.52.self_attn.k_proj",
      "language_model.model.layers.52.self_attn.v_proj",
      "language_model.model.layers.52.self_attn.f_a_proj",
      "language_model.model.layers.52.self_attn.f_b_proj",
      "language_model.model.layers.52.self_attn.b_proj",
      "language_model.model.layers.52.self_attn.g_proj",
      "language_model.model.layers.52.self_attn.o_norm",
      "language_model.model.layers.52.self_attn.o_proj",
      "language_model.model.layers.52.block_sparse_moe.gate",
      "language_model.model.layers.52.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.52.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.52.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.52.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.52.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.52.self_attention_res_proj",
      "language_model.model.layers.52.mlp_res_proj",
      "language_model.model.layers.53.self_attn.q_proj",
      "language_model.model.layers.53.self_attn.k_proj",
      "language_model.model.layers.53.self_attn.v_proj",
      "language_model.model.layers.53.self_attn.f_a_proj",
      "language_model.model.layers.53.self_attn.f_b_proj",
      "language_model.model.layers.53.self_attn.b_proj",
      "language_model.model.layers.53.self_attn.g_proj",
      "language_model.model.layers.53.self_attn.o_norm",
      "language_model.model.layers.53.self_attn.o_proj",
      "language_model.model.layers.53.block_sparse_moe.gate",
      "language_model.model.layers.53.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.53.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.53.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.53.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.53.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.53.self_attention_res_proj",
      "language_model.model.layers.53.mlp_res_proj",
      "language_model.model.layers.54.self_attn.q_proj",
      "language_model.model.layers.54.self_attn.k_proj",
      "language_model.model.layers.54.self_attn.v_proj",
      "language_model.model.layers.54.self_attn.f_a_proj",
      "language_model.model.layers.54.self_attn.f_b_proj",
      "language_model.model.layers.54.self_attn.b_proj",
      "language_model.model.layers.54.self_attn.g_proj",
      "language_model.model.layers.54.self_attn.o_norm",
      "language_model.model.layers.54.self_attn.o_proj",
      "language_model.model.layers.54.block_sparse_moe.gate",
      "language_model.model.layers.54.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.54.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.54.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.54.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.54.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.54.self_attention_res_proj",
      "language_model.model.layers.54.mlp_res_proj",
      "language_model.model.layers.55.self_attn.q_a_proj",
      "language_model.model.layers.55.self_attn.q_b_proj",
      "language_model.model.layers.55.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.55.self_attn.kv_b_proj",
      "language_model.model.layers.55.self_attn.o_proj",
      "language_model.model.layers.55.self_attn.g_proj",
      "language_model.model.layers.55.block_sparse_moe.gate",
      "language_model.model.layers.55.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.55.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.55.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.55.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.55.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.55.self_attention_res_proj",
      "language_model.model.layers.55.mlp_res_proj",
      "language_model.model.layers.56.self_attn.q_proj",
      "language_model.model.layers.56.self_attn.k_proj",
      "language_model.model.layers.56.self_attn.v_proj",
      "language_model.model.layers.56.self_attn.f_a_proj",
      "language_model.model.layers.56.self_attn.f_b_proj",
      "language_model.model.layers.56.self_attn.b_proj",
      "language_model.model.layers.56.self_attn.g_proj",
      "language_model.model.layers.56.self_attn.o_norm",
      "language_model.model.layers.56.self_attn.o_proj",
      "language_model.model.layers.56.block_sparse_moe.gate",
      "language_model.model.layers.56.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.56.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.56.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.56.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.56.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.56.self_attention_res_proj",
      "language_model.model.layers.56.mlp_res_proj",
      "language_model.model.layers.57.self_attn.q_proj",
      "language_model.model.layers.57.self_attn.k_proj",
      "language_model.model.layers.57.self_attn.v_proj",
      "language_model.model.layers.57.self_attn.f_a_proj",
      "language_model.model.layers.57.self_attn.f_b_proj",
      "language_model.model.layers.57.self_attn.b_proj",
      "language_model.model.layers.57.self_attn.g_proj",
      "language_model.model.layers.57.self_attn.o_norm",
      "language_model.model.layers.57.self_attn.o_proj",
      "language_model.model.layers.57.block_sparse_moe.gate",
      "language_model.model.layers.57.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.57.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.57.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.57.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.57.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.57.self_attention_res_proj",
      "language_model.model.layers.57.mlp_res_proj",
      "language_model.model.layers.58.self_attn.q_proj",
      "language_model.model.layers.58.self_attn.k_proj",
      "language_model.model.layers.58.self_attn.v_proj",
      "language_model.model.layers.58.self_attn.f_a_proj",
      "language_model.model.layers.58.self_attn.f_b_proj",
      "language_model.model.layers.58.self_attn.b_proj",
      "language_model.model.layers.58.self_attn.g_proj",
      "language_model.model.layers.58.self_attn.o_norm",
      "language_model.model.layers.58.self_attn.o_proj",
      "language_model.model.layers.58.block_sparse_moe.gate",
      "language_model.model.layers.58.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.58.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.58.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.58.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.58.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.58.self_attention_res_proj",
      "language_model.model.layers.58.mlp_res_proj",
      "language_model.model.layers.59.self_attn.q_a_proj",
      "language_model.model.layers.59.self_attn.q_b_proj",
      "language_model.model.layers.59.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.59.self_attn.kv_b_proj",
      "language_model.model.layers.59.self_attn.o_proj",
      "language_model.model.layers.59.self_attn.g_proj",
      "language_model.model.layers.59.block_sparse_moe.gate",
      "language_model.model.layers.59.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.59.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.59.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.59.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.59.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.59.self_attention_res_proj",
      "language_model.model.layers.59.mlp_res_proj",
      "language_model.model.layers.60.self_attn.q_proj",
      "language_model.model.layers.60.self_attn.k_proj",
      "language_model.model.layers.60.self_attn.v_proj",
      "language_model.model.layers.60.self_attn.f_a_proj",
      "language_model.model.layers.60.self_attn.f_b_proj",
      "language_model.model.layers.60.self_attn.b_proj",
      "language_model.model.layers.60.self_attn.g_proj",
      "language_model.model.layers.60.self_attn.o_norm",
      "language_model.model.layers.60.self_attn.o_proj",
      "language_model.model.layers.60.block_sparse_moe.gate",
      "language_model.model.layers.60.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.60.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.60.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.60.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.60.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.60.self_attention_res_proj",
      "language_model.model.layers.60.mlp_res_proj",
      "language_model.model.layers.61.self_attn.q_proj",
      "language_model.model.layers.61.self_attn.k_proj",
      "language_model.model.layers.61.self_attn.v_proj",
      "language_model.model.layers.61.self_attn.f_a_proj",
      "language_model.model.layers.61.self_attn.f_b_proj",
      "language_model.model.layers.61.self_attn.b_proj",
      "language_model.model.layers.61.self_attn.g_proj",
      "language_model.model.layers.61.self_attn.o_norm",
      "language_model.model.layers.61.self_attn.o_proj",
      "language_model.model.layers.61.block_sparse_moe.gate",
      "language_model.model.layers.61.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.61.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.61.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.61.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.61.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.61.self_attention_res_proj",
      "language_model.model.layers.61.mlp_res_proj",
      "language_model.model.layers.62.self_attn.q_proj",
      "language_model.model.layers.62.self_attn.k_proj",
      "language_model.model.layers.62.self_attn.v_proj",
      "language_model.model.layers.62.self_attn.f_a_proj",
      "language_model.model.layers.62.self_attn.f_b_proj",
      "language_model.model.layers.62.self_attn.b_proj",
      "language_model.model.layers.62.self_attn.g_proj",
      "language_model.model.layers.62.self_attn.o_norm",
      "language_model.model.layers.62.self_attn.o_proj",
      "language_model.model.layers.62.block_sparse_moe.gate",
      "language_model.model.layers.62.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.62.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.62.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.62.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.62.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.62.self_attention_res_proj",
      "language_model.model.layers.62.mlp_res_proj",
      "language_model.model.layers.63.self_attn.q_a_proj",
      "language_model.model.layers.63.self_attn.q_b_proj",
      "language_model.model.layers.63.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.63.self_attn.kv_b_proj",
      "language_model.model.layers.63.self_attn.o_proj",
      "language_model.model.layers.63.self_attn.g_proj",
      "language_model.model.layers.63.block_sparse_moe.gate",
      "language_model.model.layers.63.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.63.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.63.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.63.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.63.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.63.self_attention_res_proj",
      "language_model.model.layers.63.mlp_res_proj",
      "language_model.model.layers.64.self_attn.q_proj",
      "language_model.model.layers.64.self_attn.k_proj",
      "language_model.model.layers.64.self_attn.v_proj",
      "language_model.model.layers.64.self_attn.f_a_proj",
      "language_model.model.layers.64.self_attn.f_b_proj",
      "language_model.model.layers.64.self_attn.b_proj",
      "language_model.model.layers.64.self_attn.g_proj",
      "language_model.model.layers.64.self_attn.o_norm",
      "language_model.model.layers.64.self_attn.o_proj",
      "language_model.model.layers.64.block_sparse_moe.gate",
      "language_model.model.layers.64.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.64.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.64.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.64.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.64.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.64.self_attention_res_proj",
      "language_model.model.layers.64.mlp_res_proj",
      "language_model.model.layers.65.self_attn.q_proj",
      "language_model.model.layers.65.self_attn.k_proj",
      "language_model.model.layers.65.self_attn.v_proj",
      "language_model.model.layers.65.self_attn.f_a_proj",
      "language_model.model.layers.65.self_attn.f_b_proj",
      "language_model.model.layers.65.self_attn.b_proj",
      "language_model.model.layers.65.self_attn.g_proj",
      "language_model.model.layers.65.self_attn.o_norm",
      "language_model.model.layers.65.self_attn.o_proj",
      "language_model.model.layers.65.block_sparse_moe.gate",
      "language_model.model.layers.65.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.65.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.65.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.65.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.65.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.65.self_attention_res_proj",
      "language_model.model.layers.65.mlp_res_proj",
      "language_model.model.layers.66.self_attn.q_proj",
      "language_model.model.layers.66.self_attn.k_proj",
      "language_model.model.layers.66.self_attn.v_proj",
      "language_model.model.layers.66.self_attn.f_a_proj",
      "language_model.model.layers.66.self_attn.f_b_proj",
      "language_model.model.layers.66.self_attn.b_proj",
      "language_model.model.layers.66.self_attn.g_proj",
      "language_model.model.layers.66.self_attn.o_norm",
      "language_model.model.layers.66.self_attn.o_proj",
      "language_model.model.layers.66.block_sparse_moe.gate",
      "language_model.model.layers.66.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.66.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.66.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.66.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.66.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.66.self_attention_res_proj",
      "language_model.model.layers.66.mlp_res_proj",
      "language_model.model.layers.67.self_attn.q_a_proj",
      "language_model.model.layers.67.self_attn.q_b_proj",
      "language_model.model.layers.67.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.67.self_attn.kv_b_proj",
      "language_model.model.layers.67.self_attn.o_proj",
      "language_model.model.layers.67.self_attn.g_proj",
      "language_model.model.layers.67.block_sparse_moe.gate",
      "language_model.model.layers.67.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.67.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.67.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.67.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.67.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.67.self_attention_res_proj",
      "language_model.model.layers.67.mlp_res_proj",
      "language_model.model.layers.68.self_attn.q_proj",
      "language_model.model.layers.68.self_attn.k_proj",
      "language_model.model.layers.68.self_attn.v_proj",
      "language_model.model.layers.68.self_attn.f_a_proj",
      "language_model.model.layers.68.self_attn.f_b_proj",
      "language_model.model.layers.68.self_attn.b_proj",
      "language_model.model.layers.68.self_attn.g_proj",
      "language_model.model.layers.68.self_attn.o_norm",
      "language_model.model.layers.68.self_attn.o_proj",
      "language_model.model.layers.68.block_sparse_moe.gate",
      "language_model.model.layers.68.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.68.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.68.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.68.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.68.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.68.self_attention_res_proj",
      "language_model.model.layers.68.mlp_res_proj",
      "language_model.model.layers.69.self_attn.q_proj",
      "language_model.model.layers.69.self_attn.k_proj",
      "language_model.model.layers.69.self_attn.v_proj",
      "language_model.model.layers.69.self_attn.f_a_proj",
      "language_model.model.layers.69.self_attn.f_b_proj",
      "language_model.model.layers.69.self_attn.b_proj",
      "language_model.model.layers.69.self_attn.g_proj",
      "language_model.model.layers.69.self_attn.o_norm",
      "language_model.model.layers.69.self_attn.o_proj",
      "language_model.model.layers.69.block_sparse_moe.gate",
      "language_model.model.layers.69.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.69.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.69.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.69.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.69.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.69.self_attention_res_proj",
      "language_model.model.layers.69.mlp_res_proj",
      "language_model.model.layers.70.self_attn.q_proj",
      "language_model.model.layers.70.self_attn.k_proj",
      "language_model.model.layers.70.self_attn.v_proj",
      "language_model.model.layers.70.self_attn.f_a_proj",
      "language_model.model.layers.70.self_attn.f_b_proj",
      "language_model.model.layers.70.self_attn.b_proj",
      "language_model.model.layers.70.self_attn.g_proj",
      "language_model.model.layers.70.self_attn.o_norm",
      "language_model.model.layers.70.self_attn.o_proj",
      "language_model.model.layers.70.block_sparse_moe.gate",
      "language_model.model.layers.70.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.70.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.70.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.70.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.70.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.70.self_attention_res_proj",
      "language_model.model.layers.70.mlp_res_proj",
      "language_model.model.layers.71.self_attn.q_a_proj",
      "language_model.model.layers.71.self_attn.q_b_proj",
      "language_model.model.layers.71.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.71.self_attn.kv_b_proj",
      "language_model.model.layers.71.self_attn.o_proj",
      "language_model.model.layers.71.self_attn.g_proj",
      "language_model.model.layers.71.block_sparse_moe.gate",
      "language_model.model.layers.71.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.71.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.71.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.71.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.71.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.71.self_attention_res_proj",
      "language_model.model.layers.71.mlp_res_proj",
      "language_model.model.layers.72.self_attn.q_proj",
      "language_model.model.layers.72.self_attn.k_proj",
      "language_model.model.layers.72.self_attn.v_proj",
      "language_model.model.layers.72.self_attn.f_a_proj",
      "language_model.model.layers.72.self_attn.f_b_proj",
      "language_model.model.layers.72.self_attn.b_proj",
      "language_model.model.layers.72.self_attn.g_proj",
      "language_model.model.layers.72.self_attn.o_norm",
      "language_model.model.layers.72.self_attn.o_proj",
      "language_model.model.layers.72.block_sparse_moe.gate",
      "language_model.model.layers.72.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.72.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.72.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.72.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.72.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.72.self_attention_res_proj",
      "language_model.model.layers.72.mlp_res_proj",
      "language_model.model.layers.73.self_attn.q_proj",
      "language_model.model.layers.73.self_attn.k_proj",
      "language_model.model.layers.73.self_attn.v_proj",
      "language_model.model.layers.73.self_attn.f_a_proj",
      "language_model.model.layers.73.self_attn.f_b_proj",
      "language_model.model.layers.73.self_attn.b_proj",
      "language_model.model.layers.73.self_attn.g_proj",
      "language_model.model.layers.73.self_attn.o_norm",
      "language_model.model.layers.73.self_attn.o_proj",
      "language_model.model.layers.73.block_sparse_moe.gate",
      "language_model.model.layers.73.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.73.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.73.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.73.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.73.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.73.self_attention_res_proj",
      "language_model.model.layers.73.mlp_res_proj",
      "language_model.model.layers.74.self_attn.q_proj",
      "language_model.model.layers.74.self_attn.k_proj",
      "language_model.model.layers.74.self_attn.v_proj",
      "language_model.model.layers.74.self_attn.f_a_proj",
      "language_model.model.layers.74.self_attn.f_b_proj",
      "language_model.model.layers.74.self_attn.b_proj",
      "language_model.model.layers.74.self_attn.g_proj",
      "language_model.model.layers.74.self_attn.o_norm",
      "language_model.model.layers.74.self_attn.o_proj",
      "language_model.model.layers.74.block_sparse_moe.gate",
      "language_model.model.layers.74.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.74.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.74.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.74.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.74.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.74.self_attention_res_proj",
      "language_model.model.layers.74.mlp_res_proj",
      "language_model.model.layers.75.self_attn.q_a_proj",
      "language_model.model.layers.75.self_attn.q_b_proj",
      "language_model.model.layers.75.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.75.self_attn.kv_b_proj",
      "language_model.model.layers.75.self_attn.o_proj",
      "language_model.model.layers.75.self_attn.g_proj",
      "language_model.model.layers.75.block_sparse_moe.gate",
      "language_model.model.layers.75.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.75.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.75.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.75.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.75.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.75.self_attention_res_proj",
      "language_model.model.layers.75.mlp_res_proj",
      "language_model.model.layers.76.self_attn.q_proj",
      "language_model.model.layers.76.self_attn.k_proj",
      "language_model.model.layers.76.self_attn.v_proj",
      "language_model.model.layers.76.self_attn.f_a_proj",
      "language_model.model.layers.76.self_attn.f_b_proj",
      "language_model.model.layers.76.self_attn.b_proj",
      "language_model.model.layers.76.self_attn.g_proj",
      "language_model.model.layers.76.self_attn.o_norm",
      "language_model.model.layers.76.self_attn.o_proj",
      "language_model.model.layers.76.block_sparse_moe.gate",
      "language_model.model.layers.76.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.76.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.76.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.76.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.76.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.76.self_attention_res_proj",
      "language_model.model.layers.76.mlp_res_proj",
      "language_model.model.layers.77.self_attn.q_proj",
      "language_model.model.layers.77.self_attn.k_proj",
      "language_model.model.layers.77.self_attn.v_proj",
      "language_model.model.layers.77.self_attn.f_a_proj",
      "language_model.model.layers.77.self_attn.f_b_proj",
      "language_model.model.layers.77.self_attn.b_proj",
      "language_model.model.layers.77.self_attn.g_proj",
      "language_model.model.layers.77.self_attn.o_norm",
      "language_model.model.layers.77.self_attn.o_proj",
      "language_model.model.layers.77.block_sparse_moe.gate",
      "language_model.model.layers.77.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.77.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.77.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.77.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.77.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.77.self_attention_res_proj",
      "language_model.model.layers.77.mlp_res_proj",
      "language_model.model.layers.78.self_attn.q_proj",
      "language_model.model.layers.78.self_attn.k_proj",
      "language_model.model.layers.78.self_attn.v_proj",
      "language_model.model.layers.78.self_attn.f_a_proj",
      "language_model.model.layers.78.self_attn.f_b_proj",
      "language_model.model.layers.78.self_attn.b_proj",
      "language_model.model.layers.78.self_attn.g_proj",
      "language_model.model.layers.78.self_attn.o_norm",
      "language_model.model.layers.78.self_attn.o_proj",
      "language_model.model.layers.78.block_sparse_moe.gate",
      "language_model.model.layers.78.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.78.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.78.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.78.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.78.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.78.self_attention_res_proj",
      "language_model.model.layers.78.mlp_res_proj",
      "language_model.model.layers.79.self_attn.q_a_proj",
      "language_model.model.layers.79.self_attn.q_b_proj",
      "language_model.model.layers.79.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.79.self_attn.kv_b_proj",
      "language_model.model.layers.79.self_attn.o_proj",
      "language_model.model.layers.79.self_attn.g_proj",
      "language_model.model.layers.79.block_sparse_moe.gate",
      "language_model.model.layers.79.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.79.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.79.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.79.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.79.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.79.self_attention_res_proj",
      "language_model.model.layers.79.mlp_res_proj",
      "language_model.model.layers.80.self_attn.q_proj",
      "language_model.model.layers.80.self_attn.k_proj",
      "language_model.model.layers.80.self_attn.v_proj",
      "language_model.model.layers.80.self_attn.f_a_proj",
      "language_model.model.layers.80.self_attn.f_b_proj",
      "language_model.model.layers.80.self_attn.b_proj",
      "language_model.model.layers.80.self_attn.g_proj",
      "language_model.model.layers.80.self_attn.o_norm",
      "language_model.model.layers.80.self_attn.o_proj",
      "language_model.model.layers.80.block_sparse_moe.gate",
      "language_model.model.layers.80.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.80.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.80.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.80.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.80.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.80.self_attention_res_proj",
      "language_model.model.layers.80.mlp_res_proj",
      "language_model.model.layers.81.self_attn.q_proj",
      "language_model.model.layers.81.self_attn.k_proj",
      "language_model.model.layers.81.self_attn.v_proj",
      "language_model.model.layers.81.self_attn.f_a_proj",
      "language_model.model.layers.81.self_attn.f_b_proj",
      "language_model.model.layers.81.self_attn.b_proj",
      "language_model.model.layers.81.self_attn.g_proj",
      "language_model.model.layers.81.self_attn.o_norm",
      "language_model.model.layers.81.self_attn.o_proj",
      "language_model.model.layers.81.block_sparse_moe.gate",
      "language_model.model.layers.81.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.81.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.81.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.81.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.81.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.81.self_attention_res_proj",
      "language_model.model.layers.81.mlp_res_proj",
      "language_model.model.layers.82.self_attn.q_proj",
      "language_model.model.layers.82.self_attn.k_proj",
      "language_model.model.layers.82.self_attn.v_proj",
      "language_model.model.layers.82.self_attn.f_a_proj",
      "language_model.model.layers.82.self_attn.f_b_proj",
      "language_model.model.layers.82.self_attn.b_proj",
      "language_model.model.layers.82.self_attn.g_proj",
      "language_model.model.layers.82.self_attn.o_norm",
      "language_model.model.layers.82.self_attn.o_proj",
      "language_model.model.layers.82.block_sparse_moe.gate",
      "language_model.model.layers.82.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.82.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.82.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.82.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.82.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.82.self_attention_res_proj",
      "language_model.model.layers.82.mlp_res_proj",
      "language_model.model.layers.83.self_attn.q_a_proj",
      "language_model.model.layers.83.self_attn.q_b_proj",
      "language_model.model.layers.83.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.83.self_attn.kv_b_proj",
      "language_model.model.layers.83.self_attn.o_proj",
      "language_model.model.layers.83.self_attn.g_proj",
      "language_model.model.layers.83.block_sparse_moe.gate",
      "language_model.model.layers.83.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.83.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.83.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.83.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.83.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.83.self_attention_res_proj",
      "language_model.model.layers.83.mlp_res_proj",
      "language_model.model.layers.84.self_attn.q_proj",
      "language_model.model.layers.84.self_attn.k_proj",
      "language_model.model.layers.84.self_attn.v_proj",
      "language_model.model.layers.84.self_attn.f_a_proj",
      "language_model.model.layers.84.self_attn.f_b_proj",
      "language_model.model.layers.84.self_attn.b_proj",
      "language_model.model.layers.84.self_attn.g_proj",
      "language_model.model.layers.84.self_attn.o_norm",
      "language_model.model.layers.84.self_attn.o_proj",
      "language_model.model.layers.84.block_sparse_moe.gate",
      "language_model.model.layers.84.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.84.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.84.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.84.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.84.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.84.self_attention_res_proj",
      "language_model.model.layers.84.mlp_res_proj",
      "language_model.model.layers.85.self_attn.q_proj",
      "language_model.model.layers.85.self_attn.k_proj",
      "language_model.model.layers.85.self_attn.v_proj",
      "language_model.model.layers.85.self_attn.f_a_proj",
      "language_model.model.layers.85.self_attn.f_b_proj",
      "language_model.model.layers.85.self_attn.b_proj",
      "language_model.model.layers.85.self_attn.g_proj",
      "language_model.model.layers.85.self_attn.o_norm",
      "language_model.model.layers.85.self_attn.o_proj",
      "language_model.model.layers.85.block_sparse_moe.gate",
      "language_model.model.layers.85.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.85.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.85.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.85.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.85.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.85.self_attention_res_proj",
      "language_model.model.layers.85.mlp_res_proj",
      "language_model.model.layers.86.self_attn.q_proj",
      "language_model.model.layers.86.self_attn.k_proj",
      "language_model.model.layers.86.self_attn.v_proj",
      "language_model.model.layers.86.self_attn.f_a_proj",
      "language_model.model.layers.86.self_attn.f_b_proj",
      "language_model.model.layers.86.self_attn.b_proj",
      "language_model.model.layers.86.self_attn.g_proj",
      "language_model.model.layers.86.self_attn.o_norm",
      "language_model.model.layers.86.self_attn.o_proj",
      "language_model.model.layers.86.block_sparse_moe.gate",
      "language_model.model.layers.86.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.86.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.86.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.86.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.86.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.86.self_attention_res_proj",
      "language_model.model.layers.86.mlp_res_proj",
      "language_model.model.layers.87.self_attn.q_a_proj",
      "language_model.model.layers.87.self_attn.q_b_proj",
      "language_model.model.layers.87.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.87.self_attn.kv_b_proj",
      "language_model.model.layers.87.self_attn.o_proj",
      "language_model.model.layers.87.self_attn.g_proj",
      "language_model.model.layers.87.block_sparse_moe.gate",
      "language_model.model.layers.87.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.87.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.87.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.87.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.87.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.87.self_attention_res_proj",
      "language_model.model.layers.87.mlp_res_proj",
      "language_model.model.layers.88.self_attn.q_proj",
      "language_model.model.layers.88.self_attn.k_proj",
      "language_model.model.layers.88.self_attn.v_proj",
      "language_model.model.layers.88.self_attn.f_a_proj",
      "language_model.model.layers.88.self_attn.f_b_proj",
      "language_model.model.layers.88.self_attn.b_proj",
      "language_model.model.layers.88.self_attn.g_proj",
      "language_model.model.layers.88.self_attn.o_norm",
      "language_model.model.layers.88.self_attn.o_proj",
      "language_model.model.layers.88.block_sparse_moe.gate",
      "language_model.model.layers.88.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.88.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.88.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.88.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.88.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.88.self_attention_res_proj",
      "language_model.model.layers.88.mlp_res_proj",
      "language_model.model.layers.89.self_attn.q_proj",
      "language_model.model.layers.89.self_attn.k_proj",
      "language_model.model.layers.89.self_attn.v_proj",
      "language_model.model.layers.89.self_attn.f_a_proj",
      "language_model.model.layers.89.self_attn.f_b_proj",
      "language_model.model.layers.89.self_attn.b_proj",
      "language_model.model.layers.89.self_attn.g_proj",
      "language_model.model.layers.89.self_attn.o_norm",
      "language_model.model.layers.89.self_attn.o_proj",
      "language_model.model.layers.89.block_sparse_moe.gate",
      "language_model.model.layers.89.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.89.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.89.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.89.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.89.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.89.self_attention_res_proj",
      "language_model.model.layers.89.mlp_res_proj",
      "language_model.model.layers.90.self_attn.q_proj",
      "language_model.model.layers.90.self_attn.k_proj",
      "language_model.model.layers.90.self_attn.v_proj",
      "language_model.model.layers.90.self_attn.f_a_proj",
      "language_model.model.layers.90.self_attn.f_b_proj",
      "language_model.model.layers.90.self_attn.b_proj",
      "language_model.model.layers.90.self_attn.g_proj",
      "language_model.model.layers.90.self_attn.o_norm",
      "language_model.model.layers.90.self_attn.o_proj",
      "language_model.model.layers.90.block_sparse_moe.gate",
      "language_model.model.layers.90.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.90.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.90.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.90.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.90.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.90.self_attention_res_proj",
      "language_model.model.layers.90.mlp_res_proj",
      "language_model.model.layers.91.self_attn.q_a_proj",
      "language_model.model.layers.91.self_attn.q_b_proj",
      "language_model.model.layers.91.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.91.self_attn.kv_b_proj",
      "language_model.model.layers.91.self_attn.o_proj",
      "language_model.model.layers.91.self_attn.g_proj",
      "language_model.model.layers.91.block_sparse_moe.gate",
      "language_model.model.layers.91.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.91.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.91.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.91.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.91.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.91.self_attention_res_proj",
      "language_model.model.layers.91.mlp_res_proj",
      "language_model.model.layers.92.self_attn.q_a_proj",
      "language_model.model.layers.92.self_attn.q_b_proj",
      "language_model.model.layers.92.self_attn.kv_a_proj_with_mqa",
      "language_model.model.layers.92.self_attn.kv_b_proj",
      "language_model.model.layers.92.self_attn.o_proj",
      "language_model.model.layers.92.self_attn.g_proj",
      "language_model.model.layers.92.block_sparse_moe.gate",
      "language_model.model.layers.92.block_sparse_moe.shared_experts.gate_proj",
      "language_model.model.layers.92.block_sparse_moe.shared_experts.up_proj",
      "language_model.model.layers.92.block_sparse_moe.shared_experts.down_proj",
      "language_model.model.layers.92.block_sparse_moe.routed_expert_down_proj",
      "language_model.model.layers.92.block_sparse_moe.routed_expert_up_proj",
      "language_model.model.layers.92.self_attention_res_proj",
      "language_model.model.layers.92.mlp_res_proj",
      "language_model.model.output_attn_res_proj",
      "language_model.lm_head"
    ],
    "kv_cache_scheme": null,
    "quant_method": "compressed-tensors",
    "quantization_status": "compressed",
    "sparsity_config": {},
    "transform_config": {},
    "version": "0.17.2.dev32+g46196ce"
  },
  "text_config": {
    "_name_or_path": "",
    "activation_situ_beta": 4.0,
    "activation_situ_linear_beta": 25.0,
    "add_cross_attention": false,
    "architectures": [
      "KimiLinearForCausalLM"
    ],
    "attn_res_block_size": 12,
    "auto_map": {
      "AutoConfig": "configuration_kimi_k3.KimiLinearConfig",
      "AutoModel": "modeling_kimi_linear.KimiLinearModel",
      "AutoModelForCausalLM": "modeling_kimi_linear.KimiLinearForCausalLM"
    },
    "bos_token_id": 163584,
    "chunk_size_feed_forward": 0,
    "cross_attention_hidden_size": null,
    "decoder_start_token_id": null,
    "dtype": "bfloat16",
    "eos_token_id": 163586,
    "finetuning_task": null,
    "first_k_dense_replace": 1,
    "head_dim": 74,
    "hidden_act": "situ",
    "hidden_size": 7168,
    "id2label": {
      "0": "LABEL_0",
      "1": "LABEL_1"
    },
    "initializer_range": 0.02,
    "intermediate_size": 33792,
    "is_decoder": false,
    "is_encoder_decoder": false,
    "kv_lora_rank": 512,
    "label2id": {
      "LABEL_0": 0,
      "LABEL_1": 1
    },
    "latent_moe_use_norm": true,
    "linear_attn_config": {
      "full_attn_layers": [
        4,
        8,
        12,
        16,
        20,
        24,
        28,
        32,
        36,
        40,
        44,
        48,
        52,
        56,
        60,
        64,
        68,
        72,
        76,
        80,
        84,
        88,
        92,
        93
      ],
      "gate_lower_bound": -5.0,
      "head_dim": 128,
      "kda_layers": [
        1,
        2,
        3,
        5,
        6,
        7,
        9,
        10,
        11,
        13,
        14,
        15,
        17,
        18,
        19,
        21,
        22,
        23,
        25,
        26,
        27,
        29,
        30,
        31,
        33,
        34,
        35,
        37,
        38,
        39,
        41,
        42,
        43,
        45,
        46,
        47,
        49,
        50,
        51,
        53,
        54,
        55,
        57,
        58,
        59,
        61,
        62,
        63,
        65,
        66,
        67,
        69,
        70,
        71,
        73,
        74,
        75,
        77,
        78,
        79,
        81,
        82,
        83,
        85,
        86,
        87,
        89,
        90,
        91
      ],
      "num_heads": 96,
      "short_conv_kernel_size": 4,
      "use_full_rank_gate": true
    },
    "max_position_embeddings": 1048576,
    "mla_use_nope": true,
    "mla_use_output_gate": true,
    "model_type": "kimi_linear",
    "moe_intermediate_size": 3072,
    "moe_layer_freq": 1,
    "moe_renormalize": true,
    "moe_router_activation_func": "sigmoid",
    "num_attention_heads": 96,
    "num_expert_group": 1,
    "num_experts": 896,
    "num_experts_per_token": 16,
    "num_hidden_layers": 93,
    "num_key_value_heads": 96,
    "num_nextn_predict_layers": 0,
    "num_shared_experts": 2,
    "output_attentions": false,
    "output_hidden_states": false,
    "pad_token_id": 163839,
    "prefix": null,
    "problem_type": null,
    "pruned_heads": {},
    "q_lora_rank": 1536,
    "qk_nope_head_dim": 128,
    "qk_rope_head_dim": 64,
    "quantization_config": {
      "config_groups": {
        "group_0": {
          "format": "mxfp4-pack-quantized",
          "input_activations": null,
          "output_activations": null,
          "targets": [
            "Linear"
          ],
          "weights": {
            "actorder": null,
            "block_structure": null,
            "dynamic": false,
            "group_size": 32,
            "num_bits": 4,
            "observer": "minmax",
            "observer_kwargs": {},
            "scale_dtype": "torch.uint8",
            "strategy": "group",
            "symmetric": true,
            "type": "float",
            "zp_dtype": null
          }
        }
      },
      "format": "mxfp4-pack-quantized",
      "global_compression_ratio": null,
      "ignore": [
        "re:.*self_attn.*",
        "re:.*shared_experts.*",
        "re:.*mlp\\.(gate|up|gate_up|down)_proj.*",
        "re:.*lm_head.*",
        "re:.*vision_tower.*",
        "re:.*mm_projector.*"
      ],
      "kv_cache_scheme": null,
      "quant_method": "compressed-tensors",
      "quantization_status": "compressed"
    },
    "return_dict": true,
    "rms_norm_eps": 1e-05,
    "rope_parameters": {
      "rope_theta": 10000.0,
      "rope_type": "default"
    },
    "rope_theta": 10000.0,
    "routed_expert_hidden_size": 3584,
    "routed_scaling_factor": 1.0,
    "sep_token_id": null,
    "task_specific_params": null,
    "tf_legacy_loss": false,
    "tie_encoder_decoder": false,
    "tie_word_embeddings": false,
    "tokenizer_class": null,
    "topk_group": 1,
    "topk_method": "noaux_tc",
    "torchscript": false,
    "use_bfloat16": false,
    "use_cache": true,
    "use_grouped_topk": true,
    "v_head_dim": 128,
    "vocab_size": 163840
  },
  "tie_word_embeddings": false,
  "transformers_version": "5.15.0.dev0",
  "vision_config": {
    "_name_or_path": "",
    "activation_func": "gelu_pytorch_tanh",
    "architectures": null,
    "attn_bias": false,
    "chunk_size_feed_forward": 0,
    "dtype": null,
    "id2label": {
      "0": "LABEL_0",
      "1": "LABEL_1"
    },
    "init_pos_emb_height": 64,
    "init_pos_emb_time": 4,
    "init_pos_emb_width": 64,
    "is_encoder_decoder": false,
    "label2id": {
      "LABEL_0": 0,
      "LABEL_1": 1
    },
    "linear_bias": false,
    "merge_kernel_size": [
      2,
      2
    ],
    "merge_type": "sd2_tpool",
    "mlp_type": "mlp2",
    "mm_hidden_size": 1024,
    "mm_projector_type": "patchmergerv2",
    "model_type": "",
    "norm_type": "rmsnorm",
    "output_attentions": false,
    "output_hidden_states": false,
    "patch_embed_proj_bias": false,
    "patch_size": 14,
    "pos_emb_interpolation_mode": "bilinear",
    "pos_emb_type": "divided_fixed",
    "problem_type": null,
    "projector_hidden_act": "gelu",
    "projector_ln_eps": 1e-05,
    "qkv_hidden_size": 1536,
    "return_dict": true,
    "text_hidden_size": 7168,
    "vt_hidden_size": 1024,
    "vt_intermediate_size": 4096,
    "vt_num_attention_heads": 12,
    "vt_num_hidden_layers": 27
  }
}