| { |
| "bits": 4, |
| "data_type": "int", |
| "group_size": 128, |
| "sym": true, |
| "static_attention_granularity": "tensor", |
| "static_kv_granularity": "tensor", |
| "autoround_version": "0.15.0", |
| "block_name_to_quantize": "model.language_model.layers", |
| "quant_method": "auto-round", |
| "packing_format": "auto_round:auto_gptq", |
| "extra_config": { |
| "model.language_model.layers.0.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.0.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.0.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.0.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.0.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.1.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.1.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.1.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.1.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.1.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.2.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.2.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.2.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.2.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.2.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.3.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.3.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.3.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.4.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.4.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.4.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.4.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.4.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.5.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.5.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.5.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.5.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.5.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.6.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.6.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.6.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.6.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.6.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.7.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.7.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.7.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.8.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.8.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.8.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.8.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.8.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.9.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.9.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.9.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.9.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.9.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.10.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.10.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.10.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.10.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.10.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.11.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.11.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.11.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.12.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.12.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.12.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.12.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.12.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.13.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.13.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.13.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.13.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.13.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.14.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.14.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.14.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.14.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.14.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.15.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.15.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.15.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.16.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.16.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.16.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.16.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.16.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.17.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.17.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.17.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.17.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.17.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.18.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.18.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.18.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.18.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.18.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.19.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.19.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.19.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.20.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.20.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.20.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.20.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.20.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.21.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.21.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.21.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.21.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.21.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.22.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.22.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.22.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.22.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.22.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.23.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.23.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.23.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.24.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.24.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.24.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.24.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.24.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.25.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.25.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.25.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.25.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.25.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.26.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.26.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.26.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.26.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.26.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.27.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.27.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.27.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.28.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.28.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.28.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.28.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.28.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.29.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.29.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.29.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.29.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.29.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.30.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.30.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.30.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.30.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.30.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.31.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.31.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.31.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.32.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.32.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.32.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.32.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.32.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.33.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.33.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.33.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.33.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.33.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.34.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.34.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.34.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.34.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.34.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.35.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.35.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.35.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.36.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.36.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.36.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.36.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.36.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.37.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.37.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.37.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.37.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.37.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.38.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.38.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.38.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.38.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.38.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.39.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.39.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.39.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.40.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.40.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.40.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.40.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.40.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.41.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.41.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.41.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.41.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.41.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.42.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.42.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.42.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.42.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.42.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.43.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.43.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.43.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.44.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.44.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.44.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.44.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.44.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.45.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.45.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.45.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.45.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.45.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.46.linear_attn.in_proj_b": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.46.linear_attn.in_proj_a": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.46.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.46.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.46.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.47.mlp.shared_expert_gate": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.47.attn_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.47.mlp_hyper_connection.block_inject_weight": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.0.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.0.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.0.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.1.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.1.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.1.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.2.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.2.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.2.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.4.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.4.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.4.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.5.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.5.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.5.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.6.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.6.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.6.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.8.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.8.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.8.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.9.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.9.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.9.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.10.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.10.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.10.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.12.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.12.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.12.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.13.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.13.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.13.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.14.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.14.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.14.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.16.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.16.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.16.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.17.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.17.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.17.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.18.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.18.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.18.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.20.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.20.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.20.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.21.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.21.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.21.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.22.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.22.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.22.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.24.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.24.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.24.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.25.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.25.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.25.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.26.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.26.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.26.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.28.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.28.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.28.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.29.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.29.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.29.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.30.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.30.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.30.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.32.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.32.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.32.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.33.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.33.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.33.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.34.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.34.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.34.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.36.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.36.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.36.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.37.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.37.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.37.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.38.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.38.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.38.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.40.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.40.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.40.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.41.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.41.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.41.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.42.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.42.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.42.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.44.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.44.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.44.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.45.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.45.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.45.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.46.linear_attn.out_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.46.linear_attn.in_proj_qkv": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.46.linear_attn.in_proj_z": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.3.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.3.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.3.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.3.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.3.self_attn.indexer.index_qk_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.7.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.7.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.7.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.7.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.7.self_attn.indexer.index_qk_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.11.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.11.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.11.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.11.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.11.self_attn.indexer.index_qk_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.15.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.15.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.15.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.15.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.15.self_attn.indexer.index_qk_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.19.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.19.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.19.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.19.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.19.self_attn.indexer.index_qk_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.23.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.23.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.23.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.23.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.23.self_attn.indexer.index_qk_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.27.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.27.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.27.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.27.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.27.self_attn.indexer.index_qk_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.31.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.31.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.31.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.31.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.31.self_attn.indexer.index_qk_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.35.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.35.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.35.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.35.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.35.self_attn.indexer.index_qk_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.39.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.39.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.39.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.39.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.39.self_attn.indexer.index_qk_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.43.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.43.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.43.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.43.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.43.self_attn.indexer.index_qk_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.47.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.47.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.47.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.47.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.47.self_attn.indexer.index_qk_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.0.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.0.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.0.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.0.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.1.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.1.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.1.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.1.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.2.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.2.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.2.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.2.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.3.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.3.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.3.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.3.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.4.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.4.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.4.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.4.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.5.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.5.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.5.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.5.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.6.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.6.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.6.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.6.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.7.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.7.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.7.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.7.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.8.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.8.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.8.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.8.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.9.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.9.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.9.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.9.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.10.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.10.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.10.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.10.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.11.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.11.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.11.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.11.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.12.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.12.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.12.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.12.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.13.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.13.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.13.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.13.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.14.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.14.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.14.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.14.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.15.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.15.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.15.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.15.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.16.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.16.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.16.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.16.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.17.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.17.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.17.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.17.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.18.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.18.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.18.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.18.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.19.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.19.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.19.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.19.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.20.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.20.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.20.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.20.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.21.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.21.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.21.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.21.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.22.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.22.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.22.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.22.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.23.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.23.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.23.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.23.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.24.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.24.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.24.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.24.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.25.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.25.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.25.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.25.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.26.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.26.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.26.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.26.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.27.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.27.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.27.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.27.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.28.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.28.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.28.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.28.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.29.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.29.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.29.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.29.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.30.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.30.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.30.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.30.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.31.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.31.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.31.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.31.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.32.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.32.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.32.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.32.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.33.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.33.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.33.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.33.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.34.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.34.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.34.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.34.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.35.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.35.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.35.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.35.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.36.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.36.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.36.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.36.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.37.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.37.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.37.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.37.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.38.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.38.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.38.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.38.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.39.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.39.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.39.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.39.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.40.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.40.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.40.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.40.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.41.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.41.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.41.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.41.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.42.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.42.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.42.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.42.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.43.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.43.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.43.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.43.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.44.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.44.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.44.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.44.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.45.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.45.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.45.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.45.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.46.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.46.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.46.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.46.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.47.attn_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.47.attn_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.47.mlp_hyper_connection.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.47.mlp_hyper_connection.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.hyper_connection_mixer.input_mix_weight_down": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.hyper_connection_mixer.input_mix_weight_up": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.0.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.0.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.0.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.1.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.1.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.1.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.2.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.2.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.2.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.3.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.3.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.3.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.4.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.4.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.4.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.5.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.5.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.5.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.6.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.6.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.6.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.7.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.7.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.7.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.8.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.8.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.8.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.9.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.9.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.9.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.10.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.10.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.10.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.11.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.11.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.11.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.12.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.12.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.12.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.13.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.13.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.13.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.14.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.14.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.14.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.15.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.15.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.15.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.16.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.16.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.16.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.17.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.17.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.17.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.18.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.18.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.18.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.19.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.19.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.19.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.20.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.20.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.20.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.21.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.21.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.21.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.22.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.22.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.22.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.23.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.23.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.23.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.24.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.24.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.24.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.25.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.25.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.25.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.26.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.26.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.26.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.27.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.27.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.27.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.28.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.28.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.28.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.29.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.29.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.29.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.30.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.30.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.30.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.31.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.31.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.31.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.32.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.32.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.32.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.33.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.33.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.33.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.34.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.34.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.34.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.35.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.35.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.35.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.36.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.36.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.36.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.37.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.37.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.37.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.38.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.38.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.38.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.39.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.39.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.39.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.40.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.40.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.40.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.41.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.41.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.41.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.42.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.42.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.42.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.43.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.43.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.43.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.44.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.44.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.44.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.45.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.45.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.45.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.46.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.46.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.46.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.47.mlp.shared_expert.gate_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.47.mlp.shared_expert.up_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.47.mlp.shared_expert.down_proj": { |
| "bits": 16, |
| "data_type": "fp" |
| }, |
| "model.language_model.layers.1.ple.key_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| "model.language_model.layers.1.ple.value_proj": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*embed_tokens.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*visual.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*linear_attn.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*self_attn.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*hyper_connection.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*shared_expert.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*in_proj_a.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*in_proj_b.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*ple.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*mtp.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*indexer.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.0\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.1\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.2\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.3\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.4\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.5\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.6\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.7\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.8\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.9\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.10\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.11\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.12\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.13\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.14\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.15\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.16\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.17\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.18\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.19\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.20\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.21\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.22\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.23\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.24\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.25\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.26\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.27\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.28\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.29\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.30\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.31\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.32\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.33\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.34\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.35\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.36\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.37\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.38\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.39\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.40\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.41\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.42\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.43\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.44\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.45\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.46\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| }, |
| ".*model\\.language_model\\.layers\\.47\\.mlp\\.gate.*": { |
| "bits": 16, |
| "data_type": "float" |
| } |
| } |
| } |
